Load thumbnails with multiple threads, use sha256 for saving image to path instead of base64 (filename limit is 256 on linux...)

3 files changed, 499 insertions, 0 deletions

diff --git a/external/hash-library/LICENSE b/external/hash-library/LICENSE
new file mode 100644
index 0000000..e540da6
--- /dev/null
+++ b/external/hash-library/LICENSE
@@ -0,0 +1,10 @@
+zlib License
+
+Copyright (c) 2014,2015 Stephan Brumme
+
+This software is provided 'as-is', without any express or implied warranty. In no event will the authors be held liable for any damages arising from the use of this software.
+Permission is granted to anyone to use this software for any purpose, including commercial applications, and to alter it and redistribute it freely, subject to the following restrictions:
+1. The origin of this software must not be misrepresented; you must not claim that you wrote the original software.
+   If you use this software in a product, an acknowledgment in the product documentation would be appreciated but is not required.
+2. Altered source versions must be plainly marked as such, and must not be misrepresented as being the original software.
+3. This notice may not be removed or altered from any source distribution.
diff --git a/external/hash-library/sha256.cpp b/external/hash-library/sha256.cpp
new file mode 100644
index 0000000..0c6138c
--- /dev/null
+++ b/external/hash-library/sha256.cpp
@@ -0,0 +1,411 @@
+// //////////////////////////////////////////////////////////
+// sha256.cpp
+// Copyright (c) 2014,2015 Stephan Brumme. All rights reserved.
+// see http://create.stephan-brumme.com/disclaimer.html
+//
+
+#include "sha256.h"
+
+// big endian architectures need #define __BYTE_ORDER __BIG_ENDIAN
+#ifndef _MSC_VER
+#include <endian.h>
+#endif
+
+
+/// same as reset()
+SHA256::SHA256()
+{
+  reset();
+}
+
+
+/// restart
+void SHA256::reset()
+{
+  m_numBytes   = 0;
+  m_bufferSize = 0;
+
+  // according to RFC 1321
+  m_hash[0] = 0x6a09e667;
+  m_hash[1] = 0xbb67ae85;
+  m_hash[2] = 0x3c6ef372;
+  m_hash[3] = 0xa54ff53a;
+  m_hash[4] = 0x510e527f;
+  m_hash[5] = 0x9b05688c;
+  m_hash[6] = 0x1f83d9ab;
+  m_hash[7] = 0x5be0cd19;
+}
+
+
+namespace
+{
+  inline uint32_t rotate(uint32_t a, uint32_t c)
+  {
+    return (a >> c) | (a << (32 - c));
+  }
+
+  inline uint32_t swap(uint32_t x)
+  {
+#if defined(__GNUC__) || defined(__clang__)
+    return __builtin_bswap32(x);
+#endif
+#ifdef MSC_VER
+    return _byteswap_ulong(x);
+#endif
+
+    return (x >> 24) |
+          ((x >>  8) & 0x0000FF00) |
+          ((x <<  8) & 0x00FF0000) |
+           (x << 24);
+  }
+
+  // mix functions for processBlock()
+  inline uint32_t f1(uint32_t e, uint32_t f, uint32_t g)
+  {
+    uint32_t term1 = rotate(e, 6) ^ rotate(e, 11) ^ rotate(e, 25);
+    uint32_t term2 = (e & f) ^ (~e & g); //(g ^ (e & (f ^ g)))
+    return term1 + term2;
+  }
+
+  inline uint32_t f2(uint32_t a, uint32_t b, uint32_t c)
+  {
+    uint32_t term1 = rotate(a, 2) ^ rotate(a, 13) ^ rotate(a, 22);
+    uint32_t term2 = ((a | b) & c) | (a & b); //(a & (b ^ c)) ^ (b & c);
+    return term1 + term2;
+  }
+}
+
+
+/// process 64 bytes
+void SHA256::processBlock(const void* data)
+{
+  // get last hash
+  uint32_t a = m_hash[0];
+  uint32_t b = m_hash[1];
+  uint32_t c = m_hash[2];
+  uint32_t d = m_hash[3];
+  uint32_t e = m_hash[4];
+  uint32_t f = m_hash[5];
+  uint32_t g = m_hash[6];
+  uint32_t h = m_hash[7];
+
+  // data represented as 16x 32-bit words
+  const uint32_t* input = (uint32_t*) data;
+  // convert to big endian
+  uint32_t words[64];
+  int i;
+  for (i = 0; i < 16; i++)
+#if defined(__BYTE_ORDER) && (__BYTE_ORDER != 0) && (__BYTE_ORDER == __BIG_ENDIAN)
+    words[i] =      input[i];
+#else
+    words[i] = swap(input[i]);
+#endif
+
+  uint32_t x,y; // temporaries
+
+  // first round
+  x = h + f1(e,f,g) + 0x428a2f98 + words[ 0]; y = f2(a,b,c); d += x; h = x + y;
+  x = g + f1(d,e,f) + 0x71374491 + words[ 1]; y = f2(h,a,b); c += x; g = x + y;
+  x = f + f1(c,d,e) + 0xb5c0fbcf + words[ 2]; y = f2(g,h,a); b += x; f = x + y;
+  x = e + f1(b,c,d) + 0xe9b5dba5 + words[ 3]; y = f2(f,g,h); a += x; e = x + y;
+  x = d + f1(a,b,c) + 0x3956c25b + words[ 4]; y = f2(e,f,g); h += x; d = x + y;
+  x = c + f1(h,a,b) + 0x59f111f1 + words[ 5]; y = f2(d,e,f); g += x; c = x + y;
+  x = b + f1(g,h,a) + 0x923f82a4 + words[ 6]; y = f2(c,d,e); f += x; b = x + y;
+  x = a + f1(f,g,h) + 0xab1c5ed5 + words[ 7]; y = f2(b,c,d); e += x; a = x + y;
+
+  // secound round
+  x = h + f1(e,f,g) + 0xd807aa98 + words[ 8]; y = f2(a,b,c); d += x; h = x + y;
+  x = g + f1(d,e,f) + 0x12835b01 + words[ 9]; y = f2(h,a,b); c += x; g = x + y;
+  x = f + f1(c,d,e) + 0x243185be + words[10]; y = f2(g,h,a); b += x; f = x + y;
+  x = e + f1(b,c,d) + 0x550c7dc3 + words[11]; y = f2(f,g,h); a += x; e = x + y;
+  x = d + f1(a,b,c) + 0x72be5d74 + words[12]; y = f2(e,f,g); h += x; d = x + y;
+  x = c + f1(h,a,b) + 0x80deb1fe + words[13]; y = f2(d,e,f); g += x; c = x + y;
+  x = b + f1(g,h,a) + 0x9bdc06a7 + words[14]; y = f2(c,d,e); f += x; b = x + y;
+  x = a + f1(f,g,h) + 0xc19bf174 + words[15]; y = f2(b,c,d); e += x; a = x + y;
+
+  // extend to 24 words
+  for (; i < 24; i++)
+    words[i] = words[i-16] +
+               (rotate(words[i-15],  7) ^ rotate(words[i-15], 18) ^ (words[i-15] >>  3)) +
+               words[i-7] +
+               (rotate(words[i- 2], 17) ^ rotate(words[i- 2], 19) ^ (words[i- 2] >> 10));
+
+  // third round
+  x = h + f1(e,f,g) + 0xe49b69c1 + words[16]; y = f2(a,b,c); d += x; h = x + y;
+  x = g + f1(d,e,f) + 0xefbe4786 + words[17]; y = f2(h,a,b); c += x; g = x + y;
+  x = f + f1(c,d,e) + 0x0fc19dc6 + words[18]; y = f2(g,h,a); b += x; f = x + y;
+  x = e + f1(b,c,d) + 0x240ca1cc + words[19]; y = f2(f,g,h); a += x; e = x + y;
+  x = d + f1(a,b,c) + 0x2de92c6f + words[20]; y = f2(e,f,g); h += x; d = x + y;
+  x = c + f1(h,a,b) + 0x4a7484aa + words[21]; y = f2(d,e,f); g += x; c = x + y;
+  x = b + f1(g,h,a) + 0x5cb0a9dc + words[22]; y = f2(c,d,e); f += x; b = x + y;
+  x = a + f1(f,g,h) + 0x76f988da + words[23]; y = f2(b,c,d); e += x; a = x + y;
+
+  // extend to 32 words
+  for (; i < 32; i++)
+    words[i] = words[i-16] +
+               (rotate(words[i-15],  7) ^ rotate(words[i-15], 18) ^ (words[i-15] >>  3)) +
+               words[i-7] +
+               (rotate(words[i- 2], 17) ^ rotate(words[i- 2], 19) ^ (words[i- 2] >> 10));
+
+  // fourth round
+  x = h + f1(e,f,g) + 0x983e5152 + words[24]; y = f2(a,b,c); d += x; h = x + y;
+  x = g + f1(d,e,f) + 0xa831c66d + words[25]; y = f2(h,a,b); c += x; g = x + y;
+  x = f + f1(c,d,e) + 0xb00327c8 + words[26]; y = f2(g,h,a); b += x; f = x + y;
+  x = e + f1(b,c,d) + 0xbf597fc7 + words[27]; y = f2(f,g,h); a += x; e = x + y;
+  x = d + f1(a,b,c) + 0xc6e00bf3 + words[28]; y = f2(e,f,g); h += x; d = x + y;
+  x = c + f1(h,a,b) + 0xd5a79147 + words[29]; y = f2(d,e,f); g += x; c = x + y;
+  x = b + f1(g,h,a) + 0x06ca6351 + words[30]; y = f2(c,d,e); f += x; b = x + y;
+  x = a + f1(f,g,h) + 0x14292967 + words[31]; y = f2(b,c,d); e += x; a = x + y;
+
+  // extend to 40 words
+  for (; i < 40; i++)
+    words[i] = words[i-16] +
+               (rotate(words[i-15],  7) ^ rotate(words[i-15], 18) ^ (words[i-15] >>  3)) +
+               words[i-7] +
+               (rotate(words[i- 2], 17) ^ rotate(words[i- 2], 19) ^ (words[i- 2] >> 10));
+
+  // fifth round
+  x = h + f1(e,f,g) + 0x27b70a85 + words[32]; y = f2(a,b,c); d += x; h = x + y;
+  x = g + f1(d,e,f) + 0x2e1b2138 + words[33]; y = f2(h,a,b); c += x; g = x + y;
+  x = f + f1(c,d,e) + 0x4d2c6dfc + words[34]; y = f2(g,h,a); b += x; f = x + y;
+  x = e + f1(b,c,d) + 0x53380d13 + words[35]; y = f2(f,g,h); a += x; e = x + y;
+  x = d + f1(a,b,c) + 0x650a7354 + words[36]; y = f2(e,f,g); h += x; d = x + y;
+  x = c + f1(h,a,b) + 0x766a0abb + words[37]; y = f2(d,e,f); g += x; c = x + y;
+  x = b + f1(g,h,a) + 0x81c2c92e + words[38]; y = f2(c,d,e); f += x; b = x + y;
+  x = a + f1(f,g,h) + 0x92722c85 + words[39]; y = f2(b,c,d); e += x; a = x + y;
+
+  // extend to 48 words
+  for (; i < 48; i++)
+    words[i] = words[i-16] +
+               (rotate(words[i-15],  7) ^ rotate(words[i-15], 18) ^ (words[i-15] >>  3)) +
+               words[i-7] +
+               (rotate(words[i- 2], 17) ^ rotate(words[i- 2], 19) ^ (words[i- 2] >> 10));
+
+  // sixth round
+  x = h + f1(e,f,g) + 0xa2bfe8a1 + words[40]; y = f2(a,b,c); d += x; h = x + y;
+  x = g + f1(d,e,f) + 0xa81a664b + words[41]; y = f2(h,a,b); c += x; g = x + y;
+  x = f + f1(c,d,e) + 0xc24b8b70 + words[42]; y = f2(g,h,a); b += x; f = x + y;
+  x = e + f1(b,c,d) + 0xc76c51a3 + words[43]; y = f2(f,g,h); a += x; e = x + y;
+  x = d + f1(a,b,c) + 0xd192e819 + words[44]; y = f2(e,f,g); h += x; d = x + y;
+  x = c + f1(h,a,b) + 0xd6990624 + words[45]; y = f2(d,e,f); g += x; c = x + y;
+  x = b + f1(g,h,a) + 0xf40e3585 + words[46]; y = f2(c,d,e); f += x; b = x + y;
+  x = a + f1(f,g,h) + 0x106aa070 + words[47]; y = f2(b,c,d); e += x; a = x + y;
+
+  // extend to 56 words
+  for (; i < 56; i++)
+    words[i] = words[i-16] +
+               (rotate(words[i-15],  7) ^ rotate(words[i-15], 18) ^ (words[i-15] >>  3)) +
+               words[i-7] +
+               (rotate(words[i- 2], 17) ^ rotate(words[i- 2], 19) ^ (words[i- 2] >> 10));
+
+  // seventh round
+  x = h + f1(e,f,g) + 0x19a4c116 + words[48]; y = f2(a,b,c); d += x; h = x + y;
+  x = g + f1(d,e,f) + 0x1e376c08 + words[49]; y = f2(h,a,b); c += x; g = x + y;
+  x = f + f1(c,d,e) + 0x2748774c + words[50]; y = f2(g,h,a); b += x; f = x + y;
+  x = e + f1(b,c,d) + 0x34b0bcb5 + words[51]; y = f2(f,g,h); a += x; e = x + y;
+  x = d + f1(a,b,c) + 0x391c0cb3 + words[52]; y = f2(e,f,g); h += x; d = x + y;
+  x = c + f1(h,a,b) + 0x4ed8aa4a + words[53]; y = f2(d,e,f); g += x; c = x + y;
+  x = b + f1(g,h,a) + 0x5b9cca4f + words[54]; y = f2(c,d,e); f += x; b = x + y;
+  x = a + f1(f,g,h) + 0x682e6ff3 + words[55]; y = f2(b,c,d); e += x; a = x + y;
+
+  // extend to 64 words
+  for (; i < 64; i++)
+    words[i] = words[i-16] +
+               (rotate(words[i-15],  7) ^ rotate(words[i-15], 18) ^ (words[i-15] >>  3)) +
+               words[i-7] +
+               (rotate(words[i- 2], 17) ^ rotate(words[i- 2], 19) ^ (words[i- 2] >> 10));
+
+  // eigth round
+  x = h + f1(e,f,g) + 0x748f82ee + words[56]; y = f2(a,b,c); d += x; h = x + y;
+  x = g + f1(d,e,f) + 0x78a5636f + words[57]; y = f2(h,a,b); c += x; g = x + y;
+  x = f + f1(c,d,e) + 0x84c87814 + words[58]; y = f2(g,h,a); b += x; f = x + y;
+  x = e + f1(b,c,d) + 0x8cc70208 + words[59]; y = f2(f,g,h); a += x; e = x + y;
+  x = d + f1(a,b,c) + 0x90befffa + words[60]; y = f2(e,f,g); h += x; d = x + y;
+  x = c + f1(h,a,b) + 0xa4506ceb + words[61]; y = f2(d,e,f); g += x; c = x + y;
+  x = b + f1(g,h,a) + 0xbef9a3f7 + words[62]; y = f2(c,d,e); f += x; b = x + y;
+  x = a + f1(f,g,h) + 0xc67178f2 + words[63]; y = f2(b,c,d); e += x; a = x + y;
+
+  // update hash
+  m_hash[0] += a;
+  m_hash[1] += b;
+  m_hash[2] += c;
+  m_hash[3] += d;
+  m_hash[4] += e;
+  m_hash[5] += f;
+  m_hash[6] += g;
+  m_hash[7] += h;
+}
+
+
+/// add arbitrary number of bytes
+void SHA256::add(const void* data, size_t numBytes)
+{
+  const uint8_t* current = (const uint8_t*) data;
+
+  if (m_bufferSize > 0)
+  {
+    while (numBytes > 0 && m_bufferSize < BlockSize)
+    {
+      m_buffer[m_bufferSize++] = *current++;
+      numBytes--;
+    }
+  }
+
+  // full buffer
+  if (m_bufferSize == BlockSize)
+  {
+    processBlock(m_buffer);
+    m_numBytes  += BlockSize;
+    m_bufferSize = 0;
+  }
+
+  // no more data ?
+  if (numBytes == 0)
+    return;
+
+  // process full blocks
+  while (numBytes >= BlockSize)
+  {
+    processBlock(current);
+    current    += BlockSize;
+    m_numBytes += BlockSize;
+    numBytes   -= BlockSize;
+  }
+
+  // keep remaining bytes in buffer
+  while (numBytes > 0)
+  {
+    m_buffer[m_bufferSize++] = *current++;
+    numBytes--;
+  }
+}
+
+
+/// process final block, less than 64 bytes
+void SHA256::processBuffer()
+{
+  // the input bytes are considered as bits strings, where the first bit is the most significant bit of the byte
+
+  // - append "1" bit to message
+  // - append "0" bits until message length in bit mod 512 is 448
+  // - append length as 64 bit integer
+
+  // number of bits
+  size_t paddedLength = m_bufferSize * 8;
+
+  // plus one bit set to 1 (always appended)
+  paddedLength++;
+
+  // number of bits must be (numBits % 512) = 448
+  size_t lower11Bits = paddedLength & 511;
+  if (lower11Bits <= 448)
+    paddedLength +=       448 - lower11Bits;
+  else
+    paddedLength += 512 + 448 - lower11Bits;
+  // convert from bits to bytes
+  paddedLength /= 8;
+
+  // only needed if additional data flows over into a second block
+  unsigned char extra[BlockSize];
+
+  // append a "1" bit, 128 => binary 10000000
+  if (m_bufferSize < BlockSize)
+    m_buffer[m_bufferSize] = 128;
+  else
+    extra[0] = 128;
+
+  size_t i;
+  for (i = m_bufferSize + 1; i < BlockSize; i++)
+    m_buffer[i] = 0;
+  for (; i < paddedLength; i++)
+    extra[i - BlockSize] = 0;
+
+  // add message length in bits as 64 bit number
+  uint64_t msgBits = 8 * (m_numBytes + m_bufferSize);
+  // find right position
+  unsigned char* addLength;
+  if (paddedLength < BlockSize)
+    addLength = m_buffer + paddedLength;
+  else
+    addLength = extra + paddedLength - BlockSize;
+
+  // must be big endian
+  *addLength++ = (unsigned char)((msgBits >> 56) & 0xFF);
+  *addLength++ = (unsigned char)((msgBits >> 48) & 0xFF);
+  *addLength++ = (unsigned char)((msgBits >> 40) & 0xFF);
+  *addLength++ = (unsigned char)((msgBits >> 32) & 0xFF);
+  *addLength++ = (unsigned char)((msgBits >> 24) & 0xFF);
+  *addLength++ = (unsigned char)((msgBits >> 16) & 0xFF);
+  *addLength++ = (unsigned char)((msgBits >>  8) & 0xFF);
+  *addLength   = (unsigned char)( msgBits        & 0xFF);
+
+  // process blocks
+  processBlock(m_buffer);
+  // flowed over into a second block ?
+  if (paddedLength > BlockSize)
+    processBlock(extra);
+}
+
+
+/// return latest hash as 64 hex characters
+std::string SHA256::getHash()
+{
+  // compute hash (as raw bytes)
+  unsigned char rawHash[HashBytes];
+  getHash(rawHash);
+
+  // convert to hex string
+  std::string result;
+  result.reserve(2 * HashBytes);
+  for (int i = 0; i < HashBytes; i++)
+  {
+    static const char dec2hex[16+1] = "0123456789abcdef";
+    result += dec2hex[(rawHash[i] >> 4) & 15];
+    result += dec2hex[ rawHash[i]       & 15];
+  }
+
+  return result;
+}
+
+
+/// return latest hash as bytes
+void SHA256::getHash(unsigned char buffer[SHA256::HashBytes])
+{
+  // save old hash if buffer is partially filled
+  uint32_t oldHash[HashValues];
+  for (int i = 0; i < HashValues; i++)
+    oldHash[i] = m_hash[i];
+
+  // process remaining bytes
+  processBuffer();
+
+  unsigned char* current = buffer;
+  for (int i = 0; i < HashValues; i++)
+  {
+    *current++ = (m_hash[i] >> 24) & 0xFF;
+    *current++ = (m_hash[i] >> 16) & 0xFF;
+    *current++ = (m_hash[i] >>  8) & 0xFF;
+    *current++ =  m_hash[i]        & 0xFF;
+
+    // restore old hash
+    m_hash[i] = oldHash[i];
+  }
+}
+
+
+/// compute SHA256 of a memory block
+std::string SHA256::operator()(const void* data, size_t numBytes)
+{
+  reset();
+  add(data, numBytes);
+  return getHash();
+}
+
+
+/// compute SHA256 of a string, excluding final zero
+std::string SHA256::operator()(const std::string& text)
+{
+  reset();
+  add(text.c_str(), text.size());
+  return getHash();
+}
diff --git a/external/hash-library/sha256.h b/external/hash-library/sha256.h
new file mode 100644
index 0000000..aeaf314
--- /dev/null
+++ b/external/hash-library/sha256.h
@@ -0,0 +1,78 @@
+// //////////////////////////////////////////////////////////
+// sha256.h
+// Copyright (c) 2014,2015 Stephan Brumme. All rights reserved.
+// see http://create.stephan-brumme.com/disclaimer.html
+//
+
+#pragma once
+
+//#include "hash.h"
+#include <string>
+
+// define fixed size integer types
+#ifdef _MSC_VER
+// Windows
+typedef unsigned __int8  uint8_t;
+typedef unsigned __int32 uint32_t;
+typedef unsigned __int64 uint64_t;
+#else
+// GCC
+#include <stdint.h>
+#endif
+
+
+/// compute SHA256 hash
+/** Usage:
+    SHA256 sha256;
+    std::string myHash  = sha256("Hello World");     // std::string
+    std::string myHash2 = sha256("How are you", 11); // arbitrary data, 11 bytes
+
+    // or in a streaming fashion:
+
+    SHA256 sha256;
+    while (more data available)
+      sha256.add(pointer to fresh data, number of new bytes);
+    std::string myHash3 = sha256.getHash();
+  */
+class SHA256 //: public Hash
+{
+public:
+  /// split into 64 byte blocks (=> 512 bits), hash is 32 bytes long
+  enum { BlockSize = 512 / 8, HashBytes = 32 };
+
+  /// same as reset()
+  SHA256();
+
+  /// compute SHA256 of a memory block
+  std::string operator()(const void* data, size_t numBytes);
+  /// compute SHA256 of a string, excluding final zero
+  std::string operator()(const std::string& text);
+
+  /// add arbitrary number of bytes
+  void add(const void* data, size_t numBytes);
+
+  /// return latest hash as 64 hex characters
+  std::string getHash();
+  /// return latest hash as bytes
+  void        getHash(unsigned char buffer[HashBytes]);
+
+  /// restart
+  void reset();
+
+private:
+  /// process 64 bytes
+  void processBlock(const void* data);
+  /// process everything left in the internal buffer
+  void processBuffer();
+
+  /// size of processed data in bytes
+  uint64_t m_numBytes;
+  /// valid bytes in m_buffer
+  size_t   m_bufferSize;
+  /// bytes not processed yet
+  uint8_t  m_buffer[BlockSize];
+
+  enum { HashValues = HashBytes / 4 };
+  /// hash, stored as integers
+  uint32_t m_hash[HashValues];
+};