Deduplicate hashing code

src/common/darktable.h

@@ -652,6 +652,20 @@ static inline int dt_get_thread_num()
 #endif
 }
 
+#define DT_INITHASH 5381
+typedef uint64_t dt_hash_t;
+static inline dt_hash_t dt_hash(dt_hash_t hash, const void *data, const size_t size)
+{
+  const uint8_t* str = (uint8_t*)data;
+  // Scramble bits in str to create an (hopefully) unique hash representing the state of str
+  // Dan Bernstein algo v2 http://www.cse.yorku.ca/~oz/hash.html
+  // hash should be inited to DT_INITHASH if first run, or from a previous hash computed with this function.
+  for(size_t i = 0; i < size; i++)
+    hash = ((hash << 5) + hash) ^ str[i];
+
+  return hash;
+}
+
 // Allocate a buffer for 'n' objects each of size 'objsize' bytes for
 // each of the program's threads.  Ensures that there is no false
 // sharing among threads by aligning and rounding up the allocation to

src/develop/develop.c

@@ -3252,17 +3252,17 @@ dt_dev_pixelpipe_iop_t *dt_dev_distort_get_iop_pipe(dt_develop_t *dev,
   return NULL;
 }
 
-uint64_t dt_dev_hash(dt_develop_t *dev)
+dt_hash_t dt_dev_hash(dt_develop_t *dev)
 {
   return dt_dev_hash_plus(dev, dev->preview_pipe, 0.0f, DT_DEV_TRANSFORM_DIR_ALL);
 }
 
-uint64_t dt_dev_hash_plus(dt_develop_t *dev,
+dt_hash_t dt_dev_hash_plus(dt_develop_t *dev,
                           struct dt_dev_pixelpipe_t *pipe,
                           const double iop_order,
                           const dt_dev_transform_direction_t transf_direction)
 {
-  uint64_t hash = 5381;
+  dt_hash_t hash = DT_INITHASH;
   dt_pthread_mutex_lock(&dev->history_mutex);
   GList *modules = g_list_last(pipe->iop);
   GList *pieces = g_list_last(pipe->nodes);
@@ -3285,7 +3285,7 @@ uint64_t dt_dev_hash_plus(dt_develop_t *dev,
                           || (transf_direction == DT_DEV_TRANSFORM_DIR_BACK_EXCL
                               && module->iop_order < iop_order)))
     {
-      hash = ((hash << 5) + hash) ^ piece->hash;
+      hash = dt_hash(hash, &piece->hash, sizeof(piece->hash));
     }
     modules = g_list_previous(modules);
     pieces = g_list_previous(pieces);
@@ -3299,7 +3299,7 @@ gboolean dt_dev_wait_hash(dt_develop_t *dev,
                           const double iop_order,
                           const dt_dev_transform_direction_t transf_direction,
                           dt_pthread_mutex_t *lock,
-                          const volatile uint64_t *const hash)
+                          const volatile dt_hash_t *const hash)
 {
   const int usec = 5000;
   int nloop;
@@ -3320,7 +3320,7 @@ gboolean dt_dev_wait_hash(dt_develop_t *dev,
     if(dt_atomic_get_int(&pipe->shutdown))
       return TRUE;  // stop waiting if pipe shuts down
 
-    uint64_t probehash;
+    dt_hash_t probehash;
 
     if(lock)
     {
@@ -3345,7 +3345,7 @@ gboolean dt_dev_sync_pixelpipe_hash(dt_develop_t *dev,
                                     const double iop_order,
                                     const dt_dev_transform_direction_t transf_direction,
                                     dt_pthread_mutex_t *lock,
-                                    const volatile uint64_t *const hash)
+                                    const volatile dt_hash_t *const hash)
 {
   // first wait for matching hash values
   if(dt_dev_wait_hash(dev, pipe, iop_order, transf_direction, lock, hash))
@@ -3364,17 +3364,17 @@ gboolean dt_dev_sync_pixelpipe_hash(dt_develop_t *dev,
   return FALSE;
 }
 
-uint64_t dt_dev_hash_distort(dt_develop_t *dev)
+dt_hash_t dt_dev_hash_distort(dt_develop_t *dev)
 {
   return dt_dev_hash_distort_plus(dev, dev->preview_pipe, 0.0f, DT_DEV_TRANSFORM_DIR_ALL);
 }
 
-uint64_t dt_dev_hash_distort_plus(dt_develop_t *dev,
+dt_hash_t dt_dev_hash_distort_plus(dt_develop_t *dev,
                                   struct dt_dev_pixelpipe_t *pipe,
                                   const double iop_order,
                                   const dt_dev_transform_direction_t transf_direction)
 {
-  uint64_t hash = 5381;
+  dt_hash_t hash = DT_INITHASH;
   dt_pthread_mutex_lock(&dev->history_mutex);
   GList *modules = g_list_last(pipe->iop);
   GList *pieces = g_list_last(pipe->nodes);
@@ -3398,7 +3398,7 @@ uint64_t dt_dev_hash_distort_plus(dt_develop_t *dev,
            || (transf_direction == DT_DEV_TRANSFORM_DIR_BACK_EXCL
                && module->iop_order < iop_order)))
     {
-      hash = ((hash << 5) + hash) ^ piece->hash;
+      hash = dt_hash(hash, &piece->hash, sizeof(piece->hash));
     }
     modules = g_list_previous(modules);
     pieces = g_list_previous(pieces);

src/develop/develop.h

@@ -580,9 +580,9 @@ struct dt_dev_pixelpipe_iop_t *dt_dev_distort_get_iop_pipe(dt_develop_t *dev,
  * hash functions
  */
 /** generate hash value out of all module settings of pixelpipe */
-uint64_t dt_dev_hash(dt_develop_t *dev);
+dt_hash_t dt_dev_hash(dt_develop_t *dev);
 /** same function, but we can specify iop with priority between pmin and pmax */
-uint64_t dt_dev_hash_plus(dt_develop_t *dev,
+dt_hash_t dt_dev_hash_plus(dt_develop_t *dev,
                           struct dt_dev_pixelpipe_t *pipe,
                           const double iop_order,
                           const dt_dev_transform_direction_t transf_direction);
@@ -593,19 +593,19 @@ gboolean dt_dev_wait_hash(dt_develop_t *dev,
                      const double iop_order,
                      const dt_dev_transform_direction_t transf_direction,
                      dt_pthread_mutex_t *lock,
-                     const volatile uint64_t *const hash);
+                     const volatile dt_hash_t *const hash);
 /** synchronize pixelpipe by means hash values by waiting with timeout
  * and potential reprocessing */
 gboolean dt_dev_sync_pixelpipe_hash(dt_develop_t *dev,
                                struct dt_dev_pixelpipe_t *pipe,
                                const double iop_order,
                                const dt_dev_transform_direction_t transf_direction,
                                dt_pthread_mutex_t *lock,
-                               const volatile uint64_t *const hash);
+                               const volatile dt_hash_t *const hash);
 /** generate hash value out of module settings of all distorting modules of pixelpipe */
-uint64_t dt_dev_hash_distort(dt_develop_t *dev);
+dt_hash_t dt_dev_hash_distort(dt_develop_t *dev);
 /** same function, but we can specify iop with priority between pmin and pmax */
-uint64_t dt_dev_hash_distort_plus(dt_develop_t *dev,
+dt_hash_t dt_dev_hash_distort_plus(dt_develop_t *dev,
                                   struct dt_dev_pixelpipe_t *pipe,
                                   const double iop_order,
                                   const dt_dev_transform_direction_t transf_direction);

src/develop/imageop.c

@@ -2084,9 +2084,7 @@ void dt_iop_commit_params(dt_iop_module_t *module,
     /* and we add masks */
     dt_masks_group_get_hash_buffer(grp, str + pos);
 
-    uint64_t hash = 5381;
-    for(int i = 0; i < length; i++) hash = ((hash << 5) + hash) ^ str[i];
-    piece->hash = hash;
+    piece->hash = dt_hash(DT_INITHASH, str, length);
 
     free(str);
   }

src/develop/masks.h

@@ -396,7 +396,7 @@ typedef struct dt_masks_form_gui_t
 
   // ids
   dt_mask_id_t formid;
-  uint64_t pipe_hash;
+  dt_hash_t pipe_hash;
 } dt_masks_form_gui_t;
 
 /** special value to indicate an invalid or uninitialized coordinate */

src/develop/pixelpipe_cache.c

@@ -46,8 +46,8 @@ gboolean dt_dev_pixelpipe_cache_init(
   cache->data = (void **) calloc(entries, csize);
   cache->size = (size_t *)((void *)cache->data + entries * sizeof(void *));
   cache->dsc = (dt_iop_buffer_dsc_t *)((void *)cache->size + entries * sizeof(size_t));
-  cache->hash = (uint64_t *)((void *)cache->dsc + entries * sizeof(dt_iop_buffer_dsc_t));
-  cache->used = (int32_t *)((void *)cache->hash + entries * sizeof(uint64_t));
+  cache->hash = (dt_hash_t *)((void *)cache->dsc + entries * sizeof(dt_iop_buffer_dsc_t));
+  cache->used = (int32_t *)((void *)cache->hash + entries * sizeof(dt_hash_t));
   cache->ioporder = (int32_t *)((void *)cache->used + entries * sizeof(int32_t));
 
   for(int k = 0; k < entries; k++)
@@ -103,14 +103,11 @@ void dt_dev_pixelpipe_cache_cleanup(struct dt_dev_pixelpipe_t *pipe)
   cache->data = NULL;
 }
 
-static uint64_t _dev_pixelpipe_cache_basichash(
+static dt_hash_t _dev_pixelpipe_cache_basichash(
            const dt_imgid_t imgid,
            struct dt_dev_pixelpipe_t *pipe,
            const int position)
 {
-  // bernstein hash (djb2)
-  uint64_t hash = 5381;
-
   /* What do we use for the basic hash
        1) imgid as all structures using the hash might possibly contain data from other images
        2) pipe->type for the cache it's important to keep status of fast mode included here
@@ -123,10 +120,7 @@ static uint64_t _dev_pixelpipe_cache_basichash(
   const uint32_t hashing_pipemode[3] = {(uint32_t)imgid,
                                         (uint32_t)pipe->type,
                                         (uint32_t)pipe->want_detail_mask };
-
-  char *pstr = (char *)hashing_pipemode;
-  for(size_t ip = 0; ip < sizeof(hashing_pipemode); ip++)
-    hash = ((hash << 5) + hash) ^ pstr[ip];
+  dt_hash_t hash = dt_hash(DT_INITHASH, &hashing_pipemode, sizeof(hashing_pipemode));
 
   // go through all modules up to position and compute a hash using the operation and params.
   GList *pieces = pipe->nodes;
@@ -141,20 +135,16 @@ static uint64_t _dev_pixelpipe_cache_basichash(
 
     if(!skipped)
     {
-      hash = ((hash << 5) + hash) ^ piece->hash;
+      hash = dt_hash(hash, &piece->hash, sizeof(piece->hash));
       if(piece->module->request_color_pick != DT_REQUEST_COLORPICK_OFF)
       {
         if(darktable.lib->proxy.colorpicker.primary_sample->size == DT_LIB_COLORPICKER_SIZE_BOX)
         {
-          const char *str = (const char *)darktable.lib->proxy.colorpicker.primary_sample->box;
-          for(size_t i = 0; i < sizeof(float) * 4; i++)
-            hash = ((hash << 5) + hash) ^ str[i];
+          hash = dt_hash(hash, darktable.lib->proxy.colorpicker.primary_sample->box, 4 * sizeof(float));
         }
         else if(darktable.lib->proxy.colorpicker.primary_sample->size == DT_LIB_COLORPICKER_SIZE_POINT)
         {
-          const char *str = (const char *)darktable.lib->proxy.colorpicker.primary_sample->point;
-          for(size_t i = 0; i < sizeof(float) * 2; i++)
-            hash = ((hash << 5) + hash) ^ str[i];
+          hash = dt_hash(hash, darktable.lib->proxy.colorpicker.primary_sample->point, 2 * sizeof(float));
         }
       }
     }
@@ -163,28 +153,22 @@ static uint64_t _dev_pixelpipe_cache_basichash(
   return hash;
 }
 
-uint64_t dt_dev_pixelpipe_cache_hash(
+dt_hash_t dt_dev_pixelpipe_cache_hash(
            const dt_imgid_t imgid,
            const dt_iop_roi_t *roi,
            struct dt_dev_pixelpipe_t *pipe,
            const int position)
 {
-  uint64_t hash = _dev_pixelpipe_cache_basichash(imgid, pipe, position);
+  dt_hash_t hash = _dev_pixelpipe_cache_basichash(imgid, pipe, position);
   // also include roi data
-  char *str = (char *)roi;
-  for(size_t i = 0; i < sizeof(dt_iop_roi_t); i++)
-    hash = ((hash << 5) + hash) ^ str[i];
-
-  str = (char *)&pipe->scharr.hash;
-  for(size_t i = 0; i < sizeof(uint64_t); i++)
-    hash = ((hash << 5) + hash) ^ str[i];
-
-  return hash;
+  // FIXME include full roi data in cachelines
+  hash = dt_hash(hash, roi, sizeof(dt_iop_roi_t));
+  return dt_hash(hash, &pipe->scharr.hash, sizeof(pipe->scharr.hash));
 }
 
 gboolean dt_dev_pixelpipe_cache_available(
            dt_dev_pixelpipe_t *pipe,
-           const uint64_t hash,
+           const dt_hash_t hash,
            const size_t size)
 {
   if(pipe->mask_display
@@ -261,7 +245,7 @@ static int _get_cacheline(struct dt_dev_pixelpipe_t *pipe)
 static gboolean _get_by_hash(
           struct dt_dev_pixelpipe_t *pipe,
           struct dt_iop_module_t *module,
-          const uint64_t hash,
+          const dt_hash_t hash,
           const size_t size,
           void **data,
           dt_iop_buffer_dsc_t **dsc)
@@ -302,7 +286,7 @@ static gboolean _get_by_hash(
 
 gboolean dt_dev_pixelpipe_cache_get(
            struct dt_dev_pixelpipe_t *pipe,
-           const uint64_t hash,
+           const dt_hash_t hash,
            const size_t size,
            void **data,
            dt_iop_buffer_dsc_t **dsc,

src/develop/pixelpipe_cache.h

@@ -38,7 +38,7 @@ typedef struct dt_dev_pixelpipe_cache_t
   void **data;
   size_t *size;
   struct dt_iop_buffer_dsc_t *dsc;
-  uint64_t *hash;
+  dt_hash_t *hash;
   int32_t *used;
   int32_t *ioporder;
   uint64_t calls;
@@ -66,19 +66,19 @@ gboolean dt_dev_pixelpipe_cache_init(struct dt_dev_pixelpipe_t *pipe, const int
 void dt_dev_pixelpipe_cache_cleanup(struct dt_dev_pixelpipe_t *pipe);
 
 /** creates a hopefully unique hash from the complete module stack up to the module-th, including current viewport. */
-uint64_t dt_dev_pixelpipe_cache_hash(const dt_imgid_t imgid, const struct dt_iop_roi_t *roi,
+dt_hash_t dt_dev_pixelpipe_cache_hash(const dt_imgid_t imgid, const struct dt_iop_roi_t *roi,
                                      struct dt_dev_pixelpipe_t *pipe, const int position);
 
 /** returns a float data buffer in 'data' for the given hash from the cache, dsc is updated too.
   If the hash does not match any cache line, use an old buffer or allocate a fresh one.
   The size of the buffer in 'data' will be at least of size bytes.
   Returned flag is TRUE for a new buffer
 */
-gboolean dt_dev_pixelpipe_cache_get(struct dt_dev_pixelpipe_t *pipe, const uint64_t hash,
+gboolean dt_dev_pixelpipe_cache_get(struct dt_dev_pixelpipe_t *pipe, const dt_hash_t hash,
                                const size_t size, void **data, struct dt_iop_buffer_dsc_t **dsc, struct dt_iop_module_t *module, const gboolean important);
 
 /** test availability of a cache line without destroying another, if it is not found. */
-gboolean dt_dev_pixelpipe_cache_available(struct dt_dev_pixelpipe_t *pipe, const uint64_t hash, const size_t size);
+gboolean dt_dev_pixelpipe_cache_available(struct dt_dev_pixelpipe_t *pipe, const dt_hash_t hash, const size_t size);
 
 /** invalidates all cachelines. */
 void dt_dev_pixelpipe_cache_flush(const struct dt_dev_pixelpipe_t *pipe);

src/develop/pixelpipe_hb.c

@@ -1385,7 +1385,7 @@ static gboolean _dev_pixelpipe_process_rec(
   if(dt_atomic_get_int(&pipe->shutdown))
     return TRUE;
 
-  uint64_t hash = dt_dev_pixelpipe_cache_hash(pipe->image.id, roi_out, pipe, pos);
+  dt_hash_t hash = dt_dev_pixelpipe_cache_hash(pipe->image.id, roi_out, pipe, pos);
 
   // we do not want data from the preview pixelpipe cache
   // for gamma so we can compute the final scope
@@ -2997,11 +2997,7 @@ gboolean dt_dev_write_scharr_mask(dt_dev_pixelpipe_iop_t *piece,
   if(dt_masks_calc_scharr_mask(&p->scharr, rgb, wb))
     goto error;
 
-  uint64_t hash = 5381;
-  const char *str = (const char *)&p->scharr.roi;
-  for(size_t i = 0; i < sizeof(dt_iop_roi_t); i++)
-    hash = ((hash << 5) + hash) ^ str[i];
-  p->scharr.hash = hash;
+  p->scharr.hash = dt_hash(DT_INITHASH, &p->scharr.roi, sizeof(dt_iop_roi_t));
 
   dt_print_pipe(DT_DEBUG_PIPE, "write scharr mask CPU", p, NULL, roi_in, NULL, "\n");
   if(darktable.dump_pfm_module && (piece->pipe->type & DT_DEV_PIXELPIPE_EXPORT))
@@ -3066,11 +3062,7 @@ gboolean dt_dev_write_scharr_mask_cl(dt_dev_pixelpipe_iop_t *piece,
   p->scharr.data = mask;
   memcpy(&p->scharr.roi, roi_in, sizeof(dt_iop_roi_t));
 
-  uint64_t hash = 5381;
-  const char *str = (const char *)&p->scharr.roi;
-  for(size_t i = 0; i < sizeof(dt_iop_roi_t); i++)
-    hash = ((hash << 5) + hash) ^ str[i];
-  p->scharr.hash = hash;
+  p->scharr.hash = dt_hash(DT_INITHASH, &p->scharr.roi, sizeof(dt_iop_roi_t));
 
   dt_print_pipe(DT_DEBUG_PIPE, "write scharr mask CL", p, NULL, roi_in, NULL, "\n");
   if(darktable.dump_pfm_module && (piece->pipe->type & DT_DEV_PIXELPIPE_EXPORT))

src/develop/pixelpipe_hb.h

@@ -57,7 +57,7 @@ typedef struct dt_dev_pixelpipe_iop_t
 
   float iscale;        // input actually just downscaled buffer? iscale*iwidth = actual width
   int iwidth, iheight; // width and height of input buffer
-  uint64_t hash;       // hash of params and enabled.
+  dt_hash_t hash;       // hash of params and enabled.
   int bpc;             // bits per channel, 32 means float
   int colors;          // how many colors per pixel
   dt_iop_roi_t buf_in,
@@ -93,7 +93,7 @@ typedef enum dt_dev_pixelpipe_status_t
 typedef struct dt_dev_detail_mask_t
 {
   dt_iop_roi_t roi;
-  uint64_t hash;
+  dt_hash_t hash;
   float *data;
 } dt_dev_detail_mask_t;
 
@@ -144,7 +144,7 @@ typedef struct dt_dev_pixelpipe_t
   int backbuf_width, backbuf_height;
   float backbuf_scale;
   float backbuf_zoom_x, backbuf_zoom_y;
-  uint64_t backbuf_hash;
+  dt_hash_t backbuf_hash;
   dt_pthread_mutex_t mutex, backbuf_mutex, busy_mutex;
   int final_width, final_height;