Import datetime detection for copy & import and from camera

data/darktableconfig.xml.in

@@ -832,6 +832,13 @@
     <shortdescription>select only new pictures</shortdescription>
     <longdescription>only select images that have not already been imported</longdescription>
   </dtconfig>
+  <dtconfig ui="yes">
+    <name>ui_last/import_select_recognized_new</name>
+    <type>bool</type>
+    <default>true</default>
+    <shortdescription>select only pictures recognized as new</shortdescription>
+    <longdescription>only select images that are not recognized as already imported.\nas based on filename and timestamp matching, the recognition can fail in certain circumstances</longdescription>
+  </dtconfig>
   <dtconfig ui="yes">
     <name>ui_last/import_ignore_jpegs</name>
     <type>bool</type>

src/common/camera_control.c

@@ -122,7 +122,8 @@ static void _camera_process_job(const dt_camctl_t *c, const dt_camera_t *camera,
 static const char *_dispatch_request_image_path(const dt_camctl_t *c, char *exif_time, const dt_camera_t *camera);
 static const char *_dispatch_request_image_filename(const dt_camctl_t *c, const char *filename,
                                                     const char *exif_time, const dt_camera_t *camera);
-static void _dispatch_camera_image_downloaded(const dt_camctl_t *c, const dt_camera_t *camera, const char *filename);
+static void _dispatch_camera_image_downloaded(const dt_camctl_t *c, const dt_camera_t *camera,
+                                              const char *in_path, const char *in_filename, const char *filename);
 static void _dispatch_camera_connected(const dt_camctl_t *c, const dt_camera_t *camera);
 static void _dispatch_camera_disconnected(const dt_camctl_t *c, const dt_camera_t *camera);
 static void _dispatch_control_status(const dt_camctl_t *c, dt_camctl_status_t status);
@@ -271,7 +272,7 @@ static void _camera_process_job(const dt_camctl_t *c, const dt_camera_t *camera,
                                 c->gpcontext) == GP_OK)
           {
             // Notify listeners of captured image
-            _dispatch_camera_image_downloaded(c, camera, output);
+            _dispatch_camera_image_downloaded(c, camera, NULL, NULL, output);
           }
           else
             dt_print(DT_DEBUG_CAMCTL, "[camera_control] failed to download file %s\n", output);
@@ -1159,7 +1160,7 @@ void dt_camctl_import(const dt_camctl_t *c, const dt_camera_t *cam, GList *image
     if(!g_file_set_contents(output, data, size, NULL))
        dt_print(DT_DEBUG_CAMCTL, "[camera_control] failed to write file %s\n", output);
     else
-      _dispatch_camera_image_downloaded(c, cam, output);
+      _dispatch_camera_image_downloaded(c, cam, folder, filename, output);
 
     gp_file_free(camfile);
     g_free(prev_output);
@@ -1180,6 +1181,24 @@ void dt_camctl_select_camera(const dt_camctl_t *c, const dt_camera_t *cam)
   _camctl_unlock(c);
 }
 
+time_t dt_camctl_get_image_file_timestamp(const dt_camctl_t *c, const char *path, const char *filename)
+{
+  time_t timestamp = 0;
+  if(!path || !filename)
+    return 0;
+  // Lets check the type of file...
+  CameraFileInfo cfi;
+  if(!(gp_camera_file_get_info(c->active_camera->gpcam, path, filename, &cfi, c->gpcontext) == GP_OK))
+  {
+    dt_print(DT_DEBUG_CAMCTL,
+    "[camera_control] failed to get file information of %s in folder %s on device\n", filename, path);
+  }
+  else
+    timestamp = cfi.file.mtime;
+
+  return timestamp;
+}
+
 static GList *_camctl_recursive_get_list(const dt_camctl_t *c, char *path)
 {
   GList *imgs = NULL;
@@ -1780,7 +1799,7 @@ static void _camera_poll_events(const dt_camctl_t *c, const dt_camera_t *cam)
                                 c->gpcontext) == GP_OK)
           {
             // Notify listeners of captured image
-            _dispatch_camera_image_downloaded(c, cam, output);
+            _dispatch_camera_image_downloaded(c, cam, NULL, NULL, output);
           }
           else
             dt_print(DT_DEBUG_CAMCTL, "[camera_control] failed to download file %s\n", output);
@@ -1947,7 +1966,6 @@ static const char *_dispatch_request_image_path(const dt_camctl_t *c, char *exif
   return path;
 }
 
-
 static void _dispatch_camera_connected(const dt_camctl_t *c, const dt_camera_t *camera)
 {
   dt_camctl_t *camctl = (dt_camctl_t *)c;
@@ -1974,15 +1992,16 @@ static void _dispatch_camera_disconnected(const dt_camctl_t *c, const dt_camera_
   dt_pthread_mutex_unlock(&camctl->listeners_lock);
 }
 
-static void _dispatch_camera_image_downloaded(const dt_camctl_t *c, const dt_camera_t *camera, const char *filename)
+static void _dispatch_camera_image_downloaded(const dt_camctl_t *c, const dt_camera_t *camera,
+                                              const char *in_path, const char *in_filename, const char *filename)
 {
   dt_camctl_t *camctl = (dt_camctl_t *)c;
   dt_pthread_mutex_lock(&camctl->listeners_lock);
   for(GList *listener = camctl->listeners; listener; listener = g_list_next(listener))
   {
     dt_camctl_listener_t *lstnr = (dt_camctl_listener_t *)listener->data;
     if(lstnr->image_downloaded)
-      lstnr->image_downloaded(camera, filename, lstnr->data);
+      lstnr->image_downloaded(camera, in_path, in_filename, filename, lstnr->data);
   }
   dt_pthread_mutex_unlock(&camctl->listeners_lock);
 }

src/common/camera_control.h

@@ -210,7 +210,8 @@ typedef struct dt_camctl_listener_t
                                         void *data);
 
   /** Invoked when a image is downloaded while in tethered mode or by import */
-  void (*image_downloaded)(const dt_camera_t *camera, const char *filename, void *data);
+  void (*image_downloaded)(const dt_camera_t *camera, const char *in_path, const char *in_filename,
+                           const char *filename, void *data);
 
   /** Invoked when a image is found on storage.. such as from dt_camctl_get_previews(), if 0 is returned the
    * recurse is stopped.. */
@@ -273,6 +274,9 @@ int dt_camctl_can_enter_tether_mode(const dt_camctl_t *c, const dt_camera_t *cam
 void dt_camctl_tether_mode(const dt_camctl_t *c, const dt_camera_t *cam, gboolean enable);
 /** Imports the images in list from specified camera */
 void dt_camctl_import(const dt_camctl_t *c, const dt_camera_t *cam, GList *images);
+/** return the timestamp for file from camera CAUTION camera mutex already own*/
+time_t dt_camctl_get_image_file_timestamp(const dt_camctl_t *c, const char *in_path,
+                                          const char *in_filename);
 /** return the list of images from camera */
 GList *dt_camctl_get_images_list(const dt_camctl_t *c, dt_camera_t *cam);
 /** return the thumbnail of a camera image */

src/common/metadata.c

@@ -49,10 +49,22 @@ static const struct
   {"Xmp.dc.description", N_("description"), DT_METADATA_TYPE_USER, 1},
   {"Xmp.dc.rights", N_("rights"), DT_METADATA_TYPE_USER, 4},
   {"Xmp.acdsee.notes", N_("notes"), DT_METADATA_TYPE_USER, 5},
-  {"Xmp.darktable.version_name", N_("version name"), DT_METADATA_TYPE_OPTIONAL, 6}
+  {"Xmp.darktable.version_name", N_("version name"), DT_METADATA_TYPE_OPTIONAL, 6},
+  {"Xmp.darktable.image_id", N_("image id"), DT_METADATA_TYPE_INTERNAL, 7}
   // clang-format on
 };
 
+unsigned int dt_metadata_get_nb_user_metadata()
+{
+  unsigned int nb = 0;
+  for(unsigned int i = 0; i < DT_METADATA_NUMBER; i++)
+  {
+    if(dt_metadata_def[i].type != DT_METADATA_TYPE_INTERNAL)
+      nb++;
+  }
+  return nb;
+}
+
 const char *dt_metadata_get_name_by_display_order(const uint32_t order)
 {
   if(order < DT_METADATA_NUMBER)
@@ -745,6 +757,31 @@ void dt_metadata_set_list_id(const GList *img, const GList *metadata, const gboo
   }
 }
 
+gboolean dt_metadata_already_imported(const char *filename, const char *datetime)
+{
+  if(!filename || !datetime)
+    return FALSE;
+  char *id = g_strconcat(filename, "-", datetime, NULL);
+  sqlite3_stmt *stmt;
+  DT_DEBUG_SQLITE3_PREPARE_V2(dt_database_get(darktable.db),
+                              "SELECT COUNT(*) FROM main.meta_data WHERE value=?1",
+                              -1, &stmt, NULL);
+  DT_DEBUG_SQLITE3_BIND_TEXT(stmt, 1, id, -1, SQLITE_TRANSIENT);
+  gboolean res = FALSE;
+  if(sqlite3_step(stmt) == SQLITE_ROW && sqlite3_column_int(stmt, 0) != 0)
+    res = TRUE;
+  sqlite3_finalize(stmt);
+  g_free(id);
+  return res;
+}
+
+void dt_metadata_unix_time_to_text(char *exif, const size_t exif_len, const time_t *unix)
+{
+  struct tm tt;
+  (void)localtime_r(unix, &tt);
+  strftime(exif, exif_len, "%Y:%m:%d %H:%M:%S", &tt);
+}
+
 // modelines: These editor modelines have been set for all relevant files by tools/update_modelines.sh
 // vim: shiftwidth=2 expandtab tabstop=2 cindent
 // kate: tab-indents: off; indent-width 2; replace-tabs on; indent-mode cstyle; remove-trailing-spaces modified;

src/common/metadata.h

@@ -33,6 +33,7 @@ typedef enum dt_metadata_t
   DT_METADATA_XMP_DC_RIGHTS,
   DT_METADATA_XMP_ACDSEE_NOTES,
   DT_METADATA_XMP_VERSION_NAME,
+  DT_METADATA_XMP_IMAGE_ID,
   DT_METADATA_NUMBER
 }
 dt_metadata_t;
@@ -61,6 +62,9 @@ typedef enum dt_metadata_flag_t
 }
 dt_metadata_flag_t;
 
+/** return the number of user metadata (!= DT_METADATA_TYPE_INTERNAL) */
+unsigned int dt_metadata_get_nb_user_metadata();
+
 /** return the metadata key by display order */
 const char *dt_metadata_get_name_by_display_order(const uint32_t order);
 
@@ -120,6 +124,12 @@ GList *dt_metadata_get_list_id(int id); // libs/image.c
 /** Remove metadata from specific images, or all selected for id == -1. */
 void dt_metadata_clear(const GList *imgs, const gboolean undo_on); // libs/metadata.c
 
+/** check if the "Xmp.darktable.image_id" already exists */
+gboolean dt_metadata_already_imported(const char *filename, const char *datetime);
+
+/** temporary routine waiting for common datetime.c */
+void dt_metadata_unix_time_to_text(char *exif, const size_t exif_len, const time_t *unix);
+
 // modelines: These editor modelines have been set for all relevant files by tools/update_modelines.sh
 // vim: shiftwidth=2 expandtab tabstop=2 cindent
 // kate: tab-indents: off; indent-width 2; replace-tabs on; indent-mode cstyle; remove-trailing-spaces modified;

src/control/jobs/camera_jobs.c

@@ -220,11 +220,24 @@ dt_job_t *dt_camera_capture_job_create(const char *jobcode, uint32_t delay, uint
 }
 
 /** Listener interface for import job */
-void _camera_import_image_downloaded(const dt_camera_t *camera, const char *filename, void *data)
+void _camera_import_image_downloaded(const dt_camera_t *camera, const char *in_path,
+                                     const char *in_filename, const char *filename, void *data)
 {
   // Import downloaded image to import filmroll
   dt_camera_import_t *t = (dt_camera_import_t *)data;
   const int32_t imgid = dt_image_import(dt_import_session_film_id(t->shared.session), filename, FALSE, TRUE);
+
+  const time_t timestamp = (!in_path || !in_filename) ? 0 :
+               dt_camctl_get_image_file_timestamp(darktable.camctl, in_path, in_filename);
+  if(timestamp && imgid >= 0)
+  {
+    char dt_txt[DT_DATETIME_LENGTH];
+    dt_metadata_unix_time_to_text(dt_txt, sizeof(dt_txt), &timestamp);
+    gchar *id = g_strconcat(in_filename, "-", dt_txt, NULL);
+    dt_metadata_set(imgid, "Xmp.darktable.image_id", id, FALSE);
+    g_free(id);
+  }
+
   dt_control_queue_redraw_center();
   gchar *basename = g_path_get_basename(filename);
   const int num_images = g_list_length(t->images);

src/control/jobs/control_jobs.c

@@ -2083,6 +2083,22 @@ static int _control_import_image_copy(const char *filename,
     if(!imgid) dt_control_log(_("error loading file `%s'"), output);
     else
     {
+      GError *error = NULL;
+      GFile *gfile = g_file_new_for_path(filename);
+      GFileInfo *info = g_file_query_info(gfile,
+                                G_FILE_ATTRIBUTE_STANDARD_NAME ","
+                                G_FILE_ATTRIBUTE_TIME_MODIFIED,
+                                G_FILE_QUERY_INFO_NONE, NULL, &error);
+      const char *fn = g_file_info_get_name(info);
+      // FIXME set a routine common with import.c
+      const time_t datetime = g_file_info_get_attribute_uint64(info, G_FILE_ATTRIBUTE_TIME_MODIFIED);
+      char dt_txt[DT_DATETIME_LENGTH];
+      dt_metadata_unix_time_to_text(dt_txt, sizeof(dt_txt), &datetime);
+      char *id = g_strconcat(fn, "-", dt_txt, NULL);
+      dt_metadata_set(imgid, "Xmp.darktable.image_id", id, FALSE);
+      g_free(id);
+      g_object_unref(info);
+      g_object_unref(gfile);
       *imgs = g_list_prepend(*imgs, GINT_TO_POINTER(imgid));
       if((imgid & 3) == 3)
       {

src/gui/import_metadata.c

@@ -245,6 +245,8 @@ static void _import_metadata_presets_update(dt_import_metadata_t *metadata)
     uint32_t total_len = 0;
     for(unsigned int i = 0; i < DT_METADATA_NUMBER; i++)
     {
+      if(dt_metadata_get_type_by_display_order(i) == DT_METADATA_TYPE_INTERNAL)
+        continue;
       metadata_param[i] = buf;
       metadata_len[i] = strlen(metadata_param[i]) + 1;
       buf += metadata_len[i];
@@ -258,6 +260,8 @@ static void _import_metadata_presets_update(dt_import_metadata_t *metadata)
       gtk_list_store_set(metadata->m_model, &iter, 0, (char *)sqlite3_column_text(stmt, 0), -1);
       for(unsigned int i = 0; i < DT_METADATA_NUMBER; i++)
       {
+        if(dt_metadata_get_type_by_display_order(i) == DT_METADATA_TYPE_INTERNAL)
+          continue;
         gtk_list_store_set(metadata->m_model, &iter, i+1, metadata_param[i], -1);
       }
     }