Add option to download only maps that are more recent on server

src/babel.c

@@ -276,7 +276,7 @@ gboolean a_babel_convert_from_shellcommand ( VikTrwLayer *vt, const char *input_
 
 gboolean a_babel_convert_from_url ( VikTrwLayer *vt, const char *url, const char *input_type, BabelStatusFunc cb, gpointer user_data )
 {
-  static DownloadOptions options = {NULL, 0, a_check_html_file};
+  static DownloadOptions options = { FALSE, NULL, 0, a_check_html_file };
   gint fd_src;
   int fetch_ret;
   gboolean ret = FALSE;

src/curl_download.c

@@ -25,6 +25,10 @@
 
 #include <stdio.h>
 
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
 #include <glib.h>
 #include <glib/gstdio.h>
 #include <glib/gi18n.h>
@@ -107,7 +111,7 @@ void curl_download_init()
   get_cookie_file(TRUE);
 }
 
-int curl_download_uri ( const char *uri, FILE *f, DownloadOptions *options )
+int curl_download_uri ( const char *uri, FILE *f, DownloadOptions *options, time_t time_condition )
 {
   CURL *curl;
   CURLcode res = CURLE_FAILED_INIT;
@@ -116,40 +120,67 @@ int curl_download_uri ( const char *uri, FILE *f, DownloadOptions *options )
   g_debug("%s: uri=%s", __PRETTY_FUNCTION__, uri);
 
   curl = curl_easy_init ();
-  if ( curl )
-    {
-      if (vik_verbose)
-        curl_easy_setopt ( curl, CURLOPT_VERBOSE, 1 );
-      curl_easy_setopt ( curl, CURLOPT_URL, uri );
-      curl_easy_setopt ( curl, CURLOPT_FILE, f );
-      curl_easy_setopt ( curl, CURLOPT_WRITEFUNCTION, curl_write_func);
-      if (options != NULL) {
-        if(options->referer != NULL)
-          curl_easy_setopt ( curl, CURLOPT_REFERER, options->referer);
-        if(options->follow_location != 0) {
-          curl_easy_setopt ( curl, CURLOPT_FOLLOWLOCATION, 1);
-          curl_easy_setopt ( curl, CURLOPT_MAXREDIRS, options->follow_location);
-        }
-      }
-      curl_easy_setopt ( curl, CURLOPT_USERAGENT, PACKAGE "/" VERSION " libcurl/7.15.4" );
-      if ((cookie_file = get_cookie_file(FALSE)) != NULL)
-        curl_easy_setopt(curl, CURLOPT_COOKIEFILE, cookie_file);
-      res = curl_easy_perform ( curl );
-      curl_easy_cleanup ( curl );
+  if ( !curl ) {
+    return DOWNLOAD_ERROR;
+  }
+
+  if (vik_verbose)
+    curl_easy_setopt ( curl, CURLOPT_VERBOSE, 1 );
+  curl_easy_setopt ( curl, CURLOPT_URL, uri );
+  curl_easy_setopt ( curl, CURLOPT_FILE, f );
+  curl_easy_setopt ( curl, CURLOPT_WRITEFUNCTION, curl_write_func);
+  if (options != NULL) {
+    if(options->referer != NULL)
+      curl_easy_setopt ( curl, CURLOPT_REFERER, options->referer);
+    if(options->follow_location != 0) {
+      curl_easy_setopt ( curl, CURLOPT_FOLLOWLOCATION, 1);
+      curl_easy_setopt ( curl, CURLOPT_MAXREDIRS, options->follow_location);
     }
-  return(res);
+    if(options->check_file_server_time && time_condition != 0) {
+      /* if file exists, check against server if file is recent enough */
+      curl_easy_setopt ( curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_IFMODSINCE);
+      curl_easy_setopt ( curl, CURLOPT_TIMEVALUE, time_condition);
+    }
+  }
+  curl_easy_setopt ( curl, CURLOPT_USERAGENT, PACKAGE "/" VERSION " libcurl/7.15.4" );
+  if ((cookie_file = get_cookie_file(FALSE)) != NULL)
+    curl_easy_setopt(curl, CURLOPT_COOKIEFILE, cookie_file);
+  res = curl_easy_perform ( curl );
+  if (res == 0) {
+    glong response;
+    curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response);
+    if (response == 304) {         // 304 = Not Modified
+      res = DOWNLOAD_NO_NEWER_FILE;
+    } else if (response == 200) {  // 200 = Ok
+      gdouble size;
+      /* verify if curl sends us any data - this is a workaround on using CURLOPT_TIMECONDITION 
+         when the server has a (incorrect) time earlier than the time on the file we already have */
+      curl_easy_getinfo(curl, CURLINFO_SIZE_DOWNLOAD, &size);
+      if (size == 0)
+        res = DOWNLOAD_ERROR;
+      else
+        res = DOWNLOAD_NO_ERROR;
+    } else {
+      g_warning("%s: http response: %ld for uri %s (time_condition = %ld)\n", __FUNCTION__, response, uri, time_condition);
+      res = DOWNLOAD_ERROR;
+    }
+  } else {
+    res = DOWNLOAD_ERROR;
+  }
+  curl_easy_cleanup ( curl );
+  return res;
 }
 
-int curl_download_get_url ( const char *hostname, const char *uri, FILE *f, DownloadOptions *options, gboolean ftp )
+int curl_download_get_url ( const char *hostname, const char *uri, FILE *f, DownloadOptions *options, gboolean ftp, time_t time_condition )
 {
   int ret;
   gchar *full = NULL;
 
   /* Compose the full url */
   full = g_strdup_printf ( "%s://%s%s", (ftp?"ftp":"http"), hostname, uri );
-  ret = curl_download_uri ( full, f, options );
+  ret = curl_download_uri ( full, f, options, time_condition );
   g_free ( full );
   full = NULL;
 
-  return (ret ? -2 : 0);   /* -2 HTTP error */
+  return ret;
 }

src/curl_download.h

@@ -27,7 +27,7 @@
 #include "download.h"
 
 void curl_download_init ();
-int curl_download_get_url ( const char *hostname, const char *uri, FILE *f, DownloadOptions *options, gboolean ftp );
-int curl_download_uri ( const char *uri, FILE *f, DownloadOptions *options );
+int curl_download_get_url ( const char *hostname, const char *uri, FILE *f, DownloadOptions *options, gboolean ftp, time_t time_condition );
+int curl_download_uri ( const char *uri, FILE *f, DownloadOptions *options, time_t time_condition );
 
 #endif

src/download.c

@@ -25,11 +25,15 @@
 
 #include <stdio.h>
 #include <ctype.h>
+#include <errno.h>
 #include <string.h>
+#include <sys/types.h>
+#include <utime.h>
 #include <glib.h>
 #include <glib/gstdio.h>
 #include <glib/gi18n.h>
 
+
 #include "download.h"
 
 #include "curl_download.h"
@@ -78,35 +82,39 @@ static int download( const char *hostname, const char *uri, const char *fn, Down
   int ret;
   gchar *tmpfilename;
   gboolean failure = FALSE;
+  time_t time_condition = 0;
 
   /* Check file */
   if ( g_file_test ( fn, G_FILE_TEST_EXISTS ) == TRUE )
   {
-    /* File exists: return */
-    return -3;
+    if (options != NULL && options->check_file_server_time) {
+      /* Get the modified time of this file */
+      struct stat buf;
+      g_stat ( fn, &buf );
+      time_condition = buf.st_mtime;
+    } else {
+      /* Nothing to do as file already exists, so return */
+      return -3;
+    }
   } else {
     gchar *dir = g_path_get_dirname ( fn );
     g_mkdir_with_parents ( dir , 0777 );
     g_free ( dir );
-
-    /* create placeholder file */
-    if ( ! (f = g_fopen ( fn, "w+b" )) ) /* immediately open file so other threads won't -- prevents race condition */
-      return -4;
-    fclose ( f );
-    f = NULL;
   }
 
   tmpfilename = g_strdup_printf("%s.tmp", fn);
-  f = g_fopen ( tmpfilename, "w+b" );
+  f = g_fopen ( tmpfilename, "w+bx" );  /* truncate file and open it in exclusive mode */
   if ( ! f ) {
+    if (errno == EEXIST)
+      g_debug("%s: Couldn't take lock on temporary file \"%s\"\n", __FUNCTION__, tmpfilename);
     g_free ( tmpfilename );
-    g_remove ( fn ); /* couldn't create temporary. delete 0-byte file. */
     return -4;
   }
 
   /* Call the backend function */
-  ret = curl_download_get_url ( hostname, uri, f, options, ftp );
-  if (ret == -1 || ret == 1 || ret == -2) {
+  ret = curl_download_get_url ( hostname, uri, f, options, ftp, time_condition );
+
+  if (ret != DOWNLOAD_NO_ERROR && ret != DOWNLOAD_NO_NEWER_FILE) {
     g_debug("%s: download failed: curl_download_get_url=%d", __FUNCTION__, ret);
     failure = TRUE;
   }
@@ -119,19 +127,22 @@ static int download( const char *hostname, const char *uri, const char *fn, Down
   if (failure)
   {
     g_warning(_("Download error: %s"), fn);
-    fclose ( f );
-    f = NULL;
     g_remove ( tmpfilename );
     g_free ( tmpfilename );
+    fclose ( f );
+    f = NULL;
     g_remove ( fn ); /* couldn't create temporary. delete 0-byte file. */
     return -1;
   }
 
+  if (ret == DOWNLOAD_NO_NEWER_FILE)
+    g_remove ( tmpfilename );
+  else
+    g_rename ( tmpfilename, fn ); /* move completely-downloaded file to permanent location */
+  g_free ( tmpfilename );
   fclose ( f );
   f = NULL;
-  g_rename ( tmpfilename, fn ); /* move completely-downloaded file to permanent location */
-  g_free ( tmpfilename );
-  return ret;
+  return 0;
 }
 
 /* success = 0, -1 = couldn't connect, -2 HTTP error, -3 file exists, -4 couldn't write to file... */

src/download.h

@@ -30,6 +30,12 @@ gboolean a_check_map_file(FILE*);
 gboolean a_check_html_file(FILE*);
 
 typedef struct {
+  /**
+   * Check if the server has a more recent file than the one we have before downloading it
+   * This uses http header If-Modified-Since
+   */
+  gboolean check_file_server_time;
+
   /**
    * The REFERER string to use.
    * Could be NULL.
@@ -46,10 +52,16 @@ typedef struct {
    * File content checker.
    */
   VikFileContentCheckerFunc check_file;
+
 } DownloadOptions;
 
 /* TODO: convert to Glib */
 int a_http_download_get_url ( const char *hostname, const char *uri, const char *fn, DownloadOptions *opt );
 int a_ftp_download_get_url ( const char *hostname, const char *uri, const char *fn, DownloadOptions *opt );
 
+/* Error messages returned by download functions */
+enum { DOWNLOAD_NO_ERROR = 0,
+       DOWNLOAD_NO_NEWER_FILE,
+       DOWNLOAD_ERROR };
+
 #endif

src/expedia.c

@@ -44,10 +44,10 @@ static gboolean expedia_coord_to_mapcoord ( const VikCoord *src, gdouble xzoom,
 static void expedia_mapcoord_to_center_coord ( MapCoord *src, VikCoord *dest );
 static int expedia_download ( MapCoord *src, const gchar *dest_fn );
 
-static DownloadOptions expedia_options = { NULL, 2, a_check_map_file };
+static DownloadOptions expedia_options = { FALSE, NULL, 2, a_check_map_file };
 
 void expedia_init() {
-  VikMapsLayer_MapType map_type = { 5, 0, 0, VIK_VIEWPORT_DRAWMODE_EXPEDIA, expedia_coord_to_mapcoord, expedia_mapcoord_to_center_coord, expedia_download };
+  VikMapsLayer_MapType map_type = { 5, 0, 0, VIK_VIEWPORT_DRAWMODE_EXPEDIA, expedia_coord_to_mapcoord, expedia_mapcoord_to_center_coord, expedia_download, &expedia_options };
   maps_layer_register_type(_("Expedia Street Maps"), 5, &map_type);
 }
 

src/geonamessearch.c

@@ -512,7 +512,7 @@ gchar *download_url(gchar *uri)
   tmp_file = fdopen(tmp_fd, "r+");
 
   // TODO: curl may not be available
-  if (curl_download_uri(uri, tmp_file, NULL)) {  // error
+  if (curl_download_uri(uri, tmp_file, NULL, 0)) {  // error
     fclose(tmp_file);
     tmp_file = NULL;
     g_remove(tmpname);

src/googlesearch.c

@@ -43,7 +43,7 @@ static gchar *last_search_str = NULL;
 static VikCoord *last_coord = NULL;
 static gchar *last_successful_search_str = NULL;
 
-static DownloadOptions googlesearch_options = { "http://maps.google.com/", 0, a_check_map_file };
+static DownloadOptions googlesearch_options = { FALSE, "http://maps.google.com/", 0, a_check_map_file };
 
 gchar * a_googlesearch_get_search_string_for_this_place(VikWindow *vw)
 {
@@ -206,7 +206,7 @@ static int google_search_get_coord(VikWindow *vw, VikViewport *vvp, gchar *srch_
   uri = g_strdup_printf(GOOGLE_SEARCH_URL_FMT, escaped_srch_str);
 
   /* TODO: curl may not be available */
-  if (curl_download_uri(uri, tmp_file, &googlesearch_options)) {  /* error */
+  if (curl_download_uri(uri, tmp_file, &googlesearch_options, 0)) {  /* error */
     fclose(tmp_file);
     tmp_file = NULL;
     ret = -1;

src/osm.c

@@ -52,20 +52,21 @@ static int bluemarble_download ( MapCoord *src, const gchar *dest_fn );
 static int openaerialmap_download ( MapCoord *src, const gchar *dest_fn );
 #endif
 
-static DownloadOptions osm_options = { NULL, 0, a_check_map_file };
+static DownloadOptions osm_options = { TRUE, NULL, 0, a_check_map_file };
 
 /* initialisation */
 void osm_init () {
-  VikMapsLayer_MapType osmarender_type = { 12, 256, 256, VIK_VIEWPORT_DRAWMODE_MERCATOR, osm_coord_to_mapcoord, osm_mapcoord_to_center_coord, osm_osmarender_download };
-  VikMapsLayer_MapType mapnik_type = { 13, 256, 256, VIK_VIEWPORT_DRAWMODE_MERCATOR, osm_coord_to_mapcoord, osm_mapcoord_to_center_coord, osm_mapnik_download };  VikMapsLayer_MapType maplint_type = { 14, 256, 256, VIK_VIEWPORT_DRAWMODE_MERCATOR, osm_coord_to_mapcoord, osm_mapcoord_to_center_coord, osm_maplint_download };
-  VikMapsLayer_MapType cycle_type = { 17, 256, 256, VIK_VIEWPORT_DRAWMODE_MERCATOR, osm_coord_to_mapcoord, osm_mapcoord_to_center_coord, osm_cycle_download };
+  VikMapsLayer_MapType osmarender_type = { 12, 256, 256, VIK_VIEWPORT_DRAWMODE_MERCATOR, osm_coord_to_mapcoord, osm_mapcoord_to_center_coord, osm_osmarender_download, &osm_options };
+  VikMapsLayer_MapType mapnik_type = { 13, 256, 256, VIK_VIEWPORT_DRAWMODE_MERCATOR, osm_coord_to_mapcoord, osm_mapcoord_to_center_coord, osm_mapnik_download, &osm_options };
+  VikMapsLayer_MapType maplint_type = { 14, 256, 256, VIK_VIEWPORT_DRAWMODE_MERCATOR, osm_coord_to_mapcoord, osm_mapcoord_to_center_coord, osm_maplint_download, &osm_options };
+  VikMapsLayer_MapType cycle_type = { 17, 256, 256, VIK_VIEWPORT_DRAWMODE_MERCATOR, osm_coord_to_mapcoord, osm_mapcoord_to_center_coord, osm_cycle_download, &osm_options };
 
 #ifdef VIK_CONFIG_BLUEMARBLE
-  VikMapsLayer_MapType bluemarble_type = { 15, 256, 256, VIK_VIEWPORT_DRAWMODE_MERCATOR, osm_coord_to_mapcoord, osm_mapcoord_to_center_coord, bluemarble_download };
+  VikMapsLayer_MapType bluemarble_type = { 15, 256, 256, VIK_VIEWPORT_DRAWMODE_MERCATOR, osm_coord_to_mapcoord, osm_mapcoord_to_center_coord, bluemarble_download, &osm_options };
 #endif
 
 #ifdef VIK_CONFIG_OPENAERIAL
-  VikMapsLayer_MapType openaerialmap_type = { 20, 256, 256, VIK_VIEWPORT_DRAWMODE_MERCATOR, osm_coord_to_mapcoord, osm_mapcoord_to_center_coord, openaerialmap_download };
+  VikMapsLayer_MapType openaerialmap_type = { 20, 256, 256, VIK_VIEWPORT_DRAWMODE_MERCATOR, osm_coord_to_mapcoord, osm_mapcoord_to_center_coord, openaerialmap_download, &osm_options };
 #endif
 
   maps_layer_register_type("OpenStreetMap (Osmarender)", 12, &osmarender_type);

src/terraserver.c

@@ -47,12 +47,12 @@ static int terraserver_urban_download ( MapCoord *src, const gchar *dest_fn );
 
 static void terraserver_mapcoord_to_center_coord ( MapCoord *src, VikCoord *dest );
 
-static DownloadOptions terraserver_options = { NULL, 0, a_check_map_file };
+static DownloadOptions terraserver_options = { FALSE, NULL, 0, a_check_map_file };
 
 void terraserver_init () {
-  VikMapsLayer_MapType map_type_1 = { 2, 200, 200, VIK_VIEWPORT_DRAWMODE_UTM, terraserver_topo_coord_to_mapcoord, terraserver_mapcoord_to_center_coord, terraserver_topo_download };
-  VikMapsLayer_MapType map_type_2 = { 1, 200, 200, VIK_VIEWPORT_DRAWMODE_UTM, terraserver_aerial_coord_to_mapcoord, terraserver_mapcoord_to_center_coord, terraserver_aerial_download };
-  VikMapsLayer_MapType map_type_3 = { 4, 200, 200, VIK_VIEWPORT_DRAWMODE_UTM, terraserver_urban_coord_to_mapcoord, terraserver_mapcoord_to_center_coord, terraserver_urban_download };
+  VikMapsLayer_MapType map_type_1 = { 2, 200, 200, VIK_VIEWPORT_DRAWMODE_UTM, terraserver_topo_coord_to_mapcoord, terraserver_mapcoord_to_center_coord, terraserver_topo_download, &terraserver_options };
+  VikMapsLayer_MapType map_type_2 = { 1, 200, 200, VIK_VIEWPORT_DRAWMODE_UTM, terraserver_aerial_coord_to_mapcoord, terraserver_mapcoord_to_center_coord, terraserver_aerial_download, &terraserver_options };
+  VikMapsLayer_MapType map_type_3 = { 4, 200, 200, VIK_VIEWPORT_DRAWMODE_UTM, terraserver_urban_coord_to_mapcoord, terraserver_mapcoord_to_center_coord, terraserver_urban_download, &terraserver_options };
 
   maps_layer_register_type("Terraserver Topos", 2, &map_type_1);
   maps_layer_register_type("Terraserver Aerials", 1, &map_type_2);

src/vikdemlayer.c

@@ -763,7 +763,7 @@ static void srtm_dem_download_thread ( DEMDownloadParams *p, gpointer threaddata
 		(intlon >= 0) ? 'E' : 'W',
 		ABS(intlon) );
 
-  static DownloadOptions options = { NULL, 0, a_check_map_file };
+  static DownloadOptions options = { FALSE, NULL, 0, a_check_map_file };
   a_ftp_download_get_url ( SRTM_FTP_SITE, src_fn, p->dest, &options );
   g_free ( src_fn );
 }