ETag cache validation patches

configure.ac

@@ -1,5 +1,5 @@
 m4_define([version_major], [0])
-m4_define([version_minor], [7])
+m4_define([version_minor], [8])
 m4_define([version_micro], [0])
 m4_define([version_micro_extra], version_micro)
 m4_append([version_micro_extra], [])
@@ -156,6 +156,16 @@ AC_MSG_RESULT([$enable_test_apps])
 
 AC_DEFINE(FUSE_USE_VERSION, 26, [Fuse API Version])
 
+# check if we should enable strict compile warnings
+AC_ARG_ENABLE(strict-compile,
+     AS_HELP_STRING(--enable-strict-compile, enable support for strict compiler warnings),
+     [], [enable_strict_compile=no]
+)
+
+if test "x$enable_strict_compile" = "xyes" ; then
+    CFLAGS="$CFLAGS -Wextra -pedantic -Wall"
+fi
+
 # check if we should enable verbose debugging
 AC_ARG_ENABLE(debug,
      AS_HELP_STRING(--enable-debug, enable support for running in debug mode),

include/cache_mng.h

@@ -44,14 +44,9 @@ guint64 cache_mng_size (CacheMng *cmng);
 // return total size of cached file
 guint64 cache_mng_get_file_length (CacheMng *cmng, fuse_ino_t ino);
 
-// return MD5 of cached file.
-// if result is TRUE then md5str will containd string with MD5 sum
-gboolean cache_mng_get_md5 (CacheMng *cmng, fuse_ino_t ino, gchar **md5str);
-
-// return version ID of cached file
-// return NULL if version ID is not set
-const gchar *cache_mng_get_version_id (CacheMng *cmng, fuse_ino_t ino);
-void cache_mng_update_version_id (CacheMng *cmng, fuse_ino_t ino, const gchar *version_id);
+// return and update local copy of AWS ETag for this file
+const char *cache_mng_get_etag(CacheMng *cmng, fuse_ino_t ino);
+gboolean cache_mng_update_etag(CacheMng *cmng, fuse_ino_t ino, const char *etag);
 
 void cache_mng_get_stats (CacheMng *cmng, guint32 *entries_num, guint64 *total_size, guint64 *cache_hits, guint64 *cache_miss);
 #endif

include/conf_keys.h

@@ -40,10 +40,8 @@ const gchar *conf_keys_str[] = {
     "connection.max_retries",
     "filesystem.dir_cache_max_time",
     "filesystem.file_cache_max_time",
-    "filesystem.md5_enabled",
     "filesystem.cache_enabled",
     "filesystem.cache_dir",
-    "filesystem.cache_dir_max_size",
     "filesystem.cache_object_ttl",
     "filesystem.uid",
     "filesystem.gid",

include/dir_tree.h

@@ -33,8 +33,6 @@ void dir_tree_destroy (DirTree *dtree);
 DirEntry *dir_tree_update_entry (DirTree *dtree, const gchar *path, DirEntryType type,
     fuse_ino_t parent_ino, const gchar *entry_name, long long size, time_t last_modified);
 
-void dir_tree_entry_update_xattrs (DirEntry *en, struct evkeyvalq *headers);
-
 // mark that DirTree is being updated
 
 void dir_tree_start_update (DirEntry *en, G_GNUC_UNUSED const gchar *dir_path);

include/global.h

@@ -61,6 +61,7 @@
 #include <arpa/inet.h>
 #include <netinet/tcp.h>
 #include <sys/stat.h>
+#include <inttypes.h>
 #include <fcntl.h>
 #include <math.h>
 #include <ftw.h>

include/http_connection.h

@@ -81,14 +81,14 @@ typedef void (*HttpConnection_on_entry_sent_cb) (gpointer ctx, gboolean success)
 void http_connection_file_send (HttpConnection *con, int fd, const gchar *resource_path,
     HttpConnection_on_entry_sent_cb on_entry_sent_cb, gpointer ctx);
 
-typedef void (*HttpConnection_responce_cb) (HttpConnection *con, gpointer ctx, gboolean success,
+typedef void (*HttpConnection_response_cb) (HttpConnection *con, gpointer ctx, gboolean success,
         const gchar *buf, size_t buf_len, struct evkeyvalq *headers);
 gboolean http_connection_make_request (HttpConnection *con,
     const gchar *resource_path,
     const gchar *http_cmd,
     struct evbuffer *out_buffer,
     gboolean enable_retry, gpointer parent_request_data,
-    HttpConnection_responce_cb responce_cb,
+    HttpConnection_response_cb response_cb,
     gpointer ctx);
 
 #endif

include/log.h

@@ -35,6 +35,9 @@ void logger_set_color (gboolean use);
 void logger_set_file (FILE *f);
 void logger_destroy (void);
 
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wvariadic-macros"
+
 #define LOG_debug(subsystem, x...) \
 G_STMT_START { \
     logger_log_msg (__FILE__, __LINE__, __func__, LOG_debug, subsystem, x); \
@@ -50,4 +53,6 @@ G_STMT_START { \
     logger_log_msg (__FILE__, __LINE__, __func__, LOG_err, subsystem, x); \
 } G_STMT_END
 
+#pragma GCC diagnostic pop
+
 #endif

riofs.conf.xml

@@ -80,17 +80,18 @@
     <!-- time to keep file attributes cache (seconds) -->
     <file_cache_max_time type="uint">10</file_cache_max_time>
 
-    <!-- set True to enable calculating MD5 sum of file content, increases CPU load -->
-    <md5_enabled type="boolean">True</md5_enabled>
-
     <!-- set True to enable objects caching -->
     <cache_enabled type="boolean">True</cache_enabled>
 
     <!-- directory for storing cache objects -->
     <cache_dir type="string">/tmp/riofs</cache_dir>
 
-    <!-- maximum size of cache directory (1Gb) -->
+    <!-- If cache_dir_max_megabyte_size is set, it applies, -->
+    <!-- ... otherwise cache_dir_max_size applies and must be set. -- >
+    <!-- maximum size of cache directory (1Gb default, in byte units, 4 GByte max) -->
     <cache_dir_max_size type="uint">1073741824</cache_dir_max_size>
+    <!-- maximum size of cache directory (1Gb default, in MByte units, 4 PetaByte max) -->
+    <!-- <cache_dir_max_megabyte_size type="uint">1024</cache_dir_max_megabyte_size> -->
 
     <!-- maximum time of cached object, 10 min -->
     <cache_object_ttl type="uint">600</cache_object_ttl>

src/cache_mng.c

@@ -41,7 +41,7 @@ struct _CacheEntry {
     Range *avail_range;
     time_t modification_time;
     GList *ll_lru;
-    gchar *version_id;
+    gchar *etag;
 };
 
 struct _CacheContext {
@@ -74,7 +74,14 @@ CacheMng *cache_mng_create (Application *app)
     cmng->q_lru = g_queue_new ();
     cmng->size = 0;
     cmng->check_time = time (NULL);
-    cmng->max_size = conf_get_uint (application_get_conf (cmng->app), "filesystem.cache_dir_max_size");
+    // If "filesystem.cache_dir_max_megabyte_size" is set, use it, else use "filesystem.cache_dir_max_size"
+    if (conf_node_exists (application_get_conf (cmng->app), "filesystem.cache_dir_max_megabyte_size")) {
+        cmng->max_size = conf_get_uint (application_get_conf (cmng->app), "filesystem.cache_dir_max_megabyte_size");
+        cmng->max_size *= 1024 * 1024;      // Convert from megabytes to bytes
+    } else {
+        cmng->max_size = conf_get_uint (application_get_conf (cmng->app), "filesystem.cache_dir_max_size");
+    }
+    LOG_debug (CMNG_LOG, "Maximum cache size (bytes): %"PRId64, cmng->max_size);
     // generate random folder name for storing cache
     rnd_str = get_random_string (20, TRUE);
     cmng->cache_dir = g_strdup_printf ("%s/%s",
@@ -110,7 +117,7 @@ static struct _CacheEntry* cache_entry_create (fuse_ino_t ino)
     entry->avail_range = range_create ();
     entry->ll_lru = NULL;
     entry->modification_time = time (NULL);
-    entry->version_id = NULL; // version not set
+    entry->etag = NULL;
 
     return entry;
 }
@@ -120,8 +127,8 @@ static void cache_entry_destroy (gpointer data)
     struct _CacheEntry * entry = (struct _CacheEntry*) data;
 
     range_destroy(entry->avail_range);
-    if (entry->version_id)
-        g_free (entry->version_id);
+    if (entry->etag)
+        g_free (entry->etag);
     g_free(entry);
 }
 
@@ -179,80 +186,35 @@ static void cache_mng_rm_cache_dir (CacheMng *cmng)
     }
 }
 
-
-// we can only get md5 of an object containing 1 range
-// XXX: move code to separate thread
-gboolean cache_mng_get_md5 (CacheMng *cmng, fuse_ino_t ino, gchar **md5str)
-{
-    struct _CacheEntry *entry;
-    unsigned char digest[MD5_DIGEST_LENGTH];
-    MD5_CTX md5ctx;
-    ssize_t bytes;
-    unsigned char data[1024];
-    char path[PATH_MAX];
-    size_t i;
-    gchar *out;
-    FILE *in;
-
-    entry = g_hash_table_lookup (cmng->h_entries, GUINT_TO_POINTER (ino));
-    if (!entry)
-        return FALSE;
-
-    if (range_count (entry->avail_range) != 1) {
-        LOG_debug (CMNG_LOG, INO_H"Entry contains more than 1 range, can't take MD5 sum of such object !", INO_T (ino));
-        return FALSE;
-    }
-
-    cache_mng_file_name (cmng, path, sizeof (path), ino);
-    in = fopen (path, "rb");
-    if (in == NULL) {
-        LOG_debug (CMNG_LOG, INO_H"Can't open file for reading: %s", INO_T (ino), path);
-        return FALSE;
-    }
-
-    MD5_Init (&md5ctx);
-    while ((bytes = fread (data, 1, 1024, in)) != 0)
-        MD5_Update (&md5ctx, data, bytes);
-    MD5_Final (digest, &md5ctx);
-    fclose (in);
-
-    out = g_malloc (33);
-    for (i = 0; i < 16; ++i)
-        sprintf (&out[i*2], "%02x", (unsigned int)digest[i]);
-
-    *md5str = out;
-
-    return TRUE;
-}
-
-// return version ID of cached file
-// return NULL if version ID is not set
-const gchar *cache_mng_get_version_id (CacheMng *cmng, fuse_ino_t ino)
+// What was Amazon's AWS ETag for this inode, when we cached it?
+const gchar *cache_mng_get_etag (CacheMng *cmng, fuse_ino_t ino)
 {
     struct _CacheEntry *entry;
 
     entry = g_hash_table_lookup (cmng->h_entries, GUINT_TO_POINTER (ino));
     if (!entry)
         return NULL;
 
-    return entry->version_id;
+    return entry->etag;
 }
 
-void cache_mng_update_version_id (CacheMng *cmng, fuse_ino_t ino, const gchar *version_id)
+gboolean cache_mng_update_etag (CacheMng *cmng, fuse_ino_t ino, const gchar *etag)
 {
     struct _CacheEntry *entry;
 
     entry = g_hash_table_lookup (cmng->h_entries, GUINT_TO_POINTER (ino));
     if (!entry)
-        return;
+        return FALSE;
 
-    if (entry->version_id) {
-        if (strcmp (entry->version_id, version_id)) {
-            g_free (entry->version_id);
-            entry->version_id = g_strdup (version_id);
+    if (entry->etag) {
+        if (strcmp (entry->etag, etag)) {
+            g_free (entry->etag);
+            entry->etag = g_strdup (etag);
         }
     } else
-        entry->version_id = g_strdup (version_id);
+        entry->etag = g_strdup (etag);
+
+    return TRUE;
 }
 /*}}}*/
 

src/conf.c

@@ -199,6 +199,7 @@ static void text_handler (G_GNUC_UNUSED GMarkupParseContext *context, const gcha
         }
     } else if (conf_node->type == CT_LIST) {
         l = NULL;
+        saved = NULL; // workaround for https://gcc.gnu.org/bugzilla/show_bug.cgi?id=71701
         // split string and remove whitespaces
         for (tok = strtok_r (tmp_text, ",", &saved); tok; tok = strtok_r (NULL, ",", &saved)) {
             tok_striped = g_strstrip (tok);