Skip to content

Commit

Permalink
ETag fix skoobe#5: remove version consistency checking
Browse files Browse the repository at this point in the history 
This patch removes the second no longer used consistency
checking method from fileio_read_on_head_cb(), along with
what seems to be a substantial amount of no longer used
supporting code.
  • Loading branch information
@ThePythonicCow
ThePythonicCow committed Jan 15, 2017
1 parent 0a86470 commit d4a45ef
Show file tree
Hide file tree
Showing 4 changed files with 2 additions and 200 deletions.
5 changes: 0 additions & 5 deletions include/cache_mng.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,11 +44,6 @@ guint64 cache_mng_size (CacheMng *cmng);
// return total size of cached file
guint64 cache_mng_get_file_length (CacheMng *cmng, fuse_ino_t ino);

// return version ID of cached file
// return NULL if version ID is not set
const gchar *cache_mng_get_version_id (CacheMng *cmng, fuse_ino_t ino);
void cache_mng_update_version_id (CacheMng *cmng, fuse_ino_t ino, const gchar *version_id);

// return and update local copy of AWS ETag for this file
const char *cache_mng_get_etag(CacheMng *cmng, fuse_ino_t ino);
gboolean cache_mng_update_etag(CacheMng *cmng, fuse_ino_t ino, const char *etag);
Expand Down
34 changes: 0 additions & 34 deletions src/cache_mng.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,6 @@ struct _CacheEntry {
Range *avail_range;
time_t modification_time;
GList *ll_lru;
gchar *version_id;
gchar *etag;
};

Expand Down Expand Up @@ -118,7 +117,6 @@ static struct _CacheEntry* cache_entry_create (fuse_ino_t ino)
entry->avail_range = range_create ();
entry->ll_lru = NULL;
entry->modification_time = time (NULL);
entry->version_id = NULL; // version not set
entry->etag = NULL;

return entry;
Expand All @@ -129,8 +127,6 @@ static void cache_entry_destroy (gpointer data)
struct _CacheEntry * entry = (struct _CacheEntry*) data;

range_destroy(entry->avail_range);
if (entry->version_id)
g_free (entry->version_id);
if (entry->etag)
g_free (entry->etag);
g_free(entry);
Expand Down Expand Up @@ -190,36 +186,6 @@ static void cache_mng_rm_cache_dir (CacheMng *cmng)
}
}

// return version ID of cached file
// return NULL if version ID is not set
const gchar *cache_mng_get_version_id (CacheMng *cmng, fuse_ino_t ino)
{
struct _CacheEntry *entry;

entry = g_hash_table_lookup (cmng->h_entries, GUINT_TO_POINTER (ino));
if (!entry)
return NULL;

return entry->version_id;
}

void cache_mng_update_version_id (CacheMng *cmng, fuse_ino_t ino, const gchar *version_id)
{
struct _CacheEntry *entry;

entry = g_hash_table_lookup (cmng->h_entries, GUINT_TO_POINTER (ino));
if (!entry)
return;

if (entry->version_id) {
if (strcmp (entry->version_id, version_id)) {
g_free (entry->version_id);
entry->version_id = g_strdup (version_id);
}
} else
entry->version_id = g_strdup (version_id);
}

// What was Amazon's AWS ETag for this inode, when we cached it?
const gchar *cache_mng_get_etag (CacheMng *cmng, fuse_ino_t ino)
{
Expand Down
126 changes: 0 additions & 126 deletions src/file_io_ops.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,89 +99,10 @@ void fileio_destroy (FileIO *fop)

/*{{{ fileio_release*/

/*{{{ update headers on uploaded object */
static void fileio_release_on_update_header_cb (HttpConnection *con, void *ctx, gboolean success,
G_GNUC_UNUSED const gchar *buf, G_GNUC_UNUSED size_t buf_len,
G_GNUC_UNUSED struct evkeyvalq *headers)
{
FileIO *fop = (FileIO *) ctx;

http_connection_release (con);

if (!success) {
LOG_err (FIO_LOG, INO_CON_H"Failed to update headers on the server !", INO_T (fop->ino), (void *)con);
fileio_destroy (fop);
return;
}

// done
LOG_debug (FIO_LOG, INO_CON_H"Headers are updated !", INO_T (fop->ino), (void *)con);

fileio_destroy (fop);
}

// got HttpConnection object
static void fileio_release_on_update_headers_con_cb (gpointer client, gpointer ctx)
{
HttpConnection *con = (HttpConnection *) client;
FileIO *fop = (FileIO *) ctx;
gchar *path;
gchar *cpy_path;
gboolean res;
unsigned char digest[16];
gchar *md5str;
size_t i;

LOG_debug (FIO_LOG, INO_CON_H"Updating object's headers..", INO_T (fop->ino), (void *)con);

http_connection_acquire (con);

if (fop->content_type)
http_connection_add_output_header (con, "Content-Type", fop->content_type);
http_connection_add_output_header (con, "x-amz-metadata-directive", "REPLACE");
http_connection_add_output_header (con, "x-amz-storage-class", conf_get_string (application_get_conf (con->app), "s3.storage_type"));

MD5_Final (digest, &fop->md5);
md5str = g_malloc (33);
for (i = 0; i < 16; ++i)
sprintf(&md5str[i*2], "%02x", (unsigned int)digest[i]);
http_connection_add_output_header (con, "x-amz-meta-md5", md5str);
g_free (md5str);

cpy_path = g_strdup_printf ("%s%s", conf_get_string (application_get_conf (fop->app), "s3.bucket_name"), fop->fname);
http_connection_add_output_header (con, "x-amz-copy-source", cpy_path);
g_free (cpy_path);

path = g_strdup_printf ("%s", fop->fname);
res = http_connection_make_request (con,
path, "PUT", NULL, TRUE, NULL,
fileio_release_on_update_header_cb,
fop
);
g_free (path);

if (!res) {
LOG_err (FIO_LOG, INO_CON_H"Failed to create HTTP request !", INO_T (fop->ino), (void *)con);
http_connection_release (con);
fileio_destroy (fop);
return;
}
}

static void fileio_release_update_headers (FileIO *fop)
{
// update MD5 headers only if versioning is disabled
if (conf_get_boolean (application_get_conf (fop->app), "s3.versioning")) {
LOG_debug (FIO_LOG, INO_H"File uploaded !", INO_T (fop->ino));
fileio_destroy (fop);
} else {
if (!client_pool_get_client (application_get_write_client_pool (fop->app),
fileio_release_on_update_headers_con_cb, fop)) {
LOG_err (FIO_LOG, INO_H"Failed to get HTTP client !", INO_T (fop->ino));
fileio_destroy (fop);
return;
}
}
}
/*}}}*/

Expand All @@ -192,7 +113,6 @@ static void fileio_release_on_complete_cb (HttpConnection *con, void *ctx, gbool
G_GNUC_UNUSED struct evkeyvalq *headers)
{
FileIO *fop = (FileIO *) ctx;
const gchar *versioning_header;

http_connection_release (con);

Expand All @@ -202,12 +122,6 @@ static void fileio_release_on_complete_cb (HttpConnection *con, void *ctx, gbool
return;
}

versioning_header = http_find_header (headers, "x-amz-version-id");
if (versioning_header) {
cache_mng_update_version_id (application_get_cache_mng (fop->app),
fop->ino, versioning_header);
}

// done
LOG_debug (FIO_LOG, INO_CON_H"Multipart Upload is done !", INO_T (fop->ino), (void *)con);

Expand Down Expand Up @@ -281,7 +195,6 @@ static void fileio_release_on_part_sent_cb (HttpConnection *con, void *ctx, gboo
G_GNUC_UNUSED struct evkeyvalq *headers)
{
FileIO *fop = (FileIO *) ctx;
const gchar *versioning_header;

http_connection_release (con);

Expand All @@ -291,11 +204,6 @@ static void fileio_release_on_part_sent_cb (HttpConnection *con, void *ctx, gboo
return;
}

versioning_header = http_find_header (headers, "x-amz-version-id");
if (versioning_header) {
cache_mng_update_version_id (application_get_cache_mng (fop->app),
fop->ino, versioning_header);
}
// if it's a multi part upload - Complete Multipart Upload
if (fop->multipart_initiated) {
fileio_release_complete_multipart (fop);
Expand Down Expand Up @@ -443,7 +351,6 @@ static void fileio_write_on_send_cb (HttpConnection *con, void *ctx, gboolean su
G_GNUC_UNUSED struct evkeyvalq *headers)
{
FileWriteData *wdata = (FileWriteData *) ctx;
const char *versioning_header;

http_connection_release (con);

Expand All @@ -454,12 +361,6 @@ static void fileio_write_on_send_cb (HttpConnection *con, void *ctx, gboolean su
return;
}

versioning_header = http_find_header (headers, "x-amz-version-id");
if (versioning_header) {
cache_mng_update_version_id (application_get_cache_mng (wdata->fop->app),
wdata->ino, versioning_header);
}

// empty part buffer
evbuffer_drain (wdata->fop->write_buf, -1);

Expand Down Expand Up @@ -783,7 +684,6 @@ static void fileio_read_on_get_cb (HttpConnection *con, void *ctx, gboolean succ
const gchar *buf, size_t buf_len, struct evkeyvalq *headers)
{
FileReadData *rdata = (FileReadData *) ctx;
const char *versioning_header = NULL;
const char *cached_etag;

// release HttpConnection
Expand All @@ -804,12 +704,6 @@ static void fileio_read_on_get_cb (HttpConnection *con, void *ctx, gboolean succ
rdata->ino, buf_len, rdata->request_offset, (unsigned char *) buf,
NULL, NULL);

// update version ID
versioning_header = http_find_header (headers, "x-amz-version-id");
if (versioning_header) {
cache_mng_update_version_id (application_get_cache_mng (rdata->fop->app), rdata->ino, versioning_header);
}

cached_etag = cache_mng_get_etag (application_get_cache_mng (rdata->fop->app), rdata->ino);
LOG_debug (FIO_LOG, INO_H"Read from server done, AWS etag %.8s..., cache etag %.8s...",
INO_T (rdata->ino), rdata->aws_etag+1, cached_etag ? cached_etag+1 : "not set");
Expand Down Expand Up @@ -985,26 +879,6 @@ static void fileio_read_on_head_cb (HttpConnection *con, void *ctx, gboolean suc
}
}

// 3. if versioning is enabled: compare version IDs to check that local and remote files are identical
if (conf_get_boolean (application_get_conf (rdata->fop->app), "s3.versioning")) {
const char *versioning_header = http_find_header (headers, "x-amz-version-id");
if (versioning_header) {
const gchar *local_version_id = cache_mng_get_version_id (application_get_cache_mng (rdata->fop->app), rdata->ino);
if (local_version_id && !strcmp (local_version_id, versioning_header)) {
LOG_debug (FIO_LOG, INO_H"Both version IDs match, using local cached file!", INO_T (rdata->ino));
} else {
LOG_debug (FIO_LOG, INO_H"Version IDs do not match, invalidating local cached file!: %s %s",
INO_T (rdata->ino), local_version_id, versioning_header);
cache_mng_remove_file (application_get_cache_mng (rdata->fop->app), rdata->ino);
}

// header was not found
} else {
LOG_debug (FIO_LOG, INO_H"Versioning header was not found, invalidating local cached file!", INO_T (rdata->ino));
cache_mng_remove_file (application_get_cache_mng (rdata->fop->app), rdata->ino);
}
}

// resume downloading file
fileio_read_get_buf (rdata);

Expand Down
37 changes: 2 additions & 35 deletions src/main.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -289,7 +289,7 @@ static void sigusr1_cb (G_GNUC_UNUSED evutil_socket_t sig, G_GNUC_UNUSED short e
LOG_err (APP_LOG, "Failed to parse configuration file: %s", _app->conf_path);
conf_destroy(conf_new);
} else {
const gchar *copy_entries[] = {"s3.host", "s3.port", "s3.versioning", "s3.access_key_id", "s3.secret_access_key", "s3.bucket_name", NULL};
const gchar *copy_entries[] = {"s3.host", "s3.port", "s3.access_key_id", "s3.secret_access_key", "s3.bucket_name", NULL};
int i;

_app->conf = conf_new;
Expand Down Expand Up @@ -483,39 +483,6 @@ static gint application_finish_initialization_and_run (Application *app)
}
/*}}}*/

/*{{{ application_on_bucket_versioning_cb */
// replies on bucket versioning information
static void application_on_bucket_versioning_cb (gpointer ctx, gboolean success,
const gchar *buf, size_t buf_len)
{
Application *app = (Application *)ctx;
gchar *tmp;

if (!success) {
LOG_err (APP_LOG, "Failed to get bucket versioning!");
application_exit (app);
return;
}

if (buf_len > 1) {
tmp = (gchar *)buf;
tmp[buf_len - 1] = '\0';

if (strstr (buf, "<Status>Enabled</Status>")) {
LOG_debug (APP_LOG, "Bucket has versioning enabled !");
conf_set_boolean (app->conf, "s3.versioning", TRUE);
} else {
LOG_debug (APP_LOG, "Bucket has versioning disabled !");
conf_set_boolean (app->conf, "s3.versioning", FALSE);
}
} else {
conf_set_boolean (app->conf, "s3.versioning", FALSE);
}

application_finish_initialization_and_run (app);
}
/*}}}*/

/*{{{ application_on_bucket_acl_cb */
// replies on bucket ACL
static void application_on_bucket_acl_cb (gpointer ctx, gboolean success,
Expand All @@ -531,7 +498,7 @@ static void application_on_bucket_acl_cb (gpointer ctx, gboolean success,

// XXX: check ACL permissions

bucket_client_get (app->service_con, "/?versioning", application_on_bucket_versioning_cb, app);
application_finish_initialization_and_run (app);
}
/*}}}*/

Expand Down

0 comments on commit d4a45ef

Please sign in to comment.