Permalink
Browse files

Implemented ICANHAZ 'cache' field

  • Loading branch information...
hintjens committed Nov 15, 2012
1 parent b6556f3 commit fa3e77454b55ed33719cfe95c1e681f5f68fc15c
View
@@ -31,3 +31,7 @@ fmq_selftest
src/fmqroot/send/*
src/fmqroot/recv/*
src/fmqroot/logs/*
+.cache
+mymusic
+testit
+
View
@@ -79,7 +79,7 @@ path = string ; Full path or path prefix
options = dictionary
dictionary = size *key-value
key-value = string ; Formatted as name=value
-cache = dictionary ; File SHA1 signatures
+cache = dictionary ; File SHA-1 signatures
; The server confirms the subscription
ICANHAZ-OK = signature %x06
@@ -161,7 +161,13 @@ When the server grants the client access after an OHAI or YARL command, it SHALL
The client MAY subscribe to any number of virtual paths by sending ICANHAZ commands to the server. The client MAY specify a full path, or it MAY specify a partial path, which is used as a prefix match. Paths MUST start with "/", thus the path "/" subscribes to //everything//.
-The options field is reserved for future use. The path does not have to exist in the server. That is, clients can request paths which will exist in the server at some future time.
+The 'path' does not have to exist in the server. That is, clients can request paths which will exist in the server at some future time.
+
+The 'options' field provides additional information to the server. The server SHOULD implement these options:
+
+* {{RESYNC=1}} - if the client sets this, the server SHALL send the full contents of the virtual path to the client.
+
+The 'cache' dictionary field tells the server which files the client already has. The server SHOULD respect this information. Each entry in the 'cache' dictionary is a "filename=digest" key/value pair where the digest SHALL be a SHA-1 digest in printable hexadecimal format. If the filename starts with '/' then it SHOULD start with the path, otherwise the server MUST ignore it. If the filename does not start with '/' then the server SHALL treat it as relative to the path.
++++ The ICANHAZ-OK Command
View
@@ -36,6 +36,11 @@
#define FMQ_VERSION \
FMQ_MAKE_VERSION(FMQ_VERSION_MAJOR, FMQ_VERSION_MINOR, FMQ_VERSION_PATCH)
+// Maximum length of a path + filename
+#ifndef PATH_MAX
+#define PATH_MAX 1024
+#endif
+
// These are reusable utility classes
// TODO: the generators and required classes should not be part of FMQ
// as such, but a separate project so they can be reused more widely.
@@ -44,7 +49,7 @@
#include "fmq_dir.h"
#include "fmq_patch.h"
#include "fmq_sasl.h"
-#include "fmq_sha.h"
+#include "fmq_hash.h"
#include "fmq_config.h"
// These are specific to the FileMQ implementation
View
@@ -72,6 +72,12 @@ void
void
fmq_dir_dump (fmq_dir_t *self, int indent);
+// Load directory cache; returns a hash table containing the SHA-1 digests
+// of every file in the tree. The cache is saved between runs in .cache.
+// The caller must destroy the hash table when done with it.
+zhash_t *
+ fmq_dir_cache (fmq_dir_t *self);
+
// Self test of this class
int
fmq_dir_test (bool verbose);
View
@@ -96,6 +96,14 @@ int
void
fmq_file_close (fmq_file_t *self);
+// Return file handle, if opened
+FILE *
+ fmq_file_handle (fmq_file_t *self);
+
+// Return file SHA-1 hash as string; caller has to free it
+char *
+ fmq_file_hash (fmq_file_t *self);
+
// Self test of this class
int
fmq_file_test (bool verbose);
View
@@ -1,5 +1,5 @@
/* =========================================================================
- fmq_sha - wraps the sha-1.1 library
+ fmq_hash - provides hashing functions (SHA-1 at present)
-------------------------------------------------------------------------
Copyright (c) 1991-2012 iMatix Corporation -- http://www.imatix.com
@@ -22,39 +22,39 @@
=========================================================================
*/
-#ifndef __FMQ_SHA_H_INCLUDED__
-#define __FMQ_SHA_H_INCLUDED__
+#ifndef __FMQ_HASH_H_INCLUDED__
+#define __FMQ_HASH_H_INCLUDED__
#ifdef __cplusplus
extern "C" {
#endif
// Opaque class structure
-typedef struct _fmq_sha_t fmq_sha_t;
+typedef struct _fmq_hash_t fmq_hash_t;
-// Create new SHA object
-fmq_sha_t *
- fmq_sha_new (void);
+// Create new HASH object
+fmq_hash_t *
+ fmq_hash_new (void);
-// Destroy a SHA object
+// Destroy a HASH object
void
- fmq_sha_destroy (fmq_sha_t **self_p);
+ fmq_hash_destroy (fmq_hash_t **self_p);
-// Add buffer into SHA calculation
+// Add buffer into HASH calculation
void
- fmq_sha_update (fmq_sha_t *self, byte *buffer, size_t length);
+ fmq_hash_update (fmq_hash_t *self, byte *buffer, size_t length);
-// Return final SHA hash data
+// Return final HASH hash data
byte *
- fmq_sha_hash_data (fmq_sha_t *self);
+ fmq_hash_data (fmq_hash_t *self);
-// Return final SHA hash size
+// Return final HASH hash size
size_t
- fmq_sha_hash_size (fmq_sha_t *self);
+ fmq_hash_size (fmq_hash_t *self);
// Self test of this class
int
- fmq_sha_test (bool verbose);
+ fmq_hash_test (bool verbose);
#ifdef __cplusplus
}
View
@@ -40,6 +40,7 @@
ICANHAZ - Client subscribes to a path
path string
options dictionary
+ cache dictionary
ICANHAZ_OK - Server confirms the subscription
NOM - Client sends credit to the server
credit number 8
@@ -127,7 +128,8 @@ int
int
fmq_msg_send_icanhaz (void *output,
char *path,
- zhash_t *options);
+ zhash_t *options,
+ zhash_t *cache);
// Send the ICANHAZ_OK to the output in one step
int
@@ -249,6 +251,22 @@ void
size_t
fmq_msg_options_size (fmq_msg_t *self);
+// Get/set the cache field
+zhash_t *
+ fmq_msg_cache (fmq_msg_t *self);
+void
+ fmq_msg_cache_set (fmq_msg_t *self, zhash_t *cache);
+
+// Get/set a value in the cache dictionary
+char *
+ fmq_msg_cache_string (fmq_msg_t *self, char *key, char *default_value);
+uint64_t
+ fmq_msg_cache_number (fmq_msg_t *self, char *key, uint64_t default_value);
+void
+ fmq_msg_cache_insert (fmq_msg_t *self, char *key, char *format, ...);
+size_t
+ fmq_msg_cache_size (fmq_msg_t *self);
+
// Get/set the credit field
uint64_t
fmq_msg_credit (fmq_msg_t *self);
View
@@ -61,6 +61,18 @@ fmq_file_t *
fmq_patch_op_t
fmq_patch_op (fmq_patch_t *self);
+// Return patch virtual file name
+char *
+ fmq_patch_virtual (fmq_patch_t *self);
+
+// Set patch virtual file name
+void
+ fmq_patch_virtual_set (fmq_patch_t *self, char *virtual);
+
+// Return hash digest for patch file (create only)
+char *
+ fmq_patch_hashstr (fmq_patch_t *self);
+
// Self test of this class
int
fmq_patch_test (bool verbose);
View
@@ -354,7 +354,7 @@ client_apply_config (client_t *self)
entry = fmq_config_next (entry);
}
.for class.method
- if (streq (fmq_config_name (section), "$(name)")) {
+ if (streq (fmq_config_name (section), "$(name:c)")) {
. for argument
. if type = "string"
char *$(name) = fmq_config_resolve (section, "$(name)", "?");
View
@@ -505,14 +505,14 @@ $(class.name)_recv (void *input)
. elsif type = "dictionary"
GET_NUMBER1 (hash_size);
self->$(name) = zhash_new ();
+ zhash_autofree (self->$(name));
while (hash_size--) {
char *string;
GET_STRING (string);
char *value = strchr (string, '=');
if (value)
*value++ = 0;
zhash_insert (self->$(name), string, strdup (value));
- zhash_freefn (self->$(name), string, free);
free (string);
}
. elsif type = "frame"
@@ -1145,10 +1145,11 @@ $(class.name)_$(name)_insert ($(class.name)_t *self, char *key, char *format, ..
va_end (argptr);
// Store string in hash table
- if (!self->$(name))
+ if (!self->$(name)) {
self->$(name) = zhash_new ();
- if (zhash_insert (self->$(name), key, string) == 0)
- zhash_freefn (self->$(name), key, free);
+ zhash_autofree (self->$(name));
+ }
+ zhash_update (self->$(name), key, string);
}
size_t
View
@@ -23,6 +23,7 @@ This is the FILEMQ/1.0 client protocol handler
<state name = "subscribing">
<event name = "ok" next = "subscribing">
+ <action name = "format icanhaz command" />
<action name = "send" message = "ICANHAZ" />
<action name = "get next subscription" />
</event>
@@ -40,6 +41,7 @@ This is the FILEMQ/1.0 client protocol handler
<action name = "send" message = "HUGZ-OK" />
</event>
<event name = "subscribe">
+ <action name = "format icanhaz command" />
<action name = "send" message = "ICANHAZ" />
</event>
<event name = "send credit">
@@ -74,6 +76,7 @@ This is the FILEMQ/1.0 client protocol handler
<context>
bool connected; // Are we connected to server?
zlist_t *subs; // Subscriptions
+sub_t *sub; // Subscription we want to send
size_t credit; // Current credit pending
fmq_file_t *file; // File we're writing to
</context>
@@ -113,25 +116,26 @@ self->connected = true;
</action>
<action name = "get first subscription">
-sub_t *sub = (sub_t *) zlist_first (self->subs);
-if (sub) {
- fmq_msg_path_set (self->request, sub->path);
+self->sub = (sub_t *) zlist_first (self->subs);
+if (self->sub)
self->next_event = ok_event;
-}
else
self->next_event = finished_event;
</action>
<action name = "get next subscription">
-sub_t *sub = (sub_t *) zlist_next (self->subs);
-if (sub) {
- fmq_msg_path_set (self->request, sub->path);
+self->sub = (sub_t *) zlist_next (self->subs);
+if (self->sub)
self->next_event = ok_event;
-}
else
self->next_event = finished_event;
</action>
+<action name = "format icanhaz command">
+fmq_msg_path_set (self->request, self->sub->path);
+fmq_msg_cache_set (self->request, zhash_dup (self->sub->cache));
+</action>
+
<action name = "refill credit as needed">
// If credit has fallen too low, send more credit
size_t credit_to_send = 0;
@@ -149,6 +153,10 @@ if (credit_to_send) {
char *inbox = fmq_config_resolve (self->config, "client/inbox", ".inbox");
char *filename = fmq_msg_filename (self->reply);
+// Filenames from server must start with slash, which we skip
+assert (*filename == '/');
+filename++;
+
if (fmq_msg_operation (self->reply) == FMQ_MSG_FILE_CREATE) {
if (self->file == NULL) {
zclock_log ("I: create %s/%s", inbox, filename);
@@ -169,6 +177,7 @@ if (fmq_msg_operation (self->reply) == FMQ_MSG_FILE_CREATE) {
else
// Zero-sized chunk means end of file
fmq_file_destroy (&self->file);
+
fmq_chunk_destroy (&chunk);
}
else
@@ -201,27 +210,30 @@ self->next_event = terminate_event;
<argument name = "path" type = "string" />
// Store subscription along with any previous ones
// Check we don't already have a subscription for this path
-sub_t *sub = (sub_t *) zlist_first (self->subs);
-while (sub) {
- if (streq (path, sub->path))
+self->sub = (sub_t *) zlist_first (self->subs);
+while (self->sub) {
+ if (streq (path, self->sub->path))
return;
- sub = (sub_t *) zlist_next (self->subs);
+ self->sub = (sub_t *) zlist_next (self->subs);
}
// Subscription path must start with '/'
// We'll do better error handling later
assert (*path == '/');
-// New subscription, so store it for later replay
-sub = sub_new (self, path);
-zlist_append (self->subs, sub);
-zclock_log ("I: subscribing directory %s as '%s'",
- fmq_config_resolve (self->config, "client/inbox", ".inbox"), path, path);
-
+// Get directory cache for this path
+char *inbox = fmq_config_resolve (self->config, "client/inbox", ".inbox");
+fmq_dir_t *dir = fmq_dir_new (path + 1, inbox);
+zhash_t *cache = dir? fmq_dir_cache (dir): NULL;
+fmq_dir_destroy (&dir);
+
+// New subscription, store it for later replay
+self->sub = sub_new (self, path, cache);
+zlist_append (self->subs, self->sub);
+zclock_log ("I: subscribe to %s as %s%s", path, inbox, path);
+
// If we're connected, then also send to server
-if (self->connected) {
- fmq_msg_path_set (self->request, path);
+if (self->connected)
self->next_event = subscribe_event;
-}
</method>
<method name = "set inbox">
View
@@ -2,13 +2,15 @@
// Subscription in memory
typedef struct {
char *path; // Path we subscribe to
+ zhash_t *cache; // Existing files, as digests
} sub_t;
static sub_t *
-sub_new (client_t *client, char *path)
+sub_new (client_t *client, char *path, zhash_t *cache)
{
sub_t *self = (sub_t *) zmalloc (sizeof (sub_t));
self->path = strdup (path);
+ self->cache = cache; // Take ownership
return self;
}
@@ -18,6 +20,7 @@ sub_destroy (sub_t **self_p)
assert (self_p);
if (*self_p) {
sub_t *self = *self_p;
+ zhash_destroy (&self->cache);
free (self->path);
free (self);
*self_p = NULL;
View
@@ -35,6 +35,7 @@ Server grants the client access
Client subscribes to a path
<field name = "path" type = "string" />
<field name = "options" type = "dictionary" />
+ <field name = "cache" type = "dictionary" />
</message>
<message name = "ICANHAZ-OK" id = "6">
Oops, something went wrong.

0 comments on commit fa3e774

Please sign in to comment.