diff --git a/.gitignore b/.gitignore index 93dfb05..8151295 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ *.a *.dSYM *.pyc +*.deps build dist sortdb/sortdb diff --git a/conftest.py b/conftest.py new file mode 100644 index 0000000..51ec0d0 --- /dev/null +++ b/conftest.py @@ -0,0 +1,3 @@ +# needed for py.test to accept the --valgrind option +def pytest_addoption(parser): + parser.addoption("--no-valgrind", action="store_true", help="disable valgrind analysis") diff --git a/queuereader/queuereader.c b/queuereader/queuereader.c index 8176b8e..bc8b49d 100644 --- a/queuereader/queuereader.c +++ b/queuereader/queuereader.c @@ -1,3 +1,4 @@ +#define _GNU_SOURCE // for strndup() #include #include #include @@ -28,7 +29,6 @@ void queuereader_increment_backoff(); void queuereader_decrement_backoff(); static char *message = NULL; -static size_t message_len; static int backoff_counter = 0; static struct GlobalData *data = NULL; static struct event ev; @@ -137,7 +137,6 @@ void queuereader_source_cb(struct evhttp_request *req, void *cbarg) struct GlobalData *client_data = (struct GlobalData *)cbarg; char *line; size_t line_len; - char *tmp = NULL; struct evbuffer *evb; int ret = QR_EMPTY; @@ -150,9 +149,7 @@ void queuereader_source_cb(struct evhttp_request *req, void *cbarg) line = (char *)EVBUFFER_DATA(evb); line_len = EVBUFFER_LENGTH(evb); if (line_len) { - message = line; - message_len = line_len; - DUPE_N_TERMINATE(message, message_len, tmp); + message = strndup(line, line_len); ret = (*client_data->message_cb)(message, client_data->cbarg); } diff --git a/shared_tests/test_shunt.py b/shared_tests/test_shunt.py index cf94423..0bb90e7 100644 --- a/shared_tests/test_shunt.py +++ b/shared_tests/test_shunt.py @@ -34,26 +34,26 @@ def http_fetch(endpoint, params=None, response_code=200, body=None): assert res.code == response_code return res.body - -def valgrind_cmd(cmd, *options): +def valgrind_cmd(test_output_dir, *options): assert isinstance(options, (list, tuple)) - assert cmd.startswith("/"), "valgrind_cmd must take a fully qualified executible path not %s" % cmd - test_output_dir = os.path.join(os.path.dirname(cmd), "test_output") - return [ + cmdlist = list(options) + if '--no-valgrind' not in sys.argv: + cmdlist = [ 'valgrind', '-v', '--tool=memcheck', '--trace-children=yes', - # '--demangle=yes', '--log-file=%s/vg.out' % test_output_dir, '--leak-check=full', - '--show-reachable=yes', + #'--show-reachable=yes', '--run-libc-freeres=yes', - '%s' % cmd, - ] + list(options) - + ] + cmdlist + return cmdlist def check_valgrind_output(filename): + if '--no-valgrind' in sys.argv: + return + assert os.path.exists(filename) time.sleep(.15) vg_output = open(filename, 'r').readlines() @@ -75,30 +75,26 @@ def check_valgrind_output(filename): assert lost assert lost[0] == "possibly lost: 0 bytes in 0 blocks" - class SubprocessTest(unittest.TestCase): process_options = [] binary_name = "" working_dir = None - def setUp(self): - """setup method that starts up mongod instances using `self.mongo_options`""" + test_output_dir = None + + @classmethod + def setUpClass(self): + """setup method that starts up target instances using `self.process_options`""" self.temp_dirs = [] self.processes = [] assert self.binary_name, "you must override self.binary_name" assert self.working_dir, "set workign dir to os.path.dirname(__file__)" - exe = os.path.join(self.working_dir, self.binary_name) - if os.path.exists(exe): - logging.info('removing old %s' % exe) - os.unlink(exe) - + # make should update the executable if needed logging.info('running make') - pipe = subprocess.Popen(['make']) - pipe.wait() - - assert os.path.exists(exe), "compile failed" + pipe = subprocess.Popen(['make', '-C', self.working_dir]) + assert pipe.wait() == 0, "compile failed" - test_output_dir = os.path.join(self.working_dir, "test_output") + test_output_dir = self.test_output_dir if os.path.exists(test_output_dir): logging.info('removing %s' % test_output_dir) pipe = subprocess.Popen(['rm', '-rf', test_output_dir]) @@ -108,16 +104,14 @@ def setUp(self): os.makedirs(test_output_dir) for options in self.process_options: - logging.info(' '.join(options)) - # self.stdout = open(test_output_dir + '/test.out', 'w') - # self.stderr = open(test_output_dir + '/test.err', 'w') - pipe = subprocess.Popen(options)#, stdout=self.stdout, stderr=self.stderr) + pipe = subprocess.Popen(options) self.processes.append(pipe) logging.debug('started process %s' % pipe.pid) - self.wait_for('http://127.0.0.1:8080/', max_time=5) + self.wait_for('http://127.0.0.1:8080/', max_time=9) + @classmethod def wait_for(self, url, max_time): # check up to 15 times till the endpoint specified is available waiting max_time step = max_time / float(15) @@ -131,6 +125,7 @@ def wait_for(self, url, max_time): pass time.sleep(step) + @classmethod def graceful_shutdown(self): try: http_fetch('/exit', dict()) @@ -138,8 +133,9 @@ def graceful_shutdown(self): # we never get a reply if this works correctly time.sleep(1) - def tearDown(self): - """teardown method that cleans up child mongod instances, and removes their temporary data files""" + @classmethod + def tearDownClass(self): + """teardown method that cleans up child target instances, and removes their temporary data files""" logging.debug('teardown') try: self.graceful_shutdown() diff --git a/simplehttp/simplehttp.h b/simplehttp/simplehttp.h index ab9198f..afd2ac2 100644 --- a/simplehttp/simplehttp.h +++ b/simplehttp/simplehttp.h @@ -7,13 +7,6 @@ #include #define SIMPLEHTTP_VERSION "0.1.3" -#ifndef DUPE_N_TERMINATE -#define DUPE_N_TERMINATE(buf, len, tmp) \ - tmp = malloc((len) + 1); \ - memcpy(tmp, buf, (len)); \ - tmp[(len)] = '\0'; \ - buf = tmp; -#endif #if _POSIX_TIMERS > 0 diff --git a/simplehttp/util.c b/simplehttp/util.c index 909d6d0..c0c1a02 100644 --- a/simplehttp/util.c +++ b/simplehttp/util.c @@ -1,3 +1,4 @@ +#define _GNU_SOURCE // for strndup() #include #include #include @@ -58,7 +59,6 @@ uint64_t ninety_five_percent(int64_t *int_array, int length) int simplehttp_parse_url(char *endpoint, size_t endpoint_len, char **address, int *port, char **path) { // parse out address, port, path - char *tmp = NULL; char *tmp_port = NULL; char *tmp_pointer; size_t address_len; @@ -98,8 +98,8 @@ int simplehttp_parse_url(char *endpoint, size_t endpoint_len, char **address, in } path_len = (endpoint + endpoint_len) - *path; - DUPE_N_TERMINATE(*address, address_len, tmp); - DUPE_N_TERMINATE(*path, path_len, tmp); + *address = strndup(*address, address_len); + *path = strndup(*path, path_len); return 1; } diff --git a/simpleleveldb/Makefile b/simpleleveldb/Makefile index 55928d1..31d9eca 100644 --- a/simpleleveldb/Makefile +++ b/simpleleveldb/Makefile @@ -11,16 +11,19 @@ AR = ar AR_FLAGS = rc RANLIB = ranlib -all: simpleleveldb leveldb_to_csv csv_to_leveldb +TARGETS = simpleleveldb leveldb_to_csv csv_to_leveldb -leveldb_to_csv: leveldb_to_csv.c - $(CC) $(CFLAGS) -o $@ leveldb_to_csv.c $(LIBS) +SOURCES_simpleleveldb = simpleleveldb.c str_list_set.c +SOURCES_leveldb_to_csv = leveldb_to_csv.c +SOURCES_csv_to_leveldb = csv_to_leveldb.c -csv_to_leveldb: csv_to_leveldb.c - $(CC) $(CFLAGS) -o $@ csv_to_leveldb.c $(LIBS) +all: $(TARGETS) -simpleleveldb: simpleleveldb.c - $(CC) $(CFLAGS) -o $@ simpleleveldb.c $(LIBS) +-include $(TARGETS:%=%.deps) + +$(TARGETS): %: %.c + $(CC) $(CFLAGS) -MM -MT $@ -MF $@.deps $(SOURCES_$@) + $(CC) $(CFLAGS) -o $@ $(SOURCES_$@) $(LIBS) install: /usr/bin/install -d $(TARGET)/bin/ @@ -29,4 +32,4 @@ install: /usr/bin/install csv_to_leveldb $(TARGET)/bin/ clean: - rm -rf *.a *.o simpleleveldb leveldb_to_csv csv_to_leveldb *.dSYM + rm -rf *.a *.o *.deps *.dSYM $(TARGETS) diff --git a/simpleleveldb/README.md b/simpleleveldb/README.md index 72eb846..efb9be4 100644 --- a/simpleleveldb/README.md +++ b/simpleleveldb/README.md @@ -51,39 +51,95 @@ OPTIONS API endpoints: * /get - - parameters: `key`, `format` - + + parameters: `key` + * /mget - parameters: `key` (multiple), `format` + parameters: `key` (multiple) * /fwmatch - parameters: `key`, `limit` + parameters: `key`, `limit` (default 500) * /range_match - parameters: `start`, `end`, `limit` + parameters: `start`, `end`, `limit` (default 500) * /put - parameters: `key`, `value`, `format` - + parameters: `key`, `value` + Note: `value` can also be specified as the raw POST body content * /mput - - takes CSV values in the body of the request. + + Note: takes separator-separated key/value pairs in separate lines in the POST body + + * /list_append + + parameters: `key`, `value` (multiple) + + * /list_prepend + + parameters: `key`, `value` (multiple) + + * /list_remove + + parameters: `key`, `value` (multiple) + + * /list_pop + + parameters: `key`, `position` (default 0), `count` (default 1) + + Note: a negative position does a reverse count from the end of the list + + * /set_add + + parameters: `key`, `value` (multiple) + + * /set_remove + + parameters: `key`, `value` (multiple) + + * /set_pop + + parameters: `key`, `count` (default 1) + + * /dump_csv + + parameters: `key` (optional) + + Note: dumps the entire database starting at `key` or else at the beginning, in txt format csv * /del - parameters: `key`, `format` + parameters: `key` * /stats - - * /exit (cause the current process to exit) + * /exit + + Note: causes the process to exit + +All endpoints take a `format` parameter which affects whether error conditions +are represented by the HTTP response code (format=txt) or by the "status_code" +member of the json result (format=json) (in which case the HTTP response code +is always 200 if the server isn't broken). `format` also affects the output +data for all endpoints except /put, /mput, /exit, /del, and /dump_csv. + +Output data in json format is under the "data" member of the root json object, +sometimes as a string (/get), sometimes as an array (/mget), sometimes as an +object with some metadata (/list_remove). + +Most endpoints take a `separator` parameter which defaults to "," (but can be +set to any single character), which affects txt format output data. It also +affects the deserialization and serialization of lists and sets stored in the +db, and the input parsing of /mput. + +All list and set endpoints take a `return_data` parameter; set it to 1 to additionally +return the new value of the list or set. However, this doesn't work for list_pop +or set_pop endpoints in txt format. Utilities --------- diff --git a/simpleleveldb/simpleleveldb.c b/simpleleveldb/simpleleveldb.c index 2b6a9d3..47344f4 100644 --- a/simpleleveldb/simpleleveldb.c +++ b/simpleleveldb/simpleleveldb.c @@ -1,4 +1,4 @@ -#define _GNU_SOURCE +#define _GNU_SOURCE // for strndup() #include #include #include @@ -7,24 +7,43 @@ #include #include #include +#include #include #include #include #include #include "http-internal.h" +#include "str_list_set.h" +// defined values #define NAME "simpleleveldb" -#define VERSION "0.7" +#define VERSION "0.8" #define DUMP_CSV_ITERS_CHECK 10 #define DUMP_CSV_MSECS_WORK 10 #define DUMP_CSV_MSECS_SLEEP 100 #define DUMP_CSV_MAX_BUFFER (8*1024*1024) +const char default_sep = ','; + +// extra types +enum LIST_ADD_TYPE { + LIST_APPEND, + LIST_PREPEND, +}; + +struct HashValue { + const char *value; + int count; + UT_hash_handle hh; +}; + +// function prototypes void finalize_request(int response_code, char *error, struct evhttp_request *req, struct evbuffer *evb, struct evkeyvalq *args, struct json_object *jsobj); -int db_open(); -void db_close(); +char get_argument_separator(struct evkeyvalq *args); +int db_open(void); +void db_close(void); void del_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx); void put_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx); void mput_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx); @@ -34,13 +53,18 @@ void fwmatch_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx); void range_match_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx); void stats_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx); void exit_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx); -void list_append_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx); +void list_add_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx); void list_remove_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx); +void list_pop_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx); +void set_add_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx); +void set_remove_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx); +void set_pop_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx); void dump_csv_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx); void do_dump_csv(int fd, short what, void *ctx); void set_dump_csv_timer(struct evhttp_request *req); void cleanup_dump_csv_cb(struct evhttp_connection *evcon, void *arg); +// global variables leveldb_t *ldb; leveldb_options_t *ldb_options; leveldb_cache_t *ldb_cache; @@ -52,7 +76,6 @@ leveldb_iterator_t *dump_iter; struct event dump_ev; int is_currently_dumping = 0; char *dump_fwmatch_key; -char *MPUT_SEP = ","; void finalize_request(int response_code, char *error, struct evhttp_request *req, struct evbuffer *evb, struct evkeyvalq *args, struct json_object *jsobj) { @@ -70,6 +93,9 @@ void finalize_request(int response_code, char *error, struct evhttp_request *req evbuffer_add_printf(evb, "DB_ERROR: %s", error); } } else { + if (!jsobj) { + jsobj = json_object_new_object(); + } if (error) { json_object_object_add(jsobj, "status_txt", json_object_new_string(error)); json_object_object_add(jsobj, "status_code", json_object_new_int(response_code)); @@ -105,14 +131,31 @@ void finalize_request(int response_code, char *error, struct evhttp_request *req evhttp_clear_headers(args); } -void db_close() +char get_argument_separator(struct evkeyvalq *args) +{ + char *sep_str; + + sep_str = (char *)evhttp_find_header(args, "separator"); + if (sep_str) { + if (strlen(sep_str) == 1) { + return sep_str[0]; + } else { + // invalid separator + return 0; + } + } + + return default_sep; +} + +void db_close(void) { leveldb_close(ldb); leveldb_options_destroy(ldb_options); leveldb_cache_destroy(ldb_cache); } -int db_open() +int db_open(void) { char *error = NULL; char *filename = option_get_str("db_file"); @@ -136,6 +179,7 @@ int db_open() ldb = leveldb_open(ldb_options, filename, &error); if (error) { fprintf(stderr, "ERROR opening db:%s\n", error); + free(error); return 0; } return 1; @@ -146,15 +190,13 @@ void del_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx) { char *key; struct evkeyvalq args; - struct json_object *jsobj; + struct json_object *jsobj = NULL; int response_code = HTTP_OK; char *error = NULL; leveldb_writeoptions_t *write_options; evhttp_parse_query(req->uri, &args); - jsobj = json_object_new_object(); - key = (char *)evhttp_find_header(&args, "key"); if (key == NULL) { finalize_request(400, "MISSING_ARG_KEY", req, evb, &args, jsobj); @@ -171,23 +213,20 @@ void del_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx) void mput_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx) { - char *term_key, *term_value, *sep; + char *buffer_data, *term_key, *term_value; + size_t req_len, term_key_len, term_value_len; + char sep; struct evkeyvalq args; - struct json_object *jsobj; + struct json_object *jsobj = NULL; int response_code = HTTP_OK; char *error = NULL; - size_t req_len, sep_pos = 0, line_offset = 0, j, sep_len; + size_t sep_pos = 0, line_offset = 0, j; leveldb_writeoptions_t *write_options; evhttp_parse_query(req->uri, &args); - jsobj = json_object_new_object(); - - sep = (char*)evhttp_find_header(&args, "separator"); - if (sep == NULL) { - sep = MPUT_SEP; - } - sep_len = strlen(sep); - + + sep = get_argument_separator(&args); + req_len = EVBUFFER_LENGTH(req->input_buffer); if (req->type != EVHTTP_REQ_POST) { finalize_request(400, "MUST_POST_DATA", req, evb, &args, jsobj); @@ -195,34 +234,39 @@ void mput_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx) } else if (req_len <= 2) { finalize_request(400, "MISSING_ARG_VALUE", req, evb, &args, jsobj); return; + } else if (sep == 0) { + finalize_request(400, "INVALID_SEPARATOR", req, evb, &args, jsobj); + return; } - + write_options = leveldb_writeoptions_create(); for (j = 0; j <= req_len; j++) { - if ( strncmp((char*)(EVBUFFER_DATA(req->input_buffer) + j), sep, sep_len) == 0 ) { + buffer_data = (char *) EVBUFFER_DATA(req->input_buffer); + if (sep_pos == 0 && j < req_len && *(buffer_data + j) == sep) { sep_pos = j; - j += sep_len - 1; - } else if (j == req_len || *(EVBUFFER_DATA(req->input_buffer) + j) == '\n') { + } else if (j == req_len || *(buffer_data + j) == '\n') { if (line_offset == j) { // Do nothing... just skip this blank line } else if (sep_pos == 0) { response_code = 400; - error = "MALFORMED_CSV"; + error = strdup("MALFORMED_CSV"); break; // everything. } else { - term_key = strndup((const char*)(EVBUFFER_DATA(req->input_buffer)+line_offset), sep_pos - line_offset); - term_value = strndup((const char*)(EVBUFFER_DATA(req->input_buffer)+sep_pos+sep_len), j - (sep_pos+sep_len)); - - leveldb_put(ldb, write_options, term_key, strlen(term_key), term_value, strlen(term_value), &error); - - free(term_key); - free(term_value); + term_key = buffer_data + line_offset; + term_key_len = sep_pos - line_offset; + term_value = buffer_data + sep_pos + 1; + term_value_len = j - (sep_pos + 1); + + leveldb_put(ldb, write_options, term_key, term_key_len, term_value, term_value_len, &error); + if (error) { + break; + } } - line_offset = j+1; + line_offset = j + 1; sep_pos = 0; } } - + leveldb_writeoptions_destroy(write_options); finalize_request(response_code, error, req, evb, &args, jsobj); free(error); @@ -232,7 +276,7 @@ void put_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx) { char *key, *value; struct evkeyvalq args; - struct json_object *jsobj; + struct json_object *jsobj = NULL; int response_code = HTTP_OK; char *error = NULL; size_t value_len; @@ -242,8 +286,6 @@ void put_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx) key = (char *)evhttp_find_header(&args, "key"); - - jsobj = json_object_new_object(); if (key == NULL) { finalize_request(400, "MISSING_ARG_KEY", req, evb, &args, jsobj); return; @@ -268,25 +310,29 @@ void put_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx) void get_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx) { - char *key, *value, *terminated_value; + char *key, *value; + char sep; struct evkeyvalq args; - struct json_object *jsobj; + struct json_object *jsobj = NULL; int response_code = HTTP_OK; leveldb_readoptions_t *read_options; char *error = NULL; size_t vallen; - char *tmp; int format; evhttp_parse_query(req->uri, &args); format = get_argument_format(&args); key = (char *)evhttp_find_header(&args, "key"); + sep = get_argument_separator(&args); - jsobj = json_object_new_object(); if (key == NULL) { finalize_request(400, "MISSING_ARG_KEY", req, evb, &args, jsobj); return; } + if (sep == 0) { + finalize_request(400, "INVALID_SEPARATOR", req, evb, &args, jsobj); + return; + } read_options = leveldb_readoptions_create(); leveldb_readoptions_set_verify_checksums(read_options, option_get_int("verify_checksums")); @@ -294,14 +340,15 @@ void get_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx) leveldb_readoptions_destroy(read_options); if (value) { - terminated_value = value; - DUPE_N_TERMINATE(terminated_value, vallen, tmp); + value = realloc(value, vallen + 1); + value[vallen] = '\0'; + if (format == txt_format) { - evbuffer_add_printf(evb, "%s,%s\n", key, terminated_value); + evbuffer_add_printf(evb, "%s%c%s\n", key, sep, value); } else { - json_object_object_add(jsobj, "data", json_object_new_string(terminated_value)); + jsobj = json_object_new_object(); + json_object_object_add(jsobj, "data", json_object_new_string(value)); } - free(terminated_value); finalize_request(response_code, error, req, evb, &args, jsobj); } else { @@ -309,28 +356,36 @@ void get_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx) } free(value); free(error); - } void mget_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx) { - char *key, *value, *terminated_value; + char *key, *value; + char sep; int format; struct evkeyvalq args; struct evkeyval *pair; - struct json_object *jsobj = NULL; + struct json_object *jsobj = NULL, *result_array = NULL, *tmp_obj; int nkeys = 0; int response_code = HTTP_OK; size_t vallen; char *error = NULL; - char *tmp; leveldb_readoptions_t *read_options; evhttp_parse_query(req->uri, &args); format = get_argument_format(&args); + sep = get_argument_separator(&args); + + if (sep == 0) { + finalize_request(400, "INVALID_SEPARATOR", req, evb, &args, jsobj); + return; + } + if (format == json_format) { jsobj = json_object_new_object(); + result_array = json_object_new_array(); + json_object_object_add(jsobj, "data", result_array); } read_options = leveldb_readoptions_create(); @@ -349,22 +404,24 @@ void mget_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx) } if (value) { - terminated_value = value; - DUPE_N_TERMINATE(terminated_value, vallen, tmp); + value = realloc(value, vallen + 1); + value[vallen] = '\0'; if (format == json_format) { - json_object_object_add(jsobj, key, json_object_new_string(terminated_value)); + tmp_obj = json_object_new_object(); + json_object_object_add(tmp_obj, "key", json_object_new_string(key)); + json_object_object_add(tmp_obj, "value", json_object_new_string(value)); + json_object_array_add(result_array, tmp_obj); } else { - evbuffer_add_printf(evb, "%s,%s\n", key, terminated_value); + evbuffer_add_printf(evb, "%s%c%s\n", key, sep, value); } - free(terminated_value); + free(value); } - free(value); } leveldb_readoptions_destroy(read_options); if (!nkeys) { - finalize_request(400, "key is required", req, evb, &args, jsobj); + finalize_request(400, "MISSING_ARG_KEY", req, evb, &args, jsobj); return; } @@ -374,22 +431,24 @@ void mget_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx) void range_match_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx) { - char *start_key, *end_key, *key_clean, *value_clean, *tmp; + char *start_key, *end_key, *key_clean; const char *key, *value; + char sep; size_t key_len, value_len; struct evkeyvalq args; - struct json_object *jsobj, *tmp_obj, *result_array; + struct json_object *jsobj = NULL, *result_array = NULL, *tmp_obj; const leveldb_snapshot_t *bt_snapshot; leveldb_readoptions_t *bt_read_options; leveldb_iterator_t *bt_iter; - int result_count = 0, result_limit = 0; + int result_count = 0, result_limit = 0, format; evhttp_parse_query(req->uri, &args); + format = get_argument_format(&args); + sep = get_argument_separator(&args); start_key = (char *)evhttp_find_header(&args, "start"); end_key = (char *)evhttp_find_header(&args, "end"); result_limit = get_int_argument(&args, "limit", 500); - jsobj = json_object_new_object(); if (start_key == NULL || end_key == NULL) { finalize_request(400, "MISSING_ARG_KEY", req, evb, &args, jsobj); return; @@ -398,9 +457,16 @@ void range_match_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx) finalize_request(400, "INVALID_START_KEY", req, evb, &args, jsobj); return; } - - result_array = json_object_new_array(); - tmp_obj = NULL; + if (sep == 0) { + finalize_request(400, "INVALID_SEPARATOR", req, evb, &args, jsobj); + return; + } + + if (format == json_format) { + jsobj = json_object_new_object(); + result_array = json_object_new_array(); + json_object_object_add(jsobj, "data", result_array); + } bt_read_options = leveldb_readoptions_create(); bt_snapshot = leveldb_create_snapshot(ldb); @@ -411,30 +477,33 @@ void range_match_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx) while (leveldb_iter_valid(bt_iter) && (result_limit == 0 || result_count < result_limit)) { key = leveldb_iter_key(bt_iter, &key_len); - key_clean = (char *)key; - DUPE_N_TERMINATE(key_clean, key_len, tmp); - + key_clean = strndup(key, key_len); + if (strcmp(key_clean, end_key) > 0) { free(key_clean); break; } - + value = leveldb_iter_value(bt_iter, &value_len); - value_clean = (char *)value; - DUPE_N_TERMINATE(value_clean, value_len, tmp); - tmp_obj = json_object_new_object(); - json_object_object_add(tmp_obj, key_clean, json_object_new_string(value_clean)); - json_object_array_add(result_array, tmp_obj); + if (format == json_format) { + tmp_obj = json_object_new_object(); + json_object_object_add(tmp_obj, key_clean, json_object_new_string_len(value, value_len)); + json_object_array_add(result_array, tmp_obj); + } else { + evbuffer_add_printf(evb, "%s%c", key_clean, sep); + evbuffer_add(evb, value, value_len); + evbuffer_add_printf(evb, "\n"); + } leveldb_iter_next(bt_iter); result_count ++; free(key_clean); - free(value_clean); } - json_object_object_add(jsobj, "data", result_array); - json_object_object_add(jsobj, "status", json_object_new_string(result_count ? "ok" : "no results")); + if (format == json_format) { + json_object_object_add(jsobj, "status", json_object_new_string(result_count ? "ok" : "no results")); + } finalize_request(200, NULL, req, evb, &args, jsobj); @@ -445,28 +514,37 @@ void range_match_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx) void fwmatch_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx) { - char *fw_key, *key_clean, *value_clean, *tmp; + char *fw_key, *key_clean; const char *key, *value; + char sep; size_t key_len, value_len; struct evkeyvalq args; - struct json_object *jsobj, *tmp_obj, *result_array; + struct json_object *jsobj = NULL, *result_array = NULL, *tmp_obj; const leveldb_snapshot_t *fw_snapshot; leveldb_readoptions_t *fw_read_options; leveldb_iterator_t *fw_iter; - int result_count = 0, result_limit = 0; + int result_count = 0, result_limit = 0, format; evhttp_parse_query(req->uri, &args); + format = get_argument_format(&args); + sep = get_argument_separator(&args); fw_key = (char *)evhttp_find_header(&args, "key"); result_limit = get_int_argument(&args, "limit", 500); - jsobj = json_object_new_object(); - result_array = json_object_new_array(); - tmp_obj = NULL; - if (fw_key == NULL) { finalize_request(400, "MISSING_ARG_KEY", req, evb, &args, jsobj); return; } + if (sep == 0) { + finalize_request(400, "INVALID_SEPARATOR", req, evb, &args, jsobj); + return; + } + + if (format == json_format) { + jsobj = json_object_new_object(); + result_array = json_object_new_array(); + json_object_object_add(jsobj, "data", result_array); + } fw_read_options = leveldb_readoptions_create(); fw_snapshot = leveldb_create_snapshot(ldb); @@ -477,8 +555,7 @@ void fwmatch_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx) while (leveldb_iter_valid(fw_iter) && (result_limit == 0 || result_count < result_limit)) { key = leveldb_iter_key(fw_iter, &key_len); - key_clean = (char *)key; - DUPE_N_TERMINATE(key_clean, key_len, tmp); + key_clean = strndup(key, key_len); // this is the case where we are only fwing keys of this prefix // so we need to break out of the loop at the last key @@ -487,21 +564,25 @@ void fwmatch_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx) break; } value = leveldb_iter_value(fw_iter, &value_len); - value_clean = (char *)value; - DUPE_N_TERMINATE(value_clean, value_len, tmp); - tmp_obj = json_object_new_object(); - json_object_object_add(tmp_obj, key_clean, json_object_new_string(value_clean)); - json_object_array_add(result_array, tmp_obj); + if (format == json_format) { + tmp_obj = json_object_new_object(); + json_object_object_add(tmp_obj, key_clean, json_object_new_string_len(value, value_len)); + json_object_array_add(result_array, tmp_obj); + } else { + evbuffer_add_printf(evb, "%s%c", key_clean, sep); + evbuffer_add(evb, value, value_len); + evbuffer_add_printf(evb, "\n"); + } leveldb_iter_next(fw_iter); result_count ++; free(key_clean); - free(value_clean); } - json_object_object_add(jsobj, "data", result_array); - json_object_object_add(jsobj, "status", json_object_new_string(result_count ? "ok" : "no results")); + if (format == json_format) { + json_object_object_add(jsobj, "status", json_object_new_string(result_count ? "ok" : "no results")); + } finalize_request(200, NULL, req, evb, &args, jsobj); @@ -510,110 +591,500 @@ void fwmatch_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx) leveldb_release_snapshot(ldb, fw_snapshot); } -/* append a `value` string on to the end of a string value */ -void list_append_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx) +/* append or prepend multiple `value` strings to a string value */ +void list_add_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx) { - char *key, *append_value, *orig_value, *new_value, *terminated_value; + char *key, *add_value, *orig_value; + size_t orig_valuelen; + char sep; + int format, ret_data; + struct evbuffer *new_value; struct evkeyvalq args; - struct json_object *jsobj; + struct evkeyval *arg_pair; + struct json_object *jsobj = NULL, *jsobj_data = NULL, *jsobj_value = NULL; int response_code = HTTP_OK; char *error = NULL; - size_t orig_valuelen; - char *tmp; - int format; - int echo_data; leveldb_readoptions_t *read_options; leveldb_writeoptions_t *write_options; + enum LIST_ADD_TYPE type = (enum LIST_ADD_TYPE) ctx; evhttp_parse_query(req->uri, &args); format = get_argument_format(&args); - echo_data = get_int_argument(&args, "echo_data", 0); + ret_data = get_int_argument(&args, "return_data", 0); key = (char *)evhttp_find_header(&args, "key"); - append_value = (char *)evhttp_find_header(&args, "value"); - // separator = (char *)evhttp_find_header(&args, "separator"); + add_value = (char *)evhttp_find_header(&args, "value"); + sep = get_argument_separator(&args); - jsobj = json_object_new_object(); if (key == NULL) { finalize_request(400, "MISSING_ARG_KEY", req, evb, &args, jsobj); return; } - if (append_value == NULL) { + if (add_value == NULL) { finalize_request(400, "MISSING_ARG_VALUE", req, evb, &args, jsobj); return; } + if (sep == 0) { + finalize_request(400, "INVALID_SEPARATOR", req, evb, &args, jsobj); + return; + } + + if (format == json_format) { + jsobj = json_object_new_object(); + + if (ret_data) { + jsobj_data = json_object_new_object(); + jsobj_value = json_object_new_array(); + json_object_object_add(jsobj, "data", jsobj_data); + json_object_object_add(jsobj_data, "key", json_object_new_string(key)); + json_object_object_add(jsobj_data, "value", jsobj_value); + } + } + + new_value = evbuffer_new(); + evbuffer_add_printf(new_value, "%s", ""); // null terminate read_options = leveldb_readoptions_create(); leveldb_readoptions_set_verify_checksums(read_options, option_get_int("verify_checksums")); orig_value = leveldb_get(ldb, read_options, key, strlen(key), &orig_valuelen, &error); leveldb_readoptions_destroy(read_options); - // null terminate orig_value - if (orig_value) { - terminated_value = orig_value; - DUPE_N_TERMINATE(terminated_value, orig_valuelen, tmp); - free(orig_value); - orig_value = terminated_value; + if (!error) { + // append case - orig_value goes first + if (type == LIST_APPEND && orig_value) { + reserialize_list(new_value, jsobj_value, &orig_value, orig_valuelen, sep); + } + + TAILQ_FOREACH(arg_pair, &args, next) { + if (strcmp(arg_pair->key, "value") != 0) { + continue; + } + // skip empty values + if (strlen(arg_pair->value) == 0) { + continue; + } + serialize_list_item(new_value, arg_pair->value, sep); + + if (jsobj_value) { + json_object_array_add(jsobj_value, json_object_new_string(arg_pair->value)); + } + } + + // prepend case - orig_value goes last + if (type == LIST_PREPEND && orig_value) { + reserialize_list(new_value, jsobj_value, &orig_value, orig_valuelen, sep); + } + + write_options = leveldb_writeoptions_create(); + leveldb_put(ldb, write_options, key, strlen(key), (char *)EVBUFFER_DATA(new_value), EVBUFFER_LENGTH(new_value), &error); + leveldb_writeoptions_destroy(write_options); } + if (ret_data && format == txt_format) { + evbuffer_add_printf(evb, "%s%c%s\n", key, sep, EVBUFFER_DATA(new_value)); + } - if (orig_value) { - new_value = calloc(1, (orig_valuelen + 1 + strlen(append_value) + 1) * sizeof(char *)); - sprintf(new_value, "%s,%s", orig_value, append_value); - } else { - new_value = calloc(1, (strlen(append_value) + 1) * sizeof(char *)); - sprintf(new_value, "%s", append_value); + finalize_request(response_code, error, req, evb, &args, jsobj); + evbuffer_free(new_value); + free(orig_value); + free(error); +} + +/* remove `value` strings from a string value */ +void list_remove_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx) +{ + char *key, *remove_value, *orig_value, *token; + size_t orig_valuelen; + char sep; + int format, ret_data; + struct evbuffer *new_value; + struct evkeyvalq args; + struct evkeyval *arg_pair; + struct json_object *jsobj = NULL, *jsobj_data = NULL, *jsobj_removed = NULL, *jsobj_value = NULL; + struct HashValue *remove_values_hash, *hash_value, *hash_tmp; + struct ListInfo list_info; + int response_code = HTTP_OK; + char *error = NULL; + int updated = 0; + leveldb_readoptions_t *read_options; + leveldb_writeoptions_t *write_options; + + evhttp_parse_query(req->uri, &args); + format = get_argument_format(&args); + ret_data = get_int_argument(&args, "return_data", 0); + + key = (char *)evhttp_find_header(&args, "key"); + remove_value = (char *)evhttp_find_header(&args, "value"); + sep = get_argument_separator(&args); + + if (key == NULL) { + finalize_request(400, "MISSING_ARG_KEY", req, evb, &args, jsobj); + return; + } + if (remove_value == NULL) { + finalize_request(400, "MISSING_ARG_VALUE", req, evb, &args, jsobj); + return; + } + if (sep == 0) { + finalize_request(400, "INVALID_SEPARATOR", req, evb, &args, jsobj); + return; + } + + if (format == json_format) { + jsobj = json_object_new_object(); + jsobj_data = json_object_new_object(); + jsobj_removed = json_object_new_array(); + json_object_object_add(jsobj, "data", jsobj_data); + json_object_object_add(jsobj_data, "key", json_object_new_string(key)); + json_object_object_add(jsobj_data, "removed", jsobj_removed); + + if (ret_data) { + jsobj_value = json_object_new_array(); + json_object_object_add(jsobj_data, "value", jsobj_value); + } } + // put values to remove in a hash for quick lookup as list is walked + remove_values_hash = NULL; + TAILQ_FOREACH(arg_pair, &args, next) { + if (strcmp(arg_pair->key, "value") != 0) { + continue; + } + // skip empty values + if (strlen(arg_pair->value) == 0) { + continue; + } + HASH_FIND_STR(remove_values_hash, arg_pair->value, hash_value); + if (!hash_value) { + hash_value = calloc(1, sizeof(struct HashValue)); + hash_value->value = arg_pair->value; + HASH_ADD_KEYPTR(hh, remove_values_hash, hash_value->value, strlen(hash_value->value), hash_value); + } + hash_value->count++; + } + + new_value = evbuffer_new(); + evbuffer_add_printf(new_value, "%s", ""); // null terminate + + read_options = leveldb_readoptions_create(); + leveldb_readoptions_set_verify_checksums(read_options, option_get_int("verify_checksums")); + orig_value = leveldb_get(ldb, read_options, key, strlen(key), &orig_valuelen, &error); + leveldb_readoptions_destroy(read_options); + + if (orig_value && !error) { + prepare_token_list(&list_info, &orig_value, orig_valuelen, sep); + + TOKEN_LIST_FOREACH(token, &list_info) { + HASH_FIND_STR(remove_values_hash, token, hash_value); + if (hash_value) { + updated = 1; + hash_value->count--; + if (hash_value->count == 0) { + HASH_DEL(remove_values_hash, hash_value); + free(hash_value); + } + if (jsobj_removed) { + json_object_array_add(jsobj_removed, json_object_new_string(token)); + } + // don't copy this item + } else { + serialize_list_item(new_value, token, sep); + if (jsobj_value) { + json_object_array_add(jsobj_value, json_object_new_string(token)); + } + } + } + + if (updated == 1) { + write_options = leveldb_writeoptions_create(); + leveldb_put(ldb, write_options, key, strlen(key), (char *)EVBUFFER_DATA(new_value), EVBUFFER_LENGTH(new_value), &error); + leveldb_writeoptions_destroy(write_options); + } + } + + HASH_ITER(hh, remove_values_hash, hash_value, hash_tmp) { + HASH_DEL(remove_values_hash, hash_value); + free(hash_value); + } + + if (ret_data && format == txt_format) { + evbuffer_add_printf(evb, "%s%c%s\n", key, sep, (char *)EVBUFFER_DATA(new_value)); + } + + finalize_request(response_code, error, req, evb, &args, jsobj); + evbuffer_free(new_value); + free(orig_value); free(error); +} + +/* pop `count` strings from a string representation of a list */ +void list_pop_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx) +{ + char *key, *orig_value, *token, *origptr; + size_t orig_valuelen; + char sep; + int position, count, cur_pos, format, ret_data; + struct evbuffer *new_value, *pop_value = NULL; + struct evkeyvalq args; + struct json_object *jsobj = NULL, *jsobj_data = NULL, *jsobj_popped = NULL, *jsobj_value = NULL; + struct ListInfo list_info; + int response_code = HTTP_OK; + char *error = NULL; + int updated = 0; + leveldb_readoptions_t *read_options; + leveldb_writeoptions_t *write_options; - write_options = leveldb_writeoptions_create(); - leveldb_put(ldb, write_options, key, strlen(key), new_value, strlen(new_value), &error); - leveldb_writeoptions_destroy(write_options); + evhttp_parse_query(req->uri, &args); + format = get_argument_format(&args); + ret_data = get_int_argument(&args, "return_data", 0); - if (echo_data) { - if (format == json_format) { - json_object_object_add(jsobj, "data", json_object_new_string(new_value)); + key = (char *)evhttp_find_header(&args, "key"); + position = get_int_argument(&args, "position", 0); + count = get_int_argument(&args, "count", 1); + sep = get_argument_separator(&args); + + if (key == NULL) { + finalize_request(400, "MISSING_ARG_KEY", req, evb, &args, jsobj); + return; + } + if (sep == 0) { + finalize_request(400, "INVALID_SEPARATOR", req, evb, &args, jsobj); + return; + } + if (count < 1) { + finalize_request(400, "INVALID_COUNT", req, evb, &args, jsobj); + return; + } + + new_value = evbuffer_new(); + evbuffer_add_printf(new_value, "%s", ""); // null terminate + + if (format == json_format) { + jsobj = json_object_new_object(); + jsobj_data = json_object_new_object(); + jsobj_popped = json_object_new_array(); + json_object_object_add(jsobj, "data", jsobj_data); + json_object_object_add(jsobj_data, "key", json_object_new_string(key)); + json_object_object_add(jsobj_data, "popped", jsobj_popped); + + if (ret_data) { + jsobj_value = json_object_new_array(); + json_object_object_add(jsobj_data, "value", jsobj_value); + } + } else { + pop_value = evbuffer_new(); + evbuffer_add_printf(pop_value, "%s", ""); // null terminate + } + + read_options = leveldb_readoptions_create(); + leveldb_readoptions_set_verify_checksums(read_options, option_get_int("verify_checksums")); + orig_value = leveldb_get(ldb, read_options, key, strlen(key), &orig_valuelen, &error); + leveldb_readoptions_destroy(read_options); + + if (orig_value && !error) { + prepare_token_list(&list_info, &orig_value, orig_valuelen, sep); + + if (position >= 0) { + cur_pos = 0; + TOKEN_LIST_FOREACH(token, &list_info) { + if (cur_pos == position && count > 0) { + if (format == json_format) { + json_object_array_add(jsobj_popped, json_object_new_string(token)); + } else { + serialize_list_item(pop_value, token, sep); + } + count--; + updated = 1; + } else { + if (jsobj_value) { + json_object_array_add(jsobj_value, json_object_new_string(token)); + } + serialize_list_item(new_value, token, sep); + cur_pos++; + } + } } else { - evbuffer_add_printf(evb, "%s,%s\n", key, new_value); + cur_pos = -1; + TOKEN_LIST_FOREACH_REVERSE(token, &list_info) { + if (cur_pos == position && count > 0) { + if (format == json_format) { + json_object_array_add(jsobj_popped, json_object_new_string(token)); + } else { + serialize_list_item(pop_value, token, sep); + } + count--; + updated = 1; + // blank out, will not be in reconstituted list + memset(token, '\0', strlen(token)); + } else { + // will be in reconstituted list + cur_pos--; + } + } + + // reconstitute + if (updated) { + origptr = orig_value; + while (origptr < orig_value + orig_valuelen) { + if (*origptr != '\0') { + if (jsobj_value) { + json_object_array_add(jsobj_value, json_object_new_string(origptr)); + } + serialize_list_item(new_value, origptr, sep); + origptr += strlen(origptr); + } + origptr++; + } + } + } + + if (updated == 1) { + write_options = leveldb_writeoptions_create(); + leveldb_put(ldb, write_options, key, strlen(key), (char *)EVBUFFER_DATA(new_value), EVBUFFER_LENGTH(new_value), &error); + leveldb_writeoptions_destroy(write_options); } } + if (format == txt_format) { + evbuffer_add_printf(evb, "%s\n", (char *)EVBUFFER_DATA(pop_value)); + evbuffer_free(pop_value); + } + finalize_request(response_code, error, req, evb, &args, jsobj); - free(new_value); + evbuffer_free(new_value); free(orig_value); free(error); +} + +void set_add_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx) +{ + char *key, *add_value, *orig_value; + size_t orig_valuelen; + char sep; + int format, ret_data; + struct evbuffer *new_value; + struct evkeyvalq args; + struct evkeyval *arg_pair; + struct SetItem *set = NULL, *set_item; + struct json_object *jsobj = NULL, *jsobj_data = NULL, *jsobj_added = NULL, *jsobj_value = NULL; + int response_code = HTTP_OK; + char *error = NULL; + int updated = 0; + leveldb_readoptions_t *read_options; + leveldb_writeoptions_t *write_options; + + evhttp_parse_query(req->uri, &args); + format = get_argument_format(&args); + ret_data = get_int_argument(&args, "return_data", 0); + + key = (char *)evhttp_find_header(&args, "key"); + add_value = (char *)evhttp_find_header(&args, "value"); + sep = get_argument_separator(&args); + + if (key == NULL) { + finalize_request(400, "MISSING_ARG_KEY", req, evb, &args, jsobj); + return; + } + if (add_value == NULL) { + finalize_request(400, "MISSING_ARG_VALUE", req, evb, &args, jsobj); + return; + } + if (sep == 0) { + finalize_request(400, "INVALID_SEPARATOR", req, evb, &args, jsobj); + return; + } + + if (format == json_format) { + jsobj = json_object_new_object(); + jsobj_data = json_object_new_object(); + jsobj_added = json_object_new_array(); + json_object_object_add(jsobj, "data", jsobj_data); + json_object_object_add(jsobj_data, "key", json_object_new_string(key)); + json_object_object_add(jsobj_data, "added", jsobj_added); + + if (ret_data) { + jsobj_value = json_object_new_array(); + json_object_object_add(jsobj_data, "value", jsobj_value); + } + } + + read_options = leveldb_readoptions_create(); + leveldb_readoptions_set_verify_checksums(read_options, option_get_int("verify_checksums")); + orig_value = leveldb_get(ldb, read_options, key, strlen(key), &orig_valuelen, &error); + leveldb_readoptions_destroy(read_options); + + new_value = evbuffer_new(); + evbuffer_add_printf(new_value, "%s", ""); // null terminate + + if (!error) { + if (orig_value) { + deserialize_alloc_set(&set, &orig_value, orig_valuelen, sep); + } + + TAILQ_FOREACH(arg_pair, &args, next) { + if (strcmp(arg_pair->key, "value") != 0) { + continue; + } + // skip empty values + if (strlen(arg_pair->value) == 0) { + continue; + } + + HASH_FIND_STR(set, arg_pair->value, set_item); + if (!set_item) { + add_new_set_item(&set, arg_pair->value); + updated = 1; + + if (jsobj_added) { + json_object_array_add(jsobj_added, json_object_new_string(arg_pair->value)); + } + } + } + + serialize_free_set(new_value, jsobj_value, &set, sep); + + if (updated) { + write_options = leveldb_writeoptions_create(); + leveldb_put(ldb, write_options, key, strlen(key), (char *)EVBUFFER_DATA(new_value), EVBUFFER_LENGTH(new_value), &error); + leveldb_writeoptions_destroy(write_options); + } + } + + if (ret_data && format == txt_format) { + evbuffer_add_printf(evb, "%s%c%s\n", key, sep, EVBUFFER_DATA(new_value)); + } + finalize_request(response_code, error, req, evb, &args, jsobj); + evbuffer_free(new_value); + free(orig_value); + free(error); } -/* remove a `value` from string */ -void list_remove_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx) +void set_remove_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx) { - char *key, *remove_value, *orig_value, *terminated_value; + char *key, *remove_value, *orig_value; + size_t orig_valuelen; + char sep; + int format, ret_data; + struct evbuffer *new_value; struct evkeyvalq args; - struct json_object *jsobj; + struct evkeyval *arg_pair; + struct SetItem *set = NULL, *set_item; + struct json_object *jsobj = NULL, *jsobj_data = NULL, *jsobj_removed = NULL, *jsobj_value = NULL; int response_code = HTTP_OK; char *error = NULL; - size_t orig_valuelen; - char *tmp; - int i; int updated = 0; - char *token; - struct evbuffer *new_value; - int format; - int echo_data; leveldb_readoptions_t *read_options; leveldb_writeoptions_t *write_options; evhttp_parse_query(req->uri, &args); format = get_argument_format(&args); - echo_data = get_int_argument(&args, "echo_data", 0); + ret_data = get_int_argument(&args, "return_data", 0); key = (char *)evhttp_find_header(&args, "key"); remove_value = (char *)evhttp_find_header(&args, "value"); - // separator = (char *)evhttp_find_header(&args, "separator"); + sep = get_argument_separator(&args); - jsobj = json_object_new_object(); if (key == NULL) { finalize_request(400, "MISSING_ARG_KEY", req, evb, &args, jsobj); return; @@ -622,70 +1093,174 @@ void list_remove_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx) finalize_request(400, "MISSING_ARG_VALUE", req, evb, &args, jsobj); return; } + if (sep == 0) { + finalize_request(400, "INVALID_SEPARATOR", req, evb, &args, jsobj); + return; + } + + if (format == json_format) { + jsobj = json_object_new_object(); + jsobj_data = json_object_new_object(); + jsobj_removed = json_object_new_array(); + json_object_object_add(jsobj, "data", jsobj_data); + json_object_object_add(jsobj_data, "key", json_object_new_string(key)); + json_object_object_add(jsobj_data, "removed", jsobj_removed); + + if (ret_data) { + jsobj_value = json_object_new_array(); + json_object_object_add(jsobj_data, "value", jsobj_value); + } + } read_options = leveldb_readoptions_create(); leveldb_readoptions_set_verify_checksums(read_options, option_get_int("verify_checksums")); orig_value = leveldb_get(ldb, read_options, key, strlen(key), &orig_valuelen, &error); leveldb_readoptions_destroy(read_options); - // null terminate orig_value - if (orig_value) { - terminated_value = orig_value; - DUPE_N_TERMINATE(terminated_value, orig_valuelen, tmp); - free(orig_value); - orig_value = terminated_value; - } - - if (orig_value) { - new_value = evbuffer_new(); - token = strtok(orig_value, ","); - i = 0; - while (token) { - if (strcmp(token, remove_value) == 0) { - // we found the token + new_value = evbuffer_new(); + evbuffer_add_printf(new_value, "%s", ""); // null terminate + + if (orig_value && !error) { + deserialize_alloc_set(&set, &orig_value, orig_valuelen, sep); + + TAILQ_FOREACH(arg_pair, &args, next) { + if (strcmp(arg_pair->key, "value") != 0) { + continue; + } + // skip empty values + if (strlen(arg_pair->value) == 0) { + continue; + } + + HASH_FIND_STR(set, arg_pair->value, set_item); + if (set_item) { + HASH_DEL(set, set_item); + free(set_item); updated = 1; - } else { - if (i == 0) { - evbuffer_add_printf(new_value, "%s", token); - } else { - evbuffer_add_printf(new_value, ",%s", token); + + if (jsobj_removed) { + json_object_array_add(jsobj_removed, json_object_new_string(arg_pair->value)); } - i++; } - token = strtok(NULL, ","); } - if (updated == 1) { + + serialize_free_set(new_value, jsobj_value, &set, sep); + + if (updated) { write_options = leveldb_writeoptions_create(); leveldb_put(ldb, write_options, key, strlen(key), (char *)EVBUFFER_DATA(new_value), EVBUFFER_LENGTH(new_value), &error); leveldb_writeoptions_destroy(write_options); } + } + + if (ret_data && format == txt_format) { + evbuffer_add_printf(evb, "%s%c%s\n", key, sep, EVBUFFER_DATA(new_value)); + } + + finalize_request(response_code, error, req, evb, &args, jsobj); + evbuffer_free(new_value); + free(orig_value); + free(error); +} + +void set_pop_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx) +{ + char *key, *orig_value; + size_t orig_valuelen; + char sep; + int format, count, ret_data; + struct evkeyvalq args; + struct evbuffer *new_value, *pop_value = NULL; + struct SetItem *set = NULL, *set_item, *set_tmp; + struct json_object *jsobj = NULL, *jsobj_data = NULL, *jsobj_popped = NULL, *jsobj_value = NULL; + int response_code = HTTP_OK; + char *error = NULL; + int updated = 0; + leveldb_readoptions_t *read_options; + leveldb_writeoptions_t *write_options; + + evhttp_parse_query(req->uri, &args); + format = get_argument_format(&args); + ret_data = get_int_argument(&args, "return_data", 0); + + key = (char *)evhttp_find_header(&args, "key"); + count = get_int_argument(&args, "count", 1); + sep = get_argument_separator(&args); + + if (key == NULL) { + finalize_request(400, "MISSING_ARG_KEY", req, evb, &args, jsobj); + return; + } + if (count < 1) { + finalize_request(400, "INVALID_COUNT", req, evb, &args, jsobj); + return; + } + if (sep == 0) { + finalize_request(400, "INVALID_SEPARATOR", req, evb, &args, jsobj); + return; + } + + if (format == json_format) { + jsobj = json_object_new_object(); + jsobj_data = json_object_new_object(); + jsobj_popped = json_object_new_array(); + json_object_object_add(jsobj, "data", jsobj_data); + json_object_object_add(jsobj_data, "key", json_object_new_string(key)); + json_object_object_add(jsobj_data, "popped", jsobj_popped); + + if (ret_data) { + jsobj_value = json_object_new_array(); + json_object_object_add(jsobj_data, "value", jsobj_value); + } + } else { + pop_value = evbuffer_new(); + evbuffer_add_printf(pop_value, "%s", ""); // null terminate + } + + read_options = leveldb_readoptions_create(); + leveldb_readoptions_set_verify_checksums(read_options, option_get_int("verify_checksums")); + orig_value = leveldb_get(ldb, read_options, key, strlen(key), &orig_valuelen, &error); + leveldb_readoptions_destroy(read_options); + + new_value = evbuffer_new(); + evbuffer_add_printf(new_value, "%s", ""); // null terminate + + if (orig_value && !error) { + deserialize_alloc_set(&set, &orig_value, orig_valuelen, sep); - if (echo_data) { + HASH_ITER(hh, set, set_item, set_tmp) { + if (count <= 0) { + break; + } if (format == json_format) { - json_object_object_add(jsobj, "data", json_object_new_string((char *)EVBUFFER_DATA(new_value))); + json_object_array_add(jsobj_popped, json_object_new_string(set_item->value)); } else { - evbuffer_add_printf(evb, "%s,%s\n", key, (char *)EVBUFFER_DATA(new_value)); + serialize_list_item(pop_value, set_item->value, sep); } + HASH_DEL(set, set_item); + free(set_item); + count--; + updated = 1; } - evbuffer_free(new_value); + serialize_free_set(new_value, jsobj_value, &set, sep); - } else { - if (echo_data) { - if (format == json_format) { - json_object_object_add(jsobj, "data", json_object_new_string("")); - } else { - evbuffer_add_printf(evb, "%s,\n", key); - } + if (updated) { + write_options = leveldb_writeoptions_create(); + leveldb_put(ldb, write_options, key, strlen(key), (char *)EVBUFFER_DATA(new_value), EVBUFFER_LENGTH(new_value), &error); + leveldb_writeoptions_destroy(write_options); } } - free(error); + if (format == txt_format) { + evbuffer_add_printf(evb, "%s\n", EVBUFFER_DATA(pop_value)); + evbuffer_free(pop_value); + } finalize_request(response_code, error, req, evb, &args, jsobj); + evbuffer_free(new_value); free(orig_value); free(error); - } /* @@ -695,13 +1270,9 @@ note: this makes a snapshot of the database and may return after other data has void dump_csv_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx) { struct evkeyvalq args; - int format; - struct json_object *jsobj; - - jsobj = json_object_new_object(); + struct json_object *jsobj = NULL; evhttp_parse_query(req->uri, &args); - format = get_argument_format(&args); dump_fwmatch_key = (char *)evhttp_find_header(&args, "key"); if (dump_fwmatch_key) { dump_fwmatch_key = strdup(dump_fwmatch_key); @@ -727,7 +1298,6 @@ void dump_csv_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx) } evhttp_clear_headers(&args); - json_object_put(jsobj); evhttp_send_reply_start(req, 200, "OK"); evhttp_connection_set_closecb(req->evcon, cleanup_dump_csv_cb, NULL); @@ -906,8 +1476,13 @@ int main(int argc, char **argv) } simplehttp_init(); - simplehttp_set_cb("/list_append*", list_append_cb, NULL); + simplehttp_set_cb("/list_append*", list_add_cb, (void *) LIST_APPEND); + simplehttp_set_cb("/list_prepend*", list_add_cb, (void *) LIST_PREPEND); simplehttp_set_cb("/list_remove*", list_remove_cb, NULL); + simplehttp_set_cb("/list_pop*", list_pop_cb, NULL); + simplehttp_set_cb("/set_add*", set_add_cb, NULL); + simplehttp_set_cb("/set_remove*", set_remove_cb, NULL); + simplehttp_set_cb("/set_pop*", set_pop_cb, NULL); simplehttp_set_cb("/get*", get_cb, NULL); simplehttp_set_cb("/mget*", mget_cb, NULL); simplehttp_set_cb("/range_match*", range_match_cb, NULL); diff --git a/simpleleveldb/str_list_set.c b/simpleleveldb/str_list_set.c new file mode 100644 index 0000000..6f888e7 --- /dev/null +++ b/simpleleveldb/str_list_set.c @@ -0,0 +1,90 @@ +#include +#include +#include "str_list_set.h" + +void prepare_token_list(struct ListInfo *list_info, char **db_data, size_t db_data_len, char sep) +{ + // null terminate + *db_data = realloc(*db_data, db_data_len + 1); + *(*db_data + db_data_len) = '\0'; + + list_info->sep = sep; + list_info->buf = db_data; + list_info->buflen = db_data_len; + + // for strtok_r() + list_info->sepstr[0] = sep; + list_info->sepstr[1] = '\0'; + + // for reverse_tokenize() + list_info->saveptr = *db_data + db_data_len; +} + +// somewhat like strtok_r() but depends on prepare_token_list() for setup +char *reverse_tokenize(struct ListInfo *list_info) +{ + while (list_info->saveptr > *list_info->buf) { + list_info->saveptr--; + if (list_info->saveptr == *list_info->buf) { + if (*list_info->saveptr != list_info->sep) { + return list_info->saveptr; + } else { + return NULL; + } + } + if (*list_info->saveptr == list_info->sep) { + *list_info->saveptr = '\0'; + if (*(list_info->saveptr + 1) != '\0') { + return list_info->saveptr + 1; + } + } + } + return NULL; +} + +void reserialize_list(struct evbuffer *output, struct json_object *array, char **db_data, size_t db_data_len, char sep) +{ + struct ListInfo list_info; + char *item; + + prepare_token_list(&list_info, db_data, db_data_len, sep); + + TOKEN_LIST_FOREACH(item, &list_info) { + serialize_list_item(output, item, sep); + + if (array) { + json_object_array_add(array, json_object_new_string(item)); + } + } +} + +void deserialize_alloc_set(struct SetItem **set, char **db_data, size_t db_data_len, char sep) +{ + char *token; + struct SetItem *set_item; + struct ListInfo list_info; + + prepare_token_list(&list_info, db_data, db_data_len, sep); + + TOKEN_LIST_FOREACH(token, &list_info) { + HASH_FIND_STR(*set, token, set_item); + if (!set_item) { + add_new_set_item(set, token); + } + } +} + +void serialize_free_set(struct evbuffer *output, struct json_object *array, struct SetItem **set, char sep) +{ + struct SetItem *set_item, *set_tmp; + + HASH_ITER(hh, *set, set_item, set_tmp) { + if (array) { + json_object_array_add(array, json_object_new_string(set_item->value)); + } + serialize_list_item(output, set_item->value, sep); + HASH_DEL(*set, set_item); + free(set_item); + } +} + diff --git a/simpleleveldb/str_list_set.h b/simpleleveldb/str_list_set.h new file mode 100644 index 0000000..f20591b --- /dev/null +++ b/simpleleveldb/str_list_set.h @@ -0,0 +1,54 @@ +#ifndef _STR_LIST_SET_H +#define _STR_LIST_SET_H + +#include +#include + +struct SetItem { + const char *value; + UT_hash_handle hh; +}; + +struct ListInfo { + char sep; + char sepstr[2]; + char *saveptr; + char **buf; + size_t buflen; +}; + +void prepare_token_list(struct ListInfo *list_info, char **db_data, size_t db_data_len, char sep); +char *reverse_tokenize(struct ListInfo *list_info); +void reserialize_list(struct evbuffer *output, struct json_object *array, char **db_data, size_t db_data_len, char sep); +void deserialize_alloc_set(struct SetItem **set, char **db_data, size_t db_data_len, char sep); +void serialize_free_set(struct evbuffer *output, struct json_object *array, struct SetItem **set, char sep); + +static inline void serialize_list_item(struct evbuffer *output, const char *item, char sep) +{ + if (EVBUFFER_LENGTH(output) > 0) { + evbuffer_add_printf(output, "%c", sep); + } + evbuffer_add_printf(output, "%s", item); +} + +static inline void add_new_set_item(struct SetItem **set, char *value_ptr) +{ + struct SetItem *set_item; + set_item = calloc(1, sizeof(struct SetItem)); + set_item->value = value_ptr; + HASH_ADD_KEYPTR(hh, *set, value_ptr, strlen(value_ptr), set_item); +} + +#define TOKEN_LIST_FOREACH(item, list_info) \ + for (item = strtok_r(*(list_info)->buf, \ + (list_info)->sepstr, \ + &(list_info)->saveptr); \ + item != NULL; \ + item = strtok_r(NULL, \ + (list_info)->sepstr, \ + &(list_info)->saveptr) )\ + +#define TOKEN_LIST_FOREACH_REVERSE(item, list_info) \ + while ((item = reverse_tokenize(list_info)) != NULL) \ + +#endif diff --git a/simpleleveldb/test_simpleleveldb.py b/simpleleveldb/test_simpleleveldb.py index 46da9d7..7a0a6d9 100644 --- a/simpleleveldb/test_simpleleveldb.py +++ b/simpleleveldb/test_simpleleveldb.py @@ -1,47 +1,95 @@ import os +import re import sys sys.path.append(os.path.join(os.path.dirname(__file__), "../shared_tests")) -import logging from test_shunt import valgrind_cmd, SubprocessTest, http_fetch, http_fetch_json class SimpleLeveldbTest(SubprocessTest): binary_name = "simpleleveldb" - process_options = [valgrind_cmd(os.path.abspath('simpleleveldb'), '--db-file=%s/db' % os.path.join(os.path.dirname(__file__), "test_output"), '--enable-logging')] working_dir = os.path.dirname(__file__) + test_output_dir = os.path.join(working_dir, "test_output") + process_options = [valgrind_cmd(test_output_dir, os.path.join(working_dir, binary_name), '--db-file=%s/db' % test_output_dir, '--enable-logging')] + # /put, /get, /del def test_basic(self): - data = http_fetch_json('/put', dict(key='test', value='12345')) - data = http_fetch_json('/get', dict(key='test')) + data = http_fetch_json('/put', dict(key='test_basic', value='12345')) + data = http_fetch_json('/get', dict(key='test_basic')) assert data == '12345' - data = http_fetch('/get', dict(key='test', format='txt')) - assert data == 'test,12345\n' - data = http_fetch_json('/del', dict(key='test')) - data = http_fetch_json('/get', dict(key='test'), 404, 'NOT_FOUND') - + data = http_fetch_json('/get', dict(key='test_basic', format='json')) + assert data == '12345' + data = http_fetch('/get', dict(key='test_basic', format='txt')) + assert data == 'test_basic,12345\n' + data = http_fetch('/get', dict(key='test_basic', format='txt', separator='/')) + assert data == 'test_basic/12345\n' + data = http_fetch('/put', dict(key='test_basic', value='22', format='txt')) + data = http_fetch_json('/get', dict(key='test_basic')) + assert data == '22' + data = http_fetch_json('/put', dict(key='test_basic', value='33', format='json')) + data = http_fetch_json('/get', dict(key='test_basic')) + assert data == '33' + data = http_fetch_json('/put', dict(key='test_basic'), body="44") + data = http_fetch_json('/get', dict(key='test_basic')) + assert data == '44' + data = http_fetch_json('/del', dict(key='test_basic')) + data = http_fetch_json('/get', dict(key='test_basic'), 404, 'NOT_FOUND') + data = http_fetch('/get', dict(key='test_basic', format='txt'), 404) + data = http_fetch_json('/get', dict(key='test_basic', format='json'), 404, 'NOT_FOUND') + data = http_fetch_json('/put', dict(key='test_basic', value='a')) + data = http_fetch('/del', dict(key='test_basic', format='txt')) + data = http_fetch_json('/get', dict(key='test_basic'), 404, 'NOT_FOUND') + + def test_missing(self): http_fetch_json("/put", dict(), 400, 'MISSING_ARG_KEY') - http_fetch_json("/put", dict(key='test'), 400, 'MISSING_ARG_VALUE') + http_fetch_json("/put", dict(key='test_missing'), 400, 'MISSING_ARG_VALUE') http_fetch_json("/get", dict(), 400, 'MISSING_ARG_KEY') - + http_fetch_json("/del", dict(), 400, 'MISSING_ARG_KEY') http_fetch_json("/fwmatch", dict(), 400, 'MISSING_ARG_KEY') - - http_fetch_json('/put', dict(key='test1', value='asdf1')) - http_fetch_json('/put', dict(key='test2', value='asdf2')) - - data = http_fetch('/mget', dict(key=['test1', 'test2', 'test3'], format='txt')) - print data - assert data == 'test1,asdf1\ntest2,asdf2\n' - - data = http_fetch_json("/fwmatch", dict(key="test")) - print data - assert data == [{'test1': 'asdf1'}, {'test2': 'asdf2'}] - - - # test list stuff + http_fetch("/put", dict(format="txt"), 400) + http_fetch("/put", dict(format='txt', key='test_missing'), 400) + http_fetch("/get", dict(format='txt'), 400) + http_fetch("/del", dict(format='txt'), 400) + http_fetch_json("/get", dict(key="test_missing", separator="asdf"), 400, 'INVALID_SEPARATOR') + http_fetch_json("/get", dict(key="test_missing", separator=""), 400, 'INVALID_SEPARATOR') + http_fetch("/get", dict(format="txt", key="test_missing", separator=""), 400) + http_fetch_json('/mput', dict(), 400, 'MISSING_ARG_VALUE', body='.') + http_fetch_json('/mput', dict(), 400, 'MALFORMED_CSV', body='vvvvvvvvvv') + http_fetch_json('/mput', dict(separator='|'), 400, 'MALFORMED_CSV', body='test_missing,val1\n') + http_fetch_json('/mget', dict(separator=';;'), 400, 'INVALID_SEPARATOR') + http_fetch_json('/mget', dict(), 400, 'MISSING_ARG_KEY') + http_fetch('/mget', dict(format="txt"), 400) + http_fetch_json('/fwmatch', dict(), 400, 'MISSING_ARG_KEY') + http_fetch_json('/range_match', dict(), 400, 'MISSING_ARG_KEY') + http_fetch_json('/range_match', dict(start="a"), 400, 'MISSING_ARG_KEY') + http_fetch_json('/range_match', dict(end="b"), 400, 'MISSING_ARG_KEY') + http_fetch_json('/range_match', dict(start="b", end="a"), 400, 'INVALID_START_KEY') + + def test_multikey(self): + http_fetch_json('/put', dict(key='test_multikey_1', value='asdf1')) + http_fetch_json('/put', dict(key='test_multikey_2', value='asdf2')) + data = http_fetch_json('/mget', dict(key=['test_multikey_1', 'test_multikey_2', 'test_multikey_3'])) + assert data == [{'key': 'test_multikey_1', 'value': 'asdf1'}, + {'key': 'test_multikey_2', 'value': 'asdf2'}] + data = http_fetch('/mget', dict(key=['test_multikey_1', 'test_multikey_2', 'test_multikey_3'], format='txt')) + assert data == 'test_multikey_1,asdf1\ntest_multikey_2,asdf2\n' + data = http_fetch_json("/fwmatch", dict(key="test_multikey")) + assert data == [{'test_multikey_1': 'asdf1'}, {'test_multikey_2': 'asdf2'}] + data = http_fetch("/fwmatch", dict(key="test_multikey", format="txt")) + assert data == "test_multikey_1,asdf1\ntest_multikey_2,asdf2\n" + http_fetch_json('/mput', body='test_multikey_3,mv1a;mv1b\ntest_multikey_4,mv2a;mv2b\ntest_multikey_5,mv3a') + data = http_fetch_json('/get', dict(key='test_multikey_3')) + assert data == 'mv1a;mv1b' + data = http_fetch('/mget', dict(key=['test_multikey_4', 'test_multikey_5'], format='txt')) + assert data == 'test_multikey_4,mv2a;mv2b\ntest_multikey_5,mv3a\n' + http_fetch_json('/mput', body='test_multikey_ms,a,b,c') + data = http_fetch_json('/get', dict(key='test_multikey_ms')) + assert data == 'a,b,c' + + def test_lists_1(self): http_fetch_json('/get', dict(key='list_test'), 404, 'NOT_FOUND') - data = http_fetch_json('/list_append', dict(key='list_test', value='testvalue1', echo_data='1')) - assert data == 'testvalue1' + data = http_fetch_json('/list_append', dict(key='list_test', value='testvalue1', return_data='1')) + assert data['value'] == ['testvalue1'] data = http_fetch_json('/list_append', dict(key='list_test', value='testvalue2')) assert data == '' data = http_fetch_json('/get', dict(key='list_test')) @@ -53,24 +101,90 @@ def test_basic(self): data = http_fetch_json('/list_remove', dict(key='list_test', value='testvalue1')) data = http_fetch_json('/get', dict(key='list_test')) assert data == 'testvalue3' - - # try a /put with a POST body + + def test_post(self): http_fetch_json('/put', dict(key='testpost'), body='asdfpost') data = http_fetch_json('/get', dict(key='testpost')) assert data == 'asdfpost' data = http_fetch_json('/del', dict(key='testpost')) - - # test dump to csv + + def test_dump_csv(self): # we need to check more than 500 entries + http_fetch_json('/put', dict(key='zzz.dump', value='bump.value')) for x in range(505): http_fetch_json('/put', dict(key='dump.%d' % x, value='dump.value.%d' % x)) - data = http_fetch('/dump_csv') assert data.startswith("dump.0,dump.value.0\n") - assert data.endswith("test2,asdf2\n") assert data.count("\n") > 505 - data = http_fetch('/dump_csv', dict(key="dump.")) assert data.count("\n") == 505 + + def test_lists_2(self): + http_fetch_json('/list_prepend', dict(key='new_list', value='appval1')) + data = http_fetch_json('/list_prepend', dict(key='new_list', value='appvala', return_data='1')) + assert data['value'] == ['appvala', 'appval1'] + data = http_fetch('/list_prepend', dict(key='new_list', value='appvalb', return_data='1', format='txt')) + assert data == 'new_list,appvalb,appvala,appval1\n' + data = http_fetch_json('/get', dict(key='new_list')) + assert data == 'appvalb,appvala,appval1' + data = http_fetch_json('/list_pop', dict(key='new_list')) + assert data['popped'] == ['appvalb'] + data = http_fetch_json('/get', dict(key='new_list')) + assert data == 'appvala,appval1' + data = http_fetch('/list_pop', dict(key='new_list', position='-1', format='txt')) + assert data == 'appval1\n' + data = http_fetch_json('/get', dict(key='new_list')) + assert data == 'appvala' + http_fetch_json('/list_append', dict(key='new_list', value=["blah1", "blah2", "blah3"])) + data = http_fetch_json('/get', dict(key='new_list')) + assert data == 'appvala,blah1,blah2,blah3' + data = http_fetch_json('/list_pop', dict(key='new_list', position='2', count='2')) + assert data['popped'] == ['blah2', 'blah3'] + + def test_sets_1(self): + http_fetch_json('/set_add', dict(key='testset', value=['si1', 'si2'])) + http_fetch_json('/set_remove', dict(key='testset', value='si1')) + data = http_fetch_json('/get', dict(key='testset')) + assert data == 'si2' + http_fetch_json('/set_add', dict(key='testset', value=['si1', 'si2', 'si3'])) + data = http_fetch_json('/get', dict(key='testset')) + datalist = re.split(',', data) + assert 'si1' in datalist + assert 'si2' in datalist + assert 'si3' in datalist + assert ','.join(sorted(datalist)) == 'si1,si2,si3' + http_fetch_json('/set_remove', dict(key='testset', value=['si3', 'si1', 'si2'])) + data = http_fetch_json('/get', dict(key='testset')) + assert data == '' + set_list = ['sl1', 'sl2', 'sl3', 'sl4'] + http_fetch_json('/set_add', dict(key='testset', value=set_list)) + for x in range(len(set_list)): + data = http_fetch_json('/set_pop', dict(key='testset')) + assert len(data['popped']) == 1 + assert data['popped'][0] in set_list + set_list.remove(data['popped'][0]) + data = http_fetch_json('/set_pop', dict(key='testset')) + assert data['popped'] == [] + + def test_separators(self): + http_fetch_json('/list_append', dict(key='testsep', value=['3,9', '4,16', '2,4'], separator='|')) + data = http_fetch_json('/get', dict(key='testsep')) + assert data == '3,9|4,16|2,4' + http_fetch_json('/list_remove', dict(key='testsep', value='2,4', separator='|')) + data = http_fetch_json('/get', dict(key='testsep')) + assert data == '3,9|4,16' + http_fetch_json('/set_add', dict(key='testsep', value=['5,25', '7,49'], separator='|')) + http_fetch_json('/set_remove', dict(key='testsep', value=['3,9', '5,25'], separator='|')) + data = http_fetch_json('/get', dict(key='testsep')) + datalist = re.split('|', data) + assert '|'.join(sorted(datalist)) == '4,16|7,49' + http_fetch_json('/list_pop', dict(key='testsep', separator='long'), 400, 'INVALID_SEPARATOR') + http_fetch_json('/mput', dict(separator='|'), body='testsep2|sepv2\ntestsep1|sepv1\n') + data = http_fetch_json('/get', dict(key='testsep1')) + assert data == 'sepv1' + data = http_fetch_json('/get', dict(key='testsep2')) + assert data == 'sepv2' + - +if __name__ == "__main__": + print "usage: py.test" diff --git a/simplequeue/test_simplequeue.py b/simplequeue/test_simplequeue.py index ccc33b9..68efc3c 100644 --- a/simplequeue/test_simplequeue.py +++ b/simplequeue/test_simplequeue.py @@ -2,14 +2,14 @@ import sys sys.path.append(os.path.join(os.path.dirname(__file__), "../shared_tests")) -import logging import simplejson as json from test_shunt import valgrind_cmd, SubprocessTest, http_fetch, http_fetch_json class SimplequeueTest(SubprocessTest): binary_name = "simplequeue" - process_options = [valgrind_cmd(os.path.abspath('simplequeue'), '--enable-logging')] working_dir = os.path.dirname(__file__) + test_output_dir = os.path.join(working_dir, "test_output") + process_options = [valgrind_cmd(test_output_dir, os.path.join(working_dir, binary_name), '--enable-logging')] def test_basic(self): # put/get in order