Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

Merge pull request #47 from ploxiln/simpleleveldb_47

simpleleveldb list/set endpoints
  • Loading branch information...
commit 449795d3fdba38c78924dd71aa8ef9b9770c63bd 2 parents d32a797 + 94b75ab
@mreiferson mreiferson authored
View
1  .gitignore
@@ -2,6 +2,7 @@
*.a
*.dSYM
*.pyc
+*.deps
build
dist
sortdb/sortdb
View
3  conftest.py
@@ -0,0 +1,3 @@
+# needed for py.test to accept the --valgrind option
+def pytest_addoption(parser):
+ parser.addoption("--no-valgrind", action="store_true", help="disable valgrind analysis")
View
7 queuereader/queuereader.c
@@ -1,3 +1,4 @@
+#define _GNU_SOURCE // for strndup()
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
@@ -28,7 +29,6 @@ void queuereader_increment_backoff();
void queuereader_decrement_backoff();
static char *message = NULL;
-static size_t message_len;
static int backoff_counter = 0;
static struct GlobalData *data = NULL;
static struct event ev;
@@ -137,7 +137,6 @@ void queuereader_source_cb(struct evhttp_request *req, void *cbarg)
struct GlobalData *client_data = (struct GlobalData *)cbarg;
char *line;
size_t line_len;
- char *tmp = NULL;
struct evbuffer *evb;
int ret = QR_EMPTY;
@@ -150,9 +149,7 @@ void queuereader_source_cb(struct evhttp_request *req, void *cbarg)
line = (char *)EVBUFFER_DATA(evb);
line_len = EVBUFFER_LENGTH(evb);
if (line_len) {
- message = line;
- message_len = line_len;
- DUPE_N_TERMINATE(message, message_len, tmp);
+ message = strndup(line, line_len);
ret = (*client_data->message_cb)(message, client_data->cbarg);
}
View
56 shared_tests/test_shunt.py
@@ -34,26 +34,26 @@ def http_fetch(endpoint, params=None, response_code=200, body=None):
assert res.code == response_code
return res.body
-
-def valgrind_cmd(cmd, *options):
+def valgrind_cmd(test_output_dir, *options):
assert isinstance(options, (list, tuple))
- assert cmd.startswith("/"), "valgrind_cmd must take a fully qualified executible path not %s" % cmd
- test_output_dir = os.path.join(os.path.dirname(cmd), "test_output")
- return [
+ cmdlist = list(options)
+ if '--no-valgrind' not in sys.argv:
+ cmdlist = [
'valgrind',
'-v',
'--tool=memcheck',
'--trace-children=yes',
- # '--demangle=yes',
'--log-file=%s/vg.out' % test_output_dir,
'--leak-check=full',
- '--show-reachable=yes',
+ #'--show-reachable=yes',
'--run-libc-freeres=yes',
- '%s' % cmd,
- ] + list(options)
-
+ ] + cmdlist
+ return cmdlist
def check_valgrind_output(filename):
+ if '--no-valgrind' in sys.argv:
+ return
+
assert os.path.exists(filename)
time.sleep(.15)
vg_output = open(filename, 'r').readlines()
@@ -75,30 +75,26 @@ def check_valgrind_output(filename):
assert lost
assert lost[0] == "possibly lost: 0 bytes in 0 blocks"
-
class SubprocessTest(unittest.TestCase):
process_options = []
binary_name = ""
working_dir = None
- def setUp(self):
- """setup method that starts up mongod instances using `self.mongo_options`"""
+ test_output_dir = None
+
+ @classmethod
+ def setUpClass(self):
+ """setup method that starts up target instances using `self.process_options`"""
self.temp_dirs = []
self.processes = []
assert self.binary_name, "you must override self.binary_name"
assert self.working_dir, "set workign dir to os.path.dirname(__file__)"
- exe = os.path.join(self.working_dir, self.binary_name)
- if os.path.exists(exe):
- logging.info('removing old %s' % exe)
- os.unlink(exe)
-
+ # make should update the executable if needed
logging.info('running make')
- pipe = subprocess.Popen(['make'])
- pipe.wait()
-
- assert os.path.exists(exe), "compile failed"
+ pipe = subprocess.Popen(['make', '-C', self.working_dir])
+ assert pipe.wait() == 0, "compile failed"
- test_output_dir = os.path.join(self.working_dir, "test_output")
+ test_output_dir = self.test_output_dir
if os.path.exists(test_output_dir):
logging.info('removing %s' % test_output_dir)
pipe = subprocess.Popen(['rm', '-rf', test_output_dir])
@@ -108,16 +104,14 @@ def setUp(self):
os.makedirs(test_output_dir)
for options in self.process_options:
-
logging.info(' '.join(options))
- # self.stdout = open(test_output_dir + '/test.out', 'w')
- # self.stderr = open(test_output_dir + '/test.err', 'w')
- pipe = subprocess.Popen(options)#, stdout=self.stdout, stderr=self.stderr)
+ pipe = subprocess.Popen(options)
self.processes.append(pipe)
logging.debug('started process %s' % pipe.pid)
- self.wait_for('http://127.0.0.1:8080/', max_time=5)
+ self.wait_for('http://127.0.0.1:8080/', max_time=9)
+ @classmethod
def wait_for(self, url, max_time):
# check up to 15 times till the endpoint specified is available waiting max_time
step = max_time / float(15)
@@ -131,6 +125,7 @@ def wait_for(self, url, max_time):
pass
time.sleep(step)
+ @classmethod
def graceful_shutdown(self):
try:
http_fetch('/exit', dict())
@@ -138,8 +133,9 @@ def graceful_shutdown(self):
# we never get a reply if this works correctly
time.sleep(1)
- def tearDown(self):
- """teardown method that cleans up child mongod instances, and removes their temporary data files"""
+ @classmethod
+ def tearDownClass(self):
+ """teardown method that cleans up child target instances, and removes their temporary data files"""
logging.debug('teardown')
try:
self.graceful_shutdown()
View
7 simplehttp/simplehttp.h
@@ -7,13 +7,6 @@
#include <evhttp.h>
#define SIMPLEHTTP_VERSION "0.1.3"
-#ifndef DUPE_N_TERMINATE
-#define DUPE_N_TERMINATE(buf, len, tmp) \
- tmp = malloc((len) + 1); \
- memcpy(tmp, buf, (len)); \
- tmp[(len)] = '\0'; \
- buf = tmp;
-#endif
#if _POSIX_TIMERS > 0
View
6 simplehttp/util.c
@@ -1,3 +1,4 @@
+#define _GNU_SOURCE // for strndup()
#include <stdlib.h>
#include <stdio.h>
#include <unistd.h>
@@ -58,7 +59,6 @@ uint64_t ninety_five_percent(int64_t *int_array, int length)
int simplehttp_parse_url(char *endpoint, size_t endpoint_len, char **address, int *port, char **path)
{
// parse out address, port, path
- char *tmp = NULL;
char *tmp_port = NULL;
char *tmp_pointer;
size_t address_len;
@@ -98,8 +98,8 @@ int simplehttp_parse_url(char *endpoint, size_t endpoint_len, char **address, in
}
path_len = (endpoint + endpoint_len) - *path;
- DUPE_N_TERMINATE(*address, address_len, tmp);
- DUPE_N_TERMINATE(*path, path_len, tmp);
+ *address = strndup(*address, address_len);
+ *path = strndup(*path, path_len);
return 1;
}
View
19 simpleleveldb/Makefile
@@ -11,16 +11,19 @@ AR = ar
AR_FLAGS = rc
RANLIB = ranlib
-all: simpleleveldb leveldb_to_csv csv_to_leveldb
+TARGETS = simpleleveldb leveldb_to_csv csv_to_leveldb
-leveldb_to_csv: leveldb_to_csv.c
- $(CC) $(CFLAGS) -o $@ leveldb_to_csv.c $(LIBS)
+SOURCES_simpleleveldb = simpleleveldb.c str_list_set.c
+SOURCES_leveldb_to_csv = leveldb_to_csv.c
+SOURCES_csv_to_leveldb = csv_to_leveldb.c
-csv_to_leveldb: csv_to_leveldb.c
- $(CC) $(CFLAGS) -o $@ csv_to_leveldb.c $(LIBS)
+all: $(TARGETS)
-simpleleveldb: simpleleveldb.c
- $(CC) $(CFLAGS) -o $@ simpleleveldb.c $(LIBS)
+-include $(TARGETS:%=%.deps)
+
+$(TARGETS): %: %.c
+ $(CC) $(CFLAGS) -MM -MT $@ -MF $@.deps $(SOURCES_$@)
+ $(CC) $(CFLAGS) -o $@ $(SOURCES_$@) $(LIBS)
install:
/usr/bin/install -d $(TARGET)/bin/
@@ -29,4 +32,4 @@ install:
/usr/bin/install csv_to_leveldb $(TARGET)/bin/
clean:
- rm -rf *.a *.o simpleleveldb leveldb_to_csv csv_to_leveldb *.dSYM
+ rm -rf *.a *.o *.deps *.dSYM $(TARGETS)
View
82 simpleleveldb/README.md
@@ -51,39 +51,95 @@ OPTIONS
API endpoints:
* /get
-
- parameters: `key`, `format`
-
+
+ parameters: `key`
+
* /mget
- parameters: `key` (multiple), `format`
+ parameters: `key` (multiple)
* /fwmatch
- parameters: `key`, `limit`
+ parameters: `key`, `limit` (default 500)
* /range_match
- parameters: `start`, `end`, `limit`
+ parameters: `start`, `end`, `limit` (default 500)
* /put
- parameters: `key`, `value`, `format`
-
+ parameters: `key`, `value`
+
Note: `value` can also be specified as the raw POST body content
* /mput
-
- takes CSV values in the body of the request.
+
+ Note: takes separator-separated key/value pairs in separate lines in the POST body
+
+ * /list_append
+
+ parameters: `key`, `value` (multiple)
+
+ * /list_prepend
+
+ parameters: `key`, `value` (multiple)
+
+ * /list_remove
+
+ parameters: `key`, `value` (multiple)
+
+ * /list_pop
+
+ parameters: `key`, `position` (default 0), `count` (default 1)
+
+ Note: a negative position does a reverse count from the end of the list
+
+ * /set_add
+
+ parameters: `key`, `value` (multiple)
+
+ * /set_remove
+
+ parameters: `key`, `value` (multiple)
+
+ * /set_pop
+
+ parameters: `key`, `count` (default 1)
+
+ * /dump_csv
+
+ parameters: `key` (optional)
+
+ Note: dumps the entire database starting at `key` or else at the beginning, in txt format csv
* /del
- parameters: `key`, `format`
+ parameters: `key`
* /stats
-
- * /exit (cause the current process to exit)
+ * /exit
+
+ Note: causes the process to exit
+
+All endpoints take a `format` parameter which affects whether error conditions
+are represented by the HTTP response code (format=txt) or by the "status_code"
+member of the json result (format=json) (in which case the HTTP response code
+is always 200 if the server isn't broken). `format` also affects the output
+data for all endpoints except /put, /mput, /exit, /del, and /dump_csv.
+
+Output data in json format is under the "data" member of the root json object,
+sometimes as a string (/get), sometimes as an array (/mget), sometimes as an
+object with some metadata (/list_remove).
+
+Most endpoints take a `separator` parameter which defaults to "," (but can be
+set to any single character), which affects txt format output data. It also
+affects the deserialization and serialization of lists and sets stored in the
+db, and the input parsing of /mput.
+
+All list and set endpoints take a `return_data` parameter; set it to 1 to additionally
+return the new value of the list or set. However, this doesn't work for list_pop
+or set_pop endpoints in txt format.
Utilities
---------
View
945 simpleleveldb/simpleleveldb.c
@@ -1,4 +1,4 @@
-#define _GNU_SOURCE
+#define _GNU_SOURCE // for strndup()
#include <stdlib.h>
#include <stdbool.h>
#include <stdint.h>
@@ -7,24 +7,43 @@
#include <unistd.h>
#include <inttypes.h>
#include <simplehttp/queue.h>
+#include <simplehttp/uthash.h>
#include <simplehttp/simplehttp.h>
#include <json/json.h>
#include <leveldb/c.h>
#include <sys/socket.h>
#include "http-internal.h"
+#include "str_list_set.h"
+// defined values
#define NAME "simpleleveldb"
-#define VERSION "0.7"
+#define VERSION "0.8"
#define DUMP_CSV_ITERS_CHECK 10
#define DUMP_CSV_MSECS_WORK 10
#define DUMP_CSV_MSECS_SLEEP 100
#define DUMP_CSV_MAX_BUFFER (8*1024*1024)
+const char default_sep = ',';
+
+// extra types
+enum LIST_ADD_TYPE {
+ LIST_APPEND,
+ LIST_PREPEND,
+};
+
+struct HashValue {
+ const char *value;
+ int count;
+ UT_hash_handle hh;
+};
+
+// function prototypes
void finalize_request(int response_code, char *error, struct evhttp_request *req, struct evbuffer *evb, struct evkeyvalq *args, struct json_object *jsobj);
-int db_open();
-void db_close();
+char get_argument_separator(struct evkeyvalq *args);
+int db_open(void);
+void db_close(void);
void del_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx);
void put_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx);
void mput_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx);
@@ -34,13 +53,18 @@ void fwmatch_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx);
void range_match_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx);
void stats_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx);
void exit_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx);
-void list_append_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx);
+void list_add_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx);
void list_remove_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx);
+void list_pop_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx);
+void set_add_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx);
+void set_remove_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx);
+void set_pop_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx);
void dump_csv_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx);
void do_dump_csv(int fd, short what, void *ctx);
void set_dump_csv_timer(struct evhttp_request *req);
void cleanup_dump_csv_cb(struct evhttp_connection *evcon, void *arg);
+// global variables
leveldb_t *ldb;
leveldb_options_t *ldb_options;
leveldb_cache_t *ldb_cache;
@@ -52,7 +76,6 @@ leveldb_iterator_t *dump_iter;
struct event dump_ev;
int is_currently_dumping = 0;
char *dump_fwmatch_key;
-char *MPUT_SEP = ",";
void finalize_request(int response_code, char *error, struct evhttp_request *req, struct evbuffer *evb, struct evkeyvalq *args, struct json_object *jsobj)
{
@@ -70,6 +93,9 @@ void finalize_request(int response_code, char *error, struct evhttp_request *req
evbuffer_add_printf(evb, "DB_ERROR: %s", error);
}
} else {
+ if (!jsobj) {
+ jsobj = json_object_new_object();
+ }
if (error) {
json_object_object_add(jsobj, "status_txt", json_object_new_string(error));
json_object_object_add(jsobj, "status_code", json_object_new_int(response_code));
@@ -105,14 +131,31 @@ void finalize_request(int response_code, char *error, struct evhttp_request *req
evhttp_clear_headers(args);
}
-void db_close()
+char get_argument_separator(struct evkeyvalq *args)
+{
+ char *sep_str;
+
+ sep_str = (char *)evhttp_find_header(args, "separator");
+ if (sep_str) {
+ if (strlen(sep_str) == 1) {
+ return sep_str[0];
+ } else {
+ // invalid separator
+ return 0;
+ }
+ }
+
+ return default_sep;
+}
+
+void db_close(void)
{
leveldb_close(ldb);
leveldb_options_destroy(ldb_options);
leveldb_cache_destroy(ldb_cache);
}
-int db_open()
+int db_open(void)
{
char *error = NULL;
char *filename = option_get_str("db_file");
@@ -136,6 +179,7 @@ int db_open()
ldb = leveldb_open(ldb_options, filename, &error);
if (error) {
fprintf(stderr, "ERROR opening db:%s\n", error);
+ free(error);
return 0;
}
return 1;
@@ -146,15 +190,13 @@ void del_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
{
char *key;
struct evkeyvalq args;
- struct json_object *jsobj;
+ struct json_object *jsobj = NULL;
int response_code = HTTP_OK;
char *error = NULL;
leveldb_writeoptions_t *write_options;
evhttp_parse_query(req->uri, &args);
- jsobj = json_object_new_object();
-
key = (char *)evhttp_find_header(&args, "key");
if (key == NULL) {
finalize_request(400, "MISSING_ARG_KEY", req, evb, &args, jsobj);
@@ -171,23 +213,20 @@ void del_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
void mput_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
{
- char *term_key, *term_value, *sep;
+ char *buffer_data, *term_key, *term_value;
+ size_t req_len, term_key_len, term_value_len;
+ char sep;
struct evkeyvalq args;
- struct json_object *jsobj;
+ struct json_object *jsobj = NULL;
int response_code = HTTP_OK;
char *error = NULL;
- size_t req_len, sep_pos = 0, line_offset = 0, j, sep_len;
+ size_t sep_pos = 0, line_offset = 0, j;
leveldb_writeoptions_t *write_options;
evhttp_parse_query(req->uri, &args);
- jsobj = json_object_new_object();
-
- sep = (char*)evhttp_find_header(&args, "separator");
- if (sep == NULL) {
- sep = MPUT_SEP;
- }
- sep_len = strlen(sep);
-
+
+ sep = get_argument_separator(&args);
+
req_len = EVBUFFER_LENGTH(req->input_buffer);
if (req->type != EVHTTP_REQ_POST) {
finalize_request(400, "MUST_POST_DATA", req, evb, &args, jsobj);
@@ -195,34 +234,39 @@ void mput_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
} else if (req_len <= 2) {
finalize_request(400, "MISSING_ARG_VALUE", req, evb, &args, jsobj);
return;
+ } else if (sep == 0) {
+ finalize_request(400, "INVALID_SEPARATOR", req, evb, &args, jsobj);
+ return;
}
-
+
write_options = leveldb_writeoptions_create();
for (j = 0; j <= req_len; j++) {
- if ( strncmp((char*)(EVBUFFER_DATA(req->input_buffer) + j), sep, sep_len) == 0 ) {
+ buffer_data = (char *) EVBUFFER_DATA(req->input_buffer);
+ if (sep_pos == 0 && j < req_len && *(buffer_data + j) == sep) {
sep_pos = j;
- j += sep_len - 1;
- } else if (j == req_len || *(EVBUFFER_DATA(req->input_buffer) + j) == '\n') {
+ } else if (j == req_len || *(buffer_data + j) == '\n') {
if (line_offset == j) {
// Do nothing... just skip this blank line
} else if (sep_pos == 0) {
response_code = 400;
- error = "MALFORMED_CSV";
+ error = strdup("MALFORMED_CSV");
break; // everything.
} else {
- term_key = strndup((const char*)(EVBUFFER_DATA(req->input_buffer)+line_offset), sep_pos - line_offset);
- term_value = strndup((const char*)(EVBUFFER_DATA(req->input_buffer)+sep_pos+sep_len), j - (sep_pos+sep_len));
-
- leveldb_put(ldb, write_options, term_key, strlen(term_key), term_value, strlen(term_value), &error);
-
- free(term_key);
- free(term_value);
+ term_key = buffer_data + line_offset;
+ term_key_len = sep_pos - line_offset;
+ term_value = buffer_data + sep_pos + 1;
+ term_value_len = j - (sep_pos + 1);
+
+ leveldb_put(ldb, write_options, term_key, term_key_len, term_value, term_value_len, &error);
+ if (error) {
+ break;
+ }
}
- line_offset = j+1;
+ line_offset = j + 1;
sep_pos = 0;
}
}
-
+
leveldb_writeoptions_destroy(write_options);
finalize_request(response_code, error, req, evb, &args, jsobj);
free(error);
@@ -232,7 +276,7 @@ void put_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
{
char *key, *value;
struct evkeyvalq args;
- struct json_object *jsobj;
+ struct json_object *jsobj = NULL;
int response_code = HTTP_OK;
char *error = NULL;
size_t value_len;
@@ -242,8 +286,6 @@ void put_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
key = (char *)evhttp_find_header(&args, "key");
-
- jsobj = json_object_new_object();
if (key == NULL) {
finalize_request(400, "MISSING_ARG_KEY", req, evb, &args, jsobj);
return;
@@ -268,25 +310,29 @@ void put_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
void get_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
{
- char *key, *value, *terminated_value;
+ char *key, *value;
+ char sep;
struct evkeyvalq args;
- struct json_object *jsobj;
+ struct json_object *jsobj = NULL;
int response_code = HTTP_OK;
leveldb_readoptions_t *read_options;
char *error = NULL;
size_t vallen;
- char *tmp;
int format;
evhttp_parse_query(req->uri, &args);
format = get_argument_format(&args);
key = (char *)evhttp_find_header(&args, "key");
+ sep = get_argument_separator(&args);
- jsobj = json_object_new_object();
if (key == NULL) {
finalize_request(400, "MISSING_ARG_KEY", req, evb, &args, jsobj);
return;
}
+ if (sep == 0) {
+ finalize_request(400, "INVALID_SEPARATOR", req, evb, &args, jsobj);
+ return;
+ }
read_options = leveldb_readoptions_create();
leveldb_readoptions_set_verify_checksums(read_options, option_get_int("verify_checksums"));
@@ -294,14 +340,15 @@ void get_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
leveldb_readoptions_destroy(read_options);
if (value) {
- terminated_value = value;
- DUPE_N_TERMINATE(terminated_value, vallen, tmp);
+ value = realloc(value, vallen + 1);
+ value[vallen] = '\0';
+
if (format == txt_format) {
- evbuffer_add_printf(evb, "%s,%s\n", key, terminated_value);
+ evbuffer_add_printf(evb, "%s%c%s\n", key, sep, value);
} else {
- json_object_object_add(jsobj, "data", json_object_new_string(terminated_value));
+ jsobj = json_object_new_object();
+ json_object_object_add(jsobj, "data", json_object_new_string(value));
}
- free(terminated_value);
finalize_request(response_code, error, req, evb, &args, jsobj);
} else {
@@ -309,28 +356,36 @@ void get_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
}
free(value);
free(error);
-
}
void mget_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
{
- char *key, *value, *terminated_value;
+ char *key, *value;
+ char sep;
int format;
struct evkeyvalq args;
struct evkeyval *pair;
- struct json_object *jsobj = NULL;
+ struct json_object *jsobj = NULL, *result_array = NULL, *tmp_obj;
int nkeys = 0;
int response_code = HTTP_OK;
size_t vallen;
char *error = NULL;
- char *tmp;
leveldb_readoptions_t *read_options;
evhttp_parse_query(req->uri, &args);
format = get_argument_format(&args);
+ sep = get_argument_separator(&args);
+
+ if (sep == 0) {
+ finalize_request(400, "INVALID_SEPARATOR", req, evb, &args, jsobj);
+ return;
+ }
+
if (format == json_format) {
jsobj = json_object_new_object();
+ result_array = json_object_new_array();
+ json_object_object_add(jsobj, "data", result_array);
}
read_options = leveldb_readoptions_create();
@@ -349,22 +404,24 @@ void mget_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
}
if (value) {
- terminated_value = value;
- DUPE_N_TERMINATE(terminated_value, vallen, tmp);
+ value = realloc(value, vallen + 1);
+ value[vallen] = '\0';
if (format == json_format) {
- json_object_object_add(jsobj, key, json_object_new_string(terminated_value));
+ tmp_obj = json_object_new_object();
+ json_object_object_add(tmp_obj, "key", json_object_new_string(key));
+ json_object_object_add(tmp_obj, "value", json_object_new_string(value));
+ json_object_array_add(result_array, tmp_obj);
} else {
- evbuffer_add_printf(evb, "%s,%s\n", key, terminated_value);
+ evbuffer_add_printf(evb, "%s%c%s\n", key, sep, value);
}
- free(terminated_value);
+ free(value);
}
- free(value);
}
leveldb_readoptions_destroy(read_options);
if (!nkeys) {
- finalize_request(400, "key is required", req, evb, &args, jsobj);
+ finalize_request(400, "MISSING_ARG_KEY", req, evb, &args, jsobj);
return;
}
@@ -374,22 +431,24 @@ void mget_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
void range_match_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
{
- char *start_key, *end_key, *key_clean, *value_clean, *tmp;
+ char *start_key, *end_key, *key_clean;
const char *key, *value;
+ char sep;
size_t key_len, value_len;
struct evkeyvalq args;
- struct json_object *jsobj, *tmp_obj, *result_array;
+ struct json_object *jsobj = NULL, *result_array = NULL, *tmp_obj;
const leveldb_snapshot_t *bt_snapshot;
leveldb_readoptions_t *bt_read_options;
leveldb_iterator_t *bt_iter;
- int result_count = 0, result_limit = 0;
+ int result_count = 0, result_limit = 0, format;
evhttp_parse_query(req->uri, &args);
+ format = get_argument_format(&args);
+ sep = get_argument_separator(&args);
start_key = (char *)evhttp_find_header(&args, "start");
end_key = (char *)evhttp_find_header(&args, "end");
result_limit = get_int_argument(&args, "limit", 500);
- jsobj = json_object_new_object();
if (start_key == NULL || end_key == NULL) {
finalize_request(400, "MISSING_ARG_KEY", req, evb, &args, jsobj);
return;
@@ -398,9 +457,16 @@ void range_match_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
finalize_request(400, "INVALID_START_KEY", req, evb, &args, jsobj);
return;
}
-
- result_array = json_object_new_array();
- tmp_obj = NULL;
+ if (sep == 0) {
+ finalize_request(400, "INVALID_SEPARATOR", req, evb, &args, jsobj);
+ return;
+ }
+
+ if (format == json_format) {
+ jsobj = json_object_new_object();
+ result_array = json_object_new_array();
+ json_object_object_add(jsobj, "data", result_array);
+ }
bt_read_options = leveldb_readoptions_create();
bt_snapshot = leveldb_create_snapshot(ldb);
@@ -411,30 +477,33 @@ void range_match_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
while (leveldb_iter_valid(bt_iter) && (result_limit == 0 || result_count < result_limit)) {
key = leveldb_iter_key(bt_iter, &key_len);
- key_clean = (char *)key;
- DUPE_N_TERMINATE(key_clean, key_len, tmp);
-
+ key_clean = strndup(key, key_len);
+
if (strcmp(key_clean, end_key) > 0) {
free(key_clean);
break;
}
-
+
value = leveldb_iter_value(bt_iter, &value_len);
- value_clean = (char *)value;
- DUPE_N_TERMINATE(value_clean, value_len, tmp);
- tmp_obj = json_object_new_object();
- json_object_object_add(tmp_obj, key_clean, json_object_new_string(value_clean));
- json_object_array_add(result_array, tmp_obj);
+ if (format == json_format) {
+ tmp_obj = json_object_new_object();
+ json_object_object_add(tmp_obj, key_clean, json_object_new_string_len(value, value_len));
+ json_object_array_add(result_array, tmp_obj);
+ } else {
+ evbuffer_add_printf(evb, "%s%c", key_clean, sep);
+ evbuffer_add(evb, value, value_len);
+ evbuffer_add_printf(evb, "\n");
+ }
leveldb_iter_next(bt_iter);
result_count ++;
free(key_clean);
- free(value_clean);
}
- json_object_object_add(jsobj, "data", result_array);
- json_object_object_add(jsobj, "status", json_object_new_string(result_count ? "ok" : "no results"));
+ if (format == json_format) {
+ json_object_object_add(jsobj, "status", json_object_new_string(result_count ? "ok" : "no results"));
+ }
finalize_request(200, NULL, req, evb, &args, jsobj);
@@ -445,28 +514,37 @@ void range_match_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
void fwmatch_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
{
- char *fw_key, *key_clean, *value_clean, *tmp;
+ char *fw_key, *key_clean;
const char *key, *value;
+ char sep;
size_t key_len, value_len;
struct evkeyvalq args;
- struct json_object *jsobj, *tmp_obj, *result_array;
+ struct json_object *jsobj = NULL, *result_array = NULL, *tmp_obj;
const leveldb_snapshot_t *fw_snapshot;
leveldb_readoptions_t *fw_read_options;
leveldb_iterator_t *fw_iter;
- int result_count = 0, result_limit = 0;
+ int result_count = 0, result_limit = 0, format;
evhttp_parse_query(req->uri, &args);
+ format = get_argument_format(&args);
+ sep = get_argument_separator(&args);
fw_key = (char *)evhttp_find_header(&args, "key");
result_limit = get_int_argument(&args, "limit", 500);
- jsobj = json_object_new_object();
- result_array = json_object_new_array();
- tmp_obj = NULL;
-
if (fw_key == NULL) {
finalize_request(400, "MISSING_ARG_KEY", req, evb, &args, jsobj);
return;
}
+ if (sep == 0) {
+ finalize_request(400, "INVALID_SEPARATOR", req, evb, &args, jsobj);
+ return;
+ }
+
+ if (format == json_format) {
+ jsobj = json_object_new_object();
+ result_array = json_object_new_array();
+ json_object_object_add(jsobj, "data", result_array);
+ }
fw_read_options = leveldb_readoptions_create();
fw_snapshot = leveldb_create_snapshot(ldb);
@@ -477,8 +555,7 @@ void fwmatch_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
while (leveldb_iter_valid(fw_iter) && (result_limit == 0 || result_count < result_limit)) {
key = leveldb_iter_key(fw_iter, &key_len);
- key_clean = (char *)key;
- DUPE_N_TERMINATE(key_clean, key_len, tmp);
+ key_clean = strndup(key, key_len);
// this is the case where we are only fwing keys of this prefix
// so we need to break out of the loop at the last key
@@ -487,21 +564,25 @@ void fwmatch_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
break;
}
value = leveldb_iter_value(fw_iter, &value_len);
- value_clean = (char *)value;
- DUPE_N_TERMINATE(value_clean, value_len, tmp);
- tmp_obj = json_object_new_object();
- json_object_object_add(tmp_obj, key_clean, json_object_new_string(value_clean));
- json_object_array_add(result_array, tmp_obj);
+ if (format == json_format) {
+ tmp_obj = json_object_new_object();
+ json_object_object_add(tmp_obj, key_clean, json_object_new_string_len(value, value_len));
+ json_object_array_add(result_array, tmp_obj);
+ } else {
+ evbuffer_add_printf(evb, "%s%c", key_clean, sep);
+ evbuffer_add(evb, value, value_len);
+ evbuffer_add_printf(evb, "\n");
+ }
leveldb_iter_next(fw_iter);
result_count ++;
free(key_clean);
- free(value_clean);
}
- json_object_object_add(jsobj, "data", result_array);
- json_object_object_add(jsobj, "status", json_object_new_string(result_count ? "ok" : "no results"));
+ if (format == json_format) {
+ json_object_object_add(jsobj, "status", json_object_new_string(result_count ? "ok" : "no results"));
+ }
finalize_request(200, NULL, req, evb, &args, jsobj);
@@ -510,110 +591,500 @@ void fwmatch_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
leveldb_release_snapshot(ldb, fw_snapshot);
}
-/* append a `value` string on to the end of a string value */
-void list_append_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
+/* append or prepend multiple `value` strings to a string value */
+void list_add_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
{
- char *key, *append_value, *orig_value, *new_value, *terminated_value;
+ char *key, *add_value, *orig_value;
+ size_t orig_valuelen;
+ char sep;
+ int format, ret_data;
+ struct evbuffer *new_value;
struct evkeyvalq args;
- struct json_object *jsobj;
+ struct evkeyval *arg_pair;
+ struct json_object *jsobj = NULL, *jsobj_data = NULL, *jsobj_value = NULL;
int response_code = HTTP_OK;
char *error = NULL;
- size_t orig_valuelen;
- char *tmp;
- int format;
- int echo_data;
leveldb_readoptions_t *read_options;
leveldb_writeoptions_t *write_options;
+ enum LIST_ADD_TYPE type = (enum LIST_ADD_TYPE) ctx;
evhttp_parse_query(req->uri, &args);
format = get_argument_format(&args);
- echo_data = get_int_argument(&args, "echo_data", 0);
+ ret_data = get_int_argument(&args, "return_data", 0);
key = (char *)evhttp_find_header(&args, "key");
- append_value = (char *)evhttp_find_header(&args, "value");
- // separator = (char *)evhttp_find_header(&args, "separator");
+ add_value = (char *)evhttp_find_header(&args, "value");
+ sep = get_argument_separator(&args);
- jsobj = json_object_new_object();
if (key == NULL) {
finalize_request(400, "MISSING_ARG_KEY", req, evb, &args, jsobj);
return;
}
- if (append_value == NULL) {
+ if (add_value == NULL) {
finalize_request(400, "MISSING_ARG_VALUE", req, evb, &args, jsobj);
return;
}
+ if (sep == 0) {
+ finalize_request(400, "INVALID_SEPARATOR", req, evb, &args, jsobj);
+ return;
+ }
+
+ if (format == json_format) {
+ jsobj = json_object_new_object();
+
+ if (ret_data) {
+ jsobj_data = json_object_new_object();
+ jsobj_value = json_object_new_array();
+ json_object_object_add(jsobj, "data", jsobj_data);
+ json_object_object_add(jsobj_data, "key", json_object_new_string(key));
+ json_object_object_add(jsobj_data, "value", jsobj_value);
+ }
+ }
+
+ new_value = evbuffer_new();
+ evbuffer_add_printf(new_value, "%s", ""); // null terminate
read_options = leveldb_readoptions_create();
leveldb_readoptions_set_verify_checksums(read_options, option_get_int("verify_checksums"));
orig_value = leveldb_get(ldb, read_options, key, strlen(key), &orig_valuelen, &error);
leveldb_readoptions_destroy(read_options);
- // null terminate orig_value
- if (orig_value) {
- terminated_value = orig_value;
- DUPE_N_TERMINATE(terminated_value, orig_valuelen, tmp);
- free(orig_value);
- orig_value = terminated_value;
+ if (!error) {
+ // append case - orig_value goes first
+ if (type == LIST_APPEND && orig_value) {
+ reserialize_list(new_value, jsobj_value, &orig_value, orig_valuelen, sep);
+ }
+
+ TAILQ_FOREACH(arg_pair, &args, next) {
+ if (strcmp(arg_pair->key, "value") != 0) {
+ continue;
+ }
+ // skip empty values
+ if (strlen(arg_pair->value) == 0) {
+ continue;
+ }
+ serialize_list_item(new_value, arg_pair->value, sep);
+
+ if (jsobj_value) {
+ json_object_array_add(jsobj_value, json_object_new_string(arg_pair->value));
+ }
+ }
+
+ // prepend case - orig_value goes last
+ if (type == LIST_PREPEND && orig_value) {
+ reserialize_list(new_value, jsobj_value, &orig_value, orig_valuelen, sep);
+ }
+
+ write_options = leveldb_writeoptions_create();
+ leveldb_put(ldb, write_options, key, strlen(key), (char *)EVBUFFER_DATA(new_value), EVBUFFER_LENGTH(new_value), &error);
+ leveldb_writeoptions_destroy(write_options);
}
+ if (ret_data && format == txt_format) {
+ evbuffer_add_printf(evb, "%s%c%s\n", key, sep, EVBUFFER_DATA(new_value));
+ }
- if (orig_value) {
- new_value = calloc(1, (orig_valuelen + 1 + strlen(append_value) + 1) * sizeof(char *));
- sprintf(new_value, "%s,%s", orig_value, append_value);
- } else {
- new_value = calloc(1, (strlen(append_value) + 1) * sizeof(char *));
- sprintf(new_value, "%s", append_value);
+ finalize_request(response_code, error, req, evb, &args, jsobj);
+ evbuffer_free(new_value);
+ free(orig_value);
+ free(error);
+}
+
+/* remove `value` strings from a string value */
+void list_remove_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
+{
+ char *key, *remove_value, *orig_value, *token;
+ size_t orig_valuelen;
+ char sep;
+ int format, ret_data;
+ struct evbuffer *new_value;
+ struct evkeyvalq args;
+ struct evkeyval *arg_pair;
+ struct json_object *jsobj = NULL, *jsobj_data = NULL, *jsobj_removed = NULL, *jsobj_value = NULL;
+ struct HashValue *remove_values_hash, *hash_value, *hash_tmp;
+ struct ListInfo list_info;
+ int response_code = HTTP_OK;
+ char *error = NULL;
+ int updated = 0;
+ leveldb_readoptions_t *read_options;
+ leveldb_writeoptions_t *write_options;
+
+ evhttp_parse_query(req->uri, &args);
+ format = get_argument_format(&args);
+ ret_data = get_int_argument(&args, "return_data", 0);
+
+ key = (char *)evhttp_find_header(&args, "key");
+ remove_value = (char *)evhttp_find_header(&args, "value");
+ sep = get_argument_separator(&args);
+
+ if (key == NULL) {
+ finalize_request(400, "MISSING_ARG_KEY", req, evb, &args, jsobj);
+ return;
+ }
+ if (remove_value == NULL) {
+ finalize_request(400, "MISSING_ARG_VALUE", req, evb, &args, jsobj);
+ return;
+ }
+ if (sep == 0) {
+ finalize_request(400, "INVALID_SEPARATOR", req, evb, &args, jsobj);
+ return;
+ }
+
+ if (format == json_format) {
+ jsobj = json_object_new_object();
+ jsobj_data = json_object_new_object();
+ jsobj_removed = json_object_new_array();
+ json_object_object_add(jsobj, "data", jsobj_data);
+ json_object_object_add(jsobj_data, "key", json_object_new_string(key));
+ json_object_object_add(jsobj_data, "removed", jsobj_removed);
+
+ if (ret_data) {
+ jsobj_value = json_object_new_array();
+ json_object_object_add(jsobj_data, "value", jsobj_value);
+ }
}
+ // put values to remove in a hash for quick lookup as list is walked
+ remove_values_hash = NULL;
+ TAILQ_FOREACH(arg_pair, &args, next) {
+ if (strcmp(arg_pair->key, "value") != 0) {
+ continue;
+ }
+ // skip empty values
+ if (strlen(arg_pair->value) == 0) {
+ continue;
+ }
+ HASH_FIND_STR(remove_values_hash, arg_pair->value, hash_value);
+ if (!hash_value) {
+ hash_value = calloc(1, sizeof(struct HashValue));
+ hash_value->value = arg_pair->value;
+ HASH_ADD_KEYPTR(hh, remove_values_hash, hash_value->value, strlen(hash_value->value), hash_value);
+ }
+ hash_value->count++;
+ }
+
+ new_value = evbuffer_new();
+ evbuffer_add_printf(new_value, "%s", ""); // null terminate
+
+ read_options = leveldb_readoptions_create();
+ leveldb_readoptions_set_verify_checksums(read_options, option_get_int("verify_checksums"));
+ orig_value = leveldb_get(ldb, read_options, key, strlen(key), &orig_valuelen, &error);
+ leveldb_readoptions_destroy(read_options);
+
+ if (orig_value && !error) {
+ prepare_token_list(&list_info, &orig_value, orig_valuelen, sep);
+
+ TOKEN_LIST_FOREACH(token, &list_info) {
+ HASH_FIND_STR(remove_values_hash, token, hash_value);
+ if (hash_value) {
+ updated = 1;
+ hash_value->count--;
+ if (hash_value->count == 0) {
+ HASH_DEL(remove_values_hash, hash_value);
+ free(hash_value);
+ }
+ if (jsobj_removed) {
+ json_object_array_add(jsobj_removed, json_object_new_string(token));
+ }
+ // don't copy this item
+ } else {
+ serialize_list_item(new_value, token, sep);
+ if (jsobj_value) {
+ json_object_array_add(jsobj_value, json_object_new_string(token));
+ }
+ }
+ }
+
+ if (updated == 1) {
+ write_options = leveldb_writeoptions_create();
+ leveldb_put(ldb, write_options, key, strlen(key), (char *)EVBUFFER_DATA(new_value), EVBUFFER_LENGTH(new_value), &error);
+ leveldb_writeoptions_destroy(write_options);
+ }
+ }
+
+ HASH_ITER(hh, remove_values_hash, hash_value, hash_tmp) {
+ HASH_DEL(remove_values_hash, hash_value);
+ free(hash_value);
+ }
+
+ if (ret_data && format == txt_format) {
+ evbuffer_add_printf(evb, "%s%c%s\n", key, sep, (char *)EVBUFFER_DATA(new_value));
+ }
+
+ finalize_request(response_code, error, req, evb, &args, jsobj);
+ evbuffer_free(new_value);
+ free(orig_value);
free(error);
+}
+
+/* pop `count` strings from a string representation of a list */
+void list_pop_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
+{
+ char *key, *orig_value, *token, *origptr;
+ size_t orig_valuelen;
+ char sep;
+ int position, count, cur_pos, format, ret_data;
+ struct evbuffer *new_value, *pop_value = NULL;
+ struct evkeyvalq args;
+ struct json_object *jsobj = NULL, *jsobj_data = NULL, *jsobj_popped = NULL, *jsobj_value = NULL;
+ struct ListInfo list_info;
+ int response_code = HTTP_OK;
+ char *error = NULL;
+ int updated = 0;
+ leveldb_readoptions_t *read_options;
+ leveldb_writeoptions_t *write_options;
- write_options = leveldb_writeoptions_create();
- leveldb_put(ldb, write_options, key, strlen(key), new_value, strlen(new_value), &error);
- leveldb_writeoptions_destroy(write_options);
+ evhttp_parse_query(req->uri, &args);
+ format = get_argument_format(&args);
+ ret_data = get_int_argument(&args, "return_data", 0);
- if (echo_data) {
- if (format == json_format) {
- json_object_object_add(jsobj, "data", json_object_new_string(new_value));
+ key = (char *)evhttp_find_header(&args, "key");
+ position = get_int_argument(&args, "position", 0);
+ count = get_int_argument(&args, "count", 1);
+ sep = get_argument_separator(&args);
+
+ if (key == NULL) {
+ finalize_request(400, "MISSING_ARG_KEY", req, evb, &args, jsobj);
+ return;
+ }
+ if (sep == 0) {
+ finalize_request(400, "INVALID_SEPARATOR", req, evb, &args, jsobj);
+ return;
+ }
+ if (count < 1) {
+ finalize_request(400, "INVALID_COUNT", req, evb, &args, jsobj);
+ return;
+ }
+
+ new_value = evbuffer_new();
+ evbuffer_add_printf(new_value, "%s", ""); // null terminate
+
+ if (format == json_format) {
+ jsobj = json_object_new_object();
+ jsobj_data = json_object_new_object();
+ jsobj_popped = json_object_new_array();
+ json_object_object_add(jsobj, "data", jsobj_data);
+ json_object_object_add(jsobj_data, "key", json_object_new_string(key));
+ json_object_object_add(jsobj_data, "popped", jsobj_popped);
+
+ if (ret_data) {
+ jsobj_value = json_object_new_array();
+ json_object_object_add(jsobj_data, "value", jsobj_value);
+ }
+ } else {
+ pop_value = evbuffer_new();
+ evbuffer_add_printf(pop_value, "%s", ""); // null terminate
+ }
+
+ read_options = leveldb_readoptions_create();
+ leveldb_readoptions_set_verify_checksums(read_options, option_get_int("verify_checksums"));
+ orig_value = leveldb_get(ldb, read_options, key, strlen(key), &orig_valuelen, &error);
+ leveldb_readoptions_destroy(read_options);
+
+ if (orig_value && !error) {
+ prepare_token_list(&list_info, &orig_value, orig_valuelen, sep);
+
+ if (position >= 0) {
+ cur_pos = 0;
+ TOKEN_LIST_FOREACH(token, &list_info) {
+ if (cur_pos == position && count > 0) {
+ if (format == json_format) {
+ json_object_array_add(jsobj_popped, json_object_new_string(token));
+ } else {
+ serialize_list_item(pop_value, token, sep);
+ }
+ count--;
+ updated = 1;
+ } else {
+ if (jsobj_value) {
+ json_object_array_add(jsobj_value, json_object_new_string(token));
+ }
+ serialize_list_item(new_value, token, sep);
+ cur_pos++;
+ }
+ }
} else {
- evbuffer_add_printf(evb, "%s,%s\n", key, new_value);
+ cur_pos = -1;
+ TOKEN_LIST_FOREACH_REVERSE(token, &list_info) {
+ if (cur_pos == position && count > 0) {
+ if (format == json_format) {
+ json_object_array_add(jsobj_popped, json_object_new_string(token));
+ } else {
+ serialize_list_item(pop_value, token, sep);
+ }
+ count--;
+ updated = 1;
+ // blank out, will not be in reconstituted list
+ memset(token, '\0', strlen(token));
+ } else {
+ // will be in reconstituted list
+ cur_pos--;
+ }
+ }
+
+ // reconstitute
+ if (updated) {
+ origptr = orig_value;
+ while (origptr < orig_value + orig_valuelen) {
+ if (*origptr != '\0') {
+ if (jsobj_value) {
+ json_object_array_add(jsobj_value, json_object_new_string(origptr));
+ }
+ serialize_list_item(new_value, origptr, sep);
+ origptr += strlen(origptr);
+ }
+ origptr++;
+ }
+ }
+ }
+
+ if (updated == 1) {
+ write_options = leveldb_writeoptions_create();
+ leveldb_put(ldb, write_options, key, strlen(key), (char *)EVBUFFER_DATA(new_value), EVBUFFER_LENGTH(new_value), &error);
+ leveldb_writeoptions_destroy(write_options);
}
}
+ if (format == txt_format) {
+ evbuffer_add_printf(evb, "%s\n", (char *)EVBUFFER_DATA(pop_value));
+ evbuffer_free(pop_value);
+ }
+
finalize_request(response_code, error, req, evb, &args, jsobj);
- free(new_value);
+ evbuffer_free(new_value);
free(orig_value);
free(error);
+}
+
+void set_add_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
+{
+ char *key, *add_value, *orig_value;
+ size_t orig_valuelen;
+ char sep;
+ int format, ret_data;
+ struct evbuffer *new_value;
+ struct evkeyvalq args;
+ struct evkeyval *arg_pair;
+ struct SetItem *set = NULL, *set_item;
+ struct json_object *jsobj = NULL, *jsobj_data = NULL, *jsobj_added = NULL, *jsobj_value = NULL;
+ int response_code = HTTP_OK;
+ char *error = NULL;
+ int updated = 0;
+ leveldb_readoptions_t *read_options;
+ leveldb_writeoptions_t *write_options;
+
+ evhttp_parse_query(req->uri, &args);
+ format = get_argument_format(&args);
+ ret_data = get_int_argument(&args, "return_data", 0);
+
+ key = (char *)evhttp_find_header(&args, "key");
+ add_value = (char *)evhttp_find_header(&args, "value");
+ sep = get_argument_separator(&args);
+
+ if (key == NULL) {
+ finalize_request(400, "MISSING_ARG_KEY", req, evb, &args, jsobj);
+ return;
+ }
+ if (add_value == NULL) {
+ finalize_request(400, "MISSING_ARG_VALUE", req, evb, &args, jsobj);
+ return;
+ }
+ if (sep == 0) {
+ finalize_request(400, "INVALID_SEPARATOR", req, evb, &args, jsobj);
+ return;
+ }
+
+ if (format == json_format) {
+ jsobj = json_object_new_object();
+ jsobj_data = json_object_new_object();
+ jsobj_added = json_object_new_array();
+ json_object_object_add(jsobj, "data", jsobj_data);
+ json_object_object_add(jsobj_data, "key", json_object_new_string(key));
+ json_object_object_add(jsobj_data, "added", jsobj_added);
+
+ if (ret_data) {
+ jsobj_value = json_object_new_array();
+ json_object_object_add(jsobj_data, "value", jsobj_value);
+ }
+ }
+
+ read_options = leveldb_readoptions_create();
+ leveldb_readoptions_set_verify_checksums(read_options, option_get_int("verify_checksums"));
+ orig_value = leveldb_get(ldb, read_options, key, strlen(key), &orig_valuelen, &error);
+ leveldb_readoptions_destroy(read_options);
+
+ new_value = evbuffer_new();
+ evbuffer_add_printf(new_value, "%s", ""); // null terminate
+
+ if (!error) {
+ if (orig_value) {
+ deserialize_alloc_set(&set, &orig_value, orig_valuelen, sep);
+ }
+
+ TAILQ_FOREACH(arg_pair, &args, next) {
+ if (strcmp(arg_pair->key, "value") != 0) {
+ continue;
+ }
+ // skip empty values
+ if (strlen(arg_pair->value) == 0) {
+ continue;
+ }
+
+ HASH_FIND_STR(set, arg_pair->value, set_item);
+ if (!set_item) {
+ add_new_set_item(&set, arg_pair->value);
+ updated = 1;
+
+ if (jsobj_added) {
+ json_object_array_add(jsobj_added, json_object_new_string(arg_pair->value));
+ }
+ }
+ }
+
+ serialize_free_set(new_value, jsobj_value, &set, sep);
+
+ if (updated) {
+ write_options = leveldb_writeoptions_create();
+ leveldb_put(ldb, write_options, key, strlen(key), (char *)EVBUFFER_DATA(new_value), EVBUFFER_LENGTH(new_value), &error);
+ leveldb_writeoptions_destroy(write_options);
+ }
+ }
+
+ if (ret_data && format == txt_format) {
+ evbuffer_add_printf(evb, "%s%c%s\n", key, sep, EVBUFFER_DATA(new_value));
+ }
+ finalize_request(response_code, error, req, evb, &args, jsobj);
+ evbuffer_free(new_value);
+ free(orig_value);
+ free(error);
}
-/* remove a `value` from string */
-void list_remove_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
+void set_remove_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
{
- char *key, *remove_value, *orig_value, *terminated_value;
+ char *key, *remove_value, *orig_value;
+ size_t orig_valuelen;
+ char sep;
+ int format, ret_data;
+ struct evbuffer *new_value;
struct evkeyvalq args;
- struct json_object *jsobj;
+ struct evkeyval *arg_pair;
+ struct SetItem *set = NULL, *set_item;
+ struct json_object *jsobj = NULL, *jsobj_data = NULL, *jsobj_removed = NULL, *jsobj_value = NULL;
int response_code = HTTP_OK;
char *error = NULL;
- size_t orig_valuelen;
- char *tmp;
- int i;
int updated = 0;
- char *token;
- struct evbuffer *new_value;
- int format;
- int echo_data;
leveldb_readoptions_t *read_options;
leveldb_writeoptions_t *write_options;
evhttp_parse_query(req->uri, &args);
format = get_argument_format(&args);
- echo_data = get_int_argument(&args, "echo_data", 0);
+ ret_data = get_int_argument(&args, "return_data", 0);
key = (char *)evhttp_find_header(&args, "key");
remove_value = (char *)evhttp_find_header(&args, "value");
- // separator = (char *)evhttp_find_header(&args, "separator");
+ sep = get_argument_separator(&args);
- jsobj = json_object_new_object();
if (key == NULL) {
finalize_request(400, "MISSING_ARG_KEY", req, evb, &args, jsobj);
return;
@@ -622,70 +1093,174 @@ void list_remove_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
finalize_request(400, "MISSING_ARG_VALUE", req, evb, &args, jsobj);
return;
}
+ if (sep == 0) {
+ finalize_request(400, "INVALID_SEPARATOR", req, evb, &args, jsobj);
+ return;
+ }
+
+ if (format == json_format) {
+ jsobj = json_object_new_object();
+ jsobj_data = json_object_new_object();
+ jsobj_removed = json_object_new_array();
+ json_object_object_add(jsobj, "data", jsobj_data);
+ json_object_object_add(jsobj_data, "key", json_object_new_string(key));
+ json_object_object_add(jsobj_data, "removed", jsobj_removed);
+
+ if (ret_data) {
+ jsobj_value = json_object_new_array();
+ json_object_object_add(jsobj_data, "value", jsobj_value);
+ }
+ }
read_options = leveldb_readoptions_create();
leveldb_readoptions_set_verify_checksums(read_options, option_get_int("verify_checksums"));
orig_value = leveldb_get(ldb, read_options, key, strlen(key), &orig_valuelen, &error);
leveldb_readoptions_destroy(read_options);
- // null terminate orig_value
- if (orig_value) {
- terminated_value = orig_value;
- DUPE_N_TERMINATE(terminated_value, orig_valuelen, tmp);
- free(orig_value);
- orig_value = terminated_value;
- }
-
- if (orig_value) {
- new_value = evbuffer_new();
- token = strtok(orig_value, ",");
- i = 0;
- while (token) {
- if (strcmp(token, remove_value) == 0) {
- // we found the token
+ new_value = evbuffer_new();
+ evbuffer_add_printf(new_value, "%s", ""); // null terminate
+
+ if (orig_value && !error) {
+ deserialize_alloc_set(&set, &orig_value, orig_valuelen, sep);
+
+ TAILQ_FOREACH(arg_pair, &args, next) {
+ if (strcmp(arg_pair->key, "value") != 0) {
+ continue;
+ }
+ // skip empty values
+ if (strlen(arg_pair->value) == 0) {
+ continue;
+ }
+
+ HASH_FIND_STR(set, arg_pair->value, set_item);
+ if (set_item) {
+ HASH_DEL(set, set_item);
+ free(set_item);
updated = 1;
- } else {
- if (i == 0) {
- evbuffer_add_printf(new_value, "%s", token);
- } else {
- evbuffer_add_printf(new_value, ",%s", token);
+
+ if (jsobj_removed) {
+ json_object_array_add(jsobj_removed, json_object_new_string(arg_pair->value));
}
- i++;
}
- token = strtok(NULL, ",");
}
- if (updated == 1) {
+
+ serialize_free_set(new_value, jsobj_value, &set, sep);
+
+ if (updated) {
write_options = leveldb_writeoptions_create();
leveldb_put(ldb, write_options, key, strlen(key), (char *)EVBUFFER_DATA(new_value), EVBUFFER_LENGTH(new_value), &error);
leveldb_writeoptions_destroy(write_options);
}
+ }
+
+ if (ret_data && format == txt_format) {
+ evbuffer_add_printf(evb, "%s%c%s\n", key, sep, EVBUFFER_DATA(new_value));
+ }
+
+ finalize_request(response_code, error, req, evb, &args, jsobj);
+ evbuffer_free(new_value);
+ free(orig_value);
+ free(error);
+}
+
+void set_pop_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
+{
+ char *key, *orig_value;
+ size_t orig_valuelen;
+ char sep;
+ int format, count, ret_data;
+ struct evkeyvalq args;
+ struct evbuffer *new_value, *pop_value = NULL;
+ struct SetItem *set = NULL, *set_item, *set_tmp;
+ struct json_object *jsobj = NULL, *jsobj_data = NULL, *jsobj_popped = NULL, *jsobj_value = NULL;
+ int response_code = HTTP_OK;
+ char *error = NULL;
+ int updated = 0;
+ leveldb_readoptions_t *read_options;
+ leveldb_writeoptions_t *write_options;
+
+ evhttp_parse_query(req->uri, &args);
+ format = get_argument_format(&args);
+ ret_data = get_int_argument(&args, "return_data", 0);
+
+ key = (char *)evhttp_find_header(&args, "key");
+ count = get_int_argument(&args, "count", 1);
+ sep = get_argument_separator(&args);
+
+ if (key == NULL) {
+ finalize_request(400, "MISSING_ARG_KEY", req, evb, &args, jsobj);
+ return;
+ }
+ if (count < 1) {
+ finalize_request(400, "INVALID_COUNT", req, evb, &args, jsobj);
+ return;
+ }
+ if (sep == 0) {
+ finalize_request(400, "INVALID_SEPARATOR", req, evb, &args, jsobj);
+ return;
+ }
+
+ if (format == json_format) {
+ jsobj = json_object_new_object();
+ jsobj_data = json_object_new_object();
+ jsobj_popped = json_object_new_array();
+ json_object_object_add(jsobj, "data", jsobj_data);
+ json_object_object_add(jsobj_data, "key", json_object_new_string(key));
+ json_object_object_add(jsobj_data, "popped", jsobj_popped);
+
+ if (ret_data) {
+ jsobj_value = json_object_new_array();
+ json_object_object_add(jsobj_data, "value", jsobj_value);
+ }
+ } else {
+ pop_value = evbuffer_new();
+ evbuffer_add_printf(pop_value, "%s", ""); // null terminate
+ }
+
+ read_options = leveldb_readoptions_create();
+ leveldb_readoptions_set_verify_checksums(read_options, option_get_int("verify_checksums"));
+ orig_value = leveldb_get(ldb, read_options, key, strlen(key), &orig_valuelen, &error);
+ leveldb_readoptions_destroy(read_options);
+
+ new_value = evbuffer_new();
+ evbuffer_add_printf(new_value, "%s", ""); // null terminate
+
+ if (orig_value && !error) {
+ deserialize_alloc_set(&set, &orig_value, orig_valuelen, sep);
- if (echo_data) {
+ HASH_ITER(hh, set, set_item, set_tmp) {
+ if (count <= 0) {
+ break;
+ }
if (format == json_format) {
- json_object_object_add(jsobj, "data", json_object_new_string((char *)EVBUFFER_DATA(new_value)));
+ json_object_array_add(jsobj_popped, json_object_new_string(set_item->value));
} else {
- evbuffer_add_printf(evb, "%s,%s\n", key, (char *)EVBUFFER_DATA(new_value));
+ serialize_list_item(pop_value, set_item->value, sep);
}
+ HASH_DEL(set, set_item);
+ free(set_item);
+ count--;
+ updated = 1;
}
- evbuffer_free(new_value);
+ serialize_free_set(new_value, jsobj_value, &set, sep);
- } else {
- if (echo_data) {
- if (format == json_format) {
- json_object_object_add(jsobj, "data", json_object_new_string(""));
- } else {
- evbuffer_add_printf(evb, "%s,\n", key);
- }
+ if (updated) {
+ write_options = leveldb_writeoptions_create();
+ leveldb_put(ldb, write_options, key, strlen(key), (char *)EVBUFFER_DATA(new_value), EVBUFFER_LENGTH(new_value), &error);
+ leveldb_writeoptions_destroy(write_options);
}
}
- free(error);
+ if (format == txt_format) {
+ evbuffer_add_printf(evb, "%s\n", EVBUFFER_DATA(pop_value));
+ evbuffer_free(pop_value);
+ }
finalize_request(response_code, error, req, evb, &args, jsobj);
+ evbuffer_free(new_value);
free(orig_value);
free(error);
-
}
/*
@@ -695,13 +1270,9 @@ note: this makes a snapshot of the database and may return after other data has
void dump_csv_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
{
struct evkeyvalq args;
- int format;
- struct json_object *jsobj;
-
- jsobj = json_object_new_object();
+ struct json_object *jsobj = NULL;
evhttp_parse_query(req->uri, &args);
- format = get_argument_format(&args);
dump_fwmatch_key = (char *)evhttp_find_header(&args, "key");
if (dump_fwmatch_key) {
dump_fwmatch_key = strdup(dump_fwmatch_key);
@@ -727,7 +1298,6 @@ void dump_csv_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
}
evhttp_clear_headers(&args);
- json_object_put(jsobj);
evhttp_send_reply_start(req, 200, "OK");
evhttp_connection_set_closecb(req->evcon, cleanup_dump_csv_cb, NULL);
@@ -906,8 +1476,13 @@ int main(int argc, char **argv)
}
simplehttp_init();
- simplehttp_set_cb("/list_append*", list_append_cb, NULL);
+ simplehttp_set_cb("/list_append*", list_add_cb, (void *) LIST_APPEND);
+ simplehttp_set_cb("/list_prepend*", list_add_cb, (void *) LIST_PREPEND);
simplehttp_set_cb("/list_remove*", list_remove_cb, NULL);
+ simplehttp_set_cb("/list_pop*", list_pop_cb, NULL);
+ simplehttp_set_cb("/set_add*", set_add_cb, NULL);
+ simplehttp_set_cb("/set_remove*", set_remove_cb, NULL);
+ simplehttp_set_cb("/set_pop*", set_pop_cb, NULL);
simplehttp_set_cb("/get*", get_cb, NULL);
simplehttp_set_cb("/mget*", mget_cb, NULL);
simplehttp_set_cb("/range_match*", range_match_cb, NULL);
View
90 simpleleveldb/str_list_set.c
@@ -0,0 +1,90 @@
+#include <stdlib.h>
+#include <json/json.h>
+#include "str_list_set.h"
+
+void prepare_token_list(struct ListInfo *list_info, char **db_data, size_t db_data_len, char sep)
+{
+ // null terminate
+ *db_data = realloc(*db_data, db_data_len + 1);
+ *(*db_data + db_data_len) = '\0';
+
+ list_info->sep = sep;
+ list_info->buf = db_data;
+ list_info->buflen = db_data_len;
+
+ // for strtok_r()
+ list_info->sepstr[0] = sep;
+ list_info->sepstr[1] = '\0';
+
+ // for reverse_tokenize()
+ list_info->saveptr = *db_data + db_data_len;
+}
+
+// somewhat like strtok_r() but depends on prepare_token_list() for setup
+char *reverse_tokenize(struct ListInfo *list_info)
+{
+ while (list_info->saveptr > *list_info->buf) {
+ list_info->saveptr--;
+ if (list_info->saveptr == *list_info->buf) {
+ if (*list_info->saveptr != list_info->sep) {
+ return list_info->saveptr;
+ } else {
+ return NULL;
+ }
+ }
+ if (*list_info->saveptr == list_info->sep) {
+ *list_info->saveptr = '\0';
+ if (*(list_info->saveptr + 1) != '\0') {
+ return list_info->saveptr + 1;
+ }
+ }
+ }
+ return NULL;
+}
+
+void reserialize_list(struct evbuffer *output, struct json_object *array, char **db_data, size_t db_data_len, char sep)
+{
+ struct ListInfo list_info;
+ char *item;
+
+ prepare_token_list(&list_info, db_data, db_data_len, sep);
+
+ TOKEN_LIST_FOREACH(item, &list_info) {
+ serialize_list_item(output, item, sep);
+
+ if (array) {
+ json_object_array_add(array, json_object_new_string(item));
+ }
+ }
+}
+
+void deserialize_alloc_set(struct SetItem **set, char **db_data, size_t db_data_len, char sep)
+{
+ char *token;
+ struct SetItem *set_item;
+ struct ListInfo list_info;
+
+ prepare_token_list(&list_info, db_data, db_data_len, sep);
+
+ TOKEN_LIST_FOREACH(token, &list_info) {
+ HASH_FIND_STR(*set, token, set_item);
+ if (!set_item) {
+ add_new_set_item(set, token);
+ }
+ }
+}
+
+void serialize_free_set(struct evbuffer *output, struct json_object *array, struct SetItem **set, char sep)
+{
+ struct SetItem *set_item, *set_tmp;
+
+ HASH_ITER(hh, *set, set_item, set_tmp) {
+ if (array) {
+ json_object_array_add(array, json_object_new_string(set_item->value));
+ }
+ serialize_list_item(output, set_item->value, sep);
+ HASH_DEL(*set, set_item);
+ free(set_item);
+ }
+}
+
View
54 simpleleveldb/str_list_set.h
@@ -0,0 +1,54 @@
+#ifndef _STR_LIST_SET_H
+#define _STR_LIST_SET_H
+
+#include <event.h>
+#include <simplehttp/uthash.h>
+
+struct SetItem {
+ const char *value;
+ UT_hash_handle hh;
+};
+
+struct ListInfo {
+ char sep;
+ char sepstr[2];
+ char *saveptr;
+ char **buf;
+ size_t buflen;
+};
+
+void prepare_token_list(struct ListInfo *list_info, char **db_data, size_t db_data_len, char sep);
+char *reverse_tokenize(struct ListInfo *list_info);
+void reserialize_list(struct evbuffer *output, struct json_object *array, char **db_data, size_t db_data_len, char sep);
+void deserialize_alloc_set(struct SetItem **set, char **db_data, size_t db_data_len, char sep);
+void serialize_free_set(struct evbuffer *output, struct json_object *array, struct SetItem **set, char sep);
+
+static inline void serialize_list_item(struct evbuffer *output, const char *item, char sep)
+{
+ if (EVBUFFER_LENGTH(output) > 0) {
+ evbuffer_add_printf(output, "%c", sep);
+ }
+ evbuffer_add_printf(output, "%s", item);
+}
+
+static inline void add_new_set_item(struct SetItem **set, char *value_ptr)
+{
+ struct SetItem *set_item;
+ set_item = calloc(1, sizeof(struct SetItem));
+ set_item->value = value_ptr;
+ HASH_ADD_KEYPTR(hh, *set, value_ptr, strlen(value_ptr), set_item);
+}
+
+#define TOKEN_LIST_FOREACH(item, list_info) \
+ for (item = strtok_r(*(list_info)->buf, \
+ (list_info)->sepstr, \
+ &(list_info)->saveptr); \
+ item != NULL; \
+ item = strtok_r(NULL, \
+ (list_info)->sepstr, \
+ &(list_info)->saveptr) )\
+
+#define TOKEN_LIST_FOREACH_REVERSE(item, list_info) \
+ while ((item = reverse_tokenize(list_info)) != NULL) \
+
+#endif
View
184 simpleleveldb/test_simpleleveldb.py
@@ -1,47 +1,95 @@
import os
+import re
import sys
sys.path.append(os.path.join(os.path.dirname(__file__), "../shared_tests"))
-import logging
from test_shunt import valgrind_cmd, SubprocessTest, http_fetch, http_fetch_json
class SimpleLeveldbTest(SubprocessTest):
binary_name = "simpleleveldb"
- process_options = [valgrind_cmd(os.path.abspath('simpleleveldb'), '--db-file=%s/db' % os.path.join(os.path.dirname(__file__), "test_output"), '--enable-logging')]
working_dir = os.path.dirname(__file__)
+ test_output_dir = os.path.join(working_dir, "test_output")
+ process_options = [valgrind_cmd(test_output_dir, os.path.join(working_dir, binary_name), '--db-file=%s/db' % test_output_dir, '--enable-logging')]
+ # /put, /get, /del
def test_basic(self):
- data = http_fetch_json('/put', dict(key='test', value='12345'))
- data = http_fetch_json('/get', dict(key='test'))
+ data = http_fetch_json('/put', dict(key='test_basic', value='12345'))
+ data = http_fetch_json('/get', dict(key='test_basic'))
assert data == '12345'
- data = http_fetch('/get', dict(key='test', format='txt'))
- assert data == 'test,12345\n'
- data = http_fetch_json('/del', dict(key='test'))
- data = http_fetch_json('/get', dict(key='test'), 404, 'NOT_FOUND')
-
+ data = http_fetch_json('/get', dict(key='test_basic', format='json'))
+ assert data == '12345'
+ data = http_fetch('/get', dict(key='test_basic', format='txt'))
+ assert data == 'test_basic,12345\n'
+ data = http_fetch('/get', dict(key='test_basic', format='txt', separator='/'))
+ assert data == 'test_basic/12345\n'
+ data = http_fetch('/put', dict(key='test_basic', value='22', format='txt'))
+ data = http_fetch_json('/get', dict(key='test_basic'))
+ assert data == '22'
+ data = http_fetch_json('/put', dict(key='test_basic', value='33', format='json'))
+ data = http_fetch_json('/get', dict(key='test_basic'))
+ assert data == '33'
+ data = http_fetch_json('/put', dict(key='test_basic'), body="44")
+ data = http_fetch_json('/get', dict(key='test_basic'))
+ assert data == '44'
+ data = http_fetch_json('/del', dict(key='test_basic'))
+ data = http_fetch_json('/get', dict(key='test_basic'), 404, 'NOT_FOUND')
+ data = http_fetch('/get', dict(key='test_basic', format='txt'), 404)
+ data = http_fetch_json('/get', dict(key='test_basic', format='json'), 404, 'NOT_FOUND')
+ data = http_fetch_json('/put', dict(key='test_basic', value='a'))
+ data = http_fetch('/del', dict(key='test_basic', format='txt'))
+ data = http_fetch_json('/get', dict(key='test_basic'), 404, 'NOT_FOUND')
+
+ def test_missing(self):
http_fetch_json("/put", dict(), 400, 'MISSING_ARG_KEY')
- http_fetch_json("/put", dict(key='test'), 400, 'MISSING_ARG_VALUE')
+ http_fetch_json("/put", dict(key='test_missing'), 400, 'MISSING_ARG_VALUE')
http_fetch_json("/get", dict(), 400, 'MISSING_ARG_KEY')
-
+ http_fetch_json("/del", dict(), 400, 'MISSING_ARG_KEY')
http_fetch_json("/fwmatch", dict(), 400, 'MISSING_ARG_KEY')
-
- http_fetch_json('/put', dict(key='test1', value='asdf1'))
- http_fetch_json('/put', dict(key='test2', value='asdf2'))
-
- data = http_fetch('/mget', dict(key=['test1', 'test2', 'test3'], format='txt'))
- print data
- assert data == 'test1,asdf1\ntest2,asdf2\n'
-
- data = http_fetch_json("/fwmatch", dict(key="test"))
- print data
- assert data == [{'test1': 'asdf1'}, {'test2': 'asdf2'}]
-
-
- # test list stuff
+ http_fetch("/put", dict(format="txt"), 400)
+ http_fetch("/put", dict(format='txt', key='test_missing'), 400)
+ http_fetch("/get", dict(format='txt'), 400)
+ http_fetch("/del", dict(format='txt'), 400)
+ http_fetch_json("/get", dict(key="test_missing", separator="asdf"), 400, 'INVALID_SEPARATOR')
+ http_fetch_json("/get", dict(key="test_missing", separator=""), 400, 'INVALID_SEPARATOR')
+ http_fetch("/get", dict(format="txt", key="test_missing", separator=""), 400)
+ http_fetch_json('/mput', dict(), 400, 'MISSING_ARG_VALUE', body='.')
+ http_fetch_json('/mput', dict(), 400, 'MALFORMED_CSV', body='vvvvvvvvvv')
+ http_fetch_json('/mput', dict(separator='|'), 400, 'MALFORMED_CSV', body='test_missing,val1\n')
+ http_fetch_json('/mget', dict(separator=';;'), 400, 'INVALID_SEPARATOR')
+ http_fetch_json('/mget', dict(), 400, 'MISSING_ARG_KEY')
+ http_fetch('/mget', dict(format="txt"), 400)
+ http_fetch_json('/fwmatch', dict(), 400, 'MISSING_ARG_KEY')
+ http_fetch_json('/range_match', dict(), 400, 'MISSING_ARG_KEY')
+ http_fetch_json('/range_match', dict(start="a"), 400, 'MISSING_ARG_KEY')
+ http_fetch_json('/range_match', dict(end="b"), 400, 'MISSING_ARG_KEY')
+ http_fetch_json('/range_match', dict(start="b", end="a"), 400, 'INVALID_START_KEY')
+
+ def test_multikey(self):
+ http_fetch_json('/put', dict(key='test_multikey_1', value='asdf1'))
+ http_fetch_json('/put', dict(key='test_multikey_2', value='asdf2'))
+ data = http_fetch_json('/mget', dict(key=['test_multikey_1', 'test_multikey_2', 'test_multikey_3']))
+ assert data == [{'key': 'test_multikey_1', 'value': 'asdf1'},
+ {'key': 'test_multikey_2', 'value': 'asdf2'}]
+ data = http_fetch('/mget', dict(key=['test_multikey_1', 'test_multikey_2', 'test_multikey_3'], format='txt'))
+ assert data == 'test_multikey_1,asdf1\ntest_multikey_2,asdf2\n'
+ data = http_fetch_json("/fwmatch", dict(key="test_multikey"))
+ assert data == [{'test_multikey_1': 'asdf1'}, {'test_multikey_2': 'asdf2'}]
+ data = http_fetch("/fwmatch", dict(key="test_multikey", format="txt"))
+ assert data == "test_multikey_1,asdf1\ntest_multikey_2,asdf2\n"
+ http_fetch_json('/mput', body='test_multikey_3,mv1a;mv1b\ntest_multikey_4,mv2a;mv2b\ntest_multikey_5,mv3a')
+ data = http_fetch_json('/get', dict(key='test_multikey_3'))
+ assert data == 'mv1a;mv1b'
+ data = http_fetch('/mget', dict(key=['test_multikey_4', 'test_multikey_5'], format='txt'))
+ assert data == 'test_multikey_4,mv2a;mv2b\ntest_multikey_5,mv3a\n'
+ http_fetch_json('/mput', body='test_multikey_ms,a,b,c')
+ data = http_fetch_json('/get', dict(key='test_multikey_ms'))
+ assert data == 'a,b,c'
+
+ def test_lists_1(self):
http_fetch_json('/get', dict(key='list_test'), 404, 'NOT_FOUND')
- data = http_fetch_json('/list_append', dict(key='list_test', value='testvalue1', echo_data='1'))
- assert data == 'testvalue1'
+ data = http_fetch_json('/list_append', dict(key='list_test', value='testvalue1', return_data='1'))
+ assert data['value'] == ['testvalue1']
data = http_fetch_json('/list_append', dict(key='list_test', value='testvalue2'))
assert data == ''
data = http_fetch_json('/get', dict(key='list_test'))
@@ -53,24 +101,90 @@ def test_basic(self):
data = http_fetch_json('/list_remove', dict(key='list_test', value='testvalue1'))
data = http_fetch_json('/get', dict(key='list_test'))
assert data == 'testvalue3'
-
- # try a /put with a POST body
+
+ def test_post(self):
http_fetch_json('/put', dict(key='testpost'), body='asdfpost')
data = http_fetch_json('/get', dict(key='testpost'))
assert data == 'asdfpost'
data = http_fetch_json('/del', dict(key='testpost'))
-
- # test dump to csv
+
+ def test_dump_csv(self):
# we need to check more than 500 entries
+ http_fetch_json('/put', dict(key='zzz.dump', value='bump.value'))
for x in range(505):
http_fetch_json('/put', dict(key='dump.%d' % x, value='dump.value.%d' % x))
-
data = http_fetch('/dump_csv')
assert data.startswith("dump.0,dump.value.0\n")
- assert data.endswith("test2,asdf2\n")
assert data.count("\n") > 505
-
data = http_fetch('/dump_csv', dict(key="dump."))
assert data.count("\n") == 505
+
+ def test_lists_2(self):
+ http_fetch_json('/list_prepend', dict(key='new_list', value='appval1'))
+ data = http_fetch_json('/list_prepend', dict(key='new_list', value='appvala', return_data='1'))
+ assert data['value'] == ['appvala', 'appval1']
+ data = http_fetch('/list_prepend', dict(key='new_list', value='appvalb', return_data='1', format='txt'))
+ assert data == 'new_list,appvalb,appvala,appval1\n'
+ data = http_fetch_json('/get', dict(key='new_list'))
+ assert data == 'appvalb,appvala,appval1'
+ data = http_fetch_json('/list_pop', dict(key='new_list'))
+ assert data['popped'] == ['appvalb']
+ data = http_fetch_json('/get', dict(key='new_list'))
+ assert data == 'appvala,appval1'
+ data = http_fetch('/list_pop', dict(key='new_list', position='-1', format='txt'))
+ assert data == 'appval1\n'
+ data = http_fetch_json('/get', dict(key='new_list'))
+ assert data == 'appvala'
+ http_fetch_json('/list_append', dict(key='new_list', value=["blah1", "blah2", "blah3"]))
+ data = http_fetch_json('/get', dict(key='new_list'))
+ assert data == 'appvala,blah1,blah2,blah3'
+ data = http_fetch_json('/list_pop', dict(key='new_list', position='2', count='2'))
+ assert data['popped'] == ['blah2', 'blah3']
+
+ def test_sets_1(self):
+ http_fetch_json('/set_add', dict(key='testset', value=['si1', 'si2']))
+ http_fetch_json('/set_remove', dict(key='testset', value='si1'))
+ data = http_fetch_json('/get', dict(key='testset'))
+ assert data == 'si2'
+ http_fetch_json('/set_add', dict(key='testset', value=['si1', 'si2', 'si3']))
+ data = http_fetch_json('/get', dict(key='testset'))
+ datalist = re.split(',', data)
+ assert 'si1' in datalist
+ assert 'si2' in datalist
+ assert 'si3' in datalist
+ assert ','.join(sorted(datalist)) == 'si1,si2,si3'
+ http_fetch_json('/set_remove', dict(key='testset', value=['si3', 'si1', 'si2']))
+ data = http_fetch_json('/get', dict(key='testset'))
+ assert data == ''
+ set_list = ['sl1', 'sl2', 'sl3', 'sl4']
+ http_fetch_json('/set_add', dict(key='testset', value=set_list))
+ for x in range(len(set_list)):
+ data = http_fetch_json('/set_pop', dict(key='testset'))
+ assert len(data['popped']) == 1
+ assert data['popped'][0] in set_list
+ set_list.remove(data['popped'][0])
+ data = http_fetch_json('/set_pop', dict(key='testset'))
+ assert data['popped'] == []
+
+ def test_separators(self):
+ http_fetch_json('/list_append', dict(key='testsep', value=['3,9', '4,16', '2,4'], separator='|'))
+ data = http_fetch_json('/get', dict(key='testsep'))
+ assert data == '3,9|4,16|2,4'
+ http_fetch_json('/list_remove', dict(key='testsep', value='2,4', separator='|'))
+ data = http_fetch_json('/get', dict(key='testsep'))
+ assert data == '3,9|4,16'
+ http_fetch_json('/set_add', dict(key='testsep', value=['5,25', '7,49'], separator='|'))
+ http_fetch_json('/set_remove', dict(key='testsep', value=['3,9', '5,25'], separator='|'))
+ data = http_fetch_json('/get', dict(key='testsep'))
+ datalist = re.split('|', data)
+ assert '|'.join(sorted(datalist)) == '4,16|7,49'
+ http_fetch_json('/list_pop', dict(key='testsep', separator='long'), 400, 'INVALID_SEPARATOR')
+ http_fetch_json('/mput', dict(separator='|'), body='testsep2|sepv2\ntestsep1|sepv1\n')
+ data = http_fetch_json('/get', dict(key='testsep1'))
+ assert data == 'sepv1'
+ data = http_fetch_json('/get', dict(key='testsep2'))
+ assert data == 'sepv2'
+
-
+if __name__ == "__main__":
+ print "usage: py.test"
View
4 simplequeue/test_simplequeue.py
@@ -2,14 +2,14 @@
import sys
sys.path.append(os.path.join(os.path.dirname(__file__), "../shared_tests"))
-import logging
import simplejson as json
from test_shunt import valgrind_cmd, SubprocessTest, http_fetch, http_fetch_json
class SimplequeueTest(SubprocessTest):
binary_name = "simplequeue"
- process_options = [valgrind_cmd(os.path.abspath('simplequeue'), '--enable-logging')]
working_dir = os.path.dirname(__file__)
+ test_output_dir = os.path.join(working_dir, "test_output")
+ process_options = [valgrind_cmd(test_output_dir, os.path.join(working_dir, binary_name), '--enable-logging')]
def test_basic(self):
# put/get in order
Please sign in to comment.
Something went wrong with that request. Please try again.