Skip to content
Browse files

refactor stats collection into simplehttp; add generic logging to sim…

…plehttp with -V option; update all apps to compile; add gitignore
  • Loading branch information...
1 parent 0b11575 commit e91b668e03a7b14d43b95ba61fb5cd38d1662ff1 @mreiferson mreiferson committed
View
4 .gitignore
@@ -0,0 +1,4 @@
+*.o
+*.a
+*.dSYM
+sortdb/sortdb
View
1 lib/log.h
@@ -1 +0,0 @@
-void simplehttp_log(const char type, const char *host, struct evhttp_request *req, uint64_t req_time, const char *id);
View
15 lib/timer.h
@@ -1,15 +0,0 @@
-#if _POSIX_TIMERS > 0
-
-static struct timespec ts1, ts2;
-
-void _gettime(struct timespec *ts);
-unsigned int _ts_diff(struct timespec start, struct timespec end);
-
-#else
-
-static struct timeval ts1, ts2;
-
-void _gettime(struct timeval *ts);
-unsigned int _ts_diff(struct timeval start, struct timeval end);
-
-#endif
View
6 ps_to_file/ps_to_file.c
@@ -4,10 +4,8 @@
#include <string.h>
#include <unistd.h>
#include <time.h>
-
-#include "simplehttp/pubsubclient.h"
-#include "event.h"
-#include "evhttp.h"
+#include <simplehttp/pubsubclient.h>
+#include <simplehttp/simplehttp.h>
#define DEBUG 0
View
2 pubsub/Makefile
@@ -5,7 +5,7 @@ LIBSIMPLEHTTP_INC ?= $(LIBSIMPLEHTTP)/..
LIBSIMPLEHTTP_LIB ?= $(LIBSIMPLEHTTP)
CFLAGS = -I. -I$(LIBSIMPLEHTTP_INC) -I$(LIBEVENT)/include -O2 -g
-LIBS = -L. -L$(LIBSIMPLEHTTP_LIB) -L$(LIBEVENT)/lib -levent -lsimplehttp -lpcre
+LIBS = -L. -L$(LIBSIMPLEHTTP_LIB) -L$(LIBEVENT)/lib -levent -lsimplehttp -lpcre -lm
pubsub: pubsub.c
$(CC) $(CFLAGS) -o $@ $< $(LIBS)
View
4 pubsub/pubsub.c
@@ -2,8 +2,8 @@
#include <stdio.h>
#include <string.h>
#include <time.h>
-#include "simplehttp/queue.h"
-#include "simplehttp/simplehttp.h"
+#include <simplehttp/queue.h>
+#include <simplehttp/simplehttp.h>
#include "http-internal.h"
#define BOUNDARY "xXPubSubXx"
View
2 pubsub_filtered/Makefile
@@ -5,7 +5,7 @@ LIBSIMPLEHTTP_INC ?= $(LIBSIMPLEHTTP)/..
LIBSIMPLEHTTP_LIB ?= $(LIBSIMPLEHTTP)
CFLAGS = -I. -I$(LIBSIMPLEHTTP_INC) -I$(LIBEVENT)/include -g -O2
-LIBS = -L. -L$(LIBSIMPLEHTTP_LIB) -L$(LIBEVENT)/lib -levent -lsimplehttp -ljson -lpcre
+LIBS = -L. -L$(LIBSIMPLEHTTP_LIB) -L$(LIBEVENT)/lib -levent -lsimplehttp -ljson -lpcre -lm
pubsub_filtered: pubsub_filtered.c md5.c
$(CC) $(CFLAGS) -o $@ md5.c $< $(LIBS)
View
7 pubsub_filtered/pubsub_filtered.c
@@ -3,11 +3,10 @@
#include <string.h>
#include <unistd.h>
#include <time.h>
-#include "simplehttp/queue.h"
-#include "simplehttp/simplehttp.h"
-#include "event.h"
+#include <simplehttp/queue.h>
+#include <simplehttp/simplehttp.h>
+#include <json/json.h>
#include "http-internal.h"
-#include "json/json.h"
#include "md5.h"
#include "pcre.h"
View
4 pubsub_to_pubsub/Makefile
@@ -5,7 +5,7 @@ LIBSIMPLEHTTP_INC ?= $(LIBSIMPLEHTTP)/..
LIBSIMPLEHTTP_LIB ?= $(LIBSIMPLEHTTP)
CFLAGS = -I. -I$(LIBSIMPLEHTTP_INC) -I$(LIBEVENT)/include -g -O2
-LIBS = -L. -L$(LIBSIMPLEHTTP_LIB) -L$(LIBEVENT)/lib -levent -ljson -ldl -lm -lc
+LIBS = -L. -L$(LIBSIMPLEHTTP_LIB) -L$(LIBEVENT)/lib -levent -ljson -ldl -lm -lc -lpubsub_to_pubsub
AR = ar
AR_FLAGS = rc
RANLIB = ranlib
@@ -16,7 +16,7 @@ libpubsub_to_pubsub.a: pubsub_to_pubsub.o
$(RANLIB) $@
ps_to_ps: ps_to_ps.c
- $(CC) $(CFLAGS) -o $@ $< $(LIBS) -lpubsub_to_pubsub
+ $(CC) $(CFLAGS) -o $@ $< $(LIBS)
all: libpubsub_to_pubsub.a ps_to_ps
View
7 pubsub_to_pubsub/ps_to_ps.c
@@ -3,11 +3,8 @@
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
-
-
-#include "simplehttp/pubsub_to_pubsub.h"
-#include "event.h"
-#include "evhttp.h"
+#include <simplehttp/pubsub_to_pubsub.h>
+#include <simplehttp/simplehttp.h>
#define DEBUG 0
View
4 pubsubclient/Makefile
@@ -5,7 +5,7 @@ LIBSIMPLEHTTP_INC ?= $(LIBSIMPLEHTTP)/..
LIBSIMPLEHTTP_LIB ?= $(LIBSIMPLEHTTP)
CFLAGS = -I. -I$(LIBSIMPLEHTTP_INC) -I$(LIBEVENT)/include -g
-LIBS = -L. -L$(LIBSIMPLEHTTP_LIB) -L$(LIBEVENT)/lib -levent -ldl -lm -lc
+LIBS = -L. -L$(LIBSIMPLEHTTP_LIB) -L$(LIBEVENT)/lib -levent -ldl -lm -lc -lsimplehttp -lpubsubclient -lpthread -lcurl
AR = ar
AR_FLAGS = rc
RANLIB = ranlib
@@ -16,7 +16,7 @@ libpubsubclient.a: pubsubclient.o
$(RANLIB) $@
pubsub-reader: pubsub-reader.o
- $(CC) $(CFLAGS) -o $@ $< $(LIBS) -lsimplehttp -lpubsubclient -lpthread -lcurl
+ $(CC) $(CFLAGS) -o $@ $< $(LIBS)
all: libpubsubclient.a pubsub-reader
View
6 pubsubclient/pubsubclient.c
@@ -1,11 +1,11 @@
-
#include <sys/types.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
-#include "event.h"
-#include "evhttp.h"
+#include <simplehttp/queue.h>
+#include <simplehttp/simplehttp.h>
#include "pubsubclient.h"
+
#define DEBUG 0
#define BOUNDARY_LENGTH 14
// \r\n--xXPubSubXx\r\n
View
5 qrencode/qrencode.c
@@ -1,11 +1,12 @@
-#include "queue.h"
#include <stdlib.h>
#include <stdio.h>
#include <sys/mman.h>
#include <sys/stat.h>
+#include <simplehttp/queue.h>
+#include <simplehttp/simplehttp.h>
+#include <json/json.h>
#include "png.h"
#include "qrencode.h"
-#include "simplehttp.h"
/*
* libqrencode
View
4 simpleattributes/Makefile
@@ -5,11 +5,11 @@ LIBSIMPLEHTTP_INC ?= $(LIBSIMPLEHTTP)/..
LIBSIMPLEHTTP_LIB ?= $(LIBSIMPLEHTTP)
CFLAGS = -I. -I$(LIBSIMPLEHTTP_INC) -I$(LIBEVENT)/include -g
-LIBS = -L. -L$(LIBSIMPLEHTTP_LIB) -L$(LIBEVENT)/lib -L/usr/local/lib -levent -ljson -ltokyotyrant -ltokyocabinet -lz -lbz2 -lresolv -ldl -lpthread -lm -lc
+LIBS = -L. -L$(LIBSIMPLEHTTP_LIB) -L$(LIBEVENT)/lib -L/usr/local/lib -lsimplehttp -levent -ljson -ltokyotyrant -ltokyocabinet -lz -lbz2 -lresolv -ldl -lpthread -lm -lc
simpleattributes: simpleattributes.c
- $(CC) $(CFLAGS) -o $@ $< $(LIBS) -lsimplehttp
+ $(CC) $(CFLAGS) -o $@ $^ $(LIBS)
install:
/usr/bin/install -d $(TARGET)/bin
View
16 simpleattributes/simpleattributes.c
@@ -1,12 +1,12 @@
#include <tcrdb.h>
-#include "queue.h"
#include <stdlib.h>
#include <stdbool.h>
#include <stdint.h>
#include <stdio.h>
#include <string.h>
-#include "simplehttp.h"
-#include "json/json.h"
+#include <simplehttp/queue.h>
+#include <simplehttp/simplehttp.h>
+#include <json/json.h>
#define RECONNECT 5
#define MAXRES 1000
@@ -37,7 +37,7 @@ void finalize_json(struct evhttp_request *req, struct evbuffer *evb,
char *json, *jsonp;
jsonp = (char *)evhttp_find_header(args, "jsonp");
- json = json_object_to_json_string(jsobj);
+ json = (char *)json_object_to_json_string(jsobj);
if (jsonp) {
evbuffer_add_printf(evb, "%s(%s)\n", jsonp, json);
} else {
@@ -228,8 +228,8 @@ void put_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
for (i=0; i < json_object_array_length(jsonPtr); i++) {
jsonPtr2 = json_object_array_get_idx(jsonPtr, i);
- key = json_object_get_string(json_object_array_get_idx(jsonPtr2, 0));
- value = json_object_get_string(json_object_array_get_idx(jsonPtr2, 1));
+ key = (char *)json_object_get_string(json_object_array_get_idx(jsonPtr2, 0));
+ value = (char *)json_object_get_string(json_object_array_get_idx(jsonPtr2, 1));
tcmapput2(cols, key, value);
}
@@ -280,7 +280,7 @@ void get_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
if (key) {
- value = tcmapget2(cols, key);
+ value = (char *)tcmapget2(cols, key);
if (!value) {
value = "";
@@ -288,7 +288,7 @@ void get_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
json_object_object_add(jsobj2, key, json_object_new_string(value));
} else {
- while ((name = tcmapiternext2(cols)) != NULL) {
+ while ((name = (char *)tcmapiternext2(cols)) != NULL) {
json_object_object_add(jsobj2, name, json_object_new_string(tcmapget2(cols, name)));
}
}
View
4 simplegeo/Makefile
@@ -5,10 +5,10 @@ LIBSIMPLEHTTP_INC ?= $(LIBSIMPLEHTTP)/..
LIBSIMPLEHTTP_LIB ?= $(LIBSIMPLEHTTP)
CFLAGS = -I. -I$(LIBSIMPLEHTTP_INC) -I$(LIBEVENT)/include -g
-LIBS = -L. -L$(LIBSIMPLEHTTP_INC) -L$(LIBEVENT)/lib -L/usr/local/lib -levent -ljson -ltokyotyrant -ltokyocabinet -lz -lbz2 -lresolv -ldl -lpthread -lm -lc
+LIBS = -L. -L$(LIBSIMPLEHTTP_INC) -L$(LIBEVENT)/lib -L/usr/local/lib -lsimplehttp -levent -ljson -ltokyotyrant -ltokyocabinet -lz -lbz2 -lresolv -ldl -lpthread -lm -lc
simplegeo: simplegeo.c
- $(CC) $(CFLAGS) -o $@ $< $(LIBS) -lsimplehttp
+ $(CC) $(CFLAGS) -o $@ $^ $(LIBS)
install:
/usr/bin/install -d $(TARGET)/bin
View
6 simplegeo/simplegeo.c
@@ -1,13 +1,13 @@
#include <tcrdb.h>
-#include "queue.h"
#include <stdlib.h>
#include <stdbool.h>
#include <stdint.h>
#include <stdio.h>
#include <string.h>
#include <math.h>
-#include "simplehttp.h"
-#include "json/json.h"
+#include <simplehttp/queue.h>
+#include <simplehttp/simplehttp.h>
+#include <json/json.h>
#define RECONNECT 5
#define MAXRES 1000
View
6 simplehttp/Makefile
@@ -2,15 +2,15 @@ LIBEVENT ?= /usr/local
TARGET ?= /usr/local
CFLAGS = -I. -I$(LIBEVENT)/include -Wall -g -O2
-LIBS = -L. -L$(LIBEVENT)/lib -levent
+LIBS = -L. -L$(LIBEVENT)/lib -levent -lm
AR = ar
AR_FLAGS = rc
RANLIB = ranlib
-libsimplehttp.a: simplehttp.o
+libsimplehttp.a: simplehttp.o timer.o log.o util.o stat.o
/bin/rm -f $@
- $(AR) $(AR_FLAGS) $@ $<
+ $(AR) $(AR_FLAGS) $@ $^
$(RANLIB) $@
testserver: testserver.c
View
0 lib/log.c → simplehttp/log.c
File renamed without changes.
View
175 simplehttp/simplehttp.c
@@ -1,10 +1,10 @@
#include <sys/types.h>
#include <sys/time.h>
#include <sys/stat.h>
-
#include <signal.h>
#include <stdlib.h>
#include <unistd.h>
+#include <inttypes.h>
#include <pwd.h>
#include <grp.h>
#include <err.h>
@@ -12,9 +12,9 @@
#include <string.h>
#include <stdio.h>
#include <fnmatch.h>
-
#include "queue.h"
#include "simplehttp.h"
+#include "stat.h"
typedef struct cb_entry {
char *path;
@@ -24,25 +24,26 @@ typedef struct cb_entry {
} cb_entry;
TAILQ_HEAD(, cb_entry) callbacks;
-int debug = 0;
+static int debug = 0;
+static int verbose = 0;
-static void
-ignore_cb(int sig, short what, void *arg)
+int callback_count = 0;
+uint64_t request_count = 0;
+
+static void ignore_cb(int sig, short what, void *arg)
{
}
-void
-termination_handler (int signum)
+void termination_handler(int signum)
{
event_loopbreak();
}
-int
-get_uid(char *user)
+int get_uid(char *user)
{
int retcode;
struct passwd *pw;
-
+
pw = getpwnam(user);
if (pw == NULL) {
retcode = -1;
@@ -52,12 +53,11 @@ get_uid(char *user)
return retcode;
}
-int
-get_gid(char *group)
+int get_gid(char *group)
{
int retcode;
struct group *grent;
-
+
grent = getgrnam(group);
if (grent == NULL) {
retcode = -1;
@@ -67,12 +67,11 @@ get_gid(char *group)
return retcode;
}
-int
-get_user_gid(char *user)
+int get_user_gid(char *user)
{
int retcode;
struct passwd *pw;
-
+
pw = getpwnam(user);
if (pw == NULL) {
retcode = -1;
@@ -82,40 +81,82 @@ get_user_gid(char *user)
return retcode;
}
-void
-generic_request_handler(struct evhttp_request *req, void *arg)
+char **simplehttp_callback_names()
+{
+ char **callback_names;
+ char *tmp;
+ struct cb_entry *entry;
+ int i = 0, len;
+
+ callback_names = malloc(callback_count * sizeof(*callback_names));
+ TAILQ_FOREACH(entry, &callbacks, entries) {
+ len = strlen(entry->path);
+ callback_names[i] = malloc(len + 1);
+ tmp = entry->path;
+ if (tmp[0] == '/') {
+ tmp++;
+ len--;
+ }
+ if (tmp[len - 1] == '*') {
+ len--;
+ }
+ memcpy(callback_names[i], tmp, len);
+ callback_names[i][len] = '\0';
+ i++;
+ }
+
+ return callback_names;
+}
+
+void generic_request_handler(struct evhttp_request *req, void *arg)
{
- int found_cb = 0;
+ int found_cb = 0, i = 0;
struct cb_entry *entry;
struct evbuffer *evb = evbuffer_new();
+ simplehttp_ts start_ts, end_ts;
+ uint64_t req_time;
+ char id_buf[64];
if (debug) {
fprintf(stderr, "request for %s from %s\n", req->uri, req->remote_host);
}
-
+
TAILQ_FOREACH(entry, &callbacks, entries) {
if (fnmatch(entry->path, req->uri, FNM_NOESCAPE) == 0) {
+ request_count++;
+
+ simplehttp_ts_get(&start_ts);
+
(*entry->cb)(req, evb, entry->ctx);
+
+ simplehttp_ts_get(&end_ts);
+ req_time = simplehttp_ts_diff(start_ts, end_ts);
+ simplehttp_stats_store(i, req_time);
+ if (verbose) {
+ sprintf(id_buf, "%"PRIu64, request_count);
+ simplehttp_log('I', "", req, req_time, id_buf);
+ }
+
found_cb = 1;
break;
}
+ i++;
}
-
+
if (!found_cb) {
evhttp_send_reply(req, HTTP_NOTFOUND, "", evb);
}
+
evbuffer_free(evb);
}
-void
-simplehttp_init()
+void simplehttp_init()
{
event_init();
TAILQ_INIT(&callbacks);
}
-void
-simplehttp_free()
+void simplehttp_free()
{
struct cb_entry *entry;
@@ -126,22 +167,22 @@ simplehttp_free()
}
}
-void
-simplehttp_set_cb(char *path, void (*cb)(struct evhttp_request *, struct evbuffer *, void *), void *ctx)
+void simplehttp_set_cb(char *path, void (*cb)(struct evhttp_request *, struct evbuffer *, void *), void *ctx)
{
struct cb_entry *cbPtr;
-
+
cbPtr = (cb_entry *)malloc(sizeof(*cbPtr));
cbPtr->path = strdup(path);
cbPtr->cb = cb;
cbPtr->ctx = ctx;
TAILQ_INSERT_TAIL(&callbacks, cbPtr, entries);
-
+
+ callback_count++;
+
printf("registering callback for path \"%s\"\n", path);
}
-int
-simplehttp_main(int argc, char **argv)
+int simplehttp_main(int argc, char **argv)
{
uid_t uid = 0;
gid_t gid = 0;
@@ -158,29 +199,32 @@ simplehttp_main(int argc, char **argv)
address = "0.0.0.0";
port = 8080;
opterr = 0;
- while ((ch = getopt(argc, argv, "a:p:d:D:r:u:g:")) != -1) {
+ while ((ch = getopt(argc, argv, "a:p:d:D:r:u:g:V")) != -1) {
switch (ch) {
- case 'a':
- address = optarg;
- break;
- case 'p':
- port = atoi(optarg);
- break;
- case 'd':
- debug = 1;
- break;
- case 'r':
- root = optarg;
- break;
- case 'D':
- daemon = 1;
- break;
- case 'g':
- garg = optarg;
- break;
- case 'u':
- uarg = optarg;
- break;
+ case 'a':
+ address = optarg;
+ break;
+ case 'p':
+ port = atoi(optarg);
+ break;
+ case 'd':
+ debug = 1;
+ break;
+ case 'r':
+ root = optarg;
+ break;
+ case 'D':
+ daemon = 1;
+ break;
+ case 'g':
+ garg = optarg;
+ break;
+ case 'u':
+ uarg = optarg;
+ break;
+ case 'V':
+ verbose = 1;
+ break;
}
}
@@ -191,14 +235,14 @@ simplehttp_main(int argc, char **argv)
} else if (pid > 0) {
exit(EXIT_SUCCESS);
}
-
+
umask(0);
sid = setsid();
if (sid < 0) {
exit(EXIT_FAILURE);
}
}
-
+
if (uarg != NULL) {
uid = get_uid(uarg);
gid = get_user_gid(uarg);
@@ -219,7 +263,7 @@ simplehttp_main(int argc, char **argv)
}
}
}
-
+
if (root != NULL) {
if (chroot(root) != 0) {
err(1, strerror(errno));
@@ -250,25 +294,30 @@ simplehttp_main(int argc, char **argv)
signal(SIGINT, termination_handler);
signal(SIGQUIT, termination_handler);
signal(SIGTERM, termination_handler);
-
+
signal_set(&pipe_ev, SIGPIPE, ignore_cb, NULL);
signal_add(&pipe_ev, NULL);
-
+
+ simplehttp_stats_init();
+
httpd = evhttp_start(address, port);
if (!httpd) {
printf("could not bind to %s:%d\n", address, port);
return 1;
}
-
+
printf("listening on %s:%d\n", address, port);
-
+
evhttp_set_gencb(httpd, generic_request_handler, NULL);
event_dispatch();
-
+
printf("exiting\n");
- /* Not reached in this code as it is now. */
+
evhttp_free(httpd);
+
+ simplehttp_stats_destruct();
+
simplehttp_free();
-
+
return 0;
}
View
35 simplehttp/simplehttp.h
@@ -1,7 +1,42 @@
+#ifndef _SIMPLEHTTP_H
+#define _SIMPLEHTTP_H
+
#include <event.h>
#include <evhttp.h>
+struct simplehttp_stats {
+ uint64_t requests;
+ uint64_t *stats_counts;
+ uint64_t *average_requests;
+ uint64_t *ninety_five_percents;
+ char **stats_labels;
+ int callback_count;
+};
+
void simplehttp_init();
int simplehttp_main(int argc, char **argv);
void simplehttp_set_cb(char *path, void (*cb)(struct evhttp_request *, struct evbuffer *,void *), void *ctx);
+void simplehttp_log(const char type, const char *host, struct evhttp_request *req, uint64_t req_time, const char *id);
+struct simplehttp_stats *simplehttp_stats_new();
+void simplehttp_stats(struct simplehttp_stats *st);
+void simplehttp_stats_free(struct simplehttp_stats *st);
+uint64_t ninety_five_percent(int64_t *int_array, int length);
+char **simplehttp_callback_names();
+
+#if _POSIX_TIMERS > 0
+
+typedef struct timespec simplehttp_ts;
+
+void simplehttp_ts_get(struct timespec *ts);
+unsigned int simplehttp_ts_diff(struct timespec start, struct timespec end);
+
+#else
+
+typedef struct timeval simplehttp_ts;
+
+void simplehttp_ts_get(struct timeval *ts);
+unsigned int simplehttp_ts_diff(struct timeval start, struct timeval end);
+
+#endif
+#endif
View
105 simplehttp/stat.c
@@ -0,0 +1,105 @@
+#include <stdlib.h>
+#include <string.h>
+#include <inttypes.h>
+#include "stat.h"
+#include "simplehttp.h"
+#include "util.h"
+
+static int64_t *stats = NULL;
+static int *stats_idx = NULL;
+static uint64_t *stats_counts = NULL;
+
+extern int callback_count;
+extern uint64_t request_count;
+
+void simplehttp_stats_store(int index, unsigned int val)
+{
+ stats[(index * STAT_WINDOW) + stats_idx[index]] = val;
+ stats_idx[index]++;
+ stats_counts[index]++;
+
+ if (stats_idx[index] >= STAT_WINDOW) {
+ stats_idx[index] = 0;
+ }
+}
+
+void simplehttp_stats_init()
+{
+ int i;
+ stats = malloc(STAT_WINDOW * callback_count * sizeof(int64_t));
+ for (i = 0; i < (STAT_WINDOW * callback_count); i++) {
+ stats[i] = -1;
+ }
+ stats_idx = calloc(callback_count, sizeof(int));
+ stats_counts = calloc(callback_count, sizeof(uint64_t));
+}
+
+void simplehttp_stats_destruct()
+{
+ free(stats);
+ free(stats_idx);
+ free(stats_counts);
+}
+
+struct simplehttp_stats *simplehttp_stats_new()
+{
+ struct simplehttp_stats *st;
+
+ st = malloc(sizeof(struct simplehttp_stats));
+
+ return st;
+}
+
+void simplehttp_stats_free(struct simplehttp_stats *st)
+{
+ int i;
+
+ if (st) {
+ if (st->stats_counts) {
+ free(st->stats_counts);
+ }
+
+ if (st->average_requests) {
+ free(st->average_requests);
+ }
+
+ if (st->ninety_five_percents) {
+ free(st->ninety_five_percents);
+ }
+
+ if (st->stats_labels) {
+ for (i = 0; i < st->callback_count; i++) {
+ free(st->stats_labels[i]);
+ }
+ free(st->stats_labels);
+ }
+
+ free(st);
+ }
+}
+
+void simplehttp_stats(struct simplehttp_stats *st)
+{
+ uint64_t request_total;
+ int i, j, c, request_array_end;
+
+ st->requests = request_count;
+ st->callback_count = callback_count;
+ st->stats_counts = malloc(callback_count * sizeof(uint64_t));
+ memcpy(st->stats_counts, stats_counts, callback_count * sizeof(uint64_t));
+ st->average_requests = calloc(callback_count, sizeof(uint64_t));
+ st->ninety_five_percents = calloc(callback_count, sizeof(uint64_t));
+ st->stats_labels = simplehttp_callback_names();
+
+ for (i = 0; i < callback_count; i++) {
+ request_total = 0;
+ for (j = (i * STAT_WINDOW), request_array_end = j + STAT_WINDOW, c = 0;
+ (j < request_array_end) && (stats[j] != -1); j++, c++) {
+ request_total += stats[j];
+ }
+ if (c) {
+ st->average_requests[i] = request_total / c;
+ st->ninety_five_percents[i] = ninety_five_percent(stats + (i * STAT_WINDOW), c);
+ }
+ }
+}
View
10 simplehttp/stat.h
@@ -0,0 +1,10 @@
+#ifndef _STAT_H
+#define _STAT_H
+
+#define STAT_WINDOW 1000
+
+void simplehttp_stats_store(int index, unsigned int val);
+void simplehttp_stats_init();
+void simplehttp_stats_destruct();
+
+#endif
View
8 lib/timer.c → simplehttp/timer.c
@@ -3,12 +3,12 @@
#if _POSIX_TIMERS > 0
-void _gettime(struct timespec *ts)
+void simplehttp_ts_get(struct timespec *ts)
{
clock_gettime(CLOCK_REALTIME, ts);
}
-unsigned int _ts_diff(struct timespec start, struct timespec end)
+unsigned int simplehttp_ts_diff(struct timespec start, struct timespec end)
{
struct timespec temp;
@@ -26,12 +26,12 @@ unsigned int _ts_diff(struct timespec start, struct timespec end)
#else
-void _gettime(struct timeval *ts)
+void simplehttp_ts_get(struct timeval *ts)
{
gettimeofday(ts, NULL);
}
-unsigned int _ts_diff(struct timeval start, struct timeval end)
+unsigned int simplehttp_ts_diff(struct timeval start, struct timeval end)
{
struct timeval temp;
View
5 lib/util.c → simplehttp/util.c
@@ -18,9 +18,12 @@ uint64_t ninety_five_percent(int64_t *int_array, int length)
int index_of_95;
sorted_requests = calloc(length, sizeof(int64_t));
- memcpy(sorted_requests, int_array, length);
+ memcpy(sorted_requests, int_array, length * sizeof(int64_t));
qsort(sorted_requests, length, sizeof(int64_t), int_cmp);
index_of_95 = (int)ceil(((95.0 / 100.0) * length) + 0.5);
+ if (index_of_95 >= length) {
+ index_of_95 = length - 1;
+ }
value = sorted_requests[index_of_95];
free(sorted_requests);
View
5 lib/util.h → simplehttp/util.h
@@ -1 +1,6 @@
+#ifndef _UTIL_H
+#define _UTIL_H
+
uint64_t ninety_five_percent(int64_t *int_array, int length);
+
+#endif
View
4 simplememdb/Makefile
@@ -7,7 +7,7 @@ LIBSIMPLEHTTP_LIB ?= $(LIBSIMPLEHTTP)
CFLAGS = -I$(LIBSIMPLEHTTP_INC) -I$(LIBEVENT)/include -Wall -g -O2
LIBS = -L$(LIBSIMPLEHTTP_LIB) -L$(LIBEVENT)/lib -levent -lsimplehttp -ltokyocabinet -ljson -lpcre
-simplememdb: simplememdb.c ../lib/timer.c ../lib/util.c ../lib/log.c
+simplememdb: simplememdb.c
$(CC) $(CFLAGS) -o $@ $^ $(LIBS)
install:
@@ -15,4 +15,4 @@ install:
/usr/bin/install simplememdb $(TARGET)/bin
clean:
- rm -rf *.a *.o simplememdb *.dSYM
+ rm -rf *.a *.o simplememdb *.dSYM
View
211 simplememdb/simplememdb.c
@@ -14,29 +14,13 @@
#include <simplehttp/queue.h>
#include <simplehttp/simplehttp.h>
#include <json/json.h>
-#include "lib/timer.h"
-#include "lib/util.h"
-#include "lib/log.h"
#include "pcre.h"
#define NAME "simplememdb"
-#define VERSION "1.2"
+#define VERSION "1.3"
#define BUFFER_SZ 1048576
#define SM_BUFFER_SZ 4096
-#define NUM_REQUEST_TYPES 10
-#define NUM_REQUESTS_FOR_STATS 1000
-#define STATS_GET 0
-#define STATS_GET_INT 1
-#define STATS_PUT 2
-#define STATS_INCR 3
-#define STATS_DEL 4
-#define STATS_FWMATCH 5
-#define STATS_FWMATCH_INT 6
-#define STATS_VANISH 7
-#define STATS_MGET 8
-#define STATS_MGET_INT 9
-
void set_dump_timer();
void fwmatch_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx);
void fwmatch_int_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx);
@@ -51,27 +35,11 @@ void exit_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx);
void usage();
void info();
-static uint64_t requests = 0;
-static uint64_t stats_request_counts[NUM_REQUEST_TYPES];
-static int64_t stats_request[NUM_REQUESTS_FOR_STATS * NUM_REQUEST_TYPES];
-static int stats_request_idx[NUM_REQUEST_TYPES];
-static char *stats_request_labels[] = { "get", "get_int", "put", "incr", "del", "fwmatch", "fwmatch_int", "vanish", "mget", "mget_int" };
-
static TCADB *adb;
static int is_currently_dumping = 0;
static struct event ev;
static pcre *dump_regex;
-void stats_store_request(int index, unsigned int diff)
-{
- stats_request[(index * NUM_REQUESTS_FOR_STATS) + stats_request_idx[index]] = diff;
- stats_request_idx[index]++;
-
- if (stats_request_idx[index] >= NUM_REQUESTS_FOR_STATS) {
- stats_request_idx[index] = 0;
- }
-}
-
void finalize_json(struct evhttp_request *req, struct evbuffer *evb, struct evkeyvalq *args, struct json_object *jsobj)
{
const char *json, *jsonp;
@@ -115,12 +83,6 @@ void fwmatch_int_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
int txt_format = 0;
int status_code = 200;
char *status_txt = "OK";
- uint64_t req_time;
-
- _gettime(&ts1);
-
- requests++;
- stats_request_counts[STATS_FWMATCH_INT]++;
evhttp_parse_query(req->uri, &args);
key = (char *)evhttp_find_header(&args, "key");
@@ -184,14 +146,6 @@ void fwmatch_int_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
evhttp_send_reply(req, status_code, status_txt, evb);
evhttp_clear_headers(&args);
-
- _gettime(&ts2);
- req_time = _ts_diff(ts1, ts2);
- stats_store_request(STATS_FWMATCH_INT, req_time);
-
- char id_buf[256];
- sprintf(id_buf, "%"PRIu64, requests);
- simplehttp_log('I', "", req, req_time, id_buf);
}
void fwmatch_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
@@ -204,12 +158,6 @@ void fwmatch_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
int txt_format = 0;
int status_code = 200;
char *status_txt = "OK";
- uint64_t req_time;
-
- _gettime(&ts1);
-
- requests++;
- stats_request_counts[STATS_FWMATCH]++;
evhttp_parse_query(req->uri, &args);
@@ -276,30 +224,16 @@ void fwmatch_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
evhttp_send_reply(req, status_code, status_txt, evb);
evhttp_clear_headers(&args);
-
- _gettime(&ts2);
- req_time = _ts_diff(ts1, ts2);
- stats_store_request(STATS_FWMATCH, req_time);
-
- char id_buf[256];
- sprintf(id_buf, "%"PRIu64, requests);
- simplehttp_log('I', "", req, req_time, id_buf);
}
void del_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
{
char *key, *format;
struct evkeyvalq args;
- struct json_object *jsobj;
+ struct json_object *jsobj = NULL;
int txt_format = 0;
int status_code = 200;
char *status_txt = "OK";
- uint64_t req_time;
-
- _gettime(&ts1);
-
- requests++;
- stats_request_counts[STATS_DEL]++;
evhttp_parse_query(req->uri, &args);
@@ -342,14 +276,6 @@ void del_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
evhttp_send_reply(req, status_code, status_txt, evb);
evhttp_clear_headers(&args);
-
- _gettime(&ts2);
- req_time = _ts_diff(ts1, ts2);
- stats_store_request(STATS_DEL, req_time);
-
- char id_buf[256];
- sprintf(id_buf, "%"PRIu64, requests);
- simplehttp_log('I', "", req, req_time, id_buf);
}
void put_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
@@ -360,12 +286,6 @@ void put_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
int txt_format = 0;
int status_code = 200;
char *status_txt = "OK";
- uint64_t req_time;
-
- _gettime(&ts1);
-
- requests++;
- stats_request_counts[STATS_PUT]++;
evhttp_parse_query(req->uri, &args);
@@ -416,14 +336,6 @@ void put_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
evhttp_send_reply(req, status_code, status_txt, evb);
evhttp_clear_headers(&args);
-
- _gettime(&ts2);
- req_time = _ts_diff(ts1, ts2);
- stats_store_request(STATS_PUT, req_time);
-
- char id_buf[256];
- sprintf(id_buf, "%"PRIu64, requests);
- simplehttp_log('I', "", req, req_time, id_buf);
}
void get_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
@@ -435,12 +347,6 @@ void get_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
int txt_format = 0;
int status_code = 200;
char *status_txt = "OK";
- uint64_t req_time;
-
- _gettime(&ts1);
-
- requests++;
- stats_request_counts[STATS_GET]++;
evhttp_parse_query(req->uri, &args);
@@ -485,14 +391,6 @@ void get_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
evhttp_send_reply(req, status_code, status_txt, evb);
evhttp_clear_headers(&args);
-
- _gettime(&ts2);
- req_time = _ts_diff(ts1, ts2);
- stats_store_request(STATS_GET, req_time);
-
- char id_buf[256];
- sprintf(id_buf, "%"PRIu64, requests);
- simplehttp_log('I', "", req, req_time, id_buf);
}
void get_int_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
@@ -505,12 +403,6 @@ void get_int_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
int txt_format = 0;
int status_code = 200;
char *status_txt = "OK";
- uint64_t req_time;
-
- _gettime(&ts1);
-
- requests++;
- stats_request_counts[STATS_GET_INT]++;
evhttp_parse_query(req->uri, &args);
@@ -555,14 +447,6 @@ void get_int_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
evhttp_send_reply(req, status_code, status_txt, evb);
evhttp_clear_headers(&args);
-
- _gettime(&ts2);
- req_time = _ts_diff(ts1, ts2);
- stats_store_request(STATS_GET_INT, req_time);
-
- char id_buf[256];
- sprintf(id_buf, "%"PRIu64, requests);
- simplehttp_log('I', "", req, req_time, id_buf);
}
void mget_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
@@ -576,12 +460,6 @@ void mget_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
int txt_format = 0;
int status_code = 200;
char *status_txt = "OK";
- uint64_t req_time;
-
- _gettime(&ts1);
-
- requests++;
- stats_request_counts[STATS_MGET]++;
evhttp_parse_query(req->uri, &args);
@@ -623,14 +501,6 @@ void mget_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
evhttp_send_reply(req, status_code, status_txt, evb);
evhttp_clear_headers(&args);
-
- _gettime(&ts2);
- req_time = _ts_diff(ts1, ts2);
- stats_store_request(STATS_MGET, req_time);
-
- char id_buf[256];
- sprintf(id_buf, "%"PRIu64, requests);
- simplehttp_log('I', "", req, req_time, id_buf);
}
void mget_int_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
@@ -645,12 +515,6 @@ void mget_int_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
int txt_format = 0;
int status_code = 200;
char *status_txt = "OK";
- uint64_t req_time;
-
- _gettime(&ts1);
-
- requests++;
- stats_request_counts[STATS_MGET_INT]++;
evhttp_parse_query(req->uri, &args);
@@ -692,14 +556,6 @@ void mget_int_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
evhttp_send_reply(req, status_code, status_txt, evb);
evhttp_clear_headers(&args);
-
- _gettime(&ts2);
- req_time = _ts_diff(ts1, ts2);
- stats_store_request(STATS_MGET_INT, req_time);
-
- char id_buf[256];
- sprintf(id_buf, "%"PRIu64, requests);
- simplehttp_log('I', "", req, req_time, id_buf);
}
void incr_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
@@ -715,12 +571,6 @@ void incr_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
int txt_format = 0;
int status_code = 200;
char *status_txt = "OK";
- uint64_t req_time;
-
- _gettime(&ts1);
-
- requests++;
- stats_request_counts[STATS_INCR]++;
evhttp_parse_query(req->uri, &args);
@@ -778,14 +628,6 @@ void incr_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
evhttp_send_reply(req, status_code, status_txt, evb);
evhttp_clear_headers(&args);
-
- _gettime(&ts2);
- req_time = _ts_diff(ts1, ts2);
- stats_store_request(STATS_INCR, req_time);
-
- char id_buf[256];
- sprintf(id_buf, "%"PRIu64, requests);
- simplehttp_log('I', "", req, req_time, id_buf);
}
void vanish_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
@@ -793,8 +635,6 @@ void vanish_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
struct json_object *jsobj;
const char *json;
- requests++;
-
tcadbvanish(adb);
jsobj = json_object_new_object();
@@ -810,49 +650,38 @@ void vanish_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
void stats_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
{
- uint64_t request_total;
- uint64_t average_requests[NUM_REQUEST_TYPES];
- uint64_t ninety_five_percents[NUM_REQUEST_TYPES];
- int i, j, c, request_array_end;
+ int i;
struct evkeyvalq args;
const char *format;
- memset(&average_requests, 0, sizeof(average_requests));
- memset(&ninety_five_percents, 0, sizeof(ninety_five_percents));
+ struct simplehttp_stats *st;
- for (i = 0; i < NUM_REQUEST_TYPES; i++) {
- request_total = 0;
- for (j = (i * NUM_REQUESTS_FOR_STATS), request_array_end = j + NUM_REQUESTS_FOR_STATS, c = 0;
- (j < request_array_end) && (stats_request[j] != -1); j++, c++) {
- request_total += stats_request[j];
- }
- if (c) {
- average_requests[i] = request_total / c;
- ninety_five_percents[i] = ninety_five_percent(stats_request + (i * NUM_REQUESTS_FOR_STATS), c);
- }
- }
+ st = simplehttp_stats_new();
+ simplehttp_stats(st);
evhttp_parse_query(req->uri, &args);
format = (char *)evhttp_find_header(&args, "format");
if ((format != NULL) && (strcmp(format, "json") == 0)) {
evbuffer_add_printf(evb, "{");
- for (i = 0; i < NUM_REQUEST_TYPES; i++) {
- evbuffer_add_printf(evb, "\"%s_95\": %"PRIu64",", stats_request_labels[i], ninety_five_percents[i]);
- evbuffer_add_printf(evb, "\"%s_average_request\": %"PRIu64",", stats_request_labels[i], average_requests[i]);
- evbuffer_add_printf(evb, "\"%s_requests\": %"PRIu64",", stats_request_labels[i], stats_request_counts[i]);
+ for (i = 0; i < st->callback_count; i++) {
+ evbuffer_add_printf(evb, "\"%s_95\": %"PRIu64",", st->stats_labels[i], st->ninety_five_percents[i]);
+ evbuffer_add_printf(evb, "\"%s_average_request\": %"PRIu64",", st->stats_labels[i], st->average_requests[i]);
+ evbuffer_add_printf(evb, "\"%s_requests\": %"PRIu64",", st->stats_labels[i], st->stats_counts[i]);
}
- evbuffer_add_printf(evb, "\"total_requests\": %"PRIu64, requests);
+ evbuffer_add_printf(evb, "\"total_requests\": %"PRIu64, st->requests);
evbuffer_add_printf(evb, "}\n");
} else {
- evbuffer_add_printf(evb, "total requests: %"PRIu64"\n", requests);
- for (i = 0; i < NUM_REQUEST_TYPES; i++) {
- evbuffer_add_printf(evb, "/%s 95%%: %"PRIu64"\n", stats_request_labels[i], ninety_five_percents[i]);
- evbuffer_add_printf(evb, "/%s average request (usec): %"PRIu64"\n", stats_request_labels[i], average_requests[i]);
- evbuffer_add_printf(evb, "/%s requests: %"PRIu64"\n", stats_request_labels[i], stats_request_counts[i]);
+ evbuffer_add_printf(evb, "total requests: %"PRIu64"\n", st->requests);
+ for (i = 0; i < st->callback_count; i++) {
+ evbuffer_add_printf(evb, "/%s 95%%: %"PRIu64"\n", st->stats_labels[i], st->ninety_five_percents[i]);
+ evbuffer_add_printf(evb, "/%s average request (usec): %"PRIu64"\n", st->stats_labels[i], st->average_requests[i]);
+ evbuffer_add_printf(evb, "/%s requests: %"PRIu64"\n", st->stats_labels[i], st->stats_counts[i]);
}
}
+ simplehttp_stats_free(st);
+
evhttp_send_reply(req, HTTP_OK, "OK", evb);
evhttp_clear_headers(&args);
}
@@ -1023,10 +852,6 @@ int main(int argc, char **argv)
}
}
- memset(&stats_request, -1, sizeof(stats_request));
- memset(&stats_request_idx, 0, sizeof(stats_request_idx));
- memset(&stats_request_counts, 0, sizeof(stats_request_counts));
-
sprintf(buf, "+#bnum=%lu", bnum);
adb = tcadbnew();
if (!tcadbopen(adb, buf)) {
View
2 simplequeue/Makefile
@@ -5,7 +5,7 @@ LIBSIMPLEHTTP_INC ?= $(LIBSIMPLEHTTP)/..
LIBSIMPLEHTTP_LIB ?= $(LIBSIMPLEHTTP)
CFLAGS = -I$(LIBSIMPLEHTTP_INC) -I$(LIBEVENT)/include -Wall -g -O2
-LIBS = -L$(LIBSIMPLEHTTP_LIB) -L$(LIBEVENT)/lib -levent -lsimplehttp
+LIBS = -L$(LIBSIMPLEHTTP_LIB) -L$(LIBEVENT)/lib -levent -lsimplehttp -lm
simplequeue: simplequeue.c
$(CC) $(CFLAGS) -o $@ $< $(LIBS)
View
2 simpletokyo/Makefile
@@ -7,7 +7,7 @@ LIBSIMPLEHTTP_LIB ?= $(LIBSIMPLEHTTP)
CFLAGS = -I. -I$(LIBSIMPLEHTTP_INC) -I$(LIBEVENT)/include -Wall -g -O2
LIBS = -L. -L$(LIBSIMPLEHTTP_LIB) -L$(LIBEVENT)/lib -L/usr/local/lib -levent -ljson -ltokyotyrant -ltokyocabinet -lsimplehttp
-simpletokyo: simpletokyo.c ../lib/timer.c ../lib/util.c
+simpletokyo: simpletokyo.c
$(CC) $(CFLAGS) -o $@ $^ $(LIBS)
install:
View
249 simpletokyo/simpletokyo.c
@@ -9,24 +9,11 @@
#include <simplehttp/queue.h>
#include <simplehttp/simplehttp.h>
#include <json/json.h>
-#include "lib/timer.h"
-#include "lib/util.h"
+#define NAME "simpletokyo"
+#define VERSION "1.6"
#define RECONNECT 5
-#define NUM_REQUEST_TYPES 10
-#define NUM_REQUESTS_FOR_STATS 1000
-#define STATS_GET 0
-#define STATS_GET_INT 1
-#define STATS_PUT 2
-#define STATS_INCR 3
-#define STATS_DEL 4
-#define STATS_FWMATCH 5
-#define STATS_FWMATCH_INT 6
-#define STATS_VANISH 7
-#define STATS_MGET 8
-#define STATS_MGET_INT 9
-
void finalize_json(struct evhttp_request *req, struct evbuffer *evb, struct evkeyvalq *args, struct json_object *jsobj);
int open_db(char *addr, int port, TCRDB **rdb);
void close_db(TCRDB **rdb);
@@ -46,9 +33,6 @@ void vanish_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx);
void stats_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx);
void exit_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx);
-static char *g_progname = "simpletokyo";
-static char *version = "1.5";
-
struct event ev;
struct timeval tv = {RECONNECT,0};
static char *db_host = "127.0.0.1";
@@ -57,22 +41,6 @@ static TCRDB *rdb;
static int db_status;
static uint64_t db_opened = 0;
-static uint64_t requests = 0;
-static uint64_t stats_request_counts[NUM_REQUEST_TYPES];
-static int64_t stats_request[NUM_REQUESTS_FOR_STATS * NUM_REQUEST_TYPES];
-static int stats_request_idx[NUM_REQUEST_TYPES];
-static char *stats_request_labels[] = { "get", "get_int", "put", "incr", "del", "fwmatch", "fwmatch_int", "vanish", "mget", "mget_int" };
-
-void stats_store_request(int index, unsigned int diff)
-{
- stats_request[(index * NUM_REQUESTS_FOR_STATS) + stats_request_idx[index]] = diff;
- stats_request_idx[index]++;
-
- if (stats_request_idx[index] >= NUM_REQUESTS_FOR_STATS) {
- stats_request_idx[index] = 0;
- }
-}
-
void finalize_json(struct evhttp_request *req, struct evbuffer *evb, struct evkeyvalq *args, struct json_object *jsobj)
{
const char *json, *jsonp;
@@ -90,7 +58,8 @@ void finalize_json(struct evhttp_request *req, struct evbuffer *evb, struct evke
evhttp_clear_headers(args);
}
-void close_db(TCRDB **rdb) {
+void close_db(TCRDB **rdb)
+{
int ecode=0;
if (*rdb != NULL) {
if(!tcrdbclose(*rdb)){
@@ -167,19 +136,19 @@ void fwmatch_int_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
struct evkeyvalq args;
struct json_object *jsobj, *jsobj2, *jsarr;
- requests++;
- stats_request_counts[STATS_FWMATCH_INT]++;
-
if (rdb == NULL) {
evhttp_send_error(req, 503, "database not connected");
return;
}
+
evhttp_parse_query(req->uri, &args);
-
+
key = (char *)evhttp_find_header(&args, "key");
+
argtoi(&args, "max", &max, 1000);
argtoi(&args, "length", &len, 10);
argtoi(&args, "offset", &off, 0);
+
if (key == NULL) {
evhttp_send_error(req, 400, "key is required");
evhttp_clear_headers(&args);
@@ -190,15 +159,15 @@ void fwmatch_int_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
jsarr = json_object_new_array();
keylist = tcrdbfwmkeys2(rdb, key, max);
- for (i=off; keylist!=NULL && i<(len+off) && i<tclistnum(keylist); i++){
- kbuf = (char *)tclistval2(keylist, i);
- value = (int *)tcrdbget2(rdb, kbuf);
- if (value) {
- jsobj2 = json_object_new_object();
- json_object_object_add(jsobj2, kbuf, json_object_new_int((int) *value));
- json_object_array_add(jsarr, jsobj2);
- tcfree(value);
- }
+ for (i=off; keylist!=NULL && i<(len+off) && i<tclistnum(keylist); i++) {
+ kbuf = (char *)tclistval2(keylist, i);
+ value = (int *)tcrdbget2(rdb, kbuf);
+ if (value) {
+ jsobj2 = json_object_new_object();
+ json_object_object_add(jsobj2, kbuf, json_object_new_int((int) *value));
+ json_object_array_add(jsarr, jsobj2);
+ tcfree(value);
+ }
}
if(keylist) tcfree(keylist);
json_object_object_add(jsobj, "results", jsarr);
@@ -209,7 +178,7 @@ void fwmatch_int_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
db_status = tcrdbecode(rdb);
db_error_to_json(db_status, jsobj);
}
-
+
finalize_json(req, evb, &args, jsobj);
}
@@ -221,19 +190,19 @@ void fwmatch_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
struct evkeyvalq args;
struct json_object *jsobj, *jsobj2, *jsarr;
- requests++;
- stats_request_counts[STATS_FWMATCH]++;
-
if (rdb == NULL) {
evhttp_send_error(req, 503, "database not connected");
return;
}
+
evhttp_parse_query(req->uri, &args);
-
+
key = (char *)evhttp_find_header(&args, "key");
+
argtoi(&args, "max", &max, 1000);
argtoi(&args, "length", &len, 10);
argtoi(&args, "offset", &off, 0);
+
if (key == NULL) {
evhttp_send_error(req, 400, "key is required");
evhttp_clear_headers(&args);
@@ -245,14 +214,14 @@ void fwmatch_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
keylist = tcrdbfwmkeys2(rdb, key, max);
for (i=off; keylist!=NULL && i<(len+off) && i<tclistnum(keylist); i++){
- kbuf = (char *)tclistval2(keylist, i);
- value = tcrdbget2(rdb, kbuf);
- if (value) {
- jsobj2 = json_object_new_object();
- json_object_object_add(jsobj2, kbuf, json_object_new_string(value));
- json_object_array_add(jsarr, jsobj2);
- tcfree(value);
- }
+ kbuf = (char *)tclistval2(keylist, i);
+ value = tcrdbget2(rdb, kbuf);
+ if (value) {
+ jsobj2 = json_object_new_object();
+ json_object_object_add(jsobj2, kbuf, json_object_new_string(value));
+ json_object_array_add(jsarr, jsobj2);
+ tcfree(value);
+ }
}
if(keylist) tcfree(keylist);
json_object_object_add(jsobj, "results", jsarr);
@@ -263,7 +232,7 @@ void fwmatch_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
db_status = tcrdbecode(rdb);
db_error_to_json(db_status, jsobj);
}
-
+
finalize_json(req, evb, &args, jsobj);
}
@@ -273,15 +242,11 @@ void del_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
struct evkeyvalq args;
struct json_object *jsobj;
- _gettime(&ts1);
-
- requests++;
- stats_request_counts[STATS_DEL]++;
-
if (rdb == NULL) {
evhttp_send_error(req, 503, "database not connected");
return;
}
+
evhttp_parse_query(req->uri, &args);
key = (char *)evhttp_find_header(&args, "key");
@@ -300,9 +265,6 @@ void del_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
}
finalize_json(req, evb, &args, jsobj);
-
- _gettime(&ts2);
- stats_store_request(STATS_DEL, _ts_diff(ts1, ts2));
}
void put_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
@@ -311,24 +273,22 @@ void put_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
struct evkeyvalq args;
struct json_object *jsobj;
- _gettime(&ts1);
-
- requests++;
- stats_request_counts[STATS_PUT]++;
-
if (rdb == NULL) {
evhttp_send_error(req, 503, "database not connected");
return;
}
+
evhttp_parse_query(req->uri, &args);
-
+
key = (char *)evhttp_find_header(&args, "key");
value = (char *)evhttp_find_header(&args, "value");
+
if (key == NULL) {
evhttp_send_error(req, 400, "key is required");
evhttp_clear_headers(&args);
return;
}
+
if (value == NULL) {
evhttp_send_error(req, 400, "value is required");
evhttp_clear_headers(&args);
@@ -343,11 +303,8 @@ void put_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
db_status = tcrdbecode(rdb);
db_error_to_json(db_status, jsobj);
}
-
- finalize_json(req, evb, &args, jsobj);
- _gettime(&ts2);
- stats_store_request(STATS_PUT, _ts_diff(ts1, ts2));
+ finalize_json(req, evb, &args, jsobj);
}
void get_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
@@ -356,19 +313,15 @@ void get_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
struct evkeyvalq args;
struct json_object *jsobj;
- _gettime(&ts1);
-
- requests++;
- stats_request_counts[STATS_GET]++;
-
if (rdb == NULL) {
evhttp_send_error(req, 503, "database not connected");
return;
}
-
+
evhttp_parse_query(req->uri, &args);
-
+
key = (char *)evhttp_find_header(&args, "key");
+
if (key == NULL) {
evhttp_send_error(req, 400, "key is required");
evhttp_clear_headers(&args);
@@ -385,11 +338,8 @@ void get_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
db_status = tcrdbecode(rdb);
db_error_to_json(db_status, jsobj);
}
-
- finalize_json(req, evb, &args, jsobj);
- _gettime(&ts2);
- stats_store_request(STATS_GET, _ts_diff(ts1, ts2));
+ finalize_json(req, evb, &args, jsobj);
}
void get_int_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
@@ -399,19 +349,15 @@ void get_int_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
struct evkeyvalq args;
struct json_object *jsobj;
- _gettime(&ts1);
-
- requests++;
- stats_request_counts[STATS_GET_INT]++;
-
if (rdb == NULL) {
evhttp_send_error(req, 503, "database not connected");
return;
}
-
+
evhttp_parse_query(req->uri, &args);
-
+
key = (char *)evhttp_find_header(&args, "key");
+
if (key == NULL) {
evhttp_send_error(req, 400, "key is required");
evhttp_clear_headers(&args);
@@ -428,11 +374,8 @@ void get_int_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
db_status = tcrdbecode(rdb);
db_error_to_json(db_status, jsobj);
}
-
- finalize_json(req, evb, &args, jsobj);
- _gettime(&ts2);
- stats_store_request(STATS_GET_INT, _ts_diff(ts1, ts2));
+ finalize_json(req, evb, &args, jsobj);
}
void mget_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
@@ -443,19 +386,15 @@ void mget_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
struct json_object *jsobj, *jserr;
int nkeys = 0;
- _gettime(&ts1);
-
- requests++;
- stats_request_counts[STATS_MGET]++;
-
if (rdb == NULL) {
evhttp_send_error(req, 503, "database not connected");
return;
}
+ evhttp_parse_query(req->uri, &args);
+
jsobj = json_object_new_object();
- evhttp_parse_query(req->uri, &args);
TAILQ_FOREACH(pair, &args, next) {
if (pair->key[0] != 'k') continue;
key = (char *)pair->value;
@@ -480,9 +419,6 @@ void mget_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
}
finalize_json(req, evb, &args, jsobj);
-
- _gettime(&ts2);
- stats_store_request(STATS_MGET, _ts_diff(ts1, ts2));
}
void mget_int_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
@@ -494,19 +430,15 @@ void mget_int_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
struct json_object *jsobj, *jserr;
int nkeys = 0;
- _gettime(&ts1);
-
- requests++;
- stats_request_counts[STATS_MGET_INT]++;
-
if (rdb == NULL) {
evhttp_send_error(req, 503, "database not connected");
return;
}
+ evhttp_parse_query(req->uri, &args);
+
jsobj = json_object_new_object();
- evhttp_parse_query(req->uri, &args);
TAILQ_FOREACH(pair, &args, next) {
if (pair->key[0] != 'k') continue;
key = (char *)pair->value;
@@ -531,9 +463,6 @@ void mget_int_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
}
finalize_json(req, evb, &args, jsobj);
-
- _gettime(&ts2);
- stats_store_request(STATS_MGET_INT, _ts_diff(ts1, ts2));
}
void incr_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
@@ -547,23 +476,21 @@ void incr_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
bool has_key_arg = false;
bool error = false;
- _gettime(&ts1);
-
- requests++;
- stats_request_counts[STATS_INCR]++;
-
if (rdb == NULL) {
evhttp_send_error(req, 503, "database not connected");
return;
}
+
evhttp_parse_query(req->uri, &args);
-
+
incr_value = (char *)evhttp_find_header(&args, "value");
+
if (incr_value != NULL) {
value = atoi(incr_value);
}
-
+
jsobj = json_object_new_object();
+
TAILQ_FOREACH(arg, &args, next) {
if (strcasecmp(arg->key, "key") == 0) {
has_key_arg = true;
@@ -575,7 +502,7 @@ void incr_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
}
}
}
-
+
if (!has_key_arg) {
evhttp_send_error(req, 400, "key is required");
evhttp_clear_headers(&args);
@@ -584,11 +511,8 @@ void incr_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
}
if (!error) json_object_object_add(jsobj, "status", json_object_new_string("ok"));
-
- finalize_json(req, evb, &args, jsobj);
- _gettime(&ts2);
- stats_store_request(STATS_INCR, _ts_diff(ts1, ts2));
+ finalize_json(req, evb, &args, jsobj);
}
void vanish_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
@@ -596,11 +520,6 @@ void vanish_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
struct json_object *jsobj;
const char *json;
- _gettime(&ts1);
-
- requests++;
- stats_request_counts[STATS_VANISH]++;
-
if (rdb == NULL) {
evhttp_send_error(req, 503, "database not connected");
return;
@@ -616,58 +535,42 @@ void vanish_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
json_object_put(jsobj); // Odd free function
evhttp_send_reply(req, HTTP_OK, "OK", evb);
-
- _gettime(&ts2);
- stats_store_request(STATS_VANISH, _ts_diff(ts1, ts2));
}
void stats_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
{
- uint64_t request_total;
- uint64_t average_requests[NUM_REQUEST_TYPES];
- uint64_t ninety_five_percents[NUM_REQUEST_TYPES];
- int i, j, c, request_array_end;
+ int i;
struct evkeyvalq args;
const char *format;
- memset(&average_requests, 0, sizeof(average_requests));
- memset(&ninety_five_percents, 0, sizeof(ninety_five_percents));
+ struct simplehttp_stats *st;
- for (i = 0; i < NUM_REQUEST_TYPES; i++) {
- request_total = 0;
- for (j = (i * NUM_REQUESTS_FOR_STATS), request_array_end = j + NUM_REQUESTS_FOR_STATS, c = 0;
- (j < request_array_end) && (stats_request[j] != -1); j++, c++) {
- request_total += stats_request[j];
- }
- if (c) {
- average_requests[i] = request_total / c;
- ninety_five_percents[i] = ninety_five_percent(stats_request + (i * NUM_REQUESTS_FOR_STATS), c);
- }
- }
+ st = simplehttp_stats_new();
+ simplehttp_stats(st);
evhttp_parse_query(req->uri, &args);
format = (char *)evhttp_find_header(&args, "format");
if ((format != NULL) && (strcmp(format, "json") == 0)) {
evbuffer_add_printf(evb, "{");
- evbuffer_add_printf(evb, "\"db_opens\": %"PRIu64",", db_opened);
- for (i = 0; i < NUM_REQUEST_TYPES; i++) {
- evbuffer_add_printf(evb, "\"%s_95\": %"PRIu64",", stats_request_labels[i], ninety_five_percents[i]);
- evbuffer_add_printf(evb, "\"%s_average_request\": %"PRIu64",", stats_request_labels[i], average_requests[i]);
- evbuffer_add_printf(evb, "\"%s_requests\": %"PRIu64",", stats_request_labels[i], stats_request_counts[i]);
+ for (i = 0; i < st->callback_count; i++) {
+ evbuffer_add_printf(evb, "\"%s_95\": %"PRIu64",", st->stats_labels[i], st->ninety_five_percents[i]);
+ evbuffer_add_printf(evb, "\"%s_average_request\": %"PRIu64",", st->stats_labels[i], st->average_requests[i]);
+ evbuffer_add_printf(evb, "\"%s_requests\": %"PRIu64",", st->stats_labels[i], st->stats_counts[i]);
}
- evbuffer_add_printf(evb, "\"total_requests\": %"PRIu64, requests);
+ evbuffer_add_printf(evb, "\"total_requests\": %"PRIu64, st->requests);
evbuffer_add_printf(evb, "}\n");
} else {
- evbuffer_add_printf(evb, "db opens: %"PRIu64"\n", db_opened);
- evbuffer_add_printf(evb, "total requests: %"PRIu64"\n", requests);
- for (i = 0; i < NUM_REQUEST_TYPES; i++) {
- evbuffer_add_printf(evb, "/%s 95%%: %"PRIu64"\n", stats_request_labels[i], ninety_five_percents[i]);
- evbuffer_add_printf(evb, "/%s average request (usec): %"PRIu64"\n", stats_request_labels[i], average_requests[i]);
- evbuffer_add_printf(evb, "/%s requests: %"PRIu64"\n", stats_request_labels[i], stats_request_counts[i]);
+ evbuffer_add_printf(evb, "total requests: %"PRIu64"\n", st->requests);
+ for (i = 0; i < st->callback_count; i++) {
+ evbuffer_add_printf(evb, "/%s 95%%: %"PRIu64"\n", st->stats_labels[i], st->ninety_five_percents[i]);
+ evbuffer_add_printf(evb, "/%s average request (usec): %"PRIu64"\n", st->stats_labels[i], st->average_requests[i]);
+ evbuffer_add_printf(evb, "/%s requests: %"PRIu64"\n", st->stats_labels[i], st->stats_counts[i]);
}
}
+ simplehttp_stats_free(st);
+
evhttp_send_reply(req, HTTP_OK, "OK", evb);
evhttp_clear_headers(&args);
}
@@ -681,12 +584,12 @@ void exit_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx) {
void info()
{
fprintf(stdout, "simpletokyo: a light http interface to Tokyo Tyrant.\n");
- fprintf(stdout, "Version %s, https://github.com/bitly/simplehttp/tree/master/simpletokyo\n", version);
+ fprintf(stdout, "Version %s, https://github.com/bitly/simplehttp/tree/master/simpletokyo\n", VERSION);
}
void usage()
{
- fprintf(stderr, "%s: http wrapper for Tokyo Tyrant\n", g_progname);
+ fprintf(stderr, "%s: http wrapper for Tokyo Tyrant\n", NAME);
fprintf(stderr, "\n");
fprintf(stderr, "usage:\n");
fprintf(stderr, "\t-A ttserver address\n");
@@ -723,10 +626,6 @@ int main(int argc, char **argv)
}
}
- memset(&stats_request, -1, sizeof(stats_request));
- memset(&stats_request_idx, 0, sizeof(stats_request_idx));
- memset(&stats_request_counts, 0, sizeof(stats_request_counts));
-
memset(&db_status, -1, sizeof(db_status));
simplehttp_init();
View
2 sortdb/Makefile
@@ -7,7 +7,7 @@ LIBSIMPLEHTTP_LIB ?= $(LIBSIMPLEHTTP)
CFLAGS = -I$(LIBSIMPLEHTTP_INC) -I$(LIBEVENT)/include -Wall -g -O2
LIBS = -L$(LIBSIMPLEHTTP_LIB) -L$(LIBEVENT)/lib -levent -lsimplehttp -lm
-sortdb: sortdb.c ../lib/timer.c ../lib/util.c
+sortdb: sortdb.c
$(CC) $(CFLAGS) -o $@ $^ $(LIBS)
install:
View
121 sortdb/sortdb.c
@@ -11,15 +11,10 @@
#include <inttypes.h>
#include <simplehttp/queue.h>
#include <simplehttp/simplehttp.h>
-#include "lib/timer.h"
-#include "lib/util.h"
-#define DEBUG 1
-
-#define NUM_REQUEST_TYPES 2
-#define NUM_REQUESTS_FOR_STATS 1000
-#define STATS_GET 0
-#define STATS_MGET 1
+#define NAME "sortdb"
+#define VERSION "1.4"
+#define DEBUG 1
void stats_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx);
void get_cb(struct evhttp_request *req, struct evbuffer *evb,void *ctx);
@@ -34,7 +29,6 @@ void close_dbfile();
void open_dbfile();
void hup_handler(int signum);
-static char *version = "1.3";
static void *map_base = NULL;
static char *db_filename;
static struct stat st;
@@ -45,22 +39,6 @@ static uint64_t get_hits = 0;
static uint64_t get_misses = 0;
static uint64_t total_seeks = 0;
-static uint64_t requests = 0;
-static uint64_t stats_request_counts[NUM_REQUEST_TYPES];
-static int64_t stats_request[NUM_REQUESTS_FOR_STATS * NUM_REQUEST_TYPES];
-static int stats_request_idx[NUM_REQUEST_TYPES];
-static char *stats_request_labels[] = { "get", "get_int", "put", "incr", "del", "fwmatch", "fwmatch_int", "vanish" };
-
-void stats_store_request(int index, unsigned int diff)
-{
- stats_request[(index * NUM_REQUESTS_FOR_STATS) + stats_request_idx[index]] = diff;
- stats_request_idx[index]++;
-
- if (stats_request_idx[index] >= NUM_REQUESTS_FOR_STATS) {
- stats_request_idx[index] = 0;
- }
-}
-
char *prev_line(char *pos)
{
if (!pos) return NULL;
@@ -76,16 +54,16 @@ char *map_search(char *key, size_t keylen, char *lower, char *upper, int *seeks)
char *current;
char *line;
int rc;
-
+
distance = (upper - lower);
if (distance <= 1) return NULL;
-
+
*seeks += 1;
total_seeks++;
current = lower + (distance/2);
line = prev_line(current);
if (!line) return NULL;
-
+
rc = strncmp(key, line, keylen);
if (rc < 0) {
return map_search(key, keylen, lower, current, seeks);
@@ -104,11 +82,6 @@ void get_cb(struct evhttp_request *req, struct evbuffer *evb,void *ctx)
char *key, *line, *newline, *delim, buf[32];
int seeks = 0;
- _gettime(&ts1);
-
- requests++;
- stats_request_counts[STATS_GET]++;
-
evhttp_parse_query(req->uri, &args);
key = (char *)evhttp_find_header(&args, "key");
@@ -135,15 +108,9 @@ void get_cb(struct evhttp_request *req, struct evbuffer *evb,void *ctx)
} else {
get_misses++;
evhttp_send_reply(req, HTTP_NOTFOUND, "OK", evb);
-
}