Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

more code shuffling, preparing for showdown with ujson

  • Loading branch information...
commit 85342530248c276c22ce58c667c1e3479107a3e7 1 parent 6333e56
Ryan Kelly authored
2  tnetstring/_tnetstring.c
View
@@ -98,7 +98,7 @@ _tnetstring_dumps(PyObject* self, PyObject *args, PyObject *kwds)
}
Py_DECREF(object);
- string = PyString_FromStringAndSize(NULL,outbuf.used_size);
+ string = PyString_FromStringAndSize(NULL,tns_outbuf_size(&outbuf));
if(string == NULL) {
return NULL;
}
10 tnetstring/dbg.h
View
@@ -1,7 +1,15 @@
+//
+// dbg.h: minimal checking and debugging functions
+//
+// This is a small compatability shim for the Mongrel2 "dbg.h" interface,
+// to make it easier to port code back and forth between the tnetstring
+// implementation in Mongrel2 and this module.
+//
+
#ifndef __dbg_h__
#define __dbg_h__
-#define check(A, M, ...) if(!(A)) { PyErr_Format(_tnetstring_Error,M, ##__VA_ARGS__); goto error; }
+#define check(A, M, ...) if(!(A)) { PyErr_Format(_tnetstring_Error, M, ##__VA_ARGS__); goto error; }
#define sentinel(M, ...) check(0, M, ##__VA_ARGS__)
49 tnetstring/tns_core.c
View
@@ -10,18 +10,34 @@
// and then #include "tns_core.c" to stitch them into a tnetstring parser.
//
+#include <stdlib.h>
+#include <stddef.h>
+#include <ctype.h>
+
+#include "dbg.h"
+
#include "tns_core.h"
-#include "tns_outbuf_rev.c"
-static void*
-tns_parse(const char *data, size_t len, char **remain)
+// These are our internal-use functions.
+
+static int tns_parse_dict(void *dict, const char *data, size_t len);
+static int tns_parse_list(void *list, const char *data, size_t len);
+static int tns_outbuf_putc(tns_outbuf *outbuf, char c);
+static int tns_outbuf_puts(tns_outbuf *outbuf, const char *data, size_t len);
+static int tns_outbuf_clamp(tns_outbuf *outbuf, size_t orig_size);
+static char* tns_outbuf_finalize(tns_outbuf *outbuf, size_t *len);
+static void tns_outbuf_free(tns_outbuf *outbuf);
+
+#include "tns_outbuf_back.c"
+
+static void* tns_parse(const char *data, size_t len, char **remain)
{
void *val = NULL;
char *valstr = NULL;
tns_type_tag type = tns_tag_null;
size_t vallen = 0;
- // Read the length of the value, and verify that is ends in a colon.
+ // Read the length of the value, and verify that it ends in a colon.
vallen = strtol(data, &valstr, 10);
check(valstr != data, "Not a tnetstring: no length prefix.");
check((valstr+vallen+1) < (data+len), "Not a tnetstring: bad length prefix.");
@@ -94,8 +110,7 @@ tns_parse(const char *data, size_t len, char **remain)
}
-static char*
-tns_render(void *val, size_t *len)
+static char* tns_render(void *val, size_t *len)
{
tns_outbuf outbuf;
@@ -121,7 +136,7 @@ static int tns_render_value(void *val, tns_outbuf *outbuf)
check(type != 0, "type not serializable.");
tns_outbuf_putc(outbuf,type);
- orig_size = outbuf->used_size;
+ orig_size = tns_outbuf_size(outbuf);
// Render it into the output buffer, leaving space for the
// type tag at the end.
@@ -156,18 +171,10 @@ static int tns_render_value(void *val, tns_outbuf *outbuf)
}
-#define tns_rotate_buffer(data, remain, len, orig_len) {\
- len = len - (remain - data);\
- check(len < orig_len, "Error parsing data, buffer math is off.");\
- data = remain;\
-}
-
-
static int tns_parse_list(void *val, const char *data, size_t len)
{
void *item = NULL;
char *remain = NULL;
- size_t orig_len = len;
assert(value != NULL && "value cannot be NULL");
assert(data != NULL && "data cannot be NULL");
@@ -175,7 +182,8 @@ static int tns_parse_list(void *val, const char *data, size_t len)
while(len > 0) {
item = tns_parse(data, len, &remain);
check(item != NULL, "Failed to parse list.");
- tns_rotate_buffer(data, remain, len, orig_len);
+ len = len - (remain - data);
+ data = remain;
check(tns_add_to_list(val, item) != -1,
"Failed to add item to list.");
item = NULL;
@@ -196,19 +204,20 @@ static int tns_parse_dict(void *val, const char *data, size_t len)
void *key = NULL;
void *item = NULL;
char *remain = NULL;
- size_t orig_len = len;
- assert(value != NULL && "value cannot be NULL");
+ assert(val != NULL && "value cannot be NULL");
assert(data != NULL && "data cannot be NULL");
while(len > 0) {
key = tns_parse(data, len, &remain);
check(key != NULL, "Failed to parse dict key from tnetstring.");
- tns_rotate_buffer(data, remain, len, orig_len);
+ len = len - (remain - data);
+ data = remain;
item = tns_parse(data, len, &remain);
check(item != NULL, "Failed to parse dict item from tnetstring.");
- tns_rotate_buffer(data, remain, len, orig_len);
+ len = len - (remain - data);
+ data = remain;
check(tns_add_to_dict(val,key,item) != -1,
"Failed to add element to dict.");
38 tnetstring/tns_core.h
View
@@ -17,14 +17,15 @@
#include <stddef.h>
#include <ctype.h>
-#include "dbg.h"
-
-typedef struct tns_outbuf_s {
- char *buffer;
- size_t used_size;
- size_t alloc_size;
-} tns_outbuf;
-
+// tnetstring rendering is done using an "outbuf" struct, which combines
+// a malloced string with its allocation information. Rendering is done
+// from front to back; the details are deliberately hidden here since
+// I'm experimenting with multiple implementations.
+struct tns_outbuf_s;
+typedef struct tns_outbuf_s tns_outbuf;
+
+// This enumeration gives the type tag for each data type in the
+// tnetstring encoding.
typedef enum tns_type_tag_e {
tns_tag_string = ',',
tns_tag_number = '#',
@@ -34,6 +35,7 @@ typedef enum tns_type_tag_e {
tns_tag_list = ']',
} tns_type_tag;
+
// You must provide implementations for the following functions.
// They provide the low-level data manipulation routines from which
// we can build a parser and renderer.
@@ -82,29 +84,13 @@ static void* tns_parse(const char *data, size_t len, char** remain);
// On failure this function returns NULL and 'len' is unmodified.
static char* tns_render(void *val, size_t *len);
-
// If you need to copy the final result off somewhere else, you
// might like to build your own rendering function from the following.
-// It will avoid some double-copying that tns_render does in order
-// to return a proper C string.
+// It will avoid some double-copying that tns_render does internally.
// Basic plan: Initialize an outbuf, pass it to tns_render_value, then
// copy the bytes away using tns_outbuf_memmove.
-static inline int tns_outbuf_init(tns_outbuf *outbuf);
+static int tns_outbuf_init(tns_outbuf *outbuf);
static int tns_render_value(void *val, tns_outbuf *outbuf);
static void tns_outbuf_memmove(tns_outbuf *outbuf, char *dest);
-
-
-// The rest of these are for internal use only.
-
-static inline int tns_parse_dict(void *dict, const char *data, size_t len);
-static inline int tns_parse_list(void *list, const char *data, size_t len);
-static inline int tns_outbuf_itoa(tns_outbuf *outbuf, size_t n);
-static inline int tns_outbuf_extend(tns_outbuf *outbuf);
-static inline int tns_outbuf_putc(tns_outbuf *outbuf, char c);
-static int tns_outbuf_puts(tns_outbuf *outbuf, const char *data, size_t len);
-static inline int tns_outbuf_clamp(tns_outbuf *outbuf, size_t orig_size);
-static char* tns_outbuf_finalize(tns_outbuf *outbuf, size_t *len);
-static inline void tns_outbuf_free(tns_outbuf *outbuf);
-
#endif
83 tnetstring/tns_outbuf_push.c → tnetstring/tns_outbuf_back.c
View
@@ -1,5 +1,25 @@
-
-#include "tns_core.h"
+//
+// tns_outbuf_back: tns_outbuf implemented by writing from back of buffer.
+//
+// This outbuf implementation writes data starting at the back of the
+// allocated buffer. To finalize it you need to memmove it to the start
+// of the buffer so it can be treated like an ordinary malloced string.
+//
+// The advantage of this scheme is that the data is the right way round,
+// so you can shuffle it about using memmove. The disadvantage is that
+// reallocating the buffer is tricker as you must move the data by hand.
+//
+
+struct tns_outbuf_s {
+ char *buffer;
+ char *head;
+ size_t alloc_size;
+};
+
+static inline size_t tns_outbuf_size(tns_outbuf *outbuf)
+{
+ return outbuf->alloc_size - (outbuf->head - outbuf->buffer);
+}
static inline int tns_outbuf_itoa(tns_outbuf *outbuf, size_t n)
{
@@ -21,13 +41,13 @@ static inline int tns_outbuf_init(tns_outbuf *outbuf)
outbuf->buffer = malloc(64);
check_mem(outbuf->buffer);
+ outbuf->head = outbuf->buffer + 64;
outbuf->alloc_size = 64;
- outbuf->used_size = 0;
return 0;
error:
+ outbuf->head = NULL;
outbuf->alloc_size = 0;
- outbuf->used_size = 0;
return -1;
}
@@ -37,24 +57,33 @@ static inline void tns_outbuf_free(tns_outbuf *outbuf)
if(outbuf) {
free(outbuf->buffer);
outbuf->buffer = NULL;
+ outbuf->head = 0;
outbuf->alloc_size = 0;
- outbuf->used_size = 0;
}
}
-static inline int tns_outbuf_extend(tns_outbuf *outbuf)
+static inline int tns_outbuf_extend(tns_outbuf *outbuf, size_t free_size)
{
char *new_buf = NULL;
+ char *new_head = NULL;
size_t new_size = outbuf->alloc_size * 2;
+ size_t used_size;
+
+ used_size = tns_outbuf_size(outbuf);
+
+ while(new_size < free_size + used_size) {
+ new_size = new_size * 2;
+ }
new_buf = malloc(new_size);
check_mem(new_buf);
- memmove(new_buf + new_size - outbuf->used_size,
- outbuf->buffer + outbuf->alloc_size - outbuf->used_size,
- outbuf->used_size);
+
+ new_head = new_buf + new_size - used_size;
+ memmove(new_head, outbuf->head, used_size);
outbuf->buffer = new_buf;
+ outbuf->head = new_head;
outbuf->alloc_size = new_size;
return 0;
@@ -66,11 +95,11 @@ static inline int tns_outbuf_extend(tns_outbuf *outbuf)
static inline int tns_outbuf_putc(tns_outbuf *outbuf, char c)
{
- if(outbuf->alloc_size == outbuf->used_size) {
- check(tns_outbuf_extend(outbuf) != -1, "Failed to extend buffer");
+ if(outbuf->buffer == outbuf->head) {
+ check(tns_outbuf_extend(outbuf, 1) != -1, "Failed to extend buffer");
}
- outbuf->buffer[outbuf->alloc_size - ++outbuf->used_size] = c;
+ *(--outbuf->head) = c;
return 0;
@@ -81,14 +110,12 @@ static inline int tns_outbuf_putc(tns_outbuf *outbuf, char c)
static int tns_outbuf_puts(tns_outbuf *outbuf, const char *data, size_t len)
{
- while(outbuf->alloc_size - outbuf->used_size < len) {
- check(tns_outbuf_extend(outbuf) != -1, "Failed to extend buffer");
+ if(outbuf->head - outbuf->buffer < len) {
+ check(tns_outbuf_extend(outbuf, len) != -1, "Failed to extend buffer");
}
- memmove(outbuf->buffer + outbuf->alloc_size - outbuf->used_size - len,
- data, len);
-
- outbuf->used_size += len;
+ outbuf->head -= len;
+ memmove(outbuf->head, data, len);
return 0;
@@ -99,21 +126,22 @@ static int tns_outbuf_puts(tns_outbuf *outbuf, const char *data, size_t len)
static char* tns_outbuf_finalize(tns_outbuf *outbuf, size_t *len)
{
char *new_buf = NULL;
+ size_t used_size;
- memmove(outbuf->buffer,
- outbuf->buffer + outbuf->alloc_size - outbuf->used_size,
- outbuf->used_size);
+ used_size = tns_outbuf_size(outbuf);
+
+ memmove(outbuf->buffer, outbuf->head, used_size);
if(len != NULL) {
- *len = outbuf->used_size;
+ *len = used_size;
} else {
- if(outbuf->alloc_size == outbuf->used_size) {
+ if(outbuf->head == outbuf->buffer) {
new_buf = realloc(outbuf->buffer, outbuf->alloc_size*2);
check_mem(new_buf);
outbuf->buffer = new_buf;
outbuf->alloc_size = outbuf->alloc_size * 2;
}
- outbuf->buffer[outbuf->used_size++] = '\0';
+ outbuf->buffer[used_size] = '\0';
}
return outbuf->buffer;
@@ -121,13 +149,14 @@ static char* tns_outbuf_finalize(tns_outbuf *outbuf, size_t *len)
error:
free(outbuf->buffer);
outbuf->buffer = NULL;
+ outbuf->alloc_size = 0;
return NULL;
}
static inline int tns_outbuf_clamp(tns_outbuf *outbuf, size_t orig_size)
{
- size_t datalen = outbuf->used_size - orig_size;
+ size_t datalen = tns_outbuf_size(outbuf) - orig_size;
check(tns_outbuf_putc(outbuf, ':') != -1, "Failed to clamp outbuf");
check(tns_outbuf_itoa(outbuf, datalen) != -1, "Failed to clamp outbuf");
@@ -141,8 +170,6 @@ static inline int tns_outbuf_clamp(tns_outbuf *outbuf, size_t orig_size)
static inline void tns_outbuf_memmove(tns_outbuf *outbuf, char *dest)
{
- memmove(dest,
- outbuf->buffer + outbuf->alloc_size - outbuf->used_size,
- outbuf->used_size);
+ memmove(dest, outbuf->head, tns_outbuf_size(outbuf));
}
40 tnetstring/tns_outbuf_rev.c
View
@@ -1,9 +1,33 @@
+//
+// tns_outbuf_rev: tns_outbuf implemented by writing in reverse.
+//
+// This outbuf implementation writes data starting at the front of the
+// allocated buffer, but writes it in reverse. To finalize it you do
+// an in-place reverse before returning the buffer.
+//
+// The advantage of this scheme is that the data is at the beginning
+// of the allocated buffer, so you can extend it using realloc and pass
+// it around as if it were a normal string. However, the final copying
+// of the data has to be done in reverse so we can't use memmove.
+
+struct tns_outbuf_s {
+ char *buffer;
+ size_t used_size;
+ size_t alloc_size;
+};
+
+
+static inline size_t tns_outbuf_size(tns_outbuf *outbuf)
+{
+ return outbuf->used_size;
+}
-#include "tns_core.h"
static inline void tns_inplace_reverse(char *data, size_t len)
{
- char *dend, c;
+ char *dend = NULL;
+ char c = '0';
+
dend = data + len - 1;
while(dend > data) {
c = *data;
@@ -57,11 +81,15 @@ static inline void tns_outbuf_free(tns_outbuf *outbuf)
}
-static inline int tns_outbuf_extend(tns_outbuf *outbuf)
+static inline int tns_outbuf_extend(tns_outbuf *outbuf, size_t free_size)
{
char *new_buf = NULL;
size_t new_size = outbuf->alloc_size * 2;
+ while(new_size < free_size + outbuf->used_size) {
+ new_size = new_size * 2;
+ }
+
new_buf = realloc(outbuf->buffer, new_size);
check_mem(new_buf);
@@ -78,7 +106,7 @@ static inline int tns_outbuf_extend(tns_outbuf *outbuf)
static inline int tns_outbuf_putc(tns_outbuf *outbuf, char c)
{
if(outbuf->alloc_size == outbuf->used_size) {
- check(tns_outbuf_extend(outbuf) != -1, "Failed to extend buffer");
+ check(tns_outbuf_extend(outbuf, 1) != -1, "Failed to extend buffer");
}
outbuf->buffer[outbuf->used_size++] = c;
@@ -96,8 +124,8 @@ static int tns_outbuf_puts(tns_outbuf *outbuf, const char *data, size_t len)
char *buffer = NULL;
// Make sure we have enough room.
- while(outbuf->alloc_size - outbuf->used_size < len) {
- check(tns_outbuf_extend(outbuf) != -1, "Failed to extend buffer");
+ if(outbuf->alloc_size - outbuf->used_size < len) {
+ check(tns_outbuf_extend(outbuf, len) != -1, "Failed to extend buffer");
}
// Copy the data in reverse.
12 tools/shootout.py
View
@@ -28,23 +28,27 @@ def add_test(v):
def thrash_tnetstring():
for obj, tns, json in TESTS:
+# tnetstring.dumps(obj)
assert tnetstring.loads(tns) == obj
- assert tnetstring.loads(tnetstring.dumps(obj)) == obj
+# assert tnetstring.loads(tnetstring.dumps(obj)) == obj
def thrash_cjson():
for obj, tns, json in TESTS:
+# cjson.encode(obj)
assert cjson.decode(json) == obj
- assert cjson.decode(cjson.encode(obj)) == obj
+# assert cjson.decode(cjson.encode(obj)) == obj
def thrash_yajl():
for obj, tns, json in TESTS:
+# yajl.dumps(obj)
assert yajl.loads(json) == obj
- assert yajl.loads(yajl.dumps(obj)) == obj
+# assert yajl.loads(yajl.dumps(obj)) == obj
def thrash_ujson():
for obj, tns, json in TESTS:
+# ujson.dumps(obj)
assert ujson.loads(json) == obj
- assert ujson.loads(ujson.dumps(obj)) == obj
+# assert ujson.loads(ujson.dumps(obj)) == obj
if __name__ == "__main__":
Please sign in to comment.
Something went wrong with that request. Please try again.