Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

refactor to use an explicit struct of function pointers.

This costs a little in performance, since the compiler can't inline
trivial functions like tns_get_null().  But, it will make it much easier
to provide slightly different parsing/rending strategies e.g.
automatically turning unicode strings into utf8 bytes.
  • Loading branch information...
commit 83fd4ad4442e7a0996215bf43984aa065dccdaff 1 parent 3cc8e00
@rfk authored
View
214 tnetstring/_tnetstring.c
@@ -15,6 +15,7 @@
#define TNS_MAX_LENGTH 999999999
#include "tns_core.c"
+static tns_ops _tnetstring_ops;
static PyObject*
_tnetstring_loads(PyObject* self, PyObject *args)
@@ -34,7 +35,7 @@ _tnetstring_loads(PyObject* self, PyObject *args)
data = PyString_AS_STRING(string);
len = PyString_GET_SIZE(string);
- val = tns_parse(data, len, NULL);
+ val = tns_parse(&_tnetstring_ops, data, len, NULL);
Py_DECREF(string);
if(val == NULL) {
return NULL;
@@ -149,7 +150,7 @@ _tnetstring_load(PyObject* self, PyObject *args)
// Parse out the payload object
data = PyString_AS_STRING(res);
- val = tns_parse_payload(data[datalen], data, datalen);
+ val = tns_parse_payload(&_tnetstring_ops, data[datalen], data, datalen);
Py_DECREF(res); res = NULL;
return val;
@@ -192,7 +193,7 @@ _tnetstring_pop(PyObject* self, PyObject *args)
data = PyString_AS_STRING(string);
len = PyString_GET_SIZE(string);
- val = tns_parse(data, len, &remain);
+ val = tns_parse(&_tnetstring_ops, data, len, &remain);
Py_DECREF(string);
if(val == NULL) {
return NULL;
@@ -219,7 +220,7 @@ _tnetstring_dumps(PyObject* self, PyObject *args, PyObject *kwds)
Py_DECREF(object);
return NULL;
}
- if(tns_render_value(object, &outbuf) == -1) {
+ if(tns_render_value(&_tnetstring_ops, object, &outbuf) == -1) {
Py_DECREF(object);
return NULL;
}
@@ -269,47 +270,16 @@ static PyMethodDef _tnetstring_methods[] = {
};
-PyDoc_STRVAR(module_doc,
-"Fast encoding/decoding of typed-netstrings."
-);
-
-
-PyMODINIT_FUNC
-init_tnetstring(void)
-{
- Py_InitModule3("_tnetstring", _tnetstring_methods, module_doc);
-}
-
-
// Functions to hook the parser core up to python.
-static inline void*
-tns_parse_string(const char *data, size_t len)
+static void* tns_parse_string(const char *data, size_t len)
{
return PyString_FromStringAndSize(data, len);
}
-static inline int
-tns_str_is_float(const char *data, size_t len)
-{
- const char* dend = data + len;
- while(data < dend) {
- switch(*data++) {
- case '.':
- case 'e':
- case 'E':
- return 1;
- }
- }
- return 0;
-}
-
-
-static inline void*
-tns_parse_number(const char *data, size_t len)
+static void* tns_parse_integer(const char *data, size_t len)
{
- double d = 0;
long l = 0;
long long ll = 0;
int sign = 1;
@@ -318,16 +288,7 @@ tns_parse_number(const char *data, size_t len)
const char *pos, *eod;
PyObject *v = NULL;
- if(tns_str_is_float(data, len)) {
- // Technically this allows whitespace around the float, which
- // isn't valid in a tnetstring. But I don't want to waste the
- // time checking and I am *not* reimplementing strtod.
- d = strtod(data, &dataend);
- if(dataend != data + len) {
- return NULL;
- }
- return PyFloat_FromDouble(d);
- } else if (len < 10) {
+ if (len < 10) {
// Anything with less than 10 digits, we can fit into a long.
// Hand-parsing, as we need tighter error-checking than strtol.
pos = data;
@@ -352,7 +313,7 @@ tns_parse_number(const char *data, size_t len)
sign = -1;
break;
default:
- return NULL;
+ sentinel("invalid integer literal");
}
while(pos < eod) {
c = *pos++;
@@ -370,7 +331,7 @@ tns_parse_number(const char *data, size_t len)
l = (l * 10) + (c - '0');
break;
default:
- return NULL;
+ sentinel("invalid integer literal");
}
}
return PyLong_FromLong(l * sign);
@@ -399,7 +360,7 @@ tns_parse_number(const char *data, size_t len)
sign = -1;
break;
default:
- return NULL;
+ sentinel("invalid integer literal");
}
while(pos < eod) {
c = *pos++;
@@ -417,7 +378,7 @@ tns_parse_number(const char *data, size_t len)
ll = (ll * 10) + (c - '0');
break;
default:
- return NULL;
+ sentinel("invalid integer literal");
}
}
return PyLong_FromLongLong(ll * sign);
@@ -425,58 +386,75 @@ tns_parse_number(const char *data, size_t len)
// Really big numbers must be parsed by python.
// Technically this allows whitespace around the number, which
// isn't valid in a tnetstring. But I don't want to waste the
- // time checking and I am *not* reimplementing strtod.
+ // time checking and I am *not* reimplementing arbitrary-precision
+ // strtod for python.
v = PyLong_FromString((char *)data, &dataend, 10);
- if(dataend != data + len) {
- return NULL;
- }
+ check(dataend == data + len, "invalid integer literal");
return v;
}
+ sentinel("invalid code branch, check your compiler...");
+
+error:
return NULL;
}
-static inline void*
-tns_get_null(void)
+static void* tns_parse_float(const char *data, size_t len)
+{
+ double d = 0;
+ char *dataend;
+
+ // Technically this allows whitespace around the float, which
+ // isn't valid in a tnetstring. But I don't want to waste the
+ // time checking and I am *not* reimplementing strtod.
+ d = strtod(data, &dataend);
+ if(dataend != data + len) {
+ return NULL;
+ }
+ return PyFloat_FromDouble(d);
+}
+
+
+static void* tns_get_null(void)
{
Py_INCREF(Py_None);
return Py_None;
}
-static inline void*
-tns_get_true(void)
+
+static void* tns_get_true(void)
{
Py_INCREF(Py_True);
return Py_True;
}
-static inline void*
-tns_get_false(void)
+
+static void* tns_get_false(void)
{
Py_INCREF(Py_False);
return Py_False;
}
-static inline void*
-tns_new_dict(void)
+
+static void* tns_new_dict(void)
{
return PyDict_New();
}
-static inline void*
-tns_new_list(void)
+
+static void* tns_new_list(void)
{
return PyList_New(0);
}
-static inline void
-tns_free_value(void *value)
+
+static void tns_free_value(void *value)
{
Py_XDECREF(value);
}
-static inline int
-tns_add_to_dict(void *dict, void *key, void *item)
+
+static int tns_add_to_dict(void *dict, void *key, void *item)
{
int res;
res = PyDict_SetItem(dict, key, item);
@@ -488,8 +466,8 @@ tns_add_to_dict(void *dict, void *key, void *item)
return 0;
}
-static inline int
-tns_add_to_list(void *list, void *item)
+
+static int tns_add_to_list(void *list, void *item)
{
int res;
res = PyList_Append(list, item);
@@ -501,31 +479,46 @@ tns_add_to_list(void *list, void *item)
}
-static inline int
-tns_render_string(void *val, tns_outbuf *outbuf)
+static int tns_render_string(void *val, tns_outbuf *outbuf)
{
return tns_outbuf_puts(outbuf, PyString_AS_STRING(val),
PyString_GET_SIZE(val));
}
-static inline int
-tns_render_number(void *val, tns_outbuf *outbuf)
+
+static int tns_render_integer(void *val, tns_outbuf *outbuf)
{
- PyObject *string;
+ PyObject *string = NULL;
+ int res = 0;
- if(PyFloat_Check((PyObject*)val)) {
- string = PyObject_Repr(val);
- } else {
- string = PyObject_Str(val);
+ string = PyObject_Str(val);
+ if(string == NULL) {
+ return -1;
}
+
+ res = tns_render_string(string, outbuf);
+ Py_DECREF(string);
+ return res;
+}
+
+
+static int tns_render_float(void *val, tns_outbuf *outbuf)
+{
+ PyObject *string;
+ int res = 0;
+
+ string = PyObject_Repr(val);
if(string == NULL) {
return -1;
}
- return tns_render_string(string, outbuf);
+
+ res = tns_render_string(string, outbuf);
+ Py_DECREF(string);
+ return res;
}
-static inline int
-tns_render_bool(void *val, tns_outbuf *outbuf)
+
+static int tns_render_bool(void *val, tns_outbuf *outbuf)
{
if(val == Py_True) {
return tns_outbuf_puts(outbuf, "true", 4);
@@ -534,17 +527,17 @@ tns_render_bool(void *val, tns_outbuf *outbuf)
}
}
-static inline int
-tns_render_dict(void *val, tns_outbuf *outbuf)
+
+static int tns_render_dict(const tns_ops *ops, void *val, tns_outbuf *outbuf)
{
PyObject *key, *item;
Py_ssize_t pos = 0;
while(PyDict_Next(val, &pos, &key, &item)) {
- if(tns_render_value(item, outbuf) == -1) {
+ if(tns_render_value(ops, item, outbuf) == -1) {
return -1;
}
- if(tns_render_value(key, outbuf) == -1) {
+ if(tns_render_value(ops, key, outbuf) == -1) {
return -1;
}
}
@@ -552,8 +545,7 @@ tns_render_dict(void *val, tns_outbuf *outbuf)
}
-static inline int
-tns_render_list(void *val, tns_outbuf *outbuf)
+static int tns_render_list(const tns_ops *ops, void *val, tns_outbuf *outbuf)
{
PyObject *item;
Py_ssize_t idx;
@@ -563,7 +555,7 @@ tns_render_list(void *val, tns_outbuf *outbuf)
idx = PyList_GET_SIZE(val) - 1;
while(idx >= 0) {
item = PyList_GET_ITEM(val, idx);
- if(tns_render_value(item, outbuf) == -1) {
+ if(tns_render_value(ops, item, outbuf) == -1) {
return -1;
}
idx--;
@@ -572,8 +564,7 @@ tns_render_list(void *val, tns_outbuf *outbuf)
}
-static inline tns_type_tag
-tns_get_type(void *val)
+static tns_type_tag tns_get_type(void *val)
{
if(val == Py_True || val == Py_False) {
return tns_tag_bool;
@@ -582,10 +573,10 @@ tns_get_type(void *val)
return tns_tag_null;
}
if(PyInt_Check((PyObject*)val) || PyLong_Check((PyObject*)val)) {
- return tns_tag_number;
+ return tns_tag_integer;
}
if(PyFloat_Check((PyObject*)val)) {
- return tns_tag_number;
+ return tns_tag_float;
}
if(PyString_Check((PyObject*)val)) {
return tns_tag_string;
@@ -599,3 +590,38 @@ tns_get_type(void *val)
return 0;
}
+
+PyDoc_STRVAR(module_doc,
+"Fast encoding/decoding of typed-netstrings."
+);
+
+
+PyMODINIT_FUNC
+init_tnetstring(void)
+{
+ Py_InitModule3("_tnetstring", _tnetstring_methods, module_doc);
+
+ _tnetstring_ops.get_type = &tns_get_type;
+ _tnetstring_ops.free_value = &tns_free_value;
+
+ _tnetstring_ops.parse_string = tns_parse_string;
+ _tnetstring_ops.parse_integer = tns_parse_integer;
+ _tnetstring_ops.parse_float = tns_parse_float;
+ _tnetstring_ops.get_null = tns_get_null;
+ _tnetstring_ops.get_true = tns_get_true;
+ _tnetstring_ops.get_false = tns_get_false;
+
+ _tnetstring_ops.render_string = tns_render_string;
+ _tnetstring_ops.render_integer = tns_render_integer;
+ _tnetstring_ops.render_float = tns_render_float;
+ _tnetstring_ops.render_bool = tns_render_bool;
+
+ _tnetstring_ops.new_dict = tns_new_dict;
+ _tnetstring_ops.add_to_dict = tns_add_to_dict;
+ _tnetstring_ops.render_dict = tns_render_dict;
+
+ _tnetstring_ops.new_list = tns_new_list;
+ _tnetstring_ops.add_to_list = tns_add_to_list;
+ _tnetstring_ops.render_list = tns_render_list;
+}
+
View
4 tnetstring/dbg.h
@@ -9,10 +9,10 @@
#ifndef __dbg_h__
#define __dbg_h__
-#define check(A, M, ...) if(!(A)) { PyErr_Format(PyExc_ValueError, M, ##__VA_ARGS__); goto error; }
+#define check(A, M, ...) if(!(A)) { if(PyErr_Occurred() == NULL) { PyErr_Format(PyExc_ValueError, M, ##__VA_ARGS__); }; goto error; }
#define sentinel(M, ...) check(0, M, ##__VA_ARGS__)
-#define check_mem(A) if(A==NULL) { PyErr_SetString(PyExc_MemoryError, "Out of memory."); goto error; }
+#define check_mem(A) if(A==NULL) { if(PyErr_Occurred() == NULL) { PyErr_SetString(PyExc_MemoryError, "Out of memory."); }; goto error; }
#endif
View
293 tnetstring/tns_core.c
@@ -5,44 +5,33 @@
// typed-netstring format proposed for inclusion in Mongrel2. You can
// think of it like a JSON library that uses a simpler wire format.
//
-// This code is *not* designed to be compiled as a standalone library.
-// Instead, you provide a suite of low-level data manipulation functions
-// and then #include "tns_core.c" to stitch them into a tnetstring parser.
-//
-
-#include <stdlib.h>
-#include <stddef.h>
-#include <ctype.h>
#include "dbg.h"
-
#include "tns_core.h"
#ifndef TNS_MAX_LENGTH
#define TNS_MAX_LENGTH 999999999
#endif
-// These are our internal-use functions.
-
-static int tns_parse_dict(void *dict, const char *data, size_t len);
-static int tns_parse_list(void *list, const char *data, size_t len);
-static int tns_outbuf_putc(tns_outbuf *outbuf, char c);
-static int tns_outbuf_puts(tns_outbuf *outbuf, const char *data, size_t len);
+// Current outbuf implementations writes data starting at the back of
+// the allocaed buffer. When finished we simply memmove it to the front.
+// Here *buffer points to the allocated buffer, while *head points to the
+// last characer written to the buffer.
+struct tns_outbuf_s {
+ char *buffer;
+ char *head;
+ size_t alloc_size;
+};
+
+static int tns_parse_dict(const tns_ops *ops, void *dict, const char *data, size_t len);
+static int tns_parse_list(const tns_ops *ops, void *list, const char *data, size_t len);
static int tns_outbuf_clamp(tns_outbuf *outbuf, size_t orig_size);
static char* tns_outbuf_finalize(tns_outbuf *outbuf, size_t *len);
static void tns_outbuf_free(tns_outbuf *outbuf);
static size_t tns_strtosz(const char *data, size_t len, size_t *sz, char **end);
-#include "tns_outbuf_back.c"
-
-// This appears to be faster than using strncmp to compare
-// against a small string constant.
-#define STR_EQ_TRUE(s) (s[0]=='t' && s[1]=='r' && s[2]=='u' && s[3]=='e')
-#define STR_EQ_FALSE(s) (s[0]=='f' && s[1]=='a' && s[2]=='l' \
- && s[3]=='s' && s[4] == 'e')
-
-static void* tns_parse(const char *data, size_t len, char **remain)
+void* tns_parse(const tns_ops *ops, const char *data, size_t len, char **remain)
{
char *valstr = NULL;
tns_type_tag type = tns_tag_null;
@@ -51,7 +40,8 @@ static void* tns_parse(const char *data, size_t len, char **remain)
// Read the length of the value, and verify that it ends in a colon.
check(tns_strtosz(data, len, &vallen, &valstr) != -1,
"Not a tnetstring: invalid length prefix.");
- check(*valstr == ':', "Not a tnetstring: invalid length prefix.");
+ check(*valstr == ':',
+ "Not a tnetstring: invalid length prefix.");
valstr++;
check((valstr+vallen) < (data+len),
"Not a tnetstring: invalid length prefix.");
@@ -65,36 +55,48 @@ static void* tns_parse(const char *data, size_t len, char **remain)
}
// Now dispatch type parsing based on the type tag.
- return tns_parse_payload(type, valstr, vallen);
+ return tns_parse_payload(ops, type, valstr, vallen);
error:
return NULL;
}
-static void* tns_parse_payload(tns_type_tag type, const char *data, size_t len)
+// This appears to be faster than using strncmp to compare
+// against a small string constant. Ugly but fast.
+#define STR_EQ_TRUE(s) (s[0]=='t' && s[1]=='r' && s[2]=='u' && s[3]=='e')
+#define STR_EQ_FALSE(s) (s[0]=='f' && s[1]=='a' && s[2]=='l' \
+ && s[3]=='s' && s[4] == 'e')
+
+void* tns_parse_payload(const tns_ops *ops,tns_type_tag type, const char *data, size_t len)
{
void *val = NULL;
+ assert(ops != NULL && "ops struct cannot be NULL");
+
switch(type) {
// Primitive type: a string blob.
case tns_tag_string:
- val = tns_parse_string(data, len);
+ val = ops->parse_string(data, len);
check(val != NULL, "Not a tnetstring: invalid string literal.");
break;
- // Primitive type: a number.
- // I'm branching out here and allowing both floats and ints.
- case tns_tag_number:
- val = tns_parse_number(data, len);
- check(val != NULL, "Not a tnetstring: invalid number literal.");
+ // Primitive type: an integer.
+ case tns_tag_integer:
+ val = ops->parse_integer(data, len);
+ check(val != NULL, "Not a tnetstring: invalid integer literal.");
+ break;
+ // Primitive type: a float.
+ case tns_tag_float:
+ val = ops->parse_float(data, len);
+ check(val != NULL, "Not a tnetstring: invalid float literal.");
break;
// Primitive type: a boolean.
// The only acceptable values are "true" and "false".
case tns_tag_bool:
if(len == 4 && STR_EQ_TRUE(data)) {
- val = tns_get_true();
+ val = ops->get_true();
} else if(len == 5 && STR_EQ_FALSE(data)) {
- val = tns_get_false();
+ val = ops->get_false();
} else {
sentinel("Not a tnetstring: invalid boolean literal.");
val = NULL;
@@ -104,20 +106,20 @@ static void* tns_parse_payload(tns_type_tag type, const char *data, size_t len)
// This must be a zero-length string.
case tns_tag_null:
check(len == 0, "Not a tnetstring: invalid null literal");
- val = tns_get_null();
+ val = ops->get_null();
break;
// Compound type: a dict.
// The data is written <key><value><key><value>
case tns_tag_dict:
- val = tns_new_dict();
- check(tns_parse_dict(val,data,len) != -1,
+ val = ops->new_dict();
+ check(tns_parse_dict(ops, val, data, len) != -1,
"Not a tnetstring: broken dict items.");
break;
// Compound type: a list.
// The data is written <item><item><item>
case tns_tag_list:
- val = tns_new_list();
- check(tns_parse_list(val,data,len) != -1,
+ val = ops->new_list();
+ check(tns_parse_list(ops, val, data, len) != -1,
"Not a tnetstring: broken list items.");
break;
// Whoops, that ain't a tnetstring.
@@ -128,7 +130,7 @@ static void* tns_parse_payload(tns_type_tag type, const char *data, size_t len)
return val;
error:
- tns_free_value(val);
+ ops->free_value(val);
return NULL;
}
@@ -136,12 +138,12 @@ static void* tns_parse_payload(tns_type_tag type, const char *data, size_t len)
#undef STR_EQ_FALSE
-static char* tns_render(void *val, size_t *len)
+char* tns_render(const tns_ops *ops, void *val, size_t *len)
{
tns_outbuf outbuf;
check(tns_outbuf_init(&outbuf) != -1, "Failed to initialize outbuf.");
- check(tns_render_value(val, &outbuf) != -1, "Failed to render value.");
+ check(tns_render_value(ops, val, &outbuf) != -1, "Failed to render value.");
return tns_outbuf_finalize(&outbuf, len);
@@ -151,39 +153,44 @@ static char* tns_render(void *val, size_t *len)
}
-static int tns_render_value(void *val, tns_outbuf *outbuf)
+int tns_render_value(const tns_ops *ops, void *val, tns_outbuf *outbuf)
{
tns_type_tag type = tns_tag_null;
int res = -1;
size_t orig_size = 0;
+ assert(ops != NULL && "ops struct cannot be NULL");
+
// Find out the type tag for the given value.
- type = tns_get_type(val);
+ type = ops->get_type(val);
check(type != 0, "type not serializable.");
- tns_outbuf_putc(outbuf,type);
+ tns_outbuf_putc(outbuf, type);
orig_size = tns_outbuf_size(outbuf);
// Render it into the output buffer, leaving space for the
// type tag at the end.
switch(type) {
case tns_tag_string:
- res = tns_render_string(val, outbuf);
+ res = ops->render_string(val, outbuf);
+ break;
+ case tns_tag_integer:
+ res = ops->render_integer(val, outbuf);
break;
- case tns_tag_number:
- res = tns_render_number(val, outbuf);
+ case tns_tag_float:
+ res = ops->render_float(val, outbuf);
break;
case tns_tag_bool:
- res = tns_render_bool(val, outbuf);
+ res = ops->render_bool(val, outbuf);
break;
case tns_tag_null:
res = 0;
break;
case tns_tag_dict:
- res = tns_render_dict(val, outbuf);
+ res = ops->render_dict(ops, val, outbuf);
break;
case tns_tag_list:
- res = tns_render_list(val, outbuf);
+ res = ops->render_list(ops, val, outbuf);
break;
default:
sentinel("unknown type tag: '%c'.", type);
@@ -197,7 +204,7 @@ static int tns_render_value(void *val, tns_outbuf *outbuf)
}
-static int tns_parse_list(void *val, const char *data, size_t len)
+static int tns_parse_list(const tns_ops *ops, void *val, const char *data, size_t len)
{
void *item = NULL;
char *remain = NULL;
@@ -206,11 +213,11 @@ static int tns_parse_list(void *val, const char *data, size_t len)
assert(data != NULL && "data cannot be NULL");
while(len > 0) {
- item = tns_parse(data, len, &remain);
+ item = tns_parse(ops, data, len, &remain);
check(item != NULL, "Failed to parse list.");
len = len - (remain - data);
data = remain;
- check(tns_add_to_list(val, item) != -1,
+ check(ops->add_to_list(val, item) != -1,
"Failed to add item to list.");
item = NULL;
}
@@ -219,13 +226,13 @@ static int tns_parse_list(void *val, const char *data, size_t len)
error:
if(item) {
- tns_free_value(item);
+ ops->free_value(item);
}
return -1;
}
-static int tns_parse_dict(void *val, const char *data, size_t len)
+static int tns_parse_dict(const tns_ops *ops, void *val, const char *data, size_t len)
{
void *key = NULL;
void *item = NULL;
@@ -235,17 +242,17 @@ static int tns_parse_dict(void *val, const char *data, size_t len)
assert(data != NULL && "data cannot be NULL");
while(len > 0) {
- key = tns_parse(data, len, &remain);
+ key = tns_parse(ops, data, len, &remain);
check(key != NULL, "Failed to parse dict key from tnetstring.");
len = len - (remain - data);
data = remain;
- item = tns_parse(data, len, &remain);
+ item = tns_parse(ops, data, len, &remain);
check(item != NULL, "Failed to parse dict item from tnetstring.");
len = len - (remain - data);
data = remain;
- check(tns_add_to_dict(val,key,item) != -1,
+ check(ops->add_to_dict(val, key, item) != -1,
"Failed to add element to dict.");
key = NULL;
@@ -256,10 +263,10 @@ static int tns_parse_dict(void *val, const char *data, size_t len)
error:
if(key) {
- tns_free_value(key);
+ ops->free_value(key);
}
if(item) {
- tns_free_value(item);
+ ops->free_value(item);
}
return -1;
}
@@ -278,7 +285,7 @@ tns_strtosz(const char *data, size_t len, size_t *sz, char **end)
// The first character must be a digit.
// The netstring spec explicitly forbits padding zeros.
- // If it's a zero, it must be the only char in the string.
+ // So if it's a zero, it must be the only char in the string.
c = *pos++;
switch(c) {
case '0':
@@ -328,3 +335,163 @@ tns_strtosz(const char *data, size_t len, size_t *sz, char **end)
return -1;
}
+size_t tns_outbuf_size(tns_outbuf *outbuf)
+{
+ return outbuf->alloc_size - (outbuf->head - outbuf->buffer);
+}
+
+
+static inline int tns_outbuf_itoa(tns_outbuf *outbuf, size_t n)
+{
+ do {
+ check(tns_outbuf_putc(outbuf, n%10+'0') != -1,
+ "Failed to write int to tnetstring buffer.");
+ n = n / 10;
+ } while(n > 0);
+
+ return 0;
+
+error:
+ return -1;
+}
+
+
+int tns_outbuf_init(tns_outbuf *outbuf)
+{
+ outbuf->buffer = malloc(64);
+ check_mem(outbuf->buffer);
+
+ outbuf->head = outbuf->buffer + 64;
+ outbuf->alloc_size = 64;
+ return 0;
+
+error:
+ outbuf->head = NULL;
+ outbuf->alloc_size = 0;
+ return -1;
+}
+
+
+static inline void tns_outbuf_free(tns_outbuf *outbuf)
+{
+ if(outbuf) {
+ free(outbuf->buffer);
+ outbuf->buffer = NULL;
+ outbuf->head = 0;
+ outbuf->alloc_size = 0;
+ }
+}
+
+
+static inline int tns_outbuf_extend(tns_outbuf *outbuf, size_t free_size)
+{
+ char *new_buf = NULL;
+ char *new_head = NULL;
+ size_t new_size = outbuf->alloc_size * 2;
+ size_t used_size;
+
+ used_size = tns_outbuf_size(outbuf);
+
+ while(new_size < free_size + used_size) {
+ new_size = new_size * 2;
+ }
+
+ new_buf = malloc(new_size);
+ check_mem(new_buf);
+
+ new_head = new_buf + new_size - used_size;
+ memmove(new_head, outbuf->head, used_size);
+
+ free(outbuf->buffer);
+ outbuf->buffer = new_buf;
+ outbuf->head = new_head;
+ outbuf->alloc_size = new_size;
+
+ return 0;
+
+error:
+ return -1;
+}
+
+
+int tns_outbuf_putc(tns_outbuf *outbuf, char c)
+{
+ if(outbuf->buffer == outbuf->head) {
+ check(tns_outbuf_extend(outbuf, 1) != -1, "Failed to extend buffer");
+ }
+
+ *(--outbuf->head) = c;
+
+ return 0;
+
+error:
+ return -1;
+}
+
+
+int tns_outbuf_puts(tns_outbuf *outbuf, const char *data, size_t len)
+{
+ if(outbuf->head - outbuf->buffer < len) {
+ check(tns_outbuf_extend(outbuf, len) != -1, "Failed to extend buffer");
+ }
+
+ outbuf->head -= len;
+ memmove(outbuf->head, data, len);
+
+ return 0;
+
+error:
+ return -1;
+}
+
+
+static char* tns_outbuf_finalize(tns_outbuf *outbuf, size_t *len)
+{
+ char *new_buf = NULL;
+ size_t used_size;
+
+ used_size = tns_outbuf_size(outbuf);
+
+ memmove(outbuf->buffer, outbuf->head, used_size);
+
+ if(len != NULL) {
+ *len = used_size;
+ } else {
+ if(outbuf->head == outbuf->buffer) {
+ new_buf = realloc(outbuf->buffer, outbuf->alloc_size*2);
+ check_mem(new_buf);
+ outbuf->buffer = new_buf;
+ outbuf->alloc_size = outbuf->alloc_size * 2;
+ }
+ outbuf->buffer[used_size] = '\0';
+ }
+
+ return outbuf->buffer;
+
+error:
+ free(outbuf->buffer);
+ outbuf->buffer = NULL;
+ outbuf->alloc_size = 0;
+ return NULL;
+}
+
+
+static inline int tns_outbuf_clamp(tns_outbuf *outbuf, size_t orig_size)
+{
+ size_t datalen = tns_outbuf_size(outbuf) - orig_size;
+
+ check(tns_outbuf_putc(outbuf, ':') != -1, "Failed to clamp outbuf");
+ check(tns_outbuf_itoa(outbuf, datalen) != -1, "Failed to clamp outbuf");
+
+ return 0;
+
+error:
+ return -1;
+}
+
+
+void tns_outbuf_memmove(tns_outbuf *outbuf, char *dest)
+{
+ memmove(dest, outbuf->head, tns_outbuf_size(outbuf));
+}
+
View
91 tnetstring/tns_core.h
@@ -5,10 +5,6 @@
// typed-netstring format proposed for inclusion in Mongrel2. You can
// think of it like a JSON library that uses a simpler wire format.
//
-// This code is *not* designed to be compiled as a standalone library.
-// Instead, you provide a suite of low-level data manipulation functions
-// and then #include "tns_core.c" to stitch them into a tnetstring parser.
-//
#ifndef _tns_core_h
#define _tns_core_h
@@ -20,7 +16,7 @@
// tnetstring rendering is done using an "outbuf" struct, which combines
// a malloced string with its allocation information. Rendering is done
// from front to back; the details are deliberately hidden here since
-// I'm experimenting with multiple implementations.
+// I'm experimenting with multiple implementations and it might change.
struct tns_outbuf_s;
typedef struct tns_outbuf_s tns_outbuf;
@@ -28,7 +24,8 @@ typedef struct tns_outbuf_s tns_outbuf;
// tnetstring encoding.
typedef enum tns_type_tag_e {
tns_tag_string = ',',
- tns_tag_number = '#',
+ tns_tag_integer = '#',
+ tns_tag_float = '^',
tns_tag_bool = '!',
tns_tag_null = '~',
tns_tag_dict = '}',
@@ -36,49 +33,64 @@ typedef enum tns_type_tag_e {
} tns_type_tag;
-// You must provide implementations for the following functions.
-// They provide the low-level data manipulation routines from which
-// we can build a parser and renderer.
+// To convert between tnetstrings and the data structures of your application
+// you provide the following struct filled with function pointers. They
+// will be called by the core parser/renderer as necessary.
+
+struct tns_ops_s;
+typedef struct tns_ops_s tns_ops;
+
+struct tns_ops_s {
+
+ // Get the type of a data object.
+ tns_type_tag (*get_type)(void *val);
+
+ // Parse various types of object from a string.
+ void* (*parse_string)(const char *data, size_t len);
+ void* (*parse_integer)(const char *data, size_t len);
+ void* (*parse_float)(const char *data, size_t len);
-// Functions to introspect the type of a data object.
-static tns_type_tag tns_get_type(void *val);
+ // Constructors for constant primitive datatypes.
+ void* (*get_null)(void);
+ void* (*get_true)(void);
+ void* (*get_false)(void);
-// Functions for parsing and rendering primitive datatypes.
-static void *tns_parse_string(const char *data, size_t len);
-static int tns_render_string(void *val, tns_outbuf *outbuf);
-static void *tns_parse_number(const char *data, size_t len);
-static int tns_render_number(void *val, tns_outbuf *outbuf);
-static int tns_render_bool(void *val, tns_outbuf *outbuf);
+ // Render various types of object into a tns_outbuf.
+ int (*render_string)(void *val, tns_outbuf *outbuf);
+ int (*render_integer)(void *val, tns_outbuf *outbuf);
+ int (*render_float)(void *val, tns_outbuf *outbuf);
+ int (*render_bool)(void *val, tns_outbuf *outbuf);
-// Constructors to get constant primitive datatypes.
-static void *tns_get_null(void);
-static void *tns_get_true(void);
-static void *tns_get_false(void);
+ // Functions for building and rendering list values.
+ // Remember that rendering is done from back to front, so
+ // you must write the last list element first.
+ void* (*new_list)(void);
+ int (*add_to_list)(void* list, void* item);
+ int (*render_list)(const tns_ops *ops, void* list, tns_outbuf *outbuf);
-// Functions for manipulating compound datatypes.
-static void *tns_new_dict(void);
-static int tns_add_to_dict(void *dict, void *key, void *item);
-static int tns_render_dict(void *dict, tns_outbuf *outbuf);
-static void *tns_new_list(void);
-static int tns_add_to_list(void *list, void *item);
-static int tns_render_list(void *dict, tns_outbuf *outbuf);
+ // Functions for building and rendering dict values
+ // Remember that rendering is done from back to front, so
+ // you must write each value first, follow by its key.
+ void* (*new_dict)(void);
+ int (*add_to_dict)(void* dict, void* key, void* item);
+ int (*render_dict)(const tns_ops *ops, void* dict, tns_outbuf *outbuf);
-// Functions for manaing value lifecycle
-static void tns_free_value(void *value);
+ // Free values that are no longer in use
+ void (*free_value)(void *value);
+};
-// In return, you get the following functions.
// Parse an object off the front of a tnetstring.
// Returns a pointer to the parsed object, or NULL if an error occurs.
// The third argument is an output parameter; if non-NULL it will
// receive the unparsed remainder of the string.
-static void* tns_parse(const char *data, size_t len, char** remain);
+extern void* tns_parse(const tns_ops *ops, const char *data, size_t len, char** remain);
// If you need to read the length prefix yourself, e.g. because you're
// reading data off a socket, you can use this function to get just
// the payload parsing logic.
-static void* tns_parse_payload(tns_type_tag type, const char *data, size_t len);
+extern void* tns_parse_payload(const tns_ops *ops, tns_type_tag type, const char *data, size_t len);
// Render an object into a string.
// On success this function returns a malloced string containing
@@ -87,15 +99,20 @@ static void* tns_parse_payload(tns_type_tag type, const char *data, size_t len);
// the string; if NULL then the string will be null-terminated.
// The caller is responsible for freeing the returned string.
// On failure this function returns NULL and 'len' is unmodified.
-static char* tns_render(void *val, size_t *len);
+extern char* tns_render(const tns_ops *ops, void *val, size_t *len);
// If you need to copy the final result off somewhere else, you
// might like to build your own rendering function from the following.
// It will avoid some double-copying that tns_render does internally.
// Basic plan: Initialize an outbuf, pass it to tns_render_value, then
// copy the bytes away using tns_outbuf_memmove.
-static int tns_outbuf_init(tns_outbuf *outbuf);
-static int tns_render_value(void *val, tns_outbuf *outbuf);
-static void tns_outbuf_memmove(tns_outbuf *outbuf, char *dest);
+extern int tns_render_value(const tns_ops *ops, void *val, tns_outbuf *outbuf);
+extern int tns_outbuf_init(tns_outbuf *outbuf);
+extern void tns_outbuf_memmove(tns_outbuf *outbuf, char *dest);
+
+// Use these functions for rendering into an outbuf.
+extern size_t tns_outbuf_size(tns_outbuf *outbuf);
+extern int tns_outbuf_putc(tns_outbuf *outbuf, char c);
+extern int tns_outbuf_puts(tns_outbuf *outbuf, const char *data, size_t len);
#endif
View
178 tnetstring/tns_outbuf_back.c
@@ -1,178 +0,0 @@
-//
-// tns_outbuf_back: tns_outbuf implemented by writing from back of buffer.
-//
-// This outbuf implementation writes data starting at the back of the
-// allocated buffer. To finalize it you need to memmove it to the start
-// of the buffer so it can be treated like an ordinary malloced string.
-//
-// The advantage of this scheme is that the data is the right way round,
-// so you can shuffle it about using memmove. The disadvantage is that
-// reallocating the buffer is tricker as you must move the data by hand.
-// On my machines, the ability to use memmove seems to be a very slight
-// win over tns_outbuf_rev.c.
-//
-
-struct tns_outbuf_s {
- char *buffer;
- char *head;
- size_t alloc_size;
-};
-
-static inline size_t tns_outbuf_size(tns_outbuf *outbuf)
-{
- return outbuf->alloc_size - (outbuf->head - outbuf->buffer);
-}
-
-static inline int tns_outbuf_itoa(tns_outbuf *outbuf, size_t n)
-{
- do {
- check(tns_outbuf_putc(outbuf, n%10+'0') != -1,
- "Failed to write int to tnetstring buffer.");
- n = n / 10;
- } while(n > 0);
-
- return 0;
-
-error:
- return -1;
-}
-
-
-static inline int tns_outbuf_init(tns_outbuf *outbuf)
-{
- outbuf->buffer = malloc(64);
- check_mem(outbuf->buffer);
-
- outbuf->head = outbuf->buffer + 64;
- outbuf->alloc_size = 64;
- return 0;
-
-error:
- outbuf->head = NULL;
- outbuf->alloc_size = 0;
- return -1;
-}
-
-
-static inline void tns_outbuf_free(tns_outbuf *outbuf)
-{
- if(outbuf) {
- free(outbuf->buffer);
- outbuf->buffer = NULL;
- outbuf->head = 0;
- outbuf->alloc_size = 0;
- }
-}
-
-
-static inline int tns_outbuf_extend(tns_outbuf *outbuf, size_t free_size)
-{
- char *new_buf = NULL;
- char *new_head = NULL;
- size_t new_size = outbuf->alloc_size * 2;
- size_t used_size;
-
- used_size = tns_outbuf_size(outbuf);
-
- while(new_size < free_size + used_size) {
- new_size = new_size * 2;
- }
-
- new_buf = malloc(new_size);
- check_mem(new_buf);
-
- new_head = new_buf + new_size - used_size;
- memmove(new_head, outbuf->head, used_size);
-
- free(outbuf->buffer);
- outbuf->buffer = new_buf;
- outbuf->head = new_head;
- outbuf->alloc_size = new_size;
-
- return 0;
-
-error:
- return -1;
-}
-
-
-static inline int tns_outbuf_putc(tns_outbuf *outbuf, char c)
-{
- if(outbuf->buffer == outbuf->head) {
- check(tns_outbuf_extend(outbuf, 1) != -1, "Failed to extend buffer");
- }
-
- *(--outbuf->head) = c;
-
- return 0;
-
-error:
- return -1;
-}
-
-
-static int tns_outbuf_puts(tns_outbuf *outbuf, const char *data, size_t len)
-{
- if(outbuf->head - outbuf->buffer < len) {
- check(tns_outbuf_extend(outbuf, len) != -1, "Failed to extend buffer");
- }
-
- outbuf->head -= len;
- memmove(outbuf->head, data, len);
-
- return 0;
-
-error:
- return -1;
-}
-
-static char* tns_outbuf_finalize(tns_outbuf *outbuf, size_t *len)
-{
- char *new_buf = NULL;
- size_t used_size;
-
- used_size = tns_outbuf_size(outbuf);
-
- memmove(outbuf->buffer, outbuf->head, used_size);
-
- if(len != NULL) {
- *len = used_size;
- } else {
- if(outbuf->head == outbuf->buffer) {
- new_buf = realloc(outbuf->buffer, outbuf->alloc_size*2);
- check_mem(new_buf);
- outbuf->buffer = new_buf;
- outbuf->alloc_size = outbuf->alloc_size * 2;
- }
- outbuf->buffer[used_size] = '\0';
- }
-
- return outbuf->buffer;
-
-error:
- free(outbuf->buffer);
- outbuf->buffer = NULL;
- outbuf->alloc_size = 0;
- return NULL;
-}
-
-
-static inline int tns_outbuf_clamp(tns_outbuf *outbuf, size_t orig_size)
-{
- size_t datalen = tns_outbuf_size(outbuf) - orig_size;
-
- check(tns_outbuf_putc(outbuf, ':') != -1, "Failed to clamp outbuf");
- check(tns_outbuf_itoa(outbuf, datalen) != -1, "Failed to clamp outbuf");
-
- return 0;
-
-error:
- return -1;
-}
-
-
-static inline void tns_outbuf_memmove(tns_outbuf *outbuf, char *dest)
-{
- memmove(dest, outbuf->head, tns_outbuf_size(outbuf));
-}
-
View
182 tnetstring/tns_outbuf_rev.c
@@ -1,182 +0,0 @@
-//
-// tns_outbuf_rev: tns_outbuf implemented by writing in reverse.
-//
-// This outbuf implementation writes data starting at the front of the
-// allocated buffer, but writes it in reverse. To finalize it you do
-// an in-place reverse before returning the buffer.
-//
-// The advantage of this scheme is that the data is at the beginning
-// of the allocated buffer, so you can extend it using realloc and pass
-// it around as if it were a normal string. However, the final copying
-// of the data has to be done in reverse so we can't use memmove.
-
-struct tns_outbuf_s {
- char *buffer;
- size_t used_size;
- size_t alloc_size;
-};
-
-
-static inline size_t tns_outbuf_size(tns_outbuf *outbuf)
-{
- return outbuf->used_size;
-}
-
-
-static inline void tns_inplace_reverse(char *data, size_t len)
-{
- char *dend = NULL;
- char c = '0';
-
- dend = data + len - 1;
- while(dend > data) {
- c = *data;
- *data = *dend;
- *dend = c;
- data++;
- dend--;
- }
-}
-
-
-static inline int tns_outbuf_itoa(tns_outbuf *outbuf, size_t n)
-{
- do {
- check(tns_outbuf_putc(outbuf, n%10+'0') != -1,
- "Failed to write int to tnetstring buffer.");
- n = n / 10;
- } while(n > 0);
-
- return 0;
-
-error:
- return -1;
-}
-
-
-static inline int tns_outbuf_init(tns_outbuf *outbuf)
-{
- outbuf->buffer = malloc(64);
- check_mem(outbuf->buffer);
-
- outbuf->alloc_size = 64;
- outbuf->used_size = 0;
- return 0;
-
-error:
- outbuf->alloc_size = 0;
- outbuf->used_size = 0;
- return -1;
-}
-
-
-static inline void tns_outbuf_free(tns_outbuf *outbuf)
-{
- if(outbuf) {
- free(outbuf->buffer);
- outbuf->buffer = NULL;
- outbuf->alloc_size = 0;
- outbuf->used_size = 0;
- }
-}
-
-
-static inline int tns_outbuf_extend(tns_outbuf *outbuf, size_t free_size)
-{
- char *new_buf = NULL;
- size_t new_size = outbuf->alloc_size * 2;
-
- while(new_size < free_size + outbuf->used_size) {
- new_size = new_size * 2;
- }
-
- new_buf = realloc(outbuf->buffer, new_size);
- check_mem(new_buf);
-
- outbuf->buffer = new_buf;
- outbuf->alloc_size = new_size;
-
- return 0;
-
-error:
- return -1;
-}
-
-
-static inline int tns_outbuf_putc(tns_outbuf *outbuf, char c)
-{
- if(outbuf->alloc_size == outbuf->used_size) {
- check(tns_outbuf_extend(outbuf, 1) != -1, "Failed to extend buffer");
- }
-
- outbuf->buffer[outbuf->used_size++] = c;
-
- return 0;
-
-error:
- return -1;
-}
-
-
-static int tns_outbuf_puts(tns_outbuf *outbuf, const char *data, size_t len)
-{
- const char *dend = NULL;
- char *buffer = NULL;
-
- // Make sure we have enough room.
- if(outbuf->alloc_size - outbuf->used_size < len) {
- check(tns_outbuf_extend(outbuf, len) != -1, "Failed to extend buffer");
- }
-
- // Copy the data in reverse.
- buffer = outbuf->buffer + outbuf->used_size;
- dend = data + len - 1;
- while(dend >= data) {
- *buffer = *dend;
- buffer++;
- dend--;
- }
-
- outbuf->used_size += len;
-
- return 0;
-
-error:
- return -1;
-}
-
-static char* tns_outbuf_finalize(tns_outbuf *outbuf, size_t *len)
-{
- tns_inplace_reverse(outbuf->buffer, outbuf->used_size);
- if(len == NULL) {
- tns_outbuf_putc(outbuf, '\0');
- } else {
- *len = outbuf->used_size;
- }
- return outbuf->buffer;
-}
-
-
-static inline int tns_outbuf_clamp(tns_outbuf *outbuf, size_t orig_size)
-{
- size_t datalen = outbuf->used_size - orig_size;
-
- check(tns_outbuf_putc(outbuf, ':') != -1, "Failed to clamp outbuf");
- check(tns_outbuf_itoa(outbuf, datalen) != -1, "Failed to clamp outbuf");
-
- return 0;
-
-error:
- return -1;
-}
-
-static inline void tns_outbuf_memmove(tns_outbuf *outbuf, char *dest)
-{
- char *buffer = outbuf->buffer + outbuf->used_size - 1;
- while(buffer >= outbuf->buffer) {
- *dest = *buffer;
- dest++;
- buffer--;
- }
-}
-
Please sign in to comment.
Something went wrong with that request. Please try again.