Skip to content

Commit

Permalink
feat(radix): Reduce memory footprint of the C implementation
Browse files Browse the repository at this point in the history
  • Loading branch information
@Darkheir
Darkheir committed Jul 9, 2020
1 parent 52d70b0 commit 2a56665
Show file tree
Hide file tree
Showing 2 changed files with 30 additions and 43 deletions.
37 changes: 18 additions & 19 deletions README.rst
View file
@@ -1,11 +1,14 @@
py-radix
========

.. image:: https://travis-ci.org/mjschultz/py-radix.svg?branch=master
:target: https://travis-ci.org/mjschultz/py-radix
Fork of the original library to reduce the memory footprint of the C implementation.

.. image:: https://coveralls.io/repos/mjschultz/py-radix/badge.png?branch=master
:target: https://coveralls.io/r/mjschultz/py-radix?branch=master
In order to reduce the memory usage the following attributes have been removed from the `Node` objects:

* network
* prefix

The `data` object is now `None` by default instead of being an empty dict. It can be used to store any type of data.

py-radix implements the radix tree data structure for the storage and
retrieval of IPv4 and IPv6 network prefixes.
Expand Down Expand Up @@ -37,15 +40,17 @@ Usage

A simple example that demonstrates most of the features: ::

import socket
import radix


# Create a new tree
rtree = radix.Radix()

# Adding a node returns a RadixNode object. You can create
# arbitrary members in its 'data' dict to store your data
rnode = rtree.add("10.0.0.0/8")
rnode.data["blah"] = "whatever you want"
rnode.data = {"blah": "whatever you want"}

# You can specify nodes as CIDR addresses, or networks with
# separate mask lengths. The following three invocations are
Expand All @@ -59,16 +64,16 @@ A simple example that demonstrates most of the features: ::
# functions. In this case, the radix module will assume that
# a four-byte address is an IPv4 address and a sixteen-byte
# address is an IPv6 address. For example:
binary_addr = inet_ntoa("172.18.22.0")
binary_addr = socket.inet_aton("172.18.22.0")
rnode = rtree.add(packed = binary_addr, masklen = 23)

# Exact search will only return prefixes you have entered
# You can use all of the above ways to specify the address
rnode = rtree.search_exact("10.0.0.0/8")
# Get your data back out
print rnode.data["blah"]
print(rnode.data["blah"])
# Use a packed address
addr = socket.inet_ntoa("10.0.0.0")
addr = socket.inet_aton("10.0.0.0")
rnode = rtree.search_exact(packed = addr, masklen = 8)

# Best-match search will return the longest matching prefix
Expand All @@ -85,11 +90,9 @@ A simple example that demonstrates most of the features: ::
rnodes = rtree.search_covered("10.123.0.0/16")

# There are a couple of implicit members of a RadixNode:
print rnode.network # -> "10.0.0.0"
print rnode.prefix # -> "10.0.0.0/8"
print rnode.prefixlen # -> 8
print rnode.family # -> socket.AF_INET
print rnode.packed # -> '\n\x00\x00\x00'
print(rnode.family) # -> socket.AF_INET
print(rnode.packed) # -> '\n\x00\x00\x00'
print(rnode.data) # -> {'blah': 'whatever you want'}

# IPv6 prefixes are fully supported in the same tree
rnode = rtree.add("2001:DB8::/3")
Expand All @@ -98,11 +101,7 @@ A simple example that demonstrates most of the features: ::
# Use the nodes() method to return all RadixNodes created
nodes = rtree.nodes()
for rnode in nodes:
print rnode.prefix

# The prefixes() method will return all the prefixes (as a
# list of strings) that have been entered
prefixes = rtree.prefixes()
print(rnode.packed)

# You can also directly iterate over the tree itself
# this would save some memory if the tree is big
Expand All @@ -111,7 +110,7 @@ A simple example that demonstrates most of the features: ::
# receive a RuntimeWarning. Changing a node's data dict
# is permitted.
for rnode in rtree:
print rnode.prefix
print(rnode.packed)


License
Expand Down
36 changes: 12 additions & 24 deletions radix/_radix.c
View file
Expand Up @@ -50,8 +50,6 @@ PyObject *radix_constructor;
typedef struct {
PyObject_HEAD
PyObject *user_attr; /* User-specified attributes */
PyObject *network;
PyObject *prefix;
PyObject *prefixlen;
PyObject *family;
PyObject *packed;
Expand All @@ -64,7 +62,6 @@ static RadixNodeObject *
newRadixNodeObject(radix_node_t *rn)
{
RadixNodeObject *self;
char network[256], prefix[256];

/* Sanity check */
if (rn == NULL || rn->prefix == NULL ||
Expand All @@ -77,24 +74,16 @@ newRadixNodeObject(radix_node_t *rn)

self->rn = rn;

/* Format addresses for packing into objects */
prefix_addr_ntop(rn->prefix, network, sizeof(network));
prefix_ntop(rn->prefix, prefix, sizeof(prefix));

self->user_attr = PyDict_New();
self->network = PyString_FromString(network);
self->prefix = PyString_FromString(prefix);
self->user_attr = NULL;
self->prefixlen = PyInt_FromLong(rn->prefix->bitlen);
self->family = PyInt_FromLong(rn->prefix->family);
self->packed = PyString_FromStringAndSize((char*)&rn->prefix->add,
rn->prefix->family == AF_INET ? 4 : 16);

if (self->user_attr == NULL || self->prefixlen == NULL ||
self->family == NULL || self->network == NULL ||
self->prefix == NULL) {
/* RadixNode_dealloc will clean up for us */
Py_XDECREF(self);
return (NULL);
if (self->family == NULL || self->prefixlen == NULL) {
/* RadixNode_dealloc will clean up for us */
Py_XDECREF(self);
return (NULL);
}

return self;
Expand All @@ -108,8 +97,6 @@ RadixNode_dealloc(RadixNodeObject *self)
Py_XDECREF(self->user_attr);
Py_XDECREF(self->prefixlen);
Py_XDECREF(self->family);
Py_XDECREF(self->network);
Py_XDECREF(self->prefix);
Py_XDECREF(self->packed);
PyObject_Del(self);
}
Expand Down Expand Up @@ -139,9 +126,7 @@ Radix_parent(RadixNodeObject *self, void *closure)
Py_RETURN_NONE;
}
static PyMemberDef RadixNode_members[] = {
{"data", T_OBJECT, offsetof(RadixNodeObject, user_attr), READONLY},
{"network", T_OBJECT, offsetof(RadixNodeObject, network), READONLY},
{"prefix", T_OBJECT, offsetof(RadixNodeObject, prefix), READONLY},
{"data", T_OBJECT, offsetof(RadixNodeObject, user_attr), 0},
{"prefixlen", T_OBJECT, offsetof(RadixNodeObject, prefixlen), READONLY},
{"family", T_OBJECT, offsetof(RadixNodeObject, family), READONLY},
{"packed", T_OBJECT, offsetof(RadixNodeObject, packed), READONLY},
Expand Down Expand Up @@ -524,7 +509,7 @@ add_node_to_list(radix_node_t *node, void *arg)
PyObject *ret = arg;

if (node->data != NULL)
PyList_Append(ret, ((RadixNodeObject *)node->data));
PyList_Append(ret, (PyObject *)((RadixNodeObject *)node->data));
return (0);
}

Expand Down Expand Up @@ -643,8 +628,11 @@ Radix_prefixes(RadixObject *self, PyObject *args)

RADIX_TREE_WALK(self->rt, node) {
if (node->data != NULL) {
PyList_Append(ret,
((RadixNodeObject *)node->data)->prefix);
char buf[256];
PyObject *prefix = PyString_FromString(
prefix_ntop(node->prefix, buf, sizeof(buf))
);
PyList_Append(ret, prefix);
}
} RADIX_TREE_WALK_END;

Expand Down

0 comments on commit 2a56665

Please sign in to comment.