diff --git a/.clang-format b/.clang-format new file mode 100644 index 0000000..a72e2df --- /dev/null +++ b/.clang-format @@ -0,0 +1,5 @@ +BasedOnStyle: LLVM +IndentWidth: 4 +BinPackArguments: false +BinPackParameters: false +IndentCaseLabels: true diff --git a/.uncrustify.cfg b/.uncrustify.cfg deleted file mode 100644 index 46b4428..0000000 --- a/.uncrustify.cfg +++ /dev/null @@ -1,78 +0,0 @@ -# -# based on uncrustify config file for the linux kernel -# - -code_width = 80 -indent_case_brace = 4 -indent_columns = 4 -indent_label = 2 # pos: absolute col, neg: relative column -indent_with_tabs = 0 - -# -# inter-symbol newlines -# -nl_brace_else = remove # "} else" vs "} \n else" - cuddle else -nl_brace_while = remove # "} while" vs "} \n while" - cuddle while -nl_do_brace = remove # "do {" vs "do \n {" -nl_else_brace = remove # "else {" vs "else \n {" -nl_enum_brace = remove # "enum {" vs "enum \n {" -nl_fcall_brace = remove # "list_for_each() {" vs "list_for_each()\n{" -nl_fdef_brace = force # "int foo() {" vs "int foo()\n{" -nl_for_brace = remove # "for () {" vs "for () \n {" -nl_func_var_def_blk = 0 # don't add newlines after a block of var declarations -nl_if_brace = remove # "if () {" vs "if () \n {" -nl_multi_line_define = true -nl_struct_brace = remove # "struct {" vs "struct \n {" -nl_switch_brace = remove # "switch () {" vs "switch () \n {" -nl_union_brace = remove # "union {" vs "union \n {" -nl_while_brace = remove # "while () {" vs "while () \n {" - - -# -# Source code modifications -# -mod_full_brace_do = force # "do a--; while ();" vs "do { a--; } while ();" -mod_full_brace_for = force # "for () a--;" vs "for () { a--; }" -mod_full_brace_if = force # "if (a) a--;" vs "if (a) { a--; }" -mod_full_brace_nl = 3 # don't remove if more than 3 newlines -mod_full_brace_while = force # "while (a) a--;" vs "while (a) { a--; }" -mod_paren_on_return = remove # "return 1;" vs "return (1);" - - -# -# inter-character spacing options -# -sp_after_cast = remove # "(int) a" vs "(int)a" -sp_after_comma = force -sp_after_sparen = force # "if () {" vs "if (){" -sp_arith = force -sp_assign = force -sp_assign = force -sp_before_comma = remove -sp_before_ptr_star = force # "char *foo" vs "char* foo -sp_before_sparen = force # "if (" vs "if(" -sp_between_ptr_star = remove # "char * *foo" vs "char **foo" -sp_bool = force -sp_compare = force -sp_func_call_paren = remove # "foo (" vs "foo(" -sp_func_def_paren = remove # "int foo (){" vs "int foo(){" -sp_func_proto_paren = remove # "int foo ();" vs "int foo();" -sp_inside_braces = force # "{ 1 }" vs "{1}" -sp_inside_braces_enum = force # "{ 1 }" vs "{1}" -sp_inside_braces_struct = force # "{ 1 }" vs "{1}" -sp_inside_sparen = remove -sp_paren_brace = force -sp_sizeof_paren = remove # "sizeof (int)" vs "sizeof(int)" - -# -# Aligning stuff -# -align_enum_equ_span = 4 # '=' in enum definition -align_nl_cont = true -align_on_tabstop = FALSE # align on tabstops -align_right_cmt_span = 3 -align_struct_init_span = 1 -align_struct_init_span = 3 # align stuff in a structure init '= { }' -align_var_def_star_style = 2 # void *foo; -align_var_struct_span = 0 -align_with_tabs = FALSE # use tabs to align diff --git a/HISTORY.rst b/HISTORY.rst index d648bcd..3374ca0 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -14,6 +14,8 @@ History ``ipaddress.IPv6Address`` objects in addition to strings. This works with both the pure Python implementation as well as the extension. Based on a pull request #48 by Eric Pruitt. GitHub #50. +* A new method, ``get_with_prefix_len`` was added. This method returns a + tuple containing the record and the prefix length. 1.4.1 (2018-06-22) ++++++++++++++++++ diff --git a/README.rst b/README.rst index ab1b9a7..e905493 100644 --- a/README.rst +++ b/README.rst @@ -62,6 +62,10 @@ corresponding values for the IP address from the database (e.g., a dictionary for GeoIP2/GeoLite2 databases). If the database does not contain a record for that IP address, the method will return ``None``. +If you wish to also retrieve the prefix length for the record, use the +``get_with_prefix_len`` method. This returns a tuple containing the record +followed by the network prefix length associated with the record. + Example ------- @@ -70,9 +74,13 @@ Example >>> import maxminddb >>> >>> reader = maxminddb.open_database('GeoLite2-City.mmdb') + >>> >>> reader.get('1.1.1.1') {'country': ... } >>> + >>> reader.get_with_prefix_len('1.1.1.1') + ({'country': ... }, 24) + >>> >>> reader.close() Exceptions diff --git a/dev-bin/clang-format-all.sh b/dev-bin/clang-format-all.sh new file mode 100755 index 0000000..71a1bdf --- /dev/null +++ b/dev-bin/clang-format-all.sh @@ -0,0 +1,15 @@ +#!/bin/sh + +format="clang-format -i -style=file" + +for dir in extension; do + c_files=`find $dir -maxdepth 1 -name '*.c'` + if [ "$c_files" != "" ]; then + $format $dir/*.c; + fi + + h_files=`find $dir -maxdepth 1 -name '*.h'` + if [ "$h_files" != "" ]; then + $format $dir/*.h; + fi +done diff --git a/docs/conf.py b/docs/conf.py index 9b107b4..f0ae2c9 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -51,7 +51,7 @@ # General information about the project. project = 'maxminddb' -copyright = '2013-2018, MaxMind, Inc.' +copyright = '2013-2019, MaxMind, Inc.' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the diff --git a/docs/index.rst b/docs/index.rst index a003c74..20fdb5a 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -35,6 +35,6 @@ Indices and tables * :ref:`modindex` * :ref:`search` -:copyright: (c) 2013-2018 by MaxMind, Inc. +:copyright: (c) 2013-2019 by MaxMind, Inc. :license: Apache License, Version 2.0 diff --git a/extension/maxminddb.c b/extension/maxminddb.c index 94114b1..f24095e 100644 --- a/extension/maxminddb.c +++ b/extension/maxminddb.c @@ -1,9 +1,9 @@ #include -#include #include +#include #include +#include #include -#include "structmember.h" #define __STDC_FORMAT_MACROS #include @@ -12,14 +12,15 @@ static PyTypeObject Reader_Type; static PyTypeObject Metadata_Type; static PyObject *MaxMindDB_error; +// clang-format off typedef struct { - PyObject_HEAD /* no semicolon */ + PyObject_HEAD /* no semicolon */ MMDB_s *mmdb; PyObject *closed; } Reader_obj; typedef struct { - PyObject_HEAD /* no semicolon */ + PyObject_HEAD /* no semicolon */ PyObject *binary_format_major_version; PyObject *binary_format_minor_version; PyObject *build_epoch; @@ -30,7 +31,9 @@ typedef struct { PyObject *node_count; PyObject *record_size; } Metadata_obj; +// clang-format on +static int get_record(PyObject *self, PyObject *args, PyObject **record); static bool format_sockaddr(struct sockaddr *addr, char *dst); static PyObject *from_entry_data_list(MMDB_entry_data_list_s **entry_data_list); static PyObject *from_map(MMDB_entry_data_list_s **entry_data_list); @@ -39,43 +42,44 @@ static PyObject *from_uint128(const MMDB_entry_data_list_s *entry_data_list); static int ip_converter(PyObject *obj, struct sockaddr_storage *ip_address); #if PY_MAJOR_VERSION >= 3 - #define MOD_INIT(name) PyMODINIT_FUNC PyInit_ ## name(void) - #define RETURN_MOD_INIT(m) return (m) - #define FILE_NOT_FOUND_ERROR PyExc_FileNotFoundError +#define MOD_INIT(name) PyMODINIT_FUNC PyInit_##name(void) +#define RETURN_MOD_INIT(m) return (m) +#define FILE_NOT_FOUND_ERROR PyExc_FileNotFoundError #else - #define MOD_INIT(name) PyMODINIT_FUNC init ## name(void) - #define RETURN_MOD_INIT(m) return - #define PyInt_FromLong PyLong_FromLong - #define FILE_NOT_FOUND_ERROR PyExc_IOError +#define MOD_INIT(name) PyMODINIT_FUNC init##name(void) +#define RETURN_MOD_INIT(m) return +#define PyInt_FromLong PyLong_FromLong +#define FILE_NOT_FOUND_ERROR PyExc_IOError #endif #ifdef __GNUC__ - # define UNUSED(x) UNUSED_ ## x __attribute__((__unused__)) +#define UNUSED(x) UNUSED_##x __attribute__((__unused__)) #else - # define UNUSED(x) UNUSED_ ## x +#define UNUSED(x) UNUSED_##x #endif -static int Reader_init(PyObject *self, PyObject *args, PyObject *kwds) -{ +static int Reader_init(PyObject *self, PyObject *args, PyObject *kwds) { char *filename; int mode = 0; static char *kwlist[] = {"database", "mode", NULL}; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "s|i", kwlist, &filename, &mode)) { + if (!PyArg_ParseTupleAndKeywords( + args, kwds, "s|i", kwlist, &filename, &mode)) { return -1; } if (mode != 0 && mode != 1) { - PyErr_Format(PyExc_ValueError, "Unsupported open mode (%i). Only " + PyErr_Format( + PyExc_ValueError, + "Unsupported open mode (%i). Only " "MODE_AUTO and MODE_MMAP_EXT are supported by this extension.", mode); return -1; } if (0 != access(filename, R_OK)) { - PyErr_Format(FILE_NOT_FOUND_ERROR, - "No such file or directory: '%s'", - filename); + PyErr_Format( + FILE_NOT_FOUND_ERROR, "No such file or directory: '%s'", filename); return -1; } @@ -96,11 +100,10 @@ static int Reader_init(PyObject *self, PyObject *args, PyObject *kwds) if (MMDB_SUCCESS != status) { free(mmdb); - PyErr_Format( - MaxMindDB_error, - "Error opening database file (%s). Is this a valid MaxMind DB file?", - filename - ); + PyErr_Format(MaxMindDB_error, + "Error opening database file (%s). Is this a valid " + "MaxMind DB file?", + filename); return -1; } @@ -109,26 +112,41 @@ static int Reader_init(PyObject *self, PyObject *args, PyObject *kwds) return 0; } -static PyObject *Reader_get(PyObject *self, PyObject *args) -{ - MMDB_s *mmdb = ((Reader_obj *)self)->mmdb; +static PyObject *Reader_get(PyObject *self, PyObject *args) { + PyObject *record = NULL; + if (get_record(self, args, &record) == -1) { + return NULL; + } + return record; +} + +static PyObject *Reader_get_with_prefix_len(PyObject *self, PyObject *args) { + PyObject *record = NULL; + int prefix_len = get_record(self, args, &record); + if (prefix_len == -1) { + return NULL; + } + + return PyTuple_Pack(2, record, PyLong_FromLong(prefix_len)); +} +static int get_record(PyObject *self, PyObject *args, PyObject **record) { + MMDB_s *mmdb = ((Reader_obj *)self)->mmdb; if (NULL == mmdb) { PyErr_SetString(PyExc_ValueError, "Attempt to read from a closed MaxMind DB."); - return NULL; + return -1; } - struct sockaddr_storage ip_address_ss = { 0 }; + struct sockaddr_storage ip_address_ss = {0}; struct sockaddr *ip_address = (struct sockaddr *)&ip_address_ss; if (!PyArg_ParseTuple(args, "O&", ip_converter, &ip_address_ss)) { - return NULL; + return -1; } if (!ip_address->sa_family) { - PyErr_SetString(PyExc_ValueError, - "Error parsing argument"); - return NULL; + PyErr_SetString(PyExc_ValueError, "Error parsing argument"); + return -1; } int mmdb_error = MMDB_SUCCESS; @@ -142,44 +160,55 @@ static PyObject *Reader_get(PyObject *self, PyObject *args) } else { exception = MaxMindDB_error; } - char ipstr[INET6_ADDRSTRLEN] = { 0 }; + char ipstr[INET6_ADDRSTRLEN] = {0}; if (format_sockaddr(ip_address, ipstr)) { - PyErr_Format(exception, "Error looking up %s. %s", - ipstr, MMDB_strerror(mmdb_error)); + PyErr_Format(exception, + "Error looking up %s. %s", + ipstr, + MMDB_strerror(mmdb_error)); } - return NULL; + return -1; + } + + int prefix_len = result.netmask; + if (ip_address->sa_family == AF_INET && mmdb->metadata.ip_version == 6) { + // We return the prefix length given the IPv4 address. If there is + // no IPv4 subtree, we return a prefix length of 0. + prefix_len = prefix_len >= 96 ? prefix_len - 96 : 0; } if (!result.found_entry) { - Py_RETURN_NONE; + *record = Py_None; + return prefix_len; } MMDB_entry_data_list_s *entry_data_list = NULL; int status = MMDB_get_entry_data_list(&result.entry, &entry_data_list); if (MMDB_SUCCESS != status) { - char ipstr[INET6_ADDRSTRLEN] = { 0 }; + char ipstr[INET6_ADDRSTRLEN] = {0}; if (format_sockaddr(ip_address, ipstr)) { PyErr_Format(MaxMindDB_error, "Error while looking up data for %s. %s", - ipstr, MMDB_strerror(status)); + ipstr, + MMDB_strerror(status)); } MMDB_free_entry_data_list(entry_data_list); - return NULL; + return -1; } MMDB_entry_data_list_s *original_entry_data_list = entry_data_list; - PyObject *py_obj = from_entry_data_list(&entry_data_list); + *record = from_entry_data_list(&entry_data_list); MMDB_free_entry_data_list(original_entry_data_list); - return py_obj; + + return prefix_len; } -static int ip_converter(PyObject *obj, struct sockaddr_storage *ip_address) -{ +static int ip_converter(PyObject *obj, struct sockaddr_storage *ip_address) { #if PY_MAJOR_VERSION >= 3 if (PyUnicode_Check(obj)) { Py_ssize_t len; const char *ipstr = PyUnicode_AsUTF8AndSize(obj, &len); -# else +#else if (PyUnicode_Check(obj) || PyString_Check(obj)) { // Although this should work on Python 3, we will hopefully delete // this soon and the Python 3 version is cleaner. @@ -198,11 +227,10 @@ static int ip_converter(PyObject *obj, struct sockaddr_storage *ip_address) } struct addrinfo hints = { - .ai_family = AF_UNSPEC, - .ai_flags = AI_NUMERICHOST, + .ai_family = AF_UNSPEC, + .ai_flags = AI_NUMERICHOST, // We set ai_socktype so that we only get one result back - .ai_socktype = SOCK_STREAM - }; + .ai_socktype = SOCK_STREAM}; struct addrinfo *addresses = NULL; int gai_status = getaddrinfo(ipstr, NULL, &hints, &addresses); @@ -213,9 +241,10 @@ static int ip_converter(PyObject *obj, struct sockaddr_storage *ip_address) return 0; } if (!addresses) { - PyErr_SetString(PyExc_RuntimeError, - "getaddrinfo was successful but failed to set the addrinfo"); - + PyErr_SetString( + PyExc_RuntimeError, + "getaddrinfo was successful but failed to set the addrinfo"); + return 0; } memcpy(ip_address, addresses->ai_addr, addresses->ai_addrlen); freeaddrinfo(addresses); @@ -238,30 +267,30 @@ static int ip_converter(PyObject *obj, struct sockaddr_storage *ip_address) } switch (len) { - case 16: { + case 16: { ip_address->ss_family = AF_INET6; struct sockaddr_in6 *sin = (struct sockaddr_in6 *)ip_address; memcpy(sin->sin6_addr.s6_addr, bytes, len); Py_DECREF(packed); return 1; } - case 4: { + case 4: { ip_address->ss_family = AF_INET; struct sockaddr_in *sin = (struct sockaddr_in *)ip_address; memcpy(&(sin->sin_addr.s_addr), bytes, len); Py_DECREF(packed); return 1; } - default: - PyErr_SetString(PyExc_ValueError, - "argument 1 returned an unexpected packed length for address"); - Py_DECREF(packed); - return 0; + default: + PyErr_SetString( + PyExc_ValueError, + "argument 1 returned an unexpected packed length for address"); + Py_DECREF(packed); + return 0; } } -static bool format_sockaddr(struct sockaddr *sa, char *dst) -{ +static bool format_sockaddr(struct sockaddr *sa, char *dst) { char *addr; if (sa->sa_family == AF_INET) { struct sockaddr_in *sin = (struct sockaddr_in *)sa; @@ -274,14 +303,11 @@ static bool format_sockaddr(struct sockaddr *sa, char *dst) if (inet_ntop(sa->sa_family, addr, dst, INET6_ADDRSTRLEN)) { return true; } - PyErr_SetString(PyExc_RuntimeError, - "unable to format IP address"); + PyErr_SetString(PyExc_RuntimeError, "unable to format IP address"); return false; } - -static PyObject *Reader_metadata(PyObject *self, PyObject *UNUSED(args)) -{ +static PyObject *Reader_metadata(PyObject *self, PyObject *UNUSED(args)) { Reader_obj *mmdb_obj = (Reader_obj *)self; if (NULL == mmdb_obj->mmdb) { @@ -297,8 +323,7 @@ static PyObject *Reader_metadata(PyObject *self, PyObject *UNUSED(args)) PyObject *metadata_dict = from_entry_data_list(&entry_data_list); MMDB_free_entry_data_list(original_entry_data_list); if (NULL == metadata_dict || !PyDict_Check(metadata_dict)) { - PyErr_SetString(MaxMindDB_error, - "Error decoding metadata."); + PyErr_SetString(MaxMindDB_error, "Error decoding metadata."); return NULL; } @@ -308,15 +333,14 @@ static PyObject *Reader_metadata(PyObject *self, PyObject *UNUSED(args)) return NULL; } - PyObject *metadata = PyObject_Call((PyObject *)&Metadata_Type, args, - metadata_dict); + PyObject *metadata = + PyObject_Call((PyObject *)&Metadata_Type, args, metadata_dict); Py_DECREF(metadata_dict); return metadata; } -static PyObject *Reader_close(PyObject *self, PyObject *UNUSED(args)) -{ +static PyObject *Reader_close(PyObject *self, PyObject *UNUSED(args)) { Reader_obj *mmdb_obj = (Reader_obj *)self; if (NULL != mmdb_obj->mmdb) { @@ -330,11 +354,10 @@ static PyObject *Reader_close(PyObject *self, PyObject *UNUSED(args)) Py_RETURN_NONE; } -static PyObject *Reader__enter__(PyObject *self, PyObject *UNUSED(args)) -{ +static PyObject *Reader__enter__(PyObject *self, PyObject *UNUSED(args)) { Reader_obj *mmdb_obj = (Reader_obj *)self; - if(mmdb_obj->closed == Py_True) { + if (mmdb_obj->closed == Py_True) { PyErr_SetString(PyExc_ValueError, "Attempt to reopen a closed MaxMind DB."); return NULL; @@ -344,14 +367,12 @@ static PyObject *Reader__enter__(PyObject *self, PyObject *UNUSED(args)) return (PyObject *)self; } -static PyObject *Reader__exit__(PyObject *self, PyObject *UNUSED(args)) -{ +static PyObject *Reader__exit__(PyObject *self, PyObject *UNUSED(args)) { Reader_close(self, NULL); Py_RETURN_NONE; } -static void Reader_dealloc(PyObject *self) -{ +static void Reader_dealloc(PyObject *self) { Reader_obj *obj = (Reader_obj *)self; if (NULL != obj->mmdb) { Reader_close(self, NULL); @@ -360,34 +381,27 @@ static void Reader_dealloc(PyObject *self) PyObject_Del(self); } -static int Metadata_init(PyObject *self, PyObject *args, PyObject *kwds) -{ - - PyObject - *binary_format_major_version, - *binary_format_minor_version, - *build_epoch, - *database_type, - *description, - *ip_version, - *languages, - *node_count, - *record_size; - - static char *kwlist[] = { - "binary_format_major_version", - "binary_format_minor_version", - "build_epoch", - "database_type", - "description", - "ip_version", - "languages", - "node_count", - "record_size", - NULL - }; - - if (!PyArg_ParseTupleAndKeywords(args, kwds, "|OOOOOOOOO", kwlist, +static int Metadata_init(PyObject *self, PyObject *args, PyObject *kwds) { + + PyObject *binary_format_major_version, *binary_format_minor_version, + *build_epoch, *database_type, *description, *ip_version, *languages, + *node_count, *record_size; + + static char *kwlist[] = {"binary_format_major_version", + "binary_format_minor_version", + "build_epoch", + "database_type", + "description", + "ip_version", + "languages", + "node_count", + "record_size", + NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, + kwds, + "|OOOOOOOOO", + kwlist, &binary_format_major_version, &binary_format_minor_version, &build_epoch, @@ -425,8 +439,7 @@ static int Metadata_init(PyObject *self, PyObject *args, PyObject *kwds) return 0; } -static void Metadata_dealloc(PyObject *self) -{ +static void Metadata_dealloc(PyObject *self) { Metadata_obj *obj = (Metadata_obj *)self; Py_DECREF(obj->binary_format_major_version); Py_DECREF(obj->binary_format_minor_version); @@ -440,58 +453,57 @@ static void Metadata_dealloc(PyObject *self) PyObject_Del(self); } -static PyObject *from_entry_data_list(MMDB_entry_data_list_s **entry_data_list) -{ +static PyObject * +from_entry_data_list(MMDB_entry_data_list_s **entry_data_list) { if (NULL == entry_data_list || NULL == *entry_data_list) { - PyErr_SetString( - MaxMindDB_error, - "Error while looking up data. Your database may be corrupt or you have found a bug in libmaxminddb." - ); + PyErr_SetString(MaxMindDB_error, + "Error while looking up data. Your database may be " + "corrupt or you have found a bug in libmaxminddb."); return NULL; } switch ((*entry_data_list)->entry_data.type) { - case MMDB_DATA_TYPE_MAP: - return from_map(entry_data_list); - case MMDB_DATA_TYPE_ARRAY: - return from_array(entry_data_list); - case MMDB_DATA_TYPE_UTF8_STRING: - return PyUnicode_FromStringAndSize( - (*entry_data_list)->entry_data.utf8_string, - (*entry_data_list)->entry_data.data_size - ); - case MMDB_DATA_TYPE_BYTES: - return PyByteArray_FromStringAndSize( - (const char *)(*entry_data_list)->entry_data.bytes, - (Py_ssize_t)(*entry_data_list)->entry_data.data_size); - case MMDB_DATA_TYPE_DOUBLE: - return PyFloat_FromDouble((*entry_data_list)->entry_data.double_value); - case MMDB_DATA_TYPE_FLOAT: - return PyFloat_FromDouble((*entry_data_list)->entry_data.float_value); - case MMDB_DATA_TYPE_UINT16: - return PyLong_FromLong( (*entry_data_list)->entry_data.uint16); - case MMDB_DATA_TYPE_UINT32: - return PyLong_FromLong((*entry_data_list)->entry_data.uint32); - case MMDB_DATA_TYPE_BOOLEAN: - return PyBool_FromLong((*entry_data_list)->entry_data.boolean); - case MMDB_DATA_TYPE_UINT64: - return PyLong_FromUnsignedLongLong( - (*entry_data_list)->entry_data.uint64); - case MMDB_DATA_TYPE_UINT128: - return from_uint128(*entry_data_list); - case MMDB_DATA_TYPE_INT32: - return PyLong_FromLong((*entry_data_list)->entry_data.int32); - default: - PyErr_Format(MaxMindDB_error, - "Invalid data type arguments: %d", - (*entry_data_list)->entry_data.type); - return NULL; + case MMDB_DATA_TYPE_MAP: + return from_map(entry_data_list); + case MMDB_DATA_TYPE_ARRAY: + return from_array(entry_data_list); + case MMDB_DATA_TYPE_UTF8_STRING: + return PyUnicode_FromStringAndSize( + (*entry_data_list)->entry_data.utf8_string, + (*entry_data_list)->entry_data.data_size); + case MMDB_DATA_TYPE_BYTES: + return PyByteArray_FromStringAndSize( + (const char *)(*entry_data_list)->entry_data.bytes, + (Py_ssize_t)(*entry_data_list)->entry_data.data_size); + case MMDB_DATA_TYPE_DOUBLE: + return PyFloat_FromDouble( + (*entry_data_list)->entry_data.double_value); + case MMDB_DATA_TYPE_FLOAT: + return PyFloat_FromDouble( + (*entry_data_list)->entry_data.float_value); + case MMDB_DATA_TYPE_UINT16: + return PyLong_FromLong((*entry_data_list)->entry_data.uint16); + case MMDB_DATA_TYPE_UINT32: + return PyLong_FromLong((*entry_data_list)->entry_data.uint32); + case MMDB_DATA_TYPE_BOOLEAN: + return PyBool_FromLong((*entry_data_list)->entry_data.boolean); + case MMDB_DATA_TYPE_UINT64: + return PyLong_FromUnsignedLongLong( + (*entry_data_list)->entry_data.uint64); + case MMDB_DATA_TYPE_UINT128: + return from_uint128(*entry_data_list); + case MMDB_DATA_TYPE_INT32: + return PyLong_FromLong((*entry_data_list)->entry_data.int32); + default: + PyErr_Format(MaxMindDB_error, + "Invalid data type arguments: %d", + (*entry_data_list)->entry_data.type); + return NULL; } return NULL; } -static PyObject *from_map(MMDB_entry_data_list_s **entry_data_list) -{ +static PyObject *from_map(MMDB_entry_data_list_s **entry_data_list) { PyObject *py_obj = PyDict_New(); if (NULL == py_obj) { PyErr_NoMemory(); @@ -509,8 +521,7 @@ static PyObject *from_map(MMDB_entry_data_list_s **entry_data_list) PyObject *key = PyUnicode_FromStringAndSize( (char *)(*entry_data_list)->entry_data.utf8_string, - (*entry_data_list)->entry_data.data_size - ); + (*entry_data_list)->entry_data.data_size); *entry_data_list = (*entry_data_list)->next; @@ -528,8 +539,7 @@ static PyObject *from_map(MMDB_entry_data_list_s **entry_data_list) return py_obj; } -static PyObject *from_array(MMDB_entry_data_list_s **entry_data_list) -{ +static PyObject *from_array(MMDB_entry_data_list_s **entry_data_list) { const uint32_t size = (*entry_data_list)->entry_data.data_size; PyObject *py_obj = PyList_New(size); @@ -555,8 +565,7 @@ static PyObject *from_array(MMDB_entry_data_list_s **entry_data_list) return py_obj; } -static PyObject *from_uint128(const MMDB_entry_data_list_s *entry_data_list) -{ +static PyObject *from_uint128(const MMDB_entry_data_list_s *entry_data_list) { uint64_t high = 0; uint64_t low = 0; #if MMDB_UINT128_IS_BYTE_ARRAY @@ -588,21 +597,34 @@ static PyObject *from_uint128(const MMDB_entry_data_list_s *entry_data_list) } static PyMethodDef Reader_methods[] = { - { "get", Reader_get, METH_VARARGS, - "Get record for IP address" }, - { "metadata", Reader_metadata, METH_NOARGS, - "Returns metadata object for database" }, - { "close", Reader_close, METH_NOARGS, "Closes database"}, - { "__exit__", Reader__exit__, METH_VARARGS, "Called when exiting a with-context. Calls close"}, - { "__enter__", Reader__enter__, METH_NOARGS, "Called when entering a with-context."}, - { NULL, NULL, 0, NULL } -}; + {"get", + Reader_get, + METH_VARARGS, + "Return the record for the ip_address in the MaxMind DB"}, + {"get_with_prefix_len", + Reader_get_with_prefix_len, + METH_VARARGS, + "Return a tuple with the record and the associated prefix length"}, + {"metadata", + Reader_metadata, + METH_NOARGS, + "Return metadata object for database"}, + {"close", Reader_close, METH_NOARGS, "Closes database"}, + {"__exit__", + Reader__exit__, + METH_VARARGS, + "Called when exiting a with-context. Calls close"}, + {"__enter__", + Reader__enter__, + METH_NOARGS, + "Called when entering a with-context."}, + {NULL, NULL, 0, NULL}}; static PyMemberDef Reader_members[] = { - { "closed", T_OBJECT, offsetof(Reader_obj, closed), READONLY, NULL }, - { NULL, 0, 0, 0, NULL } -}; + {"closed", T_OBJECT, offsetof(Reader_obj, closed), READONLY, NULL}, + {NULL, 0, 0, 0, NULL}}; +// clang-format off static PyTypeObject Reader_Type = { PyVarObject_HEAD_INIT(NULL, 0) .tp_basicsize = sizeof(Reader_obj), @@ -614,35 +636,55 @@ static PyTypeObject Reader_Type = { .tp_name = "Reader", .tp_init = Reader_init, }; +// clang-format on -static PyMethodDef Metadata_methods[] = { - { NULL, NULL, 0, NULL } -}; +static PyMethodDef Metadata_methods[] = {{NULL, NULL, 0, NULL}}; -/* *INDENT-OFF* */ static PyMemberDef Metadata_members[] = { - { "binary_format_major_version", T_OBJECT, offsetof( - Metadata_obj, binary_format_major_version), READONLY, NULL }, - { "binary_format_minor_version", T_OBJECT, offsetof( - Metadata_obj, binary_format_minor_version), READONLY, NULL }, - { "build_epoch", T_OBJECT, offsetof(Metadata_obj, build_epoch), - READONLY, NULL }, - { "database_type", T_OBJECT, offsetof(Metadata_obj, database_type), - READONLY, NULL }, - { "description", T_OBJECT, offsetof(Metadata_obj, description), - READONLY, NULL }, - { "ip_version", T_OBJECT, offsetof(Metadata_obj, ip_version), - READONLY, NULL }, - { "languages", T_OBJECT, offsetof(Metadata_obj, languages), READONLY, - NULL }, - { "node_count", T_OBJECT, offsetof(Metadata_obj, node_count), - READONLY, NULL }, - { "record_size", T_OBJECT, offsetof(Metadata_obj, record_size), - READONLY, NULL }, - { NULL, 0, 0, 0, NULL } -}; -/* *INDENT-ON* */ - + {"binary_format_major_version", + T_OBJECT, + offsetof(Metadata_obj, binary_format_major_version), + READONLY, + NULL}, + {"binary_format_minor_version", + T_OBJECT, + offsetof(Metadata_obj, binary_format_minor_version), + READONLY, + NULL}, + {"build_epoch", + T_OBJECT, + offsetof(Metadata_obj, build_epoch), + READONLY, + NULL}, + {"database_type", + T_OBJECT, + offsetof(Metadata_obj, database_type), + READONLY, + NULL}, + {"description", + T_OBJECT, + offsetof(Metadata_obj, description), + READONLY, + NULL}, + {"ip_version", + T_OBJECT, + offsetof(Metadata_obj, ip_version), + READONLY, + NULL}, + {"languages", T_OBJECT, offsetof(Metadata_obj, languages), READONLY, NULL}, + {"node_count", + T_OBJECT, + offsetof(Metadata_obj, node_count), + READONLY, + NULL}, + {"record_size", + T_OBJECT, + offsetof(Metadata_obj, record_size), + READONLY, + NULL}, + {NULL, 0, 0, 0, NULL}}; + +// clang-format off static PyTypeObject Metadata_Type = { PyVarObject_HEAD_INIT(NULL, 0) .tp_basicsize = sizeof(Metadata_obj), @@ -652,13 +694,10 @@ static PyTypeObject Metadata_Type = { .tp_members = Metadata_members, .tp_methods = Metadata_methods, .tp_name = "Metadata", - .tp_init = Metadata_init -}; - -static PyMethodDef MaxMindDB_methods[] = { - { NULL, NULL, 0, NULL } -}; + .tp_init = Metadata_init}; +// clang-format on +static PyMethodDef MaxMindDB_methods[] = {{NULL, NULL, 0, NULL}}; #if PY_MAJOR_VERSION >= 3 static struct PyModuleDef MaxMindDB_module = { @@ -669,7 +708,7 @@ static struct PyModuleDef MaxMindDB_module = { }; #endif -MOD_INIT(extension){ +MOD_INIT(extension) { PyObject *m; #if PY_MAJOR_VERSION >= 3 @@ -695,7 +734,7 @@ MOD_INIT(extension){ } PyModule_AddObject(m, "extension", (PyObject *)&Metadata_Type); - PyObject* error_mod = PyImport_ImportModule("maxminddb.errors"); + PyObject *error_mod = PyImport_ImportModule("maxminddb.errors"); if (error_mod == NULL) { RETURN_MOD_INIT(NULL); } diff --git a/maxminddb/__init__.py b/maxminddb/__init__.py index 8af0633..c26e808 100644 --- a/maxminddb/__init__.py +++ b/maxminddb/__init__.py @@ -51,4 +51,4 @@ def Reader(database): # pylint: disable=invalid-name __version__ = '1.4.1' __author__ = 'Gregory Oschwald' __license__ = 'Apache License, Version 2.0' -__copyright__ = 'Copyright 2013-2018 Maxmind, Inc.' +__copyright__ = 'Copyright 2013-2019 Maxmind, Inc.' diff --git a/maxminddb/decoder.py b/maxminddb/decoder.py index be6d15d..7908577 100644 --- a/maxminddb/decoder.py +++ b/maxminddb/decoder.py @@ -125,13 +125,15 @@ def decode(self, offset): if not type_num: (type_num, new_offset) = self._read_extended(new_offset) - if type_num not in self._type_decoder: + try: + decoder = self._type_decoder[type_num] + except KeyError: raise InvalidDatabaseError('Unexpected type number ({type}) ' 'encountered'.format(type=type_num)) (size, new_offset) = self._size_from_ctrl_byte(ctrl_byte, new_offset, type_num) - return self._type_decoder[type_num](self, size, new_offset) + return decoder(self, size, new_offset) def _read_extended(self, offset): (next_byte, ) = struct.unpack(b'!B', self._buffer[offset:offset + 1]) diff --git a/maxminddb/reader.py b/maxminddb/reader.py index 34703f2..6d5b916 100644 --- a/maxminddb/reader.py +++ b/maxminddb/reader.py @@ -101,6 +101,16 @@ def get(self, ip_address): """Return the record for the ip_address in the MaxMind DB + Arguments: + ip_address -- an IP address in the standard string notation + """ + (record, _) = self.get_with_prefix_len(ip_address) + return record + + def get_with_prefix_len(self, ip_address): + """Return a tuple with the record and the associated prefix length + + Arguments: ip_address -- an IP address in the standard string notation """ @@ -118,24 +128,29 @@ def get(self, ip_address): raise ValueError( 'Error looking up {0}. You attempted to look up ' 'an IPv6 address in an IPv4-only database.'.format(ip_address)) - pointer = self._find_address_in_tree(packed_address) - return self._resolve_data_pointer(pointer) if pointer else None + (pointer, prefix_len) = self._find_address_in_tree(packed_address) + + if pointer: + return self._resolve_data_pointer(pointer), prefix_len + return None, prefix_len def _find_address_in_tree(self, packed): bit_count = len(packed) * 8 node = self._start_node(bit_count) + node_count = self._metadata.node_count - for i in range(bit_count): - if node >= self._metadata.node_count: - break + i = 0 + while i < bit_count and node < node_count: bit = 1 & (packed[i >> 3] >> 7 - (i % 8)) node = self._read_node(node, bit) - if node == self._metadata.node_count: + i = i + 1 + + if node == node_count: # Record is empty - return 0 - if node > self._metadata.node_count: - return node + return 0, i + if node > node_count: + return node, i raise InvalidDatabaseError('Invalid node in search tree') diff --git a/setup.py b/setup.py index b146f1d..73f74ba 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ from distutils.errors import (CCompilerError, DistutilsExecError, DistutilsPlatformError) -from setuptools import setup, Extension, Feature +from setuptools import setup, Extension cmdclass = {} PYPY = hasattr(sys, 'pypy_version_info') @@ -107,15 +107,7 @@ def find_packages(location): def run_setup(with_cext): kwargs = {} if with_cext: - if Feature: - kwargs['features'] = { - 'extension': - Feature("optional C implementation", - standard=True, - ext_modules=ext_module) - } - else: - kwargs['ext_modules'] = ext_module + kwargs['ext_modules'] = ext_module setup(name='maxminddb', version=VERSION, diff --git a/tests/reader_test.py b/tests/reader_test.py index 5b731b0..e2b4fc7 100644 --- a/tests/reader_test.py +++ b/tests/reader_test.py @@ -3,18 +3,15 @@ from __future__ import unicode_literals -import logging import ipaddress -import mock import os import sys import threading - from multiprocessing import Process, Pipe -import maxminddb +import mock -from maxminddb.compat import compat_ip_address +import maxminddb try: import maxminddb.extension @@ -72,6 +69,111 @@ def test_reader(self): self._check_ip_v6(reader, file_name) reader.close() + def test_get_with_prefix_len(self): + decoder_record = { + "array": [1, 2, 3], + "boolean": True, + "bytes": b'\x00\x00\x00*', + "double": 42.123456, + "float": 1.100000023841858, + "int32": -268435456, + "map": { + "mapX": { + "arrayX": [7, 8, 9], + "utf8_stringX": "hello", + }, + }, + "uint128": 1329227995784915872903807060280344576, + "uint16": 0x64, + "uint32": 0x10000000, + "uint64": 0x1000000000000000, + "utf8_string": "unicode! ☯ - ♫", + } + + tests = [{ + 'ip': '1.1.1.1', + 'file_name': 'MaxMind-DB-test-ipv6-32.mmdb', + 'expected_prefix_len': 8, + 'expected_record': None, + }, { + 'ip': '::1:ffff:ffff', + 'file_name': 'MaxMind-DB-test-ipv6-24.mmdb', + 'expected_prefix_len': 128, + 'expected_record': { + "ip": "::1:ffff:ffff" + }, + }, { + 'ip': '::2:0:1', + 'file_name': 'MaxMind-DB-test-ipv6-24.mmdb', + 'expected_prefix_len': 122, + 'expected_record': { + "ip": "::2:0:0" + }, + }, { + 'ip': '1.1.1.1', + 'file_name': 'MaxMind-DB-test-ipv4-24.mmdb', + 'expected_prefix_len': 32, + 'expected_record': { + "ip": "1.1.1.1" + }, + }, { + 'ip': '1.1.1.3', + 'file_name': 'MaxMind-DB-test-ipv4-24.mmdb', + 'expected_prefix_len': 31, + 'expected_record': { + "ip": "1.1.1.2" + }, + }, { + 'ip': '1.1.1.3', + 'file_name': 'MaxMind-DB-test-decoder.mmdb', + 'expected_prefix_len': 24, + 'expected_record': decoder_record, + }, { + 'ip': '::ffff:1.1.1.128', + 'file_name': 'MaxMind-DB-test-decoder.mmdb', + 'expected_prefix_len': 120, + 'expected_record': decoder_record, + }, { + 'ip': '::1.1.1.128', + 'file_name': 'MaxMind-DB-test-decoder.mmdb', + 'expected_prefix_len': 120, + 'expected_record': decoder_record, + }, { + 'ip': '200.0.2.1', + 'file_name': 'MaxMind-DB-no-ipv4-search-tree.mmdb', + 'expected_prefix_len': 0, + 'expected_record': "::0/64", + }, { + 'ip': '::200.0.2.1', + 'file_name': 'MaxMind-DB-no-ipv4-search-tree.mmdb', + 'expected_prefix_len': 64, + 'expected_record': "::0/64", + }, { + 'ip': '0:0:0:0:ffff:ffff:ffff:ffff', + 'file_name': 'MaxMind-DB-no-ipv4-search-tree.mmdb', + 'expected_prefix_len': 64, + 'expected_record': "::0/64", + }, { + 'ip': 'ef00::', + 'file_name': 'MaxMind-DB-no-ipv4-search-tree.mmdb', + 'expected_prefix_len': 1, + 'expected_record': None, + }] + + for test in tests: + with open_database('tests/data/test-data/' + test['file_name'], + self.mode) as reader: + (record, prefix_len) = reader.get_with_prefix_len(test['ip']) + + self.assertEqual( + prefix_len, test['expected_prefix_len'], + 'expected prefix_len of {} for {} in {} but got {}'.format( + test['expected_prefix_len'], test['ip'], + test['file_name'], prefix_len)) + self.assertEqual( + record, test['expected_record'], 'expected_record for ' + + test['ip'] + ' in ' + test['file_name']) + def test_decoder(self): reader = open_database( 'tests/data/test-data/MaxMind-DB-test-decoder.mmdb', self.mode)