diff --git a/HISTORY.rst b/HISTORY.rst index 697b8e2..0868a48 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -3,7 +3,7 @@ History ------- -2.9.0 +3.0.0 ++++++++++++++++++ * IMPORTANT: Python 3.10 or greater is required. If you are using an older @@ -15,6 +15,29 @@ History thread-safe for concurrent reads on platforms with pthread support (such as Linux and macOS) and Windows. On other platforms, the extension will use GIL-based protection. +* The C extension now uses PEP 489 multi-phase initialization, enabling + proper subinterpreter support and module isolation for Python 3.12+. This + modernizes the extension to use heap types instead of static types and + implements per-module state management. Key benefits include support for + Python 3.12+ isolated subinterpreters, multiple independent module + instances, and future-proofing for Python 3.14's InterpreterPoolExecutor. + Requested by R. Christian McDonald in GitHub #105. +* **BREAKING**: The pure Python ``maxminddb.reader.Metadata`` class has been + converted to a frozen dataclass. The ``__repr__`` format has changed from + ``maxminddb.reader.Metadata(...)`` to ``Metadata(...)``. More importantly, + all Metadata attributes are now readonly and cannot be modified after + creation. If you were modifying metadata attributes after object creation, + you will need to update your code. All functionality remains the same, + including the ``node_byte_size`` and ``search_tree_size`` properties. Note: + The C extension's Metadata class has always been readonly, so this change + brings the pure Python implementation into consistency with the C extension. +* MODE constants have been converted to an ``IntEnum`` (``maxminddb.const.Mode``). + The old constants (``MODE_AUTO``, ``MODE_FILE``, etc.) remain available for + backward compatibility and are now aliases to the enum members. This provides + better IDE support and type safety while maintaining full backward + compatibility. You can now use either ``Mode.FILE`` or ``MODE_FILE`` - both + work identically. Since ``IntEnum`` is int-compatible, existing code using + the constants will continue to work without modification. 2.8.2 (2025-07-25) ++++++++++++++++++ diff --git a/README.rst b/README.rst index 1f72e3e..3e064e4 100644 --- a/README.rst +++ b/README.rst @@ -39,19 +39,19 @@ provide `free GeoLite2 databases files must be decompressed with ``gunzip``. After you have obtained a database and imported the module, call -``open_database`` with a path, or file descriptor (in the case of ``MODE_FD``), +``open_database`` with a path, or file descriptor (in the case of ``Mode.FD``), to the database as the first argument. Optionally, you may pass a mode as the -second argument. The modes are exported from ``maxminddb``. Valid modes are: - -* ``MODE_MMAP_EXT`` - use the C extension with memory map. -* ``MODE_MMAP`` - read from memory map. Pure Python. -* ``MODE_FILE`` - read database as standard file. Pure Python. -* ``MODE_MEMORY`` - load database into memory. Pure Python. -* ``MODE_FD`` - load database into memory from a file descriptor. Pure Python. -* ``MODE_AUTO`` - try ``MODE_MMAP_EXT``, ``MODE_MMAP``, ``MODE_FILE`` in that +second argument. The modes are available from ``maxminddb.Mode``. Valid modes are: + +* ``Mode.MMAP_EXT`` - use the C extension with memory map. +* ``Mode.MMAP`` - read from memory map. Pure Python. +* ``Mode.FILE`` - read database as standard file. Pure Python. +* ``Mode.MEMORY`` - load database into memory. Pure Python. +* ``Mode.FD`` - load database into memory from a file descriptor. Pure Python. +* ``Mode.AUTO`` - try ``Mode.MMAP_EXT``, ``Mode.MMAP``, ``Mode.FILE`` in that order. Default. -**NOTE**: When using ``MODE_FD``, it is the *caller's* responsibility to be +**NOTE**: When using ``Mode.FD``, it is the *caller's* responsibility to be sure that the file descriptor gets closed properly. The caller may close the file descriptor immediately after the ``Reader`` object is created. diff --git a/dev-bin/release.sh b/dev-bin/release.sh index 1dae9a2..73b9e55 100755 --- a/dev-bin/release.sh +++ b/dev-bin/release.sh @@ -33,7 +33,6 @@ if [ -n "$(git status --porcelain)" ]; then exit 1 fi -perl -pi -e "s/(?<=__version__ = \").+?(?=\")/$version/gsm" maxminddb/__init__.py perl -pi -e "s/(?<=^version = \").+?(?=\")/$version/gsm" pyproject.toml echo $"Test results:" diff --git a/extension/maxminddb.c b/extension/maxminddb.c index a883769..c2dd73e 100644 --- a/extension/maxminddb.c +++ b/extension/maxminddb.c @@ -16,12 +16,6 @@ #define __STDC_FORMAT_MACROS #include -static PyTypeObject Reader_Type; -static PyTypeObject ReaderIter_Type; -static PyTypeObject Metadata_Type; -static PyObject *MaxMindDB_error; -static PyObject *ipaddress_ip_network; - // ============================================================================= // Platform-specific lock type definition // ============================================================================= @@ -105,12 +99,43 @@ typedef struct { } Metadata_obj; // clang-format on +// ============================================================================= +// Module state structure for PEP 489 multi-phase initialization +// ============================================================================= + +typedef struct { + PyObject *Reader_Type; + PyObject *ReaderIter_Type; + PyObject *Metadata_Type; + PyObject *MaxMindDB_error; + PyObject *ipaddress_ip_network; +} maxminddb_state; + +// Helper function to get module state from module +static inline maxminddb_state *get_maxminddb_state(PyObject *module) { + void *state = PyModule_GetState(module); + assert(state != NULL); + return (maxminddb_state *)state; +} + +// Helper function to get module state from self (instance) +static inline maxminddb_state *get_maxminddb_state_from_self(PyObject *self) { + PyObject *module = PyType_GetModule(Py_TYPE(self)); + if (module == NULL) { + return NULL; + } + return get_maxminddb_state(module); +} + static bool can_read(const char *path); static int get_record(PyObject *self, PyObject *args, PyObject **record); static bool format_sockaddr(struct sockaddr *addr, char *dst); -static PyObject *from_entry_data_list(MMDB_entry_data_list_s **entry_data_list); -static PyObject *from_map(MMDB_entry_data_list_s **entry_data_list); -static PyObject *from_array(MMDB_entry_data_list_s **entry_data_list); +static PyObject *from_entry_data_list(maxminddb_state *state, + MMDB_entry_data_list_s **entry_data_list); +static PyObject *from_map(maxminddb_state *state, + MMDB_entry_data_list_s **entry_data_list); +static PyObject *from_array(maxminddb_state *state, + MMDB_entry_data_list_s **entry_data_list); static PyObject *from_uint128(const MMDB_entry_data_list_s *entry_data_list); static int ip_converter(PyObject *obj, struct sockaddr_storage *ip_address); @@ -284,6 +309,11 @@ static void reader_release_write_lock(Reader_obj *reader) { // ============================================================================= static int Reader_init(PyObject *self, PyObject *args, PyObject *kwds) { + maxminddb_state *state = get_maxminddb_state_from_self(self); + if (state == NULL) { + return -1; + } + PyObject *filepath = NULL; int mode = 0; @@ -345,7 +375,7 @@ static int Reader_init(PyObject *self, PyObject *args, PyObject *kwds) { if (status != MMDB_SUCCESS) { reader_lock_destroy(&mmdb_obj->rwlock); free(mmdb); - PyErr_Format(MaxMindDB_error, + PyErr_Format(state->MaxMindDB_error, "Error opening database file (%s). Is this a valid " "MaxMind DB file?", filename); @@ -382,6 +412,11 @@ static PyObject *Reader_get_with_prefix_len(PyObject *self, PyObject *args) { } static int get_record(PyObject *self, PyObject *args, PyObject **record) { + maxminddb_state *state = get_maxminddb_state_from_self(self); + if (state == NULL) { + return -1; + } + struct sockaddr_storage ip_address_ss = {0}; struct sockaddr *ip_address = (struct sockaddr *)&ip_address_ss; if (!PyArg_ParseTuple(args, "O&", ip_converter, &ip_address_ss)) { @@ -417,7 +452,7 @@ static int get_record(PyObject *self, PyObject *args, PyObject **record) { if (MMDB_IPV6_LOOKUP_IN_IPV4_DATABASE_ERROR == mmdb_error) { exception = PyExc_ValueError; } else { - exception = MaxMindDB_error; + exception = state->MaxMindDB_error; } char ipstr[INET6_ADDRSTRLEN] = {0}; if (format_sockaddr(ip_address, ipstr)) { @@ -449,7 +484,7 @@ static int get_record(PyObject *self, PyObject *args, PyObject **record) { reader_release_read_lock(reader); char ipstr[INET6_ADDRSTRLEN] = {0}; if (format_sockaddr(ip_address, ipstr)) { - PyErr_Format(MaxMindDB_error, + PyErr_Format(state->MaxMindDB_error, "Error while looking up data for %s. %s", ipstr, MMDB_strerror(status)); @@ -459,7 +494,7 @@ static int get_record(PyObject *self, PyObject *args, PyObject **record) { } MMDB_entry_data_list_s *original_entry_data_list = entry_data_list; - *record = from_entry_data_list(&entry_data_list); + *record = from_entry_data_list(state, &entry_data_list); MMDB_free_entry_data_list(original_entry_data_list); reader_release_read_lock(reader); @@ -570,6 +605,11 @@ static bool format_sockaddr(struct sockaddr *sa, char *dst) { } static PyObject *Reader_metadata(PyObject *self, PyObject *UNUSED(args)) { + maxminddb_state *state = get_maxminddb_state_from_self(self); + if (state == NULL) { + return NULL; + } + Reader_obj *mmdb_obj = (Reader_obj *)self; if (reader_acquire_read_lock(mmdb_obj) != 0) { @@ -588,18 +628,18 @@ static PyObject *Reader_metadata(PyObject *self, PyObject *UNUSED(args)) { MMDB_get_metadata_as_entry_data_list(mmdb_obj->mmdb, &entry_data_list); if (status != MMDB_SUCCESS) { reader_release_read_lock(mmdb_obj); - PyErr_Format(MaxMindDB_error, + PyErr_Format(state->MaxMindDB_error, "Error decoding metadata. %s", MMDB_strerror(status)); return NULL; } MMDB_entry_data_list_s *original_entry_data_list = entry_data_list; - PyObject *metadata_dict = from_entry_data_list(&entry_data_list); + PyObject *metadata_dict = from_entry_data_list(state, &entry_data_list); MMDB_free_entry_data_list(original_entry_data_list); if (metadata_dict == NULL || !PyDict_Check(metadata_dict)) { reader_release_read_lock(mmdb_obj); - PyErr_SetString(MaxMindDB_error, "Error decoding metadata."); + PyErr_SetString(state->MaxMindDB_error, "Error decoding metadata."); return NULL; } @@ -612,9 +652,10 @@ static PyObject *Reader_metadata(PyObject *self, PyObject *UNUSED(args)) { } PyObject *metadata = - PyObject_Call((PyObject *)&Metadata_Type, args, metadata_dict); + PyObject_Call(state->Metadata_Type, args, metadata_dict); Py_DECREF(metadata_dict); + Py_DECREF(args); return metadata; } @@ -675,6 +716,11 @@ static void Reader_dealloc(PyObject *self) { } static PyObject *Reader_iter(PyObject *obj) { + maxminddb_state *state = get_maxminddb_state_from_self(obj); + if (state == NULL) { + return NULL; + } + Reader_obj *reader = (Reader_obj *)obj; if (reader_acquire_read_lock(reader) != 0) { @@ -690,7 +736,8 @@ static PyObject *Reader_iter(PyObject *obj) { reader_release_read_lock(reader); - ReaderIter_obj *ri = PyObject_New(ReaderIter_obj, &ReaderIter_Type); + ReaderIter_obj *ri = (ReaderIter_obj *)PyType_GenericAlloc( + (PyTypeObject *)state->ReaderIter_Type, 0); if (ri == NULL) { return NULL; } @@ -719,6 +766,11 @@ static bool is_ipv6(char ip[16]) { } static PyObject *ReaderIter_next(PyObject *self) { + maxminddb_state *state = get_maxminddb_state_from_self((PyObject *)self); + if (state == NULL) { + return NULL; + } + ReaderIter_obj *ri = (ReaderIter_obj *)self; if (reader_acquire_read_lock(ri->reader) != 0) { @@ -739,7 +791,7 @@ static PyObject *ReaderIter_next(PyObject *self) { switch (cur->type) { case MMDB_RECORD_TYPE_INVALID: reader_release_read_lock(ri->reader); - PyErr_SetString(MaxMindDB_error, + PyErr_SetString(state->MaxMindDB_error, "Invalid record when reading node"); free(cur); return NULL; @@ -756,8 +808,9 @@ static PyObject *ReaderIter_next(PyObject *self) { if (status != MMDB_SUCCESS) { reader_release_read_lock(ri->reader); const char *error = MMDB_strerror(status); - PyErr_Format( - MaxMindDB_error, "Error reading node: %s", error); + PyErr_Format(state->MaxMindDB_error, + "Error reading node: %s", + error); free(cur); return NULL; } @@ -804,7 +857,7 @@ static PyObject *ReaderIter_next(PyObject *self) { if (status != MMDB_SUCCESS) { reader_release_read_lock(ri->reader); PyErr_Format( - MaxMindDB_error, + state->MaxMindDB_error, "Error looking up data while iterating over tree: %s", MMDB_strerror(status)); MMDB_free_entry_data_list(entry_data_list); @@ -814,7 +867,8 @@ static PyObject *ReaderIter_next(PyObject *self) { MMDB_entry_data_list_s *original_entry_data_list = entry_data_list; - PyObject *record = from_entry_data_list(&entry_data_list); + PyObject *record = + from_entry_data_list(state, &entry_data_list); MMDB_free_entry_data_list(original_entry_data_list); if (record == NULL) { reader_release_read_lock(ri->reader); @@ -853,7 +907,7 @@ static PyObject *ReaderIter_next(PyObject *self) { return NULL; } PyObject *network = - PyObject_CallObject(ipaddress_ip_network, args); + PyObject_CallObject(state->ipaddress_ip_network, args); Py_DECREF(args); if (network == NULL) { reader_release_read_lock(ri->reader); @@ -873,8 +927,9 @@ static PyObject *ReaderIter_next(PyObject *self) { } default: reader_release_read_lock(ri->reader); - PyErr_Format( - MaxMindDB_error, "Unknown record type: %u", cur->type); + PyErr_Format(state->MaxMindDB_error, + "Unknown record type: %u", + cur->type); free(cur); return NULL; } @@ -971,9 +1026,10 @@ static void Metadata_dealloc(PyObject *self) { } static PyObject * -from_entry_data_list(MMDB_entry_data_list_s **entry_data_list) { +from_entry_data_list(maxminddb_state *state, + MMDB_entry_data_list_s **entry_data_list) { if (entry_data_list == NULL || *entry_data_list == NULL) { - PyErr_SetString(MaxMindDB_error, + PyErr_SetString(state->MaxMindDB_error, "Error while looking up data. Your database may be " "corrupt or you have found a bug in libmaxminddb."); return NULL; @@ -981,9 +1037,9 @@ from_entry_data_list(MMDB_entry_data_list_s **entry_data_list) { switch ((*entry_data_list)->entry_data.type) { case MMDB_DATA_TYPE_MAP: - return from_map(entry_data_list); + return from_map(state, entry_data_list); case MMDB_DATA_TYPE_ARRAY: - return from_array(entry_data_list); + return from_array(state, entry_data_list); case MMDB_DATA_TYPE_UTF8_STRING: return PyUnicode_FromStringAndSize( (*entry_data_list)->entry_data.utf8_string, @@ -1012,7 +1068,7 @@ from_entry_data_list(MMDB_entry_data_list_s **entry_data_list) { case MMDB_DATA_TYPE_INT32: return PyLong_FromLong((*entry_data_list)->entry_data.int32); default: - PyErr_Format(MaxMindDB_error, + PyErr_Format(state->MaxMindDB_error, "Invalid data type arguments: %d", (*entry_data_list)->entry_data.type); return NULL; @@ -1020,7 +1076,8 @@ from_entry_data_list(MMDB_entry_data_list_s **entry_data_list) { return NULL; } -static PyObject *from_map(MMDB_entry_data_list_s **entry_data_list) { +static PyObject *from_map(maxminddb_state *state, + MMDB_entry_data_list_s **entry_data_list) { PyObject *py_obj = PyDict_New(); if (py_obj == NULL) { PyErr_NoMemory(); @@ -1044,7 +1101,7 @@ static PyObject *from_map(MMDB_entry_data_list_s **entry_data_list) { *entry_data_list = (*entry_data_list)->next; - PyObject *value = from_entry_data_list(entry_data_list); + PyObject *value = from_entry_data_list(state, entry_data_list); if (value == NULL) { Py_DECREF(key); Py_DECREF(py_obj); @@ -1058,7 +1115,8 @@ static PyObject *from_map(MMDB_entry_data_list_s **entry_data_list) { return py_obj; } -static PyObject *from_array(MMDB_entry_data_list_s **entry_data_list) { +static PyObject *from_array(maxminddb_state *state, + MMDB_entry_data_list_s **entry_data_list) { const uint32_t size = (*entry_data_list)->entry_data.data_size; PyObject *py_obj = PyList_New(size); @@ -1070,7 +1128,7 @@ static PyObject *from_array(MMDB_entry_data_list_s **entry_data_list) { uint32_t i; for (i = 0; i < size && *entry_data_list; i++) { *entry_data_list = (*entry_data_list)->next; - PyObject *value = from_entry_data_list(entry_data_list); + PyObject *value = from_entry_data_list(state, entry_data_list); if (value == NULL) { Py_DECREF(py_obj); return NULL; @@ -1150,37 +1208,8 @@ static PyMemberDef Reader_members[] = { {"closed", T_OBJECT, offsetof(Reader_obj, closed), READONLY, NULL}, {NULL, 0, 0, 0, NULL}}; -// clang-format off -static PyTypeObject Reader_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - .tp_basicsize = sizeof(Reader_obj), - .tp_dealloc = Reader_dealloc, - .tp_doc = "Reader object", - .tp_flags = Py_TPFLAGS_DEFAULT, - .tp_iter = Reader_iter, - .tp_methods = Reader_methods, - .tp_members = Reader_members, - .tp_name = "Reader", - .tp_init = Reader_init, -}; -// clang-format on - static PyMethodDef ReaderIter_methods[] = {{NULL, NULL, 0, NULL}}; -// clang-format off -static PyTypeObject ReaderIter_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - .tp_basicsize = sizeof(ReaderIter_obj), - .tp_dealloc = ReaderIter_dealloc, - .tp_doc = "Iterator for Reader object", - .tp_flags = Py_TPFLAGS_DEFAULT, - .tp_iter = PyObject_SelfIter, - .tp_iternext = ReaderIter_next, - .tp_methods = ReaderIter_methods, - .tp_name = "ReaderIter", -}; -// clang-format on - static PyMethodDef Metadata_methods[] = {{NULL, NULL, 0, NULL}}; static PyMemberDef Metadata_members[] = { @@ -1227,84 +1256,176 @@ static PyMemberDef Metadata_members[] = { NULL}, {NULL, 0, 0, 0, NULL}}; -// clang-format off -static PyTypeObject Metadata_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - .tp_basicsize = sizeof(Metadata_obj), - .tp_dealloc = Metadata_dealloc, - .tp_doc = "Metadata object", - .tp_flags = Py_TPFLAGS_DEFAULT, - .tp_members = Metadata_members, - .tp_methods = Metadata_methods, - .tp_name = "Metadata", - .tp_init = Metadata_init}; -// clang-format on +// ============================================================================= +// Type specs for heap type conversion (PEP 489) +// ============================================================================= -static PyMethodDef MaxMindDB_methods[] = {{NULL, NULL, 0, NULL}}; +static PyType_Slot Reader_Type_slots[] = { + {Py_tp_doc, "Reader object"}, + {Py_tp_dealloc, Reader_dealloc}, + {Py_tp_init, Reader_init}, + {Py_tp_iter, Reader_iter}, + {Py_tp_methods, Reader_methods}, + {Py_tp_members, Reader_members}, + {0, NULL}, +}; -static struct PyModuleDef MaxMindDB_module = { - PyModuleDef_HEAD_INIT, - .m_name = "extension", - .m_doc = "This is a C extension to read MaxMind DB file format", - .m_methods = MaxMindDB_methods, +static PyType_Spec Reader_Type_spec = { + .name = "maxminddb.extension.Reader", + .basicsize = sizeof(Reader_obj), + .flags = Py_TPFLAGS_DEFAULT, + .slots = Reader_Type_slots, }; -PyMODINIT_FUNC PyInit_extension(void) { - PyObject *m; +static PyType_Slot Metadata_Type_slots[] = { + {Py_tp_doc, "Metadata object"}, + {Py_tp_dealloc, Metadata_dealloc}, + {Py_tp_init, Metadata_init}, + {Py_tp_methods, Metadata_methods}, + {Py_tp_members, Metadata_members}, + {0, NULL}, +}; - m = PyModule_Create(&MaxMindDB_module); +static PyType_Spec Metadata_Type_spec = { + .name = "maxminddb.extension.Metadata", + .basicsize = sizeof(Metadata_obj), + .flags = Py_TPFLAGS_DEFAULT, + .slots = Metadata_Type_slots, +}; - if (!m) { - return NULL; - } +static PyType_Slot ReaderIter_Type_slots[] = { + {Py_tp_doc, "Iterator for Reader object"}, + {Py_tp_dealloc, ReaderIter_dealloc}, + {Py_tp_iter, PyObject_SelfIter}, + {Py_tp_iternext, ReaderIter_next}, + {Py_tp_methods, ReaderIter_methods}, + {0, NULL}, +}; -#ifndef MAXMINDDB_USE_GIL_ONLY - // Only declare module as GIL-free when we have proper locks available - PyUnstable_Module_SetGIL(m, Py_MOD_GIL_NOT_USED); -#endif +static PyType_Spec ReaderIter_Type_spec = { + .name = "maxminddb.extension.ReaderIter", + .basicsize = sizeof(ReaderIter_obj), + .flags = Py_TPFLAGS_DEFAULT, + .slots = ReaderIter_Type_slots, +}; - Reader_Type.tp_new = PyType_GenericNew; - if (PyType_Ready(&Reader_Type)) { - return NULL; +static PyMethodDef MaxMindDB_methods[] = {{NULL, NULL, 0, NULL}}; + +// ============================================================================= +// Module state management functions for PEP 489 +// ============================================================================= + +static int maxminddb_traverse(PyObject *module, visitproc visit, void *arg) { + maxminddb_state *state = get_maxminddb_state(module); + Py_VISIT(state->Reader_Type); + Py_VISIT(state->ReaderIter_Type); + Py_VISIT(state->Metadata_Type); + Py_VISIT(state->MaxMindDB_error); + Py_VISIT(state->ipaddress_ip_network); + return 0; +} + +static int maxminddb_clear(PyObject *module) { + maxminddb_state *state = get_maxminddb_state(module); + Py_CLEAR(state->Reader_Type); + Py_CLEAR(state->ReaderIter_Type); + Py_CLEAR(state->Metadata_Type); + Py_CLEAR(state->MaxMindDB_error); + Py_CLEAR(state->ipaddress_ip_network); + return 0; +} + +static void maxminddb_free(void *module) { + maxminddb_clear((PyObject *)module); +} + +static int maxminddb_exec(PyObject *module) { + maxminddb_state *state = get_maxminddb_state(module); + + // Create heap types + state->Reader_Type = + PyType_FromModuleAndSpec(module, &Reader_Type_spec, NULL); + if (state->Reader_Type == NULL) { + return -1; + } + if (PyModule_AddObject(module, "Reader", Py_NewRef(state->Reader_Type)) < + 0) { + return -1; } - Py_INCREF(&Reader_Type); - PyModule_AddObject(m, "Reader", (PyObject *)&Reader_Type); - Metadata_Type.tp_new = PyType_GenericNew; - if (PyType_Ready(&Metadata_Type)) { - return NULL; + state->Metadata_Type = + PyType_FromModuleAndSpec(module, &Metadata_Type_spec, NULL); + if (state->Metadata_Type == NULL) { + return -1; } - Py_INCREF(&Metadata_Type); - PyModule_AddObject(m, "Metadata", (PyObject *)&Metadata_Type); + if (PyModule_AddObject( + module, "Metadata", Py_NewRef(state->Metadata_Type)) < 0) { + return -1; + } + + state->ReaderIter_Type = + PyType_FromModuleAndSpec(module, &ReaderIter_Type_spec, NULL); + if (state->ReaderIter_Type == NULL) { + return -1; + } + // ReaderIter is not exposed to Python directly, so no need to add to module + // Import error class from maxminddb.errors PyObject *error_mod = PyImport_ImportModule("maxminddb.errors"); if (error_mod == NULL) { - return NULL; + return -1; } - - MaxMindDB_error = PyObject_GetAttrString(error_mod, "InvalidDatabaseError"); + state->MaxMindDB_error = + PyObject_GetAttrString(error_mod, "InvalidDatabaseError"); Py_DECREF(error_mod); - - if (MaxMindDB_error == NULL) { - return NULL; + if (state->MaxMindDB_error == NULL) { + return -1; } - Py_INCREF(MaxMindDB_error); + // Import ip_network from ipaddress PyObject *ipaddress_mod = PyImport_ImportModule("ipaddress"); if (ipaddress_mod == NULL) { - return NULL; + return -1; } - - ipaddress_ip_network = PyObject_GetAttrString(ipaddress_mod, "ip_network"); + state->ipaddress_ip_network = + PyObject_GetAttrString(ipaddress_mod, "ip_network"); Py_DECREF(ipaddress_mod); + if (state->ipaddress_ip_network == NULL) { + return -1; + } - if (ipaddress_ip_network == NULL) { - return NULL; + // Add error class to module for backwards compatibility + if (PyModule_AddObject(module, + "InvalidDatabaseError", + Py_NewRef(state->MaxMindDB_error)) < 0) { + return -1; } - Py_INCREF(ipaddress_ip_network); - /* We primarily add it to the module for backwards compatibility */ - PyModule_AddObject(m, "InvalidDatabaseError", MaxMindDB_error); + return 0; +} + +static PyModuleDef_Slot maxminddb_slots[] = { + {Py_mod_exec, maxminddb_exec}, +#if (PY_MAJOR_VERSION > 3) || (PY_MAJOR_VERSION == 3 && PY_MINOR_VERSION >= 12) + {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_SUPPORTED}, +#endif +#ifndef MAXMINDDB_USE_GIL_ONLY + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, +#endif + {0, NULL}}; - return m; +static struct PyModuleDef MaxMindDB_module = { + PyModuleDef_HEAD_INIT, + .m_name = "extension", + .m_doc = "This is a C extension to read MaxMind DB file format", + .m_size = sizeof(maxminddb_state), + .m_methods = MaxMindDB_methods, + .m_slots = maxminddb_slots, + .m_traverse = maxminddb_traverse, + .m_clear = maxminddb_clear, + .m_free = maxminddb_free, +}; + +PyMODINIT_FUNC PyInit_extension(void) { + return PyModuleDef_Init(&MaxMindDB_module); } diff --git a/maxminddb/__init__.py b/maxminddb/__init__.py index d807d65..7654967 100644 --- a/maxminddb/__init__.py +++ b/maxminddb/__init__.py @@ -2,6 +2,7 @@ from __future__ import annotations +from importlib.metadata import version from typing import IO, TYPE_CHECKING, AnyStr, cast from .const import ( @@ -87,8 +88,4 @@ def open_database( return cast("Reader", _extension.Reader(database, mode)) -__title__ = "maxminddb" -__version__ = "2.8.2" -__author__ = "Gregory Oschwald" -__license__ = "Apache License, Version 2.0" -__copyright__ = "Copyright 2013-2025 MaxMind, Inc." +__version__ = version("maxminddb") diff --git a/maxminddb/const.py b/maxminddb/const.py index 959078d..0f7e982 100644 --- a/maxminddb/const.py +++ b/maxminddb/const.py @@ -1,8 +1,47 @@ """Constants used in the API.""" -MODE_AUTO = 0 -MODE_MMAP_EXT = 1 -MODE_MMAP = 2 -MODE_FILE = 4 -MODE_MEMORY = 8 -MODE_FD = 16 +from enum import IntEnum + + +class Mode(IntEnum): + """Database open modes. + + These modes control how the MaxMind DB file is opened and read. + """ + + AUTO = 0 + """Try MODE_MMAP_EXT, MODE_MMAP, MODE_FILE in that order. Default mode.""" + + MMAP_EXT = 1 + """Use the C extension with memory map.""" + + MMAP = 2 + """Read from memory map. Pure Python.""" + + FILE = 4 + """Read database as standard file. Pure Python.""" + + MEMORY = 8 + """Load database into memory. Pure Python.""" + + FD = 16 + """Database is a file descriptor, not a path. This mode implies MODE_MEMORY.""" + + +# Backward compatibility: export both enum members and old-style constants +MODE_AUTO = Mode.AUTO +MODE_MMAP_EXT = Mode.MMAP_EXT +MODE_MMAP = Mode.MMAP +MODE_FILE = Mode.FILE +MODE_MEMORY = Mode.MEMORY +MODE_FD = Mode.FD + +__all__ = [ + "MODE_AUTO", + "MODE_FD", + "MODE_FILE", + "MODE_MEMORY", + "MODE_MMAP", + "MODE_MMAP_EXT", + "Mode", +] diff --git a/maxminddb/decoder.py b/maxminddb/decoder.py index fc03958..8f67a7d 100644 --- a/maxminddb/decoder.py +++ b/maxminddb/decoder.py @@ -10,13 +10,16 @@ except ImportError: mmap = None # type: ignore[assignment] - from maxminddb.errors import InvalidDatabaseError if TYPE_CHECKING: + from collections.abc import Callable + from maxminddb.file import FileBuffer from maxminddb.types import Record + DecoderFunc = Callable[["Decoder", int, int], tuple[Record, int]] + class Decoder: """Decoder for the data section of the MaxMind DB.""" @@ -118,7 +121,7 @@ def _decode_utf8_string(self, size: int, offset: int) -> tuple[str, int]: new_offset = offset + size return self._buffer[offset:new_offset].decode("utf-8"), new_offset - _type_decoder: ClassVar = { + _type_decoder: ClassVar[dict[int, DecoderFunc]] = { 1: _decode_pointer, 2: _decode_utf8_string, 3: _decode_double, diff --git a/maxminddb/extension.pyi b/maxminddb/extension.pyi index ca1b983..1758e81 100644 --- a/maxminddb/extension.pyi +++ b/maxminddb/extension.pyi @@ -1,6 +1,7 @@ """C extension database reader and related classes.""" -from ipaddress import IPv4Address, IPv6Address +from collections.abc import Iterator +from ipaddress import IPv4Address, IPv4Network, IPv6Address, IPv6Network from os import PathLike from typing import IO, Any, AnyStr @@ -56,6 +57,7 @@ class Reader: def metadata(self) -> Metadata: """Return the metadata associated with the MaxMind DB file.""" + def __iter__(self) -> Iterator[tuple[IPv4Network | IPv6Network, Record]]: ... def __enter__(self) -> Self: ... def __exit__(self, *args) -> None: ... # noqa: ANN002 diff --git a/maxminddb/reader.py b/maxminddb/reader.py index 9997dff..bf98f53 100644 --- a/maxminddb/reader.py +++ b/maxminddb/reader.py @@ -10,6 +10,7 @@ import contextlib import ipaddress import struct +from dataclasses import dataclass from ipaddress import IPv4Address, IPv6Address from typing import IO, TYPE_CHECKING, Any, AnyStr @@ -307,6 +308,7 @@ def __enter__(self) -> Self: return self +@dataclass(kw_only=True, frozen=True) class Metadata: """Metadata for the MaxMind DB reader.""" @@ -323,19 +325,13 @@ class Metadata: """ build_epoch: int - """ - The Unix epoch for the build time of the database. - """ + """The Unix epoch for the build time of the database.""" database_type: str - """ - A string identifying the database type, e.g., "GeoIP2-City". - """ + """A string identifying the database type, e.g., "GeoIP2-City".""" description: dict[str, str] - """ - A map from locales to text descriptions of the database. - """ + """A map from locales to text descriptions of the database.""" ip_version: int """ @@ -345,50 +341,20 @@ class Metadata: """ languages: list[str] - """ - A list of locale codes supported by the database. - """ + """A list of locale codes supported by the database.""" node_count: int - """ - The number of nodes in the database. - """ + """The number of nodes in the database.""" record_size: int - """ - The bit size of a record in the search tree. - """ - - def __init__(self, **kwargs) -> None: - """Create new Metadata object. kwargs are key/value pairs from spec.""" - # Although I could just update __dict__, that is less obvious and it - # doesn't work well with static analysis tools and some IDEs - self.node_count = kwargs["node_count"] - self.record_size = kwargs["record_size"] - self.ip_version = kwargs["ip_version"] - self.database_type = kwargs["database_type"] - self.languages = kwargs["languages"] - self.binary_format_major_version = kwargs["binary_format_major_version"] - self.binary_format_minor_version = kwargs["binary_format_minor_version"] - self.build_epoch = kwargs["build_epoch"] - self.description = kwargs["description"] + """The bit size of a record in the search tree.""" @property def node_byte_size(self) -> int: - """The size of a node in bytes. - - :type: int - """ + """The size of a node in bytes.""" return self.record_size // 4 @property def search_tree_size(self) -> int: - """The size of the search tree. - - :type: int - """ + """The size of the search tree.""" return self.node_count * self.node_byte_size - - def __repr__(self) -> str: - args = ", ".join(f"{k}={v!r}" for k, v in self.__dict__.items()) - return f"{self.__module__}.{self.__class__.__name__}({args})" diff --git a/maxminddb/types.py b/maxminddb/types.py index 4c0c478..fa14eef 100644 --- a/maxminddb/types.py +++ b/maxminddb/types.py @@ -6,13 +6,10 @@ Primitive: TypeAlias = AnyStr | bool | float | int +RecordList: TypeAlias = list["Record"] +"""RecordList is a type for lists in a database record.""" -class RecordList(list["Record"]): - """RecordList is a type for lists in a database record.""" - - -class RecordDict(dict[str, "Record"]): - """RecordDict is a type for dicts in a database record.""" - +RecordDict: TypeAlias = dict[str, "Record"] +"""RecordDict is a type for dicts in a database record.""" Record: TypeAlias = Primitive | RecordList | RecordDict diff --git a/setup.py b/setup.py index d36cbce..3d966db 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,4 @@ import os -import re import sys from setuptools import Extension, setup @@ -9,7 +8,6 @@ cmdclass = {} PYPY = hasattr(sys, "pypy_version_info") -JYTHON = sys.platform.startswith("java") if os.name == "nt": # Disable unknown pragma warning @@ -65,8 +63,9 @@ class BuildFailed(Exception): - def __init__(self) -> None: - self.cause = sys.exc_info()[1] + def __init__(self, cause: Exception) -> None: + super().__init__() + self.cause = cause class ve_build_ext(build_ext): @@ -75,38 +74,24 @@ class ve_build_ext(build_ext): def run(self) -> None: try: build_ext.run(self) - except PlatformError: - raise BuildFailed + except PlatformError as ex: + raise BuildFailed(ex) from ex def build_extension(self, ext) -> None: try: build_ext.build_extension(self, ext) - except ext_errors: - raise BuildFailed - except ValueError: + except ext_errors as ex: + raise BuildFailed(ex) from ex + except ValueError as ex: # this can happen on Windows 64 bit, see Python issue 7511 - if "'path'" in str(sys.exc_info()[1]): - raise BuildFailed + if "'path'" in str(ex): + raise BuildFailed(ex) from ex raise cmdclass["build_ext"] = ve_build_ext -ROOT = os.path.dirname(__file__) - -with open(os.path.join(ROOT, "README.rst"), "rb") as fd: - README = fd.read().decode("utf8") - -with open(os.path.join(ROOT, "maxminddb", "__init__.py"), "rb") as fd: - maxminddb_text = fd.read().decode("utf8") - VERSION = ( - re.compile(r".*__version__ = \"(.*?)\"", re.DOTALL) - .match(maxminddb_text) - .group(1) - ) - - def status_msgs(*msgs): print("*" * 75) for msg in msgs: @@ -131,33 +116,26 @@ def run_setup(with_cext) -> None: kwargs["ext_modules"] = ext_module loc_cmdclass["bdist_wheel"] = bdist_wheel - setup(version=VERSION, cmdclass=loc_cmdclass, **kwargs) + setup(cmdclass=loc_cmdclass, **kwargs) -if JYTHON: +try: + run_setup(True) +except BuildFailed as exc: + if os.getenv("MAXMINDDB_REQUIRE_EXTENSION"): + raise + status_msgs( + exc.cause, + "WARNING: The C extension could not be compiled, " + + "speedups are not enabled.", + "Failure information, if any, is above.", + "Retrying the build without the C extension now.", + ) + run_setup(False) + status_msgs( - "WARNING: Disabling C extension due to Python platform.", + "WARNING: The C extension could not be compiled, " + + "speedups are not enabled.", "Plain-Python build succeeded.", ) -else: - try: - run_setup(True) - except BuildFailed as exc: - if os.getenv("MAXMINDDB_REQUIRE_EXTENSION"): - raise - status_msgs( - exc.cause, - "WARNING: The C extension could not be compiled, " - + "speedups are not enabled.", - "Failure information, if any, is above.", - "Retrying the build without the C extension now.", - ) - - run_setup(False) - - status_msgs( - "WARNING: The C extension could not be compiled, " - + "speedups are not enabled.", - "Plain-Python build succeeded.", - ) diff --git a/tests/reader_test.py b/tests/reader_test.py index 3cea5d9..b2c3edd 100644 --- a/tests/reader_test.py +++ b/tests/reader_test.py @@ -441,7 +441,7 @@ def test_metadata_unknown_attribute(self) -> None: metadata = reader.metadata() with self.assertRaisesRegex( AttributeError, - "'Metadata' object has no attribute 'blah'", + r"'(maxminddb\.extension\.)?Metadata' object has no attribute 'blah'", ): metadata.blah # type: ignore[attr-defined] # noqa: B018 reader.close()