From d7dd96822d80479b02ac580e1ed0c86823e24e45 Mon Sep 17 00:00:00 2001 From: ddelange <14880945+ddelange@users.noreply.github.com> Date: Tue, 14 Oct 2025 14:09:18 +0300 Subject: [PATCH 1/5] Add python 3.14 to CI --- .github/workflows/ci.yml | 2 +- tox.ini | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 049880d..0a9cfe9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,7 +6,7 @@ jobs: fail-fast: false matrix: os: ["ubuntu-latest"] - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13", "3.14"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13", "3.13t", "3.14", "3.14t"] include: - os: macos-latest python-version: "3.x" diff --git a/tox.ini b/tox.ini index 5c1648b..51b12cf 100644 --- a/tox.ini +++ b/tox.ini @@ -10,6 +10,9 @@ envlist = py311, py312, py313, + py313t, + py314, + py314t, mypy [testenv] From f2ac98d8aa7464165984068de9e484d0321cd4f3 Mon Sep 17 00:00:00 2001 From: ddelange <14880945+ddelange@users.noreply.github.com> Date: Tue, 14 Oct 2025 14:23:53 +0300 Subject: [PATCH 2/5] Move lock to global scope --- magic/__init__.py | 67 +++++++++++++++++++++++++---------------------- 1 file changed, 36 insertions(+), 31 deletions(-) diff --git a/magic/__init__.py b/magic/__init__.py index 851b717..86fe4a6 100644 --- a/magic/__init__.py +++ b/magic/__init__.py @@ -109,7 +109,6 @@ def __init__( self.flags |= MAGIC_NO_CHECK_SIMH self.cookie = magic_open(self.flags) - self.lock = threading.Lock() magic_load(self.cookie, magic_file) @@ -138,34 +137,31 @@ def from_buffer(self, buf): """ Identify the contents of `buf` """ - with self.lock: - try: - # if we're on python3, convert buf to bytes - # otherwise this string is passed as wchar* - # which is not what libmagic expects - # NEXTBREAK: only take bytes - if type(buf) == str and str != bytes: - buf = buf.encode("utf-8", errors="replace") - return maybe_decode(magic_buffer(self.cookie, buf)) - except MagicException as e: - return self._handle509Bug(e) + try: + # if we're on python3, convert buf to bytes + # otherwise this string is passed as wchar* + # which is not what libmagic expects + # NEXTBREAK: only take bytes + if type(buf) == str and str != bytes: + buf = buf.encode("utf-8", errors="replace") + return maybe_decode(magic_buffer(self.cookie, buf)) + except MagicException as e: + return self._handle509Bug(e) def from_file(self, filename): # raise FileNotFoundException or IOError if the file does not exist os.stat(filename, follow_symlinks=self.flags & MAGIC_SYMLINK) - with self.lock: - try: - return maybe_decode(magic_file(self.cookie, filename)) - except MagicException as e: - return self._handle509Bug(e) + try: + return maybe_decode(magic_file(self.cookie, filename)) + except MagicException as e: + return self._handle509Bug(e) def from_descriptor(self, fd): - with self.lock: - try: - return maybe_decode(magic_descriptor(self.cookie, fd)) - except MagicException as e: - return self._handle509Bug(e) + try: + return maybe_decode(magic_descriptor(self.cookie, fd)) + except MagicException as e: + return self._handle509Bug(e) def _handle509Bug(self, e): # libmagic 5.09 has a bug where it might fail to identify the @@ -317,6 +313,9 @@ def coerce_filename(filename): return filename +# libmagic is not thread-safe: guard for concurrent calls on a global scope +LOCK = threading.Lock() + magic_open = libmagic.magic_open magic_open.restype = magic_t magic_open.argtypes = [c_int] @@ -340,7 +339,8 @@ def coerce_filename(filename): def magic_file(cookie, filename): - return _magic_file(cookie, coerce_filename(filename)) + with LOCK: + return _magic_file(cookie, coerce_filename(filename)) _magic_buffer = libmagic.magic_buffer @@ -350,7 +350,8 @@ def magic_file(cookie, filename): def magic_buffer(cookie, buf): - return _magic_buffer(cookie, buf, len(buf)) + with LOCK: + return _magic_buffer(cookie, buf, len(buf)) magic_descriptor = libmagic.magic_descriptor @@ -365,7 +366,8 @@ def magic_buffer(cookie, buf): def magic_descriptor(cookie, fd): - return _magic_descriptor(cookie, fd) + with LOCK: + return _magic_descriptor(cookie, fd) _magic_load = libmagic.magic_load @@ -375,7 +377,8 @@ def magic_descriptor(cookie, fd): def magic_load(cookie, filename): - return _magic_load(cookie, coerce_filename(filename)) + with LOCK: + return _magic_load(cookie, coerce_filename(filename)) magic_setflags = libmagic.magic_setflags @@ -408,15 +411,16 @@ def magic_setparam(cookie, param, val): if not _has_param: raise NotImplementedError("magic_setparam not implemented") v = c_size_t(val) - return _magic_setparam(cookie, param, byref(v)) + with LOCK: + return _magic_setparam(cookie, param, byref(v)) def magic_getparam(cookie, param): if not _has_param: raise NotImplementedError("magic_getparam not implemented") val = c_size_t() - _magic_getparam(cookie, param, byref(val)) - return val.value + with LOCK: + return _magic_getparam(cookie, param, byref(val)).value _has_version = False @@ -427,10 +431,11 @@ def magic_getparam(cookie, param): magic_version.argtypes = [] -def version(): +def version(lock=None): if not _has_version: raise NotImplementedError("magic_version not implemented") - return magic_version() + with LOCK: + return magic_version() MAGIC_NONE = 0x000000 # No flags From 5b1d27cbd2d1aaad4f79a199387a8ac1a960c9a5 Mon Sep 17 00:00:00 2001 From: ddelange <14880945+ddelange@users.noreply.github.com> Date: Tue, 14 Oct 2025 14:39:45 +0300 Subject: [PATCH 3/5] Add test --- magic/__init__.py | 5 +++-- test/python_magic_test.py | 25 +++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/magic/__init__.py b/magic/__init__.py index 86fe4a6..cd5b279 100644 --- a/magic/__init__.py +++ b/magic/__init__.py @@ -420,7 +420,8 @@ def magic_getparam(cookie, param): raise NotImplementedError("magic_getparam not implemented") val = c_size_t() with LOCK: - return _magic_getparam(cookie, param, byref(val)).value + _magic_getparam(cookie, param, byref(val)) + return val.value _has_version = False @@ -431,7 +432,7 @@ def magic_getparam(cookie, param): magic_version.argtypes = [] -def version(lock=None): +def version(): if not _has_version: raise NotImplementedError("magic_version not implemented") with LOCK: diff --git a/test/python_magic_test.py b/test/python_magic_test.py index 5076044..41c1d07 100755 --- a/test/python_magic_test.py +++ b/test/python_magic_test.py @@ -10,6 +10,12 @@ import pytest +try: + from concurrent.futures import ThreadPoolExecutor + HAS_CONCURRENT_FUTURES = True +except ImportError: # python 2.7 + HAS_CONCURRENT_FUTURES = False + # for output which reports a local time os.environ["TZ"] = "GMT" @@ -321,6 +327,25 @@ def test_symlink(self): self.assertRaises(IOError, m_follow.from_file, tmp_broken) + @unittest.skipIf(not HAS_CONCURRENT_FUTURES, "concurrent.futures not available in Python 2.7") + def test_thread_safety(self): + """Test that concurrent from_file calls don't crash (would SEGV without global lock)""" + filename = os.path.join(self.TESTDATA_DIR, "test.pdf") + + m = magic.Magic(mime=True) + + def check_file(_): + result = m.from_file(filename) + self.assertEqual(result, "application/pdf") + return result + + with ThreadPoolExecutor(100) as executor: + results = list(executor.map(check_file, range(100))) + + # All calls should complete successfully + self.assertEqual(len(results), 100) + self.assertTrue(all(r == "application/pdf" for r in results)) + if __name__ == "__main__": unittest.main() From cde4a95c98fe5dabaf9753326a4340f306b971db Mon Sep 17 00:00:00 2001 From: ddelange <14880945+ddelange@users.noreply.github.com> Date: Tue, 14 Oct 2025 13:44:09 +0200 Subject: [PATCH 4/5] Apply suggestions from code review --- .github/workflows/ci.yml | 2 +- tox.ini | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0a9cfe9..c478602 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,7 +6,7 @@ jobs: fail-fast: false matrix: os: ["ubuntu-latest"] - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13", "3.13t", "3.14", "3.14t"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13", "3.14", "3.14t"] include: - os: macos-latest python-version: "3.x" diff --git a/tox.ini b/tox.ini index 51b12cf..01cb7b2 100644 --- a/tox.ini +++ b/tox.ini @@ -10,7 +10,6 @@ envlist = py311, py312, py313, - py313t, py314, py314t, mypy From 46bc96796563ed8728c8d25202d51fb0011f5734 Mon Sep 17 00:00:00 2001 From: ddelange <14880945+ddelange@users.noreply.github.com> Date: Fri, 17 Oct 2025 20:04:26 +0300 Subject: [PATCH 5/5] Revert "Move lock to global scope" This reverts commit f2ac98d8aa7464165984068de9e484d0321cd4f3. --- magic/__init__.py | 64 +++++++++++++++++++++-------------------------- 1 file changed, 29 insertions(+), 35 deletions(-) diff --git a/magic/__init__.py b/magic/__init__.py index cd5b279..851b717 100644 --- a/magic/__init__.py +++ b/magic/__init__.py @@ -109,6 +109,7 @@ def __init__( self.flags |= MAGIC_NO_CHECK_SIMH self.cookie = magic_open(self.flags) + self.lock = threading.Lock() magic_load(self.cookie, magic_file) @@ -137,31 +138,34 @@ def from_buffer(self, buf): """ Identify the contents of `buf` """ - try: - # if we're on python3, convert buf to bytes - # otherwise this string is passed as wchar* - # which is not what libmagic expects - # NEXTBREAK: only take bytes - if type(buf) == str and str != bytes: - buf = buf.encode("utf-8", errors="replace") - return maybe_decode(magic_buffer(self.cookie, buf)) - except MagicException as e: - return self._handle509Bug(e) + with self.lock: + try: + # if we're on python3, convert buf to bytes + # otherwise this string is passed as wchar* + # which is not what libmagic expects + # NEXTBREAK: only take bytes + if type(buf) == str and str != bytes: + buf = buf.encode("utf-8", errors="replace") + return maybe_decode(magic_buffer(self.cookie, buf)) + except MagicException as e: + return self._handle509Bug(e) def from_file(self, filename): # raise FileNotFoundException or IOError if the file does not exist os.stat(filename, follow_symlinks=self.flags & MAGIC_SYMLINK) - try: - return maybe_decode(magic_file(self.cookie, filename)) - except MagicException as e: - return self._handle509Bug(e) + with self.lock: + try: + return maybe_decode(magic_file(self.cookie, filename)) + except MagicException as e: + return self._handle509Bug(e) def from_descriptor(self, fd): - try: - return maybe_decode(magic_descriptor(self.cookie, fd)) - except MagicException as e: - return self._handle509Bug(e) + with self.lock: + try: + return maybe_decode(magic_descriptor(self.cookie, fd)) + except MagicException as e: + return self._handle509Bug(e) def _handle509Bug(self, e): # libmagic 5.09 has a bug where it might fail to identify the @@ -313,9 +317,6 @@ def coerce_filename(filename): return filename -# libmagic is not thread-safe: guard for concurrent calls on a global scope -LOCK = threading.Lock() - magic_open = libmagic.magic_open magic_open.restype = magic_t magic_open.argtypes = [c_int] @@ -339,8 +340,7 @@ def coerce_filename(filename): def magic_file(cookie, filename): - with LOCK: - return _magic_file(cookie, coerce_filename(filename)) + return _magic_file(cookie, coerce_filename(filename)) _magic_buffer = libmagic.magic_buffer @@ -350,8 +350,7 @@ def magic_file(cookie, filename): def magic_buffer(cookie, buf): - with LOCK: - return _magic_buffer(cookie, buf, len(buf)) + return _magic_buffer(cookie, buf, len(buf)) magic_descriptor = libmagic.magic_descriptor @@ -366,8 +365,7 @@ def magic_buffer(cookie, buf): def magic_descriptor(cookie, fd): - with LOCK: - return _magic_descriptor(cookie, fd) + return _magic_descriptor(cookie, fd) _magic_load = libmagic.magic_load @@ -377,8 +375,7 @@ def magic_descriptor(cookie, fd): def magic_load(cookie, filename): - with LOCK: - return _magic_load(cookie, coerce_filename(filename)) + return _magic_load(cookie, coerce_filename(filename)) magic_setflags = libmagic.magic_setflags @@ -411,16 +408,14 @@ def magic_setparam(cookie, param, val): if not _has_param: raise NotImplementedError("magic_setparam not implemented") v = c_size_t(val) - with LOCK: - return _magic_setparam(cookie, param, byref(v)) + return _magic_setparam(cookie, param, byref(v)) def magic_getparam(cookie, param): if not _has_param: raise NotImplementedError("magic_getparam not implemented") val = c_size_t() - with LOCK: - _magic_getparam(cookie, param, byref(val)) + _magic_getparam(cookie, param, byref(val)) return val.value @@ -435,8 +430,7 @@ def magic_getparam(cookie, param): def version(): if not _has_version: raise NotImplementedError("magic_version not implemented") - with LOCK: - return magic_version() + return magic_version() MAGIC_NONE = 0x000000 # No flags