Merge pull request #1344 from tahoe-lafs/3072-python-3.12-support

Python 3.12 support, compatability with Eliot 1.15

Fixes ticket:4071

.circleci/create-virtualenv.sh

@@ -46,8 +46,8 @@ export PIP_FIND_LINKS="file://${WHEELHOUSE_PATH}"
 # setuptools 45 requires Python 3.5 or newer.  Even though we upgraded pip
 # above, it may still not be able to get us a compatible version unless we
 # explicitly ask for one.
-"${PIP}" install --upgrade setuptools==44.0.0 wheel
+"${PIP}" install --upgrade setuptools wheel
 
 # Just about every user of this image wants to use tox from the bootstrap
 # virtualenv so go ahead and install it now.
-"${PIP}" install "tox~=3.0"
+"${PIP}" install "tox~=4.0"

.github/workflows/ci.yml

@@ -45,16 +45,17 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          # On macOS don't bother with 3.8, just to get faster builds.
           - os: macos-12
-            python-version: "3.9"
-          - os: macos-12
-            python-version: "3.11"
+            python-version: "3.12"
           # We only support PyPy on Linux at the moment.
           - os: ubuntu-latest
             python-version: "pypy-3.8"
           - os: ubuntu-latest
             python-version: "pypy-3.9"
+          - os: ubuntu-latest
+            python-version: "3.12"
+          - os: windows-latest
+            python-version: "3.12"
 
     steps:
       # See https://github.com/actions/checkout. A fetch-depth of 0
@@ -72,7 +73,7 @@ jobs:
 
       - name: Install Python packages
         run: |
-          pip install --upgrade "tox<4" tox-gh-actions setuptools
+          pip install --upgrade tox tox-gh-actions setuptools
           pip list
 
       - name: Display tool versions
@@ -169,7 +170,7 @@ jobs:
           - false
         include:
           - os: ubuntu-20.04
-            python-version: "3.10"
+            python-version: "3.12"
             force-foolscap: true
     steps:
 
@@ -204,7 +205,7 @@ jobs:
 
       - name: Install Python packages
         run: |
-          pip install --upgrade "tox<4"
+          pip install --upgrade tox
           pip list
 
       - name: Display tool versions
@@ -264,7 +265,7 @@ jobs:
 
       - name: Install Python packages
         run: |
-          pip install --upgrade "tox<4"
+          pip install --upgrade tox
           pip list
 
       - name: Display tool versions

integration/test_tor.py

@@ -30,6 +30,7 @@
 if sys.platform.startswith('win'):
     pytest.skip('Skipping Tor tests on Windows', allow_module_level=True)
 
+@pytest.mark.skipif(sys.version_info[:2] > (3, 11), reason='Chutney still does not support 3.12')
 @pytest_twisted.inlineCallbacks
 def test_onion_service_storage(reactor, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl):
     """
@@ -140,6 +141,7 @@ def _create_anonymous_node(reactor, name, web_port, request, temp_dir, flog_gath
     print("okay, launched")
     return result
 
+@pytest.mark.skipif(sys.version_info[:2] > (3, 11), reason='Chutney still does not support 3.12')
 @pytest.mark.skipif(sys.platform.startswith('darwin'), reason='This test has issues on macOS')
 @pytest_twisted.inlineCallbacks
 def test_anonymous_client(reactor, request, temp_dir, flog_gatherer, tor_network, introducer_furl):

newsfragments/3072.feature

@@ -0,0 +1 @@
+Added support for Python 3.12, and work with Eliot 1.15

pyproject.toml

@@ -0,0 +1,3 @@
+[build-system]
+requires = ["setuptools"]
+build-backend = "setuptools.build_meta"

setup.py

@@ -112,7 +112,7 @@ def read_version_py(infname):
     "magic-wormhole >= 0.10.2",
 
     # We want a new enough version to support custom JSON encoders.
-    "eliot < 1.15.0",  # temporary cap, to be fixed in PR #1344
+    "eliot >= 1.14.0",
 
     "pyrsistent",
 
@@ -157,10 +157,6 @@ def read_version_py(infname):
     "filelock",
 ]
 
-setup_requires = [
-    'setuptools >= 28.8.0',  # for PEP-440 style versions
-]
-
 tor_requires = [
     # 23.5 added support for custom TLS contexts in web_agent(), which is
     # needed for the HTTP storage client to run over Tor.
@@ -384,8 +380,8 @@ def run(self):
       package_dir = {'':'src'},
       packages=find_packages('src') + ['allmydata.test.plugins'],
       classifiers=trove_classifiers,
-      # We support Python 3.8 or later, 3.12 is untested for now
-      python_requires=">=3.8, <3.12",
+      # We support Python 3.8 or later, 3.13 is untested for now
+      python_requires=">=3.8, <3.13",
       install_requires=install_requires,
       extras_require={
           # Duplicate the Twisted pywin32 dependency here.  See
@@ -409,8 +405,8 @@ def run(self):
               # selected here are just the current versions at the time.
               # Bumping them to keep up with future releases is fine as long
               # as those releases are known to actually work.
-              "pip==22.0.3",
-              "wheel==0.37.1",
+              "pip==23.3.1",
+              "wheel==0.41.3",
               "subunitreporter==23.8.0",
               "python-subunit==1.4.2",
               "junitxml==0.7",
@@ -446,7 +442,6 @@ def run(self):
                     "allmydata": ["ported-modules.txt"],
                     },
       include_package_data=True,
-      setup_requires=setup_requires,
       entry_points={
           'console_scripts': [
               'tahoe = allmydata.scripts.runner:run',

src/allmydata/__init__.py

@@ -3,16 +3,6 @@
 
 community web site: U{https://tahoe-lafs.org/}
 """
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-from __future__ import unicode_literals
-
-from future.utils import PY2, PY3
-if PY2:
-    # Don't import future str() so we don't break Foolscap serialization on Python 2.
-    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, max, min  # noqa: F401
-    from past.builtins import unicode as str
 
 __all__ = [
     "__version__",
@@ -52,12 +42,6 @@
 # https://tahoe-lafs.org/trac/tahoe-lafs/wiki/Versioning
 __full_version__ = __appname__ + '/' + str(__version__)
 
-
-# Install Python 3 module locations in Python 2:
-from future import standard_library
-standard_library.install_aliases()
-
-
 # Monkey-patch 3rd party libraries:
 from ._monkeypatch import patch
 patch()
@@ -72,8 +56,7 @@
 #
 # Also note that BytesWarnings only happen if Python is run with -b option, so
 # in practice this should only affect tests.
-if PY3:
-    import warnings
-    # Error on BytesWarnings, to catch things like str(b""), but only for
-    # allmydata code.
-    warnings.filterwarnings("error", category=BytesWarning, module=".*allmydata.*")
+import warnings
+# Error on BytesWarnings, to catch things like str(b""), but only for
+# allmydata code.
+warnings.filterwarnings("error", category=BytesWarning, module=".*allmydata.*")

src/allmydata/client.py

@@ -12,6 +12,7 @@
 from functools import partial
 from configparser import NoSectionError
 
+from six import ensure_text
 from foolscap.furl import (
     decode_furl,
 )
@@ -989,6 +990,9 @@ def load_static_servers(self):
             static_servers = servers_yaml.get("storage", {})
             log.msg("found %d static servers in private/servers.yaml" %
                     len(static_servers))
+            static_servers = {
+                ensure_text(key): value for (key, value) in static_servers.items()
+            }
             self.storage_broker.set_static_servers(static_servers)
         except EnvironmentError:
             pass

src/allmydata/immutable/upload.py

@@ -78,7 +78,7 @@ def _serialize_peers(peers):
 
 def _serialize_existing_shares(existing_shares):
     return {
-        server: list(shares)
+        ensure_str(server): list(shares)
         for (server, shares)
         in existing_shares.items()
     }
@@ -91,7 +91,7 @@ def _serialize_existing_shares(existing_shares):
 
 def _serialize_happiness_mappings(happiness_mappings):
     return {
-        sharenum: base32.b2a(serverid)
+        str(sharenum): ensure_str(base32.b2a(serverid))
         for (sharenum, serverid)
         in happiness_mappings.items()
     }
@@ -112,7 +112,7 @@ def _serialize_happiness_mappings(happiness_mappings):
     u"upload_trackers",
     lambda trackers: list(
         dict(
-            server=tracker.get_name(),
+            server=ensure_str(tracker.get_name()),
             shareids=sorted(tracker.buckets.keys()),
         )
         for tracker
@@ -123,7 +123,7 @@ def _serialize_happiness_mappings(happiness_mappings):
 
 _ALREADY_SERVERIDS = Field(
     u"already_serverids",
-    lambda d: d,
+    lambda d: {str(k): v for k, v in d.items()},
     u"Some servers which are already holding some shares that we were interested in uploading.",
 )
 

src/allmydata/storage_client.py

@@ -32,7 +32,6 @@
 
 from __future__ import annotations
 
-from six import ensure_text
 from typing import Union, Callable, Any, Optional, cast, Dict
 from os import urandom
 import re
@@ -273,7 +272,6 @@ def set_static_servers(self, servers):
         # doesn't really matter but it makes the logging behavior more
         # predictable and easier to test (and at least one test does depend on
         # this sorted order).
-        servers = {ensure_text(key): value for (key, value) in servers.items()}
         for (server_id, server) in sorted(servers.items()):
             try:
                 storage_server = self._make_storage_server(

src/allmydata/test/__init__.py

@@ -125,5 +125,5 @@ def _listening(lp):
     initialize()
 
 from eliot import to_file
-from allmydata.util.eliotutil import eliot_json_encoder
-to_file(open("eliot.log", "wb"), encoder=eliot_json_encoder)
+from allmydata.util.jsonbytes import AnyBytesJSONEncoder
+to_file(open("eliot.log", "wb"), encoder=AnyBytesJSONEncoder)

src/allmydata/test/common.py

@@ -1352,6 +1352,26 @@ def mktemp(self):
     def assertRaises(self, *a, **kw):
         return self._dummyCase.assertRaises(*a, **kw)
 
+    def failUnless(self, *args, **kwargs):
+        """Backwards compatibility method."""
+        self.assertTrue(*args, **kwargs)
+
+    def failIf(self, *args, **kwargs):
+        """Backwards compatibility method."""
+        self.assertFalse(*args, **kwargs)
+
+    def failIfEqual(self, *args, **kwargs):
+        """Backwards compatibility method."""
+        self.assertNotEqual(*args, **kwargs)
+
+    def failUnlessEqual(self, *args, **kwargs):
+        """Backwards compatibility method."""
+        self.assertEqual(*args, **kwargs)
+
+    def failUnlessReallyEqual(self, *args, **kwargs):
+        """Backwards compatibility method."""
+        self.assertReallyEqual(*args, **kwargs)
+
 
 class SyncTestCase(_TestCaseMixin, TestCase):
     """

src/allmydata/test/eliotutil.py

@@ -29,6 +29,7 @@
     ILogger,
 )
 from eliot.testing import (
+    MemoryLogger,
     swap_logger,
     check_for_errors,
 )
@@ -37,8 +38,8 @@
     MonkeyPatcher,
 )
 
-from ..util.eliotutil import (
-    MemoryLogger,
+from ..util.jsonbytes import (
+    AnyBytesJSONEncoder
 )
 
 _NAME = Field.for_types(
@@ -146,7 +147,7 @@ def with_logging(
     """
     @wraps(test_method)
     def run_with_logging(*args, **kwargs):
-        validating_logger = MemoryLogger()
+        validating_logger = MemoryLogger(encoder=AnyBytesJSONEncoder)
         original = swap_logger(None)
         try:
             swap_logger(_TwoLoggers(original, validating_logger))

src/allmydata/test/test_client.py

@@ -850,6 +850,7 @@ def setUp(self):
             actionType=u"storage-client:broker:set-static-servers",
             succeeded=True,
         ),
+        encoder_=json.AnyBytesJSONEncoder
     )
     def test_static_servers(self, logger):
         """
@@ -884,6 +885,7 @@ def test_static_servers(self, logger):
             actionType=u"storage-client:broker:make-storage-server",
             succeeded=False,
         ),
+        encoder_=json.AnyBytesJSONEncoder
     )
     def test_invalid_static_server(self, logger):
         """

src/allmydata/test/test_crypto.py

@@ -507,7 +507,7 @@ def test_remove_prefix_good(self):
         """
         remove a simple prefix properly
         """
-        self.assertEquals(
+        self.assertEqual(
             remove_prefix(b"foobar", b"foo"),
             b"bar"
         )
@@ -523,7 +523,7 @@ def test_remove_prefix_zero(self):
         """
         removing a zero-length prefix does nothing
         """
-        self.assertEquals(
+        self.assertEqual(
             remove_prefix(b"foobar", b""),
             b"foobar",
         )
@@ -532,7 +532,7 @@ def test_remove_prefix_entire_string(self):
         """
         removing a prefix which is the whole string is empty
         """
-        self.assertEquals(
+        self.assertEqual(
             remove_prefix(b"foobar", b"foobar"),
             b"",
         )

src/allmydata/test/test_eliotutil.py

@@ -47,7 +47,6 @@
     Message,
     MessageType,
     fields,
-    FileDestination,
     MemoryLogger,
 )
 from eliot.twisted import DeferredContext
@@ -64,7 +63,6 @@
 from twisted.internet import reactor
 
 from ..util.eliotutil import (
-    eliot_json_encoder,
     log_call_deferred,
     _parse_destination_description,
     _EliotLogging,
@@ -188,8 +186,8 @@ def test_stdout(self):
         """
         reactor = object()
         self.assertThat(
-            _parse_destination_description("file:-")(reactor),
-            Equals(FileDestination(stdout, encoder=eliot_json_encoder)),
+            _parse_destination_description("file:-")(reactor).file,
+            Equals(stdout),
         )