From 53084f76ced048e5516ce1b79d8a3f055eadd389 Mon Sep 17 00:00:00 2001 From: Alexandre Detiste Date: Wed, 28 Feb 2024 00:49:07 +0100 Subject: [PATCH 01/11] remove more Python2 compatibility --- misc/coding_tools/graph-deps.py | 2 +- src/allmydata/_monkeypatch.py | 8 -- src/allmydata/crypto/aes.py | 10 +-- src/allmydata/frontends/sftpd.py | 3 - src/allmydata/node.py | 4 +- src/allmydata/scripts/default_nodedir.py | 7 +- src/allmydata/scripts/runner.py | 6 +- src/allmydata/scripts/slow_operation.py | 6 +- src/allmydata/scripts/tahoe_check.py | 12 ++- src/allmydata/scripts/tahoe_get.py | 4 +- src/allmydata/scripts/tahoe_manifest.py | 10 +-- src/allmydata/storage/common.py | 6 +- src/allmydata/storage/crawler.py | 12 +-- src/allmydata/test/cli/test_backup.py | 9 +-- src/allmydata/test/cli/test_backupdb.py | 2 +- src/allmydata/test/cli/test_check.py | 2 +- src/allmydata/test/cli/test_cli.py | 2 +- src/allmydata/test/cli/test_create_alias.py | 2 +- src/allmydata/test/cli/test_list.py | 13 +-- src/allmydata/test/cli/test_run.py | 2 +- src/allmydata/test/common_util.py | 33 ++------ src/allmydata/test/mutable/test_filenode.py | 2 +- src/allmydata/test/mutable/test_roundtrip.py | 2 +- src/allmydata/test/test_common_util.py | 4 +- src/allmydata/test/test_crawler.py | 16 ++-- src/allmydata/test/test_crypto.py | 15 ++-- src/allmydata/test/test_download.py | 4 - src/allmydata/test/test_encodingutil.py | 53 +++--------- src/allmydata/test/test_i2p_provider.py | 4 +- src/allmydata/test/test_runner.py | 17 +--- src/allmydata/test/test_statistics.py | 2 +- src/allmydata/test/test_storage_web.py | 2 +- src/allmydata/test/test_tor_provider.py | 7 +- src/allmydata/test/test_util.py | 6 +- src/allmydata/test/web/test_grid.py | 2 +- src/allmydata/util/base62.py | 11 +-- src/allmydata/util/encodingutil.py | 59 ++------------ src/allmydata/util/hashutil.py | 9 +-- src/allmydata/util/log.py | 13 +-- src/allmydata/windows/fixups.py | 84 +------------------- 40 files changed, 96 insertions(+), 371 deletions(-) diff --git a/misc/coding_tools/graph-deps.py b/misc/coding_tools/graph-deps.py index 400f3912a5..faa94450a6 100755 --- a/misc/coding_tools/graph-deps.py +++ b/misc/coding_tools/graph-deps.py @@ -24,7 +24,7 @@ import os, sys, subprocess, json, tempfile, zipfile, re, itertools import email.parser from pprint import pprint -from six.moves import StringIO +from io import StringIO import click all_packages = {} # name -> version diff --git a/src/allmydata/_monkeypatch.py b/src/allmydata/_monkeypatch.py index cbd3ddd135..61f12750f3 100644 --- a/src/allmydata/_monkeypatch.py +++ b/src/allmydata/_monkeypatch.py @@ -4,13 +4,5 @@ Ported to Python 3. """ -from future.utils import PY2 - - - def patch(): """Path third-party libraries to make Tahoe-LAFS work.""" - - if not PY2: - # Python 3 doesn't need to monkey patch Foolscap - return diff --git a/src/allmydata/crypto/aes.py b/src/allmydata/crypto/aes.py index 2fa5f86848..fd5617b1d5 100644 --- a/src/allmydata/crypto/aes.py +++ b/src/allmydata/crypto/aes.py @@ -10,8 +10,6 @@ Ported to Python 3. """ -import six - from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives.ciphers import ( Cipher, @@ -79,7 +77,7 @@ def encrypt_data(encryptor, plaintext): """ _validate_cryptor(encryptor, encrypt=True) - if not isinstance(plaintext, six.binary_type): + if not isinstance(plaintext, bytes): raise ValueError('Plaintext must be bytes') return encryptor.update(plaintext) @@ -118,7 +116,7 @@ def decrypt_data(decryptor, plaintext): """ _validate_cryptor(decryptor, encrypt=False) - if not isinstance(plaintext, six.binary_type): + if not isinstance(plaintext, bytes): raise ValueError('Plaintext must be bytes') return decryptor.update(plaintext) @@ -160,7 +158,7 @@ def _validate_key(key): """ confirm `key` is suitable for AES encryption, or raise ValueError """ - if not isinstance(key, six.binary_type): + if not isinstance(key, bytes): raise TypeError('Key must be bytes') if len(key) not in (16, 32): raise ValueError('Key must be 16 or 32 bytes long') @@ -175,7 +173,7 @@ def _validate_iv(iv): """ if iv is None: return DEFAULT_IV - if not isinstance(iv, six.binary_type): + if not isinstance(iv, bytes): raise TypeError('IV must be bytes') if len(iv) != 16: raise ValueError('IV must be 16 bytes long') diff --git a/src/allmydata/frontends/sftpd.py b/src/allmydata/frontends/sftpd.py index eaa89160cf..b775fa49dd 100644 --- a/src/allmydata/frontends/sftpd.py +++ b/src/allmydata/frontends/sftpd.py @@ -45,9 +45,6 @@ from allmydata.util.log import NOISY, OPERATIONAL, WEIRD, \ msg as logmsg, PrefixingLogMixin -if six.PY3: - long = int - def createSFTPError(errorCode, errorMessage): """ diff --git a/src/allmydata/node.py b/src/allmydata/node.py index fdb89e13fa..601f64b930 100644 --- a/src/allmydata/node.py +++ b/src/allmydata/node.py @@ -112,8 +112,8 @@ def formatTimeTahoeStyle(self, when): """ d = datetime.datetime.utcfromtimestamp(when) if d.microsecond: - return d.isoformat(ensure_str(" "))[:-3]+"Z" - return d.isoformat(ensure_str(" ")) + ".000Z" + return d.isoformat(" ")[:-3]+"Z" + return d.isoformat(" ") + ".000Z" PRIV_README = """ This directory contains files which contain private data for the Tahoe node, diff --git a/src/allmydata/scripts/default_nodedir.py b/src/allmydata/scripts/default_nodedir.py index c1711b55c0..fff1201408 100644 --- a/src/allmydata/scripts/default_nodedir.py +++ b/src/allmydata/scripts/default_nodedir.py @@ -3,7 +3,6 @@ """ import sys -import six from allmydata.util.assertutil import precondition from allmydata.util.fileutil import abspath_expanduser_unicode @@ -13,10 +12,10 @@ from allmydata.windows import registry path = registry.get_base_dir_path() if path: - precondition(isinstance(path, six.text_type), path) + precondition(isinstance(path, str), path) _default_nodedir = abspath_expanduser_unicode(path) if _default_nodedir is None: - path = abspath_expanduser_unicode(u"~/.tahoe") - precondition(isinstance(path, six.text_type), path) + path = abspath_expanduser_unicode("~/.tahoe") + precondition(isinstance(path, str), path) _default_nodedir = path diff --git a/src/allmydata/scripts/runner.py b/src/allmydata/scripts/runner.py index 73e2e4b599..16f43e9d84 100644 --- a/src/allmydata/scripts/runner.py +++ b/src/allmydata/scripts/runner.py @@ -65,8 +65,8 @@ class Options(usage.Options): ] optParameters = [ ["node-directory", "d", None, NODEDIR_HELP], - ["wormhole-server", None, u"ws://wormhole.tahoe-lafs.org:4000/v1", "The magic wormhole server to use.", six.text_type], - ["wormhole-invite-appid", None, u"tahoe-lafs.org/invite", "The appid to use on the wormhole server.", six.text_type], + ["wormhole-server", None, u"ws://wormhole.tahoe-lafs.org:4000/v1", "The magic wormhole server to use.", str], + ["wormhole-invite-appid", None, u"tahoe-lafs.org/invite", "The appid to use on the wormhole server.", str], ] def opt_version(self): @@ -262,7 +262,7 @@ def _setup_coverage(reactor, argv): # can we put this _setup_coverage call after we hit # argument-parsing? # ensure_str() only necessary on Python 2. - if six.ensure_str('--coverage') not in sys.argv: + if '--coverage' not in sys.argv: return argv.remove('--coverage') diff --git a/src/allmydata/scripts/slow_operation.py b/src/allmydata/scripts/slow_operation.py index 236c2bf0be..9596fd805e 100644 --- a/src/allmydata/scripts/slow_operation.py +++ b/src/allmydata/scripts/slow_operation.py @@ -2,8 +2,6 @@ Ported to Python 3. """ -from future.utils import PY3 - from six import ensure_str import os, time @@ -81,9 +79,7 @@ def poll(self): if not data["finished"]: return False if self.options.get("raw"): - if PY3: - # need to write bytes! - stdout = stdout.buffer + stdout = stdout.buffer if is_printable_ascii(jdata): stdout.write(jdata) stdout.write(b"\n") diff --git a/src/allmydata/scripts/tahoe_check.py b/src/allmydata/scripts/tahoe_check.py index 91125a3a38..c5ba07db97 100644 --- a/src/allmydata/scripts/tahoe_check.py +++ b/src/allmydata/scripts/tahoe_check.py @@ -2,7 +2,7 @@ Ported to Python 3. """ -from six import ensure_str, ensure_text +from six import ensure_text from urllib.parse import quote as url_quote import json @@ -168,7 +168,7 @@ def lineReceived(self, line): # LIT files and directories do not have a "summary" field. summary = cr.get("summary", "Healthy (LIT)") # When Python 2 is dropped the ensure_text()/ensure_str() will be unnecessary. - print(ensure_text(ensure_str("%s: %s") % (quote_path(path), quote_output(summary, quotemarks=False)), + print(ensure_text("%s: %s" % (quote_path(path), quote_output(summary, quotemarks=False)), encoding=get_io_encoding()), file=stdout) # always print out corrupt shares @@ -246,13 +246,11 @@ def lineReceived(self, line): if not path: path = [""] # we don't seem to have a summary available, so build one - # When Python 2 is dropped the ensure_text/ensure_str crap can be - # dropped. if was_healthy: - summary = ensure_str("healthy") + summary = "healthy" else: - summary = ensure_str("not healthy") - print(ensure_text(ensure_str("%s: %s") % (quote_path(path), summary), + summary = "not healthy" + print(ensure_text("%s: %s" % (quote_path(path), summary), encoding=get_io_encoding()), file=stdout) # always print out corrupt shares diff --git a/src/allmydata/scripts/tahoe_get.py b/src/allmydata/scripts/tahoe_get.py index f22d3d2936..8e688e4328 100644 --- a/src/allmydata/scripts/tahoe_get.py +++ b/src/allmydata/scripts/tahoe_get.py @@ -2,8 +2,6 @@ Ported to Python 3. """ -from future.utils import PY3 - from urllib.parse import quote as url_quote from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, escape_path, \ UnknownAliasError @@ -36,7 +34,7 @@ def get(options): outf = stdout # Make sure we can write bytes; on Python 3 stdout is Unicode by # default. - if PY3 and getattr(outf, "encoding", None) is not None: + if getattr(outf, "encoding", None) is not None: outf = outf.buffer while True: data = resp.read(4096) diff --git a/src/allmydata/scripts/tahoe_manifest.py b/src/allmydata/scripts/tahoe_manifest.py index 0cd6100bf4..ebff2e893b 100644 --- a/src/allmydata/scripts/tahoe_manifest.py +++ b/src/allmydata/scripts/tahoe_manifest.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from future.utils import PY3 - -from six import ensure_str - from urllib.parse import quote as url_quote import json from twisted.protocols.basic import LineOnlyReceiver @@ -56,8 +52,7 @@ def run(self, options): # use Twisted to split this into lines self.in_error = False # Writing bytes, so need binary stdout. - if PY3: - stdout = stdout.buffer + stdout = stdout.buffer while True: chunk = resp.read(100) if not chunk: @@ -99,8 +94,7 @@ def lineReceived(self, line): if vc: print(quote_output(vc, quotemarks=False), file=stdout) else: - # ensure_str() only necessary for Python 2. - print(ensure_str("%s %s") % ( + print("%s %s" % ( quote_output(d["cap"], quotemarks=False), quote_path(d["path"], quotemarks=False)), file=stdout) diff --git a/src/allmydata/storage/common.py b/src/allmydata/storage/common.py index c0d77a6b72..c76e01052d 100644 --- a/src/allmydata/storage/common.py +++ b/src/allmydata/storage/common.py @@ -2,8 +2,6 @@ Ported to Python 3. """ -from future.utils import PY3 - import os.path from allmydata.util import base32 @@ -43,7 +41,5 @@ def storage_index_to_dir(storageindex): Returns native string. """ sia = si_b2a(storageindex) - if PY3: - # On Python 3 we expect paths to be unicode. - sia = sia.decode("ascii") + sia = sia.decode("ascii") return os.path.join(sia[:2], sia) diff --git a/src/allmydata/storage/crawler.py b/src/allmydata/storage/crawler.py index a464449edf..613f04bfbe 100644 --- a/src/allmydata/storage/crawler.py +++ b/src/allmydata/storage/crawler.py @@ -4,9 +4,6 @@ Ported to Python 3. """ - -from future.utils import PY2, PY3 - import os import time import json @@ -150,10 +147,7 @@ def _dump_json_to_file(js, afile): """ with afile.open("wb") as f: data = json.dumps(js) - if PY2: - f.write(data) - else: - f.write(data.encode("utf8")) + f.write(data.encode("utf8")) class _LeaseStateSerializer(object): @@ -249,9 +243,7 @@ def __init__(self, server, statefile, allowed_cpu_percentage=None): self._state_serializer = _LeaseStateSerializer(statefile) self.prefixes = [si_b2a(struct.pack(">H", i << (16-10)))[:2] for i in range(2**10)] - if PY3: - # On Python 3 we expect the paths to be unicode, not bytes. - self.prefixes = [p.decode("ascii") for p in self.prefixes] + self.prefixes = [p.decode("ascii") for p in self.prefixes] self.prefixes.sort() self.timer = None self.bucket_cache = (None, []) diff --git a/src/allmydata/test/cli/test_backup.py b/src/allmydata/test/cli/test_backup.py index c951e7ee35..7ff1a14d0b 100644 --- a/src/allmydata/test/cli/test_backup.py +++ b/src/allmydata/test/cli/test_backup.py @@ -2,10 +2,8 @@ Ported to Python 3. """ -from future.utils import PY2 - import os.path -from six.moves import cStringIO as StringIO +from io import StringIO from datetime import timedelta import re @@ -421,10 +419,7 @@ def call_file(name, *args, **kwargs): else: return original_open(name, *args, **kwargs) - if PY2: - from allmydata.scripts import cli as module_to_patch - else: - import builtins as module_to_patch + import builtins as module_to_patch patcher = MonkeyPatcher((module_to_patch, 'open', call_file)) patcher.runWithPatches(parse_options, basedir, "backup", ['--exclude-from-utf-8', unicode_to_argv(exclude_file), 'from', 'to']) self.failUnless(ns.called) diff --git a/src/allmydata/test/cli/test_backupdb.py b/src/allmydata/test/cli/test_backupdb.py index 359b06f4f1..53cc3225a4 100644 --- a/src/allmydata/test/cli/test_backupdb.py +++ b/src/allmydata/test/cli/test_backupdb.py @@ -4,7 +4,7 @@ import sys import os.path, time -from six.moves import cStringIO as StringIO +from io import StringIO from twisted.trial import unittest from allmydata.util import fileutil diff --git a/src/allmydata/test/cli/test_check.py b/src/allmydata/test/cli/test_check.py index 56b36a8d5a..c895451ea6 100644 --- a/src/allmydata/test/cli/test_check.py +++ b/src/allmydata/test/cli/test_check.py @@ -3,7 +3,7 @@ import os.path import json from twisted.trial import unittest -from six.moves import cStringIO as StringIO +from io import StringIO from allmydata import uri from allmydata.util import base32 diff --git a/src/allmydata/test/cli/test_cli.py b/src/allmydata/test/cli/test_cli.py index ce51cb4f39..432437b612 100644 --- a/src/allmydata/test/cli/test_cli.py +++ b/src/allmydata/test/cli/test_cli.py @@ -2,7 +2,7 @@ Ported to Python 3. """ -from six.moves import cStringIO as StringIO +from io import StringIO import re from six import ensure_text diff --git a/src/allmydata/test/cli/test_create_alias.py b/src/allmydata/test/cli/test_create_alias.py index 862b2896a0..02978deca5 100644 --- a/src/allmydata/test/cli/test_create_alias.py +++ b/src/allmydata/test/cli/test_create_alias.py @@ -2,7 +2,7 @@ Ported to Python 3. """ -from six.moves import StringIO +from io import StringIO import os.path from twisted.trial import unittest from urllib.parse import quote as url_quote diff --git a/src/allmydata/test/cli/test_list.py b/src/allmydata/test/cli/test_list.py index 55e1d7cc17..f9ab52c53f 100644 --- a/src/allmydata/test/cli/test_list.py +++ b/src/allmydata/test/cli/test_list.py @@ -2,9 +2,6 @@ Ported to Python 3. """ -from future.utils import PY3 -from six import ensure_str - from twisted.trial import unittest from twisted.internet import defer @@ -26,10 +23,6 @@ def test_list(self): good_arg = u"g\u00F6\u00F6d" good_out = u"g\u00F6\u00F6d" - # On Python 2 we get bytes, so we need encoded version. On Python 3 - # stdio is unicode so can leave unchanged. - good_out_encoded = good_out if PY3 else good_out.encode(get_io_encoding()) - d = c0.create_dirnode() def _stash_root_and_create_file(n): self.rootnode = n @@ -52,7 +45,7 @@ def _check1(args): (rc, out, err) = args self.failUnlessReallyEqual(rc, 0) self.assertEqual(len(err), 0, err) - expected = sorted([ensure_str("0share"), ensure_str("1share"), good_out_encoded]) + expected = sorted(["0share", "1share", good_out]) self.assertEqual(sorted(out.splitlines()), expected) d.addCallback(_check1) d.addCallback(lambda ign: self.do_cli("ls", "missing")) @@ -85,8 +78,8 @@ def _check4(args): # listing a file (as dir/filename) should have the edge metadata, # including the filename self.failUnlessReallyEqual(rc, 0) - self.failUnlessIn(good_out_encoded, out) - self.failIfIn(ensure_str("-r-- %d -" % len(small)), out, + self.failUnlessIn(good_out, out) + self.failIfIn("-r-- %d -" % len(small), out, "trailing hyphen means unknown date") if good_arg is not None: diff --git a/src/allmydata/test/cli/test_run.py b/src/allmydata/test/cli/test_run.py index c7ebcd8c0d..18ee8f67d5 100644 --- a/src/allmydata/test/cli/test_run.py +++ b/src/allmydata/test/cli/test_run.py @@ -5,7 +5,7 @@ from __future__ import annotations import re -from six.moves import ( +from io import ( StringIO, ) diff --git a/src/allmydata/test/common_util.py b/src/allmydata/test/common_util.py index fce2881b05..9b194f657d 100644 --- a/src/allmydata/test/common_util.py +++ b/src/allmydata/test/common_util.py @@ -2,7 +2,7 @@ Ported to Python 3. """ -from future.utils import PY2, PY3, bchr, binary_type +from future.utils import bchr from future.builtins import str as future_str import os @@ -13,8 +13,6 @@ partial, ) from random import randrange -if PY2: - from StringIO import StringIO from io import ( TextIOWrapper, BytesIO, @@ -101,22 +99,7 @@ def run_cli_native(verb, *args, **kwargs): ) argv = ["tahoe"] + nodeargs + [verb] + list(args) stdin = kwargs.get("stdin", "") - if PY2: - # The original behavior, the Python 2 behavior, is to accept either - # bytes or unicode and try to automatically encode or decode as - # necessary. This works okay for ASCII and if LANG is set - # appropriately. These aren't great constraints so we should move - # away from this behavior. - # - # The encoding attribute doesn't change StringIO behavior on Python 2, - # but it's there for realism of the emulation. - stdin = StringIO(stdin) - stdin.encoding = encoding - stdout = StringIO() - stdout.encoding = encoding - stderr = StringIO() - stderr.encoding = encoding - else: + if True: # The new behavior, the Python 3 behavior, is to accept unicode and # encode it using a specific encoding. For older versions of Python 3, # the encoding is determined from LANG (bad) but for newer Python 3, @@ -146,13 +129,13 @@ def run_cli_native(verb, *args, **kwargs): stderr=stderr, ) def _done(rc, stdout=stdout, stderr=stderr): - if return_bytes and PY3: + if return_bytes: stdout = stdout.buffer stderr = stderr.buffer return 0, _getvalue(stdout), _getvalue(stderr) def _err(f, stdout=stdout, stderr=stderr): f.trap(SystemExit) - if return_bytes and PY3: + if return_bytes: stdout = stdout.buffer stderr = stderr.buffer return f.value.code, _getvalue(stdout), _getvalue(stderr) @@ -189,11 +172,7 @@ def run_cli_unicode(verb, argv, nodeargs=None, stdin=None, encoding=None): argv=argv, ) codec = encoding or "ascii" - if PY2: - encode = lambda t: None if t is None else t.encode(codec) - else: - # On Python 3 command-line parsing expects Unicode! - encode = lambda t: t + encode = lambda t: t d = run_cli_native( encode(verb), nodeargs=list(encode(arg) for arg in nodeargs), @@ -238,7 +217,7 @@ def flip_bit(good, which): def flip_one_bit(s, offset=0, size=None): """ flip one random bit of the string s, in a byte greater than or equal to offset and less than offset+size. """ - precondition(isinstance(s, binary_type)) + precondition(isinstance(s, bytes)) if size is None: size=len(s)-offset i = randrange(offset, offset+size) diff --git a/src/allmydata/test/mutable/test_filenode.py b/src/allmydata/test/mutable/test_filenode.py index 89881111aa..82f1e50729 100644 --- a/src/allmydata/test/mutable/test_filenode.py +++ b/src/allmydata/test/mutable/test_filenode.py @@ -2,7 +2,7 @@ Ported to Python 3. """ -from six.moves import cStringIO as StringIO +from io import StringIO from twisted.internet import defer, reactor from ..common import AsyncBrokenTestCase from testtools.matchers import ( diff --git a/src/allmydata/test/mutable/test_roundtrip.py b/src/allmydata/test/mutable/test_roundtrip.py index 238e69c610..4052193479 100644 --- a/src/allmydata/test/mutable/test_roundtrip.py +++ b/src/allmydata/test/mutable/test_roundtrip.py @@ -2,7 +2,7 @@ Ported to Python 3. """ -from six.moves import cStringIO as StringIO +from io import StringIO from ..common import AsyncTestCase from testtools.matchers import Equals, HasLength, Contains from twisted.internet import defer diff --git a/src/allmydata/test/test_common_util.py b/src/allmydata/test/test_common_util.py index 7f865d7432..01982473a2 100644 --- a/src/allmydata/test/test_common_util.py +++ b/src/allmydata/test/test_common_util.py @@ -2,8 +2,6 @@ This module has been ported to Python 3. """ -from future.utils import PY2 - import sys import random @@ -31,7 +29,7 @@ def setUp(self): def test_accepts_byte_string(self): actual = flip_one_bit(b'foo') - self.assertEqual(actual, b'fno' if PY2 else b'fom') + self.assertEqual(actual, b'fom') def test_rejects_unicode_string(self): self.assertRaises(AssertionError, flip_one_bit, u'foo') diff --git a/src/allmydata/test/test_crawler.py b/src/allmydata/test/test_crawler.py index 71b0f3deef..bf08828bdf 100644 --- a/src/allmydata/test/test_crawler.py +++ b/src/allmydata/test/test_crawler.py @@ -5,8 +5,6 @@ """ -from future.utils import PY3 - import time import os.path from twisted.trial import unittest @@ -28,10 +26,9 @@ def __init__(self, *args, **kwargs): self.all_buckets = [] self.finished_d = defer.Deferred() def process_bucket(self, cycle, prefix, prefixdir, storage_index_b32): - if PY3: - # Bucket _inputs_ are bytes, and that's what we will compare this - # to: - storage_index_b32 = storage_index_b32.encode("ascii") + # Bucket _inputs_ are bytes, and that's what we will compare this + # to: + storage_index_b32 = storage_index_b32.encode("ascii") self.all_buckets.append(storage_index_b32) def finished_cycle(self, cycle): eventually(self.finished_d.callback, None) @@ -46,10 +43,9 @@ def __init__(self, *args, **kwargs): self.finished_d = defer.Deferred() self.yield_cb = None def process_bucket(self, cycle, prefix, prefixdir, storage_index_b32): - if PY3: - # Bucket _inputs_ are bytes, and that's what we will compare this - # to: - storage_index_b32 = storage_index_b32.encode("ascii") + # Bucket _inputs_ are bytes, and that's what we will compare this + # to: + storage_index_b32 = storage_index_b32.encode("ascii") self.all_buckets.append(storage_index_b32) self.countdown -= 1 if self.countdown == 0: diff --git a/src/allmydata/test/test_crypto.py b/src/allmydata/test/test_crypto.py index aee9b41569..dd5b53e5f3 100644 --- a/src/allmydata/test/test_crypto.py +++ b/src/allmydata/test/test_crypto.py @@ -1,6 +1,3 @@ - -from future.utils import native_bytes - import unittest from base64 import b64decode @@ -40,7 +37,7 @@ class TestRegression(unittest.TestCase): # priv_str = b64encode(priv.serialize()) # pub_str = b64encode(priv.get_verifying_key().serialize()) RSA_2048_PRIV_KEY = b64decode(f.read().strip()) - assert isinstance(RSA_2048_PRIV_KEY, native_bytes) + assert isinstance(RSA_2048_PRIV_KEY, bytes) with RESOURCE_DIR.child('pycryptopp-rsa-2048-sig.txt').open('r') as f: # Signature created using `RSA_2048_PRIV_KEY` via: @@ -61,7 +58,7 @@ class TestRegression(unittest.TestCase): # priv_str = b64encode(priv.serialize()) # pub_str = b64encode(priv.get_verifying_key().serialize()) RSA_TINY_PRIV_KEY = b64decode(f.read().strip()) - assert isinstance(RSA_TINY_PRIV_KEY, native_bytes) + assert isinstance(RSA_TINY_PRIV_KEY, bytes) with RESOURCE_DIR.child('pycryptopp-rsa-32768-priv.txt').open('r') as f: # Created using `pycryptopp`: @@ -72,7 +69,7 @@ class TestRegression(unittest.TestCase): # priv_str = b64encode(priv.serialize()) # pub_str = b64encode(priv.get_verifying_key().serialize()) RSA_HUGE_PRIV_KEY = b64decode(f.read().strip()) - assert isinstance(RSA_HUGE_PRIV_KEY, native_bytes) + assert isinstance(RSA_HUGE_PRIV_KEY, bytes) def test_old_start_up_test(self): """ @@ -324,7 +321,7 @@ def test_key_serialization(self): private_key, public_key = ed25519.create_signing_keypair() private_key_str = ed25519.string_from_signing_key(private_key) - self.assertIsInstance(private_key_str, native_bytes) + self.assertIsInstance(private_key_str, bytes) private_key2, public_key2 = ed25519.signing_keypair_from_string(private_key_str) @@ -340,7 +337,7 @@ def test_key_serialization(self): # ditto, but for the verifying keys public_key_str = ed25519.string_from_verifying_key(public_key) - self.assertIsInstance(public_key_str, native_bytes) + self.assertIsInstance(public_key_str, bytes) public_key2 = ed25519.verifying_key_from_string(public_key_str) self.assertEqual( @@ -444,7 +441,7 @@ def test_keys(self): priv_key, pub_key = rsa.create_signing_keypair(2048) priv_key_str = rsa.der_string_from_signing_key(priv_key) - self.assertIsInstance(priv_key_str, native_bytes) + self.assertIsInstance(priv_key_str, bytes) priv_key2, pub_key2 = rsa.create_signing_keypair_from_string(priv_key_str) diff --git a/src/allmydata/test/test_download.py b/src/allmydata/test/test_download.py index 4d57fa8287..709786f0ea 100644 --- a/src/allmydata/test/test_download.py +++ b/src/allmydata/test/test_download.py @@ -10,7 +10,6 @@ from typing import Any -import six import os from twisted.trial import unittest from twisted.internet import defer, reactor @@ -30,9 +29,6 @@ from allmydata.codec import CRSDecoder from foolscap.eventual import eventually, fireEventually, flushEventualQueue -if six.PY3: - long = int - plaintext = b"This is a moderate-sized file.\n" * 10 mutable_plaintext = b"This is a moderate-sized mutable file.\n" * 10 diff --git a/src/allmydata/test/test_encodingutil.py b/src/allmydata/test/test_encodingutil.py index 64b9a243dc..4e16ef2b7d 100644 --- a/src/allmydata/test/test_encodingutil.py +++ b/src/allmydata/test/test_encodingutil.py @@ -1,8 +1,4 @@ -from future.utils import PY2, PY3 - -from past.builtins import unicode - lumiere_nfc = u"lumi\u00E8re" Artonwall_nfc = u"\u00C4rtonwall.mp3" Artonwall_nfd = u"A\u0308rtonwall.mp3" @@ -46,13 +42,7 @@ for fname in TEST_FILENAMES: open(os.path.join(tmpdir, fname), 'w').close() - # On Python 2, listing directories returns unicode under Windows or - # MacOS X if the input is unicode. On Python 3, it always returns - # Unicode. - if PY2 and sys.platform in ('win32', 'darwin'): - dirlist = os.listdir(unicode(tmpdir)) - else: - dirlist = os.listdir(tmpdir) + dirlist = os.listdir(tmpdir) print(" dirlist = %s" % repr(dirlist)) except: @@ -64,7 +54,6 @@ import os, sys -from unittest import skipIf from twisted.trial import unittest @@ -87,15 +76,6 @@ class MockStdout(object): # The following tests apply only to platforms that don't store filenames as # Unicode entities on the filesystem. class EncodingUtilNonUnicodePlatform(unittest.TestCase): - @skipIf(PY3, "Python 3 is always Unicode, regardless of OS.") - def setUp(self): - # Make sure everything goes back to the way it was at the end of the - # test. - self.addCleanup(_reload) - - # Mock sys.platform because unicode_platform() uses it. Cleanups run - # in reverse order so we do this second so it gets undone first. - self.patch(sys, "platform", "linux") def test_listdir_unicode(self): # What happens if latin1-encoded filenames are encountered on an UTF-8 @@ -143,10 +123,7 @@ def test_unicode_to_argv(self): converts to bytes using UTF-8 elsewhere. """ result = unicode_to_argv(lumiere_nfc) - if PY3 or self.platform == "win32": - expected_value = lumiere_nfc - else: - expected_value = lumiere_nfc.encode(self.io_encoding) + expected_value = lumiere_nfc self.assertIsInstance(result, type(expected_value)) self.assertEqual(result, expected_value) @@ -167,13 +144,10 @@ def test_listdir_unicode(self): % (self.filesystem_encoding,)) def call_os_listdir(path): - if PY2: - return self.dirlist - else: - # Python 3 always lists unicode filenames: - return [d.decode(self.filesystem_encoding) if isinstance(d, bytes) - else d - for d in self.dirlist] + # Python 3 always lists unicode filenames: + return [d.decode(self.filesystem_encoding) if isinstance(d, bytes) + else d + for d in self.dirlist] self.patch(os, 'listdir', call_os_listdir) @@ -204,10 +178,7 @@ def test_mkdir_open_exists_abspath_listdir_expanduser(self): fn = lumiere_nfc + u'/' + lumiere_nfc + u'.txt' open(fn, 'wb').close() self.failUnless(os.path.exists(fn)) - if PY2: - getcwdu = os.getcwdu - else: - getcwdu = os.getcwd + getcwdu = os.getcwd self.failUnless(os.path.exists(os.path.join(getcwdu(), fn))) filenames = listdir_unicode(lumiere_nfc) @@ -237,7 +208,7 @@ def tearDown(self): _reload() def _check(self, inp, out, enc, optional_quotes, quote_newlines): - if PY3 and isinstance(out, bytes): + if isinstance(out, bytes): out = out.decode(enc or encodingutil.io_encoding) out2 = out if optional_quotes: @@ -266,9 +237,7 @@ def _check(self, inp, out, enc, optional_quotes, quote_newlines): def _test_quote_output_all(self, enc): def check(inp, out, optional_quotes=False, quote_newlines=None): - if PY3: - # Result is always Unicode on Python 3 - out = out.decode("ascii") + out = out.decode("ascii") self._check(inp, out, enc, optional_quotes, quote_newlines) # optional single quotes @@ -354,9 +323,7 @@ def win32_other(win32, other): class QuotePaths(ReallyEqualMixin, unittest.TestCase): def assertPathsEqual(self, actual, expected): - if PY3: - # On Python 3, results should be unicode: - expected = expected.decode("ascii") + expected = expected.decode("ascii") self.failUnlessReallyEqual(actual, expected) def test_quote_path(self): diff --git a/src/allmydata/test/test_i2p_provider.py b/src/allmydata/test/test_i2p_provider.py index 3b99646bfa..f470e77af1 100644 --- a/src/allmydata/test/test_i2p_provider.py +++ b/src/allmydata/test/test_i2p_provider.py @@ -6,8 +6,8 @@ from twisted.trial import unittest from twisted.internet import defer, error from twisted.python.usage import UsageError -from six.moves import StringIO -import mock +from io import StringIO +from unittest import mock from ..util import i2p_provider from ..scripts import create_node, runner diff --git a/src/allmydata/test/test_runner.py b/src/allmydata/test/test_runner.py index f62afd0b08..bc55d507df 100644 --- a/src/allmydata/test/test_runner.py +++ b/src/allmydata/test/test_runner.py @@ -2,10 +2,6 @@ Ported to Python 3 """ -from future.utils import PY2 - -from six import ensure_text - import os.path, re, sys from os import linesep import locale @@ -129,18 +125,14 @@ def run_bintahoe(extra_argv, python_options=None): :return: A three-tuple of stdout (unicode), stderr (unicode), and the child process "returncode" (int). """ - executable = ensure_text(sys.executable) - argv = [executable] + argv = [sys.executable] if python_options is not None: argv.extend(python_options) argv.extend([u"-b", u"-m", u"allmydata.scripts.runner"]) argv.extend(extra_argv) argv = list(unicode_to_argv(arg) for arg in argv) p = Popen(argv, stdout=PIPE, stderr=PIPE) - if PY2: - encoding = "utf-8" - else: - encoding = locale.getpreferredencoding(False) + encoding = locale.getpreferredencoding(False) out = p.stdout.read().decode(encoding) err = p.stderr.read().decode(encoding) returncode = p.wait() @@ -154,10 +146,7 @@ def test_unicode_arguments_and_output(self): """ tricky = u"\u00F6" out, err, returncode = run_bintahoe([tricky]) - if PY2: - expected = u"Unknown command: \\xf6" - else: - expected = u"Unknown command: \xf6" + expected = u"Unknown command: \xf6" self.assertEqual(returncode, 1) self.assertIn( expected, diff --git a/src/allmydata/test/test_statistics.py b/src/allmydata/test/test_statistics.py index 2442e14a24..5a382e686b 100644 --- a/src/allmydata/test/test_statistics.py +++ b/src/allmydata/test/test_statistics.py @@ -4,7 +4,7 @@ Ported to Python 3. """ -from six.moves import StringIO # native string StringIO +from io import StringIO from twisted.trial import unittest diff --git a/src/allmydata/test/test_storage_web.py b/src/allmydata/test/test_storage_web.py index e9bca2183b..71d26af541 100644 --- a/src/allmydata/test/test_storage_web.py +++ b/src/allmydata/test/test_storage_web.py @@ -9,7 +9,7 @@ import re import json from unittest import skipIf -from six.moves import StringIO +from io import StringIO from twisted.trial import unittest from twisted.internet import defer diff --git a/src/allmydata/test/test_tor_provider.py b/src/allmydata/test/test_tor_provider.py index fc29e41e4e..e31a8586b6 100644 --- a/src/allmydata/test/test_tor_provider.py +++ b/src/allmydata/test/test_tor_provider.py @@ -5,9 +5,8 @@ import os from twisted.trial import unittest from twisted.internet import defer, error -from six.moves import StringIO -from six import ensure_str -import mock +from io import StringIO +from unittest import mock from ..util import tor_provider from ..scripts import create_node, runner from foolscap.eventual import flushEventualQueue @@ -185,7 +184,7 @@ def _do_test_launch(self, executable): txtorcon = mock.Mock() ehs = mock.Mock() # This appears to be a native string in the real txtorcon object... - ehs.private_key = ensure_str("privkey") + ehs.private_key = "privkey" ehs.hostname = "ONION.onion" txtorcon.EphemeralHiddenService = mock.Mock(return_value=ehs) ehs.add_to_tor = mock.Mock(return_value=defer.succeed(None)) diff --git a/src/allmydata/test/test_util.py b/src/allmydata/test/test_util.py index b6056b7e4b..07a2bfb598 100644 --- a/src/allmydata/test/test_util.py +++ b/src/allmydata/test/test_util.py @@ -2,7 +2,6 @@ Ported to Python3. """ -import six import os, time, sys import yaml import json @@ -22,8 +21,7 @@ from allmydata.test.common_util import ReallyEqualMixin from .no_network import fireNow, LocalWrapper -if six.PY3: - long = int +long = int class IDLib(unittest.TestCase): @@ -477,7 +475,7 @@ def test_convert(self): Unicode and (ASCII) native strings get roundtripped to Unicode strings. """ data = yaml.safe_dump( - [six.ensure_str("str"), u"unicode", u"\u1234nicode"] + ["str", "unicode", "\u1234nicode"] ) back = yamlutil.safe_load(data) self.assertIsInstance(back[0], str) diff --git a/src/allmydata/test/web/test_grid.py b/src/allmydata/test/web/test_grid.py index 86404a7bfa..c782733f90 100644 --- a/src/allmydata/test/web/test_grid.py +++ b/src/allmydata/test/web/test_grid.py @@ -5,7 +5,7 @@ import os.path, re from urllib.parse import quote as url_quote import json -from six.moves import StringIO +from io import StringIO from bs4 import BeautifulSoup diff --git a/src/allmydata/util/base62.py b/src/allmydata/util/base62.py index 2c44255622..3602ef0ef2 100644 --- a/src/allmydata/util/base62.py +++ b/src/allmydata/util/base62.py @@ -4,15 +4,8 @@ Ported to Python 3. """ -from future.utils import PY2 - -if PY2: - import string - maketrans = string.maketrans - translate = string.translate -else: - maketrans = bytes.maketrans - translate = bytes.translate +maketrans = bytes.maketrans +translate = bytes.translate from past.builtins import chr as byteschr diff --git a/src/allmydata/util/encodingutil.py b/src/allmydata/util/encodingutil.py index cf8d83a42a..9bf906ad77 100644 --- a/src/allmydata/util/encodingutil.py +++ b/src/allmydata/util/encodingutil.py @@ -8,7 +8,7 @@ Unicode is the default everywhere in Python 3. """ -from future.utils import PY3, native_str +from future.utils import native_str from future.builtins import str as future_str from past.builtins import unicode @@ -56,25 +56,13 @@ def check_encoding(encoding): io_encoding = "utf-8" filesystem_encoding = None -is_unicode_platform = False -use_unicode_filepath = False +is_unicode_platform = True +use_unicode_filepath = True def _reload(): - global filesystem_encoding, is_unicode_platform, use_unicode_filepath - + global filesystem_encoding filesystem_encoding = canonical_encoding(sys.getfilesystemencoding()) check_encoding(filesystem_encoding) - is_unicode_platform = PY3 or sys.platform in ["win32", "darwin"] - - # Despite the Unicode-mode FilePath support added to Twisted in - # , we can't yet use - # Unicode-mode FilePaths with INotify on non-Windows platforms due to - # . Supposedly 7928 is fixed, - # though... and Tahoe-LAFS doesn't use inotify anymore! - # - # In the interest of not breaking anything, this logic is unchanged for - # Python 2, but on Python 3 the paths are always unicode, like it or not. - use_unicode_filepath = PY3 or sys.platform == "win32" _reload() @@ -128,9 +116,7 @@ def unicode_to_argv(s): Windows, this returns the input unmodified. """ precondition(isinstance(s, unicode), s) - if PY3: - warnings.warn("This will be unnecessary once Python 2 is dropped.", - DeprecationWarning) + warnings.warn("This is unnecessary.", DeprecationWarning) if sys.platform == "win32": return s return ensure_str(s) @@ -184,24 +170,8 @@ def unicode_to_output(s): the responsibility of stdout/stderr, they expect Unicode by default. """ precondition(isinstance(s, unicode), s) - if PY3: - warnings.warn("This will be unnecessary once Python 2 is dropped.", - DeprecationWarning) - return s - - try: - out = s.encode(io_encoding) - except (UnicodeEncodeError, UnicodeDecodeError): - raise UnicodeEncodeError(native_str(io_encoding), s, 0, 0, - native_str("A string could not be encoded as %s for output to the terminal:\n%r" % - (io_encoding, repr(s)))) - - if PRINTABLE_8BIT.search(out) is None: - raise UnicodeEncodeError(native_str(io_encoding), s, 0, 0, - native_str("A string encoded as %s for output to the terminal contained unsafe bytes:\n%r" % - (io_encoding, repr(s)))) - return out - + warnings.warn("This is unnecessary.", DeprecationWarning) + return s def _unicode_escape(m, quote_newlines): u = m.group(0) @@ -303,20 +273,7 @@ def _encode(s): return b'"%s"' % (escaped.encode(encoding, 'backslashreplace'),) result = _encode(s) - if PY3: - # On Python 3 half of what this function does is unnecessary, since - # sys.stdout typically expects Unicode. To ensure no encode errors, one - # can do: - # - # sys.stdout.reconfigure(encoding=sys.stdout.encoding, errors="backslashreplace") - # - # Although the problem is that doesn't work in Python 3.6, only 3.7 or - # later... For now not thinking about it, just returning unicode since - # that is the right thing to do on Python 3. - # - # Now that Python 3.7 is the minimum, this can in theory be done: - # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3866 - result = result.decode(encoding) + result = result.decode(encoding) return result diff --git a/src/allmydata/util/hashutil.py b/src/allmydata/util/hashutil.py index 579f55d88d..7217a2d93a 100644 --- a/src/allmydata/util/hashutil.py +++ b/src/allmydata/util/hashutil.py @@ -4,13 +4,6 @@ Ported to Python 3. """ -from future.utils import PY2 -if PY2: - # Don't import bytes to prevent leaking future's bytes. - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, dict, list, object, range, str, max, min, bytes as future_bytes # noqa: F401 -else: - future_bytes = bytes - from past.builtins import chr as byteschr import os @@ -246,7 +239,7 @@ def bucket_cancel_secret_hash(file_cancel_secret, peerid): def _xor(a, b): - return b"".join([byteschr(c ^ b) for c in future_bytes(a)]) + return b"".join([byteschr(c ^ b) for c in bytes(a)]) def hmac(tag, data): diff --git a/src/allmydata/util/log.py b/src/allmydata/util/log.py index 3589bc3668..65df01dfc3 100644 --- a/src/allmydata/util/log.py +++ b/src/allmydata/util/log.py @@ -4,7 +4,6 @@ Ported to Python 3. """ -from future.utils import PY2 from six import ensure_str from pyutil import nummedobj @@ -12,14 +11,10 @@ from foolscap.logging import log from twisted.python import log as tw_log -if PY2: - def bytes_to_unicode(ign, obj): - return obj -else: - # We want to convert bytes keys to Unicode, otherwise JSON serialization - # inside foolscap will fail (for details see - # https://github.com/warner/foolscap/issues/88) - from .jsonbytes import bytes_to_unicode +# We want to convert bytes keys to Unicode, otherwise JSON serialization +# inside foolscap will fail (for details see +# https://github.com/warner/foolscap/issues/88) +from .jsonbytes import bytes_to_unicode NOISY = log.NOISY # 10 diff --git a/src/allmydata/windows/fixups.py b/src/allmydata/windows/fixups.py index 87e32a10f5..c48db2f829 100644 --- a/src/allmydata/windows/fixups.py +++ b/src/allmydata/windows/fixups.py @@ -1,7 +1,3 @@ - -from future.utils import PY3 -from past.builtins import unicode - # This code isn't loadable or sensible except on Windows. Importers all know # this and are careful. Normally I would just let an import error from ctypes # explain any mistakes but Mypy also needs some help here. This assert @@ -123,82 +119,6 @@ def initialize(): SetErrorMode(SEM_FAILCRITICALERRORS | SEM_NOOPENFILEERRORBOX) - if PY3: - # The rest of this appears to be Python 2-specific - return - - original_stderr = sys.stderr - - # If any exception occurs in this code, we'll probably try to print it on stderr, - # which makes for frustrating debugging if stderr is directed to our wrapper. - # So be paranoid about catching errors and reporting them to original_stderr, - # so that we can at least see them. - def _complain(output_file, message): - print(isinstance(message, str) and message or repr(message), file=output_file) - log.msg(message, level=log.WEIRD) - - _complain = partial(_complain, original_stderr) - - # Work around . - codecs.register(lambda name: name == 'cp65001' and codecs.lookup('utf-8') or None) - - # Make Unicode console output work independently of the current code page. - # This also fixes . - # Credit to Michael Kaplan - # and TZOmegaTZIOY - # . - try: - old_stdout_fileno = None - old_stderr_fileno = None - if hasattr(sys.stdout, 'fileno'): - old_stdout_fileno = sys.stdout.fileno() - if hasattr(sys.stderr, 'fileno'): - old_stderr_fileno = sys.stderr.fileno() - - real_stdout = (old_stdout_fileno == STDOUT_FILENO) - real_stderr = (old_stderr_fileno == STDERR_FILENO) - - if real_stdout: - hStdout = GetStdHandle(STD_OUTPUT_HANDLE) - if not a_console(hStdout): - real_stdout = False - - if real_stderr: - hStderr = GetStdHandle(STD_ERROR_HANDLE) - if not a_console(hStderr): - real_stderr = False - - if real_stdout: - sys.stdout = UnicodeOutput(hStdout, None, STDOUT_FILENO, '', _complain) - else: - sys.stdout = UnicodeOutput(None, sys.stdout, old_stdout_fileno, '', _complain) - - if real_stderr: - sys.stderr = UnicodeOutput(hStderr, None, STDERR_FILENO, '', _complain) - else: - sys.stderr = UnicodeOutput(None, sys.stderr, old_stderr_fileno, '', _complain) - except Exception as e: - _complain("exception %r while fixing up sys.stdout and sys.stderr" % (e,)) - - argv = list(arg.encode("utf-8") for arg in get_argv()) - - # Take only the suffix with the same number of arguments as sys.argv. - # This accounts for anything that can cause initial arguments to be stripped, - # for example, the Python interpreter or any options passed to it, or runner - # scripts such as 'coverage run'. It works even if there are no such arguments, - # as in the case of a frozen executable created by bb-freeze or similar. - # - # Also, modify sys.argv in place. If any code has already taken a - # reference to the original argument list object then this ensures that - # code sees the new values. This reliance on mutation of shared state is, - # of course, awful. Why does this function even modify sys.argv? Why not - # have a function that *returns* the properly initialized argv as a new - # list? I don't know. - # - # At least Python 3 gets sys.argv correct so before very much longer we - # should be able to fix this bad design by deleting it. - sys.argv[:] = argv[-len(sys.argv):] - def a_console(handle): """ @@ -274,13 +194,13 @@ def write(self, text): # There is no Windows console available. That means we are # responsible for encoding the unicode to a byte string to # write it to a Python file object. - if isinstance(text, unicode): + if isinstance(text, str): text = text.encode('utf-8') self._stream.write(text) else: # There is a Windows console available. That means Windows is # responsible for dealing with the unicode itself. - if not isinstance(text, unicode): + if not isinstance(text, str): text = str(text).decode('utf-8') remaining = len(text) while remaining > 0: From 0e5b6daa3890978f67043e9555f3b3b21946c59c Mon Sep 17 00:00:00 2001 From: Alexandre Detiste Date: Wed, 28 Feb 2024 00:53:49 +0100 Subject: [PATCH 02/11] remove more Python2: unicode -> str, long -> int --- src/allmydata/check_results.py | 4 +--- src/allmydata/immutable/upload.py | 11 +++++------ src/allmydata/scripts/tahoe_status.py | 3 +-- src/allmydata/uri.py | 16 +++++++--------- 4 files changed, 14 insertions(+), 20 deletions(-) diff --git a/src/allmydata/check_results.py b/src/allmydata/check_results.py index 8c44fd71e2..e9ef59bdbb 100644 --- a/src/allmydata/check_results.py +++ b/src/allmydata/check_results.py @@ -1,8 +1,6 @@ """Ported to Python 3. """ -from past.builtins import unicode - from zope.interface import implementer from allmydata.interfaces import ICheckResults, ICheckAndRepairResults, \ IDeepCheckResults, IDeepCheckAndRepairResults, IURI, IDisplayableServer @@ -64,7 +62,7 @@ def __init__(self, uri, storage_index, # unicode. if isinstance(summary, bytes): summary = unicode(summary, "utf-8") - assert isinstance(summary, unicode) # should be a single string + assert isinstance(summary, str) # should be a single string self._summary = summary assert not isinstance(report, str) # should be list of strings self._report = report diff --git a/src/allmydata/immutable/upload.py b/src/allmydata/immutable/upload.py index 22210ad0a7..c139700334 100644 --- a/src/allmydata/immutable/upload.py +++ b/src/allmydata/immutable/upload.py @@ -5,7 +5,6 @@ from __future__ import annotations from future.utils import native_str -from past.builtins import long, unicode from six import ensure_str import os, time, weakref, itertools @@ -57,7 +56,7 @@ _TOTAL_SHARES = Field.for_types( u"total_shares", - [int, long], + [int, int], u"The total number of shares desired.", ) @@ -104,7 +103,7 @@ def _serialize_happiness_mappings(happiness_mappings): _HAPPINESS = Field.for_types( u"happiness", - [int, long], + [int, int], u"The computed happiness of a certain placement.", ) @@ -142,7 +141,7 @@ def _serialize_happiness_mappings(happiness_mappings): _EFFECTIVE_HAPPINESS = Field.for_types( u"effective_happiness", - [int, long], + [int, int], u"The computed happiness value of a share placement map.", ) @@ -1622,7 +1621,7 @@ def _convert_old_upload_results(self, upload_results): # abbreviated), so if we detect old results, just clobber them. sharemap = upload_results.sharemap - if any(isinstance(v, (bytes, unicode)) for v in sharemap.values()): + if any(isinstance(v, (bytes, str)) for v in sharemap.values()): upload_results.sharemap = None def _build_verifycap(self, helper_upload_results): @@ -1701,7 +1700,7 @@ def set_upload_status(self, upload_status): def set_default_encoding_parameters(self, default_params): assert isinstance(default_params, dict) for k,v in default_params.items(): - precondition(isinstance(k, (bytes, unicode)), k, v) + precondition(isinstance(k, (bytes, str)), k, v) precondition(isinstance(v, int), k, v) if "k" in default_params: self.default_encoding_param_k = default_params["k"] diff --git a/src/allmydata/scripts/tahoe_status.py b/src/allmydata/scripts/tahoe_status.py index c7f19910b7..ef8da35c09 100644 --- a/src/allmydata/scripts/tahoe_status.py +++ b/src/allmydata/scripts/tahoe_status.py @@ -24,13 +24,12 @@ def print(*args, **kwargs): encoding error handler and then write the result whereas builtin print uses the "strict" encoding error handler. """ - from past.builtins import unicode out = kwargs.pop("file", None) if out is None: out = _sys_stdout encoding = out.encoding or "ascii" def ensafe(o): - if isinstance(o, unicode): + if isinstance(o, str): return o.encode(encoding, errors="replace").decode(encoding) return o return _print( diff --git a/src/allmydata/uri.py b/src/allmydata/uri.py index fccf05db9f..34f245ac7a 100644 --- a/src/allmydata/uri.py +++ b/src/allmydata/uri.py @@ -7,8 +7,6 @@ in follow-up port. """ -from past.builtins import unicode, long - import re from typing import Type @@ -91,7 +89,7 @@ def init_from_string(cls, uri): def to_string(self): assert isinstance(self.needed_shares, int) assert isinstance(self.total_shares, int) - assert isinstance(self.size, (int,long)) + assert isinstance(self.size, int) return (b'URI:CHK:%s:%s:%d:%d:%d' % (base32.b2a(self.key), @@ -147,7 +145,7 @@ def init_from_string(cls, uri): def to_string(self): assert isinstance(self.needed_shares, int) assert isinstance(self.total_shares, int) - assert isinstance(self.size, (int,long)) + assert isinstance(self.size, int) return (b'URI:CHK-Verifier:%s:%s:%d:%d:%d' % (si_b2a(self.storage_index), @@ -742,7 +740,7 @@ def get_verify_cap(self): def from_string(u, deep_immutable=False, name=u""): """Create URI from either unicode or byte string.""" - if isinstance(u, unicode): + if isinstance(u, str): u = u.encode("utf-8") if not isinstance(u, bytes): raise TypeError("URI must be unicode string or bytes: %r" % (u,)) @@ -844,7 +842,7 @@ def is_uri(s): return False def is_literal_file_uri(s): - if isinstance(s, unicode): + if isinstance(s, str): s = s.encode("utf-8") if not isinstance(s, bytes): return False @@ -853,7 +851,7 @@ def is_literal_file_uri(s): s.startswith(ALLEGED_IMMUTABLE_PREFIX + b'URI:LIT:')) def has_uri_prefix(s): - if isinstance(s, unicode): + if isinstance(s, str): s = s.encode("utf-8") if not isinstance(s, bytes): return False @@ -895,9 +893,9 @@ def pack_extension(data): pieces = [] for k in sorted(data.keys()): value = data[k] - if isinstance(value, (int, long)): + if isinstance(value, int): value = b"%d" % value - if isinstance(k, unicode): + if isinstance(k, str): k = k.encode("utf-8") assert isinstance(value, bytes), k assert re.match(br'^[a-zA-Z_\-]+$', k) From 2243ce3187f3519ff2e92a46202e983f1fdcb91d Mon Sep 17 00:00:00 2001 From: Alexandre Detiste Date: Wed, 28 Feb 2024 01:07:08 +0100 Subject: [PATCH 03/11] remove "from past.builtins import long" --- src/allmydata/interfaces.py | 3 +-- src/allmydata/introducer/client.py | 4 +--- src/allmydata/introducer/server.py | 3 +-- src/allmydata/test/mutable/util.py | 6 ++---- src/allmydata/test/test_dirnode.py | 4 +--- src/allmydata/test/test_humanreadable.py | 4 +--- src/allmydata/test/test_spans.py | 11 +++-------- src/allmydata/test/test_time_format.py | 4 +--- src/allmydata/util/netstring.py | 4 +--- src/allmydata/web/status.py | 4 +--- 10 files changed, 13 insertions(+), 34 deletions(-) diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py index 96fccb9585..1cfbb577e1 100644 --- a/src/allmydata/interfaces.py +++ b/src/allmydata/interfaces.py @@ -8,7 +8,6 @@ from future.utils import native_str -from past.builtins import long from typing import Dict from zope.interface import Interface, Attribute @@ -2774,7 +2773,7 @@ def get_size(): return Offset def get_all_encoding_parameters(): - return (int, int, int, long) + return (int, int, int, int) def read_encrypted(offset=Offset, length=ReadSize): return ListOf(bytes) diff --git a/src/allmydata/introducer/client.py b/src/allmydata/introducer/client.py index e6eab3b9f7..fd605dd1da 100644 --- a/src/allmydata/introducer/client.py +++ b/src/allmydata/introducer/client.py @@ -2,8 +2,6 @@ Ported to Python 3. """ -from past.builtins import long - from six import ensure_text, ensure_str import time @@ -304,7 +302,7 @@ def _process_announcement(self, ann, key_s): if "seqnum" in old: # must beat previous sequence number to replace if ("seqnum" not in ann - or not isinstance(ann["seqnum"], (int,long))): + or not isinstance(ann["seqnum"], int)): self.log("not replacing old announcement, no valid seqnum: %s" % (ann,), parent=lp2, level=log.NOISY, umid="zFGH3Q") diff --git a/src/allmydata/introducer/server.py b/src/allmydata/introducer/server.py index 157a1b73cb..10048c55e6 100644 --- a/src/allmydata/introducer/server.py +++ b/src/allmydata/introducer/server.py @@ -4,7 +4,6 @@ from __future__ import annotations -from past.builtins import long from six import ensure_text import time, os.path, textwrap @@ -262,7 +261,7 @@ def _publish(self, ann_t, canary, lp): if "seqnum" in old_ann: # must beat previous sequence number to replace if ("seqnum" not in ann - or not isinstance(ann["seqnum"], (int,long))): + or not isinstance(ann["seqnum"], int)): self.log("not replacing old ann, no valid seqnum", level=log.NOISY, umid="ySbaVw") self._debug_counts["inbound_no_seqnum"] += 1 diff --git a/src/allmydata/test/mutable/util.py b/src/allmydata/test/mutable/util.py index 5a2a6a8f84..fd1fc29700 100644 --- a/src/allmydata/test/mutable/util.py +++ b/src/allmydata/test/mutable/util.py @@ -4,8 +4,6 @@ from future.utils import bchr -from past.builtins import long - from io import BytesIO import attr from twisted.internet import defer, reactor @@ -129,8 +127,8 @@ def _read(shares): continue vector = response[shnum] = [] for (offset, length) in readv: - assert isinstance(offset, (int, long)), offset - assert isinstance(length, (int, long)), length + assert isinstance(offset, int), offset + assert isinstance(length, int), length vector.append(shares[shnum][offset:offset+length]) return response d.addCallback(_read) diff --git a/src/allmydata/test/test_dirnode.py b/src/allmydata/test/test_dirnode.py index 2ef57b0afc..30fba005f7 100644 --- a/src/allmydata/test/test_dirnode.py +++ b/src/allmydata/test/test_dirnode.py @@ -3,8 +3,6 @@ Ported to Python 3. """ -from past.builtins import long - import time import unicodedata from zope.interface import implementer @@ -1854,7 +1852,7 @@ def test_stats(self): (101, 316, 216), (317, 1000, 684), (1001, 3162, 99), - (long(3162277660169), long(10000000000000), 1), + (3162277660169, 10000000000000, 1), ]) class UCWEingMutableFileNode(MutableFileNode): diff --git a/src/allmydata/test/test_humanreadable.py b/src/allmydata/test/test_humanreadable.py index 1ae95145ee..277abc2833 100644 --- a/src/allmydata/test/test_humanreadable.py +++ b/src/allmydata/test/test_humanreadable.py @@ -4,8 +4,6 @@ This module has been ported to Python 3. """ -from past.builtins import long - from twisted.trial import unittest from allmydata.util import humanreadable @@ -26,7 +24,7 @@ def test_repr(self): self.assertRegex(hr(foo), r"") self.failUnlessEqual(hr(self.test_repr), ">") - self.failUnlessEqual(hr(long(1)), "1") + self.failUnlessEqual(hr(1), "1") self.assertIn(hr(10**40), ["100000000000000000...000000000000000000", "100000000000000000...0000000000000000000"]) diff --git a/src/allmydata/test/test_spans.py b/src/allmydata/test/test_spans.py index e6e510e5d3..578075e8d6 100644 --- a/src/allmydata/test/test_spans.py +++ b/src/allmydata/test/test_spans.py @@ -2,8 +2,6 @@ Tests for allmydata.util.spans. """ -from past.builtins import long - import binascii import hashlib @@ -116,9 +114,6 @@ def test_basic(self): s1 = Spans(3, 4) # 3,4,5,6 self._check1(s1) - s1 = Spans(long(3), long(4)) # 3,4,5,6 - self._check1(s1) - s2 = Spans(s1) self._check1(s2) @@ -446,9 +441,9 @@ def do_basic(self, klass): self.failUnlessEqual(ds.get(2, 4), b"fear") ds = klass() - ds.add(long(2), b"four") - ds.add(long(3), b"ea") - self.failUnlessEqual(ds.get(long(2), long(4)), b"fear") + ds.add(2, b"four") + ds.add(3, b"ea") + self.failUnlessEqual(ds.get(2, 4), b"fear") def do_scan(self, klass): diff --git a/src/allmydata/test/test_time_format.py b/src/allmydata/test/test_time_format.py index 0b409feedf..f3b9a89903 100644 --- a/src/allmydata/test/test_time_format.py +++ b/src/allmydata/test/test_time_format.py @@ -2,8 +2,6 @@ Tests for allmydata.util.time_format. """ -from past.builtins import long - import time from twisted.trial import unittest @@ -103,7 +101,7 @@ def test_parse_duration(self): def test_parse_date(self): p = time_format.parse_date self.failUnlessEqual(p("2010-02-21"), 1266710400) - self.failUnless(isinstance(p("2009-03-18"), (int, long)), p("2009-03-18")) + self.failUnless(isinstance(p("2009-03-18"), int), p("2009-03-18")) self.failUnlessEqual(p("2009-03-18"), 1237334400) def test_format_time(self): diff --git a/src/allmydata/util/netstring.py b/src/allmydata/util/netstring.py index db913172f6..ee7849b5fa 100644 --- a/src/allmydata/util/netstring.py +++ b/src/allmydata/util/netstring.py @@ -4,8 +4,6 @@ Ported to Python 3. """ -from past.builtins import long - try: from typing import Optional, Tuple, List # noqa: F401 except ImportError: @@ -27,7 +25,7 @@ def split_netstring(data, numstrings, data does not exactly equal 'required_trailer'.""" assert isinstance(data, bytes) assert required_trailer is None or isinstance(required_trailer, bytes) - assert isinstance(position, (int, long)), (repr(position), type(position)) + assert isinstance(position, int), (repr(position), type(position)) elements = [] assert numstrings >= 0 while position < len(data): diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index 07d0256e8e..1737a4d1be 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -2,8 +2,6 @@ Ported to Python 3. """ -from past.builtins import long - import itertools import hashlib import re @@ -1393,7 +1391,7 @@ def get_op_state(op): size = op.get_size() if size is None: size = "(unknown)" - elif isinstance(size, (int, long, float)): + elif isinstance(size, (int, float)): size = abbreviate_size(size) result["total_size"] = size From f47b45ac1a5bf599b9393eb99384f733de550e73 Mon Sep 17 00:00:00 2001 From: Alexandre Detiste Date: Wed, 28 Feb 2024 01:25:06 +0100 Subject: [PATCH 04/11] remove static native_str() --- src/allmydata/immutable/upload.py | 3 +-- src/allmydata/interfaces.py | 10 ++++------ src/allmydata/introducer/interfaces.py | 10 +++++----- src/allmydata/test/common_util.py | 17 ++++++----------- src/allmydata/test/storage_plugin.py | 4 ++-- src/allmydata/util/encodingutil.py | 5 +---- src/allmydata/util/iputil.py | 6 +++--- 7 files changed, 22 insertions(+), 33 deletions(-) diff --git a/src/allmydata/immutable/upload.py b/src/allmydata/immutable/upload.py index c139700334..e62f7e0c11 100644 --- a/src/allmydata/immutable/upload.py +++ b/src/allmydata/immutable/upload.py @@ -4,7 +4,6 @@ from __future__ import annotations -from future.utils import native_str from six import ensure_str import os, time, weakref, itertools @@ -165,7 +164,7 @@ class HelperUploadResults(Copyable, RemoteCopy): # package/module/class name # # Needs to be native string to make Foolscap happy. - typeToCopy = native_str("allmydata.upload.UploadResults.tahoe.allmydata.com") + typeToCopy = "allmydata.upload.UploadResults.tahoe.allmydata.com" copytype = typeToCopy # also, think twice about changing the shape of any existing attribute, diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py index 1cfbb577e1..e44a0e8bbc 100644 --- a/src/allmydata/interfaces.py +++ b/src/allmydata/interfaces.py @@ -6,8 +6,6 @@ Note that for RemoteInterfaces, the __remote_name__ needs to be a native string because of https://github.com/warner/foolscap/blob/43f4485a42c9c28e2c79d655b3a9e24d4e6360ca/src/foolscap/remoteinterface.py#L67 """ -from future.utils import native_str - from typing import Dict from zope.interface import Interface, Attribute @@ -111,7 +109,7 @@ def advise_corrupt_share(reason=bytes): class RIStorageServer(RemoteInterface): - __remote_name__ = native_str("RIStorageServer.tahoe.allmydata.com") + __remote_name__ = "RIStorageServer.tahoe.allmydata.com" def get_version(): """ @@ -2767,7 +2765,7 @@ def measure_peer_response_time(): class RIEncryptedUploadable(RemoteInterface): - __remote_name__ = native_str("RIEncryptedUploadable.tahoe.allmydata.com") + __remote_name__ = "RIEncryptedUploadable.tahoe.allmydata.com" def get_size(): return Offset @@ -2783,7 +2781,7 @@ def close(): class RICHKUploadHelper(RemoteInterface): - __remote_name__ = native_str("RIUploadHelper.tahoe.allmydata.com") + __remote_name__ = "RIUploadHelper.tahoe.allmydata.com" def get_version(): """ @@ -2796,7 +2794,7 @@ def upload(reader=RIEncryptedUploadable): class RIHelper(RemoteInterface): - __remote_name__ = native_str("RIHelper.tahoe.allmydata.com") + __remote_name__ = "RIHelper.tahoe.allmydata.com" def get_version(): """ diff --git a/src/allmydata/introducer/interfaces.py b/src/allmydata/introducer/interfaces.py index 13cd7c3da3..e714d73402 100644 --- a/src/allmydata/introducer/interfaces.py +++ b/src/allmydata/introducer/interfaces.py @@ -2,9 +2,6 @@ Ported to Python 3. """ - -from future.utils import native_str - from zope.interface import Interface from foolscap.api import StringConstraint, SetOf, DictOf, Any, \ RemoteInterface, Referenceable @@ -34,7 +31,7 @@ Announcement_v2 = Any() class RIIntroducerSubscriberClient_v2(RemoteInterface): - __remote_name__ = native_str("RIIntroducerSubscriberClient_v2.tahoe.allmydata.com") + __remote_name__ = "RIIntroducerSubscriberClient_v2.tahoe.allmydata.com" def announce_v2(announcements=SetOf(Announcement_v2)): """I accept announcements from the publisher.""" @@ -47,11 +44,14 @@ class RIIntroducerPublisherAndSubscriberService_v2(RemoteInterface): announcement message. I will deliver a copy to all connected subscribers. To hear about services, connect to me and subscribe to a specific service_name.""" - __remote_name__ = native_str("RIIntroducerPublisherAndSubscriberService_v2.tahoe.allmydata.com") + __remote_name__ = "RIIntroducerPublisherAndSubscriberService_v2.tahoe.allmydata.com" + def get_version(): return DictOf(bytes, Any()) + def publish_v2(announcement=Announcement_v2, canary=Referenceable): return None + def subscribe_v2(subscriber=RIIntroducerSubscriberClient_v2, service_name=bytes, subscriber_info=SubscriberInfo): """Give me a subscriber reference, and I will call its announce_v2() diff --git a/src/allmydata/test/common_util.py b/src/allmydata/test/common_util.py index 9b194f657d..70713a995d 100644 --- a/src/allmydata/test/common_util.py +++ b/src/allmydata/test/common_util.py @@ -3,7 +3,6 @@ """ from future.utils import bchr -from future.builtins import str as future_str import os import sys @@ -64,13 +63,13 @@ def run_cli_native(verb, *args, **kwargs): :param runner.Options options: The options instance to use to parse the given arguments. - :param native_str verb: The command to run. For example, + :param str verb: The command to run. For example, ``"create-node"``. - :param [native_str] args: The arguments to pass to the command. For + :param [str] args: The arguments to pass to the command. For example, ``("--hostname=localhost",)``. - :param [native_str] nodeargs: Extra arguments to pass to the Tahoe + :param [str] nodeargs: Extra arguments to pass to the Tahoe executable before ``verb``. :param bytes|unicode stdin: Text or bytes to pass to the command via stdin. @@ -165,7 +164,7 @@ def run_cli_unicode(verb, argv, nodeargs=None, stdin=None, encoding=None): if nodeargs is None: nodeargs = [] precondition( - all(isinstance(arg, future_str) for arg in [verb] + nodeargs + argv), + all(isinstance(arg, str) for arg in [verb] + nodeargs + argv), "arguments to run_cli_unicode must be unicode", verb=verb, nodeargs=nodeargs, @@ -229,13 +228,9 @@ def flip_one_bit(s, offset=0, size=None): class ReallyEqualMixin(object): def failUnlessReallyEqual(self, a, b, msg=None): self.assertEqual(a, b, msg) - # Make sure unicode strings are a consistent type. Specifically there's - # Future newstr (backported Unicode type) vs. Python 2 native unicode - # type. They're equal, and _logically_ the same type, but have - # different types in practice. - if a.__class__ == future_str: + if a.__class__ == str: a = str(a) - if b.__class__ == future_str: + if b.__class__ == str: b = str(b) self.assertEqual(type(a), type(b), "a :: %r (%s), b :: %r (%s), %r" % (a, type(a), b, type(b), msg)) diff --git a/src/allmydata/test/storage_plugin.py b/src/allmydata/test/storage_plugin.py index b1950387ba..f638d298f5 100644 --- a/src/allmydata/test/storage_plugin.py +++ b/src/allmydata/test/storage_plugin.py @@ -5,7 +5,7 @@ Ported to Python 3. """ -from future.utils import native_str, native_str_to_bytes +from future.utils import native_str_to_bytes from six import ensure_str import attr @@ -40,7 +40,7 @@ class RIDummy(RemoteInterface): - __remote_name__ = native_str("RIDummy.tahoe.allmydata.com") + __remote_name__ = "RIDummy.tahoe.allmydata.com" def just_some_method(): """ diff --git a/src/allmydata/util/encodingutil.py b/src/allmydata/util/encodingutil.py index 9bf906ad77..e2131eb0dc 100644 --- a/src/allmydata/util/encodingutil.py +++ b/src/allmydata/util/encodingutil.py @@ -8,9 +8,6 @@ Unicode is the default everywhere in Python 3. """ -from future.utils import native_str -from future.builtins import str as future_str - from past.builtins import unicode from six import ensure_str @@ -124,7 +121,7 @@ def unicode_to_argv(s): # According to unicode_to_argv above, the expected type for # cli args depends on the platform, so capture that expectation. -argv_type = (future_str, native_str) if sys.platform == "win32" else native_str +argv_type = (str,) """ The expected type for args to a subprocess """ diff --git a/src/allmydata/util/iputil.py b/src/allmydata/util/iputil.py index e71e514e82..61fcfc8a07 100644 --- a/src/allmydata/util/iputil.py +++ b/src/allmydata/util/iputil.py @@ -104,7 +104,7 @@ def get_local_addresses_sync(): on the local system. """ return list( - native_str(address[native_str("addr")]) + native_str(address["addr"]) for iface_name in interfaces() for address @@ -161,7 +161,7 @@ def _foolscapEndpointForPortNumber(portnum): # approach is error prone for the reasons described on # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/2787 portnum = allocate_tcp_port() - return (portnum, native_str("tcp:%d" % (portnum,))) + return (portnum, "tcp:%d" % portnum) @implementer(IStreamServerEndpoint) @@ -210,7 +210,7 @@ def listenOnUnused(tub, portnum=None): """ portnum, endpoint = _foolscapEndpointForPortNumber(portnum) tub.listenOn(endpoint) - tub.setLocation(native_str("localhost:%d" % (portnum,))) + tub.setLocation("localhost:%d" % portnum) return portnum From 8e5e886598085bedb5fe4aad7a9875fc443cc842 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 1 Mar 2024 12:09:40 -0500 Subject: [PATCH 05/11] News fragment. --- newsfragments/4093.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/4093.minor diff --git a/newsfragments/4093.minor b/newsfragments/4093.minor new file mode 100644 index 0000000000..e69de29bb2 From ddc8e64272c87af6cb31b4bdcd002818a6b8df3f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 1 Mar 2024 12:11:54 -0500 Subject: [PATCH 06/11] We're just going to assume that in 2024 this is not an issue. --- src/allmydata/test/test_encodingutil.py | 32 ------------------------- 1 file changed, 32 deletions(-) diff --git a/src/allmydata/test/test_encodingutil.py b/src/allmydata/test/test_encodingutil.py index 4e16ef2b7d..210d462032 100644 --- a/src/allmydata/test/test_encodingutil.py +++ b/src/allmydata/test/test_encodingutil.py @@ -73,38 +73,6 @@ class MockStdout(object): pass -# The following tests apply only to platforms that don't store filenames as -# Unicode entities on the filesystem. -class EncodingUtilNonUnicodePlatform(unittest.TestCase): - - def test_listdir_unicode(self): - # What happens if latin1-encoded filenames are encountered on an UTF-8 - # filesystem? - def call_os_listdir(path): - return [ - lumiere_nfc.encode('utf-8'), - lumiere_nfc.encode('latin1') - ] - self.patch(os, 'listdir', call_os_listdir) - - sys_filesystemencoding = 'utf-8' - def call_sys_getfilesystemencoding(): - return sys_filesystemencoding - self.patch(sys, 'getfilesystemencoding', call_sys_getfilesystemencoding) - - _reload() - self.failUnlessRaises(FilenameEncodingError, - listdir_unicode, - u'/dummy') - - # We're trying to list a directory whose name cannot be represented in - # the filesystem encoding. This should fail. - sys_filesystemencoding = 'ascii' - _reload() - self.failUnlessRaises(FilenameEncodingError, - listdir_unicode, - u'/' + lumiere_nfc) - class EncodingUtil(ReallyEqualMixin): def setUp(self): From 0bbc8f6b5f2aaa626bc39fa6ef619d31043f2c36 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 1 Mar 2024 12:16:40 -0500 Subject: [PATCH 07/11] Delete Python 2 specific code. --- src/allmydata/windows/fixups.py | 194 -------------------------------- 1 file changed, 194 deletions(-) diff --git a/src/allmydata/windows/fixups.py b/src/allmydata/windows/fixups.py index c48db2f829..1b204ccf45 100644 --- a/src/allmydata/windows/fixups.py +++ b/src/allmydata/windows/fixups.py @@ -11,104 +11,19 @@ import sys assert sys.platform == "win32" -import codecs -from functools import partial - -from ctypes import WINFUNCTYPE, windll, POINTER, c_int, WinError, byref, get_last_error -from ctypes.wintypes import BOOL, HANDLE, DWORD, LPWSTR, LPCWSTR, LPVOID - # from win32api import ( - STD_OUTPUT_HANDLE, - STD_ERROR_HANDLE, SetErrorMode, - - # - # HANDLE WINAPI GetStdHandle(DWORD nStdHandle); - # returns INVALID_HANDLE_VALUE, NULL, or a valid handle - GetStdHandle, ) from win32con import ( SEM_FAILCRITICALERRORS, SEM_NOOPENFILEERRORBOX, ) -from win32file import ( - INVALID_HANDLE_VALUE, - FILE_TYPE_CHAR, - - # - # DWORD WINAPI GetFileType(DWORD hFile); - GetFileType, -) - -from allmydata.util import ( - log, -) - # Keep track of whether `initialize` has run so we don't do any of the # initialization more than once. _done = False -# -# pywin32 for Python 2.7 does not bind any of these *W variants so we do it -# ourselves. -# - -# -# BOOL WINAPI WriteConsoleW(HANDLE hOutput, LPWSTR lpBuffer, DWORD nChars, -# LPDWORD lpCharsWritten, LPVOID lpReserved); -WriteConsoleW = WINFUNCTYPE( - BOOL, HANDLE, LPWSTR, DWORD, POINTER(DWORD), LPVOID, - use_last_error=True -)(("WriteConsoleW", windll.kernel32)) - -# -GetCommandLineW = WINFUNCTYPE( - LPWSTR, - use_last_error=True -)(("GetCommandLineW", windll.kernel32)) - -# -CommandLineToArgvW = WINFUNCTYPE( - POINTER(LPWSTR), LPCWSTR, POINTER(c_int), - use_last_error=True -)(("CommandLineToArgvW", windll.shell32)) - -# -# BOOL WINAPI GetConsoleMode(HANDLE hConsole, LPDWORD lpMode); -GetConsoleMode = WINFUNCTYPE( - BOOL, HANDLE, POINTER(DWORD), - use_last_error=True -)(("GetConsoleMode", windll.kernel32)) - - -STDOUT_FILENO = 1 -STDERR_FILENO = 2 - -def get_argv(): - """ - :return [unicode]: The argument list this process was invoked with, as - unicode. - - Python 2 does not do a good job exposing this information in - ``sys.argv`` on Windows so this code re-retrieves the underlying - information using Windows API calls and massages it into the right - shape. - """ - command_line = GetCommandLineW() - argc = c_int(0) - argv_unicode = CommandLineToArgvW(command_line, byref(argc)) - if argv_unicode is None: - raise WinError(get_last_error()) - - # Convert it to a normal Python list - return list( - argv_unicode[i] - for i - in range(argc.value) - ) - def initialize(): global _done @@ -118,112 +33,3 @@ def initialize(): _done = True SetErrorMode(SEM_FAILCRITICALERRORS | SEM_NOOPENFILEERRORBOX) - - -def a_console(handle): - """ - :return: ``True`` if ``handle`` refers to a console, ``False`` otherwise. - """ - if handle == INVALID_HANDLE_VALUE: - return False - return ( - # It's a character file (eg a printer or a console) - GetFileType(handle) == FILE_TYPE_CHAR and - # Checking the console mode doesn't fail (thus it's a console) - GetConsoleMode(handle, byref(DWORD())) != 0 - ) - - -class UnicodeOutput(object): - """ - ``UnicodeOutput`` is a file-like object that encodes unicode to UTF-8 and - writes it to another file or writes unicode natively to the Windows - console. - """ - def __init__(self, hConsole, stream, fileno, name, _complain): - """ - :param hConsole: ``None`` or a handle on the console to which to write - unicode. Mutually exclusive with ``stream``. - - :param stream: ``None`` or a file-like object to which to write bytes. - - :param fileno: A result to hand back from method of the same name. - - :param name: A human-friendly identifier for this output object. - - :param _complain: A one-argument callable which accepts bytes to be - written when there's a problem. Care should be taken to not make - this do a write on this object. - """ - self._hConsole = hConsole - self._stream = stream - self._fileno = fileno - self.closed = False - self.softspace = False - self.mode = 'w' - self.encoding = 'utf-8' - self.name = name - - self._complain = _complain - - from allmydata.util.encodingutil import canonical_encoding - from allmydata.util import log - if hasattr(stream, 'encoding') and canonical_encoding(stream.encoding) != 'utf-8': - log.msg("%s: %r had encoding %r, but we're going to write UTF-8 to it" % - (name, stream, stream.encoding), level=log.CURIOUS) - self.flush() - - def isatty(self): - return False - def close(self): - # don't really close the handle, that would only cause problems - self.closed = True - def fileno(self): - return self._fileno - def flush(self): - if self._hConsole is None: - try: - self._stream.flush() - except Exception as e: - self._complain("%s.flush: %r from %r" % (self.name, e, self._stream)) - raise - - def write(self, text): - try: - if self._hConsole is None: - # There is no Windows console available. That means we are - # responsible for encoding the unicode to a byte string to - # write it to a Python file object. - if isinstance(text, str): - text = text.encode('utf-8') - self._stream.write(text) - else: - # There is a Windows console available. That means Windows is - # responsible for dealing with the unicode itself. - if not isinstance(text, str): - text = str(text).decode('utf-8') - remaining = len(text) - while remaining > 0: - n = DWORD(0) - # There is a shorter-than-documented limitation on the - # length of the string passed to WriteConsoleW (see - # #1232). - retval = WriteConsoleW(self._hConsole, text, min(remaining, 10000), byref(n), None) - if retval == 0: - raise IOError("WriteConsoleW failed with WinError: %s" % (WinError(get_last_error()),)) - if n.value == 0: - raise IOError("WriteConsoleW returned %r, n.value = 0" % (retval,)) - remaining -= n.value - if remaining == 0: break - text = text[n.value:] - except Exception as e: - self._complain("%s.write: %r" % (self.name, e)) - raise - - def writelines(self, lines): - try: - for line in lines: - self.write(line) - except Exception as e: - self._complain("%s.writelines: %r" % (self.name, e)) - raise From dfdb6c60d0e0f1fc24ca7409069db72dd4a6cf1c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 1 Mar 2024 12:17:11 -0500 Subject: [PATCH 08/11] Fix lints. --- src/allmydata/check_results.py | 2 +- src/allmydata/test/cli/test_list.py | 2 +- src/allmydata/test/test_encodingutil.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/allmydata/check_results.py b/src/allmydata/check_results.py index e9ef59bdbb..44a4a1db8b 100644 --- a/src/allmydata/check_results.py +++ b/src/allmydata/check_results.py @@ -61,7 +61,7 @@ def __init__(self, uri, storage_index, # On Python 2, we can mix bytes and Unicode. On Python 3, we want # unicode. if isinstance(summary, bytes): - summary = unicode(summary, "utf-8") + summary = str(summary, "utf-8") assert isinstance(summary, str) # should be a single string self._summary = summary assert not isinstance(report, str) # should be list of strings diff --git a/src/allmydata/test/cli/test_list.py b/src/allmydata/test/cli/test_list.py index f9ab52c53f..55f0952fe0 100644 --- a/src/allmydata/test/cli/test_list.py +++ b/src/allmydata/test/cli/test_list.py @@ -9,7 +9,7 @@ from allmydata.interfaces import MDMF_VERSION, SDMF_VERSION from allmydata.mutable.publish import MutableData from ..no_network import GridTestMixin -from allmydata.util.encodingutil import quote_output, get_io_encoding +from allmydata.util.encodingutil import quote_output from .common import CLITestMixin diff --git a/src/allmydata/test/test_encodingutil.py b/src/allmydata/test/test_encodingutil.py index 210d462032..c98ef9e401 100644 --- a/src/allmydata/test/test_encodingutil.py +++ b/src/allmydata/test/test_encodingutil.py @@ -65,7 +65,7 @@ from allmydata.util import encodingutil, fileutil from allmydata.util.encodingutil import unicode_to_url, \ unicode_to_output, quote_output, quote_path, quote_local_unicode_path, \ - quote_filepath, unicode_platform, listdir_unicode, FilenameEncodingError, \ + quote_filepath, unicode_platform, listdir_unicode, \ get_filesystem_encoding, to_bytes, from_utf8_or_none, _reload, \ to_filepath, extend_filepath, unicode_from_filepath, unicode_segments_from, \ unicode_to_argv From 6c48698f28fae226ff7f872a49fb5810302ec9d5 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 1 Mar 2024 13:00:24 -0500 Subject: [PATCH 09/11] No need for repetition. --- src/allmydata/immutable/upload.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/allmydata/immutable/upload.py b/src/allmydata/immutable/upload.py index e62f7e0c11..de59d3dc9c 100644 --- a/src/allmydata/immutable/upload.py +++ b/src/allmydata/immutable/upload.py @@ -55,7 +55,7 @@ _TOTAL_SHARES = Field.for_types( u"total_shares", - [int, int], + [int], u"The total number of shares desired.", ) @@ -102,7 +102,7 @@ def _serialize_happiness_mappings(happiness_mappings): _HAPPINESS = Field.for_types( u"happiness", - [int, int], + [int], u"The computed happiness of a certain placement.", ) @@ -140,7 +140,7 @@ def _serialize_happiness_mappings(happiness_mappings): _EFFECTIVE_HAPPINESS = Field.for_types( u"effective_happiness", - [int, int], + [int], u"The computed happiness value of a share placement map.", ) From a9015cdb5baa271f70fa1aa3a4fbc04cb7e9bf91 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 1 Mar 2024 13:06:32 -0500 Subject: [PATCH 10/11] Remove another future import. --- src/allmydata/test/common_util.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/common_util.py b/src/allmydata/test/common_util.py index 70713a995d..d52cb8afa6 100644 --- a/src/allmydata/test/common_util.py +++ b/src/allmydata/test/common_util.py @@ -2,8 +2,6 @@ Ported to Python 3. """ -from future.utils import bchr - import os import sys import time @@ -25,6 +23,9 @@ from ..scripts import runner from allmydata.util.encodingutil import unicode_platform, get_filesystem_encoding, argv_type, unicode_to_argv +def bchr(s): + return bytes([s]) + def skip_if_cannot_represent_filename(u): precondition(isinstance(u, str)) From 09854684e6a3c47e23c7aae8cfcfd676f5d8ffdd Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 1 Mar 2024 13:08:09 -0500 Subject: [PATCH 11/11] Remove another future import. --- src/allmydata/test/storage_plugin.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/storage_plugin.py b/src/allmydata/test/storage_plugin.py index f638d298f5..46088903f7 100644 --- a/src/allmydata/test/storage_plugin.py +++ b/src/allmydata/test/storage_plugin.py @@ -5,7 +5,6 @@ Ported to Python 3. """ -from future.utils import native_str_to_bytes from six import ensure_str import attr @@ -87,7 +86,7 @@ def get_client_resource(self, configuration): """ items = configuration.items(self._client_section_name, []) resource = Data( - native_str_to_bytes(dumps(dict(items))), + dumps(dict(items)).encode("utf-8"), ensure_str("text/json"), ) # Give it some dynamic stuff too. @@ -105,7 +104,7 @@ class GetCounter(Resource, object): value = 0 def render_GET(self, request): self.value += 1 - return native_str_to_bytes(dumps({"value": self.value})) + return dumps({"value": self.value}).encode("utf-8") @implementer(RIDummy)