Merge f1bd7c0 into aa22c6e

integration/conftest.py

@@ -28,7 +28,7 @@
 import pytest
 import pytest_twisted
 
-from util import (
+from .util import (
     _CollectOutputProtocol,
     _MagicTextProtocol,
     _DumpOutputProtocol,

integration/test_aaa_aardvark.py

@@ -5,6 +5,15 @@
 # You can safely skip any of these tests, it'll just appear to "take
 # longer" to start the first test as the fixtures get built
 
+from __future__ import unicode_literals
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from future.utils import PY2
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+
 
 def test_create_flogger(flog_gatherer):
     print("Created flog_gatherer")

integration/test_servers_of_happiness.py

@@ -1,9 +1,21 @@
+"""
+Ported to Python 3.
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+
 import sys
 from os.path import join
 
 from twisted.internet.error import ProcessTerminated
 
-import util
+from . import util
 
 import pytest_twisted
 
@@ -42,4 +54,4 @@ def test_upload_immutable(reactor, temp_dir, introducer_furl, flog_gatherer, sto
         assert isinstance(e, ProcessTerminated)
 
     output = proto.output.getvalue()
-    assert "shares could be placed on only" in output
+    assert b"shares could be placed on only" in output

integration/test_streaming_logs.py

@@ -1,16 +1,25 @@
+"""
+Ported to Python 3.
+"""
 from __future__ import (
     print_function,
     unicode_literals,
     absolute_import,
     division,
 )
 
+from future.utils import PY2
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+
+from six import ensure_text
+
 import json
 
 from os.path import (
     join,
 )
-from urlparse import (
+from urllib.parse import (
     urlsplit,
 )
 
@@ -68,7 +77,7 @@ def _connect_client(reactor, api_auth_token, ws_url):
     factory = WebSocketClientFactory(
         url=ws_url,
         headers={
-            "Authorization": "{} {}".format(SCHEME, api_auth_token),
+            "Authorization": "{} {}".format(str(SCHEME, "ascii"), api_auth_token),
         }
     )
     factory.protocol = _StreamingLogClientProtocol
@@ -127,7 +136,7 @@ def _test_streaming_logs(reactor, temp_dir, alice):
     node_url = cfg.get_config_from_file("node.url")
     api_auth_token = cfg.get_private_config("api_auth_token")
 
-    ws_url = node_url.replace("http://", "ws://")
+    ws_url = ensure_text(node_url).replace("http://", "ws://")
     log_url = ws_url + "private/logs/v1"
 
     print("Connecting to {}".format(log_url))

integration/util.py

@@ -5,7 +5,7 @@
 import json
 from os import mkdir, environ
 from os.path import exists, join
-from six.moves import StringIO
+from io import StringIO, BytesIO
 from functools import partial
 from subprocess import check_output
 
@@ -59,7 +59,7 @@ class _CollectOutputProtocol(ProcessProtocol):
     """
     def __init__(self):
         self.done = Deferred()
-        self.output = StringIO()
+        self.output = BytesIO()
 
     def processEnded(self, reason):
         if not self.done.called:
@@ -73,7 +73,7 @@ def outReceived(self, data):
         self.output.write(data)
 
     def errReceived(self, data):
-        print("ERR: {}".format(data))
+        print("ERR: {!r}".format(data))
         self.output.write(data)
 
 
@@ -94,9 +94,11 @@ def processExited(self, reason):
             self.done.errback(reason)
 
     def outReceived(self, data):
+        data = unicode(data, sys.stdout.encoding)
         self._out.write(data)
 
     def errReceived(self, data):
+        data = unicode(data, sys.stdout.encoding)
         self._out.write(data)
 
 
@@ -116,13 +118,15 @@ def processEnded(self, reason):
         self.exited.callback(None)
 
     def outReceived(self, data):
+        data = unicode(data, sys.stdout.encoding)
         sys.stdout.write(data)
         self._output.write(data)
         if not self.magic_seen.called and self._magic_text in self._output.getvalue():
             print("Saw '{}' in the logs".format(self._magic_text))
             self.magic_seen.callback(self)
 
     def errReceived(self, data):
+        data = unicode(data, sys.stderr.encoding)
         sys.stdout.write(data)
 
 
@@ -282,7 +286,7 @@ def created(_):
                 config,
                 u'node',
                 u'log_gatherer.furl',
-                flog_gatherer.decode("utf-8"),
+                flog_gatherer,
             )
             write_config(FilePath(config_path), config)
         created_d.addCallback(created)
@@ -528,7 +532,8 @@ def generate_ssh_key(path):
     key = RSAKey.generate(2048)
     key.write_private_key_file(path)
     with open(path + ".pub", "wb") as f:
-        f.write(b"%s %s" % (key.get_name(), key.get_base64()))
+        s = "%s %s" % (key.get_name(), key.get_base64())
+        f.write(s.encode("ascii"))
 
 
 def run_in_thread(f):

src/allmydata/client.py

@@ -928,7 +928,8 @@ def get_auth_token(self):
         random data in "api_auth_token" which must be echoed to API
         calls.
         """
-        return self.config.get_private_config('api_auth_token')
+        return self.config.get_private_config(
+            'api_auth_token').encode("ascii")
 
     def _create_auth_token(self):
         """

src/allmydata/frontends/auth.py

@@ -80,5 +80,5 @@ def _checkPassword(self, creds):
             return defer.fail(error.UnauthorizedLogin())
 
         d = defer.maybeDeferred(creds.checkPassword, correct)
-        d.addCallback(self._cbPasswordMatch, str(creds.username))
+        d.addCallback(self._cbPasswordMatch, creds.username)
         return d

src/allmydata/frontends/sftpd.py

@@ -1011,8 +1011,8 @@ def __init__(self, client, rootnode, username):
         PrefixingLogMixin.__init__(self, facility="tahoe.sftp", prefix=username)
         if noisy: self.log(".__init__(%r, %r, %r)" % (client, rootnode, username), level=NOISY)
 
-        self.channelLookup["session"] = session.SSHSession
-        self.subsystemLookup["sftp"] = FileTransferServer
+        self.channelLookup[b"session"] = session.SSHSession
+        self.subsystemLookup[b"sftp"] = FileTransferServer
 
         self._client = client
         self._root = rootnode

src/allmydata/test/test_auth.py

@@ -65,7 +65,7 @@ def test_unknown_user(self):
         avatarId = self.checker.requestAvatarId(key_credentials)
         return self.assertFailure(avatarId, error.UnauthorizedLogin)
 
-    def test_password_auth_user(self):
+    def test_password_auth_user_with_ssh_key(self):
         """
         AccountFileChecker.requestAvatarId returns a Deferred that fires with
         UnauthorizedLogin if called with an SSHPrivateKey object for a username
@@ -76,6 +76,29 @@ def test_password_auth_user(self):
         avatarId = self.checker.requestAvatarId(key_credentials)
         return self.assertFailure(avatarId, error.UnauthorizedLogin)
 
+    def test_password_auth_user_with_correct_password(self):
+        """
+        AccountFileChecker.requestAvatarId returns a Deferred that fires with
+        the user if the correct password is given.
+        """
+        key_credentials = credentials.UsernamePassword(b"alice", b"password")
+        d = self.checker.requestAvatarId(key_credentials)
+        def authenticated(avatarId):
+            self.assertEqual(
+                (b"alice",
+                 b"URI:DIR2:aaaaaaaaaaaaaaaaaaaaaaaaaa:1111111111111111111111111111111111111111111111111111"),
+                (avatarId.username, avatarId.rootcap))
+        return d
+
+    def test_password_auth_user_with_wrong_password(self):
+        """
+        AccountFileChecker.requestAvatarId returns a Deferred that fires with
+        UnauthorizedLogin if the wrong password is given.
+        """
+        key_credentials = credentials.UsernamePassword(b"alice", b"WRONG")
+        avatarId = self.checker.requestAvatarId(key_credentials)
+        return self.assertFailure(avatarId, error.UnauthorizedLogin)
+
     def test_unrecognized_key(self):
         """
         AccountFileChecker.requestAvatarId returns a Deferred that fires with

src/allmydata/test/test_client.py

@@ -415,7 +415,7 @@ def test_web_apiauthtoken(self):
             f.write("deadbeef")
 
         token = c.get_auth_token()
-        self.assertEqual("deadbeef", token)
+        self.assertEqual(b"deadbeef", token)
 
     @defer.inlineCallbacks
     def test_web_staticdir(self):

src/allmydata/test/web/test_web.py

@@ -218,7 +218,7 @@ def is_connected(self): # TODO: remove me
         return self.connected
     def get_version(self):
         return {
-            "application-version": "1.0"
+            b"application-version": b"1.0"
         }
     def get_permutation_seed(self):
         return b""

src/allmydata/util/_python3.py

@@ -16,6 +16,18 @@
 if PY2:
     from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
+
+# Every time a module is added here, also add it to tox.ini environment
+# integrations3. Bit of duplication, but it's only a handful of files and quite
+# temporary, just until we've ported them all.
+PORTED_INTEGRATION_TESTS = [
+    "integration.test_aaa_aardvark",
+    "integration.test_servers_of_happiness",
+    "integration.test_sftp",
+    "integration.test_streaming_logs",
+]
+
+
 # Keep these sorted alphabetically, to reduce merge conflicts:
 PORTED_MODULES = [
     "allmydata",

src/allmydata/web/root.py

@@ -318,7 +318,7 @@ def _describe_server(self, server):
         }
         version = server.get_version()
         if version is not None:
-            description[u"version"] = version["application-version"]
+            description[u"version"] = version[b"application-version"]
 
         return description
 

tox.ini

@@ -7,7 +7,7 @@
 [gh-actions]
 python =
     2.7: py27-coverage,codechecks
-    3.6: py36-coverage
+    3.6: py36-coverage,integration3
     3.7: py37-coverage
     3.8: py38-coverage
     3.9: py39-coverage,typechecks,codechecks3
@@ -17,7 +17,7 @@ python =
 twisted = 1
 
 [tox]
-envlist = typechecks,codechecks,codechecks3,py{27,36,37,38,39}-{coverage},pypy27,pypy3
+envlist = typechecks,codechecks,codechecks3,py{27,36,37,38,39}-{coverage},pypy27,pypy3,integration,integration3
 minversion = 2.4
 
 [testenv]
@@ -96,6 +96,17 @@ commands =
          coverage report
 
 
+[testenv:integration3]
+basepython = python3
+setenv =
+         COVERAGE_PROCESS_START=.coveragerc
+commands =
+         # NOTE: 'run with "py.test --keep-tempdir -s -v integration/" to debug failures'
+         python3 -b -m pytest --timeout=1800 --coverage -v {posargs:integration/test_aaa_aardvark.py integration/test_servers_of_happiness.py integration/test_sftp.py integration/test_streaming_logs.py}
+         coverage combine
+         coverage report
+
+
 [testenv:codechecks]
 basepython = python2.7
 # On macOS, git inside of towncrier needs $HOME.