Merge pull request #905 from tahoe-lafs/3514.test-introducer-python-3

Port test_introducer.py to Python 3

Fixes ticket:3514

src/allmydata/client.py

@@ -1,3 +1,5 @@
+from past.builtins import unicode
+
 import os, stat, time, weakref
 from base64 import urlsafe_b64encode
 from functools import partial
@@ -728,10 +730,14 @@ def get_stats(self):
         return { 'node.uptime': time.time() - self.started_timestamp }
 
     def init_secrets(self):
-        lease_s = self.config.get_or_create_private_config("secret", _make_secret)
+        # configs are always unicode
+        def _unicode_make_secret():
+            return unicode(_make_secret(), "ascii")
+        lease_s = self.config.get_or_create_private_config(
+            "secret", _unicode_make_secret).encode("utf-8")
         lease_secret = base32.a2b(lease_s)
-        convergence_s = self.config.get_or_create_private_config('convergence',
-                                                                 _make_secret)
+        convergence_s = self.config.get_or_create_private_config(
+            'convergence', _unicode_make_secret).encode("utf-8")
         self.convergence = base32.a2b(convergence_s)
         self._secret_holder = SecretHolder(lease_secret, self.convergence)
 
@@ -740,9 +746,11 @@ def init_node_key(self):
         # existing key
         def _make_key():
             private_key, _ = ed25519.create_signing_keypair()
-            return ed25519.string_from_signing_key(private_key) + b"\n"
+            # Config values are always unicode:
+            return unicode(ed25519.string_from_signing_key(private_key) + b"\n", "utf-8")
 
-        private_key_str = self.config.get_or_create_private_config("node.privkey", _make_key)
+        private_key_str = self.config.get_or_create_private_config(
+            "node.privkey", _make_key).encode("utf-8")
         private_key, public_key = ed25519.signing_keypair_from_string(private_key_str)
         public_key_str = ed25519.string_from_verifying_key(public_key)
         self.config.write_config_file("node.pubkey", public_key_str + b"\n", "wb")

src/allmydata/introducer/client.py

@@ -1,4 +1,5 @@
-from past.builtins import unicode
+from past.builtins import unicode, long
+from six import ensure_text
 
 import time
 from zope.interface import implementer
@@ -17,7 +18,7 @@
 class InvalidCacheError(Exception):
     pass
 
-V2 = "http://allmydata.org/tahoe/protocols/introducer/v2"
+V2 = b"http://allmydata.org/tahoe/protocols/introducer/v2"
 
 @implementer(RIIntroducerSubscriberClient_v2, IIntroducerClient)
 class IntroducerClient(service.Service, Referenceable):
@@ -26,6 +27,8 @@ def __init__(self, tub, introducer_furl,
                  nickname, my_version, oldest_supported,
                  sequencer, cache_filepath):
         self._tub = tub
+        if isinstance(introducer_furl, unicode):
+            introducer_furl = introducer_furl.encode("utf-8")
         self.introducer_furl = introducer_furl
 
         assert type(nickname) is unicode
@@ -35,11 +38,11 @@ def __init__(self, tub, introducer_furl,
         self._sequencer = sequencer
         self._cache_filepath = cache_filepath
 
-        self._my_subscriber_info = { "version": 0,
-                                     "nickname": self._nickname,
-                                     "app-versions": [],
-                                     "my-version": self._my_version,
-                                     "oldest-supported": self._oldest_supported,
+        self._my_subscriber_info = { b"version": 0,
+                                     b"nickname": self._nickname,
+                                     b"app-versions": [],
+                                     b"my-version": self._my_version,
+                                     b"oldest-supported": self._oldest_supported,
                                      }
 
         self._outbound_announcements = {} # not signed
@@ -113,19 +116,24 @@ def _save_announcements(self):
         announcements = []
         for _, value in self._inbound_announcements.items():
             ann, key_s, time_stamp = value
+            # On Python 2, bytes strings are encoded into YAML Unicode strings.
+            # On Python 3, bytes are encoded as YAML bytes. To minimize
+            # changes, Python 3 for now ensures the same is true.
             server_params = {
                 "ann" : ann,
-                "key_s" : key_s,
+                "key_s" : ensure_text(key_s),
                 }
             announcements.append(server_params)
         announcement_cache_yaml = yamlutil.safe_dump(announcements)
+        if isinstance(announcement_cache_yaml, unicode):
+            announcement_cache_yaml = announcement_cache_yaml.encode("utf-8")
         self._cache_filepath.setContent(announcement_cache_yaml)
 
     def _got_introducer(self, publisher):
         self.log("connected to introducer, getting versions")
-        default = { "http://allmydata.org/tahoe/protocols/introducer/v1":
+        default = { b"http://allmydata.org/tahoe/protocols/introducer/v1":
                     { },
-                    "application-version": "unknown: no get_version()",
+                    b"application-version": b"unknown: no get_version()",
                     }
         d = add_version_to_remote_reference(publisher, default)
         d.addCallback(self._got_versioned_introducer)
@@ -138,6 +146,7 @@ def _got_error(self, f):
     def _got_versioned_introducer(self, publisher):
         self.log("got introducer version: %s" % (publisher.version,))
         # we require an introducer that speaks at least V2
+        assert all(type(V2) == type(v) for v in publisher.version)
         if V2 not in publisher.version:
             raise InsufficientVersionError("V2", publisher.version)
         self._publisher = publisher
@@ -162,7 +171,7 @@ def subscribe_to(self, service_name, cb, *args, **kwargs):
         self._subscribed_service_names.add(service_name)
         self._maybe_subscribe()
         for index,(ann,key_s,when) in self._inbound_announcements.items():
-            precondition(isinstance(key_s, str), key_s)
+            precondition(isinstance(key_s, bytes), key_s)
             servicename = index[0]
             if servicename == service_name:
                 eventually(cb, key_s, ann, *args, **kwargs)
@@ -238,7 +247,7 @@ def got_announcements(self, announcements, lp=None):
                 # this might raise UnknownKeyError or bad-sig error
                 ann, key_s = unsign_from_foolscap(ann_t)
                 # key is "v0-base32abc123"
-                precondition(isinstance(key_s, str), key_s)
+                precondition(isinstance(key_s, bytes), key_s)
             except BadSignature:
                 self.log("bad signature on inbound announcement: %s" % (ann_t,),
                          parent=lp, level=log.WEIRD, umid="ZAU15Q")
@@ -248,7 +257,7 @@ def got_announcements(self, announcements, lp=None):
             self._process_announcement(ann, key_s)
 
     def _process_announcement(self, ann, key_s):
-        precondition(isinstance(key_s, str), key_s)
+        precondition(isinstance(key_s, bytes), key_s)
         self._debug_counts["inbound_announcement"] += 1
         service_name = str(ann["service-name"])
         if service_name not in self._subscribed_service_names:
@@ -257,7 +266,7 @@ def _process_announcement(self, ann, key_s):
             self._debug_counts["wrong_service"] += 1
             return
         # for ASCII values, simplejson might give us unicode *or* bytes
-        if "nickname" in ann and isinstance(ann["nickname"], str):
+        if "nickname" in ann and isinstance(ann["nickname"], bytes):
             ann["nickname"] = unicode(ann["nickname"])
         nick_s = ann.get("nickname",u"").encode("utf-8")
         lp2 = self.log(format="announcement for nickname '%(nick)s', service=%(svc)s: %(ann)s",
@@ -266,11 +275,11 @@ def _process_announcement(self, ann, key_s):
         # how do we describe this node in the logs?
         desc_bits = []
         assert key_s
-        desc_bits.append("serverid=" + key_s[:20])
+        desc_bits.append(b"serverid=" + key_s[:20])
         if "anonymous-storage-FURL" in ann:
             tubid_s = get_tubid_string_from_ann(ann)
-            desc_bits.append("tubid=" + tubid_s[:8])
-        description = "/".join(desc_bits)
+            desc_bits.append(b"tubid=" + tubid_s[:8])
+        description = b"/".join(desc_bits)
 
         # the index is used to track duplicates
         index = (service_name, key_s)
@@ -320,7 +329,7 @@ def _process_announcement(self, ann, key_s):
         self._deliver_announcements(key_s, ann)
 
     def _deliver_announcements(self, key_s, ann):
-        precondition(isinstance(key_s, str), key_s)
+        precondition(isinstance(key_s, bytes), key_s)
         service_name = str(ann["service-name"])
         for (service_name2,cb,args,kwargs) in self._local_subscribers:
             if service_name2 == service_name:

src/allmydata/introducer/common.py

@@ -1,16 +1,19 @@
+from past.builtins import unicode
+
 import re
-import json
 from allmydata.crypto.util import remove_prefix
 from allmydata.crypto import ed25519
-from allmydata.util import base32, rrefutil
+from allmydata.util import base32, rrefutil, jsonbytes as json
 
 
 def get_tubid_string_from_ann(ann):
-    return get_tubid_string(str(ann.get("anonymous-storage-FURL")
-                                or ann.get("FURL")))
+    furl = ann.get("anonymous-storage-FURL") or ann.get("FURL")
+    if isinstance(furl, unicode):
+        furl = furl.encode("utf-8")
+    return get_tubid_string(furl)
 
 def get_tubid_string(furl):
-    m = re.match(r'pb://(\w+)@', furl)
+    m = re.match(br'pb://(\w+)@', furl)
     assert m
     return m.group(1).lower()
 

src/allmydata/introducer/server.py

@@ -1,3 +1,5 @@
+from past.builtins import long
+from six import ensure_str, ensure_text
 
 import time, os.path, textwrap
 from zope.interface import implementer
@@ -7,7 +9,7 @@
 from foolscap.api import Referenceable
 import allmydata
 from allmydata import node
-from allmydata.util import log, rrefutil
+from allmydata.util import log, rrefutil, dictutil
 from allmydata.util.i2p_provider import create as create_i2p_provider
 from allmydata.util.tor_provider import create as create_tor_provider
 from allmydata.introducer.interfaces import \
@@ -122,7 +124,7 @@ def init_web(self, webport):
 
         from allmydata.webish import IntroducerWebishServer
         nodeurl_path = self.config.get_config_path(u"node.url")
-        config_staticdir = self.get_config("node", "web.static", "public_html").decode('utf-8')
+        config_staticdir = self.get_config("node", "web.static", "public_html")
         staticdir = self.config.get_config_path(config_staticdir)
         ws = IntroducerWebishServer(self, webport, nodeurl_path, staticdir)
         ws.setServiceParent(self)
@@ -133,8 +135,8 @@ class IntroducerService(service.MultiService, Referenceable):
     # v1 is the original protocol, added in 1.0 (but only advertised starting
     # in 1.3), removed in 1.12. v2 is the new signed protocol, added in 1.10
     VERSION = { #"http://allmydata.org/tahoe/protocols/introducer/v1": { },
-                "http://allmydata.org/tahoe/protocols/introducer/v2": { },
-                "application-version": str(allmydata.__full_version__),
+                b"http://allmydata.org/tahoe/protocols/introducer/v2": { },
+                b"application-version": allmydata.__full_version__.encode("utf-8"),
                 }
 
     def __init__(self):
@@ -279,6 +281,10 @@ def _publish(self, ann_t, canary, lp):
     def remote_subscribe_v2(self, subscriber, service_name, subscriber_info):
         self.log("introducer: subscription[%s] request at %s"
                  % (service_name, subscriber), umid="U3uzLg")
+        service_name = ensure_str(service_name)
+        subscriber_info = dictutil.UnicodeKeyDict({
+            ensure_text(k): v for (k, v) in subscriber_info.items()
+        })
         return self.add_subscriber(subscriber, service_name, subscriber_info)
 
     def add_subscriber(self, subscriber, service_name, subscriber_info):
@@ -302,6 +308,10 @@ def _remove():
         subscriber.notifyOnDisconnect(_remove)
 
         # now tell them about any announcements they're interested in
+        assert {type(service_name)}.issuperset(
+            set(type(k[0]) for k in self._announcements)), (
+                service_name, self._announcements.keys()
+        )
         announcements = set( [ ann_t
                                for idx,(ann_t,canary,ann,when)
                                in self._announcements.items()

src/allmydata/mutable/publish.py

@@ -914,7 +914,7 @@ def _connection_problem(self, f, writer):
 
     def log_goal(self, goal, message=""):
         logmsg = [message]
-        for (shnum, server) in sorted([(s,p) for (p,s) in goal]):
+        for (shnum, server) in sorted([(s,p) for (p,s) in goal], key=lambda t: (id(t[0]), id(t[1]))):
             logmsg.append("sh%d to [%s]" % (shnum, server.get_name()))
         self.log("current goal: %s" % (", ".join(logmsg)), level=log.NOISY)
         self.log("we are planning to push new seqnum=#%d" % self._new_seqnum,