Merge pull request tahoe-lafs#945 from tahoe-lafs/3566.web-tests-pyth…

…on-3-part-2

Port web tests to Python 3, part 2

Fixes ticket:3566

src/allmydata/blacklist.py

@@ -34,10 +34,10 @@ def read_blacklist(self):
         try:
             if self.last_mtime is None or current_mtime > self.last_mtime:
                 self.entries.clear()
-                with open(self.blacklist_fn, "r") as f:
+                with open(self.blacklist_fn, "rb") as f:
                     for line in f:
                         line = line.strip()
-                        if not line or line.startswith("#"):
+                        if not line or line.startswith(b"#"):
                             continue
                         si_s, reason = line.split(None, 1)
                         si = base32.a2b(si_s) # must be valid base32

src/allmydata/dirnode.py

@@ -18,7 +18,6 @@
 from zope.interface import implementer
 from twisted.internet import defer
 from foolscap.api import fireEventually
-import json
 
 from allmydata.crypto import aes
 from allmydata.deep_stats import DeepStats
@@ -31,7 +30,7 @@
 from allmydata.check_results import DeepCheckResults, \
      DeepCheckAndRepairResults
 from allmydata.monitor import Monitor
-from allmydata.util import hashutil, base32, log
+from allmydata.util import hashutil, base32, log, jsonbytes as json
 from allmydata.util.encodingutil import quote_output, normalize
 from allmydata.util.assertutil import precondition
 from allmydata.util.netstring import netstring, split_netstring

src/allmydata/scripts/debug.py

@@ -1,5 +1,7 @@
 from __future__ import print_function
 
+from future.utils import bchr
+
 # do not import any allmydata modules at this level. Do that from inside
 # individual functions instead.
 import struct, time, os, sys
@@ -905,7 +907,7 @@ def flip_bit(start, end):
         f = open(fn, "rb+")
         f.seek(offset)
         d = f.read(1)
-        d = chr(ord(d) ^ 0x01)
+        d = bchr(ord(d) ^ 0x01)
         f.seek(offset)
         f.write(d)
         f.close()
@@ -920,7 +922,7 @@ def flip_bit(start, end):
         f.seek(m.DATA_OFFSET)
         data = f.read(2000)
         # make sure this slot contains an SMDF share
-        assert data[0] == b"\x00", "non-SDMF mutable shares not supported"
+        assert data[0:1] == b"\x00", "non-SDMF mutable shares not supported"
         f.close()
 
         (version, ig_seqnum, ig_roothash, ig_IV, ig_k, ig_N, ig_segsize,

src/allmydata/test/common.py

@@ -11,7 +11,7 @@
     "skipIf",
 ]
 
-from past.builtins import chr as byteschr
+from past.builtins import chr as byteschr, unicode
 
 import os, random, struct
 import six
@@ -825,13 +825,18 @@ def shouldHTTPError(self, which,
                         code=None, substring=None, response_substring=None,
                         callable=None, *args, **kwargs):
         # returns a Deferred with the response body
-        assert substring is None or isinstance(substring, str)
+        if isinstance(substring, bytes):
+            substring = unicode(substring, "ascii")
+        if isinstance(response_substring, unicode):
+            response_substring = response_substring.encode("ascii")
+        assert substring is None or isinstance(substring, unicode)
+        assert response_substring is None or isinstance(response_substring, bytes)
         assert callable
         def _validate(f):
             if code is not None:
-                self.failUnlessEqual(f.value.status, str(code), which)
+                self.failUnlessEqual(f.value.status, b"%d" % code, which)
             if substring:
-                code_string = str(f)
+                code_string = unicode(f)
                 self.failUnless(substring in code_string,
                                 "%s: substring '%s' not in '%s'"
                                 % (which, substring, code_string))

src/allmydata/test/common_util.py

@@ -1,7 +1,8 @@
 from __future__ import print_function
 
-from future.utils import PY2, native_str
+from future.utils import PY2, native_str, bchr, binary_type
 from future.builtins import str as future_str
+from past.builtins import unicode
 
 import os
 import time
@@ -20,9 +21,6 @@
 from ..util.assertutil import precondition
 from ..scripts import runner
 from allmydata.util.encodingutil import unicode_platform, get_filesystem_encoding, get_io_encoding
-# Imported for backwards compatibility:
-from future.utils import bord, bchr, binary_type
-from past.builtins import unicode
 
 
 def skip_if_cannot_represent_filename(u):
@@ -183,13 +181,12 @@ def insecurerandstr(n):
     return b''.join(map(bchr, map(randrange, [0]*n, [256]*n)))
 
 def flip_bit(good, which):
-    # TODO Probs need to update with bchr/bord as with flip_one_bit, below.
-    # flip the low-order bit of good[which]
+    """Flip the low-order bit of good[which]."""
     if which == -1:
-        pieces = good[:which], good[-1:], ""
+        pieces = good[:which], good[-1:], b""
     else:
         pieces = good[:which], good[which:which+1], good[which+1:]
-    return pieces[0] + chr(ord(pieces[1]) ^ 0x01) + pieces[2]
+    return pieces[0] + bchr(ord(pieces[1]) ^ 0x01) + pieces[2]
 
 def flip_one_bit(s, offset=0, size=None):
     """ flip one random bit of the string s, in a byte greater than or equal to offset and less
@@ -198,7 +195,7 @@ def flip_one_bit(s, offset=0, size=None):
     if size is None:
         size=len(s)-offset
     i = randrange(offset, offset+size)
-    result = s[:i] + bchr(bord(s[i])^(0x01<<randrange(0, 8))) + s[i+1:]
+    result = s[:i] + bchr(ord(s[i:i+1])^(0x01<<randrange(0, 8))) + s[i+1:]
     assert result != s, "Internal error -- flip_one_bit() produced the same string as its input: %s == %s" % (result, s)
     return result
 

src/allmydata/test/no_network.py

@@ -24,6 +24,7 @@
 if PY2:
     from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 from past.builtins import unicode
+from six import ensure_text
 
 import os
 from base64 import b32encode
@@ -614,8 +615,7 @@ def GET(self, urlpath, followRedirect=False, return_response=False,
             method="GET", clientnum=0, **kwargs):
         # if return_response=True, this fires with (data, statuscode,
         # respheaders) instead of just data.
-        assert not isinstance(urlpath, unicode)
-        url = self.client_baseurls[clientnum] + urlpath
+        url = self.client_baseurls[clientnum] + ensure_text(urlpath)
 
         response = yield treq.request(method, url, persistent=False,
                                       allow_redirects=followRedirect,

src/allmydata/test/test_checker.py

@@ -173,7 +173,7 @@ def create_fake_client(self):
         return c
 
     def render_json(self, resource):
-        return self.successResultOf(render(resource, {"output": ["json"]}))
+        return self.successResultOf(render(resource, {b"output": [b"json"]}))
 
     def render_element(self, element, args=None):
         if args is None:
@@ -186,7 +186,7 @@ def test_literal(self):
         html = self.render_element(lcr)
         self.failUnlessIn(b"Literal files are always healthy", html)
 
-        html = self.render_element(lcr, args={"return_to": ["FOOURL"]})
+        html = self.render_element(lcr, args={b"return_to": [b"FOOURL"]})
         self.failUnlessIn(b"Literal files are always healthy", html)
         self.failUnlessIn(b'<a href="FOOURL">Return to file.</a>', html)
 
@@ -269,7 +269,7 @@ def test_check(self):
         self.failUnlessIn("File Check Results for SI=2k6avp", s) # abbreviated
         self.failUnlessIn("Not Recoverable! : rather dead", s)
 
-        html = self.render_element(w, args={"return_to": ["FOOURL"]})
+        html = self.render_element(w, args={b"return_to": [b"FOOURL"]})
         self.failUnlessIn(b'<a href="FOOURL">Return to file/directory.</a>',
                           html)
 

src/allmydata/test/test_storage_web.py

@@ -70,7 +70,7 @@ def renderJSON(resource):
     """
     Render a JSON from the given resource.
     """
-    return render(resource, {"t": ["json"]})
+    return render(resource, {b"t": [b"json"]})
 
 class MyBucketCountingCrawler(BucketCountingCrawler):
     def finished_prefix(self, cycle, prefix):