Ticket3252 port web directory.remaining.1

src/allmydata/scripts/tahoe_backup.py

@@ -51,8 +51,8 @@ def mkdir(contents, options):
     return dircap
 
 def put_child(dirurl, childname, childcap):
-    assert dirurl[-1] == "/"
-    url = dirurl + urllib.quote(unicode_to_url(childname)) + "?t=uri"
+    assert dirurl[-1] != "/"
+    url = dirurl + "/" + urllib.quote(unicode_to_url(childname)) + "?t=uri"
     resp = do_http("PUT", url, childcap)
     if resp.status not in (200, 201):
         raise HTTPError("Error during put_child", resp)
@@ -105,6 +105,9 @@ def run(self):
 
         archives_url = to_url + "Archives/"
 
+        archives_url = archives_url.rstrip("/")
+        to_url = to_url.rstrip("/")
+
         # first step: make sure the target directory exists, as well as the
         # Archives/ subdirectory.
         resp = do_http("GET", archives_url + "?t=json")

src/allmydata/test/cli/test_cli.py

@@ -911,8 +911,14 @@ def _check(args, st):
             self.failUnlessReallyEqual(err, "")
             self.failUnlessIn(st, out)
             return out
+
         def _mkdir(ign, mutable_type, uri_prefix, dirname):
-            d2 = self.do_cli("mkdir", "--format="+mutable_type, dirname)
+            """
+            :param str mutable_type: 'sdmf' or 'mdmf' (or uppercase versions)
+            :param str uri_prefix: kind of URI
+            :param str dirname: the directory alias
+            """
+            d2 = self.do_cli("mkdir", "--format={}".format(mutable_type), dirname)
             d2.addCallback(_check, uri_prefix)
             def _stash_filecap(cap):
                 u = uri.from_string(cap)

src/allmydata/test/common.py

@@ -4,13 +4,15 @@
     "SyncTestCase",
     "AsyncTestCase",
     "AsyncBrokenTestCase",
+    "TrialTestCase",
 
     "flush_logged_errors",
     "skip",
     "skipIf",
 ]
 
 import os, random, struct
+import six
 import tempfile
 from tempfile import mktemp
 from functools import partial
@@ -57,6 +59,7 @@
     IReactorSocket,
 )
 from twisted.internet.endpoints import AdoptedStreamServerEndpoint
+from twisted.trial.unittest import TestCase as _TrialTestCase
 
 from allmydata import uri
 from allmydata.interfaces import IMutableFileNode, IImmutableFileNode,\
@@ -1242,3 +1245,29 @@ class AsyncBrokenTestCase(_TestCaseMixin, TestCase):
     run_tests_with = EliotLoggedRunTest.make_factory(
         AsynchronousDeferredRunTestForBrokenTwisted.make_factory(timeout=60.0),
     )
+
+
+class TrialTestCase(_TrialTestCase):
+    """
+    A twisted.trial.unittest.TestCaes with Tahoe required fixes
+    applied. Currently these are:
+
+      - ensure that .fail() passes a bytes msg on Python2
+    """
+
+    def fail(self, msg):
+        """
+        Ensure our msg is a native string on Python2. If it was Unicode,
+        we encode it as utf8 and hope for the best. On Python3 we take
+        no action.
+
+        This is necessary because Twisted passes the 'msg' argument
+        along to the constructor of an exception; on Python2,
+        Exception will accept a `unicode` instance but will fail if
+        you try to turn that Exception instance into a string.
+        """
+
+        if six.PY2:
+            if isinstance(msg, six.text_type):
+                return super(self, TrialTestCase).fail(msg.encode("utf8"))
+        return super(self, TrialTestCase).fail(msg)

src/allmydata/test/test_dirnode.py

@@ -9,6 +9,7 @@
 from allmydata import uri, dirnode
 from allmydata.client import _Client
 from allmydata.immutable import upload
+from allmydata.immutable.literal import LiteralFileNode
 from allmydata.interfaces import IImmutableFileNode, IMutableFileNode, \
      ExistingChildError, NoSuchChildError, MustNotBeUnknownRWError, \
      MustBeDeepImmutableError, MustBeReadonlyError, \
@@ -27,6 +28,9 @@
 from base64 import b32decode
 import allmydata.test.common_util as testutil
 
+from hypothesis import given
+from hypothesis.strategies import text
+
 if six.PY3:
     long = int
 
@@ -1460,6 +1464,33 @@ def _make_kids(self, nm, which):
             kids[unicode(name)] = (nm.create_from_cap(caps[name]), {})
         return kids
 
+    @given(text(min_size=1, max_size=20))
+    def test_pack_unpack_unicode_hypothesis(self, name):
+        """
+        pack -> unpack results in the same objects (with a unicode name)
+        """
+        nm = NodeMaker(None, None, None, None, None, {"k": 3, "n": 10}, None, None)
+        fn = MinimalFakeMutableFile()
+
+        # FIXME TODO: we shouldn't have to do this out here, but
+        # Hypothesis found that a name with "\x2000" does not make the
+        # round-trip properly .. so for now we'll only give the packer
+        # normalized names.
+        # See also:
+        # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/2606
+        # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/1076
+        name = unicodedata.normalize('NFC', name)
+
+        kids = {
+            name: (LiteralFileNode(uri.from_string(one_uri)), {}),
+        }
+        packed = dirnode.pack_children(kids, fn.get_writekey(), deep_immutable=False)
+        write_uri = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q"
+        filenode = nm.create_from_cap(write_uri)
+        dn = dirnode.DirectoryNode(filenode, nm, None)
+        unkids = dn._unpack_contents(packed)
+        self.assertEqual(kids, unkids)
+
     def test_deep_immutable(self):
         nm = NodeMaker(None, None, None, None, None, {"k": 3, "n": 10}, None, None)
         fn = MinimalFakeMutableFile()

src/allmydata/test/web/common.py

@@ -18,6 +18,64 @@ def assert_soup_has_favicon(testcase, soup):
         any(t[u'href'] == u'/icon.png' for t in links), soup)
 
 
+def assert_soup_has_tag_with_attributes(testcase, soup, tag_name, attrs):
+    """
+    Using a ``TestCase`` object ``testcase``, assert that the passed
+    in ``BeatufulSoup`` object ``soup`` contains a tag ``tag_name``
+    (unicode) which has all the attributes in ``attrs`` (dict).
+    """
+    tags = soup.find_all(tag_name)
+    for tag in tags:
+        if all(v in tag.attrs.get(k, []) for k, v in attrs.items()):
+            return  # we found every attr in this tag; done
+    testcase.fail(
+        u"No <{}> tags contain attributes: {}".format(tag_name, attrs)
+    )
+
+
+def assert_soup_has_tag_with_attributes_and_content(testcase, soup, tag_name, content, attrs):
+    """
+    Using a ``TestCase`` object ``testcase``, assert that the passed
+    in ``BeatufulSoup`` object ``soup`` contains a tag ``tag_name``
+    (unicode) which has all the attributes in ``attrs`` (dict) and
+    contains the string ``content`` (unicode).
+    """
+    assert_soup_has_tag_with_attributes(testcase, soup, tag_name, attrs)
+    assert_soup_has_tag_with_content(testcase, soup, tag_name, content)
+
+
+def _normalized_contents(tag):
+    """
+    :returns: all the text contents of the tag with whitespace
+        normalized: all newlines removed and at most one space between
+        words.
+    """
+    return u" ".join(tag.text.split())
+
+
+def assert_soup_has_tag_with_content(testcase, soup, tag_name, content):
+    """
+    Using a ``TestCase`` object ``testcase``, assert that the passed
+    in ``BeatufulSoup`` object ``soup`` contains a tag ``tag_name``
+    (unicode) which contains the string ``content`` (unicode).
+    """
+    tags = soup.find_all(tag_name)
+    for tag in tags:
+        if content in tag.contents:
+            return
+
+        # make these "fuzzy" options?
+        for c in tag.contents:
+            if content in c:
+                return
+
+        if content in _normalized_contents(tag):
+            return
+    testcase.fail(
+        u"No <{}> tag contains the text '{}'".format(tag_name, content)
+    )
+
+
 def assert_soup_has_text(testcase, soup, text):
     """
     Using a ``TestCase`` object ``testcase``, assert that the passed in

src/allmydata/test/web/test_grid.py

@@ -3,6 +3,9 @@
 import os.path, re, urllib
 import json
 from six.moves import StringIO
+
+from bs4 import BeautifulSoup
+
 from nevow import rend
 from twisted.trial import unittest
 from allmydata import uri, dirnode
@@ -325,8 +328,8 @@ def test_unknown(self, immutable=False):
 
         def _stash_root_and_create_file(n):
             self.rootnode = n
-            self.rooturl = "uri/" + urllib.quote(n.get_uri()) + "/"
-            self.rourl = "uri/" + urllib.quote(n.get_readonly_uri()) + "/"
+            self.rooturl = "uri/" + urllib.quote(n.get_uri())
+            self.rourl = "uri/" + urllib.quote(n.get_readonly_uri())
             if not immutable:
                 return self.rootnode.set_node(name, future_node)
         d.addCallback(_stash_root_and_create_file)
@@ -386,7 +389,7 @@ def _check_info(res, expect_rw_uri, expect_ro_uri):
 
         d.addCallback(lambda ign: self.GET(expected_info_url))
         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=False)
-        d.addCallback(lambda ign: self.GET("%s%s?t=info" % (self.rooturl, str(name))))
+        d.addCallback(lambda ign: self.GET("%s/%s?t=info" % (self.rooturl, str(name))))
         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=True)
 
         def _check_json(res, expect_rw_uri):
@@ -410,7 +413,7 @@ def _check_json(res, expect_rw_uri):
             # TODO: check metadata contents
             self.failUnlessIn("metadata", data[1])
 
-        d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rooturl, str(name))))
+        d.addCallback(lambda ign: self.GET("%s/%s?t=json" % (self.rooturl, str(name))))
         d.addCallback(_check_json, expect_rw_uri=not immutable)
 
         # and make sure that a read-only version of the directory can be
@@ -425,7 +428,7 @@ def _check_json(res, expect_rw_uri):
         d.addCallback(lambda ign: self.GET(self.rourl+"?t=json"))
         d.addCallback(_check_directory_json, expect_rw_uri=False)
 
-        d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rourl, str(name))))
+        d.addCallback(lambda ign: self.GET("%s/%s?t=json" % (self.rourl, str(name))))
         d.addCallback(_check_json, expect_rw_uri=False)
 
         # TODO: check that getting t=info from the Info link in the ro directory
@@ -492,7 +495,7 @@ def _created(dn):
             self.failUnlessIn("CHK", cap.to_string())
             self.cap = cap
             self.rootnode = dn
-            self.rooturl = "uri/" + urllib.quote(dn.get_uri()) + "/"
+            self.rooturl = "uri/" + urllib.quote(dn.get_uri())
             return download_to_data(dn._node)
         d.addCallback(_created)
 
@@ -534,19 +537,28 @@ def _check_kids(children):
         # Make sure the lonely child can be listed in HTML...
         d.addCallback(lambda ign: self.GET(self.rooturl))
         def _check_html(res):
+            soup = BeautifulSoup(res, 'html5lib')
             self.failIfIn("URI:SSK", res)
-            get_lonely = "".join([r'<td>FILE</td>',
-                                  r'\s+<td>',
-                                  r'<a href="[^"]+%s[^"]+" rel="noreferrer">lonely</a>' % (urllib.quote(lonely_uri),),
-                                  r'</td>',
-                                  r'\s+<td align="right">%d</td>' % len("one"),
-                                  ])
-            self.failUnless(re.search(get_lonely, res), res)
-
-            # find the More Info link for name, should be relative
-            mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
-            info_url = mo.group(1)
-            self.failUnless(info_url.endswith(urllib.quote(lonely_uri) + "?t=info"), info_url)
+            found = False
+            for td in soup.find_all(u"td"):
+                if td.text != u"FILE":
+                    continue
+                a = td.findNextSibling()(u"a")[0]
+                self.assertIn(urllib.quote(lonely_uri), a[u"href"])
+                self.assertEqual(u"lonely", a.text)
+                self.assertEqual(a[u"rel"], [u"noreferrer"])
+                self.assertEqual(u"{}".format(len("one")), td.findNextSibling().findNextSibling().text)
+                found = True
+                break
+            self.assertTrue(found)
+
+            infos = list(
+                a[u"href"]
+                for a in soup.find_all(u"a")
+                if a.text == u"More Info"
+            )
+            self.assertEqual(1, len(infos))
+            self.assertTrue(infos[0].endswith(urllib.quote(lonely_uri) + "?t=info"))
         d.addCallback(_check_html)
 
         # ... and in JSON.
@@ -573,7 +585,7 @@ def test_deep_check(self):
         d = c0.create_dirnode()
         def _stash_root_and_create_file(n):
             self.rootnode = n
-            self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
+            self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri())
             return n.add_file(u"good", upload.Data(DATA, convergence=""))
         d.addCallback(_stash_root_and_create_file)
         def _stash_uri(fn, which):
@@ -747,7 +759,7 @@ def test_deep_check_and_repair(self):
         d = c0.create_dirnode()
         def _stash_root_and_create_file(n):
             self.rootnode = n
-            self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
+            self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri())
             return n.add_file(u"good", upload.Data(DATA, convergence=""))
         d.addCallback(_stash_root_and_create_file)
         def _stash_uri(fn, which):
@@ -960,7 +972,7 @@ def test_deep_add_lease(self):
         def _stash_root_and_create_file(n):
             self.rootnode = n
             self.uris["root"] = n.get_uri()
-            self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
+            self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri())
             return n.add_file(u"one", upload.Data(DATA, convergence=""))
         d.addCallback(_stash_root_and_create_file)
         def _stash_uri(fn, which):
@@ -1027,8 +1039,8 @@ def test_exceptions(self):
         DATA = "data" * 100
         d = c0.create_dirnode()
         def _stash_root(n):
-            self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
-            self.fileurls["imaginary"] = self.fileurls["root"] + "imaginary"
+            self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri())
+            self.fileurls["imaginary"] = self.fileurls["root"] + "/imaginary"
             return n
         d.addCallback(_stash_root)
         d.addCallback(lambda ign: c0.upload(upload.Data(DATA, convergence="")))
@@ -1044,14 +1056,14 @@ def _stash_bad(ur):
         d.addCallback(lambda ign: c0.create_dirnode())
         def _mangle_dirnode_1share(n):
             u = n.get_uri()
-            url = self.fileurls["dir-1share"] = "uri/" + urllib.quote(u) + "/"
+            url = self.fileurls["dir-1share"] = "uri/" + urllib.quote(u)
             self.fileurls["dir-1share-json"] = url + "?t=json"
             self.delete_shares_numbered(u, range(1,10))
         d.addCallback(_mangle_dirnode_1share)
         d.addCallback(lambda ign: c0.create_dirnode())
         def _mangle_dirnode_0share(n):
             u = n.get_uri()
-            url = self.fileurls["dir-0share"] = "uri/" + urllib.quote(u) + "/"
+            url = self.fileurls["dir-0share"] = "uri/" + urllib.quote(u)
             self.fileurls["dir-0share-json"] = url + "?t=json"
             self.delete_shares_numbered(u, range(0,10))
         d.addCallback(_mangle_dirnode_0share)
@@ -1330,8 +1342,8 @@ def _get_dircap(dn):
             self.dir_si_b32 = base32.b2a(dn.get_storage_index())
             self.dir_url_base = "uri/"+dn.get_write_uri()
             self.dir_url_json1 = "uri/"+dn.get_write_uri()+"?t=json"
-            self.dir_url_json2 = "uri/"+dn.get_write_uri()+"/?t=json"
-            self.dir_url_json_ro = "uri/"+dn.get_readonly_uri()+"/?t=json"
+            self.dir_url_json2 = "uri/"+dn.get_write_uri()+"?t=json"
+            self.dir_url_json_ro = "uri/"+dn.get_readonly_uri()+"?t=json"
             self.child_url = "uri/"+dn.get_readonly_uri()+"/child"
         d.addCallback(_get_dircap)
         d.addCallback(lambda ign: self.GET(self.dir_url_base, followRedirect=True))