Merge branch 'pr278': write (but don't read) YAML cache

refs ticket:2759

src/allmydata/_auto_deps.py

@@ -87,6 +87,7 @@
     # * pyOpenSSL >= 0.14 is needed in order to avoid
     #   <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/2474>.
     "pyOpenSSL >= 0.14",
+    "PyYAML >= 3.11",
 ]
 
 # Includes some indirect dependencies, but does not include allmydata.
@@ -114,6 +115,7 @@
     ('six',              'six'),
     ('enum34',           'enum'),
     ('pycparser',        'pycparser'),
+    ('PyYAML',           'yaml'),
 ]
 
 # Dependencies for which we don't know how to get a version number at run-time.

src/allmydata/client.py

@@ -6,6 +6,7 @@
 from twisted.internet import reactor, defer
 from twisted.application import service
 from twisted.application.internet import TimerService
+from twisted.python.filepath import FilePath
 from pycryptopp.publickey import rsa
 
 import allmydata
@@ -171,12 +172,13 @@ def _sequencer(self):
 
     def init_introducer_client(self):
         self.introducer_furl = self.get_config("client", "introducer.furl")
+        introducer_cache_filepath = FilePath(os.path.join(self.basedir, "private", "introducer_cache.yaml"))
         ic = IntroducerClient(self.tub, self.introducer_furl,
                               self.nickname,
                               str(allmydata.__full_version__),
                               str(self.OLDEST_SUPPORTED_VERSION),
                               self.get_app_versions(),
-                              self._sequencer)
+                              self._sequencer, introducer_cache_filepath)
         self.introducer_client = ic
         ic.setServiceParent(self)
 

src/allmydata/introducer/client.py

@@ -1,5 +1,5 @@
 
-import time
+import time, yaml
 from zope.interface import implements
 from twisted.application import service
 from foolscap.api import Referenceable, eventually, RemoteInterface
@@ -47,7 +47,7 @@ class IntroducerClient(service.Service, Referenceable):
 
     def __init__(self, tub, introducer_furl,
                  nickname, my_version, oldest_supported,
-                 app_versions, sequencer):
+                 app_versions, sequencer, cache_filepath):
         self._tub = tub
         self.introducer_furl = introducer_furl
 
@@ -57,6 +57,7 @@ def __init__(self, tub, introducer_furl,
         self._oldest_supported = oldest_supported
         self._app_versions = app_versions
         self._sequencer = sequencer
+        self._cache_filepath = cache_filepath
 
         self._my_subscriber_info = { "version": 0,
                                      "nickname": self._nickname,
@@ -113,6 +114,18 @@ def connect_failed(failure):
         d = self._tub.getReference(self.introducer_furl)
         d.addErrback(connect_failed)
 
+    def _save_announcements(self):
+        announcements = []
+        for _, value in self._inbound_announcements.items():
+            ann, key_s, time_stamp = value
+            server_params = {
+                "ann" : ann,
+                "key_s" : key_s,
+                }
+            announcements.append(server_params)
+        announcement_cache_yaml = yaml.dump(announcements)
+        self._cache_filepath.setContent(announcement_cache_yaml)
+
     def _got_introducer(self, publisher):
         self.log("connected to introducer, getting versions")
         default = { "http://allmydata.org/tahoe/protocols/introducer/v1":
@@ -343,6 +356,7 @@ def _process_announcement(self, ann, key_s):
                      parent=lp2, level=log.NOISY)
 
         self._inbound_announcements[index] = (ann, key_s, time.time())
+        self._save_announcements()
         # note: we never forget an index, but we might update its value
 
         for (service_name2,cb,args,kwargs) in self._local_subscribers:

src/allmydata/test/test_introducer.py

@@ -1,11 +1,12 @@
 
-import os, re, itertools
+import os, re, itertools, yaml
 from base64 import b32decode
 import simplejson
 
 from twisted.trial import unittest
 from twisted.internet import defer, address
 from twisted.python import log
+from twisted.python.filepath import FilePath
 
 from foolscap.api import Tub, Referenceable, fireEventually, flushEventualQueue
 from twisted.application import service
@@ -91,7 +92,8 @@ class Introducer(ServiceMixin, unittest.TestCase, pollmixin.PollMixin):
 
     def test_create(self):
         ic = IntroducerClient(None, "introducer.furl", u"my_nickname",
-                              "my_version", "oldest_version", {}, fakeseq)
+                              "my_version", "oldest_version", {}, fakeseq,
+                              FilePath(self.mktemp()))
         self.failUnless(isinstance(ic, IntroducerClient))
 
     def test_listen(self):
@@ -123,7 +125,8 @@ def test_id_collision(self):
         i = IntroducerService()
         ic = IntroducerClient(None,
                               "introducer.furl", u"my_nickname",
-                              "my_version", "oldest_version", {}, fakeseq)
+                              "my_version", "oldest_version", {}, fakeseq,
+                              FilePath(self.mktemp()))
         sk_s, vk_s = keyutil.make_keypair()
         sk, _ignored = keyutil.parse_privkey(sk_s)
         keyid = keyutil.remove_prefix(vk_s, "pub-v0-")
@@ -172,7 +175,8 @@ class Client(unittest.TestCase):
     def test_duplicate_receive_v1(self):
         ic = IntroducerClient(None,
                               "introducer.furl", u"my_nickname",
-                              "my_version", "oldest_version", {}, fakeseq)
+                              "my_version", "oldest_version", {}, fakeseq,
+                              FilePath(self.mktemp()))
         announcements = []
         ic.subscribe_to("storage",
                         lambda key_s,ann: announcements.append(ann))
@@ -221,12 +225,14 @@ def _then3(ign):
     def test_duplicate_receive_v2(self):
         ic1 = IntroducerClient(None,
                                "introducer.furl", u"my_nickname",
-                               "ver23", "oldest_version", {}, fakeseq)
+                               "ver23", "oldest_version", {}, fakeseq,
+                               FilePath(self.mktemp()))
         # we use a second client just to create a different-looking
         # announcement
         ic2 = IntroducerClient(None,
                                "introducer.furl", u"my_nickname",
-                               "ver24","oldest_version",{}, fakeseq)
+                               "ver24","oldest_version",{}, fakeseq,
+                               FilePath(self.mktemp()))
         announcements = []
         def _received(key_s, ann):
             announcements.append( (key_s, ann) )
@@ -329,7 +335,8 @@ def test_id_collision(self):
         # not replace the other)
         ic = IntroducerClient(None,
                               "introducer.furl", u"my_nickname",
-                              "my_version", "oldest_version", {}, fakeseq)
+                              "my_version", "oldest_version", {}, fakeseq,
+                              FilePath(self.mktemp()))
         announcements = []
         ic.subscribe_to("storage",
                         lambda key_s,ann: announcements.append(ann))
@@ -367,7 +374,8 @@ def test_duplicate(self):
         i = IntroducerService()
         ic1 = IntroducerClient(None,
                                "introducer.furl", u"my_nickname",
-                               "ver23", "oldest_version", {}, realseq)
+                               "ver23", "oldest_version", {}, realseq,
+                               FilePath(self.mktemp()))
         furl1 = "pb://62ubehyunnyhzs7r6vdonnm2hpi52w6y@127.0.0.1:36106/gydnp"
 
         privkey_s, _ = keyutil.make_keypair()
@@ -469,7 +477,8 @@ def test_queue_until_connected(self):
         tub2 = Tub()
         tub2.setServiceParent(self.parent)
         c = IntroducerClient(tub2, ifurl,
-                             u"nickname", "version", "oldest", {}, fakeseq)
+                             u"nickname", "version", "oldest", {}, fakeseq,
+                             FilePath(self.mktemp()))
         furl1 = "pb://onug64tu@127.0.0.1:123/short" # base32("short")
         sk_s, vk_s = keyutil.make_keypair()
         sk, _ignored = keyutil.parse_privkey(sk_s)
@@ -557,7 +566,8 @@ def do_system_test(self, server_version):
                 c = IntroducerClient(tub, self.introducer_furl,
                                      NICKNAME % str(i),
                                      "version", "oldest",
-                                     {"component": "component-v1"}, fakeseq)
+                                     {"component": "component-v1"}, fakeseq,
+                                     FilePath(self.mktemp()))
             received_announcements[c] = {}
             def got(key_s_or_tubid, ann, announcements, i):
                 if i == 0:
@@ -877,7 +887,7 @@ def test_client_v2(self):
         app_versions = {"whizzy": "fizzy"}
         client_v2 = IntroducerClient(tub, introducer_furl, NICKNAME % u"v2",
                                      "my_version", "oldest", app_versions,
-                                     fakeseq)
+                                     fakeseq, FilePath(self.mktemp()))
         #furl1 = "pb://62ubehyunnyhzs7r6vdonnm2hpi52w6y@127.0.0.1:0/swissnum"
         #ann_s = make_ann_t(client_v2, furl1, None, 10)
         #introducer.remote_publish_v2(ann_s, Referenceable())
@@ -939,7 +949,7 @@ def test_client_v2_unsigned(self):
         app_versions = {"whizzy": "fizzy"}
         client_v2 = IntroducerClient(tub, introducer_furl, u"nick-v2",
                                      "my_version", "oldest", app_versions,
-                                     fakeseq)
+                                     fakeseq, FilePath(self.mktemp()))
         furl1 = "pb://62ubehyunnyhzs7r6vdonnm2hpi52w6y@127.0.0.1:0/swissnum"
         tubid = "62ubehyunnyhzs7r6vdonnm2hpi52w6y"
         ann_s0 = make_ann_t(client_v2, furl1, None, 10)
@@ -961,7 +971,7 @@ def test_client_v2_signed(self):
         app_versions = {"whizzy": "fizzy"}
         client_v2 = IntroducerClient(tub, introducer_furl, u"nick-v2",
                                      "my_version", "oldest", app_versions,
-                                     fakeseq)
+                                     fakeseq, FilePath(self.mktemp()))
         furl1 = "pb://62ubehyunnyhzs7r6vdonnm2hpi52w6y@127.0.0.1:0/swissnum"
         sk_s, vk_s = keyutil.make_keypair()
         sk, _ignored = keyutil.parse_privkey(sk_s)
@@ -998,6 +1008,45 @@ def test_client_v1(self):
         self.failUnlessEqual(a[0].version, "my_version")
         self.failUnlessEqual(a[0].announcement["anonymous-storage-FURL"], furl1)
 
+    def test_client_cache_1(self):
+        basedir = "introducer/ClientSeqnums/test_client_cache_1"
+        fileutil.make_dirs(basedir)
+        cache_filepath = FilePath(os.path.join(basedir, "private",
+                                               "introducer_cache.yaml"))
+
+        # if storage is enabled, the Client will publish its storage server
+        # during startup (although the announcement will wait in a queue
+        # until the introducer connection is established). To avoid getting
+        # confused by this, disable storage.
+        f = open(os.path.join(basedir, "tahoe.cfg"), "w")
+        f.write("[client]\n")
+        f.write("introducer.furl = nope\n")
+        f.write("[storage]\n")
+        f.write("enabled = false\n")
+        f.close()
+
+        c = TahoeClient(basedir)
+        ic = c.introducer_client
+        sk_s, vk_s = keyutil.make_keypair()
+        sk, _ignored = keyutil.parse_privkey(sk_s)
+        keyutil.remove_prefix(vk_s, "pub-v0-")
+        furl1 = "pb://onug64tu@127.0.0.1:123/short" # base32("short")
+        ann_t = make_ann_t(ic, furl1, sk, 1)
+
+        ic.got_announcements([ann_t])
+
+        # check the cache for the announcement
+        with cache_filepath.open() as f:
+            def constructor(loader, node):
+                return node.value
+            yaml.SafeLoader.add_constructor("tag:yaml.org,2002:python/unicode", constructor)
+            announcements = yaml.safe_load(f)
+            f.close()
+
+        self.failUnlessEqual(len(announcements), 1)
+        self.failUnlessEqual("pub-" + announcements[0]['key_s'], vk_s)
+
+
 class ClientSeqnums(unittest.TestCase):
     def test_client(self):
         basedir = "introducer/ClientSeqnums/test_client"
@@ -1079,7 +1128,7 @@ def test_failure(self):
 
         c = IntroducerClient(tub, self.introducer_furl,
                              u"nickname-client", "version", "oldest", {},
-                             fakeseq)
+                             fakeseq, FilePath(self.mktemp()))
         announcements = {}
         def got(key_s, ann):
             announcements[key_s] = ann