From 649ad0b1c2642d8b8f6c21f4985e20745b7ea621 Mon Sep 17 00:00:00 2001 From: kytv Date: Sat, 18 Feb 2012 00:08:15 +0000 Subject: [PATCH 01/14] multiple introducer support author: Faruq (original darcs patches available from https://tahoe-lafs.org/trac/tahoe-lafs/ticket/68) --- src/allmydata/web/root.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/allmydata/web/root.py b/src/allmydata/web/root.py index d8d789cf37..54a5c32db9 100644 --- a/src/allmydata/web/root.py +++ b/src/allmydata/web/root.py @@ -229,6 +229,29 @@ def data_connected_to_introducer(self, ctx, data): return "yes" return "no" + # In case we configure multiple introducers + def data_introducers(self, ctx, data): + connection_status = [] + connection_status = self.client.connected_to_introducer() + s = [] + furls = self.client.introducer_furls + for furl in furls: + if connection_status: + i = furls.index(furl) + if connection_status[i]: + s.append( (furl, "Yes") ) + else: + s.append( (furl, "No") ) + s.sort() + return s + + def render_introducers_row(self, ctx, s): + (furl, connected) = s + #connected = + ctx.fillSlots("introducer_furl", "%s" % (furl)) + ctx.fillSlots("connected", "%s" % (connected)) + return ctx.tag + def data_connected_to_introducer_alt(self, ctx, data): return self._connectedalts[self.data_connected_to_introducer(ctx, data)] From be387ae2d27cb8fe213aa4bb4834d2655c08dc51 Mon Sep 17 00:00:00 2001 From: Leif Ryge Date: Mon, 18 Jan 2016 19:03:24 +0000 Subject: [PATCH 02/14] apply fixes from daira's review This addresses all of the issues daira mentioned here: https://github.com/leif/tahoe-lafs/commit/1ae5aaecbb68f13019b6bc2ba4632bb4a5623aaa plus a few small things I noticed. --- src/allmydata/test/test_multi_introducers.py | 124 +++++++++++++++++++ src/allmydata/web/root.py | 11 +- 2 files changed, 133 insertions(+), 2 deletions(-) create mode 100644 src/allmydata/test/test_multi_introducers.py diff --git a/src/allmydata/test/test_multi_introducers.py b/src/allmydata/test/test_multi_introducers.py new file mode 100644 index 0000000000..d06574836d --- /dev/null +++ b/src/allmydata/test/test_multi_introducers.py @@ -0,0 +1,124 @@ +#!/usr/bin/python +import unittest, os +from mock import Mock, patch + +from allmydata.util.fileutil import write, remove +from allmydata.client import Client, MULTI_INTRODUCERS_CFG +from allmydata.scripts.create_node import write_node_config +from allmydata.web.root import Root + +INTRODUCERS_CFG_FURLS=['furl1', 'furl2'] + +def cfg_setup(): + # setup tahoe.cfg and basedir/introducers + # create a custom tahoe.cfg + c = open(os.path.join("tahoe.cfg"), "w") + config = {} + write_node_config(c, config) + fake_furl = "furl1" + c.write("[client]\n") + c.write("introducer.furl = %s\n" % fake_furl) + c.close() + + # create a basedir/introducers + write(MULTI_INTRODUCERS_CFG, '\n'.join(INTRODUCERS_CFG_FURLS)) + +def cfg_cleanup(): + # clean-up all cfg files + remove("tahoe.cfg") + remove(MULTI_INTRODUCERS_CFG) + +class TestRoot(unittest.TestCase): + + def setUp(self): + cfg_setup() + + def tearDown(self): + cfg_cleanup() + + @patch('allmydata.web.root.Root') + def test_introducer_furls(self, MockRoot): + """Ensure that a client's 'welcome page can fetch all introducer FURLs + loaded by the Client""" + + # mock setup + mockctx = Mock() + mockdata = Mock() + + # get the Client and furl count + myclient = Client() + furls = myclient.introducer_furls + furl_count = len(furls) + + # Pass mock value to Root + myroot = Root(myclient) + + # make the call + s = myroot.data_introducers(mockctx, mockdata) + + #assertions: compare return value with preset value + self.failUnlessEqual(furl_count, len(s)) + + + +class TestClient(unittest.TestCase): + def setUp(self): + cfg_setup() + + def tearDown(self): + cfg_cleanup() + + def test_introducer_count(self): + """ Ensure that the Client creates same number of introducer clients + as found in "basedir/introducers" config file. """ + write(MULTI_INTRODUCERS_CFG, '\n'.join(INTRODUCERS_CFG_FURLS)) + + # get a client and count of introducer_clients + myclient = Client() + ic_count = len(myclient.introducer_clients) + + # assertions + self.failUnlessEqual(ic_count, 2) + + def test_read_introducer_furl_from_tahoecfg(self): + """ Ensure that the Client reads the introducer.furl config item from + the tahoe.cfg file. """ + # create a custom tahoe.cfg + c = open(os.path.join("tahoe.cfg"), "w") + config = {} + write_node_config(c, config) + fake_furl = "furl1" + c.write("[client]\n") + c.write("introducer.furl = %s\n" % fake_furl) + c.close() + + # get a client and first introducer_furl + myclient = Client() + tahoe_cfg_furl = myclient.introducer_furls[0] + + # assertions + self.failUnlessEqual(fake_furl, tahoe_cfg_furl) + + def test_warning(self): + """ Ensure that the Client warns user if the the introducer.furl config item from the tahoe.cfg file is copied to "introducers" cfg file """ + # prepare tahoe.cfg + c = open(os.path.join("tahoe.cfg"), "w") + config = {} + write_node_config(c, config) + fake_furl = "furl0" + c.write("[client]\n") + c.write("introducer.furl = %s\n" % fake_furl) + c.close() + + # prepare "basedir/introducers" + write(MULTI_INTRODUCERS_CFG, '\n'.join(INTRODUCERS_CFG_FURLS)) + + # get a client + myclient = Client() + + # assertions: we expect a warning as tahoe_cfg furl is different + self.failUnlessEqual(True, myclient.warn_flag) + + +if __name__ == "__main__": + unittest.main() diff --git a/src/allmydata/web/root.py b/src/allmydata/web/root.py index 54a5c32db9..52f2a5f7d0 100644 --- a/src/allmydata/web/root.py +++ b/src/allmydata/web/root.py @@ -11,8 +11,8 @@ from allmydata import get_package_versions_string from allmydata.util import log from allmydata.interfaces import IFileNode -from allmydata.web import filenode, directory, unlinked, status, operations -from allmydata.web import storage +from allmydata.web import filenode, directory, unlinked, status, operations, storage, \ + introducerless_config from allmydata.web.common import abbreviate_size, getxmlfile, WebError, \ get_arg, RenderMixin, get_format, get_mutable_type, render_time_delta, render_time, render_time_attr @@ -435,3 +435,10 @@ def render_incident_button(self, ctx, data): T.input(type="submit", value=u"Save \u00BB"), ]] return T.div[form] + + def render_show_introducerless_config(self, ctx, data): + if self.client.get_config("node", "web.reveal_storage_furls", default=False, boolean=True): + return ctx.tag[T.a(href="introducerless_config")["Introducerless Config"]] + else: + return "" + From bc64a6b2f802bff9b593b19292250e8a92cec9fe Mon Sep 17 00:00:00 2001 From: Leif Ryge Date: Mon, 4 Jan 2016 19:58:55 +0000 Subject: [PATCH 03/14] wui: improved columns in welcome page server list As discussed at https://tahoe-lafs.org/trac/tahoe-lafs/ticket/1973 and in previous pull request #129. - replace lengthy timestamps with human-readable deltas (eg 1h 2m 3s) - replace "announced" column with "Last RX" column - remove service column (it always said the same thing, "storage") - fix colspan on 'You are not presently connected' message Previous versions, some with github comments: https://github.com/leif/tahoe-lafs/commit/3fe9053134b2429904f673df561e602a50f83c7e , https://github.com/leif/tahoe-lafs/commit/486dbfc7bd3c0bbba42a6df8e4564601120aec0e , and https://github.com/tahoe-lafs/tahoe-lafs/commit/c89ea625803be36a18bce1af4eef95dcd78bba2b, https://github.com/tahoe-lafs/tahoe-lafs/commit/9fabb924867e164e1c6d4d805761db6c39652cf7, https://github.com/tahoe-lafs/tahoe-lafs/commit/bbd8b42a25f8617c43b8293f3b654b3d060e27b9 Unlike previous attempts, the tests on this one should pass in any timezone. (But like current master, will fail with Nevow >=0.12...) Thanks to an anonymous contributor who wrote some of the tests. --- src/allmydata/test/test_util.py | 34 +++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/src/allmydata/test/test_util.py b/src/allmydata/test/test_util.py index b6b2bf28cc..2bf0647704 100644 --- a/src/allmydata/test/test_util.py +++ b/src/allmydata/test/test_util.py @@ -1180,6 +1180,40 @@ def test_format_delta(self): self.failUnlessEqual( time_format.format_delta(time_1decimal, time_1d21h46m49s_delta), '1d 21h 46m 48s') + def test_format_delta(self): + time_1 = 1389812723 + time_5s_delta = 1389812728 + time_28m7s_delta = 1389814410 + time_1h_delta = 1389816323 + time_1d21h46m49s_delta = 1389977532 + + self.failUnlessEqual( + time_format.format_delta(time_1, time_1), '0s') + + self.failUnlessEqual( + time_format.format_delta(time_1, time_5s_delta), '5s') + self.failUnlessEqual( + time_format.format_delta(time_1, time_28m7s_delta), '28m 7s') + self.failUnlessEqual( + time_format.format_delta(time_1, time_1h_delta), '1h 0m 0s') + self.failUnlessEqual( + time_format.format_delta(time_1, time_1d21h46m49s_delta), '1d 21h 46m 49s') + + self.failUnlessEqual( + time_format.format_delta(time_1d21h46m49s_delta, time_1), '-') + + # time_1 with a decimal fraction will make the delta 1s less + time_1decimal = 1389812723.383963 + + self.failUnlessEqual( + time_format.format_delta(time_1decimal, time_5s_delta), '4s') + self.failUnlessEqual( + time_format.format_delta(time_1decimal, time_28m7s_delta), '28m 6s') + self.failUnlessEqual( + time_format.format_delta(time_1decimal, time_1h_delta), '59m 59s') + self.failUnlessEqual( + time_format.format_delta(time_1decimal, time_1d21h46m49s_delta), '1d 21h 46m 48s') + class CacheDir(unittest.TestCase): def test_basic(self): basedir = "test_util/CacheDir/test_basic" From 93fc051183f7e361546a020eebde02b0a36bb09a Mon Sep 17 00:00:00 2001 From: Leif Ryge Date: Mon, 4 Jan 2016 16:00:59 +0000 Subject: [PATCH 04/14] wui: use standard time format (#1077) --- src/allmydata/web/root.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/allmydata/web/root.py b/src/allmydata/web/root.py index 52f2a5f7d0..a30f4f11ea 100644 --- a/src/allmydata/web/root.py +++ b/src/allmydata/web/root.py @@ -344,6 +344,8 @@ def render_service_row(self, ctx, server): ctx.fillSlots("service_connection_status_rel_time", service_connection_status_rel_time) ctx.fillSlots("last_received_data_abs_time", last_received_data_abs_time) ctx.fillSlots("last_received_data_rel_time", last_received_data_rel_time) + ctx.fillSlots("since", render_time(since)) + ctx.fillSlots("announced", render_time(announced)) ctx.fillSlots("version", version) ctx.fillSlots("available_space", available_space) From a6d4e2936461b8a88d1a9386de41f6b05a4d6c4b Mon Sep 17 00:00:00 2001 From: Leif Ryge Date: Sun, 23 Nov 2014 09:42:16 +0000 Subject: [PATCH 05/14] wui: improved columns in welcome page server list As discussed at https://tahoe-lafs.org/trac/tahoe-lafs/ticket/1973 - replace lengthy timestamps with human-readable deltas (eg 1h2m3s) - replace "Since" column with "Status" (addresses usability problem in #1961) - replace "announced" column with "Last RX" column - remove service column (it always said the same thing, "storage") This is a squashed commit of 3fe9053134b2429904f673df561e602a50f83c7e Thanks to an anonymous contributor who wrote some of the tests. --- src/allmydata/web/root.py | 7 +++++-- src/allmydata/web/welcome.xhtml | 6 +++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/allmydata/web/root.py b/src/allmydata/web/root.py index a30f4f11ea..de7e0bf213 100644 --- a/src/allmydata/web/root.py +++ b/src/allmydata/web/root.py @@ -15,6 +15,7 @@ introducerless_config from allmydata.web.common import abbreviate_size, getxmlfile, WebError, \ get_arg, RenderMixin, get_format, get_mutable_type, render_time_delta, render_time, render_time_attr +from allmydata.util.time_format import format_delta class URIHandler(RenderMixin, rend.Page): @@ -145,6 +146,8 @@ def __init__(self, client, clock=None, now_fn=None): # use to test ophandle expiration. self.child_operations = operations.OphandleTable(clock) self.now_fn = now_fn + if self.now_fn is None: + self.now_fn = time.time try: s = client.getServiceNamed("storage") except KeyError: @@ -314,13 +317,13 @@ def render_service_row(self, ctx, server): else: rhost_s = str(rhost) addr = rhost_s - service_connection_status = "yes" + service_connection_status = "Connected" last_connect_time = server.get_last_connect_time() service_connection_status_rel_time = render_time_delta(last_connect_time, self.now_fn()) service_connection_status_abs_time = render_time_attr(last_connect_time) else: addr = "N/A" - service_connection_status = "no" + service_connection_status = "Disconnected" last_loss_time = server.get_last_loss_time() service_connection_status_rel_time = render_time_delta(last_loss_time, self.now_fn()) service_connection_status_abs_time = render_time_attr(last_loss_time) diff --git a/src/allmydata/web/welcome.xhtml b/src/allmydata/web/welcome.xhtml index ca2387672f..b029e7ffea 100644 --- a/src/allmydata/web/welcome.xhtml +++ b/src/allmydata/web/welcome.xhtml @@ -168,6 +168,7 @@ + @@ -176,6 +177,9 @@ + - +

Status

Nickname

Address

Last RX

+
status-indicator service-
+

img/connected-.png
@@ -187,7 +191,7 @@
You are not presently connected to any peers
You are not presently connected to any peers
From b55aae7d9bc930bb775907d1458b6585827661fc Mon Sep 17 00:00:00 2001 From: Leif Ryge Date: Thu, 6 Feb 2014 21:11:56 +0000 Subject: [PATCH 06/14] add minimal /introducerless_config page --- src/allmydata/web/welcome.xhtml | 1 + 1 file changed, 1 insertion(+) diff --git a/src/allmydata/web/welcome.xhtml b/src/allmydata/web/welcome.xhtml index b029e7ffea..e7204728d6 100644 --- a/src/allmydata/web/welcome.xhtml +++ b/src/allmydata/web/welcome.xhtml @@ -118,6 +118,7 @@
  • Recent and Active Operations
  • Operational Statistics
  • +

  • From 0924917185bd43b268c6c24254ce49920f571347 Mon Sep 17 00:00:00 2001 From: David Stainton Date: Wed, 15 Jan 2014 22:03:44 +0100 Subject: [PATCH 07/14] Teach client to talk to storage nodes in tahoe.cfg --- src/allmydata/web/root.py | 23 +++++++++++++++++++++++ src/allmydata/web/welcome.xhtml | 6 ++++++ 2 files changed, 29 insertions(+) diff --git a/src/allmydata/web/root.py b/src/allmydata/web/root.py index de7e0bf213..ff6a437094 100644 --- a/src/allmydata/web/root.py +++ b/src/allmydata/web/root.py @@ -222,6 +222,21 @@ def data_introducer_furl_prefix(self, ctx, data): else: return "%s/[censored]" % (prefix,) + def data_introducer_furl_prefix(self, ctx, data): + + if not self.client.introducer_furl: + return + + ifurl = self.client.introducer_furl + # trim off the secret swissnum + (prefix, _, swissnum) = ifurl.rpartition("/") + if not ifurl: + return None + if swissnum == "introducer": + return ifurl + else: + return "%s/[censored]" % (prefix,) + def data_introducer_description(self, ctx, data): if self.data_connected_to_introducer(ctx, data) == "no": return "Introducer not connected" @@ -339,6 +354,14 @@ def render_service_row(self, ctx, server): available_space = "N/A" else: available_space = abbreviate_size(available_space) + + service_name = announcement["service-name"] + + seed = announcement['permutation-seed-base32'] + furl = announcement['anonymous-storage-FURL'] + + ctx.fillSlots("seed", seed) + ctx.fillSlots("furl", furl) ctx.fillSlots("address", addr) ctx.fillSlots("service_connection_status", service_connection_status) ctx.fillSlots("service_connection_status_alt", self._connectedalts[service_connection_status]) diff --git a/src/allmydata/web/welcome.xhtml b/src/allmydata/web/welcome.xhtml index e7204728d6..d426f3f0a9 100644 --- a/src/allmydata/web/welcome.xhtml +++ b/src/allmydata/web/welcome.xhtml @@ -171,6 +171,8 @@

    Status

    Nickname

    +

    Seed

    +

    FURL

    Address

    Last RX

    Version

    @@ -187,6 +189,10 @@
    + + + + From c2abc1f4e53420e9d1ecd974ad1819ef4eefd119 Mon Sep 17 00:00:00 2001 From: Leif Ryge Date: Mon, 13 Jan 2014 13:01:12 +0000 Subject: [PATCH 08/14] add "Since" column to introducers table --- src/allmydata/web/root.py | 21 ++++++++++++++++----- src/allmydata/web/welcome.xhtml | 23 ++++++++++++++++++++++- 2 files changed, 38 insertions(+), 6 deletions(-) diff --git a/src/allmydata/web/root.py b/src/allmydata/web/root.py index ff6a437094..3a5640b29f 100644 --- a/src/allmydata/web/root.py +++ b/src/allmydata/web/root.py @@ -256,10 +256,9 @@ def data_introducers(self, ctx, data): for furl in furls: if connection_status: i = furls.index(furl) - if connection_status[i]: - s.append( (furl, "Yes") ) - else: - s.append( (furl, "No") ) + ic = self.client.introducer_clients[i] + since = self.client.introducer_clients[i].get_since() + s.append((display_furl, bool(connection_statuses[i]), ic)) s.sort() return s @@ -267,7 +266,19 @@ def render_introducers_row(self, ctx, s): (furl, connected) = s #connected = ctx.fillSlots("introducer_furl", "%s" % (furl)) - ctx.fillSlots("connected", "%s" % (connected)) + ctx.fillSlots("service_connection_status", "%s" % (service_connection_status,)) + ctx.fillSlots("service_connection_status_alt", + self._connectedalts[service_connection_status]) + ctx.fillSlots("service_connection_status_abs_time", service_connection_status_abs_time) + ctx.fillSlots("service_connection_status_rel_time", service_connection_status_rel_time) + ctx.fillSlots("last_received_data_abs_time", last_received_data_abs_time) + ctx.fillSlots("last_received_data_rel_time", last_received_data_rel_time) + + status = ("No", "Yes") + ctx.fillSlots("connected-bool", "%s" % (connected)) + ctx.fillSlots("connected", "%s" % (status[int(connected)])) + ctx.fillSlots("since", "%s" % (time.strftime(TIME_FORMAT, + time.localtime(since)))) return ctx.tag def data_connected_to_introducer_alt(self, ctx, data): diff --git a/src/allmydata/web/welcome.xhtml b/src/allmydata/web/welcome.xhtml index d426f3f0a9..29e14957f9 100644 --- a/src/allmydata/web/welcome.xhtml +++ b/src/allmydata/web/welcome.xhtml @@ -198,7 +198,28 @@ - You are not presently connected to any peers + You are not presently connected to any peers + +
    +

    Connected to of introducers

    +
    + + + + + + + + + + + +

    Address

    Since

    +
    status-indicator connected-
    +
    +
    + +
    no introducers!
    From dcdc60e786808402c31801750b7c7c924f4f9b7a Mon Sep 17 00:00:00 2001 From: Leif Ryge Date: Sun, 12 Jan 2014 18:14:48 +0000 Subject: [PATCH 09/14] display number of connected introducers --- src/allmydata/web/welcome.xhtml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/allmydata/web/welcome.xhtml b/src/allmydata/web/welcome.xhtml index 29e14957f9..ea89800186 100644 --- a/src/allmydata/web/welcome.xhtml +++ b/src/allmydata/web/welcome.xhtml @@ -160,6 +160,20 @@ +
    +

    Connected to of introducers

    +
    +
    + + + + + +
    +
    status-indicator connected-
    +
    +
    no introducers!
    +

    Connected to From 30797c0971f0c58423fa8d8a35324a73935ff44a Mon Sep 17 00:00:00 2001 From: Leif Ryge Date: Sat, 11 Jan 2014 21:14:41 +0000 Subject: [PATCH 10/14] move introducers list down --- src/allmydata/web/welcome.xhtml | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/allmydata/web/welcome.xhtml b/src/allmydata/web/welcome.xhtml index ea89800186..a6d8e3da04 100644 --- a/src/allmydata/web/welcome.xhtml +++ b/src/allmydata/web/welcome.xhtml @@ -139,7 +139,7 @@

    -
    img/connected-.png
    +
    img/connected-.png

    @@ -161,14 +161,19 @@
    -

    Connected to of introducers

    +

    Connected Introducer(s)

    - +
    + + + + - + From 1ca75439befaa319ab61c4091717d85062c21890 Mon Sep 17 00:00:00 2001 From: Leif Ryge Date: Sat, 11 Jan 2014 20:51:03 +0000 Subject: [PATCH 11/14] remove single-introducer censored furl --- src/allmydata/web/welcome.xhtml | 1 - 1 file changed, 1 deletion(-) diff --git a/src/allmydata/web/welcome.xhtml b/src/allmydata/web/welcome.xhtml index a6d8e3da04..7c5bad3e2b 100644 --- a/src/allmydata/web/welcome.xhtml +++ b/src/allmydata/web/welcome.xhtml @@ -142,7 +142,6 @@
    img/connected-.png
    -

    From f96bbbd0d24251118d59f4e0a7f525666b1e5dd1 Mon Sep 17 00:00:00 2001 From: David Stainton Date: Tue, 17 May 2016 16:47:57 +0000 Subject: [PATCH 12/14] All of david415/2788.multi_intro.0 squashed into one commit This commit-comment should probably be edited, but this is so I can put all david's changes on top of rebased Leif et al. changes Conflicts: src/allmydata/test/test_multi_introducers.py src/allmydata/test/test_web.py src/allmydata/web/root.py --- src/allmydata/client.py | 127 ++++++++++++------ src/allmydata/introducer/client.py | 25 +++- src/allmydata/storage_client.py | 11 ++ src/allmydata/test/test_introducer.py | 47 +++---- src/allmydata/test/test_multi_introducers.py | 130 +++++++++---------- src/allmydata/test/test_storage_client.py | 1 + src/allmydata/test/test_upload.py | 2 + src/allmydata/test/test_web.py | 7 +- src/allmydata/web/root.py | 19 +-- src/allmydata/web/welcome.xhtml | 21 ++- 10 files changed, 228 insertions(+), 162 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index 1cb76c536c..5f66a1557d 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -1,4 +1,4 @@ -import os, stat, time, weakref +import os, stat, time, weakref, yaml from allmydata import node from base64 import urlsafe_b64encode @@ -9,6 +9,8 @@ from twisted.python.filepath import FilePath from pycryptopp.publickey import rsa +from foolscap.api import eventually + import allmydata from allmydata.storage.server import StorageServer from allmydata import storage_client @@ -122,10 +124,13 @@ def __init__(self, basedir="."): node.Node.__init__(self, basedir) # All tub.registerReference must happen *after* we upcall, since # that's what does tub.setLocation() + self.warn_flag = False + self.introducer_clients = [] + self.introducer_furls = [] self.started_timestamp = time.time() self.logSource="Client" self.encoding_params = self.DEFAULT_ENCODING_PARAMETERS.copy() - self.init_introducer_client() + self.load_connections() self.init_stats_provider() self.init_secrets() self.init_node_key() @@ -170,17 +175,63 @@ def _sequencer(self): nonce = _make_secret().strip() return seqnum, nonce - def init_introducer_client(self): - self.introducer_furl = self.get_config("client", "introducer.furl") - introducer_cache_filepath = FilePath(os.path.join(self.basedir, "private", "introducer_cache.yaml")) - ic = IntroducerClient(self.tub, self.introducer_furl, - self.nickname, - str(allmydata.__full_version__), - str(self.OLDEST_SUPPORTED_VERSION), - self.get_app_versions(), - self._sequencer, introducer_cache_filepath) - self.introducer_client = ic - ic.setServiceParent(self) + def old_introducer_config_compatiblity(self): + tahoe_cfg_introducer_furl = self.get_config("client", "introducer.furl", None) + if tahoe_cfg_introducer_furl is not None: + tahoe_cfg_introducer_furl = tahoe_cfg_introducer_furl.encode('utf-8') + + for nick in self.connections_config['introducers'].keys(): + if tahoe_cfg_introducer_furl == self.connections_config['introducers'][nick]['furl']: + log.err("Introducer furl specified in both tahoe.cfg and connections.yaml; please fix impossible configuration.") + self.warn_flag = True + return + + if u"introducer" in self.connections_config['introducers'].keys(): + if tahoe_cfg_introducer_furl is not None: + log.err("Introducer nickname in connections.yaml must not be called 'introducer' if the tahoe.cfg file also specifies and introducer.") + self.warn_flag = True + + if tahoe_cfg_introducer_furl is not None: + self.connections_config['introducers'][u"introducer"] = {} + self.connections_config['introducers'][u"introducer"]['furl'] = tahoe_cfg_introducer_furl + + def load_connections(self): + """ + Load the connections.yaml file if it exists, otherwise + create a default configuration. + """ + self.warn_flag = False + connections_filepath = FilePath(os.path.join(self.basedir, "private", "connections.yaml")) + def construct_unicode(loader, node): + return node.value + yaml.SafeLoader.add_constructor("tag:yaml.org,2002:str", + construct_unicode) + try: + with connections_filepath.open() as f: + self.connections_config = yaml.safe_load(f) + except EnvironmentError: + exists = False + self.connections_config = { 'servers' : {}, + 'introducers' : {}, + } + connections_filepath.setContent(yaml.safe_dump(self.connections_config)) + + self.old_introducer_config_compatiblity() + introducers = self.connections_config['introducers'] + for nickname in introducers: + introducer_cache_filepath = FilePath(os.path.join(self.basedir, "private", nickname)) + self.introducer_furls.append(introducers[nickname]['furl']) + ic = IntroducerClient(introducers[nickname]['furl'], + nickname, + str(allmydata.__full_version__), + str(self.OLDEST_SUPPORTED_VERSION), + self.get_app_versions(), + self._sequencer, introducer_cache_filepath) + self.introducer_clients.append(ic) + + # init introducer_clients as usual + for ic in self.introducer_clients: + ic.setServiceParent(self) def init_stats_provider(self): gatherer_furl = self.get_config("client", "stats_gatherer.furl", None) @@ -298,7 +349,9 @@ def init_storage(self): ann = {"anonymous-storage-FURL": furl, "permutation-seed-base32": self._init_permutation_seed(ss), } - self.introducer_client.publish("storage", ann, self._node_key) + + for ic in self.introducer_clients: + ic.publish("storage", ann, self._node_key) def init_client(self): helper_furl = self.get_config("client", "helper.furl", None) @@ -360,32 +413,13 @@ def init_client_storage_broker(self): helper = storage_client.ConnectedEnough(sb, connection_threshold) self.upload_ready_d = helper.when_connected_enough() - # load static server specifications from tahoe.cfg, if any. - # Not quite ready yet. - #if self.config.has_section("client-server-selection"): - # server_params = {} # maps serverid to dict of parameters - # for (name, value) in self.config.items("client-server-selection"): - # pieces = name.split(".") - # if pieces[0] == "server": - # serverid = pieces[1] - # if serverid not in server_params: - # server_params[serverid] = {} - # server_params[serverid][pieces[2]] = value - # for serverid, params in server_params.items(): - # server_type = params.pop("type") - # if server_type == "tahoe-foolscap": - # s = storage_client.NativeStorageClient(*params) - # else: - # msg = ("unrecognized server type '%s' in " - # "tahoe.cfg [client-server-selection]server.%s.type" - # % (server_type, serverid)) - # raise storage_client.UnknownServerTypeError(msg) - # sb.add_server(s.serverid, s) - - # check to see if we're supposed to use the introducer too - if self.get_config("client-server-selection", "use_introducer", - default=True, boolean=True): - sb.use_introducer(self.introducer_client) + # utilize the loaded static server specifications + servers = self.connections_config['servers'] + for server_key in servers.keys(): + eventually(self.storage_broker.got_static_announcement, server_key, servers[server_id]['announcement']) + + for ic in self.introducer_clients: + sb.use_introducer(ic) def get_storage_broker(self): return self.storage_broker @@ -508,9 +542,18 @@ def _check_exit_trigger(self, exit_trigger_file): def get_encoding_parameters(self): return self.encoding_params + # In case we configure multiple introducers + def introducer_connection_statuses(self): + status = [] + if self.introducer_clients: + for ic in self.introducer_clients: + s = ic.connected_to_introducer() + status.append(s) + return status + def connected_to_introducer(self): - if self.introducer_client: - return self.introducer_client.connected_to_introducer() + if len(self.introducer_clients) > 0: + return True return False def get_renewal_secret(self): # this will go away diff --git a/src/allmydata/introducer/client.py b/src/allmydata/introducer/client.py index be1c2a381d..7c67ec65cb 100644 --- a/src/allmydata/introducer/client.py +++ b/src/allmydata/introducer/client.py @@ -3,6 +3,8 @@ from zope.interface import implements from twisted.application import service from foolscap.api import Referenceable, eventually +from foolscap.api import Tub + from allmydata.interfaces import InsufficientVersionError from allmydata.introducer.interfaces import IIntroducerClient, \ RIIntroducerSubscriberClient_v2 @@ -17,13 +19,16 @@ class InvalidCacheError(Exception): V2 = "http://allmydata.org/tahoe/protocols/introducer/v2" -class IntroducerClient(service.Service, Referenceable): +class IntroducerClient(service.MultiService, Referenceable): implements(RIIntroducerSubscriberClient_v2, IIntroducerClient) - def __init__(self, tub, introducer_furl, + def __init__(self, introducer_furl, nickname, my_version, oldest_supported, app_versions, sequencer, cache_filepath): - self._tub = tub + service.MultiService.__init__(self) + + self._tub = Tub() + self._tub.setServiceParent(self) self.introducer_furl = introducer_furl assert type(nickname) is unicode @@ -46,6 +51,7 @@ def __init__(self, tub, introducer_furl, self._canary = Referenceable() self._publisher = None + self._since = None self._local_subscribers = [] # (servicename,cb,args,kwargs) tuples self._subscribed_service_names = set() @@ -77,7 +83,7 @@ def _debug_retired(self, res): return res def startService(self): - service.Service.startService(self) + service.MultiService.startService(self) self._introducer_error = None rc = self._tub.connectTo(self.introducer_furl, self._got_introducer) self._introducer_reconnector = rc @@ -144,6 +150,7 @@ def _got_versioned_introducer(self, publisher): if V2 not in publisher.version: raise InsufficientVersionError("V2", publisher.version) self._publisher = publisher + self._since = int(time.time()) publisher.notifyOnDisconnect(self._disconnected) self._maybe_publish() self._maybe_subscribe() @@ -151,6 +158,7 @@ def _got_versioned_introducer(self, publisher): def _disconnected(self): self.log("bummer, we've lost our connection to the introducer") self._publisher = None + self._since = int(time.time()) self._subscriptions.clear() def log(self, *args, **kwargs): @@ -325,3 +333,12 @@ def _deliver_announcements(self, key_s, ann): def connected_to_introducer(self): return bool(self._publisher) + + def get_since(self): + return self._since + + def get_last_received_data_time(self): + if self._publisher is None: + return None + else: + return self._publisher.getDataLastReceivedAt() diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index d4162fdec4..d3a8a488a5 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -139,6 +139,17 @@ def _got_connection(self): # this is called by NativeStorageClient when it is connected self._server_listeners.notify() + def got_static_announcement(self, key_s, ann): + print "got static announcement" + if key_s is not None: + precondition(isinstance(key_s, str), key_s) + precondition(key_s.startswith("v0-"), key_s) + assert ann["service-name"] == "storage" + s = NativeStorageServer(key_s, ann) # XXX tub_options=... + server_id = s.get_serverid() + self.servers[server_id] = s + s.start_connecting(self._trigger_connections) + def _got_announcement(self, key_s, ann): if key_s is not None: precondition(isinstance(key_s, str), key_s) diff --git a/src/allmydata/test/test_introducer.py b/src/allmydata/test/test_introducer.py index c221c32108..a1ef38ba65 100644 --- a/src/allmydata/test/test_introducer.py +++ b/src/allmydata/test/test_introducer.py @@ -88,7 +88,7 @@ def tearDown(self): class Introducer(ServiceMixin, unittest.TestCase, pollmixin.PollMixin): def test_create(self): - ic = IntroducerClient(None, "introducer.furl", u"my_nickname", + ic = IntroducerClient("introducer.furl", u"my_nickname", "my_version", "oldest_version", {}, fakeseq, FilePath(self.mktemp())) self.failUnless(isinstance(ic, IntroducerClient)) @@ -120,14 +120,12 @@ def make_ann_t(ic, furl, privkey, seqnum): class Client(unittest.TestCase): def test_duplicate_receive_v2(self): - ic1 = IntroducerClient(None, - "introducer.furl", u"my_nickname", + ic1 = IntroducerClient("introducer.furl", u"my_nickname", "ver23", "oldest_version", {}, fakeseq, FilePath(self.mktemp())) # we use a second client just to create a different-looking # announcement - ic2 = IntroducerClient(None, - "introducer.furl", u"my_nickname", + ic2 = IntroducerClient("introducer.furl", u"my_nickname", "ver24","oldest_version",{}, fakeseq, FilePath(self.mktemp())) announcements = [] @@ -230,8 +228,7 @@ def _then5(ign): class Server(unittest.TestCase): def test_duplicate(self): i = IntroducerService() - ic1 = IntroducerClient(None, - "introducer.furl", u"my_nickname", + ic1 = IntroducerClient("introducer.furl", u"my_nickname", "ver23", "oldest_version", {}, realseq, FilePath(self.mktemp())) furl1 = "pb://62ubehyunnyhzs7r6vdonnm2hpi52w6y@127.0.0.1:36106/gydnp" @@ -334,9 +331,10 @@ def test_queue_until_connected(self): ifurl = self.central_tub.registerReference(introducer, furlFile=iff) tub2 = Tub() tub2.setServiceParent(self.parent) - c = IntroducerClient(tub2, ifurl, + c = IntroducerClient(ifurl, u"nickname", "version", "oldest", {}, fakeseq, FilePath(self.mktemp())) + c._tub = tub2 furl1 = "pb://onug64tu@127.0.0.1:123/short" # base32("short") sk_s, vk_s = keyutil.make_keypair() sk, _ignored = keyutil.parse_privkey(sk_s) @@ -413,11 +411,12 @@ def do_system_test(self): tub.setLocation("localhost:%d" % portnum) log.msg("creating client %d: %s" % (i, tub.getShortTubID())) - c = IntroducerClient(tub, self.introducer_furl, + c = IntroducerClient(self.introducer_furl, NICKNAME % str(i), "version", "oldest", {"component": "component-v1"}, fakeseq, FilePath(self.mktemp())) + c._tub = tub received_announcements[c] = {} def got(key_s_or_tubid, ann, announcements): index = key_s_or_tubid or get_tubid_string_from_ann(ann) @@ -676,7 +675,7 @@ def test_client_v2(self): introducer = IntroducerService() tub = introducer_furl = None app_versions = {"whizzy": "fizzy"} - client_v2 = IntroducerClient(tub, introducer_furl, NICKNAME % u"v2", + client_v2 = IntroducerClient(introducer_furl, NICKNAME % u"v2", "my_version", "oldest", app_versions, fakeseq, FilePath(self.mktemp())) #furl1 = "pb://62ubehyunnyhzs7r6vdonnm2hpi52w6y@127.0.0.1:0/swissnum" @@ -698,7 +697,7 @@ def test_client_v2_signed(self): introducer = IntroducerService() tub = introducer_furl = None app_versions = {"whizzy": "fizzy"} - client_v2 = IntroducerClient(tub, introducer_furl, u"nick-v2", + client_v2 = IntroducerClient(introducer_furl, u"nick-v2", "my_version", "oldest", app_versions, fakeseq, FilePath(self.mktemp())) furl1 = "pb://62ubehyunnyhzs7r6vdonnm2hpi52w6y@127.0.0.1:0/swissnum" @@ -730,9 +729,11 @@ def construct_unicode(loader, node): def test_client_cache(self): basedir = "introducer/ClientSeqnums/test_client_cache_1" fileutil.make_dirs(basedir) + privatedir = os.path.join(basedir, "private") + fileutil.make_dirs(privatedir) cache_filepath = FilePath(os.path.join(basedir, "private", "introducer_cache.yaml")) - + cache_filepath.setContent("") # if storage is enabled, the Client will publish its storage server # during startup (although the announcement will wait in a queue # until the introducer connection is established). To avoid getting @@ -745,7 +746,8 @@ def test_client_cache(self): f.close() c = TahoeClient(basedir) - ic = c.introducer_client + ic = c.introducer_clients[0] + ic._cache_filepath = cache_filepath sk_s, vk_s = keyutil.make_keypair() sk, _ignored = keyutil.parse_privkey(sk_s) pub1 = keyutil.remove_prefix(vk_s, "pub-") @@ -792,7 +794,7 @@ def test_client_cache(self): for a in announcements])) # test loading - ic2 = IntroducerClient(None, "introducer.furl", u"my_nickname", + ic2 = IntroducerClient("introducer.furl", u"my_nickname", "my_version", "oldest_version", {}, fakeseq, ic._cache_filepath) announcements = {} @@ -836,7 +838,7 @@ def test_client(self): f.close() c = TahoeClient(basedir) - ic = c.introducer_client + ic = c.introducer_clients[0] outbound = ic._outbound_announcements published = ic._published_announcements def read_seqnum(): @@ -892,16 +894,15 @@ def test_failure(self): i.setServiceParent(self.parent) self.introducer_furl = self.central_tub.registerReference(i) - tub = Tub() - tub.setOption("expose-remote-exception-types", False) - tub.setServiceParent(self.parent) - portnum = iputil.allocate_tcp_port() - tub.listenOn("tcp:%d" % portnum) - tub.setLocation("localhost:%d" % portnum) - - c = IntroducerClient(tub, self.introducer_furl, + c = IntroducerClient(self.introducer_furl, u"nickname-client", "version", "oldest", {}, fakeseq, FilePath(self.mktemp())) + c._tub.setOption("expose-remote-exception-types", False) + c._tub.setServiceParent(self.parent) + portnum = iputil.allocate_tcp_port() + c._tub.listenOn("tcp:%d" % portnum) + c._tub.setLocation("localhost:%d" % portnum) + announcements = {} def got(key_s, ann): announcements[key_s] = ann diff --git a/src/allmydata/test/test_multi_introducers.py b/src/allmydata/test/test_multi_introducers.py index d06574836d..9d354e4631 100644 --- a/src/allmydata/test/test_multi_introducers.py +++ b/src/allmydata/test/test_multi_introducers.py @@ -1,90 +1,76 @@ #!/usr/bin/python -import unittest, os -from mock import Mock, patch +import os, yaml +from twisted.python.filepath import FilePath +from twisted.trial import unittest from allmydata.util.fileutil import write, remove -from allmydata.client import Client, MULTI_INTRODUCERS_CFG +from allmydata.client import Client from allmydata.scripts.create_node import write_node_config from allmydata.web.root import Root INTRODUCERS_CFG_FURLS=['furl1', 'furl2'] +INTRODUCERS_CFG_FURLS_COMMENTED="""introducers: + 'intro1': {furl: furl1} +# 'intro2': {furl: furl4} +servers: {} +transport_plugins: {} + """ -def cfg_setup(): - # setup tahoe.cfg and basedir/introducers - # create a custom tahoe.cfg - c = open(os.path.join("tahoe.cfg"), "w") - config = {} - write_node_config(c, config) - fake_furl = "furl1" - c.write("[client]\n") - c.write("introducer.furl = %s\n" % fake_furl) - c.close() +class MultiIntroTests(unittest.TestCase): - # create a basedir/introducers - write(MULTI_INTRODUCERS_CFG, '\n'.join(INTRODUCERS_CFG_FURLS)) - -def cfg_cleanup(): - # clean-up all cfg files - remove("tahoe.cfg") - remove(MULTI_INTRODUCERS_CFG) - -class TestRoot(unittest.TestCase): - - def setUp(self): - cfg_setup() - - def tearDown(self): - cfg_cleanup() - - @patch('allmydata.web.root.Root') - def test_introducer_furls(self, MockRoot): - """Ensure that a client's 'welcome page can fetch all introducer FURLs - loaded by the Client""" - - # mock setup - mockctx = Mock() - mockdata = Mock() - - # get the Client and furl count - myclient = Client() - furls = myclient.introducer_furls - furl_count = len(furls) - - # Pass mock value to Root - myroot = Root(myclient) - - # make the call - s = myroot.data_introducers(mockctx, mockdata) - - #assertions: compare return value with preset value - self.failUnlessEqual(furl_count, len(s)) - - - -class TestClient(unittest.TestCase): def setUp(self): - cfg_setup() - - def tearDown(self): - cfg_cleanup() + # setup tahoe.cfg and basedir/private/introducers + # create a custom tahoe.cfg + self.basedir = os.path.dirname(self.mktemp()) + c = open(os.path.join(self.basedir, "tahoe.cfg"), "w") + config = {} + write_node_config(c, config) + fake_furl = "furl1" + c.write("[client]\n") + c.write("introducer.furl = %s\n" % fake_furl) + c.close() + os.mkdir(os.path.join(self.basedir,"private")) def test_introducer_count(self): """ Ensure that the Client creates same number of introducer clients - as found in "basedir/introducers" config file. """ - write(MULTI_INTRODUCERS_CFG, '\n'.join(INTRODUCERS_CFG_FURLS)) + as found in "basedir/private/introducers" config file. """ + connections = {'introducers': + { + u'intro3':{ 'furl': 'furl3', + 'subscribe_only': False }, + u'intro2':{ 'furl': 'furl4', + 'subscribe_only': False } + }, + 'servers':{}, + 'transport_plugins':{} + } + connections_filepath = FilePath(os.path.join(self.basedir, "private", "connections.yaml")) + connections_filepath.setContent(yaml.safe_dump(connections)) + # get a client and count of introducer_clients + myclient = Client(self.basedir) + ic_count = len(myclient.introducer_clients) + # assertions + self.failUnlessEqual(ic_count, 3) + + def test_introducer_count_commented(self): + """ Ensure that the Client creates same number of introducer clients + as found in "basedir/private/introducers" config file when there is one + commented.""" + connections_filepath = FilePath(os.path.join(self.basedir, "private", "connections.yaml")) + connections_filepath.setContent(INTRODUCERS_CFG_FURLS_COMMENTED) # get a client and count of introducer_clients - myclient = Client() + myclient = Client(self.basedir) ic_count = len(myclient.introducer_clients) # assertions - self.failUnlessEqual(ic_count, 2) + self.failUnlessEqual(ic_count, 1) def test_read_introducer_furl_from_tahoecfg(self): """ Ensure that the Client reads the introducer.furl config item from the tahoe.cfg file. """ # create a custom tahoe.cfg - c = open(os.path.join("tahoe.cfg"), "w") + c = open(os.path.join(self.basedir, "tahoe.cfg"), "w") config = {} write_node_config(c, config) fake_furl = "furl1" @@ -93,28 +79,30 @@ def test_read_introducer_furl_from_tahoecfg(self): c.close() # get a client and first introducer_furl - myclient = Client() + myclient = Client(self.basedir) tahoe_cfg_furl = myclient.introducer_furls[0] # assertions self.failUnlessEqual(fake_furl, tahoe_cfg_furl) def test_warning(self): - """ Ensure that the Client warns user if the the introducer.furl config item from the tahoe.cfg file is copied to "introducers" cfg file """ + """ Ensure that the Client warns user if the the introducer.furl config + item from the tahoe.cfg file is copied to "introducers" cfg file. """ # prepare tahoe.cfg - c = open(os.path.join("tahoe.cfg"), "w") + c = open(os.path.join(self.basedir,"tahoe.cfg"), "w") config = {} write_node_config(c, config) - fake_furl = "furl0" + fake_furl = "furl1" c.write("[client]\n") c.write("introducer.furl = %s\n" % fake_furl) c.close() - # prepare "basedir/introducers" - write(MULTI_INTRODUCERS_CFG, '\n'.join(INTRODUCERS_CFG_FURLS)) + # prepare "basedir/private/connections.yml + connections_filepath = FilePath(os.path.join(self.basedir, "private", "connections.yaml")) + connections_filepath.setContent(INTRODUCERS_CFG_FURLS_COMMENTED) # get a client - myclient = Client() + myclient = Client(self.basedir) # assertions: we expect a warning as tahoe_cfg furl is different self.failUnlessEqual(True, myclient.warn_flag) diff --git a/src/allmydata/test/test_storage_client.py b/src/allmydata/test/test_storage_client.py index f55592c99f..5b938cd16c 100644 --- a/src/allmydata/test/test_storage_client.py +++ b/src/allmydata/test/test_storage_client.py @@ -41,6 +41,7 @@ class TestStorageFarmBroker(unittest.TestCase): @inlineCallbacks def test_threshold_reached(self): introducer = Mock() + tub = Mock() broker = StorageFarmBroker(True) done = ConnectedEnough(broker, 5).when_connected_enough() broker.use_introducer(introducer) diff --git a/src/allmydata/test/test_upload.py b/src/allmydata/test/test_upload.py index c8c27be92a..11fa8ff756 100644 --- a/src/allmydata/test/test_upload.py +++ b/src/allmydata/test/test_upload.py @@ -185,6 +185,7 @@ def remote_abort(self): pass class FakeClient: + introducer_clients = [] DEFAULT_ENCODING_PARAMETERS = {"k":25, "happy": 25, "n": 100, @@ -204,6 +205,7 @@ def __init__(self, mode="good", num_servers=50): "permutation-seed-base32": base32.b2a(serverid) } self.storage_broker.test_add_rref(serverid, rref, ann) self.last_servers = [s[1] for s in servers] + self.introducer_clients = [] def log(self, *args, **kwargs): pass diff --git a/src/allmydata/test/test_web.py b/src/allmydata/test/test_web.py index 48cc439170..f5b7d429d1 100644 --- a/src/allmydata/test/test_web.py +++ b/src/allmydata/test/test_web.py @@ -689,7 +689,7 @@ def _check_introducer_not_connected_unguessable(res): html = res.replace('\n', ' ') self.failUnlessIn('
    pb://someIntroducer/[censored]
    ', html) self.failIfIn('pb://someIntroducer/secret', html) - self.failUnless(re.search('', html), res) + self.failUnless(re.search('

    [ ]*
    No introducers connected
    ', html), res) d.addCallback(_check_introducer_not_connected_unguessable) # introducer connected, unguessable furl @@ -702,7 +702,7 @@ def _check_introducer_connected_unguessable(res): html = res.replace('\n', ' ') self.failUnlessIn('
    pb://someIntroducer/[censored]
    ', html) self.failIfIn('pb://someIntroducer/secret', html) - self.failUnless(re.search('', html), res) + self.failUnless(re.search('
    [ ]*
    1 introducer connected
    ', html), res) d.addCallback(_check_introducer_connected_unguessable) # introducer connected, guessable furl @@ -714,7 +714,7 @@ def _set_introducer_connected_guessable(ign): def _check_introducer_connected_guessable(res): html = res.replace('\n', ' ') self.failUnlessIn('
    pb://someIntroducer/introducer
    ', html) - self.failUnless(re.search('', html), res) + self.failUnless(re.search('[ ]*
    1 introducer connected
    ', html), res) d.addCallback(_check_introducer_connected_guessable) return d @@ -5918,7 +5918,6 @@ class ErrorBoom(rend.Page): def beforeRender(self, ctx): raise CompletelyUnhandledError("whoops") - # XXX FIXME when we introduce "mock" as a dependency, these can # probably just be Mock instances @implementer(IRequest) diff --git a/src/allmydata/web/root.py b/src/allmydata/web/root.py index 3a5640b29f..875932ec2b 100644 --- a/src/allmydata/web/root.py +++ b/src/allmydata/web/root.py @@ -222,21 +222,6 @@ def data_introducer_furl_prefix(self, ctx, data): else: return "%s/[censored]" % (prefix,) - def data_introducer_furl_prefix(self, ctx, data): - - if not self.client.introducer_furl: - return - - ifurl = self.client.introducer_furl - # trim off the secret swissnum - (prefix, _, swissnum) = ifurl.rpartition("/") - if not ifurl: - return None - if swissnum == "introducer": - return ifurl - else: - return "%s/[censored]" % (prefix,) - def data_introducer_description(self, ctx, data): if self.data_connected_to_introducer(ctx, data) == "no": return "Introducer not connected" @@ -257,7 +242,6 @@ def data_introducers(self, ctx, data): if connection_status: i = furls.index(furl) ic = self.client.introducer_clients[i] - since = self.client.introducer_clients[i].get_since() s.append((display_furl, bool(connection_statuses[i]), ic)) s.sort() return s @@ -375,7 +359,8 @@ def render_service_row(self, ctx, server): ctx.fillSlots("furl", furl) ctx.fillSlots("address", addr) ctx.fillSlots("service_connection_status", service_connection_status) - ctx.fillSlots("service_connection_status_alt", self._connectedalts[service_connection_status]) + ctx.fillSlots("service_connection_status_alt", + self._connectedalts[service_connection_status]) ctx.fillSlots("connected-bool", bool(rhost)) ctx.fillSlots("service_connection_status_abs_time", service_connection_status_abs_time) ctx.fillSlots("service_connection_status_rel_time", service_connection_status_rel_time) diff --git a/src/allmydata/web/welcome.xhtml b/src/allmydata/web/welcome.xhtml index 7c5bad3e2b..f518a79762 100644 --- a/src/allmydata/web/welcome.xhtml +++ b/src/allmydata/web/welcome.xhtml @@ -239,9 +239,28 @@
    Introducer FURLConnected?
    -
    status-indicator connected-
    -
    +
    + + service-connected connected-
    no introducers!
    no introducers!
    +
    +

    Connected to of introducers

    +
    + + + + + + + + + + + + +

    Address

    Last RX

    +
    img/connected-.png
    + +
    +
    No introducers are configured.
    -
    From ef658a535270ded420e8368209eb1c1e6b51fb0c Mon Sep 17 00:00:00 2001 From: David Stainton Date: Thu, 19 May 2016 20:32:10 +0000 Subject: [PATCH 13/14] Fix unit test bit rot --- src/allmydata/test/test_web.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/src/allmydata/test/test_web.py b/src/allmydata/test/test_web.py index f5b7d429d1..16d6f9c77b 100644 --- a/src/allmydata/test/test_web.py +++ b/src/allmydata/test/test_web.py @@ -236,8 +236,6 @@ def __init__(self, nodeid, nickname): self.lease_checker = FakeLeaseChecker() def get_stats(self): return {"storage_server.accepting_immutable_shares": False} - def on_status_changed(self, cb): - cb(self) class FakeClient(Client): def __init__(self): @@ -247,7 +245,7 @@ def __init__(self): self.all_contents = {} self.nodeid = "fake_nodeid" self.nickname = u"fake_nickname \u263A" - self.introducer_furl = "None" + self.introducer_furls = [] self.stats_provider = FakeStatsProvider() self._secret_holder = SecretHolder("lease secret", "convergence secret") self.helper = None @@ -263,6 +261,7 @@ def __init__(self): serverid="other_nodeid", nickname=u"disconnected_nickname \u263B", connected = False, last_connect_time = 15, last_loss_time = 25, last_rx_time = 35)) self.introducer_client = None + self.introducer_clients = None self.history = FakeHistory() self.uploader = FakeUploader() self.uploader.all_contents = self.all_contents @@ -279,6 +278,8 @@ def get_long_nodeid(self): return "v0-nodeid" def get_long_tubid(self): return "tubid" + def get_config(self, section, option, default=None, boolean=False): + return None def startService(self): return service.MultiService.startService(self) @@ -623,6 +624,7 @@ def should302(self, res, which): (which, res)) class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixin, unittest.TestCase): + def test_create(self): pass @@ -676,13 +678,17 @@ def __init__(self, connected): self.connected = connected def connected_to_introducer(self): return self.connected + def get_since(self): + return 0 + def get_last_received_data_time(self): + return 0 d = defer.succeed(None) # introducer not connected, unguessable furl def _set_introducer_not_connected_unguessable(ign): - self.s.introducer_furl = "pb://someIntroducer/secret" - self.s.introducer_client = MockIntroducerClient(False) + self.s.introducer_furls = [ "pb://someIntroducer/secret" ] + self.s.introducer_clients = [ MockIntroducerClient(False) ] return self.GET("/") d.addCallback(_set_introducer_not_connected_unguessable) def _check_introducer_not_connected_unguessable(res): @@ -694,8 +700,8 @@ def _check_introducer_not_connected_unguessable(res): # introducer connected, unguessable furl def _set_introducer_connected_unguessable(ign): - self.s.introducer_furl = "pb://someIntroducer/secret" - self.s.introducer_client = MockIntroducerClient(True) + self.s.introducer_furls = [ "pb://someIntroducer/secret" ] + self.s.introducer_clients = [ MockIntroducerClient(True) ] return self.GET("/") d.addCallback(_set_introducer_connected_unguessable) def _check_introducer_connected_unguessable(res): @@ -707,8 +713,8 @@ def _check_introducer_connected_unguessable(res): # introducer connected, guessable furl def _set_introducer_connected_guessable(ign): - self.s.introducer_furl = "pb://someIntroducer/introducer" - self.s.introducer_client = MockIntroducerClient(True) + self.s.introducer_furls = [ "pb://someIntroducer/introducer" ] + self.s.introducer_clients = [ MockIntroducerClient(True) ] return self.GET("/") d.addCallback(_set_introducer_connected_guessable) def _check_introducer_connected_guessable(res): From 8adff8b411cdbaf1440cea2efceae9832782ba8e Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 7 Jun 2016 16:35:40 -0600 Subject: [PATCH 14/14] incorrect merge resolutions? ...or was 2788.multi_intro.0 not correct? This patch represents differences between the head of a branch started from leif/introless-multiintro to get correct attribution for 2788.multi_intro.0 However, at the end there were some differences and so this diff takes out those differences so that the tip of this branch and 2788.multi_intro.0 are the same. If any of the changes in this diff look wrong, they probably are. --- src/allmydata/web/root.py | 93 ++++++++++++++------------------- src/allmydata/web/welcome.xhtml | 52 +----------------- 2 files changed, 40 insertions(+), 105 deletions(-) diff --git a/src/allmydata/web/root.py b/src/allmydata/web/root.py index 875932ec2b..11ae6642ea 100644 --- a/src/allmydata/web/root.py +++ b/src/allmydata/web/root.py @@ -11,11 +11,9 @@ from allmydata import get_package_versions_string from allmydata.util import log from allmydata.interfaces import IFileNode -from allmydata.web import filenode, directory, unlinked, status, operations, storage, \ - introducerless_config +from allmydata.web import filenode, directory, unlinked, status, operations, storage from allmydata.web.common import abbreviate_size, getxmlfile, WebError, \ get_arg, RenderMixin, get_format, get_mutable_type, render_time_delta, render_time, render_time_attr -from allmydata.util.time_format import format_delta class URIHandler(RenderMixin, rend.Page): @@ -146,8 +144,7 @@ def __init__(self, client, clock=None, now_fn=None): # use to test ophandle expiration. self.child_operations = operations.OphandleTable(clock) self.now_fn = now_fn - if self.now_fn is None: - self.now_fn = time.time + # what if now_fn is None? try: s = client.getServiceNamed("storage") except KeyError: @@ -211,35 +208,41 @@ def render_services(self, ctx, data): return ctx.tag[ul] - def data_introducer_furl_prefix(self, ctx, data): - ifurl = self.client.introducer_furl - # trim off the secret swissnum - (prefix, _, swissnum) = ifurl.rpartition("/") - if not ifurl: - return None - if swissnum == "introducer": - return ifurl - else: - return "%s/[censored]" % (prefix,) + def data_total_introducers(self, ctx, data): + return len(self.client.introducer_furls) + + def data_connected_introducers(self, ctx, data): + return self.client.introducer_connection_statuses().count(True) def data_introducer_description(self, ctx, data): - if self.data_connected_to_introducer(ctx, data) == "no": - return "Introducer not connected" - return "Introducer" + connected_count = self.data_connected_introducers( ctx, data ) + if connected_count == 0: + return "No introducers connected" + elif connected_count == 1: + return "1 introducer connected" + else: + return "%s introducers connected" % (connected_count,) - def data_connected_to_introducer(self, ctx, data): - if self.client.connected_to_introducer(): + def data_connected_to_at_least_one_introducer(self, ctx, data): + if True in self.client.introducer_connection_statuses(): return "yes" return "no" + def data_connected_to_at_least_one_introducer_alt(self, ctx, data): + return self._connectedalts[self.data_connected_to_at_least_one_introducer(ctx, data)] + # In case we configure multiple introducers def data_introducers(self, ctx, data): - connection_status = [] - connection_status = self.client.connected_to_introducer() + connection_statuses = self.client.introducer_connection_statuses() s = [] furls = self.client.introducer_furls for furl in furls: - if connection_status: + if connection_statuses: + display_furl = furl + # trim off the secret swissnum + (prefix, _, swissnum) = furl.rpartition("/") + if swissnum != "introducer": + display_furl = "%s/[censored]" % (prefix,) i = furls.index(furl) ic = self.client.introducer_clients[i] s.append((display_furl, bool(connection_statuses[i]), ic)) @@ -247,8 +250,17 @@ def data_introducers(self, ctx, data): return s def render_introducers_row(self, ctx, s): - (furl, connected) = s - #connected = + (furl, connected, ic) = s + service_connection_status = "yes" if connected else "no" + + since = ic.get_since() + service_connection_status_rel_time = render_time_delta(since, self.now_fn()) + service_connection_status_abs_time = render_time_attr(since) + + last_received_data_time = ic.get_last_received_data_time() + last_received_data_rel_time = render_time_delta(last_received_data_time, self.now_fn()) + last_received_data_abs_time = render_time_attr(last_received_data_time) + ctx.fillSlots("introducer_furl", "%s" % (furl)) ctx.fillSlots("service_connection_status", "%s" % (service_connection_status,)) ctx.fillSlots("service_connection_status_alt", @@ -257,17 +269,8 @@ def render_introducers_row(self, ctx, s): ctx.fillSlots("service_connection_status_rel_time", service_connection_status_rel_time) ctx.fillSlots("last_received_data_abs_time", last_received_data_abs_time) ctx.fillSlots("last_received_data_rel_time", last_received_data_rel_time) - - status = ("No", "Yes") - ctx.fillSlots("connected-bool", "%s" % (connected)) - ctx.fillSlots("connected", "%s" % (status[int(connected)])) - ctx.fillSlots("since", "%s" % (time.strftime(TIME_FORMAT, - time.localtime(since)))) return ctx.tag - def data_connected_to_introducer_alt(self, ctx, data): - return self._connectedalts[self.data_connected_to_introducer(ctx, data)] - def data_helper_furl_prefix(self, ctx, data): try: uploader = self.client.getServiceNamed("uploader") @@ -327,13 +330,13 @@ def render_service_row(self, ctx, server): else: rhost_s = str(rhost) addr = rhost_s - service_connection_status = "Connected" + service_connection_status = "yes" last_connect_time = server.get_last_connect_time() service_connection_status_rel_time = render_time_delta(last_connect_time, self.now_fn()) service_connection_status_abs_time = render_time_attr(last_connect_time) else: addr = "N/A" - service_connection_status = "Disconnected" + service_connection_status = "no" last_loss_time = server.get_last_loss_time() service_connection_status_rel_time = render_time_delta(last_loss_time, self.now_fn()) service_connection_status_abs_time = render_time_attr(last_loss_time) @@ -349,25 +352,14 @@ def render_service_row(self, ctx, server): available_space = "N/A" else: available_space = abbreviate_size(available_space) - - service_name = announcement["service-name"] - - seed = announcement['permutation-seed-base32'] - furl = announcement['anonymous-storage-FURL'] - - ctx.fillSlots("seed", seed) - ctx.fillSlots("furl", furl) ctx.fillSlots("address", addr) ctx.fillSlots("service_connection_status", service_connection_status) ctx.fillSlots("service_connection_status_alt", self._connectedalts[service_connection_status]) - ctx.fillSlots("connected-bool", bool(rhost)) ctx.fillSlots("service_connection_status_abs_time", service_connection_status_abs_time) ctx.fillSlots("service_connection_status_rel_time", service_connection_status_rel_time) ctx.fillSlots("last_received_data_abs_time", last_received_data_abs_time) ctx.fillSlots("last_received_data_rel_time", last_received_data_rel_time) - ctx.fillSlots("since", render_time(since)) - ctx.fillSlots("announced", render_time(announced)) ctx.fillSlots("version", version) ctx.fillSlots("available_space", available_space) @@ -459,10 +451,3 @@ def render_incident_button(self, ctx, data): T.input(type="submit", value=u"Save \u00BB"), ]] return T.div[form] - - def render_show_introducerless_config(self, ctx, data): - if self.client.get_config("node", "web.reveal_storage_furls", default=False, boolean=True): - return ctx.tag[T.a(href="introducerless_config")["Introducerless Config"]] - else: - return "" - diff --git a/src/allmydata/web/welcome.xhtml b/src/allmydata/web/welcome.xhtml index f518a79762..ef57512677 100644 --- a/src/allmydata/web/welcome.xhtml +++ b/src/allmydata/web/welcome.xhtml @@ -159,25 +159,6 @@

    -
    -

    Connected Introducer(s)

    -
    -
    - - - - - - - - - - -
    Introducer FURLConnected?
    - - service-connected connected- -
    no introducers!
    -

    Connected to @@ -187,10 +168,7 @@ - - - @@ -198,46 +176,18 @@ - - - - - - -

    Status

    Nickname

    Seed

    FURL

    Address

    Last RX

    Version

    -
    status-indicator service-
    -

    img/connected-.png
    You are not presently connected to any peers
    -
    -

    Connected to of introducers

    -
    - - - - - - - - - - - - +

    Address

    Since

    -
    status-indicator connected-
    -
    -
    - -
    no introducers!
    You are not presently connected to any peers

    Connected to of introducers