From 14ebeba07d527a189a5cdf93c0f85392302b2ca1 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 1 Aug 2023 15:28:49 -0400 Subject: [PATCH 1/5] avoid re-computing the current time inside this loop It could lead to funny behavior if we cross a boundary at just the wrong time. Also the debug print could be misleading in such a case. --- integration/util.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/integration/util.py b/integration/util.py index 31d351bc12..7564891209 100644 --- a/integration/util.py +++ b/integration/util.py @@ -622,8 +622,9 @@ def await_client_ready(tahoe, timeout=10, liveness=60*2, minimum_number_of_serve time.sleep(1) continue + now = time.time() print( - f"Now: {time.ctime()}\n" + f"Now: {time.ctime(now)}\n" f"Server last-received-data: {[time.ctime(s['last_received_data']) for s in servers]}" ) @@ -633,7 +634,7 @@ def await_client_ready(tahoe, timeout=10, liveness=60*2, minimum_number_of_serve ] # check that all times are 'recent enough' (it's OK if _some_ servers # are down, we just want to make sure a sufficient number are up) - if len([time.time() - t <= liveness for t in server_times if t is not None]) < minimum_number_of_servers: + if len([now - t <= liveness for t in server_times if t is not None]) < minimum_number_of_servers: print("waiting because at least one server too old") time.sleep(1) continue From a0b78a134e05e33d90e802011b8dfd428d8d358d Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 1 Aug 2023 15:49:30 -0400 Subject: [PATCH 2/5] Leave a hint about what successful "bootstrap" looks like --- integration/conftest.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/integration/conftest.py b/integration/conftest.py index c94c054293..9a7a47ec4c 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -575,6 +575,32 @@ def cleanup(): request.addfinalizer(cleanup) pytest_twisted.blockon(chutney(("start", basic_network))) + + # Wait for the nodes to "bootstrap" - ie, form a network among themselves. + # Successful bootstrap is reported with a message something like: + # + # Everything bootstrapped after 151 sec + # Bootstrap finished: 151 seconds + # Node status: + # test000a : 100, done , Done + # test001a : 100, done , Done + # test002a : 100, done , Done + # test003r : 100, done , Done + # test004r : 100, done , Done + # test005r : 100, done , Done + # test006r : 100, done , Done + # test007r : 100, done , Done + # test008c : 100, done , Done + # test009c : 100, done , Done + # Published dir info: + # test000a : 100, all nodes , desc md md_cons ns_cons , Dir info cached + # test001a : 100, all nodes , desc md md_cons ns_cons , Dir info cached + # test002a : 100, all nodes , desc md md_cons ns_cons , Dir info cached + # test003r : 100, all nodes , desc md md_cons ns_cons , Dir info cached + # test004r : 100, all nodes , desc md md_cons ns_cons , Dir info cached + # test005r : 100, all nodes , desc md md_cons ns_cons , Dir info cached + # test006r : 100, all nodes , desc md md_cons ns_cons , Dir info cached + # test007r : 100, all nodes , desc md md_cons ns_cons , Dir info cached pytest_twisted.blockon(chutney(("wait_for_bootstrap", basic_network))) # print some useful stuff From 871df0b1b4e5d5264c9cd244b02eac09a690510f Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 1 Aug 2023 15:49:44 -0400 Subject: [PATCH 3/5] Dump some more details about what we're waiting for --- integration/util.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/integration/util.py b/integration/util.py index 7564891209..768741bd8d 100644 --- a/integration/util.py +++ b/integration/util.py @@ -623,18 +623,22 @@ def await_client_ready(tahoe, timeout=10, liveness=60*2, minimum_number_of_serve continue now = time.time() + server_times = [ + server['last_received_data'] + for server + in servers + if server['last_received_data'] is not None + ] print( f"Now: {time.ctime(now)}\n" - f"Server last-received-data: {[time.ctime(s['last_received_data']) for s in servers]}" + f"Liveness required: {liveness}\n" + f"Server last-received-data: {[time.ctime(s) for s in server_times]}\n" + f"Server ages: {[now - s for s in server_times]}\n" ) - server_times = [ - server['last_received_data'] - for server in servers - ] # check that all times are 'recent enough' (it's OK if _some_ servers # are down, we just want to make sure a sufficient number are up) - if len([now - t <= liveness for t in server_times if t is not None]) < minimum_number_of_servers: + if len([now - t <= liveness for t in server_times]) < minimum_number_of_servers: print("waiting because at least one server too old") time.sleep(1) continue From 9d670e54e29cb2c249100576949539cddc53b7f8 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 1 Aug 2023 15:56:02 -0400 Subject: [PATCH 4/5] Get the liveness filter condition right --- integration/util.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/integration/util.py b/integration/util.py index 768741bd8d..7e10b43151 100644 --- a/integration/util.py +++ b/integration/util.py @@ -638,8 +638,12 @@ def await_client_ready(tahoe, timeout=10, liveness=60*2, minimum_number_of_serve # check that all times are 'recent enough' (it's OK if _some_ servers # are down, we just want to make sure a sufficient number are up) - if len([now - t <= liveness for t in server_times]) < minimum_number_of_servers: - print("waiting because at least one server too old") + alive = [t for t in server_times if now - t <= liveness] + if len(alive) < minimum_number_of_servers: + print( + f"waiting because we found {len(alive)} servers " + f"and want {minimum_number_of_servers}" + ) time.sleep(1) continue From b8ee7a4e98e0968d6bc2d8a60f8638a83b1e04ce Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 1 Aug 2023 15:56:20 -0400 Subject: [PATCH 5/5] news fragment --- newsfragments/4055.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/4055.minor diff --git a/newsfragments/4055.minor b/newsfragments/4055.minor new file mode 100644 index 0000000000..e69de29bb2