From 6d785e15970d2b87d88993450460ae6e0a332c55 Mon Sep 17 00:00:00 2001 From: Erik van den Brink Date: Tue, 6 Nov 2018 15:27:55 +0100 Subject: [PATCH 1/2] bootstrap fix and changes --- neo/Prompt/Commands/Bootstrap.py | 26 +++--- neo/Prompt/Commands/tests/test_bootstrap.py | 10 +-- neo/Settings.py | 14 ++- neo/bin/bootstrap.py | 7 +- neo/data/protocol.mainnet.json | 4 +- neo/data/protocol.testnet.json | 98 ++++++++++----------- requirements.txt | 2 +- 7 files changed, 85 insertions(+), 76 deletions(-) diff --git a/neo/Prompt/Commands/Bootstrap.py b/neo/Prompt/Commands/Bootstrap.py index 5ed7efba7..d3b64664b 100644 --- a/neo/Prompt/Commands/Bootstrap.py +++ b/neo/Prompt/Commands/Bootstrap.py @@ -8,20 +8,20 @@ import os -def BootstrapBlockchainFile(target_dir, download_file, require_confirm=True): - - if download_file is None: - print("no bootstrap file specified. Please update your configuration file.") +def BootstrapBlockchainFile(target_dir, download_location, bootstrap_name, require_confirm=True): + if download_location is None: + print("no bootstrap location file specified. Please update your configuration file.") sys.exit(0) if require_confirm: print("This will overwrite any data currently in %s.\nType 'confirm' to continue" % target_dir) confirm = prompt("[confirm]> ", is_password=False) if confirm == 'confirm': - return do_bootstrap(download_file, target_dir) + return do_bootstrap(download_location, bootstrap_name, target_dir) else: - return do_bootstrap(download_file, + return do_bootstrap(download_location, + bootstrap_name, target_dir, tmp_file_name=os.path.join(settings.DATA_DIR_PATH, 'btest.tar.gz'), tmp_chain_name='btestchain') @@ -30,20 +30,22 @@ def BootstrapBlockchainFile(target_dir, download_file, require_confirm=True): sys.exit(0) -def do_bootstrap(bootstrap_file, destination_dir, tmp_file_name=None, tmp_chain_name='tmpchain'): - +def do_bootstrap(download_location, bootstrap_name, destination_dir, tmp_file_name=None, tmp_chain_name='tmpchain'): if tmp_file_name is None: tmp_file_name = os.path.join(settings.DATA_DIR_PATH, 'bootstrap.tar.gz') success = False - print('will download file %s ' % bootstrap_file) - print('') - try: - response = requests.get(bootstrap_file, stream=True) + source = requests.get(download_location) + source.raise_for_status() + source_json = source.json() + response = requests.get(source_json[bootstrap_name], stream=True) response.raise_for_status() + print('will download file %s ' % source_json[bootstrap_name]) + print('') + # Total size in bytes. total_size = int(response.headers.get('content-length', 0)) diff --git a/neo/Prompt/Commands/tests/test_bootstrap.py b/neo/Prompt/Commands/tests/test_bootstrap.py index 50f9fe944..18cb89da5 100644 --- a/neo/Prompt/Commands/tests/test_bootstrap.py +++ b/neo/Prompt/Commands/tests/test_bootstrap.py @@ -5,8 +5,7 @@ class BootstrapTestCase(TestCase): - bootstrap_file_good = 'https://s3.us-east-2.amazonaws.com/cityofzion/bootstrap_testnet/bootstraptest.tar.gz' - bootstrap_file_bad = 'https://s3.us-east-2.amazonaws.com/blah.tar.gz' + bootstrap_unittest_file_locations = 'https://s3.us-east-2.amazonaws.com/cityofzion/bootstrap_unittest_latest' bootstrap_target_dir = './fixtures/bootstrap_test' bootstrap_target_dir_bad = 'does_not_exist' @@ -21,25 +20,24 @@ def tearDown(self): pass def test_1_bad_bootstrap_file(self): - # this should exit 0 print("*** ***") print("*** This test expects a `404 Not found` ***") print("*** ***") with self.assertRaises(SystemExit): - BootstrapBlockchainFile(self.bootstrap_target_dir, self.bootstrap_file_bad, require_confirm=False) + BootstrapBlockchainFile(self.bootstrap_target_dir, self.bootstrap_unittest_file_locations, "badnet", require_confirm=False) # make sure no files are left around self.assertFalse(os.path.exists(self.bootstrap_target_dir)) def test_2_good_bootstrap_file(self): with self.assertRaises(SystemExit): - BootstrapBlockchainFile(self.bootstrap_target_dir, self.bootstrap_file_good, require_confirm=False) + BootstrapBlockchainFile(self.bootstrap_target_dir, self.bootstrap_unittest_file_locations, "goodnet", require_confirm=False) self.assertTrue(os.path.exists(self.bootstrap_target_dir)) def test_3_good_bootstrap_bad_path(self): with self.assertRaises(SystemExit): - BootstrapBlockchainFile(self.bootstrap_target_dir_bad, self.bootstrap_file_good, require_confirm=False) + BootstrapBlockchainFile(self.bootstrap_target_dir_bad, self.bootstrap_unittest_file_locations, "goodnet", require_confirm=False) self.assertFalse(os.path.exists(self.bootstrap_target_dir)) diff --git a/neo/Settings.py b/neo/Settings.py index 25b9f67f6..607b8ebe8 100644 --- a/neo/Settings.py +++ b/neo/Settings.py @@ -163,6 +163,15 @@ def net_name(self): # Setup methods def setup(self, config_file): """ Setup settings from a JSON config file """ + + def get_config_and_warn(key, default, abort=False): + value = config.get(key, default) + if value == default: + print(f"Cannot find {key} in settings, using default value: {default}") + if abort: + sys.exit(-1) + return value + if not self.DATA_DIR_PATH: # Setup default data dir self.set_data_dir(None) @@ -192,9 +201,8 @@ def setup(self, config_file): self.URI_PREFIX = config['UriPrefix'] self.ACCEPT_INCOMING_PEERS = config.get('AcceptIncomingPeers', False) - self.BOOTSTRAP_FILE = config['BootstrapFile'] - self.NOTIF_BOOTSTRAP_FILE = config['NotificationBootstrapFile'] - + self.BOOTSTRAP_NAME = get_config_and_warn('BootstrapName', "mainnet") + self.BOOTSTRAP_LOCATIONS = get_config_and_warn('BootstrapFiles', "abort", abort=True) Helper.ADDRESS_VERSION = self.ADDRESS_VERSION self.USE_DEBUG_STORAGE = config.get('DebugStorage', True) diff --git a/neo/bin/bootstrap.py b/neo/bin/bootstrap.py index 808ba3751..67b9b9fb4 100644 --- a/neo/bin/bootstrap.py +++ b/neo/bin/bootstrap.py @@ -42,10 +42,11 @@ def main(): elif args.mainnet: settings.setup_mainnet() + bootstrap_name = settings.BOOTSTRAP_NAME if args.notifications: - BootstrapBlockchainFile(settings.notification_leveldb_path, settings.NOTIF_BOOTSTRAP_FILE, require_confirm) - else: - BootstrapBlockchainFile(settings.chain_leveldb_path, settings.BOOTSTRAP_FILE, require_confirm) + bootstrap_name += "_notif" + + BootstrapBlockchainFile(settings.notification_leveldb_path, settings.BOOTSTRAP_LOCATIONS, bootstrap_name, require_confirm) if __name__ == "__main__": diff --git a/neo/data/protocol.mainnet.json b/neo/data/protocol.mainnet.json index 1a4195f11..3a47526e6 100644 --- a/neo/data/protocol.mainnet.json +++ b/neo/data/protocol.mainnet.json @@ -53,8 +53,8 @@ ], "SslCert": "", "SslCertPassword": "", - "BootstrapFile": "https://s3.us-east-2.amazonaws.com/cityofzion/bootstrap_main/Main2891xxx.tar.gz", - "NotificationBootstrapFile": "https://s3.us-east-2.amazonaws.com/cityofzion/bootstrap_main/MainNotif2891xxx.tar.gz", + "BootstrapName": "mainnet", + "BootstrapFiles": "https://s3.us-east-2.amazonaws.com/cityofzion/bootstrap_latest", "DebugStorage": 1, "AcceptIncomingPeers": false, "CompilerNep8": false diff --git a/neo/data/protocol.testnet.json b/neo/data/protocol.testnet.json index 3476968ab..f4db921f2 100644 --- a/neo/data/protocol.testnet.json +++ b/neo/data/protocol.testnet.json @@ -1,52 +1,52 @@ { - "ApplicationConfiguration": { - "DataDirectoryPath": "Chains/SC234", - "NotificationDataPath": "Chains/Test_Notif", - "RPCPort": 20332, - "NodePort": 20333, - "SslCert": "", - "SslCertPassword": "", - "UriPrefix": [ - "http://*:20332" - ], - "WsPort": 20334, - "BootstrapFile":"https://s3.us-east-2.amazonaws.com/cityofzion/bootstrap_main/Test168xxxx.tar.gz", - "NotificationBootstrapFile":"https://s3.us-east-2.amazonaws.com/cityofzion/bootstrap_main/TestNotif168xxxx.tar.gz", - "DebugStorage":1, - "AcceptIncomingPeers": false, - "CompilerNep8":false - }, - "ProtocolConfiguration": { - "AddressVersion": 23, - "Magic": 1953787457, - "SeedList": [ - "13.58.169.218:20333", - "13.58.33.157:20333", - "18.222.161.128:20333", - "18.191.171.240:20333", - "18.222.168.189:20333" - ], - "RPCList":[ - "http://18.221.221.195:8880", - "http://18.221.139.152:8880", - "http://52.15.48.60:8880", - "http://18.221.0.152:8880", - "http://52.14.184.44:8880" - ], - "StandbyValidators": [ - "0327da12b5c40200e9f65569476bbff2218da4f32548ff43b6387ec1416a231ee8", - "026ce35b29147ad09e4afe4ec4a7319095f08198fa8babbe3c56e970b143528d22", - "0209e7fd41dfb5c2f8dc72eb30358ac100ea8c72da18847befe06eade68cebfcb9", - "039dafd8571a641058ccc832c5e2111ea39b09c0bde36050914384f7a48bce9bf9", - "038dddc06ce687677a53d54f096d2591ba2302068cf123c1f2d75c2dddc5425579", - "02d02b1873a0863cd042cc717da31cea0d7cf9db32b74d4c72c01b0011503e2e22", - "034ff5ceeac41acf22cd5ed2da17a6df4dd8358fcb2bfb1a43208ad0feaab2746b" - ], - "SystemFee": { - "EnrollmentTransaction": 10, - "IssueTransaction": 5, - "PublishTransaction": 5, - "RegisterTransaction": 100 - } + "ApplicationConfiguration": { + "DataDirectoryPath": "Chains/SC234", + "NotificationDataPath": "Chains/Test_Notif", + "RPCPort": 20332, + "NodePort": 20333, + "SslCert": "", + "SslCertPassword": "", + "UriPrefix": [ + "http://*:20332" + ], + "WsPort": 20334, + "BootstrapName": "testnet", + "BootstrapFiles": "https://s3.us-east-2.amazonaws.com/cityofzion/bootstrap_latest", + "DebugStorage": 1, + "AcceptIncomingPeers": false, + "CompilerNep8": false + }, + "ProtocolConfiguration": { + "AddressVersion": 23, + "Magic": 1953787457, + "SeedList": [ + "13.58.169.218:20333", + "13.58.33.157:20333", + "18.222.161.128:20333", + "18.191.171.240:20333", + "18.222.168.189:20333" + ], + "RPCList": [ + "http://18.221.221.195:8880", + "http://18.221.139.152:8880", + "http://52.15.48.60:8880", + "http://18.221.0.152:8880", + "http://52.14.184.44:8880" + ], + "StandbyValidators": [ + "0327da12b5c40200e9f65569476bbff2218da4f32548ff43b6387ec1416a231ee8", + "026ce35b29147ad09e4afe4ec4a7319095f08198fa8babbe3c56e970b143528d22", + "0209e7fd41dfb5c2f8dc72eb30358ac100ea8c72da18847befe06eade68cebfcb9", + "039dafd8571a641058ccc832c5e2111ea39b09c0bde36050914384f7a48bce9bf9", + "038dddc06ce687677a53d54f096d2591ba2302068cf123c1f2d75c2dddc5425579", + "02d02b1873a0863cd042cc717da31cea0d7cf9db32b74d4c72c01b0011503e2e22", + "034ff5ceeac41acf22cd5ed2da17a6df4dd8358fcb2bfb1a43208ad0feaab2746b" + ], + "SystemFee": { + "EnrollmentTransaction": 10, + "IssueTransaction": 5, + "PublishTransaction": 5, + "RegisterTransaction": 100 } + } } diff --git a/requirements.txt b/requirements.txt index 8d38003bd..776149dac 100644 --- a/requirements.txt +++ b/requirements.txt @@ -56,7 +56,7 @@ pytz==2018.5 requests==2.20.0 scrypt==0.8.6 six==1.11.0 -tqdm==4.26.0 +tqdm==4.23.4 Twisted==18.7.0 urllib3==1.23 virtualenv==16.0.0 From f6f92d270923275d3777bb23274d2a4919a06eef Mon Sep 17 00:00:00 2001 From: Erik van den Brink Date: Tue, 6 Nov 2018 15:54:11 +0100 Subject: [PATCH 2/2] Update changelog and other configs --- CHANGELOG.rst | 2 ++ neo/data/protocol.coz.json | 25 ++++++++++++++----------- neo/data/protocol.unittest-net.json | 4 +++- 3 files changed, 19 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index db15b6fa9..8c3458a19 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,8 @@ All notable changes to this project are documented in this file. ----------------------- - Disallow ``Void`` type input parameters for smart contracts and increase test coverage - Fix confirmed tx not being purged from mempool `#703 `_ +- Fix bootstrap thread joining failure on Ubuntu systems +- Make bootstrap lookup dynamic such that users don't have to update their configs from here on forward [0.8.2] 2018-10-31 diff --git a/neo/data/protocol.coz.json b/neo/data/protocol.coz.json index 67cac008b..1c597ff3d 100644 --- a/neo/data/protocol.coz.json +++ b/neo/data/protocol.coz.json @@ -14,32 +14,35 @@ "188.68.34.29:10334", "188.68.34.29:10336" ], - "RPCList":[ + "RPCList": [ "http://188.68.34.29:10001", "http://188.68.34.29:10002", "http://188.68.34.29:10003", "http://188.68.34.29:10004" ], "SystemFee": { - "EnrollmentTransaction": 1000, - "IssueTransaction": 500, - "PublishTransaction": 500, - "RegisterTransaction": 10000 + "EnrollmentTransaction": 1000, + "IssueTransaction": 500, + "PublishTransaction": 500, + "RegisterTransaction": 10000 } }, - "ApplicationConfiguration": { "DataDirectoryPath": "Chains/coznet", "NotificationDataPath": "Chains/coz_notif", "RPCPort": 20332, "NodePort": 20333, "WsPort": 20334, - "UriPrefix": [ "http://*:20332"], + "UriPrefix": [ + "http://*:20332" + ], "SslCert": "", "SslCertPassword": "", - "BootstrapFile":"", - "NotificationBootstrapFile":"", - "DebugStorage":1, - "CompilerNep8":false + "BootstrapFile": "", + "NotificationBootstrapFile": "", + "DebugStorage": 1, + "CompilerNep8": false, + "BootstrapName": "fauxnet", + "BootstrapFiles": "this_does_not_exist_for_this_network" } } diff --git a/neo/data/protocol.unittest-net.json b/neo/data/protocol.unittest-net.json index 302a7fea1..b3d2a8beb 100644 --- a/neo/data/protocol.unittest-net.json +++ b/neo/data/protocol.unittest-net.json @@ -39,6 +39,8 @@ "NotificationBootstrapFile": "", "DebugStorage": 1, "AcceptIncomingPeers": false, - "CompilerNep8":true + "CompilerNep8": true, + "BootstrapName": "fauxnet", + "BootstrapFiles": "this_does_not_exist_for_this_network" } }