Merge pull request #705 from ixje/bootstrap_updates

Bootstrap fix + updates
CityOfZion · Nov 6, 2018 · 84c6bb2 · 84c6bb2
2 parents cd77b16 + f6f92d2
commit 84c6bb2
Show file tree

Hide file tree

Showing 10 changed files with 104 additions and 88 deletions.
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
@@ -7,6 +7,8 @@ All notable changes to this project are documented in this file.
 -----------------------
 - Disallow ``Void`` type input parameters for smart contracts and increase test coverage
 - Fix confirmed tx not being purged from mempool `#703 <https://github.com/CityOfZion/neo-python/issues/703>`_
+- Fix bootstrap thread joining failure on Ubuntu systems
+- Make bootstrap lookup dynamic such that users don't have to update their configs from here on forward
 
 
 [0.8.2] 2018-10-31

diff --git a/neo/Prompt/Commands/Bootstrap.py b/neo/Prompt/Commands/Bootstrap.py
@@ -8,20 +8,20 @@
 import os
 
 
-def BootstrapBlockchainFile(target_dir, download_file, require_confirm=True):
-
-    if download_file is None:
-        print("no bootstrap file specified.  Please update your configuration file.")
+def BootstrapBlockchainFile(target_dir, download_location, bootstrap_name, require_confirm=True):
+    if download_location is None:
+        print("no bootstrap location file specified. Please update your configuration file.")
         sys.exit(0)
 
     if require_confirm:
         print("This will overwrite any data currently in %s.\nType 'confirm' to continue" % target_dir)
         confirm = prompt("[confirm]> ", is_password=False)
         if confirm == 'confirm':
-            return do_bootstrap(download_file, target_dir)
+            return do_bootstrap(download_location, bootstrap_name, target_dir)
     else:
 
-        return do_bootstrap(download_file,
+        return do_bootstrap(download_location,
+                            bootstrap_name,
                             target_dir,
                             tmp_file_name=os.path.join(settings.DATA_DIR_PATH, 'btest.tar.gz'),
                             tmp_chain_name='btestchain')
@@ -30,20 +30,22 @@ def BootstrapBlockchainFile(target_dir, download_file, require_confirm=True):
     sys.exit(0)
 
 
-def do_bootstrap(bootstrap_file, destination_dir, tmp_file_name=None, tmp_chain_name='tmpchain'):
-
+def do_bootstrap(download_location, bootstrap_name, destination_dir, tmp_file_name=None, tmp_chain_name='tmpchain'):
     if tmp_file_name is None:
         tmp_file_name = os.path.join(settings.DATA_DIR_PATH, 'bootstrap.tar.gz')
 
     success = False
 
-    print('will download file %s ' % bootstrap_file)
-    print('')
-
     try:
-        response = requests.get(bootstrap_file, stream=True)
+        source = requests.get(download_location)
+        source.raise_for_status()
+        source_json = source.json()
+        response = requests.get(source_json[bootstrap_name], stream=True)
         response.raise_for_status()
 
+        print('will download file %s ' % source_json[bootstrap_name])
+        print('')
+
         # Total size in bytes.
         total_size = int(response.headers.get('content-length', 0))
 

diff --git a/neo/Prompt/Commands/tests/test_bootstrap.py b/neo/Prompt/Commands/tests/test_bootstrap.py
@@ -5,8 +5,7 @@
 
 
 class BootstrapTestCase(TestCase):
-    bootstrap_file_good = 'https://s3.us-east-2.amazonaws.com/cityofzion/bootstrap_testnet/bootstraptest.tar.gz'
-    bootstrap_file_bad = 'https://s3.us-east-2.amazonaws.com/blah.tar.gz'
+    bootstrap_unittest_file_locations = 'https://s3.us-east-2.amazonaws.com/cityofzion/bootstrap_unittest_latest'
 
     bootstrap_target_dir = './fixtures/bootstrap_test'
     bootstrap_target_dir_bad = 'does_not_exist'
@@ -21,25 +20,24 @@ def tearDown(self):
             pass
 
     def test_1_bad_bootstrap_file(self):
-
         # this should exit 0
         print("***                                     ***")
         print("*** This test expects a `404 Not found` ***")
         print("***                                     ***")
         with self.assertRaises(SystemExit):
-            BootstrapBlockchainFile(self.bootstrap_target_dir, self.bootstrap_file_bad, require_confirm=False)
+            BootstrapBlockchainFile(self.bootstrap_target_dir, self.bootstrap_unittest_file_locations, "badnet", require_confirm=False)
 
         # make sure no files are left around
         self.assertFalse(os.path.exists(self.bootstrap_target_dir))
 
     def test_2_good_bootstrap_file(self):
         with self.assertRaises(SystemExit):
-            BootstrapBlockchainFile(self.bootstrap_target_dir, self.bootstrap_file_good, require_confirm=False)
+            BootstrapBlockchainFile(self.bootstrap_target_dir, self.bootstrap_unittest_file_locations, "goodnet", require_confirm=False)
 
         self.assertTrue(os.path.exists(self.bootstrap_target_dir))
 
     def test_3_good_bootstrap_bad_path(self):
         with self.assertRaises(SystemExit):
-            BootstrapBlockchainFile(self.bootstrap_target_dir_bad, self.bootstrap_file_good, require_confirm=False)
+            BootstrapBlockchainFile(self.bootstrap_target_dir_bad, self.bootstrap_unittest_file_locations, "goodnet", require_confirm=False)
 
         self.assertFalse(os.path.exists(self.bootstrap_target_dir))
diff --git a/neo/Settings.py b/neo/Settings.py
@@ -163,6 +163,15 @@ def net_name(self):
     # Setup methods
     def setup(self, config_file):
         """ Setup settings from a JSON config file """
+
+        def get_config_and_warn(key, default, abort=False):
+            value = config.get(key, default)
+            if value == default:
+                print(f"Cannot find {key} in settings, using default value: {default}")
+                if abort:
+                    sys.exit(-1)
+            return value
+
         if not self.DATA_DIR_PATH:
             # Setup default data dir
             self.set_data_dir(None)
@@ -192,9 +201,8 @@ def setup(self, config_file):
         self.URI_PREFIX = config['UriPrefix']
         self.ACCEPT_INCOMING_PEERS = config.get('AcceptIncomingPeers', False)
 
-        self.BOOTSTRAP_FILE = config['BootstrapFile']
-        self.NOTIF_BOOTSTRAP_FILE = config['NotificationBootstrapFile']
-
+        self.BOOTSTRAP_NAME = get_config_and_warn('BootstrapName', "mainnet")
+        self.BOOTSTRAP_LOCATIONS = get_config_and_warn('BootstrapFiles', "abort", abort=True)
         Helper.ADDRESS_VERSION = self.ADDRESS_VERSION
 
         self.USE_DEBUG_STORAGE = config.get('DebugStorage', True)

diff --git a/neo/bin/bootstrap.py b/neo/bin/bootstrap.py
@@ -42,10 +42,11 @@ def main():
     elif args.mainnet:
         settings.setup_mainnet()
 
+    bootstrap_name = settings.BOOTSTRAP_NAME
     if args.notifications:
-        BootstrapBlockchainFile(settings.notification_leveldb_path, settings.NOTIF_BOOTSTRAP_FILE, require_confirm)
-    else:
-        BootstrapBlockchainFile(settings.chain_leveldb_path, settings.BOOTSTRAP_FILE, require_confirm)
+        bootstrap_name += "_notif"
+
+    BootstrapBlockchainFile(settings.notification_leveldb_path, settings.BOOTSTRAP_LOCATIONS, bootstrap_name, require_confirm)
 
 
 if __name__ == "__main__":

diff --git a/neo/data/protocol.coz.json b/neo/data/protocol.coz.json
@@ -14,32 +14,35 @@
       "188.68.34.29:10334",
       "188.68.34.29:10336"
     ],
-    "RPCList":[
+    "RPCList": [
       "http://188.68.34.29:10001",
       "http://188.68.34.29:10002",
       "http://188.68.34.29:10003",
       "http://188.68.34.29:10004"
     ],
     "SystemFee": {
-        "EnrollmentTransaction": 1000,
-        "IssueTransaction": 500,
-        "PublishTransaction": 500,
-        "RegisterTransaction": 10000
+      "EnrollmentTransaction": 1000,
+      "IssueTransaction": 500,
+      "PublishTransaction": 500,
+      "RegisterTransaction": 10000
     }
   },
-
   "ApplicationConfiguration": {
     "DataDirectoryPath": "Chains/coznet",
     "NotificationDataPath": "Chains/coz_notif",
     "RPCPort": 20332,
     "NodePort": 20333,
     "WsPort": 20334,
-    "UriPrefix": [ "http://*:20332"],
+    "UriPrefix": [
+      "http://*:20332"
+    ],
     "SslCert": "",
     "SslCertPassword": "",
-    "BootstrapFile":"",
-    "NotificationBootstrapFile":"",
-    "DebugStorage":1,
-    "CompilerNep8":false
+    "BootstrapFile": "",
+    "NotificationBootstrapFile": "",
+    "DebugStorage": 1,
+    "CompilerNep8": false,
+    "BootstrapName": "fauxnet",
+    "BootstrapFiles": "this_does_not_exist_for_this_network"
   }
 }
diff --git a/neo/data/protocol.mainnet.json b/neo/data/protocol.mainnet.json
@@ -53,8 +53,8 @@
     ],
     "SslCert": "",
     "SslCertPassword": "",
-    "BootstrapFile": "https://s3.us-east-2.amazonaws.com/cityofzion/bootstrap_main/Main2891xxx.tar.gz",
-    "NotificationBootstrapFile": "https://s3.us-east-2.amazonaws.com/cityofzion/bootstrap_main/MainNotif2891xxx.tar.gz",
+    "BootstrapName": "mainnet",
+    "BootstrapFiles": "https://s3.us-east-2.amazonaws.com/cityofzion/bootstrap_latest",
     "DebugStorage": 1,
     "AcceptIncomingPeers": false,
     "CompilerNep8": false

diff --git a/neo/data/protocol.testnet.json b/neo/data/protocol.testnet.json
@@ -1,52 +1,52 @@
 {
-    "ApplicationConfiguration": {
-        "DataDirectoryPath": "Chains/SC234",
-        "NotificationDataPath": "Chains/Test_Notif",
-        "RPCPort": 20332,
-        "NodePort": 20333,
-        "SslCert": "",
-        "SslCertPassword": "",
-        "UriPrefix": [
-            "http://*:20332"
-        ],
-        "WsPort": 20334,
-        "BootstrapFile":"https://s3.us-east-2.amazonaws.com/cityofzion/bootstrap_main/Test168xxxx.tar.gz",
-        "NotificationBootstrapFile":"https://s3.us-east-2.amazonaws.com/cityofzion/bootstrap_main/TestNotif168xxxx.tar.gz",
-        "DebugStorage":1,
-        "AcceptIncomingPeers": false,
-        "CompilerNep8":false
-    },
-    "ProtocolConfiguration": {
-        "AddressVersion": 23,
-        "Magic": 1953787457,
-        "SeedList": [
-            "13.58.169.218:20333",
-            "13.58.33.157:20333",
-            "18.222.161.128:20333",
-            "18.191.171.240:20333",
-            "18.222.168.189:20333"
-        ],
-        "RPCList":[
-            "http://18.221.221.195:8880",
-            "http://18.221.139.152:8880",
-            "http://52.15.48.60:8880",
-            "http://18.221.0.152:8880",
-            "http://52.14.184.44:8880"
-        ],
-        "StandbyValidators": [
-            "0327da12b5c40200e9f65569476bbff2218da4f32548ff43b6387ec1416a231ee8",
-            "026ce35b29147ad09e4afe4ec4a7319095f08198fa8babbe3c56e970b143528d22",
-            "0209e7fd41dfb5c2f8dc72eb30358ac100ea8c72da18847befe06eade68cebfcb9",
-            "039dafd8571a641058ccc832c5e2111ea39b09c0bde36050914384f7a48bce9bf9",
-            "038dddc06ce687677a53d54f096d2591ba2302068cf123c1f2d75c2dddc5425579",
-            "02d02b1873a0863cd042cc717da31cea0d7cf9db32b74d4c72c01b0011503e2e22",
-            "034ff5ceeac41acf22cd5ed2da17a6df4dd8358fcb2bfb1a43208ad0feaab2746b"
-        ],
-        "SystemFee": {
-            "EnrollmentTransaction": 10,
-            "IssueTransaction": 5,
-            "PublishTransaction": 5,
-            "RegisterTransaction": 100
-        }
+  "ApplicationConfiguration": {
+    "DataDirectoryPath": "Chains/SC234",
+    "NotificationDataPath": "Chains/Test_Notif",
+    "RPCPort": 20332,
+    "NodePort": 20333,
+    "SslCert": "",
+    "SslCertPassword": "",
+    "UriPrefix": [
+      "http://*:20332"
+    ],
+    "WsPort": 20334,
+    "BootstrapName": "testnet",
+    "BootstrapFiles": "https://s3.us-east-2.amazonaws.com/cityofzion/bootstrap_latest",
+    "DebugStorage": 1,
+    "AcceptIncomingPeers": false,
+    "CompilerNep8": false
+  },
+  "ProtocolConfiguration": {
+    "AddressVersion": 23,
+    "Magic": 1953787457,
+    "SeedList": [
+      "13.58.169.218:20333",
+      "13.58.33.157:20333",
+      "18.222.161.128:20333",
+      "18.191.171.240:20333",
+      "18.222.168.189:20333"
+    ],
+    "RPCList": [
+      "http://18.221.221.195:8880",
+      "http://18.221.139.152:8880",
+      "http://52.15.48.60:8880",
+      "http://18.221.0.152:8880",
+      "http://52.14.184.44:8880"
+    ],
+    "StandbyValidators": [
+      "0327da12b5c40200e9f65569476bbff2218da4f32548ff43b6387ec1416a231ee8",
+      "026ce35b29147ad09e4afe4ec4a7319095f08198fa8babbe3c56e970b143528d22",
+      "0209e7fd41dfb5c2f8dc72eb30358ac100ea8c72da18847befe06eade68cebfcb9",
+      "039dafd8571a641058ccc832c5e2111ea39b09c0bde36050914384f7a48bce9bf9",
+      "038dddc06ce687677a53d54f096d2591ba2302068cf123c1f2d75c2dddc5425579",
+      "02d02b1873a0863cd042cc717da31cea0d7cf9db32b74d4c72c01b0011503e2e22",
+      "034ff5ceeac41acf22cd5ed2da17a6df4dd8358fcb2bfb1a43208ad0feaab2746b"
+    ],
+    "SystemFee": {
+      "EnrollmentTransaction": 10,
+      "IssueTransaction": 5,
+      "PublishTransaction": 5,
+      "RegisterTransaction": 100
     }
+  }
 }
diff --git a/neo/data/protocol.unittest-net.json b/neo/data/protocol.unittest-net.json
@@ -39,6 +39,8 @@
     "NotificationBootstrapFile": "",
     "DebugStorage": 1,
     "AcceptIncomingPeers": false,
-    "CompilerNep8":true
+    "CompilerNep8": true,
+    "BootstrapName": "fauxnet",
+    "BootstrapFiles": "this_does_not_exist_for_this_network"
   }
 }
diff --git a/requirements.txt b/requirements.txt
@@ -56,7 +56,7 @@ pytz==2018.5
 requests==2.20.0
 scrypt==0.8.6
 six==1.11.0
-tqdm==4.26.0
+tqdm==4.23.4
 Twisted==18.7.0
 urllib3==1.23
 virtualenv==16.0.0