Merge branch 'hotfix/fast-sync-subdomain-queue'

blockstack/lib/nameset/namedb.py

@@ -270,7 +270,11 @@ def get_state_paths(cls, impl, working_dir):
         """
         Get the paths to the relevant db files to back up
         """
-        return super(BlockstackDB, cls).get_state_paths(impl, working_dir) + [os.path.join(working_dir, 'atlas.db'), os.path.join(working_dir, 'subdomains.db')]
+        return super(BlockstackDB, cls).get_state_paths(impl, working_dir) + [
+                os.path.join(working_dir, 'atlas.db'), 
+                os.path.join(working_dir, 'subdomains.db'),
+                os.path.join(working_dir, 'subdomains.db.queue')
+            ]
 
 
     def get_db_path( self ):
@@ -308,9 +312,11 @@ def export_db(self, dirpath):
         snapshots_path = os.path.join(dirpath, os.path.basename(virtualchain.get_snapshots_filename(virtualchain_hooks, self.working_dir)))
         atlas_path = os.path.join(dirpath, 'atlas.db')
         subdomain_path = os.path.join(dirpath, 'subdomains.db')
+        subdomain_queue_path = os.path.join(dirpath, 'subdomains.db.queue')
 
         src_atlas_path = os.path.join(self.working_dir, 'atlas.db')
         src_subdomain_path = os.path.join(self.working_dir, 'subdomains.db')
+        src_subdomain_queue_path = os.path.join(self.working_dir, 'subdomains.db.queue')
 
         virtualchain.sqlite3_backup(self.get_db_path(), db_path)
         virtualchain.sqlite3_backup(virtualchain.get_snapshots_filename(virtualchain_hooks, self.working_dir), snapshots_path)
@@ -320,6 +326,9 @@ def export_db(self, dirpath):
 
         if os.path.exists(src_subdomain_path):
             virtualchain.sqlite3_backup(src_subdomain_path, subdomain_path)
+
+        if os.path.exists(src_subdomain_queue_path):
+            virtualchain.sqlite3_backup(src_subdomain_queue_path, subdomain_queue_path)
 
 
     @classmethod

blockstack/version.py

@@ -1,5 +1,5 @@
 # this is the only place where version should be updated
 __version_major__ = '19'
 __version_minor__ = '0'
-__version_patch__ = '1'
+__version_patch__ = '2'
 __version__ = '{}.{}.{}.0'.format(__version_major__, __version_minor__, __version_patch__)

integration_tests/blockstack_integration_tests/scenarios/name_pre_regup_subdomain_fast_sync.py

@@ -68,7 +68,7 @@ def restore( working_dir, snapshot_path, restore_dir, pubkeys, num_required ):
         return False
 
     # database must be identical 
-    db_filenames = ['blockstack-server.db', 'blockstack-server.snapshots', 'atlas.db', 'subdomains.db']
+    db_filenames = ['blockstack-server.db', 'blockstack-server.snapshots', 'atlas.db', 'subdomains.db', 'subdomains.db.queue']
     src_paths = [os.path.join(working_dir, fn) for fn in db_filenames]
     backup_paths = [os.path.join(restore_dir, fn) for fn in db_filenames]
 
@@ -94,12 +94,6 @@ def restore( working_dir, snapshot_path, restore_dir, pubkeys, num_required ):
             print 'Missing import keychain {}'.format(import_keychain_path)
             return False
 
-    # all subdomains are present
-    subds = ['bar.foo_{}.test'.format(i) for i in range(0,10)]
-    subdomain_db = blockstack.lib.subdomains.SubdomainDB(os.path.join(restore_dir, 'subdomains.db'), os.path.join(restore_dir, 'zonefiles'))
-    for subd in subds:
-        rec = subdomain_db.get_subdomain_entry(subd)
-
     return True
 
 
@@ -150,119 +144,190 @@ def scenario( wallets, **kw ):
 
     testlib.next_block( **kw )
 
-    # propagate 
-    for name in zonefiles:
+    # propagate the first five subdomains 
+    for i in range(0,5):
+        name = 'foo_{}.test'.format(i)
         assert testlib.blockstack_put_zonefile(zonefiles[name])
-
-    # process subdomains
+    
+    # process the first five subdomains
     testlib.next_block( **kw )
 
+    # propagate the last five subdomains, but don't process them
+    for i in range(5,10):
+        name = 'foo_{}.test'.format(i)
+        assert testlib.blockstack_put_zonefile(zonefiles[name])
+
     print 'waiting for all zone files to replicate'
     time.sleep(10)
 
     working_dir = os.environ.get('BLOCKSTACK_WORKING_DIR')
     restore_dir = os.path.join(working_dir, "snapshot_dir")
 
-    # snapshot the latest backup
-    snapshot_path = os.path.join(working_dir, "snapshot.bsk" )
-    rc = blockstack.fast_sync_snapshot(kw['working_dir'], snapshot_path, wallets[3].privkey, None )
-    if not rc:
-        print "Failed to fast_sync_snapshot"
-        return False
-
-    if not os.path.exists(snapshot_path):
-        print "Failed to create snapshot {}".format(snapshot_path)
-        return False
+    # make a backup 
+    db = testlib.get_state_engine()
 
-    # sign with more keys 
-    for i in xrange(4, 6):
-        rc = blockstack.fast_sync_sign_snapshot( snapshot_path, wallets[i].privkey )
+    print 'begin make backups of state from {}'.format(testlib.get_current_block(**kw) - 1)
+    for name in os.listdir(os.path.join(working_dir, 'backups')):
+        if name.endswith('.{}'.format(testlib.get_current_block(**kw) - 1)):
+            os.unlink(os.path.join(working_dir, 'backups', name))
+
+    db.make_backups(testlib.get_current_block(**kw))
+    print 'end make backups'
+
+    def _backup_and_restore():
+        # snapshot the latest backup
+        snapshot_path = os.path.join(working_dir, "snapshot.bsk" )
+        rc = blockstack.fast_sync_snapshot(working_dir, snapshot_path, wallets[3].privkey, None )
         if not rc:
-            print "Failed to sign with key {}".format(i)
+            print "Failed to fast_sync_snapshot"
+            return False
+
+        if not os.path.exists(snapshot_path):
+            print "Failed to create snapshot {}".format(snapshot_path)
             return False
 
-    # restore!
-    rc = restore( kw['working_dir'], snapshot_path, restore_dir, [wallets[3].pubkey_hex, wallets[4].pubkey_hex, wallets[5].pubkey_hex], 3 )
-    if not rc:
-        print "failed to restore snapshot {}".format(snapshot_path)
-        return False
+        # sign with more keys 
+        for i in xrange(4, 6):
+            rc = blockstack.fast_sync_sign_snapshot( snapshot_path, wallets[i].privkey )
+            if not rc:
+                print "Failed to sign with key {}".format(i)
+                return False
 
-    rc = restore( kw['working_dir'], snapshot_path, restore_dir, [wallets[5].pubkey_hex, wallets[4].pubkey_hex, wallets[3].pubkey_hex], 3 )
-    if not rc:
-        print "failed to restore snapshot {}".format(snapshot_path)
-        return False
+        # restore!
+        rc = restore( kw['working_dir'], snapshot_path, restore_dir, [wallets[3].pubkey_hex, wallets[4].pubkey_hex, wallets[5].pubkey_hex], 3 )
+        if not rc:
+            print "1 failed to restore snapshot {}".format(snapshot_path)
+            return False
 
-    rc = restore( kw['working_dir'], snapshot_path, restore_dir, [wallets[3].pubkey_hex, wallets[4].pubkey_hex], 2 )
-    if not rc:
-        print "failed to restore snapshot {}".format(snapshot_path)
-        return False
+        rc = restore( kw['working_dir'], snapshot_path, restore_dir, [wallets[5].pubkey_hex, wallets[4].pubkey_hex, wallets[3].pubkey_hex], 3 )
+        if not rc:
+            print "2 failed to restore snapshot {}".format(snapshot_path)
+            return False
 
-    rc = restore( kw['working_dir'], snapshot_path, restore_dir, [wallets[3].pubkey_hex, wallets[5].pubkey_hex], 2 )
-    if not rc:
-        print "failed to restore snapshot {}".format(snapshot_path)
-        return False
+        rc = restore( kw['working_dir'], snapshot_path, restore_dir, [wallets[3].pubkey_hex, wallets[4].pubkey_hex], 2 )
+        if not rc:
+            print "3 failed to restore snapshot {}".format(snapshot_path)
+            return False
 
-    rc = restore( kw['working_dir'], snapshot_path, restore_dir, [wallets[4].pubkey_hex, wallets[5].pubkey_hex], 2 )
-    if not rc:
-        print "failed to restore snapshot {}".format(snapshot_path)
-        return False
+        rc = restore( kw['working_dir'], snapshot_path, restore_dir, [wallets[3].pubkey_hex, wallets[5].pubkey_hex], 2 )
+        if not rc:
+            print "4 failed to restore snapshot {}".format(snapshot_path)
+            return False
 
-    rc = restore( kw['working_dir'], snapshot_path, restore_dir, [wallets[3].pubkey_hex], 1 )
-    if not rc:
-        print "failed to restore snapshot {}".format(snapshot_path)
-        return False
+        rc = restore( kw['working_dir'], snapshot_path, restore_dir, [wallets[4].pubkey_hex, wallets[5].pubkey_hex], 2 )
+        if not rc:
+            print "5 failed to restore snapshot {}".format(snapshot_path)
+            return False
 
-    rc = restore( kw['working_dir'], snapshot_path, restore_dir, [wallets[4].pubkey_hex, wallets[0].pubkey_hex], 1 )
-    if not rc:
-        print "failed to restore snapshot {}".format(snapshot_path)
-        return False
+        rc = restore( kw['working_dir'], snapshot_path, restore_dir, [wallets[3].pubkey_hex], 1 )
+        if not rc:
+            print "6 failed to restore snapshot {}".format(snapshot_path)
+            return False
 
-    rc = restore( kw['working_dir'], snapshot_path, restore_dir, [wallets[0].pubkey_hex, wallets[1].pubkey_hex, wallets[5].pubkey_hex], 1 )
-    if not rc:
-        print "failed to restore snapshot {}".format(snapshot_path)
-        return False
+        rc = restore( kw['working_dir'], snapshot_path, restore_dir, [wallets[4].pubkey_hex, wallets[0].pubkey_hex], 1 )
+        if not rc:
+            print "7 failed to restore snapshot {}".format(snapshot_path)
+            return False
 
-    # should fail
-    rc = restore( kw['working_dir'], snapshot_path, restore_dir, [wallets[3].pubkey_hex], 2 )
-    if rc:
-        print "restored insufficient signatures snapshot {}".format(snapshot_path)
-        return False
+        rc = restore( kw['working_dir'], snapshot_path, restore_dir, [wallets[0].pubkey_hex, wallets[1].pubkey_hex, wallets[5].pubkey_hex], 1 )
+        if not rc:
+            print "8 failed to restore snapshot {}".format(snapshot_path)
+            return False
 
-    shutil.rmtree(restore_dir)
+        shutil.move(restore_dir, restore_dir + '.bak')
 
-    # should fail
-    rc = restore( kw['working_dir'], snapshot_path, restore_dir, [wallets[3].pubkey_hex, wallets[4].pubkey_hex], 3 )
-    if rc:
-        print "restored insufficient signatures snapshot {}".format(snapshot_path)
-        return False
+        # should fail
+        rc = restore( kw['working_dir'], snapshot_path, restore_dir, [wallets[3].pubkey_hex], 2 )
+        if rc:
+            print "restored insufficient signatures snapshot {}".format(snapshot_path)
+            return False
 
-    shutil.rmtree(restore_dir)
+        shutil.rmtree(restore_dir)
 
-    # should fail
-    rc = restore( kw['working_dir'], snapshot_path, restore_dir, [wallets[0].pubkey_hex], 1 )
-    if rc:
-        print "restored wrongly-signed snapshot {}".format(snapshot_path)
-        return False
+        # should fail
+        rc = restore( kw['working_dir'], snapshot_path, restore_dir, [wallets[3].pubkey_hex, wallets[4].pubkey_hex], 3 )
+        if rc:
+            print "restored insufficient signatures snapshot {}".format(snapshot_path)
+            return False
 
-    shutil.rmtree(restore_dir)
+        shutil.rmtree(restore_dir)
+
+        # should fail
+        rc = restore( kw['working_dir'], snapshot_path, restore_dir, [wallets[0].pubkey_hex], 1 )
+        if rc:
+            print "restored wrongly-signed snapshot {}".format(snapshot_path)
+            return False
+
+        shutil.rmtree(restore_dir)
 
-    # should fail
-    rc = restore( kw['working_dir'], snapshot_path, restore_dir, [wallets[0].pubkey_hex, wallets[3].pubkey_hex], 2 )
-    if rc:
-        print "restored wrongly-signed snapshot {}".format(snapshot_path)
+        # should fail
+        rc = restore( kw['working_dir'], snapshot_path, restore_dir, [wallets[0].pubkey_hex, wallets[3].pubkey_hex], 2 )
+        if rc:
+            print "restored wrongly-signed snapshot {}".format(snapshot_path)
+            return False
+
+        shutil.rmtree(restore_dir)
+
+        # should fail
+        rc = restore( kw['working_dir'], snapshot_path, restore_dir, [wallets[0].pubkey_hex, wallets[3].pubkey_hex, wallets[4].pubkey_hex], 3 )
+        if rc:
+            print "restored wrongly-signed snapshot {}".format(snapshot_path)
+            return False
+
+        shutil.rmtree(restore_dir)
+        shutil.move(restore_dir + '.bak', restore_dir)
+        return True
+
+    # test backup and restore
+    res = _backup_and_restore()
+    if not res:
+        return res
+
+    # first five subdomains are all present in the subdomain DB 
+    subds = ['bar.foo_{}.test'.format(i) for i in range(0,5)]
+    subdomain_db = blockstack.lib.subdomains.SubdomainDB(os.path.join(restore_dir, 'subdomains.db'), os.path.join(restore_dir, 'zonefiles'))
+    for subd in subds:
+        rec = subdomain_db.get_subdomain_entry(subd)
+        if not rec:
+            print 'not found: {}'.format(subd)
+            return False
+
+    # last 5 subdomains are queued in the subdomain DB queue 
+    queued_zfinfos = blockstack.lib.queue.queuedb_findall(os.path.join(restore_dir, 'subdomains.db.queue'), 'zonefiles')
+    if len(queued_zfinfos) != 5:
+        print 'only {} zonefiles queued'.format(queued_zfinfos)
+        print queued_zfinfos
         return False
 
+    # process the last five subdomains
+    testlib.next_block( **kw )
+
     shutil.rmtree(restore_dir)
+    os.unlink(os.path.join(working_dir, "snapshot.bsk"))
+
+    # test backup and restore
+    res = _backup_and_restore()
+    if not res:
+        return res
+
+    # all subdomains are all present in the subdomain DB 
+    subds = ['bar.foo_{}.test'.format(i) for i in range(0,10)]
+    subdomain_db = blockstack.lib.subdomains.SubdomainDB(os.path.join(restore_dir, 'subdomains.db'), os.path.join(restore_dir, 'zonefiles'))
+    for subd in subds:
+        rec = subdomain_db.get_subdomain_entry(subd)
+        if not rec:
+            print 'not found: {}'.format(subd)
+            return False
 
-    # should fail
-    rc = restore( kw['working_dir'], snapshot_path, restore_dir, [wallets[0].pubkey_hex, wallets[3].pubkey_hex, wallets[4].pubkey_hex], 3 )
-    if rc:
-        print "restored wrongly-signed snapshot {}".format(snapshot_path)
+    # nothing queued
+    queued_zfinfos = blockstack.lib.queue.queuedb_findall(os.path.join(restore_dir, 'subdomains.db.queue'), 'zonefiles')
+    if len(queued_zfinfos) != 0:
+        print '{} zonefiles queued'.format(queued_zfinfos)
+        print queued_zfinfos
         return False
 
     shutil.rmtree(restore_dir)
 
-
 def check( state_engine ):
 
     # not revealed, but ready 

integration_tests/blockstack_integration_tests/version.py

@@ -1 +1 @@
-__version__='19.0.1.0'
+__version__='19.0.2.0'