Merge pull request #819 from qtumproject/xuanyan/fix-linearize

Merge linearize updates from bitcoin

contrib/linearize/example-linearize.cfg

@@ -22,10 +22,17 @@ max_height=313000
 netmagic=f1cfa6d3
 genesis=000075aef83cf2853580f8ae8ce6f8c3096cfa21d98334d6e3f95e5582ed986c
 input=/home/example/.qtum/blocks
+
 # testnet
 #netmagic=0d221506
 #genesis=0000e803ee215c0684ca0d2f9220594d3f828617972aad66feb2ba51f5e14222
 #input=/home/example/.qtum/testnet3/blocks
+
+# regtest
+#netmagic=fdddc6e1
+#genesis=665ed5b402ac0b44efc37d8926332994363e8a7278b7ee9a58fb972efadae943
+#input=/home/example/.qtum/regtest/blocks
+
 # "output" option causes blockchain files to be written to the given location,
 # with "output_file" ignored. If not used, "output_file" is used instead.
 # output=/home/example/blockchain_directory

contrib/linearize/linearize-data.py

@@ -15,6 +15,7 @@
 import hashlib
 import datetime
 import time
+import glob
 from collections import namedtuple
 from binascii import unhexlify
 
@@ -92,6 +93,31 @@ def mkblockmap(blkindex):
         blkmap[hash] = height
     return blkmap
 
+# This gets the first block file ID that exists from the input block
+# file directory.
+def getFirstBlockFileId(block_dir_path):
+    # First, this sets up a pattern to search for block files, for
+    # example 'blkNNNNN.dat'.
+    blkFilePattern = os.path.join(
+        block_dir_path, "blk[0-9][0-9][0-9][0-9][0-9].dat")
+
+    # This search is done with glob
+    blkFnList = glob.glob(blkFilePattern)
+
+    if len(blkFnList) == 0:
+        print("blocks not pruned - starting at 0")
+        return 0
+    # We then get the lexicographic minimum, which should be the first
+    # block file name.
+    firstBlkFilePath = min(blkFnList)
+    firstBlkFn = os.path.basename(firstBlkFilePath)
+
+    # now, the string should be ['b','l','k','N','N','N','N','N','.','d','a','t']
+    # So get the ID by choosing:              3   4   5   6   7
+    # The ID is not necessarily 0 if this is a pruned node.
+    blkId = int(firstBlkFn[3:8])
+    return blkId
+
 # Block header and extent on disk
 BlockExtent = namedtuple('BlockExtent', ['fn', 'offset', 'inhdr', 'blkhdr', 'size'])
 
@@ -101,7 +127,9 @@ def __init__(self, settings, blkindex, blkmap):
         self.blkindex = blkindex
         self.blkmap = blkmap
 
-        self.inFn = 0
+        # Get first occurring block file id - for pruned nodes this
+        # will not necessarily be 0
+        self.inFn = getFirstBlockFileId(self.settings['input'])
         self.inF = None
         self.outFn = 0
         self.outsz = 0
@@ -213,8 +241,8 @@ def run(self):
 
             inMagic = inhdr[:4]
             if (inMagic != self.settings['netmagic']):
-                print("Invalid magic: " + inMagic.hex())
-                return
+                self.inF.seek(-7, os.SEEK_CUR)
+                continue
             inLenLE = inhdr[4:]
             su = struct.unpack("<I", inLenLE)
             inLen = su[0] - 180 # length without header