1- #!/usr/bin/env python
1+ #!/usr/bin/env python3
22#
33# linearize-data.py: Construct a linear, no-fork version of the chain.
44#
88#
99
1010from __future__ import print_function , division
11+ try : # Python 3
12+ import http .client as httplib
13+ except ImportError : # Python 2
14+ import httplib
1115import json
1216import struct
1317import re
1418import os
1519import os .path
1620import base64
17- import httplib
1821import sys
1922import hashlib
2023import dash_hash
2124import datetime
2225import time
2326from collections import namedtuple
27+ from binascii import hexlify , unhexlify
2428
2529settings = {}
2630
31+ ##### Switch endian-ness #####
32+ def hex_switchEndian (s ):
33+ """ Switches the endianness of a hex string (in pairs of hex chars) """
34+ pairList = [s [i :i + 2 ].encode () for i in range (0 , len (s ), 2 )]
35+ return b'' .join (pairList [::- 1 ]).decode ()
36+
2737def uint32 (x ):
28- return x & 0xffffffffL
38+ return x & 0xffffffff
2939
3040def bytereverse (x ):
3141 return uint32 (( ((x ) << 24 ) | (((x ) << 8 ) & 0x00ff0000 ) |
@@ -36,14 +46,14 @@ def bufreverse(in_buf):
3646 for i in range (0 , len (in_buf ), 4 ):
3747 word = struct .unpack ('@I' , in_buf [i :i + 4 ])[0 ]
3848 out_words .append (struct .pack ('@I' , bytereverse (word )))
39- return '' .join (out_words )
49+ return b ''join (out_words )
4050
4151def wordreverse (in_buf ):
4252 out_words = []
4353 for i in range (0 , len (in_buf ), 4 ):
4454 out_words .append (in_buf [i :i + 4 ])
4555 out_words .reverse ()
46- return '' .join (out_words )
56+ return b ''join (out_words )
4757
4858def calc_hdr_hash (blk_hdr ):
4959 #hash1 = hashlib.sha256()
@@ -62,7 +72,7 @@ def calc_hash_str(blk_hdr):
6272 hash = calc_hdr_hash (blk_hdr )
6373 hash = bufreverse (hash )
6474 hash = wordreverse (hash )
65- hash_str = hash . encode ( 'hex '
75+ hash_str = hexlify ( hash ). decode ( 'utf-8 '
6676 return hash_str
6777
6878def get_blk_dt (blk_hdr ):
@@ -72,17 +82,21 @@ def get_blk_dt(blk_hdr):
7282 dt_ym = datetime .datetime (dt .year , dt .month , 1 )
7383 return (dt_ym , nTime )
7484
85+ # When getting the list of block hashes, undo any byte reversals.
7586def get_block_hashes (settings ):
7687 blkindex = []
7788 f = open (settings ['hashlist' ], "r" )
7889 for line in f :
7990 line = line .rstrip ()
91+ if settings ['rev_hash_bytes' ] == 'true' :
92+ line = hex_switchEndian (line )
8093 blkindex .append (line )
8194
8295 print ("Read " + str (len (blkindex )) + " hashes" )
8396
8497 return blkindex
8598
99+ # The block map shouldn't give or receive byte-reversed hashes.
86100def mkblockmap (blkindex ):
87101 blkmap = {}
88102 for height ,hash in enumerate (blkindex ):
@@ -210,7 +224,7 @@ def run(self):
210224
211225 inMagic = inhdr [:4 ]
212226 if (inMagic != self .settings ['netmagic' ]):
213- print ("Invalid magic: " + inMagic . encode ( 'hex '
227+ print ("Invalid magic: " + hexlify ( inMagic ). decode ( 'utf-8 '
214228 return
215229 inLenLE = inhdr [4 :]
216230 su = struct .unpack ("<I" , inLenLE )
@@ -268,10 +282,16 @@ def run(self):
268282 settings [m .group (1 )] = m .group (2 )
269283 f .close ()
270284
285+ # Force hash byte format setting to be lowercase to make comparisons easier.
286+ # Also place upfront in case any settings need to know about it.
287+ if 'rev_hash_bytes' not in settings :
288+ settings ['rev_hash_bytes' ] = 'false'
289+ settings ['rev_hash_bytes' ] = settings ['rev_hash_bytes' ].lower ()
290+
271291 if 'netmagic' not in settings :
272- settings ['netmagic' ] = 'cee2caff '
292+ settings ['netmagic' ] = 'bf0c6bbd '
273293 if 'genesis' not in settings :
274- settings ['genesis' ] = '00000bafbc94add76cb75e2ec92894837288a481e5c005f6563d91623bf8bc2c '
294+ settings ['genesis' ] = '00000ffd590b1485b3caadc19b22e6379c733355108f107a430458cdf3407ab6 '
275295 if 'input' not in settings :
276296 settings ['input' ] = 'input'
277297 if 'hashlist' not in settings :
@@ -281,14 +301,14 @@ def run(self):
281301 if 'split_timestamp' not in settings :
282302 settings ['split_timestamp' ] = 0
283303 if 'max_out_sz' not in settings :
284- settings ['max_out_sz' ] = 1000L * 1000 * 1000
304+ settings ['max_out_sz' ] = 1000 * 1000 * 1000
285305 if 'out_of_order_cache_sz' not in settings :
286306 settings ['out_of_order_cache_sz' ] = 100 * 1000 * 1000
287307
288- settings ['max_out_sz' ] = long (settings ['max_out_sz' ])
308+ settings ['max_out_sz' ] = int (settings ['max_out_sz' ])
289309 settings ['split_timestamp' ] = int (settings ['split_timestamp' ])
290310 settings ['file_timestamp' ] = int (settings ['file_timestamp' ])
291- settings ['netmagic' ] = settings ['netmagic' ].decode ( 'hex' )
311+ settings ['netmagic' ] = unhexlify ( settings ['netmagic' ].encode ( 'utf-8' ) )
292312 settings ['out_of_order_cache_sz' ] = int (settings ['out_of_order_cache_sz' ])
293313
294314 if 'output_file' not in settings and 'output' not in settings :
@@ -298,8 +318,8 @@ def run(self):
298318 blkindex = get_block_hashes (settings )
299319 blkmap = mkblockmap (blkindex )
300320
321+ # Block hash map won't be byte-reversed. Neither should the genesis hash.
301322 if not settings ['genesis' ] in blkmap :
302323 print ("Genesis block not found in hashlist" )
303324 else :
304325 BlockDataCopier (settings , blkindex , blkmap ).run ()
305-
0 commit comments