Remove additional sources of nondeterminism from benchmark archive

qa/zcash/create_benchmark_archive.py

@@ -1,17 +1,21 @@
 import binascii
+import calendar
 import json
 import plyvel
 import progressbar
 import os
+import stat
 import struct
 import subprocess
 import sys
+import tarfile
+import time
 
 ZCASH_CLI = './src/zcash-cli'
 USAGE = """
 Requirements:
-- faketime
-- tar
+- find
+- xz
 - %s (edit ZCASH_CLI in this script to alter the path)
 - A running mainnet zcashd using the default datadir with -txindex=1
 
@@ -26,7 +30,7 @@
 """ % ZCASH_CLI
 
 def check_deps():
-    if subprocess.call(['which', 'faketime', 'tar', ZCASH_CLI], stdout=subprocess.PIPE):
+    if subprocess.call(['which', 'find', 'xz', ZCASH_CLI], stdout=subprocess.PIPE):
         print USAGE
         sys.exit()
 
@@ -132,6 +136,24 @@ def compress_script(script):
     result.extend(script)
     return bytes(result)
 
+def deterministic_filter(tarinfo):
+    tarinfo.uid = tarinfo.gid = 0
+    tarinfo.uname = tarinfo.gname = "root"
+    tarinfo.mtime = calendar.timegm(time.strptime('2017-05-17', '%Y-%m-%d'))
+    tarinfo.mode |= stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP
+    tarinfo.mode &= ~stat.S_IWGRP
+    if tarinfo.isdir():
+        tarinfo.mode |= \
+            stat.S_IXUSR | \
+            stat.S_IXGRP | \
+            stat.S_IXOTH
+    else:
+        tarinfo.mode &= \
+            ~stat.S_IXUSR & \
+            ~stat.S_IXGRP & \
+            ~stat.S_IXOTH
+    return tarinfo
+
 def create_benchmark_archive(blk_hash):
     blk = json.loads(subprocess.check_output([ZCASH_CLI, 'getblock', blk_hash]))
     print 'Height: %d' % blk['height']
@@ -226,9 +248,14 @@ def create_benchmark_archive(blk_hash):
 
     # Make reproducible archive
     os.remove('%s/LOG' % db_path)
-    archive_name = 'block-%d.tar.gz' % blk['height']
-    subprocess.check_call(['faketime', '2017-05-17T00:00:00Z', 'tar', 'czf', archive_name, '--mtime=2017-05-17T00:00:00Z', 'benchmark'])
-    print 'Created archive %s' % archive_name
+    files = subprocess.check_output(['find', 'benchmark']).strip().split('\n')
+    archive_name = 'block-%d.tar' % blk['height']
+    tar = tarfile.open(archive_name, 'w')
+    for name in sorted(files):
+        tar.add(name, recursive=False, filter=deterministic_filter)
+    tar.close()
+    subprocess.check_call(['xz', '-6', archive_name])
+    print 'Created archive %s.xz' % archive_name
     subprocess.call(['rm', '-r', 'benchmark'])
 
 if __name__ == '__main__':

qa/zcash/performance-measurements.sh

@@ -66,14 +66,14 @@ function zcashd_valgrind_stop {
 }
 
 function extract_benchmark_data {
-    if [ -f "block-107134.tar.gz" ]; then
+    if [ -f "block-107134.tar.xz" ]; then
         # Check the hash of the archive:
         "$SHA256CMD" $SHA256ARGS -c <<EOF
-299a36b3445a9a0631eb9eb0b9e76c3e9e7493a98d6621ffd6dc362d3d86cbe8  block-107134.tar.gz
+4bd5ad1149714394e8895fa536725ed5d6c32c99812b962bfa73f03b5ffad4bb  block-107134.tar.xz
 EOF
         ARCHIVE_RESULT=$?
     else
-        echo "block-107134.tar.gz not found."
+        echo "block-107134.tar.xz not found."
         ARCHIVE_RESULT=1
     fi
     if [ $ARCHIVE_RESULT -ne 0 ]; then
@@ -84,7 +84,7 @@ EOF
         echo "Usage details are inside the Python script."
         exit 1
     fi
-    tar xzf block-107134.tar.gz -C "$DATADIR/regtest"
+    xzcat block-107134.tar.xz | tar x -C "$DATADIR/regtest"
 }
 
 # Precomputation