Create .tar.gz package deterministically using Python's "tarfile"

python/servo/command_base.py

@@ -7,12 +7,16 @@
 # option. This file may not be copied, modified, or distributed
 # except according to those terms.
 
+import gzip
+import itertools
+import locale
 import os
 from os import path
 import contextlib
 import subprocess
 from subprocess import PIPE
 import sys
+import tarfile
 import platform
 
 import toml
@@ -33,6 +37,55 @@ def cd(new_path):
         os.chdir(previous_path)
 
 
+@contextlib.contextmanager
+def setlocale(name):
+    """Context manager for changing the current locale"""
+    saved_locale = locale.setlocale(locale.LC_ALL)
+    try:
+        yield locale.setlocale(locale.LC_ALL, name)
+    finally:
+        locale.setlocale(locale.LC_ALL, saved_locale)
+
+
+def archive_deterministically(dir_to_archive, dest_archive, prepend_path=None):
+    """Create a .tar.gz archive in a deterministic (reproducible) manner.
+
+    See https://reproducible-builds.org/docs/archives/ for more details."""
+
+    def reset(tarinfo):
+        """Helper to reset owner/group and modification time for tar entries"""
+        tarinfo.uid = tarinfo.gid = 0
+        tarinfo.uname = tarinfo.gname = "root"
+        tarinfo.mtime = 0
+        return tarinfo
+
+    dest_archive = os.path.abspath(dest_archive)
+    with cd(dir_to_archive):
+        current_dir = "."
+        file_list = [current_dir]
+        for root, dirs, files in os.walk(current_dir):
+            for name in itertools.chain(dirs, files):
+                file_list.append(os.path.join(root, name))
+
+        # Sort file entries with the fixed locale
+        with setlocale('C'):
+            file_list.sort(cmp=locale.strcoll)
+
+        # Use a temporary file and atomic rename to avoid partially-formed
+        # packaging (in case of exceptional situations like running out of disk space).
+        # TODO do this in a temporary folder after #11983 is fixed
+        temp_file = '{}.temp~'.format(dest_archive)
+        with os.fdopen(os.open(temp_file, os.O_WRONLY | os.O_CREAT, 0644), 'w') as out_file:
+            with gzip.GzipFile('wb', fileobj=out_file, mtime=0) as gzip_file:
+                with tarfile.open(fileobj=gzip_file, mode='w:') as tar_file:
+                    for entry in file_list:
+                        arcname = entry
+                        if prepend_path is not None:
+                            arcname = os.path.normpath(os.path.join(prepend_path, arcname))
+                        tar_file.add(entry, filter=reset, recursive=False, arcname=arcname)
+        os.rename(temp_file, dest_archive)
+
+
 def host_triple():
     os_type = platform.system().lower()
     if os_type == "linux":

python/servo/package_commands.py

@@ -16,7 +16,6 @@
 import os
 import shutil
 import subprocess
-import tarfile
 
 from mach.registrar import Registrar
 from datetime import datetime
@@ -29,7 +28,14 @@
 
 from mako.template import Template
 
-from servo.command_base import CommandBase, cd, BuildNotFound, is_macosx, is_windows
+from servo.command_base import (
+    archive_deterministically,
+    BuildNotFound,
+    cd,
+    CommandBase,
+    is_macosx,
+    is_windows,
+)
 from servo.post_build_commands import find_dep_path_newest
 
 
@@ -205,7 +211,10 @@ def package(self, release=False, dev=False, android=None, debug=False, debugger=
         else:
             dir_to_package = '/'.join(binary_path.split('/')[:-1])
             dir_to_root = '/'.join(binary_path.split('/')[:-3])
-            shutil.copytree(dir_to_root + '/resources', dir_to_package + '/resources')
+            resources_dir = dir_to_package + '/resources'
+            if os.path.exists(resources_dir):
+                delete(resources_dir)
+            shutil.copytree(dir_to_root + '/resources', resources_dir)
             browserhtml_path = find_dep_path_newest('browserhtml', binary_path)
             if browserhtml_path is None:
                 print("Could not find browserhtml package; perhaps you haven't built Servo.")
@@ -234,9 +243,9 @@ def package(self, release=False, dev=False, android=None, debug=False, debugger=
             time = datetime.utcnow().replace(microsecond=0).isoformat()
             time = time.replace(':', "-")
             tar_path += time + "-servo-tech-demo.tar.gz"
-            with tarfile.open(tar_path, "w:gz") as tar:
-                # arcname is to add by relative rather than absolute path
-                tar.add(dir_to_package, arcname='servo/')
+
+            archive_deterministically(dir_to_package, tar_path, prepend_path='servo/')
+
             print("Packaged Servo into " + tar_path)
 
     @Command('install',