make-virtio: Shrink archive/iso size with hardlinking

Switch to using a .tar.gz, since that can preserve hardlinks.

Then before generating the archive, iterate over the driver tree
and replace all identical files with hardlinks to the first
occurrence.

This cuts the iso size down 2/3 and speeds up the RPM build process

https://bugzilla.redhat.com/show_bug.cgi?id=1217644

make-fedora-virtio-win-rpm.py

@@ -294,7 +294,7 @@ def _build_latest_rpm():
     rpm_dir = tempfile.mkdtemp(prefix='virtio-win-rpm-dir-')
     atexit.register(lambda: shutil.rmtree(rpm_dir))
 
-    shellcomm("mv %s/*.zip %s" % (script_dir, rpm_dir))
+    shellcomm("mv %s/*.tar.gz %s" % (script_dir, rpm_dir))
     shellcomm("cp %s/*-sources.zip %s" % (new_builds, rpm_dir))
     shellcomm("cd %s && mkdir %s && cp *.msi %s && "
         "zip -9 -r %s/%s-installers.zip %s && rm -rf %s" %

make-virtio-win-rpm-archive.py

@@ -6,7 +6,7 @@
 # See the COPYING file in the top-level directory.
 
 
-# Script for generating .vfd and .zip for virtio-win RPM
+# Script for generating .vfd and .tar.gz for virtio-win RPM
 #
 # Note to the maintainer: This script is also used internally for the RHEL
 #   virtio-win RPM build process. Consider that when making changes to the
@@ -16,6 +16,7 @@
 import atexit
 import errno
 import glob
+import hashlib
 import os
 import shutil
 import subprocess
@@ -130,7 +131,7 @@ def build_vfd(fname, dmap, driverdir, rootdir, finaldir):
     # The temp directory where we stage the files that will go on the vfd
     floppydir = os.path.join(rootdir, "drivers")
 
-    # The directory that will end up in the .zip archive, and in /usr/share
+    # The directory that will end up in the archive, and in /usr/share
     # via the RPM. We call this 'vfddrivers' to make it explicit where
     # they are coming from, but the RPM installs it as 'drivers' for
     # historical reasons.
@@ -197,22 +198,34 @@ def build_vfd(fname, dmap, driverdir, rootdir, finaldir):
     shutil.rmtree(floppydir)
 
 
-def archive(nvr, driverdir, finaldir):
+def hardlink_identical_files(outdir):
+    print "Hardlinking identical files..."
+
+    hashmap = {}
+    for root, dirs, files in os.walk(outdir):
+        ignore = dirs
+        for f in files:
+            path = os.path.join(root, f)
+            md5 = hashlib.md5(open(path, 'rb').read()).hexdigest()
+            if md5 not in hashmap:
+                hashmap[md5] = path
+                continue
+
+            # Found a collision
+            os.unlink(path)
+            run(["ln", hashmap[md5], path])
+
+
+def archive(nvr, finaldir):
     """
-    zip up the working directory
+    tar up the working directory
     """
+
+    # Generate .tar.gz
     print 'archiving the results'
-    for fname in os.listdir(driverdir):
-        path = os.path.join(driverdir, fname)
-        if os.path.isdir(path):
-            shutil.copytree(path, os.path.join(finaldir, fname))
-        else:
-            shutil.copy2(path, os.path.join(finaldir, fname))
-
-    # Generate .zip
     archivefile = os.path.join(os.path.dirname(finaldir),
-        "%s-bin-for-rpm.zip" % nvr)
-    run('cd %s && zip -9 -r %s %s' %
+        "%s-bin-for-rpm.tar.gz" % nvr)
+    run('cd %s && tar -czvf %s %s' %
         (os.path.dirname(finaldir), archivefile, nvr), shell=True)
 
     # Copy results to cwd
@@ -228,7 +241,7 @@ def archive(nvr, driverdir, finaldir):
 def get_options():
     description = """
 Package pre-built Windows drivers into a virtual floppy disk and bundle
-it in a ZIP file. Must pass a virtio-win version string, which is used
+it in a tar file. Must pass a virtio-win version string, which is used
 in the output file names, and a directory containing the built drivers.
 
 Example: %(prog)s virtio-win-1.2.3 /path/to/built/drivers
@@ -259,7 +272,9 @@ def main():
     build_vfd(options.nvr + '_amd64.vfd', vfd_dirs_64,
         options.driverdir, rootdir, finaldir)
 
-    archive(options.nvr, options.driverdir, finaldir)
+    run(["cp", "-rp", "%s/." % options.driverdir, finaldir])
+    hardlink_identical_files(finaldir)
+    archive(options.nvr, finaldir)
 
     return 0
 

tests/compare-output.py

@@ -38,7 +38,7 @@ def run(cmd):
 
 def extract_files(filename):
     """
-    Passed in either a zip file or RPM, extract the contents, including
+    Passed in either a zip, tar.gz, or RPM, extract the contents, including
     the contents of any contained vfd or iso files. Move these to a temp
     directory for easy comparison.
     """
@@ -56,12 +56,14 @@ def extract_files(filename):
         pass
     elif filename.endswith(".zip"):
         run("unzip %s -d %s > /dev/null" % (filename, extract_dir))
+    elif filename.endswith(".tar.gz"):
+        run("tar -xvf %s --directory %s > /dev/null" % (filename, extract_dir))
     elif filename.endswith(".rpm"):
         run("cd %s && rpm2cpio %s | cpio -idm --quiet" %
             (extract_dir, filename))
     else:
-        fail("Unexpected filename %s, only expecting .zip, .rpm, or a "
-            "directory" % filename)
+        fail("Unexpected filename %s, only expecting .zip, *.tar.gz, .rpm, "
+             "or a directory" % filename)
 
 
     # Find .vfd files
@@ -92,20 +94,20 @@ def extract_files(filename):
 def parse_args():
     desc = """
 Helper for comparing the output of make-virtio-win-rpm-archive.py. Can
-either compare the raw .zip output, a virtio-win .rpm file, or two directories
-for make-driver-dir.py output. Example:
+either compare the raw .tar.gz output, a virtio-win .rpm file,
+or two directories for make-driver-dir.py output. Example:
 
 - Run make-virtio-win-rpm-archive.py ...
-- Run make-virtio-win-rpm-archive.py, then move $OUTPUT.zip to orig.zip
+- Run make-virtio-win-rpm-archive.py, then move $OUTPUT.tar.gz to orig.tar.gz
 - Make your changes to make-virtio-win-rpm-archive.py
-- Re-run make-virtio-win-rpm-archive.py, then move $OUTPUT.zip to new.zip
-- Review changes: %(prog)s orig.zip new.zip
+- Re-run make-virtio-win-rpm-archive.py, then move $OUTPUT.tar.gz to new.tar.gz
+- Review changes: %(prog)s orig.tar.gz new.tar.gz
 """
     parser = argparse.ArgumentParser(description=desc,
         formatter_class=argparse.RawDescriptionHelpFormatter)
 
-    parser.add_argument("orig", help="Original .zip/.rpm/directory output")
-    parser.add_argument("new", help="New .zip/.rpm/directory output")
+    parser.add_argument("orig", help="Original .tar.gz/.rpm/directory output")
+    parser.add_argument("new", help="New .tar.gz/.rpm/directory output")
     parser.add_argument("--treeonly", action="store_true",
         help="Only show tree diff output.")
 

virtio-win.spec

@@ -33,7 +33,7 @@ License: GPLv2
 %endif
 
 # Already built files
-Source1: %{name}-%{version}-bin-for-rpm.zip
+Source1: %{name}-%{version}-bin-for-rpm.tar.gz
 Source2: %{qemu_ga_win_build}-installers.zip
 
 # Source files shipped in the srpm