From dcf48d65f564bd9ef5d69b8da1b81763f4d00cea Mon Sep 17 00:00:00 2001 From: Andrew Leonard Date: Tue, 9 Nov 2021 13:06:14 +0000 Subject: [PATCH 1/3] 8276743: Make openjdk build Zip Archive generation "reproducible" Signed-off-by: Andrew Leonard --- make/ToolsJdk.gmk | 2 + make/common/ZipArchive.gmk | 20 +- .../build/tools/generatezip/GenerateZip.java | 284 ++++++++++++++++++ 3 files changed, 303 insertions(+), 3 deletions(-) create mode 100644 make/jdk/src/classes/build/tools/generatezip/GenerateZip.java diff --git a/make/ToolsJdk.gmk b/make/ToolsJdk.gmk index 395a78602f626..968f48fa90474 100644 --- a/make/ToolsJdk.gmk +++ b/make/ToolsJdk.gmk @@ -82,6 +82,8 @@ TOOL_GENERATECACERTS = $(JAVA_SMALL) -cp $(BUILDTOOLS_OUTPUTDIR)/jdk_tools_class TOOL_GENERATEEMOJIDATA = $(JAVA_SMALL) -cp $(BUILDTOOLS_OUTPUTDIR)/jdk_tools_classes \ build.tools.generateemojidata.GenerateEmojiData +TOOL_GENERATEZIP = $(JAVA_SMALL) -cp $(BUILDTOOLS_OUTPUTDIR)/jdk_tools_classes \ + build.tools.generatezip.GenerateZip # TODO: There are references to the jdwpgen.jar in jdk/make/netbeans/jdwpgen/build.xml # and nbproject/project.properties in the same dir. Needs to be looked at. diff --git a/make/common/ZipArchive.gmk b/make/common/ZipArchive.gmk index efa013f501ecf..d3de3b422072a 100644 --- a/make/common/ZipArchive.gmk +++ b/make/common/ZipArchive.gmk @@ -1,5 +1,5 @@ # -# Copyright (c) 2011, 2019, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2011, 2021, Oracle and/or its affiliates. All rights reserved. # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. # # This code is free software; you can redistribute it and/or modify it @@ -26,6 +26,8 @@ ifndef _ZIP_ARCHIVE_GMK _ZIP_ARCHIVE_GMK := 1 +include ../ToolsJdk.gmk + ifeq (,$(_MAKEBASE_GMK)) $(error You must include MakeBase.gmk prior to including ZipArchive.gmk) endif @@ -134,6 +136,8 @@ define SetupZipArchiveBody # dir is very small. # If zip has nothing to do, it returns 12 and would fail the build. Check for 12 # and only fail if it's not. + # For reproducible builds set the zip access & modify times to SOURCE_DATE_EPOCH + # by using a ziptmp folder to generate final zip from using GenerateZip. $$($1_ZIP) : $$($1_ALL_SRCS) $$($1_EXTRA_DEPS) $$(call LogWarn, Updating $$($1_NAME)) $$(call MakeTargetDir) @@ -156,14 +160,24 @@ define SetupZipArchiveBody ) \ ) \ ) + $(RM) -r $$(SUPPORT_OUTPUTDIR)/ziptmp/$1 + $(MKDIR) -p $$(SUPPORT_OUTPUTDIR)/ziptmp/$1/files $$(foreach s,$$($1_SRC_SLASH), $$(call ExecuteWithLog, \ $$(SUPPORT_OUTPUTDIR)/zip/$$(patsubst $$(OUTPUTDIR)/%,%, $$@), \ - (cd $$s && $(ZIPEXE) -qru $$($1_ZIP_OPTIONS) $$@ . \ + (cd $$s && $(ZIPEXE) -qru $$($1_ZIP_OPTIONS) \ + $$(SUPPORT_OUTPUTDIR)/ziptmp/$1/tmp.zip . \ $$($1_ZIP_INCLUDES) $$($1_ZIP_EXCLUDES) -x \*_the.\* \ $$($1_ZIP_EXCLUDES_$$s) \ || test "$$$$?" = "12" \ ))$$(NEWLINE) \ - ) true \ + ) true + $$(call ExecuteWithLog, \ + $$(SUPPORT_OUTPUTDIR)/generatezip/$$(patsubst $$(OUTPUTDIR)/%,%, $$@), \ + (cd $$(SUPPORT_OUTPUTDIR)/ziptmp/$1/files && \ + $(RM) $$@ && \ + $(UNZIP) -q $$(SUPPORT_OUTPUTDIR)/ziptmp/$1/tmp.zip && \ + $(TOOL_GENERATEZIP) -f $$@ -t $(SOURCE_DATE_EPOCH) . \ + ))$$(NEWLINE) \ $(TOUCH) $$@ # Add zip to target list diff --git a/make/jdk/src/classes/build/tools/generatezip/GenerateZip.java b/make/jdk/src/classes/build/tools/generatezip/GenerateZip.java new file mode 100644 index 0000000000000..556500b9c162d --- /dev/null +++ b/make/jdk/src/classes/build/tools/generatezip/GenerateZip.java @@ -0,0 +1,284 @@ +/* + * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package build.tools.generatezip; + +import java.io.*; +import java.nio.file.*; +import java.nio.file.attribute.BasicFileAttributes; +import java.nio.channels.Channels; +import java.nio.channels.FileChannel; +import java.util.*; +import java.util.zip.ZipEntry; +import java.util.zip.ZipException; +import java.util.zip.ZipFile; +import java.util.zip.ZipInputStream; +import java.util.zip.ZipOutputStream; + +/** + * Generate a zip file in a "reproducible" manner from the input files or + * directory. + * Standard zip tools rely on OS file list querying whose ordering varies + * by platform architecture, this class ensures the zip entries are ordered + * and also supports SOURCE_DATE_EPOCH timestamps. + */ +public class GenerateZip { + String fname = null; + String zname = ""; + long timestamp = -1L; + List files = new ArrayList<>();; + boolean verbose = false; + + Set entries = new LinkedHashSet<>(); + + private boolean ok; + + public GenerateZip() { + } + + public synchronized boolean run(String args[]) { + ok = true; + if (!parseArgs(args)) { + return false; + } + try { + zname = fname.replace(File.separatorChar, '/'); + if (zname.startsWith("./")) { + zname = zname.substring(2); + } + + if (verbose) System.out.println("Files or directories to zip: "+files); + + File zipFile = new File(fname); + // Check archive to create does not exist + if (!zipFile.exists()) { + // Process Files + for(String file : files) { + Path filepath = Paths.get(file); + processFiles(filepath); + } + + try (FileOutputStream out = new FileOutputStream(fname)) { + boolean createOk = create(new BufferedOutputStream(out, 4096)); + if (ok) { + ok = createOk; + } + } + } else { + error("Target zip file "+fname+" already exists."); + ok = false; + } + } catch (IOException e) { + fatalError(e); + ok = false; + } catch (Error ee) { + ee.printStackTrace(); + ok = false; + } catch (Throwable t) { + t.printStackTrace(); + ok = false; + } + return ok; + } + + boolean parseArgs(String args[]) { + try { + boolean parsingIncludes = false; + boolean parsingExcludes = false; + int count = 0; + while(count < args.length) { + if (args[count].startsWith("-")) { + String flag = args[count].substring(1); + switch (flag.charAt(0)) { + case 'f': + fname = args[++count]; + break; + case 't': + // SOURCE_DATE_EPOCH timestamp specified + timestamp = Long.parseLong(args[++count]) * 1000; + break; + case 'v': + verbose = true; + break; + default: + error(String.format("Illegal option -%s", String.valueOf(flag.charAt(0)))); + usageError(); + return false; + } + } else { + // file or dir to zip + files.add(args[count]); + } + count++; + } + } catch (ArrayIndexOutOfBoundsException e) { + usageError(); + return false; + } catch (NumberFormatException e) { + usageError(); + return false; + } + if (fname == null) { + error(String.format("-f must be specified")); + usageError(); + return false; + } + // If no files specified then default to current directory + if (files.size() == 0) { + error("No input directory or files were specified"); + usageError(); + return false; + } + + return true; + } + + // Walk tree matching files and adding to entries list + void processFiles(Path path) throws IOException { + File fpath = path.toFile(); + boolean pathIsDir = fpath.isDirectory(); + + // Keep a sorted Set of files to be processed, so that the Jmod is reproducible + // as Files.walkFileTree order is not defined + SortedMap filesToProcess = new TreeMap(); + + Files.walkFileTree(path, Set.of(FileVisitOption.FOLLOW_LINKS), + Integer.MAX_VALUE, new SimpleFileVisitor() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) + throws IOException + { + Path relPath; + String name; + if (pathIsDir) { + relPath = path.relativize(file); + name = relPath.toString(); + } else { + relPath = file; + name = file.toString(); + } + filesToProcess.put(name, file); + return FileVisitResult.CONTINUE; + } + }); + + // Process files in sorted order + for (Map.Entry entry : filesToProcess.entrySet()) { + String name = entry.getKey(); + Path filepath = entry.getValue(); + + File f = filepath.toFile(); + entries.add(f); + } + } + + // Create new zip from entries + boolean create(OutputStream out) throws IOException + { + try (ZipOutputStream zos = new ZipOutputStream(out)) { + for (File file: entries) { + addFile(zos, file); + } + } + return true; + } + + // Ensure a consistent entry name format + String entryName(String name) { + name = name.replace(File.separatorChar, '/'); + + if (name.startsWith("/")) { + name = name.substring(1); + } else if (name.startsWith("./")) { + name = name.substring(2); + } + return name; + } + + // Add File to Zip + void addFile(ZipOutputStream zos, File file) throws IOException { + String name = file.getPath(); + boolean isDir = file.isDirectory(); + if (isDir) { + name = name.endsWith(File.separator) ? name : (name + File.separator); + } + name = entryName(name); + + if (name.equals("") || name.equals(".") || name.equals(zname)) { + return; + } + + long size = isDir ? 0 : file.length(); + + if (verbose) { + System.out.println("Adding: "+name); + } + + ZipEntry e = new ZipEntry(name); + // Set to specified timestamp if set otherwise use file lastModified time + if (timestamp != -1L) { + e.setTime(timestamp); + } else { + e.setTime(file.lastModified()); + } + if (size == 0) { + e.setMethod(ZipEntry.STORED); + e.setSize(0); + e.setCrc(0); + } + zos.putNextEntry(e); + if (!isDir) { + byte[] buf = new byte[8192]; + int len; + try (FileInputStream fis = new FileInputStream(file); + FileChannel fic = fis.getChannel()) { + fic.transferTo(0, fic.size(), Channels.newChannel(zos)); + } + } + zos.closeEntry(); + } + + void usageError() { + error( + "Usage: GenerateZip [-v] -f \n" + + "Options:\n" + + " -v verbose output\n" + + " -f specify archive file name to create\n" + + " -t specific SOURCE_DATE_EPOCH value to use for timestamps\n" + + "If any file is a directory then it is processed recursively.\n"); + } + + void fatalError(Exception e) { + e.printStackTrace(); + } + + protected void error(String s) { + System.err.println(s); + } + + public static void main(String args[]) { + GenerateZip z = new GenerateZip(); + System.exit(z.run(args) ? 0 : 1); + } +} + From 8d148065952724bcb7ef0264745ecea378ec0a4b Mon Sep 17 00:00:00 2001 From: Andrew Leonard Date: Thu, 11 Nov 2021 19:44:07 +0000 Subject: [PATCH 2/3] 8276743: Make openjdk build Zip Archive generation "reproducible" Signed-off-by: Andrew Leonard --- make/Main.gmk | 10 +- make/ToolsJdk.gmk | 4 +- make/common/ZipArchive.gmk | 26 +-- .../MakeZipReproducible.java} | 186 +++++++----------- 4 files changed, 90 insertions(+), 136 deletions(-) rename make/jdk/src/classes/build/tools/{generatezip/GenerateZip.java => makezipreproducible/MakeZipReproducible.java} (52%) diff --git a/make/Main.gmk b/make/Main.gmk index 75eee65ba84af..e5ea250bf5aad 100644 --- a/make/Main.gmk +++ b/make/Main.gmk @@ -324,7 +324,7 @@ $(eval $(call SetupTarget, vscode-project-ccls, \ # aren't built until after libjava and libjvm are available to link to. $(eval $(call SetupTarget, demos-jdk, \ MAKEFILE := CompileDemos, \ - DEPS := java.base-libs exploded-image, \ + DEPS := java.base-libs exploded-image buildtools-jdk, \ )) $(eval $(call SetupTarget, test-image-demos-jdk, \ @@ -383,12 +383,12 @@ bootcycle-images: $(eval $(call SetupTarget, zip-security, \ MAKEFILE := ZipSecurity, \ - DEPS := java.base-java java.security.jgss-java java.security.jgss-libs, \ + DEPS := buildtools-jdk java.base-java java.security.jgss-java java.security.jgss-libs, \ )) $(eval $(call SetupTarget, zip-source, \ MAKEFILE := ZipSource, \ - DEPS := gensrc, \ + DEPS := buildtools-jdk gensrc, \ )) $(eval $(call SetupTarget, jrtfs-jar, \ @@ -508,13 +508,13 @@ $(eval $(call SetupTarget, docs-jdk-index, \ $(eval $(call SetupTarget, docs-zip, \ MAKEFILE := Docs, \ TARGET := docs-zip, \ - DEPS := docs-jdk, \ + DEPS := docs-jdk buildtools-jdk, \ )) $(eval $(call SetupTarget, docs-specs-zip, \ MAKEFILE := Docs, \ TARGET := docs-specs-zip, \ - DEPS := docs-jdk-specs, \ + DEPS := docs-jdk-specs buildtools-jdk, \ )) $(eval $(call SetupTarget, update-build-docs, \ diff --git a/make/ToolsJdk.gmk b/make/ToolsJdk.gmk index 968f48fa90474..af9def3a415ad 100644 --- a/make/ToolsJdk.gmk +++ b/make/ToolsJdk.gmk @@ -82,8 +82,8 @@ TOOL_GENERATECACERTS = $(JAVA_SMALL) -cp $(BUILDTOOLS_OUTPUTDIR)/jdk_tools_class TOOL_GENERATEEMOJIDATA = $(JAVA_SMALL) -cp $(BUILDTOOLS_OUTPUTDIR)/jdk_tools_classes \ build.tools.generateemojidata.GenerateEmojiData -TOOL_GENERATEZIP = $(JAVA_SMALL) -cp $(BUILDTOOLS_OUTPUTDIR)/jdk_tools_classes \ - build.tools.generatezip.GenerateZip +TOOL_MAKEZIPREPRODUCIBLE = $(JAVA_SMALL) -cp $(BUILDTOOLS_OUTPUTDIR)/jdk_tools_classes \ + build.tools.makezipreproducible.MakeZipReproducible # TODO: There are references to the jdwpgen.jar in jdk/make/netbeans/jdwpgen/build.xml # and nbproject/project.properties in the same dir. Needs to be looked at. diff --git a/make/common/ZipArchive.gmk b/make/common/ZipArchive.gmk index d3de3b422072a..41c4b5a47bb58 100644 --- a/make/common/ZipArchive.gmk +++ b/make/common/ZipArchive.gmk @@ -26,6 +26,7 @@ ifndef _ZIP_ARCHIVE_GMK _ZIP_ARCHIVE_GMK := 1 +# Depends on build tools for MakeZipReproducible include ../ToolsJdk.gmk ifeq (,$(_MAKEBASE_GMK)) @@ -137,7 +138,7 @@ define SetupZipArchiveBody # If zip has nothing to do, it returns 12 and would fail the build. Check for 12 # and only fail if it's not. # For reproducible builds set the zip access & modify times to SOURCE_DATE_EPOCH - # by using a ziptmp folder to generate final zip from using GenerateZip. + # by using a ziptmp folder to generate final zip from using MakeZipReproducible. $$($1_ZIP) : $$($1_ALL_SRCS) $$($1_EXTRA_DEPS) $$(call LogWarn, Updating $$($1_NAME)) $$(call MakeTargetDir) @@ -160,24 +161,25 @@ define SetupZipArchiveBody ) \ ) \ ) - $(RM) -r $$(SUPPORT_OUTPUTDIR)/ziptmp/$1 - $(MKDIR) -p $$(SUPPORT_OUTPUTDIR)/ziptmp/$1/files $$(foreach s,$$($1_SRC_SLASH), $$(call ExecuteWithLog, \ $$(SUPPORT_OUTPUTDIR)/zip/$$(patsubst $$(OUTPUTDIR)/%,%, $$@), \ - (cd $$s && $(ZIPEXE) -qru $$($1_ZIP_OPTIONS) \ - $$(SUPPORT_OUTPUTDIR)/ziptmp/$1/tmp.zip . \ + (cd $$s && $(ZIPEXE) -qru $$($1_ZIP_OPTIONS) $$@ . \ $$($1_ZIP_INCLUDES) $$($1_ZIP_EXCLUDES) -x \*_the.\* \ $$($1_ZIP_EXCLUDES_$$s) \ || test "$$$$?" = "12" \ ))$$(NEWLINE) \ ) true - $$(call ExecuteWithLog, \ - $$(SUPPORT_OUTPUTDIR)/generatezip/$$(patsubst $$(OUTPUTDIR)/%,%, $$@), \ - (cd $$(SUPPORT_OUTPUTDIR)/ziptmp/$1/files && \ - $(RM) $$@ && \ - $(UNZIP) -q $$(SUPPORT_OUTPUTDIR)/ziptmp/$1/tmp.zip && \ - $(TOOL_GENERATEZIP) -f $$@ -t $(SOURCE_DATE_EPOCH) . \ - ))$$(NEWLINE) \ + ifeq ($(ENABLE_REPRODUCIBLE_BUILD), true) + $$(call ExecuteWithLog, \ + $$(SUPPORT_OUTPUTDIR)/makezipreproducible/$$(patsubst $$(OUTPUTDIR)/%,%, $$@), \ + ($(RM) $$(SUPPORT_OUTPUTDIR)/ziptmp/$1/tmp.zip && \ + $(MKDIR) -p $$(SUPPORT_OUTPUTDIR)/ziptmp/$1 && \ + $(TOOL_MAKEZIPREPRODUCIBLE) -f $$(SUPPORT_OUTPUTDIR)/ziptmp/$1/tmp.zip \ + -t $(SOURCE_DATE_EPOCH) $$@ && \ + $(RM) $$@ && \ + $(MV) $$(SUPPORT_OUTPUTDIR)/ziptmp/$1/tmp.zip $$@ \ + )) + endif $(TOUCH) $$@ # Add zip to target list diff --git a/make/jdk/src/classes/build/tools/generatezip/GenerateZip.java b/make/jdk/src/classes/build/tools/makezipreproducible/MakeZipReproducible.java similarity index 52% rename from make/jdk/src/classes/build/tools/generatezip/GenerateZip.java rename to make/jdk/src/classes/build/tools/makezipreproducible/MakeZipReproducible.java index 556500b9c162d..2dac7430a304d 100644 --- a/make/jdk/src/classes/build/tools/generatezip/GenerateZip.java +++ b/make/jdk/src/classes/build/tools/makezipreproducible/MakeZipReproducible.java @@ -21,7 +21,7 @@ * questions. */ -package build.tools.generatezip; +package build.tools.makezipreproducible; import java.io.*; import java.nio.file.*; @@ -36,24 +36,24 @@ import java.util.zip.ZipOutputStream; /** - * Generate a zip file in a "reproducible" manner from the input files or - * directory. + * Generate a zip file in a "reproducible" manner from the input zip file. * Standard zip tools rely on OS file list querying whose ordering varies * by platform architecture, this class ensures the zip entries are ordered * and also supports SOURCE_DATE_EPOCH timestamps. */ -public class GenerateZip { +public class MakeZipReproducible { + String input_file = null; String fname = null; String zname = ""; long timestamp = -1L; - List files = new ArrayList<>();; boolean verbose = false; - Set entries = new LinkedHashSet<>(); + // Keep a sorted Set of ZipEntrys to be processed, so that the zip is reproducible + SortedMap entries = new TreeMap(); private boolean ok; - public GenerateZip() { + public MakeZipReproducible() { } public synchronized boolean run(String args[]) { @@ -67,26 +67,28 @@ public synchronized boolean run(String args[]) { zname = zname.substring(2); } - if (verbose) System.out.println("Files or directories to zip: "+files); + if (verbose) System.out.println("Input zip file: " + input_file); - File zipFile = new File(fname); - // Check archive to create does not exist - if (!zipFile.exists()) { - // Process Files - for(String file : files) { - Path filepath = Paths.get(file); - processFiles(filepath); - } - - try (FileOutputStream out = new FileOutputStream(fname)) { - boolean createOk = create(new BufferedOutputStream(out, 4096)); + File inFile = new File(input_file); + if (!inFile.exists()) { + error("Input zip file does not exist"); + ok = false; + } else { + File zipFile = new File(fname); + // Check archive to create does not exist + if (!zipFile.exists()) { + // Process input ZipEntries + ok = processInputEntries(inFile); if (ok) { - ok = createOk; + try (FileOutputStream out = new FileOutputStream(fname)) { + ok = create(inFile, new BufferedOutputStream(out, 4096)); + } + } else { } + } else { + error("Target zip file "+fname+" already exists."); + ok = false; } - } else { - error("Target zip file "+fname+" already exists."); - ok = false; } } catch (IOException e) { fatalError(e); @@ -126,8 +128,13 @@ boolean parseArgs(String args[]) { return false; } } else { - // file or dir to zip - files.add(args[count]); + // input zip file + if (input_file != null) { + error("Input zip file already specified"); + usageError(); + return false; + } + input_file = args[count]; } count++; } @@ -139,13 +146,13 @@ boolean parseArgs(String args[]) { return false; } if (fname == null) { - error(String.format("-f must be specified")); + error("-f must be specified"); usageError(); return false; } // If no files specified then default to current directory - if (files.size() == 0) { - error("No input directory or files were specified"); + if (input_file == null) { + error("No input zip file specified"); usageError(); return false; } @@ -153,119 +160,64 @@ boolean parseArgs(String args[]) { return true; } - // Walk tree matching files and adding to entries list - void processFiles(Path path) throws IOException { - File fpath = path.toFile(); - boolean pathIsDir = fpath.isDirectory(); - - // Keep a sorted Set of files to be processed, so that the Jmod is reproducible - // as Files.walkFileTree order is not defined - SortedMap filesToProcess = new TreeMap(); - - Files.walkFileTree(path, Set.of(FileVisitOption.FOLLOW_LINKS), - Integer.MAX_VALUE, new SimpleFileVisitor() { - @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) - throws IOException - { - Path relPath; - String name; - if (pathIsDir) { - relPath = path.relativize(file); - name = relPath.toString(); - } else { - relPath = file; - name = file.toString(); - } - filesToProcess.put(name, file); - return FileVisitResult.CONTINUE; - } - }); - - // Process files in sorted order - for (Map.Entry entry : filesToProcess.entrySet()) { - String name = entry.getKey(); - Path filepath = entry.getValue(); - - File f = filepath.toFile(); - entries.add(f); + // Process input zip file and add to sorted entries set + boolean processInputEntries(File inFile) throws IOException { + try (FileInputStream fis = new FileInputStream(inFile); + ZipInputStream zis = new ZipInputStream(fis)) { + ZipEntry entry; + while ((entry = zis.getNextEntry()) != null) { + entries.put(entry.getName(), entry); + } } + + return true; } // Create new zip from entries - boolean create(OutputStream out) throws IOException + boolean create(File inFile, OutputStream out) throws IOException { - try (ZipOutputStream zos = new ZipOutputStream(out)) { - for (File file: entries) { - addFile(zos, file); + try (ZipFile zipFile = new ZipFile(inFile); + ZipOutputStream zos = new ZipOutputStream(out)) { + for (Map.Entry entry : entries.entrySet()) { + ZipEntry zipEntry = entry.getValue(); + if (zipEntry.getSize() > 0) { + try (InputStream eis = zipFile.getInputStream(zipEntry)) { + addEntry(zos, zipEntry, eis); + } + } else { + addEntry(zos, zipEntry, null); + } } } return true; } - // Ensure a consistent entry name format - String entryName(String name) { - name = name.replace(File.separatorChar, '/'); - - if (name.startsWith("/")) { - name = name.substring(1); - } else if (name.startsWith("./")) { - name = name.substring(2); - } - return name; - } - - // Add File to Zip - void addFile(ZipOutputStream zos, File file) throws IOException { - String name = file.getPath(); - boolean isDir = file.isDirectory(); - if (isDir) { - name = name.endsWith(File.separator) ? name : (name + File.separator); - } - name = entryName(name); - - if (name.equals("") || name.equals(".") || name.equals(zname)) { - return; - } - - long size = isDir ? 0 : file.length(); - + // Add Entry and data to Zip + void addEntry(ZipOutputStream zos, ZipEntry entry, InputStream entryInputStream) throws IOException { if (verbose) { - System.out.println("Adding: "+name); + System.out.println("Adding: "+entry.getName()); } - ZipEntry e = new ZipEntry(name); - // Set to specified timestamp if set otherwise use file lastModified time + // Set to specified timestamp if set otherwise leave as original lastModified time if (timestamp != -1L) { - e.setTime(timestamp); - } else { - e.setTime(file.lastModified()); - } - if (size == 0) { - e.setMethod(ZipEntry.STORED); - e.setSize(0); - e.setCrc(0); + entry.setTime(timestamp); } - zos.putNextEntry(e); - if (!isDir) { - byte[] buf = new byte[8192]; - int len; - try (FileInputStream fis = new FileInputStream(file); - FileChannel fic = fis.getChannel()) { - fic.transferTo(0, fic.size(), Channels.newChannel(zos)); - } + + zos.putNextEntry(entry); + if (entry.getSize() > 0 && entryInputStream != null) { + entryInputStream.transferTo(zos); } zos.closeEntry(); } void usageError() { error( - "Usage: GenerateZip [-v] -f \n" + + "Usage: MakeZipReproducible [-v] [-t ] -f \n" + "Options:\n" + " -v verbose output\n" + " -f specify archive file name to create\n" + " -t specific SOURCE_DATE_EPOCH value to use for timestamps\n" + - "If any file is a directory then it is processed recursively.\n"); + " input_zip_file re-written as a reproducible zip output_zip_file.\n"); } void fatalError(Exception e) { @@ -277,7 +229,7 @@ protected void error(String s) { } public static void main(String args[]) { - GenerateZip z = new GenerateZip(); + MakeZipReproducible z = new MakeZipReproducible(); System.exit(z.run(args) ? 0 : 1); } } From ea477b9c24fa6f2aa0f23d189de43b636a709cfa Mon Sep 17 00:00:00 2001 From: Andrew Leonard Date: Fri, 12 Nov 2021 07:24:10 +0000 Subject: [PATCH 3/3] 8276743: Make openjdk build Zip Archive generation "reproducible" Signed-off-by: Andrew Leonard --- .../build/tools/makezipreproducible/MakeZipReproducible.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/make/jdk/src/classes/build/tools/makezipreproducible/MakeZipReproducible.java b/make/jdk/src/classes/build/tools/makezipreproducible/MakeZipReproducible.java index 2dac7430a304d..18931f052ee43 100644 --- a/make/jdk/src/classes/build/tools/makezipreproducible/MakeZipReproducible.java +++ b/make/jdk/src/classes/build/tools/makezipreproducible/MakeZipReproducible.java @@ -37,7 +37,7 @@ /** * Generate a zip file in a "reproducible" manner from the input zip file. - * Standard zip tools rely on OS file list querying whose ordering varies + * Standard zip tools rely on OS file list querying whose ordering can vary * by platform architecture, this class ensures the zip entries are ordered * and also supports SOURCE_DATE_EPOCH timestamps. */