Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

8277069: [REDO] JDK-8276743 Make openjdk build Zip Archive generation "reproducible" #83

Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
@@ -324,7 +324,7 @@ $(eval $(call SetupTarget, vscode-project-ccls, \
# aren't built until after libjava and libjvm are available to link to.
$(eval $(call SetupTarget, demos-jdk, \
MAKEFILE := CompileDemos, \
DEPS := java.base-libs exploded-image, \
DEPS := java.base-libs exploded-image buildtools-jdk, \
))

$(eval $(call SetupTarget, test-image-demos-jdk, \
@@ -383,12 +383,12 @@ bootcycle-images:

$(eval $(call SetupTarget, zip-security, \
MAKEFILE := ZipSecurity, \
DEPS := java.base-java java.security.jgss-java java.security.jgss-libs, \
DEPS := buildtools-jdk java.base-java java.security.jgss-java java.security.jgss-libs, \
))

$(eval $(call SetupTarget, zip-source, \
MAKEFILE := ZipSource, \
DEPS := gensrc, \
DEPS := buildtools-jdk gensrc, \
))

$(eval $(call SetupTarget, jrtfs-jar, \
@@ -508,13 +508,13 @@ $(eval $(call SetupTarget, docs-jdk-index, \
$(eval $(call SetupTarget, docs-zip, \
MAKEFILE := Docs, \
TARGET := docs-zip, \
DEPS := docs-jdk, \
DEPS := docs-jdk buildtools-jdk, \
))

$(eval $(call SetupTarget, docs-specs-zip, \
MAKEFILE := Docs, \
TARGET := docs-specs-zip, \
DEPS := docs-jdk-specs, \
DEPS := docs-jdk-specs buildtools-jdk, \
))

$(eval $(call SetupTarget, update-build-docs, \
@@ -80,6 +80,8 @@ TOOL_GENERATECACERTS = $(JAVA_SMALL) -cp $(BUILDTOOLS_OUTPUTDIR)/jdk_tools_class
TOOL_GENERATEEMOJIDATA = $(JAVA_SMALL) -cp $(BUILDTOOLS_OUTPUTDIR)/jdk_tools_classes \
build.tools.generateemojidata.GenerateEmojiData

TOOL_MAKEZIPREPRODUCIBLE = $(JAVA_SMALL) -cp $(BUILDTOOLS_OUTPUTDIR)/jdk_tools_classes \
build.tools.makezipreproducible.MakeZipReproducible

# TODO: There are references to the jdwpgen.jar in jdk/make/netbeans/jdwpgen/build.xml
# and nbproject/project.properties in the same dir. Needs to be looked at.
@@ -1,5 +1,5 @@
#
# Copyright (c) 2011, 2019, Oracle and/or its affiliates. All rights reserved.
# Copyright (c) 2011, 2021, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -26,6 +26,9 @@
ifndef _ZIP_ARCHIVE_GMK
_ZIP_ARCHIVE_GMK := 1

# Depends on build tools for MakeZipReproducible
include ../ToolsJdk.gmk

ifeq (,$(_MAKEBASE_GMK))
$(error You must include MakeBase.gmk prior to including ZipArchive.gmk)
endif
@@ -51,6 +54,8 @@ endif
# FOLLOW_SYMLINKS - Set to explicitly follow symlinks. Affects performance of
# finding files.
# ZIP_OPTIONS extra options to pass to zip
# REPRODUCIBLE override ENABLE_REPRODUCIBLE_BUILD (to make zip reproducible or not)

SetupZipArchive = $(NamedParamsMacroTemplate)
define SetupZipArchiveBody

@@ -124,6 +129,10 @@ define SetupZipArchiveBody
) \
)

ifeq ($$($1_REPRODUCIBLE), )
$1_REPRODUCIBLE := $$(ENABLE_REPRODUCIBLE_BUILD)
endif

# Use a slightly shorter name for logging, but with enough path to identify this zip.
$1_NAME:=$$(subst $$(OUTPUTDIR)/,,$$($1_ZIP))

@@ -134,6 +143,8 @@ define SetupZipArchiveBody
# dir is very small.
# If zip has nothing to do, it returns 12 and would fail the build. Check for 12
# and only fail if it's not.
# For reproducible builds set the zip access & modify times to SOURCE_DATE_EPOCH
# by using a ziptmp folder to generate final zip from using MakeZipReproducible.
$$($1_ZIP) : $$($1_ALL_SRCS) $$($1_EXTRA_DEPS)
$$(call LogWarn, Updating $$($1_NAME))
$$(call MakeTargetDir)
@@ -163,7 +174,18 @@ define SetupZipArchiveBody
$$($1_ZIP_EXCLUDES_$$s) \
|| test "$$$$?" = "12" \
))$$(NEWLINE) \
) true \
) true
ifeq ($$($1_REPRODUCIBLE), true)
$$(call ExecuteWithLog, \
$$(SUPPORT_OUTPUTDIR)/makezipreproducible/$$(patsubst $$(OUTPUTDIR)/%,%, $$@), \
($(RM) $$(SUPPORT_OUTPUTDIR)/ziptmp/$1/tmp.zip && \
$(MKDIR) -p $$(SUPPORT_OUTPUTDIR)/ziptmp/$1 && \
$(TOOL_MAKEZIPREPRODUCIBLE) -f $$(SUPPORT_OUTPUTDIR)/ziptmp/$1/tmp.zip \
-t $(SOURCE_DATE_EPOCH) $$@ && \
$(RM) $$@ && \
$(MV) $$(SUPPORT_OUTPUTDIR)/ziptmp/$1/tmp.zip $$@ \
))
endif
$(TOUCH) $$@

# Add zip to target list
@@ -0,0 +1,231 @@
/*
* Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*/

package build.tools.makezipreproducible;

import java.io.*;
import java.nio.file.*;
import java.nio.file.attribute.BasicFileAttributes;
import java.nio.channels.Channels;
import java.nio.channels.FileChannel;
import java.util.*;
import java.util.zip.ZipEntry;
import java.util.zip.ZipException;
import java.util.zip.ZipFile;
import java.util.zip.ZipInputStream;
import java.util.zip.ZipOutputStream;

/**
* Generate a zip file in a "reproducible" manner from the input zip file.
* Standard zip tools rely on OS file list querying whose ordering can vary
* by platform architecture, this class ensures the zip entries are ordered
* and also supports SOURCE_DATE_EPOCH timestamps.
*/
public class MakeZipReproducible {
String input_file = null;
String fname = null;
String zname = "";
long timestamp = -1L;
boolean verbose = false;

// Keep a sorted Set of ZipEntrys to be processed, so that the zip is reproducible
SortedMap<String, ZipEntry> entries = new TreeMap<String, ZipEntry>();

private boolean ok;

public MakeZipReproducible() {
}

public synchronized boolean run(String args[]) {
ok = true;
if (!parseArgs(args)) {
return false;
}
try {
zname = fname.replace(File.separatorChar, '/');
if (zname.startsWith("./")) {
zname = zname.substring(2);
}

if (verbose) System.out.println("Input zip file: " + input_file);

File inFile = new File(input_file);
if (!inFile.exists()) {
error("Input zip file does not exist");
ok = false;
} else {
File zipFile = new File(fname);
// Check archive to create does not exist
if (!zipFile.exists()) {
// Process input ZipEntries
ok = processInputEntries(inFile);
if (ok) {
try (FileOutputStream out = new FileOutputStream(fname)) {
ok = create(inFile, new BufferedOutputStream(out, 4096));
}
} else {
}
} else {
error("Target zip file "+fname+" already exists.");
ok = false;
}
}
} catch (IOException e) {
fatalError(e);
ok = false;
} catch (Error ee) {
ee.printStackTrace();
ok = false;
} catch (Throwable t) {
t.printStackTrace();
ok = false;
}
return ok;
}

boolean parseArgs(String args[]) {
try {
boolean parsingIncludes = false;
boolean parsingExcludes = false;
int count = 0;
while(count < args.length) {
if (args[count].startsWith("-")) {
String flag = args[count].substring(1);
switch (flag.charAt(0)) {
case 'f':
fname = args[++count];
break;
case 't':
// SOURCE_DATE_EPOCH timestamp specified
timestamp = Long.parseLong(args[++count]) * 1000;
break;
case 'v':
verbose = true;
break;
default:
error(String.format("Illegal option -%s", String.valueOf(flag.charAt(0))));
usageError();
return false;
}
} else {
// input zip file
if (input_file != null) {
error("Input zip file already specified");
usageError();
return false;
}
input_file = args[count];
}
count++;
}
} catch (ArrayIndexOutOfBoundsException e) {
usageError();
return false;
} catch (NumberFormatException e) {
usageError();
return false;
}
if (fname == null) {
error("-f <outputArchiveName> must be specified");
usageError();
return false;
}
// If no files specified then default to current directory
if (input_file == null) {
error("No input zip file specified");
usageError();
return false;
}

return true;
}

// Process input zip file and add to sorted entries set
boolean processInputEntries(File inFile) throws IOException {
ZipFile zipFile = new ZipFile(inFile);
zipFile.stream().forEach(entry -> entries.put(entry.getName(), entry));

return true;
}

// Create new zip from entries
boolean create(File inFile, OutputStream out) throws IOException
{
try (ZipFile zipFile = new ZipFile(inFile);
ZipOutputStream zos = new ZipOutputStream(out)) {
for (Map.Entry<String, ZipEntry> entry : entries.entrySet()) {
ZipEntry zipEntry = entry.getValue();
if (zipEntry.getSize() > 0) {
try (InputStream eis = zipFile.getInputStream(zipEntry)) {
addEntry(zos, zipEntry, eis);
}
} else {
addEntry(zos, zipEntry, null);
}
}
}
return true;
}

// Add Entry and data to Zip
void addEntry(ZipOutputStream zos, ZipEntry entry, InputStream entryInputStream) throws IOException {
if (verbose) {
System.out.println("Adding: "+entry.getName());
}

// Set to specified timestamp if set otherwise leave as original lastModified time
if (timestamp != -1L) {
entry.setTime(timestamp);
}

zos.putNextEntry(entry);
if (entry.getSize() > 0 && entryInputStream != null) {
entryInputStream.transferTo(zos);
}
zos.closeEntry();
}

void usageError() {
error(
"Usage: MakeZipReproducible [-v] [-t <SOURCE_DATE_EPOCH>] -f <output_zip_file> <input_zip_file>\n" +
"Options:\n" +
" -v verbose output\n" +
" -f specify archive file name to create\n" +
" -t specific SOURCE_DATE_EPOCH value to use for timestamps\n" +
" input_zip_file re-written as a reproducible zip output_zip_file.\n");
}

void fatalError(Exception e) {
e.printStackTrace();
}

protected void error(String s) {
System.err.println(s);
}

public static void main(String args[]) {
MakeZipReproducible z = new MakeZipReproducible();
System.exit(z.run(args) ? 0 : 1);
}
}