Skip to content
Permalink
Browse files
8277069: [REDO] JDK-8276743 Make openjdk build Zip Archive generation…
… "reproducible"

Co-authored-by: Andrew Leonard <aleonard@openjdk.org>
Co-authored-by: Magnus Ihse Bursie <ihse@openjdk.org>
Reviewed-by: erikj
  • Loading branch information
magicus and Andrew Leonard committed Dec 2, 2021
1 parent b8ac0d2 commit c93552c8bbcdabb6219327d67409bf63432f49d8
Show file tree
Hide file tree
Showing 4 changed files with 262 additions and 7 deletions.
@@ -324,7 +324,7 @@ $(eval $(call SetupTarget, vscode-project-ccls, \
# aren't built until after libjava and libjvm are available to link to.
$(eval $(call SetupTarget, demos-jdk, \
MAKEFILE := CompileDemos, \
DEPS := java.base-libs exploded-image, \
DEPS := java.base-libs exploded-image buildtools-jdk, \
))

$(eval $(call SetupTarget, test-image-demos-jdk, \
@@ -383,12 +383,12 @@ bootcycle-images:

$(eval $(call SetupTarget, zip-security, \
MAKEFILE := ZipSecurity, \
DEPS := java.base-java java.security.jgss-java java.security.jgss-libs, \
DEPS := buildtools-jdk java.base-java java.security.jgss-java java.security.jgss-libs, \
))

$(eval $(call SetupTarget, zip-source, \
MAKEFILE := ZipSource, \
DEPS := gensrc, \
DEPS := buildtools-jdk gensrc, \
))

$(eval $(call SetupTarget, jrtfs-jar, \
@@ -508,13 +508,13 @@ $(eval $(call SetupTarget, docs-jdk-index, \
$(eval $(call SetupTarget, docs-zip, \
MAKEFILE := Docs, \
TARGET := docs-zip, \
DEPS := docs-jdk, \
DEPS := docs-jdk buildtools-jdk, \
))

$(eval $(call SetupTarget, docs-specs-zip, \
MAKEFILE := Docs, \
TARGET := docs-specs-zip, \
DEPS := docs-jdk-specs, \
DEPS := docs-jdk-specs buildtools-jdk, \
))

$(eval $(call SetupTarget, update-build-docs, \
@@ -82,6 +82,8 @@ TOOL_GENERATECACERTS = $(JAVA_SMALL) -cp $(BUILDTOOLS_OUTPUTDIR)/jdk_tools_class
TOOL_GENERATEEMOJIDATA = $(JAVA_SMALL) -cp $(BUILDTOOLS_OUTPUTDIR)/jdk_tools_classes \
build.tools.generateemojidata.GenerateEmojiData

TOOL_MAKEZIPREPRODUCIBLE = $(JAVA_SMALL) -cp $(BUILDTOOLS_OUTPUTDIR)/jdk_tools_classes \
build.tools.makezipreproducible.MakeZipReproducible

# TODO: There are references to the jdwpgen.jar in jdk/make/netbeans/jdwpgen/build.xml
# and nbproject/project.properties in the same dir. Needs to be looked at.
@@ -1,5 +1,5 @@
#
# Copyright (c) 2011, 2019, Oracle and/or its affiliates. All rights reserved.
# Copyright (c) 2011, 2021, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -26,6 +26,9 @@
ifndef _ZIP_ARCHIVE_GMK
_ZIP_ARCHIVE_GMK := 1

# Depends on build tools for MakeZipReproducible
include ../ToolsJdk.gmk

ifeq (,$(_MAKEBASE_GMK))
$(error You must include MakeBase.gmk prior to including ZipArchive.gmk)
endif
@@ -51,6 +54,8 @@ endif
# FOLLOW_SYMLINKS - Set to explicitly follow symlinks. Affects performance of
# finding files.
# ZIP_OPTIONS extra options to pass to zip
# REPRODUCIBLE override ENABLE_REPRODUCIBLE_BUILD (to make zip reproducible or not)

SetupZipArchive = $(NamedParamsMacroTemplate)
define SetupZipArchiveBody

@@ -124,6 +129,10 @@ define SetupZipArchiveBody
) \
)

ifeq ($$($1_REPRODUCIBLE), )
$1_REPRODUCIBLE := $$(ENABLE_REPRODUCIBLE_BUILD)
endif

# Use a slightly shorter name for logging, but with enough path to identify this zip.
$1_NAME:=$$(subst $$(OUTPUTDIR)/,,$$($1_ZIP))

@@ -134,6 +143,8 @@ define SetupZipArchiveBody
# dir is very small.
# If zip has nothing to do, it returns 12 and would fail the build. Check for 12
# and only fail if it's not.
# For reproducible builds set the zip access & modify times to SOURCE_DATE_EPOCH
# by using a ziptmp folder to generate final zip from using MakeZipReproducible.
$$($1_ZIP) : $$($1_ALL_SRCS) $$($1_EXTRA_DEPS)
$$(call LogWarn, Updating $$($1_NAME))
$$(call MakeTargetDir)
@@ -163,7 +174,18 @@ define SetupZipArchiveBody
$$($1_ZIP_EXCLUDES_$$s) \
|| test "$$$$?" = "12" \
))$$(NEWLINE) \
) true \
) true
ifeq ($$($1_REPRODUCIBLE), true)
$$(call ExecuteWithLog, \
$$(SUPPORT_OUTPUTDIR)/makezipreproducible/$$(patsubst $$(OUTPUTDIR)/%,%, $$@), \
($(RM) $$(SUPPORT_OUTPUTDIR)/ziptmp/$1/tmp.zip && \
$(MKDIR) -p $$(SUPPORT_OUTPUTDIR)/ziptmp/$1 && \
$(TOOL_MAKEZIPREPRODUCIBLE) -f $$(SUPPORT_OUTPUTDIR)/ziptmp/$1/tmp.zip \
-t $(SOURCE_DATE_EPOCH) $$@ && \
$(RM) $$@ && \
$(MV) $$(SUPPORT_OUTPUTDIR)/ziptmp/$1/tmp.zip $$@ \
))
endif
$(TOUCH) $$@

# Add zip to target list
@@ -0,0 +1,231 @@
/*
* Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*/

package build.tools.makezipreproducible;

import java.io.*;
import java.nio.file.*;
import java.nio.file.attribute.BasicFileAttributes;
import java.nio.channels.Channels;
import java.nio.channels.FileChannel;
import java.util.*;
import java.util.zip.ZipEntry;
import java.util.zip.ZipException;
import java.util.zip.ZipFile;
import java.util.zip.ZipInputStream;
import java.util.zip.ZipOutputStream;

/**
* Generate a zip file in a "reproducible" manner from the input zip file.
* Standard zip tools rely on OS file list querying whose ordering can vary
* by platform architecture, this class ensures the zip entries are ordered
* and also supports SOURCE_DATE_EPOCH timestamps.
*/
public class MakeZipReproducible {
String input_file = null;
String fname = null;
String zname = "";
long timestamp = -1L;
boolean verbose = false;

// Keep a sorted Set of ZipEntrys to be processed, so that the zip is reproducible
SortedMap<String, ZipEntry> entries = new TreeMap<String, ZipEntry>();

private boolean ok;

public MakeZipReproducible() {
}

public synchronized boolean run(String args[]) {
ok = true;
if (!parseArgs(args)) {
return false;
}
try {
zname = fname.replace(File.separatorChar, '/');
if (zname.startsWith("./")) {
zname = zname.substring(2);
}

if (verbose) System.out.println("Input zip file: " + input_file);

File inFile = new File(input_file);
if (!inFile.exists()) {
error("Input zip file does not exist");
ok = false;
} else {
File zipFile = new File(fname);
// Check archive to create does not exist
if (!zipFile.exists()) {
// Process input ZipEntries
ok = processInputEntries(inFile);
if (ok) {
try (FileOutputStream out = new FileOutputStream(fname)) {
ok = create(inFile, new BufferedOutputStream(out, 4096));
}
} else {
}
} else {
error("Target zip file "+fname+" already exists.");
ok = false;
}
}
} catch (IOException e) {
fatalError(e);
ok = false;
} catch (Error ee) {
ee.printStackTrace();
ok = false;
} catch (Throwable t) {
t.printStackTrace();
ok = false;
}
return ok;
}

boolean parseArgs(String args[]) {
try {
boolean parsingIncludes = false;
boolean parsingExcludes = false;
int count = 0;
while(count < args.length) {
if (args[count].startsWith("-")) {
String flag = args[count].substring(1);
switch (flag.charAt(0)) {
case 'f':
fname = args[++count];
break;
case 't':
// SOURCE_DATE_EPOCH timestamp specified
timestamp = Long.parseLong(args[++count]) * 1000;
break;
case 'v':
verbose = true;
break;
default:
error(String.format("Illegal option -%s", String.valueOf(flag.charAt(0))));
usageError();
return false;
}
} else {
// input zip file
if (input_file != null) {
error("Input zip file already specified");
usageError();
return false;
}
input_file = args[count];
}
count++;
}
} catch (ArrayIndexOutOfBoundsException e) {
usageError();
return false;
} catch (NumberFormatException e) {
usageError();
return false;
}
if (fname == null) {
error("-f <outputArchiveName> must be specified");
usageError();
return false;
}
// If no files specified then default to current directory
if (input_file == null) {
error("No input zip file specified");
usageError();
return false;
}

return true;
}

// Process input zip file and add to sorted entries set
boolean processInputEntries(File inFile) throws IOException {
ZipFile zipFile = new ZipFile(inFile);
zipFile.stream().forEach(entry -> entries.put(entry.getName(), entry));

return true;
}

// Create new zip from entries
boolean create(File inFile, OutputStream out) throws IOException
{
try (ZipFile zipFile = new ZipFile(inFile);
ZipOutputStream zos = new ZipOutputStream(out)) {
for (Map.Entry<String, ZipEntry> entry : entries.entrySet()) {
ZipEntry zipEntry = entry.getValue();
if (zipEntry.getSize() > 0) {
try (InputStream eis = zipFile.getInputStream(zipEntry)) {
addEntry(zos, zipEntry, eis);
}
} else {
addEntry(zos, zipEntry, null);
}
}
}
return true;
}

// Add Entry and data to Zip
void addEntry(ZipOutputStream zos, ZipEntry entry, InputStream entryInputStream) throws IOException {
if (verbose) {
System.out.println("Adding: "+entry.getName());
}

// Set to specified timestamp if set otherwise leave as original lastModified time
if (timestamp != -1L) {
entry.setTime(timestamp);
}

zos.putNextEntry(entry);
if (entry.getSize() > 0 && entryInputStream != null) {
entryInputStream.transferTo(zos);
}
zos.closeEntry();
}

void usageError() {
error(
"Usage: MakeZipReproducible [-v] [-t <SOURCE_DATE_EPOCH>] -f <output_zip_file> <input_zip_file>\n" +
"Options:\n" +
" -v verbose output\n" +
" -f specify archive file name to create\n" +
" -t specific SOURCE_DATE_EPOCH value to use for timestamps\n" +
" input_zip_file re-written as a reproducible zip output_zip_file.\n");
}

void fatalError(Exception e) {
e.printStackTrace();
}

protected void error(String s) {
System.err.println(s);
}

public static void main(String args[]) {
MakeZipReproducible z = new MakeZipReproducible();
System.exit(z.run(args) ? 0 : 1);
}
}

3 comments on commit c93552c

@openjdk-notifier
Copy link

@openjdk-notifier openjdk-notifier bot commented on c93552c Dec 2, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@andrew-m-leonard
Copy link

@andrew-m-leonard andrew-m-leonard commented on c93552c Jan 12, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@openjdk
Copy link

@openjdk openjdk bot commented on c93552c Jan 12, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@andrew-m-leonard the backport was successfully created on the branch andrew-m-leonard-backport-c93552c8 in my personal fork of openjdk/jdk17u-dev. To create a pull request with this backport targeting openjdk/jdk17u-dev:master, just click the following link:

➡️ Create pull request

The title of the pull request is automatically filled in correctly and below you find a suggestion for the pull request body:

Hi all,

This pull request contains a backport of commit c93552c8 from the openjdk/jdk repository.

The commit being backported was authored by Magnus Ihse Bursie on 2 Dec 2021 and was reviewed by Erik Joelsson.

Thanks!

If you need to update the source branch of the pull then run the following commands in a local clone of your personal fork of openjdk/jdk17u-dev:

$ git fetch https://github.com/openjdk-bots/jdk17u-dev andrew-m-leonard-backport-c93552c8:andrew-m-leonard-backport-c93552c8
$ git checkout andrew-m-leonard-backport-c93552c8
# make changes
$ git add paths/to/changed/files
$ git commit --message 'Describe additional changes made'
$ git push https://github.com/openjdk-bots/jdk17u-dev andrew-m-leonard-backport-c93552c8

Please sign in to comment.