From 3e87db19bb7e2359e4b1ceaac7c123542345ecb1 Mon Sep 17 00:00:00 2001 From: Andrew Leonard Date: Mon, 15 Nov 2021 15:34:03 +0000 Subject: [PATCH 1/8] 8276764: Enable deterministic file content ordering for Jar and Jmod Signed-off-by: Andrew Leonard --- .../share/classes/sun/tools/jar/Main.java | 13 +- .../classes/jdk/tools/jmod/JmodTask.java | 19 +- test/jdk/tools/jar/ContentOrder.java | 170 ++++++++++++++++++ test/jdk/tools/jmod/JmodTest.java | 15 +- 4 files changed, 207 insertions(+), 10 deletions(-) create mode 100644 test/jdk/tools/jar/ContentOrder.java diff --git a/src/jdk.jartool/share/classes/sun/tools/jar/Main.java b/src/jdk.jartool/share/classes/sun/tools/jar/Main.java index 4458cd00b445e..8b6634ff715f5 100644 --- a/src/jdk.jartool/share/classes/sun/tools/jar/Main.java +++ b/src/jdk.jartool/share/classes/sun/tools/jar/Main.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 1996, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1996, 2021, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -127,7 +127,8 @@ public int hashCode() { Map> pathsMap = new HashMap<>(); // There's also a files array per version - Map filesMap = new HashMap<>(); + // Use a LinkedHashMap to keep original insertion ordering + Map filesMap = new LinkedHashMap<>(); // Do we think this is a multi-release jar? Set to true // if --release option found followed by at least file @@ -813,7 +814,7 @@ private void expand(File dir, String[] files, Set cpaths, int version) if (entries.add(e)) { // utilize entryMap for the duplicate dir check even in // case of cflag == true. - // dir name confilict/duplicate could happen with -C option. + // dir name conflict/duplicate could happen with -C option. // just remove the last "e" from the "entries" (zos will fail // with "duplicated" entries), but continue expanding the // sub tree @@ -822,7 +823,11 @@ private void expand(File dir, String[] files, Set cpaths, int version) } else { entryMap.put(name, e); } - expand(f, f.list(), cpaths, version); + String[] dirFiles = f.list(); + // Ensure files list is sorted for reproducible jar content + if (dirFiles != null) + Arrays.sort(dirFiles); + expand(f, dirFiles, cpaths, version); } } else { error(formatMsg("error.nosuch.fileordir", String.valueOf(f))); diff --git a/src/jdk.jlink/share/classes/jdk/tools/jmod/JmodTask.java b/src/jdk.jlink/share/classes/jdk/tools/jmod/JmodTask.java index eb541bba63fb2..d8be60d0f57d4 100644 --- a/src/jdk.jlink/share/classes/jdk/tools/jmod/JmodTask.java +++ b/src/jdk.jlink/share/classes/jdk/tools/jmod/JmodTask.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -767,6 +767,10 @@ void processSection(JmodOutputStream out, Section section, List paths) void processSection(JmodOutputStream out, Section section, Path path) throws IOException { + // Keep a sorted set of files to be processed, so that the jmod + // content is reproducible as Files.walkFileTree order is not defined + SortedMap filesToProcess = new TreeMap(); + Files.walkFileTree(path, Set.of(FileVisitOption.FOLLOW_LINKS), Integer.MAX_VALUE, new SimpleFileVisitor() { @Override @@ -782,14 +786,21 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) if (out.contains(section, name)) { warning("warn.ignore.duplicate.entry", name, section); } else { - try (InputStream in = Files.newInputStream(file)) { - out.writeEntry(in, section, name); - } + filesToProcess.put(name, file); } } return FileVisitResult.CONTINUE; } }); + + // Process files in sorted order for deterministic jmod content + for (Map.Entry entry : filesToProcess.entrySet()) { + String name = entry.getKey(); + Path file = entry.getValue(); + try (InputStream in = Files.newInputStream(file)) { + out.writeEntry(in, section, name); + } + } } boolean matches(Path path, List matchers) { diff --git a/test/jdk/tools/jar/ContentOrder.java b/test/jdk/tools/jar/ContentOrder.java new file mode 100644 index 0000000000000..31d52e3ba4326 --- /dev/null +++ b/test/jdk/tools/jar/ContentOrder.java @@ -0,0 +1,170 @@ +/* + * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +/* + * @test + * @bug 8276764 + * @summary test that the jar content ordering is sorted + * @library /test/lib + * @modules jdk.jartool + * @build jdk.test.lib.Platform + * jdk.test.lib.util.FileUtils + * @run testng ContentOrder + */ + +import org.testng.Assert; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.PrintStream; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.spi.ToolProvider; +import java.util.stream.Stream; +import java.util.zip.ZipException; + +import jdk.test.lib.util.FileUtils; + +public class ContentOrder { + private static final ToolProvider JAR_TOOL = ToolProvider.findFirst("jar") + .orElseThrow(() -> + new RuntimeException("jar tool not found") + ); + + private final String nl = System.lineSeparator(); + private final ByteArrayOutputStream baos = new ByteArrayOutputStream(); + private final PrintStream out = new PrintStream(baos); + private Runnable onCompletion; + + @BeforeMethod + public void reset() { + onCompletion = null; + } + + @AfterMethod + public void run() { + if (onCompletion != null) { + onCompletion.run(); + } + } + + @Test + public void test1() throws IOException { + mkdir("testjar/Ctest1 testjar/Btest2/subdir1 testjar/Atest3"); + touch("testjar/Ctest1/testfile1 testjar/Ctest1/testfile2 testjar/Ctest1/testfile3"); + touch("testjar/Btest2/subdir1/testfileC testjar/Btest2/subdir1/testfileB testjar/Btest2/subdir1/testfileA"); + touch("testjar/Atest3/fileZ testjar/Atest3/fileY testjar/Atest3/fileX"); + jar("cf test.jar testjar"); + jar("tf test.jar"); + println(); + String output = "META-INF/" + nl + + "META-INF/MANIFEST.MF" + nl + + "testjar/" + nl + + "testjar/Atest3/" + nl + + "testjar/Atest3/fileX" + nl + + "testjar/Atest3/fileY" + nl + + "testjar/Atest3/fileZ" + nl + + "testjar/Btest2/" + nl + + "testjar/Btest2/subdir1/" + nl + + "testjar/Btest2/subdir1/testfileA" + nl + + "testjar/Btest2/subdir1/testfileB" + nl + + "testjar/Btest2/subdir1/testfileC" + nl + + "testjar/Ctest1/" + nl + + "testjar/Ctest1/testfile1" + nl + + "testjar/Ctest1/testfile2" + nl + + "testjar/Ctest1/testfile3" + nl; + rm("test.jar testjar"); + Assert.assertEquals(baos.toByteArray(), output.getBytes()); + } + + private Stream mkpath(String... args) { + return Arrays.stream(args).map(d -> Paths.get(".", d.split("/"))); + } + + private void mkdir(String cmdline) { + System.out.println("mkdir -p " + cmdline); + mkpath(cmdline.split(" +")).forEach(p -> { + try { + Files.createDirectories(p); + } catch (IOException x) { + throw new UncheckedIOException(x); + } + }); + } + + private void touch(String cmdline) { + System.out.println("touch " + cmdline); + mkpath(cmdline.split(" +")).forEach(p -> { + try { + Files.createFile(p); + } catch (IOException x) { + throw new UncheckedIOException(x); + } + }); + } + + private void rm(String cmdline) { + System.out.println("rm -rf " + cmdline); + mkpath(cmdline.split(" +")).forEach(p -> { + try { + if (Files.isDirectory(p)) { + FileUtils.deleteFileTreeWithRetry(p); + } else { + FileUtils.deleteFileIfExistsWithRetry(p); + } + } catch (IOException x) { + throw new UncheckedIOException(x); + } + }); + } + + private void jar(String cmdline) throws IOException { + System.out.println("jar " + cmdline); + baos.reset(); + + // the run method catches IOExceptions, we need to expose them + ByteArrayOutputStream baes = new ByteArrayOutputStream(); + PrintStream err = new PrintStream(baes); + PrintStream saveErr = System.err; + System.setErr(err); + int rc = JAR_TOOL.run(out, err, cmdline.split(" +")); + System.setErr(saveErr); + if (rc != 0) { + String s = baes.toString(); + if (s.startsWith("java.util.zip.ZipException: duplicate entry: ")) { + throw new ZipException(s); + } + throw new IOException(s); + } + } + + private void println() throws IOException { + System.out.println(new String(baos.toByteArray())); + } +} diff --git a/test/jdk/tools/jmod/JmodTest.java b/test/jdk/tools/jmod/JmodTest.java index c32284faee652..bfec68b748beb 100644 --- a/test/jdk/tools/jmod/JmodTest.java +++ b/test/jdk/tools/jmod/JmodTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -23,7 +23,7 @@ /* * @test - * @bug 8142968 8166568 8166286 8170618 8168149 8240910 + * @bug 8142968 8166568 8166286 8170618 8168149 8240910 8276764 * @summary Basic test for jmod * @library /test/lib * @modules jdk.compiler @@ -197,6 +197,17 @@ public void testList() throws IOException { assertContains(r.output, CLASSES_PREFIX + "jdk/test/foo/Foo.class"); assertContains(r.output, CLASSES_PREFIX + "jdk/test/foo/internal/Message.class"); assertContains(r.output, CLASSES_PREFIX + "jdk/test/foo/resources/foo.properties"); + + // JDK-8276764: Ensure the order is sorted for reproducible jmod content + // module-info, followed by + int mod_info_i = r.output.indexOf(CLASSES_PREFIX + "module-info.class"); + int foo_cls_i = r.output.indexOf(CLASSES_PREFIX + "jdk/test/foo/Foo.class"); + int msg_i = r.output.indexOf(CLASSES_PREFIX + "jdk/test/foo/internal/Message.class"); + int res_i = r.output.indexOf(CLASSES_PREFIX + "jdk/test/foo/resources/foo.properties"); + System.out.println("jmod classes sort order check:\n"+r.output); + assertTrue(mod_info_i < foo_cls_i); + assertTrue(foo_cls_i < msg_i); + assertTrue(msg_i < res_i); }); } From 46f25d3e2e0bd12de475de24d81a049c28ba2a80 Mon Sep 17 00:00:00 2001 From: Andrew Leonard Date: Fri, 19 Nov 2021 09:49:08 +0000 Subject: [PATCH 2/8] 8276764: Enable deterministic file content ordering for Jar and Jmod Signed-off-by: Andrew Leonard --- .../share/classes/sun/tools/jar/Main.java | 21 +- test/jdk/tools/jar/ContentOrder.java | 42 ++-- test/jdk/tools/jar/CreateJarBenchmark.java | 189 ++++++++++++++++++ 3 files changed, 226 insertions(+), 26 deletions(-) create mode 100644 test/jdk/tools/jar/CreateJarBenchmark.java diff --git a/src/jdk.jartool/share/classes/sun/tools/jar/Main.java b/src/jdk.jartool/share/classes/sun/tools/jar/Main.java index 8b6634ff715f5..9058ba1d7c1b3 100644 --- a/src/jdk.jartool/share/classes/sun/tools/jar/Main.java +++ b/src/jdk.jartool/share/classes/sun/tools/jar/Main.java @@ -127,7 +127,9 @@ public int hashCode() { Map> pathsMap = new HashMap<>(); // There's also a files array per version - // Use a LinkedHashMap to keep original insertion ordering + // filesMap key entries are created in the order the versions are + // specified on the cmd line, hence LinkedHashMap keeps that order. + // String[] values for each version are also in cmd line order. Map filesMap = new LinkedHashMap<>(); // Do we think this is a multi-release jar? Set to true @@ -773,15 +775,17 @@ private void expand() throws IOException { private void expand(File dir, String[] files, Set cpaths, int version) throws IOException { - if (files == null) + if (files == null) { return; + } for (int i = 0; i < files.length; i++) { File f; - if (dir == null) + if (dir == null) { f = new File(files[i]); - else + } else { f = new File(dir, files[i]); + } boolean isDir = f.isDirectory(); String name = toEntryName(f.getPath(), cpaths, isDir); @@ -803,11 +807,13 @@ private void expand(File dir, String[] files, Set cpaths, int version) Entry e = new Entry(f, name, false); if (isModuleInfoEntry(name)) { moduleInfos.putIfAbsent(name, Files.readAllBytes(f.toPath())); - if (uflag) + if (uflag) { entryMap.put(name, e); + } } else if (entries.add(e)) { - if (uflag) + if (uflag) { entryMap.put(name, e); + } } } else if (isDir) { Entry e = new Entry(f, name, true); @@ -825,8 +831,9 @@ private void expand(File dir, String[] files, Set cpaths, int version) } String[] dirFiles = f.list(); // Ensure files list is sorted for reproducible jar content - if (dirFiles != null) + if (dirFiles != null) { Arrays.sort(dirFiles); + } expand(f, dirFiles, cpaths, version); } } else { diff --git a/test/jdk/tools/jar/ContentOrder.java b/test/jdk/tools/jar/ContentOrder.java index 31d52e3ba4326..a5f0f31f606dc 100644 --- a/test/jdk/tools/jar/ContentOrder.java +++ b/test/jdk/tools/jar/ContentOrder.java @@ -41,6 +41,7 @@ import java.io.IOException; import java.io.PrintStream; import java.io.UncheckedIOException; +import java.io.File; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -76,10 +77,13 @@ public void run() { @Test public void test1() throws IOException { - mkdir("testjar/Ctest1 testjar/Btest2/subdir1 testjar/Atest3"); - touch("testjar/Ctest1/testfile1 testjar/Ctest1/testfile2 testjar/Ctest1/testfile3"); - touch("testjar/Btest2/subdir1/testfileC testjar/Btest2/subdir1/testfileB testjar/Btest2/subdir1/testfileA"); - touch("testjar/Atest3/fileZ testjar/Atest3/fileY testjar/Atest3/fileX"); + mkdir("testjar/Ctest1", "testjar/Btest2/subdir1", "testjar/Atest3"); + touch("testjar/Ctest1/testfile1", "testjar/Ctest1/testfile2", "testjar/Ctest1/testfile3"); + touch("testjar/Btest2/subdir1/testfileC", "testjar/Btest2/subdir1/testfileB", "testjar/Btest2/subdir1/testfileA"); + touch("testjar/Atest3/fileZ", "testjar/Atest3/fileY", "testjar/Atest3/fileX"); + + onCompletion = () -> rm("test.jar", "testjar"); + jar("cf test.jar testjar"); jar("tf test.jar"); println(); @@ -99,7 +103,6 @@ public void test1() throws IOException { "testjar/Ctest1/testfile1" + nl + "testjar/Ctest1/testfile2" + nl + "testjar/Ctest1/testfile3" + nl; - rm("test.jar testjar"); Assert.assertEquals(baos.toByteArray(), output.getBytes()); } @@ -107,36 +110,37 @@ private Stream mkpath(String... args) { return Arrays.stream(args).map(d -> Paths.get(".", d.split("/"))); } - private void mkdir(String cmdline) { - System.out.println("mkdir -p " + cmdline); - mkpath(cmdline.split(" +")).forEach(p -> { + private void mkdir(String... dirs) { + System.out.println("mkdir -p " + Arrays.toString(dirs)); + Arrays.stream(dirs).forEach(p -> { try { - Files.createDirectories(p); + Files.createDirectories((new File(p)).toPath()); } catch (IOException x) { throw new UncheckedIOException(x); } }); } - private void touch(String cmdline) { - System.out.println("touch " + cmdline); - mkpath(cmdline.split(" +")).forEach(p -> { + private void touch(String... files) { + System.out.println("touch " + Arrays.toString(files)); + Arrays.stream(files).forEach(p -> { try { - Files.createFile(p); + Files.createFile((new File(p)).toPath()); } catch (IOException x) { throw new UncheckedIOException(x); } }); } - private void rm(String cmdline) { - System.out.println("rm -rf " + cmdline); - mkpath(cmdline.split(" +")).forEach(p -> { + private void rm(String... files) { + System.out.println("rm -rf " + Arrays.toString(files)); + Arrays.stream(files).forEach(p -> { try { - if (Files.isDirectory(p)) { - FileUtils.deleteFileTreeWithRetry(p); + Path path = (new File(p)).toPath(); + if (Files.isDirectory(path)) { + FileUtils.deleteFileTreeWithRetry(path); } else { - FileUtils.deleteFileIfExistsWithRetry(p); + FileUtils.deleteFileIfExistsWithRetry(path); } } catch (IOException x) { throw new UncheckedIOException(x); diff --git a/test/jdk/tools/jar/CreateJarBenchmark.java b/test/jdk/tools/jar/CreateJarBenchmark.java new file mode 100644 index 0000000000000..4ced0fa101a36 --- /dev/null +++ b/test/jdk/tools/jar/CreateJarBenchmark.java @@ -0,0 +1,189 @@ +/* + * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +/* + * @test + * @bug 8276764 + * @summary perform a jar creation benchmark + * @library /test/lib + * @modules jdk.jartool + * @build jdk.test.lib.Platform + * jdk.test.lib.util.FileUtils + * @run testng CreateJarBenchmark + */ + +import org.testng.Assert; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.PrintStream; +import java.io.UncheckedIOException; +import java.io.File; +import java.io.FileOutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.spi.ToolProvider; +import java.util.stream.Stream; +import java.util.zip.ZipException; + +import jdk.test.lib.util.FileUtils; + +public class CreateJarBenchmark { + private static final ToolProvider JAR_TOOL = ToolProvider.findFirst("jar") + .orElseThrow(() -> + new RuntimeException("jar tool not found") + ); + + private final ByteArrayOutputStream baos = new ByteArrayOutputStream(); + private final PrintStream out = new PrintStream(baos); + private Runnable onCompletion; + + @BeforeMethod + public void reset() { + onCompletion = null; + } + + @AfterMethod + public void run() { + if (onCompletion != null) { + onCompletion.run(); + } + } + + @Test + public void testSingleDir() throws IOException { + // Create a single testjar directory containing 10000 files + mkdir("testjar"); + for(int i = 0; i < 10000; i++) { + createFile("testjar/testfile"+i); + } + + onCompletion = () -> rm("test.jar", "testjar"); + + // Perform 100x jar creations + long start = System.currentTimeMillis(); + for(int i = 0; i < 100; i++) { + jar("cf test.jar testjar"); + rm("test.jar"); + } + long finish = System.currentTimeMillis(); + + System.out.println("single directory jar creation benchmark = " + (finish-start) + "ms"); + } + + @Test + public void testMultiDir() throws IOException { + // Create a nested 10x20 set of sub-dirs each containing 50 files + mkdir("testjar"); + for(int i = 0; i < 10; i++) { + mkdir("testjar/testdir" + i); + for(int j = 0; j < 20; j++) { + mkdir("testjar/testdir" + i + "/subdir" + j); + for(int k = 0; k < 50; k++) { + createFile("testjar/testdir" + i + "/subdir" + j + "/testfile" + k); + } + } + } + + onCompletion = () -> rm("test.jar", "testjar"); + + // Perform 100x jar creations + long start = System.currentTimeMillis(); + for(int i = 0; i < 100; i++) { + jar("cf test.jar testjar"); + rm("test.jar"); + } + long finish = System.currentTimeMillis(); + + System.out.println("multi directory jar creation benchmark = " + (finish-start) + "ms"); + } + + private Stream mkpath(String... args) { + return Arrays.stream(args).map(d -> Paths.get(".", d.split("/"))); + } + + private void mkdir(String... dirs) { + Arrays.stream(dirs).forEach(p -> { + try { + Files.createDirectories((new File(p)).toPath()); + } catch (IOException x) { + throw new UncheckedIOException(x); + } + }); + } + + private void createFile(String... files) { + Arrays.stream(files).forEach(p -> { + try { + try (FileOutputStream fos = new FileOutputStream(p)) { + // Create file with fixed content + byte[] bytes = new byte[10000]; + Arrays.fill(bytes, (byte)0x41); + fos.write(bytes); + } + } catch (IOException x) { + throw new UncheckedIOException(x); + } + }); + } + + private void rm(String... files) { + Arrays.stream(files).forEach(p -> { + try { + Path path = (new File(p)).toPath(); + if (Files.isDirectory(path)) { + FileUtils.deleteFileTreeWithRetry(path); + } else { + FileUtils.deleteFileIfExistsWithRetry(path); + } + } catch (IOException x) { + throw new UncheckedIOException(x); + } + }); + } + + private void jar(String cmdline) throws IOException { + System.out.println("jar " + cmdline); + baos.reset(); + + // the run method catches IOExceptions, we need to expose them + ByteArrayOutputStream baes = new ByteArrayOutputStream(); + PrintStream err = new PrintStream(baes); + PrintStream saveErr = System.err; + System.setErr(err); + int rc = JAR_TOOL.run(out, err, cmdline.split(" +")); + System.setErr(saveErr); + if (rc != 0) { + String s = baes.toString(); + if (s.startsWith("java.util.zip.ZipException: duplicate entry: ")) { + throw new ZipException(s); + } + throw new IOException(s); + } + } +} From 772b89a459b43b42599b67bb3299395952d4e38f Mon Sep 17 00:00:00 2001 From: Andrew Leonard Date: Fri, 19 Nov 2021 09:53:01 +0000 Subject: [PATCH 3/8] 8276764: Enable deterministic file content ordering for Jar and Jmod Signed-off-by: Andrew Leonard --- src/jdk.jartool/share/classes/sun/tools/jar/Main.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/jdk.jartool/share/classes/sun/tools/jar/Main.java b/src/jdk.jartool/share/classes/sun/tools/jar/Main.java index 9058ba1d7c1b3..9e0db1834f555 100644 --- a/src/jdk.jartool/share/classes/sun/tools/jar/Main.java +++ b/src/jdk.jartool/share/classes/sun/tools/jar/Main.java @@ -127,9 +127,9 @@ public int hashCode() { Map> pathsMap = new HashMap<>(); // There's also a files array per version - // filesMap key entries are created in the order the versions are - // specified on the cmd line, hence LinkedHashMap keeps that order. - // String[] values for each version are also in cmd line order. + // base version is the first entry and then follow with the version given + // from the --release option in the command-line order. + // The value of each entry is the files given in the command-line order. Map filesMap = new LinkedHashMap<>(); // Do we think this is a multi-release jar? Set to true From c8d6e90d1e00fc4a161322be1fba98b093eabe52 Mon Sep 17 00:00:00 2001 From: Andrew Leonard Date: Fri, 19 Nov 2021 10:05:06 +0000 Subject: [PATCH 4/8] 8276764: Enable deterministic file content ordering for Jar and Jmod Signed-off-by: Andrew Leonard --- src/jdk.jartool/share/classes/sun/tools/jar/Main.java | 2 +- test/jdk/tools/jar/CreateJarBenchmark.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/jdk.jartool/share/classes/sun/tools/jar/Main.java b/src/jdk.jartool/share/classes/sun/tools/jar/Main.java index 9e0db1834f555..e79b858d99017 100644 --- a/src/jdk.jartool/share/classes/sun/tools/jar/Main.java +++ b/src/jdk.jartool/share/classes/sun/tools/jar/Main.java @@ -129,7 +129,7 @@ public int hashCode() { // There's also a files array per version // base version is the first entry and then follow with the version given // from the --release option in the command-line order. - // The value of each entry is the files given in the command-line order. + // The value of each entry is the files given in the command-line order. Map filesMap = new LinkedHashMap<>(); // Do we think this is a multi-release jar? Set to true diff --git a/test/jdk/tools/jar/CreateJarBenchmark.java b/test/jdk/tools/jar/CreateJarBenchmark.java index 4ced0fa101a36..94fe05722a5fe 100644 --- a/test/jdk/tools/jar/CreateJarBenchmark.java +++ b/test/jdk/tools/jar/CreateJarBenchmark.java @@ -29,7 +29,7 @@ * @modules jdk.jartool * @build jdk.test.lib.Platform * jdk.test.lib.util.FileUtils - * @run testng CreateJarBenchmark + * @run testng CreateJarBenchmark */ import org.testng.Assert; @@ -86,7 +86,7 @@ public void testSingleDir() throws IOException { onCompletion = () -> rm("test.jar", "testjar"); // Perform 100x jar creations - long start = System.currentTimeMillis(); + long start = System.currentTimeMillis(); for(int i = 0; i < 100; i++) { jar("cf test.jar testjar"); rm("test.jar"); From 23efb28c51381ecf6647ae6805f4037e993ee9e9 Mon Sep 17 00:00:00 2001 From: Andrew Leonard Date: Fri, 19 Nov 2021 20:09:20 +0000 Subject: [PATCH 5/8] 8276764: Enable deterministic file content ordering for Jar and Jmod Signed-off-by: Andrew Leonard --- test/jdk/tools/jar/ContentOrder.java | 48 +++++- test/jdk/tools/jar/CreateJarBenchmark.java | 189 --------------------- 2 files changed, 42 insertions(+), 195 deletions(-) delete mode 100644 test/jdk/tools/jar/CreateJarBenchmark.java diff --git a/test/jdk/tools/jar/ContentOrder.java b/test/jdk/tools/jar/ContentOrder.java index a5f0f31f606dc..e9d5694fe50db 100644 --- a/test/jdk/tools/jar/ContentOrder.java +++ b/test/jdk/tools/jar/ContentOrder.java @@ -76,7 +76,7 @@ public void run() { } @Test - public void test1() throws IOException { + public void testSingleDir() throws IOException { mkdir("testjar/Ctest1", "testjar/Btest2/subdir1", "testjar/Atest3"); touch("testjar/Ctest1/testfile1", "testjar/Ctest1/testfile2", "testjar/Ctest1/testfile3"); touch("testjar/Btest2/subdir1/testfileC", "testjar/Btest2/subdir1/testfileB", "testjar/Btest2/subdir1/testfileA"); @@ -86,7 +86,7 @@ public void test1() throws IOException { jar("cf test.jar testjar"); jar("tf test.jar"); - println(); + System.out.println(new String(baos.toByteArray())); String output = "META-INF/" + nl + "META-INF/MANIFEST.MF" + nl + "testjar/" + nl + @@ -106,6 +106,46 @@ public void test1() throws IOException { Assert.assertEquals(baos.toByteArray(), output.getBytes()); } + @Test + public void testMultiDirWithReleases() throws IOException { + mkdir("testjar/foo/classes", + "testjar/foo11/classes/Zclasses", + "testjar/foo11/classes/Yclasses", + "testjar/foo17/classes/Bclasses", + "testjar/foo17/classes/Aclasses"); + touch("testjar/foo/classes/testfile1", "testjar/foo/classes/testfile2"); + touch("testjar/foo11/classes/Zclasses/testfile1", "testjar/foo11/classes/Zclasses/testfile2"); + touch("testjar/foo11/classes/Yclasses/testfileA", "testjar/foo11/classes/Yclasses/testfileB"); + touch("testjar/foo17/classes/Bclasses/testfile1", "testjar/foo17/classes/Bclasses/testfile2"); + touch("testjar/foo17/classes/Aclasses/testfileA", "testjar/foo17/classes/Aclasses/testfileB"); + + onCompletion = () -> rm("test.jar", "testjar"); + + jar("cf test.jar -C testjar/foo classes " + + "--release 17 -C testjar/foo17 classes/Bclasses -C testjar/foo17 classes/Aclasses " + + "--release 11 -C testjar/foo11 classes/Zclasses -C testjar/foo11 classes/Yclasses"); + jar("tf test.jar"); + System.out.println(new String(baos.toByteArray())); + String output = "META-INF/" + nl + + "META-INF/MANIFEST.MF" + nl + + "classes/" + nl + + "classes/testfile1" + nl + + "classes/testfile2" + nl + + "META-INF/versions/17/classes/Bclasses/" + nl + + "META-INF/versions/17/classes/Bclasses/testfile1" + nl + + "META-INF/versions/17/classes/Bclasses/testfile2" + nl + + "META-INF/versions/17/classes/Aclasses/" + nl + + "META-INF/versions/17/classes/Aclasses/testfileA" + nl + + "META-INF/versions/17/classes/Aclasses/testfileB" + nl + + "META-INF/versions/11/classes/Zclasses/" + nl + + "META-INF/versions/11/classes/Zclasses/testfile1" + nl + + "META-INF/versions/11/classes/Zclasses/testfile2" + nl + + "META-INF/versions/11/classes/Yclasses/" + nl + + "META-INF/versions/11/classes/Yclasses/testfileA" + nl + + "META-INF/versions/11/classes/Yclasses/testfileB" + nl; + Assert.assertEquals(baos.toByteArray(), output.getBytes()); + } + private Stream mkpath(String... args) { return Arrays.stream(args).map(d -> Paths.get(".", d.split("/"))); } @@ -167,8 +207,4 @@ private void jar(String cmdline) throws IOException { throw new IOException(s); } } - - private void println() throws IOException { - System.out.println(new String(baos.toByteArray())); - } } diff --git a/test/jdk/tools/jar/CreateJarBenchmark.java b/test/jdk/tools/jar/CreateJarBenchmark.java deleted file mode 100644 index 94fe05722a5fe..0000000000000 --- a/test/jdk/tools/jar/CreateJarBenchmark.java +++ /dev/null @@ -1,189 +0,0 @@ -/* - * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ - -/* - * @test - * @bug 8276764 - * @summary perform a jar creation benchmark - * @library /test/lib - * @modules jdk.jartool - * @build jdk.test.lib.Platform - * jdk.test.lib.util.FileUtils - * @run testng CreateJarBenchmark - */ - -import org.testng.Assert; -import org.testng.annotations.AfterMethod; -import org.testng.annotations.BeforeMethod; -import org.testng.annotations.Test; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.PrintStream; -import java.io.UncheckedIOException; -import java.io.File; -import java.io.FileOutputStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.Arrays; -import java.util.spi.ToolProvider; -import java.util.stream.Stream; -import java.util.zip.ZipException; - -import jdk.test.lib.util.FileUtils; - -public class CreateJarBenchmark { - private static final ToolProvider JAR_TOOL = ToolProvider.findFirst("jar") - .orElseThrow(() -> - new RuntimeException("jar tool not found") - ); - - private final ByteArrayOutputStream baos = new ByteArrayOutputStream(); - private final PrintStream out = new PrintStream(baos); - private Runnable onCompletion; - - @BeforeMethod - public void reset() { - onCompletion = null; - } - - @AfterMethod - public void run() { - if (onCompletion != null) { - onCompletion.run(); - } - } - - @Test - public void testSingleDir() throws IOException { - // Create a single testjar directory containing 10000 files - mkdir("testjar"); - for(int i = 0; i < 10000; i++) { - createFile("testjar/testfile"+i); - } - - onCompletion = () -> rm("test.jar", "testjar"); - - // Perform 100x jar creations - long start = System.currentTimeMillis(); - for(int i = 0; i < 100; i++) { - jar("cf test.jar testjar"); - rm("test.jar"); - } - long finish = System.currentTimeMillis(); - - System.out.println("single directory jar creation benchmark = " + (finish-start) + "ms"); - } - - @Test - public void testMultiDir() throws IOException { - // Create a nested 10x20 set of sub-dirs each containing 50 files - mkdir("testjar"); - for(int i = 0; i < 10; i++) { - mkdir("testjar/testdir" + i); - for(int j = 0; j < 20; j++) { - mkdir("testjar/testdir" + i + "/subdir" + j); - for(int k = 0; k < 50; k++) { - createFile("testjar/testdir" + i + "/subdir" + j + "/testfile" + k); - } - } - } - - onCompletion = () -> rm("test.jar", "testjar"); - - // Perform 100x jar creations - long start = System.currentTimeMillis(); - for(int i = 0; i < 100; i++) { - jar("cf test.jar testjar"); - rm("test.jar"); - } - long finish = System.currentTimeMillis(); - - System.out.println("multi directory jar creation benchmark = " + (finish-start) + "ms"); - } - - private Stream mkpath(String... args) { - return Arrays.stream(args).map(d -> Paths.get(".", d.split("/"))); - } - - private void mkdir(String... dirs) { - Arrays.stream(dirs).forEach(p -> { - try { - Files.createDirectories((new File(p)).toPath()); - } catch (IOException x) { - throw new UncheckedIOException(x); - } - }); - } - - private void createFile(String... files) { - Arrays.stream(files).forEach(p -> { - try { - try (FileOutputStream fos = new FileOutputStream(p)) { - // Create file with fixed content - byte[] bytes = new byte[10000]; - Arrays.fill(bytes, (byte)0x41); - fos.write(bytes); - } - } catch (IOException x) { - throw new UncheckedIOException(x); - } - }); - } - - private void rm(String... files) { - Arrays.stream(files).forEach(p -> { - try { - Path path = (new File(p)).toPath(); - if (Files.isDirectory(path)) { - FileUtils.deleteFileTreeWithRetry(path); - } else { - FileUtils.deleteFileIfExistsWithRetry(path); - } - } catch (IOException x) { - throw new UncheckedIOException(x); - } - }); - } - - private void jar(String cmdline) throws IOException { - System.out.println("jar " + cmdline); - baos.reset(); - - // the run method catches IOExceptions, we need to expose them - ByteArrayOutputStream baes = new ByteArrayOutputStream(); - PrintStream err = new PrintStream(baes); - PrintStream saveErr = System.err; - System.setErr(err); - int rc = JAR_TOOL.run(out, err, cmdline.split(" +")); - System.setErr(saveErr); - if (rc != 0) { - String s = baes.toString(); - if (s.startsWith("java.util.zip.ZipException: duplicate entry: ")) { - throw new ZipException(s); - } - throw new IOException(s); - } - } -} From 18776eb3ecade32bfe211a8e28da40d8f68b089a Mon Sep 17 00:00:00 2001 From: Andrew Leonard Date: Fri, 19 Nov 2021 21:20:12 +0000 Subject: [PATCH 6/8] 8276764: Enable deterministic file content ordering for Jar and Jmod Signed-off-by: Andrew Leonard --- src/jdk.jartool/share/man/jar.1 | 8 ++++++++ src/jdk.jlink/share/man/jmod.1 | 6 ++++++ test/jdk/tools/jar/ContentOrder.java | 4 ++++ 3 files changed, 18 insertions(+) diff --git a/src/jdk.jartool/share/man/jar.1 b/src/jdk.jartool/share/man/jar.1 index 4a8c8f4ae6697..3134e2cd96728 100644 --- a/src/jdk.jartool/share/man/jar.1 +++ b/src/jdk.jartool/share/man/jar.1 @@ -75,6 +75,14 @@ updating a modular jar or updating an existing non\-modular jar: .PP \f[B]Note:\f[R] .PP +When the jar is created or updated, the files and directories added during +the operation will be sorted within each directory that is processed. This +will ensure a deterministic jar content ordering. Where multiple directories +or releases are specified, the processing of each will be the order they +are specified on the command line. +.PP +\f[B]Note:\f[R] +.PP All mandatory or optional arguments for long options are also mandatory or optional for any corresponding short options. .SH MAIN OPERATION MODES diff --git a/src/jdk.jlink/share/man/jmod.1 b/src/jdk.jlink/share/man/jmod.1 index 16f595782a507..1888b5717a9fc 100644 --- a/src/jdk.jlink/share/man/jmod.1 +++ b/src/jdk.jlink/share/man/jmod.1 @@ -104,6 +104,12 @@ This enables a package to be exported to one or more specifically\-named modules and to no others through qualified exports. The runtime verifies if the recorded hash of a module matches the one resolved at run time; if not, the runtime returns an error. +.PP +\f[B]Note:\f[R] +.PP +When creating a jmod the files and directories within each class\-path +directory will be sorted, this will ensure a deterministic jmod content +ordering. .SH OPTIONS FOR JMOD .TP .B \f[CB]\-\-class\-path\f[R] \f[I]path\f[R] diff --git a/test/jdk/tools/jar/ContentOrder.java b/test/jdk/tools/jar/ContentOrder.java index e9d5694fe50db..dc5315efde1be 100644 --- a/test/jdk/tools/jar/ContentOrder.java +++ b/test/jdk/tools/jar/ContentOrder.java @@ -75,6 +75,7 @@ public void run() { } } + // Test that the jar content ordering when processing a single directory is sorted @Test public void testSingleDir() throws IOException { mkdir("testjar/Ctest1", "testjar/Btest2/subdir1", "testjar/Atest3"); @@ -106,6 +107,9 @@ public void testSingleDir() throws IOException { Assert.assertEquals(baos.toByteArray(), output.getBytes()); } + // Test that when specifying multiple directories or releases that the sort + // ordering is done on each directory and release, reserving the order of + // the directories/releases specified on the command line @Test public void testMultiDirWithReleases() throws IOException { mkdir("testjar/foo/classes", From 2a7668f1edf51dcdab132d78d7f6dedb4bc78686 Mon Sep 17 00:00:00 2001 From: Andrew Leonard Date: Fri, 19 Nov 2021 21:59:18 +0000 Subject: [PATCH 7/8] 8276764: Enable deterministic file content ordering for Jar and Jmod Signed-off-by: Andrew Leonard --- src/jdk.jartool/share/man/jar.1 | 8 -------- src/jdk.jlink/share/man/jmod.1 | 6 ------ 2 files changed, 14 deletions(-) diff --git a/src/jdk.jartool/share/man/jar.1 b/src/jdk.jartool/share/man/jar.1 index 3134e2cd96728..4a8c8f4ae6697 100644 --- a/src/jdk.jartool/share/man/jar.1 +++ b/src/jdk.jartool/share/man/jar.1 @@ -75,14 +75,6 @@ updating a modular jar or updating an existing non\-modular jar: .PP \f[B]Note:\f[R] .PP -When the jar is created or updated, the files and directories added during -the operation will be sorted within each directory that is processed. This -will ensure a deterministic jar content ordering. Where multiple directories -or releases are specified, the processing of each will be the order they -are specified on the command line. -.PP -\f[B]Note:\f[R] -.PP All mandatory or optional arguments for long options are also mandatory or optional for any corresponding short options. .SH MAIN OPERATION MODES diff --git a/src/jdk.jlink/share/man/jmod.1 b/src/jdk.jlink/share/man/jmod.1 index 1888b5717a9fc..16f595782a507 100644 --- a/src/jdk.jlink/share/man/jmod.1 +++ b/src/jdk.jlink/share/man/jmod.1 @@ -104,12 +104,6 @@ This enables a package to be exported to one or more specifically\-named modules and to no others through qualified exports. The runtime verifies if the recorded hash of a module matches the one resolved at run time; if not, the runtime returns an error. -.PP -\f[B]Note:\f[R] -.PP -When creating a jmod the files and directories within each class\-path -directory will be sorted, this will ensure a deterministic jmod content -ordering. .SH OPTIONS FOR JMOD .TP .B \f[CB]\-\-class\-path\f[R] \f[I]path\f[R] From 58772b58da72476275c1e60c9d40cc2cc198bbd8 Mon Sep 17 00:00:00 2001 From: Andrew Leonard Date: Tue, 23 Nov 2021 09:58:35 +0000 Subject: [PATCH 8/8] 8276764: Enable deterministic file content ordering for Jar and Jmod Signed-off-by: Andrew Leonard --- src/jdk.jlink/share/classes/jdk/tools/jmod/JmodTask.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/jdk.jlink/share/classes/jdk/tools/jmod/JmodTask.java b/src/jdk.jlink/share/classes/jdk/tools/jmod/JmodTask.java index d8be60d0f57d4..f4248bd16cef3 100644 --- a/src/jdk.jlink/share/classes/jdk/tools/jmod/JmodTask.java +++ b/src/jdk.jlink/share/classes/jdk/tools/jmod/JmodTask.java @@ -769,7 +769,7 @@ void processSection(JmodOutputStream out, Section section, Path path) { // Keep a sorted set of files to be processed, so that the jmod // content is reproducible as Files.walkFileTree order is not defined - SortedMap filesToProcess = new TreeMap(); + SortedMap filesToProcess = new TreeMap(); Files.walkFileTree(path, Set.of(FileVisitOption.FOLLOW_LINKS), Integer.MAX_VALUE, new SimpleFileVisitor() {