8276764: Enable deterministic file content ordering for Jar and Jmod

src/jdk.jartool/share/classes/sun/tools/jar/Main.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1996, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1996, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -127,7 +127,10 @@ public int hashCode() {
     Map<Integer,Set<String>> pathsMap = new HashMap<>();
 
     // There's also a files array per version
-    Map<Integer,String[]> filesMap = new HashMap<>();
+    // base version is the first entry and then follow with the version given
+    // from the --release option in the command-line order.
+    // The value of each entry is the files given in the command-line order.
+    Map<Integer,String[]> filesMap = new LinkedHashMap<>();
 
     // Do we think this is a multi-release jar?  Set to true
     // if --release option found followed by at least file
@@ -772,15 +775,17 @@ private void expand() throws IOException {
     private void expand(File dir, String[] files, Set<String> cpaths, int version)
         throws IOException
     {
-        if (files == null)
+        if (files == null) {
             return;
+        }
 
         for (int i = 0; i < files.length; i++) {
             File f;
-            if (dir == null)
+            if (dir == null) {
                 f = new File(files[i]);
-            else
+            } else {
                 f = new File(dir, files[i]);
+            }
 
             boolean isDir = f.isDirectory();
             String name = toEntryName(f.getPath(), cpaths, isDir);
@@ -802,18 +807,20 @@ private void expand(File dir, String[] files, Set<String> cpaths, int version)
                 Entry e = new Entry(f, name, false);
                 if (isModuleInfoEntry(name)) {
                     moduleInfos.putIfAbsent(name, Files.readAllBytes(f.toPath()));
-                    if (uflag)
+                    if (uflag) {
                         entryMap.put(name, e);
+                    }
                 } else if (entries.add(e)) {
-                    if (uflag)
+                    if (uflag) {
                         entryMap.put(name, e);
+                    }
                 }
             } else if (isDir) {
                 Entry e = new Entry(f, name, true);
                 if (entries.add(e)) {
                     // utilize entryMap for the duplicate dir check even in
                     // case of cflag == true.
-                    // dir name confilict/duplicate could happen with -C option.
+                    // dir name conflict/duplicate could happen with -C option.
                     // just remove the last "e" from the "entries" (zos will fail
                     // with "duplicated" entries), but continue expanding the
                     // sub tree
@@ -822,7 +829,12 @@ private void expand(File dir, String[] files, Set<String> cpaths, int version)
                     } else {
                         entryMap.put(name, e);
                     }
-                    expand(f, f.list(), cpaths, version);
+                    String[] dirFiles = f.list();
+                    // Ensure files list is sorted for reproducible jar content
+                    if (dirFiles != null) {
+                        Arrays.sort(dirFiles);
+                    }
+                    expand(f, dirFiles, cpaths, version);
                 }
             } else {
                 error(formatMsg("error.nosuch.fileordir", String.valueOf(f)));

src/jdk.jlink/share/classes/jdk/tools/jmod/JmodTask.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -767,6 +767,10 @@ void processSection(JmodOutputStream out, Section section, List<Path> paths)
         void processSection(JmodOutputStream out, Section section, Path path)
             throws IOException
         {
+            // Keep a sorted set of files to be processed, so that the jmod
+            // content is reproducible as Files.walkFileTree order is not defined
+            SortedMap<String, Path> filesToProcess = new TreeMap<String, Path>();
+
             Files.walkFileTree(path, Set.of(FileVisitOption.FOLLOW_LINKS),
                 Integer.MAX_VALUE, new SimpleFileVisitor<Path>() {
                     @Override
@@ -782,14 +786,21 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
                             if (out.contains(section, name)) {
                                 warning("warn.ignore.duplicate.entry", name, section);
                             } else {
-                                try (InputStream in = Files.newInputStream(file)) {
-                                    out.writeEntry(in, section, name);
-                                }
+                                filesToProcess.put(name, file);
                             }
                         }
                         return FileVisitResult.CONTINUE;
                     }
                 });
+
+            // Process files in sorted order for deterministic jmod content
+            for (Map.Entry<String, Path> entry : filesToProcess.entrySet()) {
+                String name = entry.getKey();
+                Path   file = entry.getValue();
+                try (InputStream in = Files.newInputStream(file)) {
+                    out.writeEntry(in, section, name);
+                }
+            }
         }
 
         boolean matches(Path path, List<PathMatcher> matchers) {

test/jdk/tools/jar/ContentOrder.java

@@ -0,0 +1,214 @@
+/*
+ * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * @test
+ * @bug 8276764
+ * @summary test that the jar content ordering is sorted
+ * @library /test/lib
+ * @modules jdk.jartool
+ * @build jdk.test.lib.Platform
+ *        jdk.test.lib.util.FileUtils
+ * @run testng ContentOrder
+ */
+
+import org.testng.Assert;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.io.UncheckedIOException;
+import java.io.File;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Arrays;
+import java.util.spi.ToolProvider;
+import java.util.stream.Stream;
+import java.util.zip.ZipException;
+
+import jdk.test.lib.util.FileUtils;
+
+public class ContentOrder {
+    private static final ToolProvider JAR_TOOL = ToolProvider.findFirst("jar")
+        .orElseThrow(() ->
+            new RuntimeException("jar tool not found")
+        );
+
+    private final String nl = System.lineSeparator();
+    private final ByteArrayOutputStream baos = new ByteArrayOutputStream();
+    private final PrintStream out = new PrintStream(baos);
+    private Runnable onCompletion;
+
+    @BeforeMethod
+    public void reset() {
+        onCompletion = null;
+    }
+
+    @AfterMethod
+    public void run() {
+        if (onCompletion != null) {
+            onCompletion.run();
+        }
+    }
+
+    // Test that the jar content ordering when processing a single directory is sorted
+    @Test
+    public void testSingleDir() throws IOException {
+        mkdir("testjar/Ctest1", "testjar/Btest2/subdir1", "testjar/Atest3");
+        touch("testjar/Ctest1/testfile1", "testjar/Ctest1/testfile2", "testjar/Ctest1/testfile3");
+        touch("testjar/Btest2/subdir1/testfileC", "testjar/Btest2/subdir1/testfileB", "testjar/Btest2/subdir1/testfileA");
+        touch("testjar/Atest3/fileZ", "testjar/Atest3/fileY", "testjar/Atest3/fileX");
+
+        onCompletion = () -> rm("test.jar", "testjar");
+
+        jar("cf test.jar testjar");
+        jar("tf test.jar");
+        System.out.println(new String(baos.toByteArray()));
+        String output = "META-INF/" + nl +
+                "META-INF/MANIFEST.MF" + nl +
+                "testjar/" + nl +
+                "testjar/Atest3/" + nl +
+                "testjar/Atest3/fileX" + nl +
+                "testjar/Atest3/fileY" + nl +
+                "testjar/Atest3/fileZ" + nl +
+                "testjar/Btest2/" + nl +
+                "testjar/Btest2/subdir1/" + nl +
+                "testjar/Btest2/subdir1/testfileA" + nl +
+                "testjar/Btest2/subdir1/testfileB" + nl +
+                "testjar/Btest2/subdir1/testfileC" + nl +
+                "testjar/Ctest1/" + nl +
+                "testjar/Ctest1/testfile1" + nl +
+                "testjar/Ctest1/testfile2" + nl +
+                "testjar/Ctest1/testfile3" + nl;
+        Assert.assertEquals(baos.toByteArray(), output.getBytes());
+    }
+
+    // Test that when specifying multiple directories or releases that the sort
+    // ordering is done on each directory and release, reserving the order of
+    // the directories/releases specified on the command line
+    @Test
+    public void testMultiDirWithReleases() throws IOException {
+        mkdir("testjar/foo/classes",
+              "testjar/foo11/classes/Zclasses",
+              "testjar/foo11/classes/Yclasses",
+              "testjar/foo17/classes/Bclasses",
+              "testjar/foo17/classes/Aclasses");
+        touch("testjar/foo/classes/testfile1", "testjar/foo/classes/testfile2");
+        touch("testjar/foo11/classes/Zclasses/testfile1", "testjar/foo11/classes/Zclasses/testfile2");
+        touch("testjar/foo11/classes/Yclasses/testfileA", "testjar/foo11/classes/Yclasses/testfileB");
+        touch("testjar/foo17/classes/Bclasses/testfile1", "testjar/foo17/classes/Bclasses/testfile2");
+        touch("testjar/foo17/classes/Aclasses/testfileA", "testjar/foo17/classes/Aclasses/testfileB");
+
+        onCompletion = () -> rm("test.jar", "testjar");
+
+        jar("cf test.jar -C testjar/foo classes " +
+            "--release 17 -C testjar/foo17 classes/Bclasses -C testjar/foo17 classes/Aclasses " +
+            "--release 11 -C testjar/foo11 classes/Zclasses -C testjar/foo11 classes/Yclasses");
+        jar("tf test.jar");
+        System.out.println(new String(baos.toByteArray()));
+        String output = "META-INF/" + nl +
+                "META-INF/MANIFEST.MF" + nl +
+                "classes/" + nl +
+                "classes/testfile1" + nl +
+                "classes/testfile2" + nl +
+                "META-INF/versions/17/classes/Bclasses/" + nl +
+                "META-INF/versions/17/classes/Bclasses/testfile1" + nl +
+                "META-INF/versions/17/classes/Bclasses/testfile2" + nl +
+                "META-INF/versions/17/classes/Aclasses/" + nl +
+                "META-INF/versions/17/classes/Aclasses/testfileA" + nl +
+                "META-INF/versions/17/classes/Aclasses/testfileB" + nl +
+                "META-INF/versions/11/classes/Zclasses/" + nl +
+                "META-INF/versions/11/classes/Zclasses/testfile1" + nl +
+                "META-INF/versions/11/classes/Zclasses/testfile2" + nl +
+                "META-INF/versions/11/classes/Yclasses/" + nl +
+                "META-INF/versions/11/classes/Yclasses/testfileA" + nl +
+                "META-INF/versions/11/classes/Yclasses/testfileB" + nl;
+        Assert.assertEquals(baos.toByteArray(), output.getBytes());
+    }
+
+    private Stream<Path> mkpath(String... args) {
+        return Arrays.stream(args).map(d -> Paths.get(".", d.split("/")));
+    }
+
+    private void mkdir(String... dirs) {
+        System.out.println("mkdir -p " + Arrays.toString(dirs));
+        Arrays.stream(dirs).forEach(p -> {
+            try {
+                Files.createDirectories((new File(p)).toPath());
+            } catch (IOException x) {
+                throw new UncheckedIOException(x);
+            }
+        });
+    }
+
+    private void touch(String... files) {
+        System.out.println("touch " + Arrays.toString(files));
+        Arrays.stream(files).forEach(p -> {
+            try {
+                Files.createFile((new File(p)).toPath());
+            } catch (IOException x) {
+                throw new UncheckedIOException(x);
+            }
+        });
+    }
+
+    private void rm(String... files) {
+        System.out.println("rm -rf " + Arrays.toString(files));
+        Arrays.stream(files).forEach(p -> {
+            try {
+                Path path = (new File(p)).toPath();
+                if (Files.isDirectory(path)) {
+                    FileUtils.deleteFileTreeWithRetry(path);
+                } else {
+                    FileUtils.deleteFileIfExistsWithRetry(path);
+                }
+            } catch (IOException x) {
+                throw new UncheckedIOException(x);
+            }
+        });
+    }
+
+    private void jar(String cmdline) throws IOException {
+        System.out.println("jar " + cmdline);
+        baos.reset();
+
+        // the run method catches IOExceptions, we need to expose them
+        ByteArrayOutputStream baes = new ByteArrayOutputStream();
+        PrintStream err = new PrintStream(baes);
+        PrintStream saveErr = System.err;
+        System.setErr(err);
+        int rc = JAR_TOOL.run(out, err, cmdline.split(" +"));
+        System.setErr(saveErr);
+        if (rc != 0) {
+            String s = baes.toString();
+            if (s.startsWith("java.util.zip.ZipException: duplicate entry: ")) {
+                throw new ZipException(s);
+            }
+            throw new IOException(s);
+        }
+    }
+}

test/jdk/tools/jmod/JmodTest.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -23,7 +23,7 @@
 
 /*
  * @test
- * @bug 8142968 8166568 8166286 8170618 8168149 8240910
+ * @bug 8142968 8166568 8166286 8170618 8168149 8240910 8276764
  * @summary Basic test for jmod
  * @library /test/lib
  * @modules jdk.compiler
@@ -197,6 +197,17 @@ public void testList() throws IOException {
                 assertContains(r.output, CLASSES_PREFIX + "jdk/test/foo/Foo.class");
                 assertContains(r.output, CLASSES_PREFIX + "jdk/test/foo/internal/Message.class");
                 assertContains(r.output, CLASSES_PREFIX + "jdk/test/foo/resources/foo.properties");
+
+                // JDK-8276764: Ensure the order is sorted for reproducible jmod content
+                // module-info, followed by <sorted classes>
+                int mod_info_i = r.output.indexOf(CLASSES_PREFIX + "module-info.class");
+                int foo_cls_i  = r.output.indexOf(CLASSES_PREFIX + "jdk/test/foo/Foo.class");
+                int msg_i      = r.output.indexOf(CLASSES_PREFIX + "jdk/test/foo/internal/Message.class");
+                int res_i      = r.output.indexOf(CLASSES_PREFIX + "jdk/test/foo/resources/foo.properties");
+                System.out.println("jmod classes sort order check:\n"+r.output);
+                assertTrue(mod_info_i < foo_cls_i);
+                assertTrue(foo_cls_i < msg_i);
+                assertTrue(msg_i < res_i);
             });
     }