From 7bc428ba1afaa9be9ea54adcea6ce8eefa5b5945 Mon Sep 17 00:00:00 2001 From: Oscar Boykin Date: Wed, 18 May 2016 14:40:59 -1000 Subject: [PATCH 1/5] Check that jar output is byte-for-byte identical --- test_run.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test_run.sh b/test_run.sh index c722f03e2..88008c283 100755 --- a/test_run.sh +++ b/test_run.sh @@ -17,6 +17,15 @@ test_disappearing_class() { set -e } +test_build_is_identical() { + bazel build test/... + md5 bazel-bin/test/*.jar > hash1 + bazel clean + bazel build test/... + md5 bazel-bin/test/*.jar > hash2 + diff hash1 hash2 +} + bazel build test/... \ && bazel run test:ScalaBinary \ && bazel run test:ScalaLibBinary \ @@ -27,4 +36,5 @@ bazel build test/... \ && find -L ./bazel-testlogs -iname "*.xml" \ && (find -L ./bazel-testlogs -iname "*.xml" | xargs -n1 xmllint > /dev/null) \ && test_disappearing_class \ + && test_build_is_identical \ && echo "all good" From 8ac7a6fec6d137c8ceed8beda5ddfdbc47c29742 Mon Sep 17 00:00:00 2001 From: Oscar Boykin Date: Wed, 25 May 2016 15:32:04 -1000 Subject: [PATCH 2/5] add bazel's jar creation code --- scala/scala.bzl | 50 ++-- src/java/io/bazel/rulesscala/jar/BUILD | 5 + .../io/bazel/rulesscala/jar/JarCreator.java | 207 ++++++++++++++++ .../io/bazel/rulesscala/jar/JarHelper.java | 223 ++++++++++++++++++ 4 files changed, 471 insertions(+), 14 deletions(-) create mode 100644 src/java/io/bazel/rulesscala/jar/BUILD create mode 100644 src/java/io/bazel/rulesscala/jar/JarCreator.java create mode 100644 src/java/io/bazel/rulesscala/jar/JarHelper.java diff --git a/scala/scala.bzl b/scala/scala.bzl index a5480be42..b74bd9ed5 100644 --- a/scala/scala.bzl +++ b/scala/scala.bzl @@ -29,31 +29,48 @@ def _adjust_resources_path(path): return dir_1 + dir_2, rel_path return "", path -def _add_resources_cmd(ctx): +def _add_resources_cmd(ctx, dest): res_cmd = "" for f in ctx.files.resources: c_dir, res_path = _adjust_resources_path(f.path) - change_dir = "-C " + c_dir if c_dir else "" - res_cmd += "\n{jar} uf {out} " + change_dir + " " + res_path + res_cmd += "\nmkdir -p $(dirname {out_dir}{res_path})\ncp {c_dir}{res_path} {out_dir}{res_path}".format( + out_dir=dest, + res_path=res_path, + c_dir=c_dir) return res_cmd +def _get_jar_path(paths): + for p in paths: + path = p.path + if path.endswith("/jar_deploy.jar"): + return path + return None + def _build_nosrc_jar(ctx, buildijar): - res_cmd = _add_resources_cmd(ctx) + cp_resources = _add_resources_cmd(ctx, "{out}_tmp".format(out=ctx.outputs.jar.path)) ijar_cmd = "" if buildijar: ijar_cmd = "\ncp {out} {ijar_out}".format( out=ctx.outputs.jar.path, ijar_out=ctx.outputs.ijar.path) cmd = """ +rm -rf {out}_tmp set -e +mkdir -p {out}_tmp +# copy any resources +{cp_resources} +# Make jar file deterministic by setting the timestamp of files +find {out}_tmp -exec touch -t 198001010000 {{}} \; # Make jar file deterministic by setting the timestamp of files touch -t 198001010000 {manifest} -{jar} cmf {manifest} {out} -""" + ijar_cmd + res_cmd +{java} -jar {jar} -m {manifest} {out} +""" + ijar_cmd cmd = cmd.format( + cp_resources=cp_resources, out=ctx.outputs.jar.path, manifest=ctx.outputs.manifest.path, - jar=ctx.file._jar.path) + java=ctx.file._java.path, + jar=_get_jar_path(ctx.files._jar)) outs = [ctx.outputs.jar] if buildijar: outs.extend([ctx.outputs.ijar]) @@ -61,7 +78,8 @@ touch -t 198001010000 {manifest} inputs= ctx.files.resources + ctx.files._jdk + - [ctx.outputs.manifest, ctx.file._jar], + ctx.files._jar + + [ctx.outputs.manifest, ctx.file._java], outputs=outs, command=cmd, progress_message="scala %s" % ctx.label, @@ -83,7 +101,7 @@ def _collect_plugin_paths(plugins): def _compile(ctx, _jars, dep_srcjars, buildijar): jars = _jars - res_cmd = _add_resources_cmd(ctx) + cp_resources = _add_resources_cmd(ctx, "{out}_tmp".format(out=ctx.outputs.jar.path)) ijar_cmd = "" if buildijar: ijar_cmd = "\n{ijar} {out} {ijar_out}".format( @@ -135,22 +153,25 @@ touch {out}_args/files_from_jar mkdir -p {out}_tmp""" + srcjar_cmd + """ cat {scalac_args} {out}_args/files_from_jar > {out}_args/args env JAVACMD={java} {scalac} {jvm_flags} @{out}_args/args +# add any resources +{cp_resources} # Make jar file deterministic by setting the timestamp of files find {out}_tmp -exec touch -t 198001010000 {{}} \; touch -t 198001010000 {manifest} -{jar} cmf {manifest} {out} -C {out}_tmp . +{java} -jar {jar} -m {manifest} {out} {out}_tmp rm -rf {out}_args rm -rf {out}_tmp rm -rf {out}_tmp_expand_srcjars -""" + ijar_cmd + res_cmd +""" + ijar_cmd cmd = cmd.format( + cp_resources=cp_resources, java=ctx.file._java.path, jvm_flags=" ".join(["-J" + flag for flag in ctx.attr.jvm_flags]), scalac=ctx.file._scalac.path, scalac_args=scalac_args_file.path, out=ctx.outputs.jar.path, manifest=ctx.outputs.manifest.path, - jar=ctx.file._jar.path, + jar=_get_jar_path(ctx.files._jar), ijar=ctx.file._ijar.path, ) outs = [ctx.outputs.jar] @@ -165,11 +186,12 @@ rm -rf {out}_tmp_expand_srcjars ctx.files.plugins + ctx.files.resources + ctx.files._jdk + + ctx.files._jar + ctx.files._scalasdk + [ctx.outputs.manifest, - ctx.file._jar, ctx.file._ijar, ctx.file._scalac, + ctx.file._java, scalac_args_file], outputs=outs, command=cmd, @@ -361,7 +383,7 @@ _implicit_deps = { "_scalasdk": attr.label(default=Label("@scala//:sdk"), allow_files=True), "_scalareflect": attr.label(default=Label("@scala//:lib/scala-reflect.jar"), single_file=True, allow_files=True), "_java": attr.label(executable=True, default=Label("@bazel_tools//tools/jdk:java"), single_file=True, allow_files=True), - "_jar": attr.label(executable=True, default=Label("@bazel_tools//tools/jdk:jar"), single_file=True, allow_files=True), + "_jar": attr.label(executable=True, default=Label("//src/java/io/bazel/rulesscala/jar:jar_deploy.jar"), allow_files=True), "_jdk": attr.label(default=Label("//tools/defaults:jdk"), allow_files=True), } diff --git a/src/java/io/bazel/rulesscala/jar/BUILD b/src/java/io/bazel/rulesscala/jar/BUILD new file mode 100644 index 000000000..009207740 --- /dev/null +++ b/src/java/io/bazel/rulesscala/jar/BUILD @@ -0,0 +1,5 @@ +java_binary(name = "jar", + srcs = ["JarCreator.java", "JarHelper.java"], + main_class = "io.bazel.rulesscala.jar.JarCreator", + visibility = ["//visibility:public"], +) diff --git a/src/java/io/bazel/rulesscala/jar/JarCreator.java b/src/java/io/bazel/rulesscala/jar/JarCreator.java new file mode 100644 index 000000000..07ffa4ee6 --- /dev/null +++ b/src/java/io/bazel/rulesscala/jar/JarCreator.java @@ -0,0 +1,207 @@ +// Copyright 2014 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package io.bazel.rulesscala.jar; + +import java.io.BufferedOutputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.Collection; +import java.util.Map; +import java.util.TreeMap; +import java.util.jar.Attributes; +import java.util.jar.JarOutputStream; +import java.util.jar.Manifest; + +/** + * A class for creating Jar files. Allows normalization of Jar entries by setting their timestamp to + * the DOS epoch. All Jar entries are sorted alphabetically. + */ +public class JarCreator extends JarHelper { + + // Map from Jar entry names to files. Use TreeMap so we can establish a canonical order for the + // entries regardless in what order they get added. + private final Map jarEntries = new TreeMap<>(); + private String manifestFile; + private String mainClass; + + public JarCreator(String fileName) { + super(fileName); + } + + /** + * Adds an entry to the Jar file, normalizing the name. + * + * @param entryName the name of the entry in the Jar file + * @param fileName the name of the input file for the entry + * @return true iff a new entry was added + */ + public boolean addEntry(String entryName, String fileName) { + if (entryName.startsWith("/")) { + entryName = entryName.substring(1); + } else if (entryName.startsWith("./")) { + entryName = entryName.substring(2); + } + return jarEntries.put(entryName, fileName) == null; + } + + /** + * Adds the contents of a directory to the Jar file. All files below this + * directory will be added to the Jar file using the name relative to the + * directory as the name for the Jar entry. + * + * @param directory the directory to add to the jar + */ + public void addDirectory(String directory) { + addDirectory(null, new File(directory)); + } + + /** + * Adds the contents of a directory to the Jar file. All files below this + * directory will be added to the Jar file using the prefix and the name + * relative to the directory as the name for the Jar entry. Always uses '/' as + * the separator char for the Jar entries. + * + * @param prefix the prefix to prepend to every Jar entry name found below the + * directory + * @param directory the directory to add to the Jar + */ + private void addDirectory(String prefix, File directory) { + File[] files = directory.listFiles(); + if (files != null) { + for (File file : files) { + String entryName = prefix != null ? prefix + "/" + file.getName() : file.getName(); + jarEntries.put(entryName, file.getAbsolutePath()); + if (file.isDirectory()) { + addDirectory(entryName, file); + } + } + } + } + + /** + * Adds a collection of entries to the jar, each with a given source path, and with + * the resulting file in the root of the jar. + * 
+   * some/long/path.foo => (path.foo, some/long/path.foo)
+   *
+ */ + public void addRootEntries(Collection entries) { + for (String entry : entries) { + jarEntries.put(new File(entry).getName(), entry); + } + } + + /** + * Sets the main.class entry for the manifest. A value of null + * (the default) will omit the entry. + * + * @param mainClass the fully qualified name of the main class + */ + public void setMainClass(String mainClass) { + this.mainClass = mainClass; + } + + /** + * Sets filename for the manifest content. If this is set the manifest will be + * read from this file otherwise the manifest content will get generated on + * the fly. + * + * @param manifestFile the filename of the manifest file. + */ + public void setManifestFile(String manifestFile) { + this.manifestFile = manifestFile; + } + + private byte[] manifestContent() throws IOException { + Manifest manifest; + if (manifestFile != null) { + FileInputStream in = new FileInputStream(manifestFile); + manifest = new Manifest(in); + } else { + manifest = new Manifest(); + } + Attributes attributes = manifest.getMainAttributes(); + attributes.put(Attributes.Name.MANIFEST_VERSION, "1.0"); + Attributes.Name createdBy = new Attributes.Name("Created-By"); + if (attributes.getValue(createdBy) == null) { + attributes.put(createdBy, "blaze"); + } + if (mainClass != null) { + attributes.put(Attributes.Name.MAIN_CLASS, mainClass); + } + ByteArrayOutputStream out = new ByteArrayOutputStream(); + manifest.write(out); + return out.toByteArray(); + } + + /** + * Executes the creation of the Jar file. + * + * @throws IOException if the Jar cannot be written or any of the entries + * cannot be read. + */ + public void execute() throws IOException { + out = new JarOutputStream(new BufferedOutputStream(new FileOutputStream(jarFile))); + + // Create the manifest entry in the Jar file + writeManifestEntry(manifestContent()); + try { + for (Map.Entry entry : jarEntries.entrySet()) { + copyEntry(entry.getKey(), new File(entry.getValue())); + } + } finally { + out.closeEntry(); + out.close(); + } + } + + /** + * A simple way to create Jar file using the JarCreator class. + */ + public static void main(String[] args) { + if (args.length < 1) { + System.err.println("usage: CreateJar [-m manifest] output [root directories]"); + System.exit(1); + } + + int idx = 0; + String manifestFile = null; + if (args[0].equals("-m")) { + manifestFile = args[1]; + idx = 2; + } + String output = args[idx]; + JarCreator createJar = new JarCreator(output); + createJar.setManifestFile(manifestFile); + for (int i = (idx+1); i < args.length; i++) { + createJar.addDirectory(args[i]); + } + createJar.setCompression(true); + createJar.setNormalize(true); + long start = System.currentTimeMillis(); + try { + createJar.execute(); + } catch (IOException e) { + e.printStackTrace(); + System.exit(1); + } + long stop = System.currentTimeMillis(); + //System.err.println((stop - start) + "ms."); + //System.err.println(output); + } +} diff --git a/src/java/io/bazel/rulesscala/jar/JarHelper.java b/src/java/io/bazel/rulesscala/jar/JarHelper.java new file mode 100644 index 000000000..62536023d --- /dev/null +++ b/src/java/io/bazel/rulesscala/jar/JarHelper.java @@ -0,0 +1,223 @@ +// Copyright 2014 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package io.bazel.rulesscala.jar; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.nio.file.Files; +import java.util.HashSet; +import java.util.Set; +import java.util.jar.JarEntry; +import java.util.jar.JarFile; +import java.util.jar.JarOutputStream; +import java.util.zip.CRC32; + +/** + * A simple helper class for creating Jar files. All Jar entries are sorted alphabetically. Allows + * normalization of Jar entries by setting the timestamp of non-.class files to the DOS epoch. + * Timestamps of .class files are set to the DOS epoch + 2 seconds (The zip timestamp granularity) + * Adjusting the timestamp for .class files is neccessary since otherwise javac will recompile java + * files if both the java file and its .class file are present. + */ +public class JarHelper { + + public static final String MANIFEST_DIR = "META-INF/"; + public static final String MANIFEST_NAME = JarFile.MANIFEST_NAME; + public static final String SERVICES_DIR = "META-INF/services/"; + + public static final long DOS_EPOCH_IN_JAVA_TIME = 315561600000L; + + // ZIP timestamps have a resolution of 2 seconds. + // see http://www.info-zip.org/FAQ.html#limits + public static final long MINIMUM_TIMESTAMP_INCREMENT = 2000L; + + // The name of the Jar file we want to create + protected final String jarFile; + + // The properties to describe how to create the Jar + protected boolean normalize; + protected int storageMethod = JarEntry.DEFLATED; + protected boolean verbose = false; + + // The state needed to create the Jar + protected final Set names = new HashSet<>(); + protected JarOutputStream out; + + public JarHelper(String filename) { + jarFile = filename; + } + + /** + * Enables or disables the Jar entry normalization. + * + * @param normalize If true the timestamps of Jar entries will be set to the + * DOS epoch. + */ + public void setNormalize(boolean normalize) { + this.normalize = normalize; + } + + /** + * Enables or disables compression for the Jar file entries. + * + * @param compression if true enables compressions for the Jar file entries. + */ + public void setCompression(boolean compression) { + storageMethod = compression ? JarEntry.DEFLATED : JarEntry.STORED; + } + + /** + * Enables or disables verbose messages. + * + * @param verbose if true enables verbose messages. + */ + public void setVerbose(boolean verbose) { + this.verbose = verbose; + } + + /** + * Returns the normalized timestamp for a jar entry based on its name. + * This is necessary since javac will, when loading a class X, prefer a + * source file to a class file, if both files have the same timestamp. + * Therefore, we need to adjust the timestamp for class files to slightly + * after the normalized time. + * @param name The name of the file for which we should return the + * normalized timestamp. + * @return the time for a new Jar file entry in milliseconds since the epoch. + */ + private long normalizedTimestamp(String name) { + if (name.endsWith(".class")) { + return DOS_EPOCH_IN_JAVA_TIME + MINIMUM_TIMESTAMP_INCREMENT; + } else { + return DOS_EPOCH_IN_JAVA_TIME; + } + } + + /** + * Returns the time for a new Jar file entry in milliseconds since the epoch. + * Uses {@link JarCreator#DOS_EPOCH_IN_JAVA_TIME} for normalized entries, + * {@link System#currentTimeMillis()} otherwise. + * + * @param filename The name of the file for which we are entering the time + * @return the time for a new Jar file entry in milliseconds since the epoch. + */ + protected long newEntryTimeMillis(String filename) { + return normalize ? normalizedTimestamp(filename) : System.currentTimeMillis(); + } + + /** + * Writes an entry with specific contents to the jar. Directory entries must + * include the trailing '/'. + */ + protected void writeEntry(JarOutputStream out, String name, byte[] content) throws IOException { + if (names.add(name)) { + // Create a new entry + JarEntry entry = new JarEntry(name); + entry.setTime(newEntryTimeMillis(name)); + int size = content.length; + entry.setSize(size); + if (size == 0) { + entry.setMethod(JarEntry.STORED); + entry.setCrc(0); + out.putNextEntry(entry); + } else { + entry.setMethod(storageMethod); + if (storageMethod == JarEntry.STORED) { + CRC32 crc = new CRC32(); + crc.update(content); + entry.setCrc(crc.getValue()); + } + out.putNextEntry(entry); + out.write(content); + } + out.closeEntry(); + } + } + + /** + * Writes a standard Java manifest entry into the JarOutputStream. This + * includes the directory entry for the "META-INF" directory + * + * @param content the Manifest content to write to the manifest entry. + * @throws IOException + */ + protected void writeManifestEntry(byte[] content) throws IOException { + int oldStorageMethod = storageMethod; + // Do not compress small manifest files, the compressed one is frequently + // larger than the original. The threshold of 256 bytes is somewhat arbitrary. + if (content.length < 256) { + storageMethod = JarEntry.STORED; + } + try { + writeEntry(out, MANIFEST_DIR, new byte[]{}); + writeEntry(out, MANIFEST_NAME, content); + } finally { + storageMethod = oldStorageMethod; + } + } + + /** + * Copies file or directory entries from the file system into the jar. + * Directory entries will be detected and their names automatically '/' + * suffixed. + */ + protected void copyEntry(String name, File file) throws IOException { + if (!names.contains(name)) { + if (!file.exists()) { + throw new FileNotFoundException(file.getAbsolutePath() + " (No such file or directory)"); + } + boolean isDirectory = file.isDirectory(); + if (isDirectory && !name.endsWith("/")) { + name = name + '/'; // always normalize directory names before checking set + } + if (names.add(name)) { + if (verbose) { + System.err.println("adding " + file); + } + // Create a new entry + long size = isDirectory ? 0 : file.length(); + JarEntry outEntry = new JarEntry(name); + long newtime = normalize ? normalizedTimestamp(name) : file.lastModified(); + outEntry.setTime(newtime); + outEntry.setSize(size); + if (size == 0L) { + outEntry.setMethod(JarEntry.STORED); + outEntry.setCrc(0); + out.putNextEntry(outEntry); + } else { + outEntry.setMethod(storageMethod); + if (storageMethod == JarEntry.STORED) { + outEntry.setCrc(hashFile(file)); + } + out.putNextEntry(outEntry); + Files.copy(file.toPath(), out); + } + out.closeEntry(); + } + } + } + protected long hashFile(File f) throws IOException { + FileInputStream fis = new FileInputStream(f); + CRC32 crc = new CRC32(); + byte[] buffer = new byte[65536]; + int bytesRead; + while((bytesRead = fis.read(buffer)) != -1) { + crc.update(buffer, 0, bytesRead); + } + return crc.getValue(); + } +} From a6b8fe38e44bc7d7ebb737efe1c01a7c293cf310 Mon Sep 17 00:00:00 2001 From: Oscar Boykin Date: Wed, 25 May 2016 21:10:43 -1000 Subject: [PATCH 3/5] Add a default output to scala_library --- scala/scala.bzl | 1 + 1 file changed, 1 insertion(+) diff --git a/scala/scala.bzl b/scala/scala.bzl index 74d7f2978..ea8c2c292 100644 --- a/scala/scala.bzl +++ b/scala/scala.bzl @@ -317,6 +317,7 @@ def _lib(ctx, non_macro_lib): files = list(rjars), collect_data = True) return struct( + files = set([ctx.outputs.jar]), # Here is the default output scala = scalaattr, runfiles=runfiles, # This is a free monoid given to the graph for the purpose of From dda93145eca4afcb6e3c01e89767bac433d4bbfe Mon Sep 17 00:00:00 2001 From: Oscar Boykin Date: Wed, 25 May 2016 22:08:22 -1000 Subject: [PATCH 4/5] Remove unneeded find/touch with new jar code --- scala/scala.bzl | 7 ------- 1 file changed, 7 deletions(-) diff --git a/scala/scala.bzl b/scala/scala.bzl index 168cd51f8..4644feb08 100644 --- a/scala/scala.bzl +++ b/scala/scala.bzl @@ -59,10 +59,6 @@ set -e mkdir -p {out}_tmp # copy any resources {cp_resources} -# Make jar file deterministic by setting the timestamp of files -find {out}_tmp -exec touch -t 198001010000 {{}} \; -# Make jar file deterministic by setting the timestamp of files -touch -t 198001010000 {manifest} {java} -jar {jar} -m {manifest} {out} """ + ijar_cmd cmd = cmd.format( @@ -155,9 +151,6 @@ cat {scalac_args} {out}_args/files_from_jar > {out}_args/args env JAVACMD={java} {scalac} {jvm_flags} @{out}_args/args # add any resources {cp_resources} -# Make jar file deterministic by setting the timestamp of files -find {out}_tmp -exec touch -t 198001010000 {{}} \; -touch -t 198001010000 {manifest} {java} -jar {jar} -m {manifest} {out} {out}_tmp rm -rf {out}_args rm -rf {out}_tmp From a5990030ac708e50d5768a5c9a3092252c93aafa Mon Sep 17 00:00:00 2001 From: "P. Oscar Boykin" Date: Wed, 25 May 2016 22:14:27 -1000 Subject: [PATCH 5/5] use md5sum not md5 in test --- test_run.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test_run.sh b/test_run.sh index 30755ad04..3479f9109 100755 --- a/test_run.sh +++ b/test_run.sh @@ -19,10 +19,10 @@ test_disappearing_class() { test_build_is_identical() { bazel build test/... - md5 bazel-bin/test/*.jar > hash1 + md5sum bazel-bin/test/*.jar > hash1 bazel clean bazel build test/... - md5 bazel-bin/test/*.jar > hash2 + md5sum bazel-bin/test/*.jar > hash2 diff hash1 hash2 }