From f36129504a171103e51d285ee1c0167abab266c1 Mon Sep 17 00:00:00 2001 From: Kenneth Knowles Date: Tue, 3 May 2016 21:27:44 -0700 Subject: [PATCH 1/3] Remove timestamp in source that caused spurious builds --- pom.xml | 7 ++ .../runners/dataflow/DataflowRunnerTest.java | 12 +++ .../org/apache/beam/sdk/util/ReleaseInfo.java | 74 +++++++++++++------ .../org/apache/beam/sdk/sdk.properties | 22 ------ 4 files changed, 69 insertions(+), 46 deletions(-) delete mode 100644 sdks/java/core/src/main/resources/org/apache/beam/sdk/sdk.properties diff --git a/pom.xml b/pom.xml index 307e43e1f978..88ada1dce9b1 100644 --- a/pom.xml +++ b/pom.xml @@ -845,6 +845,13 @@ + + + + ${timestamp} + + + diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowRunnerTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowRunnerTest.java index ddb7cf8c7dc8..e6be2f8db6de 100644 --- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowRunnerTest.java +++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowRunnerTest.java @@ -26,6 +26,7 @@ import static org.hamcrest.Matchers.startsWith; import static org.hamcrest.collection.IsIterableContainingInOrder.contains; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertThat; @@ -432,6 +433,17 @@ public void testRunWithFiles() throws IOException { assertEquals( ReleaseInfo.getReleaseInfo().getVersion(), workflowJob.getEnvironment().getUserAgent().get("version")); + + // The version should not be unknown, as the SDK should be in a separate jar + assertNotEquals( + ReleaseInfo.getReleaseInfo().getVersion(), + ReleaseInfo.UNKNOWN_VERSION_STRING); + + // We should know what SDK we have depended upon, since we put it in the user agent string + assertNotNull(ReleaseInfo.getReleaseInfo().getBuildTimestamp()); + assertEquals( + ReleaseInfo.getReleaseInfo().getBuildTimestamp(), + workflowJob.getEnvironment().getUserAgent().get("build.date")); } @Test diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ReleaseInfo.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ReleaseInfo.java index ba80de97dc4d..afcdbf72ce6b 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ReleaseInfo.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ReleaseInfo.java @@ -20,8 +20,11 @@ import com.google.api.client.json.GenericJson; import com.google.api.client.util.Key; import java.io.IOException; -import java.io.InputStream; -import java.util.Properties; +import java.net.URISyntaxException; +import java.net.URL; +import java.util.jar.JarFile; +import java.util.jar.Manifest; +import javax.annotation.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -31,14 +34,17 @@ public final class ReleaseInfo extends GenericJson { private static final Logger LOG = LoggerFactory.getLogger(ReleaseInfo.class); - private static final String PROPERTIES_PATH = - "/org/apache/beam/sdk/sdk.properties"; + private static final String MANIFEST_TIMESTAMP_ATTRIBUTE = "Build-timestamp"; private static class LazyInit { - private static final ReleaseInfo INSTANCE = - new ReleaseInfo(PROPERTIES_PATH); + private static final ReleaseInfo INSTANCE = new ReleaseInfo(); } + /** + * The string that will be used as a version if none can be ascertained. + */ + public static final String UNKNOWN_VERSION_STRING = "Unknown"; + /** * Returns an instance of {@link ReleaseInfo}. */ @@ -47,38 +53,58 @@ public static ReleaseInfo getReleaseInfo() { } @Key private String name = "Apache Beam SDK for Java"; - @Key private String version = "Unknown"; + @Key private String version = UNKNOWN_VERSION_STRING; + @Nullable @Key("build.date") private String timestamp = null; /** Provides the SDK name. */ public String getName() { return name; } - /** Provides the SDK version. */ + /** + * The version of the packaged artifact for the core Apache Beam Java SDK, or + * {@link #UNKNOWN_VERSION_STRING} if unknown. + * + *

+ * This should only be unknown if the class is used outside of a built artifact. + */ public String getVersion() { return version; } - private ReleaseInfo(String resourcePath) { - Properties properties = new Properties(); + /** + * The build time of the packaged artifact for the core Apache Beam Java SDK, or {@code null} if + * unknown. + * + *

+ * This should only be {@code null} if the class is used outside of a built artifact. + */ + @Nullable + public String getBuildTimestamp() { + return timestamp; + } - try (InputStream in = ReleaseInfo.class.getResourceAsStream(PROPERTIES_PATH)) { - if (in == null) { - LOG.warn("Beam properties resource not found: {}", resourcePath); - return; - } + private ReleaseInfo() { + // This may not actually be a jar; in that case we cannot learn anything from the manifest + URL jarLocation = getClass().getProtectionDomain().getCodeSource().getLocation(); - properties.load(in); - } catch (IOException e) { - LOG.warn("Error loading Beam properties resource: ", e); + // Get the version via standard channels + @Nullable String maybeVersion = getClass().getPackage().getImplementationVersion(); + if (maybeVersion != null) { + version = maybeVersion; } - for (String name : properties.stringPropertyNames()) { - if (name.equals("name")) { - // We don't allow the properties to override the SDK name. - continue; - } - put(name, properties.getProperty(name)); + // Read the timestamp, which is a custom manifest entry + Manifest manifest; + try (JarFile jarFile = new JarFile(jarLocation.toURI().getPath())) { + manifest = jarFile.getManifest(); + } catch (IOException exc) { + LOG.warn("Beam SDK jar manifest not found at " + jarLocation, exc); + return; + } catch (URISyntaxException exc) { + LOG.warn("Beam SDK code location appears to be invalid at " + jarLocation, exc); + return; } + timestamp = manifest.getMainAttributes().getValue(MANIFEST_TIMESTAMP_ATTRIBUTE); } } diff --git a/sdks/java/core/src/main/resources/org/apache/beam/sdk/sdk.properties b/sdks/java/core/src/main/resources/org/apache/beam/sdk/sdk.properties deleted file mode 100644 index 21e314d44da8..000000000000 --- a/sdks/java/core/src/main/resources/org/apache/beam/sdk/sdk.properties +++ /dev/null @@ -1,22 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# SDK source version - -version=${pom.version} - -build.date=${timestamp} - From 23862c80a21d67eaf6c1e3fff745dadba0d2d0c9 Mon Sep 17 00:00:00 2001 From: Kenneth Knowles Date: Wed, 4 May 2016 14:07:22 -0700 Subject: [PATCH 2/3] Add ReleaseInfo to bigtable package and use for User-agent --- .../org/apache/beam/sdk/util/ReleaseInfo.java | 6 +- .../beam/sdk/io/gcp/bigtable/BigtableIO.java | 11 ++-- .../beam/sdk/io/gcp/bigtable/ReleaseInfo.java | 55 +++++++++++++++++++ 3 files changed, 62 insertions(+), 10 deletions(-) create mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/ReleaseInfo.java diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ReleaseInfo.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ReleaseInfo.java index afcdbf72ce6b..92c6e5cef5fc 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ReleaseInfo.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ReleaseInfo.java @@ -65,8 +65,7 @@ public String getName() { * The version of the packaged artifact for the core Apache Beam Java SDK, or * {@link #UNKNOWN_VERSION_STRING} if unknown. * - *

- * This should only be unknown if the class is used outside of a built artifact. + *

This should only be unknown if the class is used outside of a built artifact. */ public String getVersion() { return version; @@ -76,8 +75,7 @@ public String getVersion() { * The build time of the packaged artifact for the core Apache Beam Java SDK, or {@code null} if * unknown. * - *

- * This should only be {@code null} if the class is used outside of a built artifact. + *

This should only be {@code null} if the class is used outside of a built artifact. */ @Nullable public String getBuildTimestamp() { diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIO.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIO.java index 90b95848b11e..c0d8e98c363c 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIO.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIO.java @@ -55,7 +55,6 @@ import org.apache.beam.sdk.transforms.PTransform; import org.apache.beam.sdk.transforms.ParDo; import org.apache.beam.sdk.transforms.display.DisplayData; -import org.apache.beam.sdk.util.ReleaseInfo; import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PBegin; import org.apache.beam.sdk.values.PCollection; @@ -1026,12 +1025,12 @@ public BigtableWriteException(KV> record, Throwab */ private static String getUserAgent() { String javaVersion = System.getProperty("java.specification.version"); - ReleaseInfo info = ReleaseInfo.getReleaseInfo(); return String.format( - "%s/%s (%s); %s", - info.getName(), - info.getVersion(), - javaVersion, + "%s/%s %s/%s java/%s", + org.apache.beam.sdk.io.gcp.bigtable.ReleaseInfo.getReleaseInfo().getName(), + org.apache.beam.sdk.io.gcp.bigtable.ReleaseInfo.getReleaseInfo().getVersion(), + org.apache.beam.sdk.util.ReleaseInfo.getReleaseInfo().getName(), + org.apache.beam.sdk.util.ReleaseInfo.getReleaseInfo().getVersion(), "0.3.0" /* TODO get Bigtable client version directly from jar. */); } diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/ReleaseInfo.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/ReleaseInfo.java new file mode 100644 index 000000000000..7202ac8c8881 --- /dev/null +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/ReleaseInfo.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.gcp.bigtable; + +import javax.annotation.Nullable; + +/** + * Utilities for working with release information. + */ +public final class ReleaseInfo { + + private static final ReleaseInfo INSTANCE = new ReleaseInfo(); + + /** + * Returns the release info of the distributed artifact for the Apache Beam GCP connectors, or + * {@code null} if unknown. + */ + public static ReleaseInfo getReleaseInfo() { + return INSTANCE; + } + + private ReleaseInfo() { } + + /** + * The name of the root java package for the Apache Beam GCP connectors. + */ + public String getName() { + return ReleaseInfo.class.getPackage().getName(); + } + + /** + * The version of the distributed artifact for the Apache Beam GCP connectors, or {@code null} + * if unknown. + * + *

This should only be {@code null} if the class is used outside of a built artifact. + */ + @Nullable public String getVersion() { + return ReleaseInfo.class.getPackage().getImplementationVersion(); + } +} From 166243dc4dedb88f6a187bbe36ccb6c56ba59953 Mon Sep 17 00:00:00 2001 From: Kenneth Knowles Date: Mon, 24 Oct 2016 19:19:16 -0700 Subject: [PATCH 3/3] Use -Xpkginfo:always flag --- pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/pom.xml b/pom.xml index 88ada1dce9b1..c28e84d9341f 100644 --- a/pom.xml +++ b/pom.xml @@ -817,6 +817,7 @@ -Xlint:-try -Xlint:-unchecked -Xlint:-varargs + -Xpkginfo:always