[FLINK-16871][Runtime] Make more build time information available at …

…runtime

.gitignore

@@ -5,7 +5,6 @@ scalastyle-output.xml
 .metadata
 .settings
 .project
-.version.properties
 filter.properties
 logs.zip
 target

flink-dist/pom.xml

@@ -842,7 +842,7 @@ under the License.
 			</plugin>
 
 			<plugin>
-				<!-- Description: https://github.com/ktoso/maven-git-commit-id-plugin -->
+				<!-- Description: https://github.com/git-commit-id/git-commit-id-maven-plugin -->
 				<groupId>pl.project13.maven</groupId>
 				<artifactId>git-commit-id-plugin</artifactId>
 				<executions>
@@ -854,12 +854,11 @@ under the License.
 				</executions>
 				<configuration>
 					<dotGitDirectory>${project.basedir}/../.git</dotGitDirectory>
-					<generateGitPropertiesFile>true</generateGitPropertiesFile>
 					<failOnNoGitDirectory>false</failOnNoGitDirectory>
 					<skipPoms>false</skipPoms>
-					<generateGitPropertiesFilename>src/main/flink-bin/.version.properties</generateGitPropertiesFilename>
 					<gitDescribe>
-						<!-- don't generate the describe property -->
+						<!-- Don't generate the describe property -->
+						<!-- It is useless due to the way Flink does branches and tags -->
 						<skip>true</skip>
 					</gitDescribe>
 				</configuration>

flink-runtime/pom.xml

@@ -402,6 +402,38 @@ under the License.
 				</configuration>
 			</plugin>
 
+			<plugin>
+				<groupId>com.google.code.maven-replacer-plugin</groupId>
+				<artifactId>replacer</artifactId>
+				<version>1.5.3</version>
+				<executions>
+					<execution>
+						<id>Generate Version Java</id>
+						<phase>generate-sources</phase>
+						<goals>
+							<goal>replace</goal>
+						</goals>
+						<configuration>
+							<file>${basedir}/src/main/code-gen/version/Version.java.template</file>
+							<outputFile>${basedir}/target/generated-sources/java/org/apache/flink/version/Version.java</outputFile>
+						</configuration>
+					</execution>
+				</executions>
+				<configuration>
+					<replacements>
+						<replacement><token>@project.version@</token><value>${project.version}</value></replacement>
+						<replacement><token>@scala.version@</token><value>${scala.binary.version}</value></replacement>
+
+						<replacement><token>@git.build.time@</token><value>${git.build.time}</value></replacement>
+						<replacement><token>@git.build.version@</token><value>${git.build.version}</value></replacement>
+						<replacement><token>@git.commit.id@</token><value>${git.commit.id}</value></replacement>
+						<replacement><token>@git.commit.id.abbrev@</token><value>${git.commit.id.abbrev}</value></replacement>
+						<replacement><token>@git.commit.time@</token><value>${git.commit.time}</value></replacement>
+						<replacement><token>@git.dirty@</token><value>${git.dirty}</value></replacement>
+					</replacements>
+				</configuration>
+			</plugin>
+
 			<!-- Adding scala source directories to build path -->
 			<plugin>
 				<groupId>org.codehaus.mojo</groupId>
@@ -433,6 +465,18 @@ under the License.
 							</sources>
 						</configuration>
 					</execution>
+					<execution>
+						<id>add-generated-sources</id>
+						<phase>generate-sources</phase>
+						<goals>
+							<goal>add-source</goal>
+						</goals>
+						<configuration>
+							<sources>
+								<source>${project.build.directory}/generated-sources/java/</source>
+							</sources>
+						</configuration>
+					</execution>
 				</executions>
 			</plugin>
 
@@ -454,25 +498,26 @@ under the License.
 				</configuration>
 			</plugin>
 			<plugin>
-				<!-- Description: https://github.com/ktoso/maven-git-commit-id-plugin
+				<!-- Description: https://github.com/git-commit-id/git-commit-id-maven-plugin
 					Used to show the git ref when starting the jobManager. -->
 				<groupId>pl.project13.maven</groupId>
 				<artifactId>git-commit-id-plugin</artifactId>
 				<executions>
 					<execution>
+						<id>get-the-git-infos</id>
+						<phase>validate</phase>
 						<goals>
 							<goal>revision</goal>
-						 </goals>
+						</goals>
 					</execution>
 				</executions>
 				<configuration>
 					<dotGitDirectory>${project.basedir}/../.git</dotGitDirectory>
-					<generateGitPropertiesFile>true</generateGitPropertiesFile>
 					<skipPoms>false</skipPoms>
 					<failOnNoGitDirectory>false</failOnNoGitDirectory>
-					<generateGitPropertiesFilename>src/main/resources/.version.properties</generateGitPropertiesFilename>
 					<gitDescribe>
-						<!-- don't generate the describe property -->
+						<!-- Don't generate the describe property -->
+						<!-- It is useless due to the way Flink does branches and tags -->
 						<skip>true</skip>
 					</gitDescribe>
 				</configuration>

flink-runtime/src/main/code-gen/version/Version.java.template

@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.version;
+
+import java.time.Instant;
+import java.time.ZoneId;
+import java.time.format.DateTimeFormatter;
+
+public final class Version {
+	public  static final String  PROJECT_VERSION       = "@project.version@";
+	public  static final String  SCALA_VERSION         = "@scala.version@";
+	public  static final Instant GIT_BUILD_TIME;
+	public  static final String  GIT_BUILD_TIME_STR;
+	public  static final String  GIT_COMMIT_ID         = "@git.commit.id@";
+	public  static final String  GIT_COMMIT_ID_ABBREV  = "@git.commit.id.abbrev@";
+	public  static final Instant GIT_COMMIT_TIME;
+	public  static final String  GIT_COMMIT_TIME_STR;
+	public  static final boolean GIT_DIRTY             = Boolean.parseBoolean("@git.dirty@");
+
+	static {
+		// This is to reliably parse the datetime format configured in the git-commit-id-plugin
+		DateTimeFormatter GIT_DATE_TIME    = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ssZ");
+		GIT_BUILD_TIME                     = GIT_DATE_TIME.parse("@git.build.time@", Instant::from);
+		GIT_COMMIT_TIME                    = GIT_DATE_TIME.parse("@git.commit.time@", Instant::from);
+
+		// Default format is in Berlin timezone because that is where Flink originated.
+		DateTimeFormatter BERLIN_DATE_TIME = DateTimeFormatter.ISO_OFFSET_DATE_TIME.withZone(ZoneId.of("Europe/Berlin"));
+		GIT_BUILD_TIME_STR                 = BERLIN_DATE_TIME.format(GIT_BUILD_TIME);
+		GIT_COMMIT_TIME_STR                = BERLIN_DATE_TIME.format(GIT_COMMIT_TIME);
+	}
+
+}

flink-runtime/src/main/java/org/apache/flink/runtime/util/EnvironmentInformation.java

@@ -18,17 +18,25 @@
 
 package org.apache.flink.runtime.util;
 
-import java.io.InputStream;
+import org.apache.flink.util.OperatingSystem;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import java.lang.management.ManagementFactory;
 import java.lang.management.RuntimeMXBean;
 import java.lang.reflect.Method;
+import java.time.Instant;
 import java.util.List;
-import java.util.Properties;
 
-import org.apache.flink.util.OperatingSystem;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import static org.apache.flink.version.Version.GIT_BUILD_TIME;
+import static org.apache.flink.version.Version.GIT_BUILD_TIME_STR;
+import static org.apache.flink.version.Version.GIT_COMMIT_ID;
+import static org.apache.flink.version.Version.GIT_COMMIT_ID_ABBREV;
+import static org.apache.flink.version.Version.GIT_COMMIT_TIME;
+import static org.apache.flink.version.Version.GIT_COMMIT_TIME_STR;
+import static org.apache.flink.version.Version.GIT_DIRTY;
+import static org.apache.flink.version.Version.PROJECT_VERSION;
+import static org.apache.flink.version.Version.SCALA_VERSION;
 
 /**
  * Utility class that gives access to the execution environment of the JVM, like
@@ -41,14 +49,77 @@ public class EnvironmentInformation {
 	public static final String UNKNOWN = "<unknown>";
 
 	/**
-	 * Returns the version of the code as String. If version == null, then the JobManager does not run from a
-	 * Maven build. An example is a source code checkout, compile, and run from inside an IDE.
-	 * 
-	 * @return The version string.
+	 * Returns the version of the code as String.
+	 * @return The project version string.
 	 */
 	public static String getVersion() {
-		String version = EnvironmentInformation.class.getPackage().getImplementationVersion();
-		return version != null ? version : UNKNOWN;
+		return PROJECT_VERSION;
+	}
+
+	/**
+	 * Returns the version of the used Scala compiler as String.
+	 * @return The scala version string.
+	 */
+	public static String getScalaVersion() {
+		return SCALA_VERSION;
+	}
+
+	/**
+	 * @return The Instant this version of the software was built.
+	 */
+	public static Instant getBuildTime() {
+		return GIT_BUILD_TIME;
+	}
+
+	/**
+	 * @return The Instant this version of the software was built as a String using the Europe/Berlin timezone.
+	 */
+	public static String getBuildTimeString() {
+		return GIT_BUILD_TIME_STR;
+	}
+
+	/**
+	 * @return The last known commit id of this version of the software.
+	 */
+	public static String getGitCommitId() {
+		return GIT_COMMIT_ID;
+	}
+
+	/**
+	 * @return The last known abbreviated commit id of this version of the software.
+	 */
+	public static String getGitCommitIdAbbrev() {
+		return GIT_COMMIT_ID_ABBREV;
+	}
+
+	/**
+	 * @return The Instant of the last commit of this code.
+	 */
+	public static Instant getGitCommitTime() {
+		return GIT_COMMIT_TIME;
+	}
+
+	/**
+	 * @return The Instant of the last commit of this code as a String using the Europe/Berlin timezone.
+	 */
+	public static String getGitCommitTimeString() {
+		return GIT_COMMIT_TIME_STR;
+	}
+
+	/**
+	 * @return Does this built have modifications that have not yet been committed.
+	 */
+	public static boolean getGitDirty() {
+		return GIT_DIRTY;
+	}
+
+	/**
+	 * Returns the exact name of the Dockerimage for Flink that is needed to run in.
+	 *
+	 * @return The "name:tag" of the Flink docker image.
+	 */
+	public static String getFlinkDockerImageNameAndTag() {
+		return "flink:" + EnvironmentInformation.getVersion() + "-scala_" + EnvironmentInformation.getScalaVersion();
 	}
 
 	/**
@@ -57,26 +128,7 @@ public static String getVersion() {
 	 * @return The code revision.
 	 */
 	public static RevisionInformation getRevisionInformation() {
-		String revision = UNKNOWN;
-		String commitDate = UNKNOWN;
-		try (InputStream propFile = EnvironmentInformation.class.getClassLoader().getResourceAsStream(".version.properties")) {
-			if (propFile != null) {
-				Properties properties = new Properties();
-				properties.load(propFile);
-				String propRevision = properties.getProperty("git.commit.id.abbrev");
-				String propCommitDate = properties.getProperty("git.commit.time");
-				revision = propRevision != null ? propRevision : UNKNOWN;
-				commitDate = propCommitDate != null ? propCommitDate : UNKNOWN;
-			}
-		} catch (Throwable t) {
-			if (LOG.isDebugEnabled()) {
-				LOG.debug("Cannot determine code revision: Unable to read version property file.", t);
-			} else {
-				LOG.info("Cannot determine code revision: Unable to read version property file.");
-			}
-		}
-
-		return new RevisionInformation(revision, commitDate);
+		return new RevisionInformation(GIT_COMMIT_ID_ABBREV, GIT_BUILD_TIME_STR);
 	}
 
 	/**
@@ -270,6 +322,7 @@ public static void logEnvironmentInfo(Logger log, String componentName, String[]
 		if (log.isInfoEnabled()) {
 			RevisionInformation rev = getRevisionInformation();
 			String version = getVersion();
+			String scalaVersion = getScalaVersion();
 
 			String jvmVersion = getJvmVersion();
 			String[] options = getJvmStartupOptionsArray();
@@ -287,7 +340,7 @@ public static void logEnvironmentInfo(Logger log, String componentName, String[]
 			}
 
 			log.info("--------------------------------------------------------------------------------");
-			log.info(" Starting " + componentName + " (Version: " + version + ", "
+			log.info(" Starting " + componentName + " (Version: " + version + ", Scala: " + scalaVersion + ", "
 					+ "Rev:" + rev.commitId + ", " + "Date:" + rev.commitDate + ")");
 			log.info(" OS current user: " + System.getProperty("user.name"));
 			log.info(" Current Hadoop/Kerberos user: " + getHadoopUser());

flink-runtime/src/test/java/org/apache/flink/runtime/util/EnvironmentInformationTest.java

@@ -54,15 +54,26 @@ public void testJavaMemory() {
 			fail(e.getMessage());
 		}
 	}
-	
+
 	@Test
 	public void testEnvironmentMethods() {
 		try {
 			assertNotNull(EnvironmentInformation.getJvmStartupOptions());
 			assertNotNull(EnvironmentInformation.getJvmStartupOptionsArray());
 			assertNotNull(EnvironmentInformation.getJvmVersion());
+
 			assertNotNull(EnvironmentInformation.getRevisionInformation());
 			assertNotNull(EnvironmentInformation.getVersion());
+			assertNotNull(EnvironmentInformation.getScalaVersion());
+			assertNotNull(EnvironmentInformation.getFlinkDockerImageNameAndTag());
+
+			assertNotNull(EnvironmentInformation.getBuildTime());
+			assertNotNull(EnvironmentInformation.getBuildTimeString());
+			assertNotNull(EnvironmentInformation.getGitCommitId());
+			assertNotNull(EnvironmentInformation.getGitCommitIdAbbrev());
+			assertNotNull(EnvironmentInformation.getGitCommitTime());
+			assertNotNull(EnvironmentInformation.getGitCommitTimeString());
+
 			EnvironmentInformation.getHadoopVersionString();
 			assertNotNull(EnvironmentInformation.getHadoopUser());
 			assertTrue(EnvironmentInformation.getOpenFileHandlesLimit() >= -1);

pom.xml

@@ -1811,14 +1811,18 @@ under the License.
 				<plugin>
 					<groupId>pl.project13.maven</groupId>
 					<artifactId>git-commit-id-plugin</artifactId>
-					<!-- Don't use 2.1.14 as it is incompatible with various maven versions -->
-					<version>2.1.10</version>
+					<version>4.0.0</version>
 					<configuration>
 						<excludeProperties>
-							<excludeProperty>git.build.*</excludeProperty>
+							<excludeProperty>git.build.user.*</excludeProperty>
+							<excludeProperty>git.commit.user.*</excludeProperty>
 							<excludeProperty>git.branch.*</excludeProperty>
 							<excludeProperty>git.remote.*</excludeProperty>
 						</excludeProperties>
+
+						<!-- This is the same as the default date time format of this plugin.      -->
+						<!-- Because we want to parse the result were fixing it to avoid problems. -->
+						<dateFormat>yyyy-MM-dd'T'HH:mm:ssZ</dateFormat>
 					</configuration>
 				</plugin>