Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[FLINK-16871][runtime] Make more build time information available at runtime #11592

Merged
merged 1 commit into from
May 6, 2020
Merged

[FLINK-16871][runtime] Make more build time information available at runtime #11592

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
23 changes: 0 additions & 23 deletions flink-dist/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -841,29 +841,6 @@ under the License.
</configuration>
</plugin>

<plugin>
<!-- Description: https://github.com/ktoso/maven-git-commit-id-plugin -->
<groupId>pl.project13.maven</groupId>
<artifactId>git-commit-id-plugin</artifactId>
<executions>
<execution>
<goals>
<goal>revision</goal>
</goals>
</execution>
</executions>
<configuration>
<dotGitDirectory>${project.basedir}/../.git</dotGitDirectory>
<generateGitPropertiesFile>true</generateGitPropertiesFile>
<failOnNoGitDirectory>false</failOnNoGitDirectory>
<skipPoms>false</skipPoms>
<generateGitPropertiesFilename>src/main/flink-bin/.version.properties</generateGitPropertiesFilename>
<gitDescribe>
<!-- don't generate the describe property -->
<skip>true</skip>
</gitDescribe>
</configuration>
</plugin>
</plugins>
</build>

Expand Down
23 changes: 18 additions & 5 deletions flink-runtime/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -316,6 +316,18 @@ under the License.
</dependencyManagement>

<build>

<resources>
<resource>
<directory>src/main/resources</directory>
<filtering>false</filtering>
</resource>
<resource>
<directory>src/main/resources-filtered</directory>
<filtering>true</filtering>
</resource>
</resources>

<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
Expand Down Expand Up @@ -454,25 +466,26 @@ under the License.
</configuration>
</plugin>
<plugin>
<!-- Description: https://github.com/ktoso/maven-git-commit-id-plugin
<!-- Description: https://github.com/git-commit-id/git-commit-id-maven-plugin
Used to show the git ref when starting the jobManager. -->
<groupId>pl.project13.maven</groupId>
<artifactId>git-commit-id-plugin</artifactId>
<executions>
<execution>
<id>get-the-git-infos</id>
<phase>validate</phase>
<goals>
<goal>revision</goal>
</goals>
</goals>
</execution>
</executions>
<configuration>
<dotGitDirectory>${project.basedir}/../.git</dotGitDirectory>
<generateGitPropertiesFile>true</generateGitPropertiesFile>
<skipPoms>false</skipPoms>
<failOnNoGitDirectory>false</failOnNoGitDirectory>
<generateGitPropertiesFilename>src/main/resources/.version.properties</generateGitPropertiesFilename>
<gitDescribe>
<!-- don't generate the describe property -->
<!-- Don't generate the describe property -->
<!-- It is useless due to the way Flink does branches and tags -->
<skip>true</skip>
</gitDescribe>
</configuration>
Expand Down
166 changes: 136 additions & 30 deletions flink-runtime/src/main/java/org/apache/flink/runtime/util/EnvironmentInformation.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,18 +18,23 @@

package org.apache.flink.runtime.util;

import org.apache.flink.util.OperatingSystem;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.io.InputStream;
import java.lang.management.ManagementFactory;
import java.lang.management.RuntimeMXBean;
import java.lang.reflect.Method;
import java.time.Instant;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeParseException;
import java.util.List;
import java.util.Properties;

import org.apache.flink.util.OperatingSystem;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
* Utility class that gives access to the execution environment of the JVM, like
* the executing user, startup options, or the JVM version.
Expand All @@ -41,42 +46,142 @@ public class EnvironmentInformation {
public static final String UNKNOWN = "<unknown>";

/**
* Returns the version of the code as String. If version == null, then the JobManager does not run from a
* Maven build. An example is a source code checkout, compile, and run from inside an IDE.
*
* @return The version string.
* Returns the version of the code as String.
* @return The project version string.
*/
public static String getVersion() {
String version = EnvironmentInformation.class.getPackage().getImplementationVersion();
return version != null ? version : UNKNOWN;
return getVersionsInstance().projectVersion;
}

/**
* Returns the version of the used Scala compiler as String.
* @return The scala version string.
*/
public static String getScalaVersion() {
return getVersionsInstance().scalaVersion;
}

/**
* @return The Instant this version of the software was built.
*/
public static Instant getBuildTime() {
return getVersionsInstance().gitBuildTime;
}

/**
* @return The Instant this version of the software was built as a String using the Europe/Berlin timezone.
*/
public static String getBuildTimeString() {
return getVersionsInstance().gitBuildTimeStr;
}

/**
* @return The last known commit id of this version of the software.
*/
public static String getGitCommitId() {
return getVersionsInstance().gitCommitId;
}

/**
* @return The last known abbreviated commit id of this version of the software.
*/
public static String getGitCommitIdAbbrev() {
return getVersionsInstance().gitCommitIdAbbrev;
}

/**
* @return The Instant of the last commit of this code.
*/
public static Instant getGitCommitTime() {
return getVersionsInstance().gitCommitTime;
}

/**
* @return The Instant of the last commit of this code as a String using the Europe/Berlin timezone.
*/
public static String getGitCommitTimeString() {
return getVersionsInstance().gitCommitTimeStr;
}

/**
* Returns the code revision (commit and commit date) of Flink, as generated by the Maven builds.
*
*
* @return The code revision.
*/
public static RevisionInformation getRevisionInformation() {
String revision = UNKNOWN;
String commitDate = UNKNOWN;
try (InputStream propFile = EnvironmentInformation.class.getClassLoader().getResourceAsStream(".version.properties")) {
if (propFile != null) {
Properties properties = new Properties();
properties.load(propFile);
String propRevision = properties.getProperty("git.commit.id.abbrev");
String propCommitDate = properties.getProperty("git.commit.time");
revision = propRevision != null ? propRevision : UNKNOWN;
commitDate = propCommitDate != null ? propCommitDate : UNKNOWN;
return new RevisionInformation(getGitCommitIdAbbrev(), getGitCommitTimeString());
}

private static final class Versions {
private static final Instant DEFAULT_TIME_INSTANT = Instant.EPOCH;
private static final String DEFAULT_TIME_STRING = "1970-01-01T00:00:00+0000";
private static final String UNKNOWN_COMMIT_ID = "DecafC0ffeeD0d0F00d";
private static final String UNKNOWN_COMMIT_ID_ABBREV = "DeadD0d0";
private String projectVersion = UNKNOWN;
private String scalaVersion = UNKNOWN;
private Instant gitBuildTime = DEFAULT_TIME_INSTANT;
private String gitBuildTimeStr = DEFAULT_TIME_STRING;
private String gitCommitId = UNKNOWN_COMMIT_ID;
private String gitCommitIdAbbrev = UNKNOWN_COMMIT_ID_ABBREV;
private Instant gitCommitTime = DEFAULT_TIME_INSTANT;
private String gitCommitTimeStr = DEFAULT_TIME_STRING;

private static final String PROP_FILE = ".flink-runtime.version.properties";

private static final String FAIL_MESSAGE = "The file " + PROP_FILE + " has not been generated correctly. You MUST run 'mvn generate-sources' in the flink-runtime module.";

private String getProperty(Properties properties, String key, String defaultValue) {
String value = properties.getProperty(key);
if (value == null || value.charAt(0) == '$') {
nielsbasjes marked this conversation as resolved.
Show resolved Hide resolved
return defaultValue;
}
} catch (Throwable t) {
if (LOG.isDebugEnabled()) {
LOG.debug("Cannot determine code revision: Unable to read version property file.", t);
} else {
LOG.info("Cannot determine code revision: Unable to read version property file.");
return value;
}

public Versions() {
ClassLoader classLoader = EnvironmentInformation.class.getClassLoader();
try (InputStream propFile = classLoader.getResourceAsStream(PROP_FILE)) {
if (propFile != null) {
Properties properties = new Properties();
properties.load(propFile);

projectVersion = getProperty(properties, "project.version", UNKNOWN);
scalaVersion = getProperty(properties, "scala.binary.version", UNKNOWN);

gitCommitId = getProperty(properties, "git.commit.id", UNKNOWN_COMMIT_ID);
gitCommitIdAbbrev = getProperty(properties, "git.commit.id.abbrev", UNKNOWN_COMMIT_ID_ABBREV);

// This is to reliably parse the datetime format configured in the git-commit-id-plugin
DateTimeFormatter gitDateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ssZ");

// Default format is in Berlin timezone because that is where Flink originated.
DateTimeFormatter berlinDateTime = DateTimeFormatter.ISO_OFFSET_DATE_TIME.withZone(ZoneId.of("Europe/Berlin"));

try {
String propGitCommitTime = getProperty(properties, "git.commit.time", DEFAULT_TIME_STRING);
gitCommitTime = gitDateTimeFormatter.parse(propGitCommitTime, Instant::from);
gitCommitTimeStr = berlinDateTime.format(gitCommitTime);

String propGitBuildTime = getProperty(properties, "git.build.time", DEFAULT_TIME_STRING);
gitBuildTime = gitDateTimeFormatter.parse(propGitBuildTime, Instant::from);
gitBuildTimeStr = berlinDateTime.format(gitBuildTime);
} catch (DateTimeParseException dtpe) {
LOG.error("{} : {}", FAIL_MESSAGE, dtpe);
throw new IllegalStateException(FAIL_MESSAGE);
}
}
} catch (IOException ioe) {
LOG.info("Cannot determine code revision: Unable to read version property file.: {}", ioe.getMessage());
}
}

return new RevisionInformation(revision, commitDate);
}

private static final class VersionsHolder {
nielsbasjes marked this conversation as resolved.
Show resolved Hide resolved
static final Versions INSTANCE = new Versions();
}

private static Versions getVersionsInstance() {
return VersionsHolder.INSTANCE;
}

/**
Expand Down Expand Up @@ -270,6 +375,7 @@ public static void logEnvironmentInfo(Logger log, String componentName, String[]
if (log.isInfoEnabled()) {
RevisionInformation rev = getRevisionInformation();
String version = getVersion();
String scalaVersion = getScalaVersion();

String jvmVersion = getJvmVersion();
String[] options = getJvmStartupOptionsArray();
Expand All @@ -287,7 +393,7 @@ public static void logEnvironmentInfo(Logger log, String componentName, String[]
}

log.info("--------------------------------------------------------------------------------");
log.info(" Starting " + componentName + " (Version: " + version + ", "
log.info(" Starting " + componentName + " (Version: " + version + ", Scala: " + scalaVersion + ", "
+ "Rev:" + rev.commitId + ", " + "Date:" + rev.commitDate + ")");
log.info(" OS current user: " + System.getProperty("user.name"));
log.info(" Current Hadoop/Kerberos user: " + getHadoopUser());
Expand Down
22 changes: 22 additions & 0 deletions flink-runtime/src/main/resources-filtered/.flink-runtime.version.properties
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

project.version=${project.version}
scala.binary.version=${scala.binary.version}

git.commit.id=${git.commit.id}
git.commit.id.abbrev=${git.commit.id.abbrev}
git.commit.time=${git.commit.time}
git.build.time=${git.build.time}
32 changes: 31 additions & 1 deletion flink-runtime/src/test/java/org/apache/flink/runtime/util/EnvironmentInformationTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,14 @@

import static org.junit.Assert.*;

import org.apache.flink.util.TestLogger;
import org.junit.Test;
import org.mockito.Mockito;
import org.slf4j.Logger;

public class EnvironmentInformationTest {
import java.util.Arrays;

public class EnvironmentInformationTest extends TestLogger {

@Test
public void testJavaMemory() {
Expand Down Expand Up @@ -63,9 +66,36 @@ public void testEnvironmentMethods() {
assertNotNull(EnvironmentInformation.getJvmVersion());
assertNotNull(EnvironmentInformation.getRevisionInformation());
assertNotNull(EnvironmentInformation.getVersion());
assertNotNull(EnvironmentInformation.getScalaVersion());
assertNotNull(EnvironmentInformation.getBuildTime());
assertNotNull(EnvironmentInformation.getBuildTimeString());
assertNotNull(EnvironmentInformation.getGitCommitId());
assertNotNull(EnvironmentInformation.getGitCommitIdAbbrev());
assertNotNull(EnvironmentInformation.getGitCommitTime());
assertNotNull(EnvironmentInformation.getGitCommitTimeString());
EnvironmentInformation.getHadoopVersionString();
assertNotNull(EnvironmentInformation.getHadoopUser());
assertTrue(EnvironmentInformation.getOpenFileHandlesLimit() >= -1);

if (log.isInfoEnabled()) {
// Visual inspection of the available Environment variables
// To actually see it set "rootLogger.level = INFO" in "log4j2-test.properties"
log.info("JvmStartupOptions : {}", EnvironmentInformation.getJvmStartupOptions());
log.info("JvmStartupOptionsArray : {}", Arrays.asList(EnvironmentInformation.getJvmStartupOptionsArray()));
log.info("JvmVersion : {}", EnvironmentInformation.getJvmVersion());
log.info("RevisionInformation : {}", EnvironmentInformation.getRevisionInformation());
log.info("Version : {}", EnvironmentInformation.getVersion());
log.info("ScalaVersion : {}", EnvironmentInformation.getScalaVersion());
log.info("BuildTime : {}", EnvironmentInformation.getBuildTime());
log.info("BuildTimeString : {}", EnvironmentInformation.getBuildTimeString());
log.info("GitCommitId : {}", EnvironmentInformation.getGitCommitId());
log.info("GitCommitIdAbbrev : {}", EnvironmentInformation.getGitCommitIdAbbrev());
log.info("GitCommitTime : {}", EnvironmentInformation.getGitCommitTime());
log.info("GitCommitTimeString : {}", EnvironmentInformation.getGitCommitTimeString());
log.info("HadoopVersionString : {}", EnvironmentInformation.getHadoopVersionString());
log.info("HadoopUser : {}", EnvironmentInformation.getHadoopUser());
log.info("OpenFileHandlesLimit : {}", EnvironmentInformation.getOpenFileHandlesLimit());
}
}
catch (Exception e) {
e.printStackTrace();
Expand Down
10 changes: 7 additions & 3 deletions pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1814,14 +1814,18 @@ under the License.
<plugin>
<groupId>pl.project13.maven</groupId>
<artifactId>git-commit-id-plugin</artifactId>
<!-- Don't use 2.1.14 as it is incompatible with various maven versions -->
<version>2.1.10</version>
<version>4.0.0</version>
nielsbasjes marked this conversation as resolved.
Show resolved Hide resolved
<configuration>
<excludeProperties>
<excludeProperty>git.build.*</excludeProperty>
<excludeProperty>git.build.user.*</excludeProperty>
<excludeProperty>git.commit.user.*</excludeProperty>
<excludeProperty>git.branch.*</excludeProperty>
<excludeProperty>git.remote.*</excludeProperty>
</excludeProperties>

<!-- This is the same as the default date time format of this plugin. -->
<!-- Because we want to parse the result we are fixing it to avoid problems. -->
<dateFormat>yyyy-MM-dd'T'HH:mm:ssZ</dateFormat>
</configuration>
</plugin>

Expand Down