PUB-225: Java 11 migration (#513)

* PUB-225: Java 11 migration

* Update to create 1.8 binaries but still default to 11 runtime

* Fix flaky unit test

README.md

@@ -5,7 +5,7 @@
 The [Wavefront Proxy](https://docs.wavefront.com/proxies.html) is a light-weight Java application that you send your metrics, histograms, and trace data to. It handles batching and transmission of your data to the Wavefront service in a secure, fast, and reliable manner.
 
 ## Requirements
-  * Java >= 1.8
+  * Java 8 or higher
   * Maven
 
 ## Overview

pkg/after-install.sh

@@ -46,23 +46,23 @@ chown $user:$group $conf_dir/$service_name
 
 if [[ ! -f $conf_dir/$service_name/wavefront.conf ]]; then
     if [[ -f $wavefront_dir/$service_name/conf/wavefront.conf ]]; then
-        echo "Copying $conf_dir/$service_name/wavefront.conf from $wavefront_dir/$service_name/conf/wavefront.conf"
+        echo "Copying $conf_dir/$service_name/wavefront.conf from $wavefront_dir/$service_name/conf/wavefront.conf" >&2
         cp $wavefront_dir/$service_name/conf/wavefront.conf $conf_dir/$service_name/wavefront.conf
     else
-        echo "Creating $conf_dir/$service_name/wavefront.conf from template"
+        echo "Creating $conf_dir/$service_name/wavefront.conf from default template" >&2
         cp $conf_dir/$service_name/wavefront.conf.default $conf_dir/$service_name/wavefront.conf
     fi
 else
    echo "$conf_dir/$service_name/wavefront.conf already exists"
 fi
 
 if [[ ! -f $conf_dir/$service_name/preprocessor_rules.yaml ]]; then
-    echo "Creating $conf_dir/$service_name/preprocessor_rules.yaml from template"
+    echo "Creating $conf_dir/$service_name/preprocessor_rules.yaml from default template" >&2
     cp $conf_dir/$service_name/preprocessor_rules.yaml.default $conf_dir/$service_name/preprocessor_rules.yaml
 fi
 
 if [[ ! -f $conf_dir/$service_name/log4j2.xml ]]; then
-    echo "Creating $conf_dir/$service_name/log4j2.xml from template"
+    echo "Creating $conf_dir/$service_name/log4j2.xml from default template" >&2
     cp $conf_dir/$service_name/log4j2.xml.default $conf_dir/$service_name/log4j2.xml
 fi
 

pkg/before-remove.sh

@@ -15,8 +15,10 @@ jre_dir="$wavefront_dir/$service_name/proxy-jre"
 #  - https://www.debian.org/doc/debian-policy/ch-maintainerscripts.html
 if [[ "$1" == "0" ]] || [[ "$1" == "remove" ]] || [[ "$1" == "purge" ]]; then
     service wavefront-proxy stop || true
-    echo "Removing installed JRE from $jre_dir"
-    rm -rf $jre_dir
+    if [ -d $jre_dir ]; then
+      [ "$(ls -A $jre_dir)" ] && echo "Removing installed JRE from $jre_dir" >&2
+      rm -rf $jre_dir
+    fi
 fi
 
 exit 0

pkg/etc/init.d/wavefront-proxy

@@ -53,8 +53,8 @@ fi
 daemon_log_file=${DAEMON_LOG_FILE:-/var/log/wavefront/wavefront-daemon.log}
 err_file="/var/log/wavefront/wavefront-error.log"
 pid_file=${PID_FILE:-/var/run/$service_name.pid}
-agent_jar=${AGENT_JAR:-$proxy_dir/bin/wavefront-push-agent.jar}
-class="com.wavefront.agent.PushAgentDaemon"
+proxy_jar=${AGENT_JAR:-$proxy_dir/bin/wavefront-proxy.jar}
+class="com.wavefront.agent.WavefrontProxyService"
 app_args=${APP_ARGS:--f $conf_file}
 
 # If JAVA_ARGS is not set, try to detect memory size and set heap to 8GB if machine has more than 8GB.
@@ -85,38 +85,51 @@ if [[ -r $proxy_launch_conf ]]; then
 	fi
 fi
 
+# Workaround for environments with locked-down internet access where approved traffic goes through HTTP proxy.
+# If JRE cannot be found in $proxy_jre_dir (most likely because its download failed during installation), and
+# $PROXY_JAVA_HOME pointing to a user-defined JRE is not defined either, we'll try to read HTTP proxy settings
+# from wavefront.conf, if any, and try to download the JRE again. Ideally we should go back to bundling JRE
+# and not having to worry about accessibility of external resources.
 download_jre() {
     [[ -d $proxy_jre_dir ]] || mkdir -p $proxy_jre_dir
 
-    echo "Checking $conf_file for HTTP proxy settings"
+    echo "Checking $conf_file for HTTP proxy settings" >&2
     proxy_host=$(grep "^\s*proxyHost=" $conf_file | cut -d'=' -f2)
     proxy_port=$(grep "^\s*proxyPort=" $conf_file | cut -d'=' -f2)
     proxy_user=$(grep "^\s*proxyUser=" $conf_file | cut -d'=' -f2)
     proxy_password=$(grep "^\s*proxyPassword=" $conf_file | cut -d'=' -f2)
 
     if [[ -n $proxy_host && -n $proxy_port ]]; then
-        echo "Using HTTP proxy $proxy_host:$proxy_port"
+        echo "Using HTTP proxy $proxy_host:$proxy_port" >&2
         proxy_args="--proxy $proxy_host:$proxy_port"
         if [[ -n $proxy_user && -n $proxy_password ]]; then
-            echo "Authenticating as $proxy_user"
+            echo "Authenticating as $proxy_user" >&2
             proxy_args+=" --proxy-user $proxy_user:$proxy_password"
         fi
     else
-        echo "No HTTP proxy configuration detected - attempting direct download"
+        echo "No HTTP proxy configuration detected - attempting direct download" >&2
     fi
-    curl -L --silent -o /tmp/jre.tar.gz $proxy_args https://s3-us-west-2.amazonaws.com/wavefront-misc/proxy-jre.tgz || true
+    curl -L --silent -o /tmp/jre.tar.gz $proxy_args https://s3-us-west-2.amazonaws.com/wavefront-misc/proxy-jre-11.0.6-linux_x64.tar.gz || true
     tar -xf /tmp/jre.tar.gz --strip 1 -C $proxy_jre_dir || true
     rm /tmp/jre.tar.gz || true
 }
 
-# Workaround for environments with locked-down internet access where approved traffic goes through HTTP proxy.
-# If JRE cannot be found in $proxy_jre_dir (most likely because its download failed during installation), and
-# $PROXY_JAVA_HOME pointing to a user-defined JRE is not defined either, we'll try to read HTTP proxy settings
-# from wavefront.conf, if any, and try to download the JRE again. Ideally we should go back to bundling JRE
-# and not having to worry about accessibility of external resources.
+# If $PROXY_JAVA_HOME is not defined and there is no JRE in $proxy_jre_dir, try to auto-detect
+# locally installed JDK first. We will accept 1.8, 9, 10, 11.
 if [[ -z "$PROXY_JAVA_HOME" && ! -r $proxy_jre_dir/bin/java ]]; then
-    echo "JRE not found - trying to download and install"
-    download_jre
+    if type -p java; then
+        JAVA_VERSION=$(java -version 2>&1 | head -1 | cut -d'"' -f2 | sed '/^1\./s///' | cut -d'.' -f1)
+        if [[ $JAVA_VERSION == "11" || $JAVA_VERSION == "10" || $JAVA_VERSION == "9" || $JAVA_VERSION == "1.8" ]]; then
+            JAVA_HOME=$($(dirname $(dirname $(readlink -f $(which javac)))))
+            echo "Using Java runtime $JAVA_VERSION detected in $JAVA_HOME" >&2
+        else
+            echo "Java runtime [1.8; 12) not found - trying to download and install" >&2
+            download_jre
+        fi
+    else
+        echo "Java runtime not found - trying to download and install" >&2
+        download_jre
+    fi
 fi
 
 jsvc=$proxy_dir/bin/jsvc
@@ -128,15 +141,15 @@ jsvc_exec()
         > $err_file
     fi
 
-	cd "$(dirname "$agent_jar")"
+	cd "$(dirname "$proxy_jar")"
 
     set +e
 	# We want word splitting below, as we're building up a command line.
 	# shellcheck disable=SC2086
 	$jsvc \
 		-user $user \
 		-home $JAVA_HOME \
-		-cp $agent_jar \
+		-cp $proxy_jar \
 		$java_args \
 		-Xss2049k \
 		-XX:OnOutOfMemoryError="kill -1 %p" \
@@ -150,7 +163,7 @@ jsvc_exec()
 		$class \
 		$app_args &> $daemon_log_file
 	if [[ $? -ne 0 ]]; then
-		echo "There was a problem, see $err_file and $daemon_log_file"
+		echo "There was a problem, see $err_file and $daemon_log_file" >&2
 	fi
 }
 

pkg/stage.sh

@@ -61,4 +61,4 @@ ln -s ../commons-daemon/src/native/unix/jsvc jsvc
 
 echo "Stage the agent jar..."
 cd $PROG_DIR
-cp $PUSH_AGENT_JAR $PROXY_DIR/bin/wavefront-push-agent.jar
+cp $PUSH_AGENT_JAR $PROXY_DIR/bin/wavefront-proxy.jar

pom.xml

@@ -10,8 +10,8 @@
   </modules>
   <packaging>pom</packaging>
 
-  <name>Wavefront All-in-One</name>
-  <description>Top-level Wavefront Public Java POM</description>
+  <name>Wavefront Proxy</name>
+  <description>Wavefront Proxy Project</description>
   <url>http://www.wavefront.com</url>
   <licenses>
     <license>
@@ -58,7 +58,7 @@
     <jackson.version>2.9.10</jackson.version>
     <jackson-databind.version>2.9.10.3</jackson-databind.version>
     <netty.version>4.1.45.Final</netty.version>
-    <java-lib.version>2020-02.2</java-lib.version>
+    <java-lib.version>2020-03.4</java-lib.version>
 
     <doclint>none</doclint>
   </properties>

proxy/docker/Dockerfile

@@ -17,7 +17,7 @@ RUN apt-get install -y sudo
 RUN apt-get install -y gnupg2
 RUN apt-get install -y debian-archive-keyring
 RUN apt-get install -y apt-transport-https
-RUN apt-get install -y openjdk-8-jdk
+RUN apt-get install -y openjdk-11-jdk
 
 # Download wavefront proxy (latest release). Merely extract the debian, don't want to try running startup scripts.
 RUN echo "deb https://packagecloud.io/wavefront/proxy/ubuntu/ bionic main" > /etc/apt/sources.list.d/wavefront_proxy.list

proxy/docker/Dockerfile-rhel

@@ -2,7 +2,6 @@
 FROM registry.access.redhat.com/ubi7
 USER root
 
-
 # This script may automatically configure wavefront without prompting, based on
 # these variables:
 #  WAVEFRONT_URL           (required)
@@ -12,17 +11,17 @@ USER root
 #  WAVEFRONT_PROXY_ARGS    (default is none)
 #  JAVA_ARGS               (default is none)
 
+RUN yum-config-manager --enable rhel-7-server-optional-rpms
 RUN yum update --disableplugin=subscription-manager -y && rm -rf /var/cache/yum
 
 RUN yum install -y sudo
 RUN yum install -y curl
 RUN yum install -y hostname
-RUN yum install -y java-1.8.0-openjdk-devel.x86_64
+RUN yum install -y java-11-openjdk-devel
 
-# Download wavefront proxy (latest release). Merely extract the debian, don't want to try running startup scripts.
+# Download wavefront proxy (latest release). Merely extract the package, don't want to try running startup scripts.
 RUN curl -s https://packagecloud.io/install/repositories/wavefront/proxy/script.rpm.sh | sudo bash
 RUN yum -y update
-
 RUN yum -y -q install wavefront-proxy
 
 # Configure agent

proxy/docker/run.sh

@@ -25,7 +25,7 @@ java \
     $jvm_container_opts $JAVA_ARGS \
 	-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager \
 	-Dlog4j.configurationFile=/etc/wavefront/wavefront-proxy/log4j2.xml \
-	-jar /opt/wavefront/wavefront-proxy/bin/wavefront-push-agent.jar \
+	-jar /opt/wavefront/wavefront-proxy/bin/wavefront-proxy.jar \
 	-h $WAVEFRONT_URL \
 	-t $WAVEFRONT_TOKEN \
 	--hostname ${WAVEFRONT_HOSTNAME:-$(hostname)} \

proxy/src/main/java/com/wavefront/agent/PushAgentDaemon.java → proxy/src/main/java/com/wavefront/agent/WavefrontProxyService.java

@@ -6,7 +6,7 @@
 /**
  * @author Mori Bellamy (mori@wavefront.com)
  */
-public class PushAgentDaemon implements Daemon {
+public class WavefrontProxyService implements Daemon {
 
     private PushAgent agent;
     private DaemonContext daemonContext;

proxy/src/test/java/com/wavefront/agent/PointMatchers.java

@@ -6,6 +6,7 @@
 
 import java.util.Map;
 
+import wavefront.report.Histogram;
 import wavefront.report.ReportPoint;
 
 /**
@@ -98,4 +99,26 @@ public void describeTo(Description description) {
     };
   }
 
+  public static Matcher<ReportPoint> histogramMatches(int samples, double weight) {
+    return new BaseMatcher<ReportPoint>() {
+
+      @Override
+      public boolean matches(Object o) {
+        ReportPoint point = (ReportPoint) o;
+        if (!(point.getValue() instanceof Histogram)) return false;
+        Histogram value = (Histogram) point.getValue();
+        double sum = 0;
+        for (int i = 0; i < value.getBins().size(); i++) {
+          sum += value.getBins().get(i) * value.getCounts().get(i);
+        }
+        return sum == weight && value.getCounts().stream().reduce(Integer::sum).get() == samples;
+      }
+
+      @Override
+      public void describeTo(Description description) {
+        description.appendText(
+            "Total histogram weight should be " + weight + ", and total samples = " + samples);
+      }
+    };
+  }
 }

proxy/src/test/java/com/wavefront/agent/logsharvesting/LogsIngesterTest.java

@@ -1,12 +1,28 @@
 package com.wavefront.agent.logsharvesting;
 
+import java.io.File;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.Consumer;
+
+import org.easymock.Capture;
+import org.easymock.CaptureType;
+import org.easymock.EasyMock;
+import org.junit.After;
+import org.junit.Test;
+import org.logstash.beats.Message;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
 import com.wavefront.agent.PointMatchers;
 import com.wavefront.agent.auth.TokenAuthenticatorBuilder;
 import com.wavefront.agent.channel.NoopHealthCheckManager;
@@ -20,23 +36,6 @@
 import com.wavefront.common.MetricConstants;
 import com.wavefront.data.ReportableEntityType;
 
-import org.easymock.Capture;
-import org.easymock.CaptureType;
-import org.easymock.EasyMock;
-import org.junit.After;
-import org.junit.Test;
-import org.logstash.beats.Message;
-
-import java.io.File;
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.function.Consumer;
-
 import io.netty.channel.Channel;
 import io.netty.channel.ChannelHandlerContext;
 import oi.thekraken.grok.api.exception.GrokException;
@@ -54,7 +53,6 @@
 import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.emptyIterable;
 import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.lessThan;
@@ -568,24 +566,9 @@ public void testWavefrontHistogramMultipleCentroids() throws Exception {
       lines[i] = "histo " + (i + 1);
     }
     List<ReportPoint> reportPoints = getPoints(mockHistogramHandler, 2, 500, this::receiveLog, lines);
-    ReportPoint reportPoint = reportPoints.get(0);
-    assertThat(reportPoint.getValue(), instanceOf(Histogram.class));
-    Histogram wavefrontHistogram = (Histogram) reportPoint.getValue();
-    double sum = 0;
-    for (int i = 0; i < wavefrontHistogram.getBins().size(); i++) {
-      sum += wavefrontHistogram.getBins().get(i) * wavefrontHistogram.getCounts().get(i);
-    }
-    assertThat(sum, equalTo(7260.0));
-    assertThat(wavefrontHistogram.getCounts().stream().reduce(Integer::sum).get(), equalTo(120));
-    reportPoint = reportPoints.get(1);
-    assertThat(reportPoint.getValue(), instanceOf(Histogram.class));
-    wavefrontHistogram = (Histogram) reportPoint.getValue();
-    sum = 0;
-    for (int i = 0; i < wavefrontHistogram.getBins().size(); i++) {
-      sum += wavefrontHistogram.getBins().get(i) * wavefrontHistogram.getCounts().get(i);
-    }
-    assertThat(sum, equalTo(21660.0));
-    assertThat(wavefrontHistogram.getCounts().stream().reduce(Integer::sum).get(), equalTo(120));
+    assertThat(reportPoints.size(), equalTo(2));
+    assertThat(reportPoints, containsInAnyOrder(PointMatchers.histogramMatches(120, 7260.0),
+        PointMatchers.histogramMatches(120, 21660.0)));
   }
 
   @Test(expected = ConfigurationException.class)