[#600] chore(operator): change JDK base from openjdk to eclipse-temur…

…in (#617)

### What changes were proposed in this pull request?
1. replace openjdk with eclipse-temurin
2. split rss-server images into two parts: base and upper
3. fix lsof location in start.sh
4. enable rss-server image build on ci

### Why are the changes needed?
Fixes #600 

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
Manually verified

.github/workflows/build.yml

@@ -82,6 +82,6 @@ jobs:
     if: needs.changes.outputs.kubernetes == 'true' ||  github.event_name == 'push'
     uses: ./.github/workflows/single.yml
     with:
-      maven-args: package -Pkubernetes -DskipUTs -DskipITs
+      maven-args: package -Pkubernetes -DskipUTs -DskipITs -DskipBuildImage=${{ needs.changes.outputs.rss_server_docker == 'false' }}
       cache-key: package
-      go-version: '1.17'
+      go-version: '1.17'

.github/workflows/changes.yml

@@ -23,6 +23,9 @@ on:
       kubernetes:
         description: "whether current workflow touches deploy/kubernetes dir or not"
         value: ${{ jobs.changes.outputs.kubernetes }}
+      rss_server_docker:
+        description: "whether current workflow touches deploy/kubernetes/docker dir or not"
+        value: ${{ jobs.changes.outputs.rss_server_docker }}
 
 jobs:
   changes:
@@ -37,5 +40,8 @@ jobs:
           filters: |
             kubernetes:
               - 'deploy/kubernetes/**'
+            rss_server_docker:
+              - 'deploy/kubernetes/docker/**'
     outputs:
       kubernetes: ${{ steps.filter.outputs.kubernetes }}
+      rss_server_docker: ${{ steps.filter.outputs.rss_server_docker }}

build_distribution.sh

@@ -1,4 +1,4 @@
-#! /usr/bin/env bash
+#!/usr/bin/env bash
 
 #
 # Licensed to the Apache Software Foundation (ASF) under one or more

deploy/kubernetes/docker/Dockerfile

@@ -15,7 +15,8 @@
 # limitations under the License.
 #
 
-FROM openjdk:8-jdk-slim
+ARG BASE_IMAGE=uniffle-base:latest
+FROM ${BASE_IMAGE}
 
 ARG HADOOP_VERSION
 ARG RSS_VERSION
@@ -27,9 +28,6 @@ LABEL author=${AUTHOR}
 LABEL git-branch=${GIT_BRANCH}
 LABEL git-commit=${GIT_COMMIT}
 
-RUN apt-get update && apt-get install -y zlib1g zlib1g-dev lzop lsof netcat dnsutils less procps iputils-ping \
-      && apt-get clean && rm -rf /var/lib/apt/lists/*
-
 RUN useradd -ms /bin/bash rssadmin
 RUN mkdir -p /data/rssadmin/
 RUN chown -R rssadmin:rssadmin /data

deploy/kubernetes/docker/base/centos/Dockerfile

@@ -0,0 +1,24 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# this docker file serves the rss server base image for CentOS/RedHat distributions
+ARG BASE_IMAGE=eclipse-temurin:8-jdk-centos7
+FROM ${BASE_IMAGE}
+
+RUN yum install lzo zlib zlib-devel lzop lsof netcat dnsutils less procps iputils-ping -y && \
+    yum clean all && \
+    rm -rf /var/cache/yum

deploy/kubernetes/docker/base/debian/Dockerfile

@@ -0,0 +1,24 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# this docker file serves the rss server base image for Debian/Ubuntu releated distributions
+ARG BASE_IMAGE=eclipse-temurin:8-jdk
+FROM ${BASE_IMAGE}
+
+RUN apt-get update && \
+    apt-get install -y zlib1g zlib1g-dev lzop lsof netcat dnsutils less procps iputils-ping && \
+    apt-get clean && rm -rf /var/lib/apt/lists/*

deploy/kubernetes/docker/build.sh

@@ -24,18 +24,20 @@ function exit_with_usage() {
   echo "Usage:"
   echo "+------------------------------------------------------------------------------------------------------+"
   echo "| ./build.sh [--hadoop-version <hadoop version>] [--registry <registry url>] [--author <author name>]  |"
-  echo "|            [--apache-mirror <apache mirror url>]                                                     |"
+  echo "|            [--base-os-distribution <os distribution>] [--base-image <base image url>]                |"
+  echo "|            [--push-image <true|false>] [--apache-mirror <apache mirror url>]                         |"
   echo "+------------------------------------------------------------------------------------------------------+"
   exit 1
 }
 
-UNKNOWN_REGISTRY="UNKNOWN_REGISTRY"
-
-REGISTRY=$UNKNOWN_REGISTRY
+REGISTRY="docker.io/library"
 HADOOP_VERSION=2.8.5
 AUTHOR=$(whoami)
 # If you are based in China, you could pass --apache-mirror <a_mirror_url> when building this.
 APACHE_MIRROR="https://dlcdn.apache.org"
+OS_DISTRIBUTION=debian
+BASE_IMAGE=""
+PUSH_IMAGE="true"
 
 while (( "$#" )); do
   case $1 in
@@ -51,6 +53,18 @@ while (( "$#" )); do
       AUTHOR="$2"
       shift
       ;;
+    --base-os-distribution)
+      OS_DISTRIBUTION="$2"
+      shift
+      ;;
+    --base-image)
+      BASE_IMAGE="$2"
+      shift
+      ;;
+    --push-image)
+      PUSH_IMAGE="$2"
+      shift
+      ;;
     --help)
       exit_with_usage
       ;;
@@ -73,10 +87,16 @@ while (( "$#" )); do
   shift
 done
 
-if [ "$REGISTRY" == $UNKNOWN_REGISTRY ]; \
-  then echo "please set registry url"; exit; \
+if [ -z "$BASE_IMAGE" ]; then
+  echo "start building base image: uniffle-base"
+  docker build -t "uniffle-base:latest" \
+    -f base/"${OS_DISTRIBUTION}"/Dockerfile .
+  BASE_IMAGE="uniffle-base:latest"
+else
+  echo "using base image(${BASE_IMAGE}) to build rss server"
 fi
 
+
 HADOOP_FILE=hadoop-${HADOOP_VERSION}.tar.gz
 ARCHIVE_HADOOP_URL=https://archive.apache.org/dist/hadoop/core/hadoop-${HADOOP_VERSION}/${HADOOP_FILE}
 HADOOP_URL=${APACHE_MIRROR}/hadoop/core/hadoop-${HADOOP_VERSION}/${HADOOP_FILE}
@@ -91,7 +111,7 @@ cd $RSS_DIR || exit
 RSS_VERSION=$(mvn help:evaluate -Dexpression=project.version 2>/dev/null | grep -v "INFO" | grep -v "WARNING" | tail -n 1)
 RSS_FILE=rss-${RSS_VERSION}.tgz
 if [ ! -e "$RSS_FILE" ]; \
-  then sh ./build_distribution.sh; \
+  then bash ./build_distribution.sh; \
   else echo "$RSS_FILE has been built"; \
 fi
 cd "$OLDPWD" || exit
@@ -108,7 +128,10 @@ docker build -t "$IMAGE" \
              --build-arg AUTHOR="$AUTHOR" \
              --build-arg GIT_COMMIT="$GIT_COMMIT" \
              --build-arg GIT_BRANCH="$GIT_BRANCH" \
+             --build-arg BASE_IMAGE="$BASE_IMAGE" \
              -f Dockerfile --no-cache .
 
-echo "pushing image: $IMAGE"
-docker push "$IMAGE"
+if [ x"${PUSH_IMAGE}" == x"true" ]; then
+  echo "pushing image: $IMAGE"
+  docker push "$IMAGE"
+fi

deploy/kubernetes/docker/start.sh

@@ -31,7 +31,7 @@ if [ "$SERVICE_NAME" == "coordinator" ];then
     bash ${basedir}/bin/start-coordinator.sh &
     sleep 10
     while : ; do
-        pid=$(/usr/bin/lsof -i:"${COORDINATOR_RPC_PORT}" -sTCP:LISTEN)
+        pid=$(lsof -i:"${COORDINATOR_RPC_PORT}" -sTCP:LISTEN)
         if [ "$pid" = "" ]; then
           break
         else
@@ -46,7 +46,7 @@ if [ "$SERVICE_NAME" == "server" ];then
     bash ${basedir}/bin/start-shuffle-server.sh &
     sleep 10
     while : ; do
-        pid=$(/usr/bin/lsof -i:"$SERVER_RPC_PORT" -sTCP:LISTEN)
+        pid=$(lsof -i:"$SERVER_RPC_PORT" -sTCP:LISTEN)
         if [ "$pid" = "" ]; then
           break
         else

deploy/kubernetes/pom.xml

@@ -45,6 +45,19 @@
               <executable>${basedir}/build-operator.sh</executable>
             </configuration>
           </execution>
+          <execution>
+            <id>Build RSS server image</id>
+            <phase>compile</phase>
+            <goals>
+              <goal>exec</goal>
+            </goals>
+            <configuration>
+              <workingDirectory>${basedir}/docker</workingDirectory>
+              <executable>${basedir}/docker/build.sh</executable>
+              <commandlineArgs>--push-image false --hadoop-version 2.10.2</commandlineArgs>
+              <skip>${skipBuildImage}</skip>
+            </configuration>
+          </execution>
           <execution>
             <id>Test Kubernetes Operator</id>
             <phase>test</phase>

pom.xml

@@ -81,6 +81,7 @@
     <trimStackTrace>false</trimStackTrace>
     <skipUTs>${skipTests}</skipUTs>
     <skipITs>${skipTests}</skipITs>
+    <skipBuildImage>true</skipBuildImage>
   </properties>
 
   <repositories>