HDDS-3990. Test Kubernetes examples with acceptance tests (#1223)

.github/workflows/post-commit.yml

@@ -258,3 +258,69 @@ jobs:
            name: coverage
            path: target/coverage
          continue-on-error: true
+  kubernetes:
+    name: kubernetes
+    runs-on: ubuntu-18.04
+    steps:
+      - name: Cache for maven dependencies
+        uses: actions/cache@v2
+        with:
+          path: ~/.m2/repository
+          key: maven-repo-${{ hashFiles('**/pom.xml') }}
+      - name: Cache for npm dependencies
+        uses: actions/cache@v2
+        with:
+          path: |
+            ~/.pnpm-store
+            **/node_modules
+          key: ${{ runner.os }}-pnpm-${{ hashFiles('**/pnpm-lock.yaml') }}
+          restore-keys: |
+            ${{ runner.os }}-pnpm-
+      - name: Checkout to /mnt/ozone
+        run: |
+          sudo chmod 777 /mnt
+          git clone 'https://github.com/${{ github.repository }}.git' /mnt/ozone
+          cd /mnt/ozone
+          if [[ '${{ github.event_name }}' == 'pull_request' ]]; then
+            git fetch --verbose origin '${{ github.ref }}'
+          else
+            git fetch --verbose origin '${{ github.sha }}'
+          fi
+          git checkout FETCH_HEAD
+          git reset --hard
+      - name: Install robotframework
+        run: sudo pip install robotframework
+      - name: Install k3s
+        run: curl -sfL https://get.k3s.io | sh -
+      - name: Copy Kubernetes config file
+        run: |
+          sudo mkdir ~/.kube
+          sudo cp /etc/rancher/k3s/k3s.yaml ~/.kube/config
+          sudo chown $(id -u) ~/.kube/config
+      - name: Install flekszible
+        run: |
+          cd /tmp
+          wget https://github.com/elek/flekszible/releases/download/v1.8.1/flekszible_1.8.1_Linux_x86_64.tar.gz -O - | tar -zx
+          chmod +x flekszible
+          sudo mv flekszible /usr/bin/flekszible
+      - name: Run a full build
+        run: |
+          cd /mnt/ozone
+          hadoop-ozone/dev-support/checks/build.sh -Pcoverage
+      - name: Execute tests
+        run: |
+          cd /mnt/ozone/hadoop-ozone/dist/target/ozone-* && sudo mkdir .aws && sudo chmod 777 .aws && sudo chown 1000 .aws
+          cd /mnt/ozone && hadoop-ozone/dev-support/checks/kubernetes.sh
+      - name: Archive build results
+        uses: actions/upload-artifact@master
+        if: always()
+        with:
+          name: kubernetes
+          path: /mnt/ozone/target/kubernetes
+        continue-on-error: true
+      - name: Delete temporary build artifacts before caching
+        run: |
+          #Never cache local artifacts
+          rm -rf ~/.m2/repository/org/apache/hadoop/hdds
+          rm -rf ~/.m2/repository/org/apache/hadoop/ozone
+        if: always()

hadoop-ozone/dev-support/checks/kubernetes.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+cd "$DIR/../../.." || exit 1
+
+REPORT_DIR=${OUTPUT_DIR:-"$DIR/../../../target/kubernetes"}
+
+OZONE_VERSION=$(grep "<ozone.version>" "pom.xml" | sed 's/<[^>]*>//g'|  sed 's/^[ \t]*//')
+DIST_DIR="$DIR/../../dist/target/ozone-$OZONE_VERSION"
+
+if [ ! -d "$DIST_DIR" ]; then
+    echo "Distribution dir is missing. Doing a full build"
+    "$DIR/build.sh" -Pcoverage
+fi
+
+mkdir -p "$REPORT_DIR"
+
+cd "$DIST_DIR/kubernetes/examples" || exit 1
+./test-all.sh
+RES=$?
+cp result/* "$REPORT_DIR/"
+cp "$REPORT_DIR/log.html" "$REPORT_DIR/summary.html"
+exit $RES

hadoop-ozone/dist/dev-support/bin/dist-layout-stitching

@@ -131,3 +131,4 @@ run cp -p -r "${ROOT}/hadoop-ozone/dist/target/Dockerfile" .
 
 #workaround for https://issues.apache.org/jira/browse/MRESOURCES-236
 find ./compose -name "*.sh" -exec chmod 755 {} \;
+find ./kubernetes -name "*.sh" -exec chmod 755 {} \;

hadoop-ozone/dist/src/main/k8s/.gitignore

@@ -0,0 +1,15 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+result

hadoop-ozone/dist/src/main/k8s/definitions/ozone/config.yaml

@@ -26,6 +26,7 @@ data:
   OZONE-SITE.XML_ozone.om.address: "om-0.om"
   OZONE-SITE.XML_ozone.scm.client.address: "scm-0.scm"
   OZONE-SITE.XML_ozone.scm.names: "scm-0.scm"
+  OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "3"
   LOG4J.PROPERTIES_log4j.rootLogger: "INFO, stdout"
   LOG4J.PROPERTIES_log4j.appender.stdout: "org.apache.log4j.ConsoleAppender"
   LOG4J.PROPERTIES_log4j.appender.stdout.layout: "org.apache.log4j.PatternLayout"

hadoop-ozone/dist/src/main/k8s/definitions/ozone/definitions/onenode.yaml

@@ -19,7 +19,7 @@ description: remove scheduling rules to make it possible to run multiple datanod
 - type: Remove
   trigger:
     metadata:
-      name: ozone-datanode
+      name: datanode
   path:
     - spec
     - template

hadoop-ozone/dist/src/main/k8s/examples/getting-started/config-configmap.yaml

@@ -26,6 +26,7 @@ data:
   OZONE-SITE.XML_ozone.om.address: om-0.om
   OZONE-SITE.XML_ozone.scm.client.address: scm-0.scm
   OZONE-SITE.XML_ozone.scm.names: scm-0.scm
+  OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "3"
   LOG4J.PROPERTIES_log4j.rootLogger: INFO, stdout
   LOG4J.PROPERTIES_log4j.appender.stdout: org.apache.log4j.ConsoleAppender
   LOG4J.PROPERTIES_log4j.appender.stdout.layout: org.apache.log4j.PatternLayout

hadoop-ozone/dist/src/main/k8s/examples/getting-started/datanode-statefulset.yaml

@@ -37,16 +37,6 @@ spec:
         prometheus.io/port: "9882"
         prometheus.io/path: /prom
     spec:
-      affinity:
-        podAntiAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-          - labelSelector:
-              matchExpressions:
-              - key: component
-                operator: In
-                values:
-                - datanode
-            topologyKey: kubernetes.io/hostname
       securityContext:
         fsGroup: 1000
       containers:

hadoop-ozone/dist/src/main/k8s/examples/getting-started/test.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+export K8S_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
+cd "$K8S_DIR"
+
+# shellcheck source=/dev/null
+source "../testlib.sh"
+
+rm -rf result
+
+regenerate_resources
+
+start_k8s_env
+
+execute_robot_test scm-0 smoketest/basic/basic.robot
+
+combine_reports
+
+stop_k8s_env
+
+revert_resources

hadoop-ozone/dist/src/main/k8s/examples/minikube/config-configmap.yaml

@@ -26,6 +26,7 @@ data:
   OZONE-SITE.XML_ozone.om.address: om-0.om
   OZONE-SITE.XML_ozone.scm.client.address: scm-0.scm
   OZONE-SITE.XML_ozone.scm.names: scm-0.scm
+  OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "3"
   LOG4J.PROPERTIES_log4j.rootLogger: INFO, stdout
   LOG4J.PROPERTIES_log4j.appender.stdout: org.apache.log4j.ConsoleAppender
   LOG4J.PROPERTIES_log4j.appender.stdout.layout: org.apache.log4j.PatternLayout

hadoop-ozone/dist/src/main/k8s/examples/minikube/test.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+export K8S_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
+cd "$K8S_DIR"
+
+# shellcheck source=/dev/null
+source "../testlib.sh"
+
+rm -rf result
+
+regenerate_resources
+
+start_k8s_env
+
+execute_robot_test scm-0 smoketest/basic/basic.robot
+
+combine_reports
+
+stop_k8s_env
+
+revert_resources

hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/Flekszible

@@ -37,11 +37,11 @@ import:
     - type: Image
       image: "@docker.image@"
     - type: ozone/tracing
-  - path: pv-test
+  - path: test-webserver
     destination: pv-test
   - path: ozone-csi
     destination: csi
-  - path: pv-test
+  - path: test-webserver
     destination: pv-test
 transformations:
   - type: Namespace

hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/config-configmap.yaml

@@ -26,6 +26,7 @@ data:
   OZONE-SITE.XML_ozone.om.address: om-0.om
   OZONE-SITE.XML_ozone.scm.client.address: scm-0.scm
   OZONE-SITE.XML_ozone.scm.names: scm-0.scm
+  OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "3"
   LOG4J.PROPERTIES_log4j.rootLogger: INFO, stdout
   LOG4J.PROPERTIES_log4j.appender.stdout: org.apache.log4j.ConsoleAppender
   LOG4J.PROPERTIES_log4j.appender.stdout.layout: org.apache.log4j.PatternLayout

hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/datanode-statefulset.yaml

@@ -37,16 +37,6 @@ spec:
         prometheus.io/port: "9882"
         prometheus.io/path: /prom
     spec:
-      affinity:
-        podAntiAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-          - labelSelector:
-              matchExpressions:
-              - key: component
-                operator: In
-                values:
-                - datanode
-            topologyKey: kubernetes.io/hostname
       securityContext:
         fsGroup: 1000
       containers:

hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/test.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+export K8S_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
+cd "$K8S_DIR"
+
+# shellcheck source=/dev/null
+source "../testlib.sh"
+
+rm -rf result
+
+regenerate_resources
+
+start_k8s_env
+
+execute_robot_test scm-0 smoketest/basic/basic.robot
+
+combine_reports
+
+stop_k8s_env
+
+revert_resources

hadoop-ozone/dist/src/main/k8s/examples/ozone/Flekszible

@@ -31,7 +31,7 @@ import:
     destination: pv-test
   - path: ozone-csi
     destination: csi
-  - path: pv-test
+  - path: test-webserver
     destination: pv-test
 transformations:
   - type: Namespace

hadoop-ozone/dist/src/main/k8s/examples/ozone/config-configmap.yaml

@@ -26,6 +26,7 @@ data:
   OZONE-SITE.XML_ozone.om.address: om-0.om
   OZONE-SITE.XML_ozone.scm.client.address: scm-0.scm
   OZONE-SITE.XML_ozone.scm.names: scm-0.scm
+  OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "3"
   LOG4J.PROPERTIES_log4j.rootLogger: INFO, stdout
   LOG4J.PROPERTIES_log4j.appender.stdout: org.apache.log4j.ConsoleAppender
   LOG4J.PROPERTIES_log4j.appender.stdout.layout: org.apache.log4j.PatternLayout

hadoop-ozone/dist/src/main/k8s/examples/ozone/test.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+export K8S_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
+cd "$K8S_DIR"
+
+# shellcheck source=/dev/null
+source "../testlib.sh"
+
+rm -rf result
+
+regenerate_resources
+
+start_k8s_env
+
+execute_robot_test scm-0 smoketest/basic/basic.robot
+
+combine_reports
+
+stop_k8s_env
+
+revert_resources