Ensure Github workflow runs on docker image used by Production Distri…

…bution Build Signed-off-by: Peter Zhu <zhujiaxi@amazon.com>
opensearch-project · Oct 11, 2023 · 9a5dea2 · 9a5dea2
1 parent 6f29716
commit 9a5dea2
Show file tree

Hide file tree

Showing 6 changed files with 154 additions and 46 deletions.
diff --git a/.github/workflows/bwc-test-workflow.yml b/.github/workflows/bwc-test-workflow.yml
@@ -9,29 +9,39 @@ on:
       - "*"
 
 jobs:
-  build:
+  Get-CI-Image-Tag:
+    uses: ./.github/workflows/get-ci-image-tag.yml
+    with:
+      platform: centos7
+      usage: opensearch
+
+  build-linux:
+    needs: Get-CI-Image-Tag
     strategy:
-      matrix:
-        java: [ 11 ]
+          matrix:
+            java: [ 11 ]
     # Job name
     name: Build and test Alerting
     # This job runs on Linux
     runs-on: ubuntu-latest
+    container:
+      # using the same image which is used by opensearch-build team to build the OpenSearch Distribution
+      # this image tag is subject to change as more dependencies and updates will arrive over time
+      image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }}
+      # need to switch to root so that github actions can install runner binary on container without permission issues.
+      options: --user root
+
     steps:
-      # This step uses the setup-java Github action: https://github.com/actions/setup-java
-      - name: Set Up JDK ${{ matrix.java }}
-        uses: actions/setup-java@v1
-        with:
-          java-version: ${{ matrix.java }}
       # This step uses the checkout Github action: https://github.com/actions/checkout
       - name: Checkout Branch
         uses: actions/checkout@v2
       # This step uses the setup-java Github action: https://github.com/actions/setup-java
-      - name: Set Up JDK 11
+      - name: Set Up JDK ${{ matrix.java }}
         uses: actions/setup-java@v1
         with:
-          java-version: 11
+          java-version: ${{ matrix.java }}
       - name: Run Alerting Backwards Compatibility Tests
         run: |
           echo "Running backwards compatibility tests..."
-          ./gradlew bwcTestSuite
+          chown -R opensearch.opensearch `pwd`
+          su opensearch -c "whoami && java -version && ./gradlew bwcTestSuite"
diff --git a/.github/workflows/dco.yml b/.github/workflows/dco.yml
diff --git a/.github/workflows/get-ci-image-tag.yml b/.github/workflows/get-ci-image-tag.yml
@@ -0,0 +1,39 @@
+name: Get CI Image Tag
+on:
+  workflow_call:
+    inputs:
+      platform:
+        required: true
+        type: string
+      usage:
+        required: true
+        type: string
+    outputs:
+      ci-image-version-linux:
+        description: The ci image version for linux build
+        value: ${{ jobs.Get-CI-Image-Tag.outputs.output-ci-image-version-linux }}
+
+jobs:
+  Get-CI-Image-Tag:
+    runs-on: ubuntu-latest
+    outputs:
+      output-ci-image-version-linux: ${{ steps.step-ci-image-version-linux.outputs.ci-image-version-linux }}
+    steps:
+      - name: Install crane
+        uses: iarekylew00t/crane-installer@v1
+        with:
+          crane-release: v0.15.2
+      - name: Checkout opensearch-build repository
+        uses: actions/checkout@v2
+        with:
+            repository: 'opensearch-project/opensearch-build'
+            ref: 'main'
+            path: 'opensearch-build'
+      - name: Get ci image version from opensearch-build repository scripts
+        id: step-ci-image-version-linux
+        run: |
+          crane version
+          CI_IMAGE_VERSION=`opensearch-build/docker/ci/get-ci-images.sh -p ${{ inputs.platform }} -u ${{ inputs.usage }} -t build | head -1`
+          echo $CI_IMAGE_VERSION
+          echo "ci-image-version-linux=$CI_IMAGE_VERSION" >> $GITHUB_OUTPUT
+
diff --git a/.github/workflows/multi-node-test-workflow.yml b/.github/workflows/multi-node-test-workflow.yml
@@ -9,14 +9,28 @@ on:
       - "*"
 
 jobs:
-  build:
+  Get-CI-Image-Tag:
+    uses: ./.github/workflows/get-ci-image-tag.yml
+    with:
+      platform: centos7
+      usage: opensearch
+
+  build-linux:
+    needs: Get-CI-Image-Tag
     strategy:
       matrix:
         java: [ 11, 17 ]
     # Job name
     name: Build and test Alerting
     # This job runs on Linux
     runs-on: ubuntu-latest
+    container:
+      # using the same image which is used by opensearch-build team to build the OpenSearch Distribution
+      # this image tag is subject to change as more dependencies and updates will arrive over time
+      image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }}
+      # need to switch to root so that github actions can install runner binary on container without permission issues.
+      options: --user root
+
     steps:
       # This step uses the setup-java Github action: https://github.com/actions/setup-java
       - name: Set Up JDK ${{ matrix.java }}
@@ -27,4 +41,6 @@ jobs:
       - name: Checkout Branch
         uses: actions/checkout@v2
       - name: Run integration tests with multi node config
-        run: ./gradlew integTest -PnumNodes=3
+        run: |
+          chown -R opensearch.opensearch `pwd`
+          su opensearch -c "whoami && java -version && ./gradlew integTest -PnumNodes=3"
diff --git a/.github/workflows/security-test-workflow.yml b/.github/workflows/security-test-workflow.yml
@@ -9,20 +9,29 @@ on:
       - "*"
 
 jobs:
-  build:
+  Get-CI-Image-Tag:
+    uses: ./.github/workflows/get-ci-image-tag.yml
+    with:
+      platform: centos7
+      usage: opensearch
+
+  build-linux:
+    needs: Get-CI-Image-Tag
     strategy:
       matrix:
         java: [ 11, 17 ]
     # Job name
     name: Build and test Alerting
     # This job runs on Linux
     runs-on: ubuntu-latest
+    container:
+      # using the same image which is used by opensearch-build team to build the OpenSearch Distribution
+      # this image tag is subject to change as more dependencies and updates will arrive over time
+      image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }}
+      # need to switch to root so that github actions can install runner binary on container without permission issues.
+      options: --user root
+
     steps:
-      # This step uses the setup-java Github action: https://github.com/actions/setup-java
-      - name: Set Up JDK ${{ matrix.java }}
-        uses: actions/setup-java@v1
-        with:
-          java-version: ${{ matrix.java }}
       # This step uses the checkout Github action: https://github.com/actions/checkout
       - name: Checkout Branch
         uses: actions/checkout@v2
@@ -33,7 +42,9 @@ jobs:
           java-version: ${{ matrix.java }}
       - name: Build Alerting
         # Only assembling since the full build is governed by other workflows
-        run: ./gradlew assemble
+        run: |
+          chown -R opensearch.opensearch `pwd`
+          su opensearch -c "whoami && java -version && ./gradlew assemble"
       - name: Pull and Run Docker
         run: |
           plugin=`basename $(ls alerting/build/distributions/*.zip)`
@@ -86,7 +97,8 @@ jobs:
           if [ $security -gt 0 ]
           then
             echo "Security plugin is available"
-            ./gradlew :alerting:integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername=docker-cluster -Dsecurity=true -Dhttps=true -Duser=admin -Dpassword=admin
+            chown -R opensearch.opensearch `pwd`
+            su opensearch -c "whoami && java -version && ./gradlew :alerting:integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername=docker-cluster -Dsecurity=true -Dhttps=true -Duser=admin -Dpassword=admin"
           else
             echo "Security plugin is NOT available skipping this run as tests without security have already been run"
           fi
diff --git a/.github/workflows/test-workflow.yml b/.github/workflows/test-workflow.yml
@@ -9,14 +9,69 @@ on:
       - "*"
 
 jobs:
+  Get-CI-Image-Tag:
+    uses: ./.github/workflows/get-ci-image-tag.yml
+    with:
+      platform: centos7
+      usage: opensearch
+
+  build-linux:
+    needs: Get-CI-Image-Tag
+    env:
+      BUILD_ARGS: ${{ matrix.os_build_args }}
+      WORKING_DIR: ${{ matrix.working_directory }}.
+    strategy:
+      matrix:
+        java: [11, 17]
+    # Job name
+    name: Build Alerting with JDK ${{ matrix.java }} on Linux
+    # This job runs on Linux
+    runs-on: ubuntu-latest
+    container:
+      # using the same image which is used by opensearch-build team to build the OpenSearch Distribution
+      # this image tag is subject to change as more dependencies and updates will arrive over time
+      image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }}
+      # need to switch to root so that github actions can install runner binary on container without permission issues.
+      options: --user root
+
+    steps:
+      # This step uses the checkout Github action: https://github.com/actions/checkout
+      - name: Checkout Branch
+        uses: actions/checkout@v2
+      # This step uses the setup-java Github action: https://github.com/actions/setup-java
+      - name: Set Up JDK ${{ matrix.java }}
+        uses: actions/setup-java@v1
+        with:
+          java-version: ${{ matrix.java }}
+      - name: Build and run with Gradle
+        run: |
+          chown -R opensearch.opensearch `pwd`
+          su opensearch -c "whoami && java -version && ./gradlew assemble integTest"
+      - name: Create Artifact Path
+        run: |
+          mkdir -p alerting-artifacts
+          cp ./alerting/build/distributions/*.zip alerting-artifacts
+      # This step uses the codecov-action Github action: https://github.com/codecov/codecov-action
+      - name: Upload Coverage Report
+        uses: codecov/codecov-action@v1
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+      # This step uses the upload-artifact Github action: https://github.com/actions/upload-artifact
+      - name: Upload Artifacts
+        uses: actions/upload-artifact@v1
+        with:
+          name: alerting-plugin-${{ matrix.os }}
+          path: alerting-artifacts
+
   build:
+    needs: Get-CI-Image-Tag
     env:
       BUILD_ARGS: ${{ matrix.os_build_args }}
       WORKING_DIR: ${{ matrix.working_directory }}.
     strategy:
       matrix:
         java: [11, 17]
-        os: [ ubuntu-latest, windows-latest, macos-latest ]
+        os: [ windows-latest, macos-latest ]
         include:
           - os: windows-latest
             os_build_args: -x integTest
@@ -49,12 +104,6 @@ jobs:
         run: |
           mkdir -p alerting-artifacts
           cp ./alerting/build/distributions/*.zip alerting-artifacts
-      # This step uses the codecov-action Github action: https://github.com/codecov/codecov-action
-      - name: Upload Coverage Report
-        if: ${{ matrix.os == 'ubuntu-latest' }}
-        uses: codecov/codecov-action@v1
-        with:
-          token: ${{ secrets.CODECOV_TOKEN }}
       # This step uses the upload-artifact Github action: https://github.com/actions/upload-artifact
       - name: Upload Artifacts
         uses: actions/upload-artifact@v1