From 9a5dea200e493ed2b31368fb1eb54c71ef54bd73 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Wed, 11 Oct 2023 16:33:50 -0400 Subject: [PATCH 1/2] Ensure Github workflow runs on docker image used by Production Distribution Build Signed-off-by: Peter Zhu --- .github/workflows/bwc-test-workflow.yml | 32 ++++++---- .github/workflows/dco.yml | 18 ------ .github/workflows/get-ci-image-tag.yml | 39 ++++++++++++ .../workflows/multi-node-test-workflow.yml | 20 +++++- .github/workflows/security-test-workflow.yml | 28 ++++++--- .github/workflows/test-workflow.yml | 63 ++++++++++++++++--- 6 files changed, 154 insertions(+), 46 deletions(-) delete mode 100644 .github/workflows/dco.yml create mode 100644 .github/workflows/get-ci-image-tag.yml diff --git a/.github/workflows/bwc-test-workflow.yml b/.github/workflows/bwc-test-workflow.yml index 66174c697..a1fa6ed4a 100644 --- a/.github/workflows/bwc-test-workflow.yml +++ b/.github/workflows/bwc-test-workflow.yml @@ -9,29 +9,39 @@ on: - "*" jobs: - build: + Get-CI-Image-Tag: + uses: ./.github/workflows/get-ci-image-tag.yml + with: + platform: centos7 + usage: opensearch + + build-linux: + needs: Get-CI-Image-Tag strategy: - matrix: - java: [ 11 ] + matrix: + java: [ 11 ] # Job name name: Build and test Alerting # This job runs on Linux runs-on: ubuntu-latest + container: + # using the same image which is used by opensearch-build team to build the OpenSearch Distribution + # this image tag is subject to change as more dependencies and updates will arrive over time + image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} + # need to switch to root so that github actions can install runner binary on container without permission issues. + options: --user root + steps: - # This step uses the setup-java Github action: https://github.com/actions/setup-java - - name: Set Up JDK ${{ matrix.java }} - uses: actions/setup-java@v1 - with: - java-version: ${{ matrix.java }} # This step uses the checkout Github action: https://github.com/actions/checkout - name: Checkout Branch uses: actions/checkout@v2 # This step uses the setup-java Github action: https://github.com/actions/setup-java - - name: Set Up JDK 11 + - name: Set Up JDK ${{ matrix.java }} uses: actions/setup-java@v1 with: - java-version: 11 + java-version: ${{ matrix.java }} - name: Run Alerting Backwards Compatibility Tests run: | echo "Running backwards compatibility tests..." - ./gradlew bwcTestSuite + chown -R opensearch.opensearch `pwd` + su opensearch -c "whoami && java -version && ./gradlew bwcTestSuite" diff --git a/.github/workflows/dco.yml b/.github/workflows/dco.yml deleted file mode 100644 index 53ed5304c..000000000 --- a/.github/workflows/dco.yml +++ /dev/null @@ -1,18 +0,0 @@ -name: Developer Certificate of Origin Check - -on: [pull_request] - -jobs: - check: - runs-on: ubuntu-latest - - steps: - - name: Get PR Commits - id: 'get-pr-commits' - uses: tim-actions/get-pr-commits@v1.1.0 - with: - token: ${{ secrets.GITHUB_TOKEN }} - - name: DCO Check - uses: tim-actions/dco@v1.1.0 - with: - commits: ${{ steps.get-pr-commits.outputs.commits }} \ No newline at end of file diff --git a/.github/workflows/get-ci-image-tag.yml b/.github/workflows/get-ci-image-tag.yml new file mode 100644 index 000000000..81c42635a --- /dev/null +++ b/.github/workflows/get-ci-image-tag.yml @@ -0,0 +1,39 @@ +name: Get CI Image Tag +on: + workflow_call: + inputs: + platform: + required: true + type: string + usage: + required: true + type: string + outputs: + ci-image-version-linux: + description: The ci image version for linux build + value: ${{ jobs.Get-CI-Image-Tag.outputs.output-ci-image-version-linux }} + +jobs: + Get-CI-Image-Tag: + runs-on: ubuntu-latest + outputs: + output-ci-image-version-linux: ${{ steps.step-ci-image-version-linux.outputs.ci-image-version-linux }} + steps: + - name: Install crane + uses: iarekylew00t/crane-installer@v1 + with: + crane-release: v0.15.2 + - name: Checkout opensearch-build repository + uses: actions/checkout@v2 + with: + repository: 'opensearch-project/opensearch-build' + ref: 'main' + path: 'opensearch-build' + - name: Get ci image version from opensearch-build repository scripts + id: step-ci-image-version-linux + run: | + crane version + CI_IMAGE_VERSION=`opensearch-build/docker/ci/get-ci-images.sh -p ${{ inputs.platform }} -u ${{ inputs.usage }} -t build | head -1` + echo $CI_IMAGE_VERSION + echo "ci-image-version-linux=$CI_IMAGE_VERSION" >> $GITHUB_OUTPUT + diff --git a/.github/workflows/multi-node-test-workflow.yml b/.github/workflows/multi-node-test-workflow.yml index b6377ed5d..b80f04a43 100644 --- a/.github/workflows/multi-node-test-workflow.yml +++ b/.github/workflows/multi-node-test-workflow.yml @@ -9,7 +9,14 @@ on: - "*" jobs: - build: + Get-CI-Image-Tag: + uses: ./.github/workflows/get-ci-image-tag.yml + with: + platform: centos7 + usage: opensearch + + build-linux: + needs: Get-CI-Image-Tag strategy: matrix: java: [ 11, 17 ] @@ -17,6 +24,13 @@ jobs: name: Build and test Alerting # This job runs on Linux runs-on: ubuntu-latest + container: + # using the same image which is used by opensearch-build team to build the OpenSearch Distribution + # this image tag is subject to change as more dependencies and updates will arrive over time + image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} + # need to switch to root so that github actions can install runner binary on container without permission issues. + options: --user root + steps: # This step uses the setup-java Github action: https://github.com/actions/setup-java - name: Set Up JDK ${{ matrix.java }} @@ -27,4 +41,6 @@ jobs: - name: Checkout Branch uses: actions/checkout@v2 - name: Run integration tests with multi node config - run: ./gradlew integTest -PnumNodes=3 + run: | + chown -R opensearch.opensearch `pwd` + su opensearch -c "whoami && java -version && ./gradlew integTest -PnumNodes=3" diff --git a/.github/workflows/security-test-workflow.yml b/.github/workflows/security-test-workflow.yml index 127962210..9a8f45bf8 100644 --- a/.github/workflows/security-test-workflow.yml +++ b/.github/workflows/security-test-workflow.yml @@ -9,7 +9,14 @@ on: - "*" jobs: - build: + Get-CI-Image-Tag: + uses: ./.github/workflows/get-ci-image-tag.yml + with: + platform: centos7 + usage: opensearch + + build-linux: + needs: Get-CI-Image-Tag strategy: matrix: java: [ 11, 17 ] @@ -17,12 +24,14 @@ jobs: name: Build and test Alerting # This job runs on Linux runs-on: ubuntu-latest + container: + # using the same image which is used by opensearch-build team to build the OpenSearch Distribution + # this image tag is subject to change as more dependencies and updates will arrive over time + image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} + # need to switch to root so that github actions can install runner binary on container without permission issues. + options: --user root + steps: - # This step uses the setup-java Github action: https://github.com/actions/setup-java - - name: Set Up JDK ${{ matrix.java }} - uses: actions/setup-java@v1 - with: - java-version: ${{ matrix.java }} # This step uses the checkout Github action: https://github.com/actions/checkout - name: Checkout Branch uses: actions/checkout@v2 @@ -33,7 +42,9 @@ jobs: java-version: ${{ matrix.java }} - name: Build Alerting # Only assembling since the full build is governed by other workflows - run: ./gradlew assemble + run: | + chown -R opensearch.opensearch `pwd` + su opensearch -c "whoami && java -version && ./gradlew assemble" - name: Pull and Run Docker run: | plugin=`basename $(ls alerting/build/distributions/*.zip)` @@ -86,7 +97,8 @@ jobs: if [ $security -gt 0 ] then echo "Security plugin is available" - ./gradlew :alerting:integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername=docker-cluster -Dsecurity=true -Dhttps=true -Duser=admin -Dpassword=admin + chown -R opensearch.opensearch `pwd` + su opensearch -c "whoami && java -version && ./gradlew :alerting:integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername=docker-cluster -Dsecurity=true -Dhttps=true -Duser=admin -Dpassword=admin" else echo "Security plugin is NOT available skipping this run as tests without security have already been run" fi diff --git a/.github/workflows/test-workflow.yml b/.github/workflows/test-workflow.yml index e5386ba41..6d248cfcb 100644 --- a/.github/workflows/test-workflow.yml +++ b/.github/workflows/test-workflow.yml @@ -9,14 +9,69 @@ on: - "*" jobs: + Get-CI-Image-Tag: + uses: ./.github/workflows/get-ci-image-tag.yml + with: + platform: centos7 + usage: opensearch + + build-linux: + needs: Get-CI-Image-Tag + env: + BUILD_ARGS: ${{ matrix.os_build_args }} + WORKING_DIR: ${{ matrix.working_directory }}. + strategy: + matrix: + java: [11, 17] + # Job name + name: Build Alerting with JDK ${{ matrix.java }} on Linux + # This job runs on Linux + runs-on: ubuntu-latest + container: + # using the same image which is used by opensearch-build team to build the OpenSearch Distribution + # this image tag is subject to change as more dependencies and updates will arrive over time + image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} + # need to switch to root so that github actions can install runner binary on container without permission issues. + options: --user root + + steps: + # This step uses the checkout Github action: https://github.com/actions/checkout + - name: Checkout Branch + uses: actions/checkout@v2 + # This step uses the setup-java Github action: https://github.com/actions/setup-java + - name: Set Up JDK ${{ matrix.java }} + uses: actions/setup-java@v1 + with: + java-version: ${{ matrix.java }} + - name: Build and run with Gradle + run: | + chown -R opensearch.opensearch `pwd` + su opensearch -c "whoami && java -version && ./gradlew assemble integTest" + - name: Create Artifact Path + run: | + mkdir -p alerting-artifacts + cp ./alerting/build/distributions/*.zip alerting-artifacts + # This step uses the codecov-action Github action: https://github.com/codecov/codecov-action + - name: Upload Coverage Report + uses: codecov/codecov-action@v1 + with: + token: ${{ secrets.CODECOV_TOKEN }} + # This step uses the upload-artifact Github action: https://github.com/actions/upload-artifact + - name: Upload Artifacts + uses: actions/upload-artifact@v1 + with: + name: alerting-plugin-${{ matrix.os }} + path: alerting-artifacts + build: + needs: Get-CI-Image-Tag env: BUILD_ARGS: ${{ matrix.os_build_args }} WORKING_DIR: ${{ matrix.working_directory }}. strategy: matrix: java: [11, 17] - os: [ ubuntu-latest, windows-latest, macos-latest ] + os: [ windows-latest, macos-latest ] include: - os: windows-latest os_build_args: -x integTest @@ -49,12 +104,6 @@ jobs: run: | mkdir -p alerting-artifacts cp ./alerting/build/distributions/*.zip alerting-artifacts - # This step uses the codecov-action Github action: https://github.com/codecov/codecov-action - - name: Upload Coverage Report - if: ${{ matrix.os == 'ubuntu-latest' }} - uses: codecov/codecov-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} # This step uses the upload-artifact Github action: https://github.com/actions/upload-artifact - name: Upload Artifacts uses: actions/upload-artifact@v1 From e7a1e04b57877a8c00a5957429907b7c4fe81703 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Wed, 11 Oct 2023 17:46:38 -0400 Subject: [PATCH 2/2] Tweak spacing Signed-off-by: Peter Zhu --- .github/workflows/bwc-test-workflow.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/bwc-test-workflow.yml b/.github/workflows/bwc-test-workflow.yml index a1fa6ed4a..bf8e8ff7c 100644 --- a/.github/workflows/bwc-test-workflow.yml +++ b/.github/workflows/bwc-test-workflow.yml @@ -18,8 +18,8 @@ jobs: build-linux: needs: Get-CI-Image-Tag strategy: - matrix: - java: [ 11 ] + matrix: + java: [ 11 ] # Job name name: Build and test Alerting # This job runs on Linux