From e81d845055f81e7af5ad5f517f65f01613a5c957 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Mon, 30 Oct 2023 19:14:31 -0400 Subject: [PATCH 1/2] Merged from main Signed-off-by: Peter Zhu --- .github/workflows/benchmark.yml | 20 +++++-- .../workflows/test_build_multi_platform.yml | 54 +++++++++++++++++-- .github/workflows/test_bwc.yml | 30 ++++++++--- .github/workflows/test_security.yml | 1 + 4 files changed, 92 insertions(+), 13 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index c5caeedd0..edf7a0bbb 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -8,7 +8,13 @@ on: - "*" jobs: + Get-CI-Image-Tag: + uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main + with: + product: opensearch + Build-ad: + needs: Get-CI-Image-Tag strategy: matrix: java: [17] @@ -16,18 +22,26 @@ jobs: name: Run Anomaly detection model performance benchmark runs-on: ubuntu-latest + container: + # using the same image which is used by opensearch-build team to build the OpenSearch Distribution + # this image tag is subject to change as more dependencies and updates will arrive over time + image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} + # need to switch to root so that github actions can install runner binary on container without permission issues. + options: --user root steps: - name: Setup Java ${{ matrix.java }} uses: actions/setup-java@v3 with: + distribution: 'temurin' java-version: ${{ matrix.java }} # anomaly-detection - name: Checkout AD - uses: actions/checkout@v4 + uses: actions/checkout@v3 - name: Build and Run Tests run: | - ./gradlew ':test' --tests "org.opensearch.ad.ml.HCADModelPerfTests" -Dtests.seed=2AEBDBBAE75AC5E0 -Dtests.security.manager=false -Dtests.locale=es-CU -Dtests.timezone=Chile/EasterIsland -Dtest.logs=true -Dmodel-benchmark=true - ./gradlew integTest --tests "org.opensearch.ad.e2e.SingleStreamModelPerfIT" -Dtests.seed=60CDDB34427ACD0C -Dtests.security.manager=false -Dtests.locale=kab-DZ -Dtests.timezone=Asia/Hebron -Dtest.logs=true -Dmodel-benchmark=true \ No newline at end of file + chown -R 1000:1000 `pwd` + su `id -un 1000` -c "./gradlew ':test' --tests 'org.opensearch.ad.ml.HCADModelPerfTests' -Dtests.seed=2AEBDBBAE75AC5E0 -Dtests.security.manager=false -Dtests.locale=es-CU -Dtests.timezone=Chile/EasterIsland -Dtest.logs=true -Dmodel-benchmark=true" + su `id -un 1000` -c "./gradlew integTest --tests 'org.opensearch.ad.e2e.SingleStreamModelPerfIT' -Dtests.seed=60CDDB34427ACD0C -Dtests.security.manager=false -Dtests.locale=kab-DZ -Dtests.timezone=Asia/Hebron -Dtest.logs=true -Dmodel-benchmark=true" diff --git a/.github/workflows/test_build_multi_platform.yml b/.github/workflows/test_build_multi_platform.yml index ebfc3f309..0f5d2f764 100644 --- a/.github/workflows/test_build_multi_platform.yml +++ b/.github/workflows/test_build_multi_platform.yml @@ -8,6 +8,11 @@ on: - "*" jobs: + Get-CI-Image-Tag: + uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main + with: + product: opensearch + Build-ad-windows: strategy: matrix: @@ -20,6 +25,7 @@ jobs: - name: Setup Java ${{ matrix.java }} uses: actions/setup-java@v3 with: + distribution: 'temurin' java-version: ${{ matrix.java }} - name: Checkout Anomaly Detection uses: actions/checkout@v4 @@ -38,15 +44,56 @@ jobs: with: file: ./build/reports/jacoco/test/jacocoTestReport.xml flags: plugin - Build-ad: + + Build-ad-linux: + needs: Get-CI-Image-Tag strategy: matrix: java: [11,17] - os: [ubuntu-latest, macos-latest] + fail-fast: false + name: Build and Test Anomaly detection Plugin + runs-on: ubuntu-latest + container: + # using the same image which is used by opensearch-build team to build the OpenSearch Distribution + # this image tag is subject to change as more dependencies and updates will arrive over time + image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} + # need to switch to root so that github actions can install runner binary on container without permission issues. + options: --user root + + env: + JENKINS_URL: build.ci.opensearch.org + + steps: + - name: Setup Java ${{ matrix.java }} + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: ${{ matrix.java }} + + - name: Checkout AD + uses: actions/checkout@v3 + + - name: Assemble / build / mavenlocal / integTest + run: | + chown -R 1000:1000 `pwd` + su `id -un 1000` -c "./gradlew assemble && + ./gradlew build && + ./gradlew publishToMavenLocal && + ./gradlew integTest -PnumNodes=3" + - name: Upload Coverage Report + uses: codecov/codecov-action@v3 + with: + file: ./build/reports/jacoco/test/jacocoTestReport.xml + flags: plugin + + Build-ad-macos: + strategy: + matrix: + java: [11,17,20] fail-fast: false name: Build and Test Anomaly detection Plugin - runs-on: ${{ matrix.os }} + runs-on: macos-latest env: JENKINS_URL: build.ci.opensearch.org @@ -54,6 +101,7 @@ jobs: - name: Setup Java ${{ matrix.java }} uses: actions/setup-java@v3 with: + distribution: 'temurin' java-version: ${{ matrix.java }} - name: Checkout AD diff --git a/.github/workflows/test_bwc.yml b/.github/workflows/test_bwc.yml index 10c42d26f..d753eea76 100644 --- a/.github/workflows/test_bwc.yml +++ b/.github/workflows/test_bwc.yml @@ -8,7 +8,13 @@ on: - "*" jobs: - Build-ad: + Get-CI-Image-Tag: + uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main + with: + product: opensearch + + Build-ad-linux: + needs: Get-CI-Image-Tag strategy: matrix: java: [11,17] @@ -16,30 +22,40 @@ jobs: name: Test Anomaly detection BWC runs-on: ubuntu-latest + container: + # using the same image which is used by opensearch-build team to build the OpenSearch Distribution + # this image tag is subject to change as more dependencies and updates will arrive over time + image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} + # need to switch to root so that github actions can install runner binary on container without permission issues. + options: --user root steps: - name: Setup Java ${{ matrix.java }} uses: actions/setup-java@v3 with: + distribution: 'temurin' java-version: ${{ matrix.java }} # anomaly-detection - name: Checkout AD - uses: actions/checkout@v4 + uses: actions/checkout@v3 - name: Assemble anomaly-detection run: | - plugin_version=`./gradlew properties -q | grep "opensearch_build:" | awk '{print $2}'` + chown -R 1000:1000 `pwd` + su `id -un 1000` -c "./gradlew properties -q | grep 'opensearch_build:' | awk '{print $2}'" > /tmp/plugin_version.txt + plugin_version=`cat /tmp/plugin_version.txt` echo plugin_version $plugin_version - ./gradlew assemble + su `id -un 1000` -c "./gradlew assemble" echo "Creating ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/$plugin_version ..." - mkdir -p ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/$plugin_version + su `id -un 1000` -c "mkdir -p ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/$plugin_version" echo "Copying ./build/distributions/*.zip to ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/$plugin_version ..." ls ./build/distributions/ - cp ./build/distributions/*.zip ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/$plugin_version + su `id -un 1000` -c "cp ./build/distributions/*.zip ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/$plugin_version" echo "Copied ./build/distributions/*.zip to ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/$plugin_version ..." ls ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/$plugin_version - name: Run AD Backwards Compatibility Tests run: | echo "Running backwards compatibility tests ..." - ./gradlew bwcTestSuite -Dtests.security.manager=false \ No newline at end of file + chown -R 1000:1000 `pwd` + su `id -un 1000` -c "./gradlew bwcTestSuite -Dtests.security.manager=false" diff --git a/.github/workflows/test_security.yml b/.github/workflows/test_security.yml index 384033710..5c1df108a 100644 --- a/.github/workflows/test_security.yml +++ b/.github/workflows/test_security.yml @@ -23,6 +23,7 @@ jobs: - name: Setup Java ${{ matrix.java }} uses: actions/setup-java@v3 with: + distribution: 'temurin' java-version: ${{ matrix.java }} # anomaly-detection From acf0135a34b04235591fce4688900ab3abf13a7f Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Mon, 30 Oct 2023 19:35:01 -0400 Subject: [PATCH 2/2] Fix the bwc test version retrieval (#1093) * Onboarding jenkins prod docker images to github actions Signed-off-by: Peter Zhu * Add temurin Signed-off-by: Peter Zhu * Add temurin Signed-off-by: Peter Zhu * Add more Signed-off-by: Peter Zhu * Add more Signed-off-by: Peter Zhu * Add more Signed-off-by: Peter Zhu --------- Signed-off-by: Peter Zhu --- .github/workflows/test_bwc.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/test_bwc.yml b/.github/workflows/test_bwc.yml index d753eea76..bc166633b 100644 --- a/.github/workflows/test_bwc.yml +++ b/.github/workflows/test_bwc.yml @@ -42,9 +42,8 @@ jobs: - name: Assemble anomaly-detection run: | + plugin_version=`./gradlew properties -q | grep "opensearch_build:" | awk '{print $2}'` chown -R 1000:1000 `pwd` - su `id -un 1000` -c "./gradlew properties -q | grep 'opensearch_build:' | awk '{print $2}'" > /tmp/plugin_version.txt - plugin_version=`cat /tmp/plugin_version.txt` echo plugin_version $plugin_version su `id -un 1000` -c "./gradlew assemble" echo "Creating ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/$plugin_version ..."