Add build to workflow; update docker image

Signed-off-by: Tyler Ohlsen <ohltyler@amazon.com>
opensearch-project · Oct 10, 2023 · ab40be3 · ab40be3
1 parent de6e677
commit ab40be3
Show file tree

Hide file tree

Showing 3 changed files with 299 additions and 59 deletions.
diff --git a/.github/workflows/build-and-test-workflow.yml b/.github/workflows/build-and-test-workflow.yml
@@ -0,0 +1,142 @@
+name: Build and test workflow
+on:
+  push:
+    branches:
+      - "*"
+  pull_request:
+    branches:
+      - "*"
+env:
+  OPENSEARCH_DASHBOARDS_VERSION: 'main'
+jobs:
+  Get-CI-Image-Tag:
+    runs-on: ubuntu-latest
+    outputs:
+      ci-image-version-linux: ${{ steps.step-ci-image-version-linux.outputs.ci-image-version-linux }}
+    steps:
+      - name: Install crane
+        uses: iarekylew00t/crane-installer@v1
+        with:
+          crane-release: v0.15.2
+      - name: Checkout opensearch-build repository
+        uses: actions/checkout@v2
+        with:
+          repository: "opensearch-project/opensearch-build"
+          ref: "main"
+          path: "opensearch-build"
+      - name: Get ci image version from opensearch-build repository scripts
+        id: step-ci-image-version-linux
+        run: |
+          crane version
+          CI_IMAGE_VERSION=`opensearch-build/docker/ci/get-ci-images.sh -p rockylinux8 -u opensearch-dashboards -t build | head -1`          echo $CI_IMAGE_VERSION
+          echo $CI_IMAGE_VERSION
+          echo "ci-image-version-linux=$CI_IMAGE_VERSION" >> $GITHUB_OUTPUT
+
+  build-and-test-linux:
+    name: Build the plugin and run unit tests
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+    runs-on: ${{ matrix.os }}
+    needs: Get-CI-Image-Tag
+    container:
+      # using the same image which is used by opensearch-build team to build the OpenSearch Distribution
+      # this image tag is subject to change as more dependencies and updates will arrive over time
+      image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }}
+      options: --user root
+    steps:
+      # Enable longer filenames for windows
+      - name: Enable longer filenames
+        if: ${{ matrix.os == 'windows-latest' }}
+        run: git config --system core.longpaths true
+      - name: Checkout OpenSearch Dashboards
+        uses: actions/checkout@v2
+        with:
+          repository: opensearch-project/OpenSearch-Dashboards
+          ref: ${{ env.OPENSEARCH_DASHBOARDS_VERSION }}
+          path: OpenSearch-Dashboards
+      - name: Setup Node
+        uses: actions/setup-node@v3
+        with:
+          node-version-file: './OpenSearch-Dashboards/.nvmrc'
+          registry-url: 'https://registry.npmjs.org'
+      - name: Install Yarn
+        # Need to use bash to avoid having a windows/linux specific step
+        shell: bash
+        run: |
+          YARN_VERSION=$(node -p "require('./OpenSearch-Dashboards/package.json').engines.yarn")
+          echo "Installing yarn@$YARN_VERSION"
+          npm i -g yarn@$YARN_VERSION
+      - run: node -v
+      - run: yarn -v
+      - name: Checkout Anomaly Detection OpenSearch Dashboards plugin
+        uses: actions/checkout@v2
+        with:
+          path: OpenSearch-Dashboards/plugins/anomaly-detection-dashboards-plugin
+      - name: Bootstrap the plugin
+        run: |
+          cd OpenSearch-Dashboards/plugins/anomaly-detection-dashboards-plugin
+          yarn osd bootstrap
+      - name: Build the plugin
+        run: |
+          cd OpenSearch-Dashboards/plugins/anomaly-detection-dashboards-plugin
+          yarn build
+      - name: Run unit tests
+        run: |
+          cd OpenSearch-Dashboards/plugins/anomaly-detection-dashboards-plugin
+          yarn run test:jest --coverage
+      - name: Uploads coverage
+        uses: codecov/codecov-action@v1
+
+  # TODO: once github actions supports windows and macos docker containers, we can
+  # merge these in to the above step's matrix.
+  build-and-test-windows-macos:
+    name: Build the plugin and run unit tests
+    strategy:
+      matrix:
+        os: [macos-latest, windows-latest]
+    runs-on: ${{ matrix.os }}
+    steps:
+      # Enable longer filenames for windows
+      - name: Enable longer filenames
+        if: ${{ matrix.os == 'windows-latest' }}
+        run: git config --system core.longpaths true
+      - name: Checkout OpenSearch Dashboards
+        uses: actions/checkout@v2
+        with:
+          repository: opensearch-project/OpenSearch-Dashboards
+          ref: ${{ env.OPENSEARCH_DASHBOARDS_VERSION }}
+          path: OpenSearch-Dashboards
+      - name: Setup Node
+        uses: actions/setup-node@v3
+        with:
+          node-version-file: './OpenSearch-Dashboards/.nvmrc'
+          registry-url: 'https://registry.npmjs.org'
+      - name: Install Yarn
+        # Need to use bash to avoid having a windows/linux specific step
+        shell: bash
+        run: |
+          YARN_VERSION=$(node -p "require('./OpenSearch-Dashboards/package.json').engines.yarn")
+          echo "Installing yarn@$YARN_VERSION"
+          npm i -g yarn@$YARN_VERSION
+      - run: node -v
+      - run: yarn -v
+      - name: Checkout Anomaly Detection OpenSearch Dashboards plugin
+        uses: actions/checkout@v2
+        with:
+          path: OpenSearch-Dashboards/plugins/anomaly-detection-dashboards-plugin
+      - name: Bootstrap the plugin
+        run: |
+          cd OpenSearch-Dashboards/plugins/anomaly-detection-dashboards-plugin
+          yarn osd bootstrap
+      - name: Build the plugin
+        run: |
+          cd OpenSearch-Dashboards/plugins/anomaly-detection-dashboards-plugin
+          yarn build
+      - name: Run unit tests
+        run: |
+          cd OpenSearch-Dashboards/plugins/anomaly-detection-dashboards-plugin
+          yarn run test:jest --coverage
+      - name: Uploads coverage
+        uses: codecov/codecov-action@v1
+
diff --git a/.github/workflows/remote-integ-tests-workflow.yml b/.github/workflows/remote-integ-tests-workflow.yml
@@ -10,18 +10,172 @@ on:
     branches:
       - "*"
 jobs:
-  test-without-security:
-    name: Run integ tests without security
+  Get-CI-Image-Tag:
+    runs-on: ubuntu-latest
+    outputs:
+      ci-image-version-linux: ${{ steps.step-ci-image-version-linux.outputs.ci-image-version-linux }}
+    steps:
+      - name: Install crane
+        uses: iarekylew00t/crane-installer@v1
+        with:
+          crane-release: v0.15.2
+      - name: Checkout opensearch-build repository
+        uses: actions/checkout@v2
+        with:
+          repository: "opensearch-project/opensearch-build"
+          ref: "main"
+          path: "opensearch-build"
+      - name: Get ci image version from opensearch-build repository scripts
+        id: step-ci-image-version-linux
+        run: |
+          crane version
+          CI_IMAGE_VERSION=`opensearch-build/docker/ci/get-ci-images.sh -p rockylinux8 -u opensearch-dashboards -t build | head -1`          echo $CI_IMAGE_VERSION
+          echo $CI_IMAGE_VERSION
+          echo "ci-image-version-linux=$CI_IMAGE_VERSION" >> $GITHUB_OUTPUT
+
+  integ-test-without-security-linux:
+    name: Run integ tests without security - linux
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        java: [11]
+        include:
+          - os: windows-latest
+            cypress_cache_folder: ~/AppData/Local/Cypress/Cache
+          - os: ubuntu-latest
+            cypress_cache_folder: ~/.cache/Cypress
+    runs-on: ${{ matrix.os }}
+    needs: Get-CI-Image-Tag
+    container:
+      # using the same image which is used by opensearch-build team to build the OpenSearch Distribution
+      # this image tag is subject to change as more dependencies and updates will arrive over time
+      image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }}
+      options: --user root
+    steps:
+      - name: Set up Java 11
+        uses: actions/setup-java@v3
+        with:
+          distribution: 'corretto'
+          java-version: '11'
+
+      - name: Enable longer filenames
+        if: ${{ matrix.os == 'windows-latest' }}
+        run: git config --system core.longpaths true
+
+      - name: Checkout OpenSearch Dashboards
+        uses: actions/checkout@v2
+        with:
+          repository: opensearch-project/OpenSearch-Dashboards
+          ref: '${{ github.base_ref }}'
+          path: OpenSearch-Dashboards
+
+      - name: Checkout Anomaly Detection OpenSearch Dashboards plugin
+        uses: actions/checkout@v2
+        with:
+          path: OpenSearch-Dashboards/plugins/anomaly-detection-dashboards-plugin
+
+      - name: Setup Node
+        uses: actions/setup-node@v3
+        with:
+          node-version-file: './OpenSearch-Dashboards/.nvmrc'
+          registry-url: 'https://registry.npmjs.org'
+
+      - name: Install Yarn
+        # Need to use bash to avoid having a windows/linux specific step
+        shell: bash
+        run: |
+          YARN_VERSION=$(node -p "require('./OpenSearch-Dashboards/package.json').engines.yarn")
+          echo "Installing yarn@$YARN_VERSION"
+          npm i -g yarn@$YARN_VERSION
+
+      - run: node -v
+      - run: yarn -v
+
+      - name: Checkout Anomaly-Detection
+        uses: actions/checkout@v2
+        with:
+          path: anomaly-detection
+          repository: opensearch-project/anomaly-detection
+          ref: '${{ github.base_ref }}'
+
+      - name: Run OpenSearch with plugin
+        run: |
+          cd anomaly-detection
+          CONFIG_PATH=../OpenSearch-Dashboards/plugins/anomaly-detection-dashboards-plugin/opensearch_dashboards.json
+          OPENSEARCH_VERSION=$(node -p "require('$CONFIG_PATH').opensearchDashboardsVersion")-SNAPSHOT
+          echo "Using OpenSearch version $OPENSEARCH_VERSION"
+          ./gradlew run -Dopensearch.version=$OPENSEARCH_VERSION &
+          timeout 300 bash -c 'while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' localhost:9200)" != "200" ]]; do sleep 5; done'
+        shell: bash
+
+      - name: Bootstrap the plugin
+        run: |
+          cd OpenSearch-Dashboards/plugins/anomaly-detection-dashboards-plugin
+          yarn osd bootstrap
+
+      - name: Run OpenSearch Dashboards server
+        run: |
+          cd OpenSearch-Dashboards
+          yarn start --no-base-path --no-watch &
+        shell: bash
+
+      # Window is slow so wait longer
+      - name: Sleep until OSD server starts - windows
+        if: ${{ matrix.os == 'windows-latest' }}
+        run: Start-Sleep -s 400
+        shell: powershell
+
+      - name: Sleep until OSD server starts - non-windows
+        if: ${{ matrix.os != 'windows-latest' }}
+        run: sleep 300
+        shell: bash
+
+      - name: Checkout opensearch-dashboards-functional-test
+        uses: actions/checkout@v2
+        with:
+          path: opensearch-dashboards-functional-test
+          repository: opensearch-project/opensearch-dashboards-functional-test
+          ref: '${{ github.base_ref }}'
+
+      - name: Get Cypress version
+        id: cypress_version
+        run: |
+          echo "::set-output name=cypress_version::$(cat ./opensearch-dashboards-functional-test/package.json | jq '.devDependencies.cypress' | tr -d '"')"
+
+      - name: Cache Cypress
+        id: cache-cypress
+        uses: actions/cache@v1
+        with:
+          path: ${{ matrix.cypress_cache_folder }}
+          key: cypress-cache-v2-${{ runner.os }}-${{ hashFiles('**/package.json') }}
+        env:
+          CYPRESS_INSTALL_BINARY: ${{ steps.cypress_version.outputs.cypress_version }}
+      - run: npx cypress cache list
+      - run: npx cypress cache path
+
+      - name: Run AD cypress tests
+        uses: cypress-io/github-action@v2
+        with:
+          working-directory: opensearch-dashboards-functional-test
+          command: yarn run cypress run --env SECURITY_ENABLED=false --spec cypress/integration/plugins/anomaly-detection-dashboards-plugin/**/*.js
+        env:
+          CYPRESS_CACHE_FOLDER: ${{ matrix.cypress_cache_folder }}   
+
+  # TODO: once github actions supports windows and macos docker containers, we can
+  # merge these in to the above step's matrix.
+  integ-test-without-security-windows:
+    name: Run integ tests without security - windows
     strategy:
       matrix:
-        os: [ubuntu-latest, windows-latest]
+        os: [windows-latest]
         java: [11]
         include:
           - os: windows-latest
             cypress_cache_folder: ~/AppData/Local/Cypress/Cache
           - os: ubuntu-latest
             cypress_cache_folder: ~/.cache/Cypress
     runs-on: ${{ matrix.os }}
+    needs: Get-CI-Image-Tag
     steps:
       - name: Set up Java 11
         uses: actions/setup-java@v3

diff --git a/.github/workflows/unit-tests-workflow.yml b/.github/workflows/unit-tests-workflow.yml