Merge pull request #777 from camerski/staging-workflows

Change release workflows to use new staging bucket
opendistro-for-elasticsearch · Feb 1, 2021 · 16f464c · 16f464c
2 parents bc40306 + 3616e97
commit 16f464c
Show file tree

Hide file tree

Showing 4 changed files with 100 additions and 45 deletions.
diff --git a/.github/workflows/sql-cli-release-workflow.yml b/.github/workflows/sql-cli-release-workflow.yml
@@ -28,29 +28,42 @@ jobs:
       - name: Configure AWS Credentials
         uses: aws-actions/configure-aws-credentials@v1
         with:
-          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-access-key-id: ${{ secrets.AWS_STAGING_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_STAGING_SECRET_ACCESS_KEY }}
           aws-region: us-east-1
 
       - name: Install Dependencies
         run: |
           python -m pip install --upgrade pip
           pip install setuptools wheel twine
-      # publish to S3 and PyPI
-      - name: Build and Publish
+
+      - name: Build
         run: |
           python setup.py sdist bdist_wheel
-          artifact=`ls ./dist/*.tar.gz`
-          wheel_artifact=`ls ./dist/*.whl`
-          renamed_wheel_artifact=`echo $wheel_artifact | sed 's/_/-/g'`
-          mv "$wheel_artifact" "$renamed_wheel_artifact"
 
-          aws s3 cp $artifact s3://artifacts.opendistroforelasticsearch.amazon.com/downloads/elasticsearch-clients/opendistro-sql-cli/
-          aws s3 cp $renamed_wheel_artifact s3://artifacts.opendistroforelasticsearch.amazon.com/downloads/elasticsearch-clients/opendistro-sql-cli/
+      - name: Publish to S3
+        shell: bash
+        run: |
+          tarball=`ls ./dist/*.tar.gz`
+          wheel=`ls ./dist/*.whl`
+
+          renamed_wheel=`echo $wheel | sed 's/_/-/g'`
+          mv "$wheel" "$renamed_wheel"
+
+          # Inject the build number before the suffix
+          tarball_outfile=`basename ${tarball%.tar.gz}-build-${GITHUB_RUN_NUMBER}.tar.gz`
+          wheel_outfile=`basename ${wheel%.whl}-build-${GITHUB_RUN_NUMBER}.whl`
+
+          s3_prefix="s3://staging.artifacts.opendistroforelasticsearch.amazon.com/snapshots/elasticsearch-clients/sql-cli/"
+
+          echo "Copying ${tarball} to ${s3_prefix}${tarball_outfile}"
+          aws s3 cp --quiet $tarball ${s3_prefix}${tarball_outfile}
 
-        # aws cloudfront create-invalidation --distribution-id ${{ secrets.DISTRIBUTION_ID }} --paths "/downloads/*"
+          echo "Copying ${wheel} to ${s3_prefix}${wheel_outfile}"
+          aws s3 cp --quiet $wheel ${s3_prefix}${wheel_outfile}
 
-        # TODO: Publish to PyPI
+      # TODO: Publish to PyPI
+      # - name: Publish to PyPI
         #     env:
         #        TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
         #        TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}

diff --git a/.github/workflows/sql-jdbc-release-workflow.yml b/.github/workflows/sql-jdbc-release-workflow.yml
@@ -24,8 +24,8 @@ jobs:
       - name: Configure AWS Credentials
         uses: aws-actions/configure-aws-credentials@v1
         with:
-          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-access-key-id: ${{ secrets.AWS_STAGING_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_STAGING_SECRET_ACCESS_KEY }}
           aws-region: us-east-1
 
       - name: Setup Java ${{ matrix.java }}
@@ -36,6 +36,16 @@ jobs:
       - name: Run build
         run: |
           ./gradlew publishShadowPublicationToInternal-releasesRepository '-Dorg.gradle.jvmargs=--add-modules java.xml.bind' -Dbuild.snapshot=false
-          artifact=`ls -1t build/libs/*.jar | grep -v "SNAPSHOT.jar" | grep -v "sources.jar" | head -1`
-          aws s3 cp $artifact s3://artifacts.opendistroforelasticsearch.amazon.com/downloads/elasticsearch-clients/opendistro-sql-jdbc/
-          aws cloudfront create-invalidation --distribution-id E2S86LO5GZAJQM --paths "/downloads/*"
+
+      - name: Publish to S3
+        shell: bash
+        run: |
+          jar=`ls -1t build/libs/*.jar | grep -v "SNAPSHOT.jar" | grep -v "sources.jar" | head -1`
+
+          # Inject the build number before the suffix
+          jar_outfile=`basename ${jar%.jar}-build-${GITHUB_RUN_NUMBER}.jar`
+
+          s3_prefix="s3://staging.artifacts.opendistroforelasticsearch.amazon.com/snapshots/elasticsearch-clients/sql-jdbc/"
+
+          echo "Copying ${jar} to ${s3_prefix}${jar_outfile}"
+          aws s3 cp --quiet $jar ${s3_prefix}${jar_outfile}
diff --git a/.github/workflows/sql-odbc-release-workflow.yml b/.github/workflows/sql-odbc-release-workflow.yml
@@ -29,8 +29,8 @@ jobs:
       - name: Configure AWS Credentials
         uses: aws-actions/configure-aws-credentials@v1
         with:
-          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-access-key-id: ${{ secrets.AWS_STAGING_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_STAGING_SECRET_ACCESS_KEY }}
           aws-region: us-east-1
       - name: upload-cppcheck-results
         if: failure()
@@ -81,11 +81,19 @@ jobs:
         if: success()
         run: |
           cd installer
-          mac_installer=`ls -1t *.pkg | grep "Open Distro for Elasticsearch SQL ODBC Driver" | head -1`
-          mv "$mac_installer" "${{ env.PLUGIN_NAME }}-${{ env.OD_VERSION }}-macos-x64.pkg"
-          mac_installer=`ls -1t *.pkg | grep "${{ env.PLUGIN_NAME }}-${{ env.OD_VERSION }}-macos-x64.pkg" | head -1`
-          echo $mac_installer
-          aws s3 cp "$mac_installer" s3://artifacts.opendistroforelasticsearch.amazon.com/downloads/elasticsearch-clients/opendistro-sql-odbc/mac/
+
+          pkg=`ls -1t *.pkg | grep "Open Distro for Elasticsearch SQL ODBC Driver" | head -1`
+          mv "$pkg" "${{ env.PLUGIN_NAME }}-${{ env.OD_VERSION }}-macos-x64.pkg"
+          pkg=`ls -1t *.pkg | grep "${{ env.PLUGIN_NAME }}-${{ env.OD_VERSION }}-macos-x64.pkg" | head -1`
+
+          # Inject the build number before the suffix
+          pkg_outfile=`basename ${pkg%.pkg}-build-${GITHUB_RUN_NUMBER}.pkg`
+
+          s3_prefix="s3://staging.artifacts.opendistroforelasticsearch.amazon.com/snapshots/elasticsearch-clients/sql-odbc/"
+
+          echo "Copying ${pkg} to ${s3_prefix}${pkg_outfile}"
+          aws s3 cp --quiet $pkg ${s3_prefix}${pkg_outfile}
+
   build-windows32:
     runs-on: windows-latest
     defaults:
@@ -100,8 +108,8 @@ jobs:
       - name: Configure AWS Credentials
         uses: aws-actions/configure-aws-credentials@v1
         with:
-          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-access-key-id: ${{ secrets.AWS_STAGING_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_STAGING_SECRET_ACCESS_KEY }}
           aws-region: us-east-1
       - name: configure-and-build-driver
         run: |
@@ -131,11 +139,19 @@ jobs:
         shell: bash
         run: |
           cd ci-output/installer
-          windows_installer=`ls -1t *.msi | grep "Open Distro for Elasticsearch SQL ODBC Driver" | head -1`
-          mv "$windows_installer" "${{ env.PLUGIN_NAME }}-${{ env.OD_VERSION }}-windows-x86.msi"
-          windows_installer=`ls -1t *.msi | grep "${{ env.PLUGIN_NAME }}-${{ env.OD_VERSION }}-windows-x86.msi" | head -1`
-          echo $windows_installer
-          aws s3 cp "$windows_installer" s3://artifacts.opendistroforelasticsearch.amazon.com/downloads/elasticsearch-clients/opendistro-sql-odbc/windows/
+          
+          msi=`ls -1t *.msi | grep "Open Distro for Elasticsearch SQL ODBC Driver" | head -1`
+          mv "$msi" "${{ env.PLUGIN_NAME }}-${{ env.OD_VERSION }}-windows-x86.msi"
+          msi=`ls -1t *.msi | grep "${{ env.PLUGIN_NAME }}-${{ env.OD_VERSION }}-windows-x86.msi" | head -1`
+
+          # Inject the build number before the suffix
+          msi_outfile=`basename ${msi%.msi}-build-${GITHUB_RUN_NUMBER}.msi`
+
+          s3_prefix="s3://staging.artifacts.opendistroforelasticsearch.amazon.com/snapshots/elasticsearch-clients/sql-odbc/"
+
+          echo "Copying ${msi} to ${s3_prefix}${msi_outfile}"
+          aws s3 cp --quiet $msi ${s3_prefix}${msi_outfile}
+
   build-windows64:
     runs-on: windows-latest
     defaults:
@@ -150,8 +166,8 @@ jobs:
       - name: Configure AWS Credentials
         uses: aws-actions/configure-aws-credentials@v1
         with:
-          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-access-key-id: ${{ secrets.AWS_STAGING_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_STAGING_SECRET_ACCESS_KEY }}
           aws-region: us-east-1
       - name: configure-and-build-driver
         run: |
@@ -181,8 +197,16 @@ jobs:
         shell: bash
         run: |
           cd ci-output/installer
-          windows_installer=`ls -1t *.msi | grep "Open Distro for Elasticsearch SQL ODBC Driver" | head -1`
-          mv "$windows_installer" "${{ env.PLUGIN_NAME }}-${{ env.OD_VERSION }}-windows-x64.msi"
-          windows_installer=`ls -1t *.msi | grep "${{ env.PLUGIN_NAME }}-${{ env.OD_VERSION }}-windows-x64.msi" | head -1`
-          echo $windows_installer
-          aws s3 cp "$windows_installer" s3://artifacts.opendistroforelasticsearch.amazon.com/downloads/elasticsearch-clients/opendistro-sql-odbc/windows/
+          
+          msi=`ls -1t *.msi | grep "Open Distro for Elasticsearch SQL ODBC Driver" | head -1`
+          mv "$msi" "${{ env.PLUGIN_NAME }}-${{ env.OD_VERSION }}-windows-x64.msi"
+          msi=`ls -1t *.msi | grep "${{ env.PLUGIN_NAME }}-${{ env.OD_VERSION }}-windows-x64.msi" | head -1`
+          
+          # Inject the build number before the suffix
+          msi_outfile=`basename ${msi%.msi}-build-${GITHUB_RUN_NUMBER}.msi`
+
+          s3_prefix="s3://staging.artifacts.opendistroforelasticsearch.amazon.com/snapshots/elasticsearch-clients/sql-odbc/"
+
+          echo "Copying ${msi} to ${s3_prefix}${msi_outfile}"
+          aws s3 cp --quiet $msi ${s3_prefix}${msi_outfile}
+
diff --git a/.github/workflows/sql-workbench-release-workflow.yml b/.github/workflows/sql-workbench-release-workflow.yml
@@ -20,8 +20,8 @@ jobs:
       - name: Configure AWS Credentials
         uses: aws-actions/configure-aws-credentials@v1
         with:
-          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-access-key-id: ${{ secrets.AWS_STAGING_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_STAGING_SECRET_ACCESS_KEY }}
           aws-region: us-east-1
 
       - name: Checkout Plugin
@@ -51,12 +51,20 @@ jobs:
           max_attempts: 3
           command: cd kibana/plugins/workbench; yarn kbn bootstrap
 
-      - name: Build Artifact
+      - name: Build
         run: |
           cd kibana/plugins/workbench
           yarn build
-          mv ./build/*.zip ./build/${{ env.PLUGIN_NAME }}-${{ env.OD_VERSION }}.zip
-          artifact=`ls ./build/*.zip`
 
-          aws s3 cp $artifact s3://artifacts.opendistroforelasticsearch.amazon.com/downloads/kibana-plugins/opendistro-query-workbench/
-          aws cloudfront create-invalidation --distribution-id ${{ secrets.DISTRIBUTION_ID }} --paths "/downloads/*"
+      - name: Upload to S3
+        shell: bash
+        run: |
+          zip=`ls ./build/*.zip`
+
+          # Inject the build number before the suffix
+          zip_outfile=`basename ${zip%.zip}-build-${GITHUB_RUN_NUMBER}.zip`
+
+          s3_prefix="s3://staging.artifacts.opendistroforelasticsearch.amazon.com/snapshots/kibana-plugins/sql-workbench/"
+
+          echo "Copying ${zip} to ${s3_prefix}${zip_outfile}"
+          aws s3 cp --quiet $zip ${s3_prefix}${zip_outfile}