From 85c367c5abc22840b9ddc8ae70d851c859132258 Mon Sep 17 00:00:00 2001 From: Cameron Skinner Date: Mon, 12 Oct 2020 20:27:56 +0000 Subject: [PATCH 1/3] Change sql-cli-, sql-jdbc-, sql-odbc-, and sql-workbench-release-workflow to use new staging bucket for artifacts --- .../workflows/sql-cli-release-workflow.yml | 28 ++++++++++---- .../workflows/sql-jdbc-release-workflow.yml | 24 ++++++++---- .../workflows/sql-odbc-release-workflow.yml | 38 ++++++++++++++----- .../sql-workbench-release-workflow.yml | 17 +++++++-- 4 files changed, 79 insertions(+), 28 deletions(-) diff --git a/.github/workflows/sql-cli-release-workflow.yml b/.github/workflows/sql-cli-release-workflow.yml index 78b843e1a7..aba99da4ef 100644 --- a/.github/workflows/sql-cli-release-workflow.yml +++ b/.github/workflows/sql-cli-release-workflow.yml @@ -36,19 +36,31 @@ jobs: run: | python -m pip install --upgrade pip pip install setuptools wheel twine - # publish to S3 and PyPI - - name: Build and Publish + + - name: Build run: | python setup.py sdist bdist_wheel - artifact=`ls ./dist/*.tar.gz` - wheel_artifact=`ls ./dist/*.whl` - aws s3 cp $artifact s3://artifacts.opendistroforelasticsearch.amazon.com/downloads/elasticsearch-clients/opendistro-sql-cli/ - aws s3 cp $wheel_artifact s3://artifacts.opendistroforelasticsearch.amazon.com/downloads/elasticsearch-clients/opendistro-sql-cli/ + - name: Publish to S3 + shell: bash + run: | + tarball=`ls ./dist/*.tar.gz` + wheel=`ls ./dist/*.whl` + + # Inject the build number before the suffix + tarball_outfile=`basename ${tarball%.tar.gz}-build-${GITHUB_RUN_NUMBER}.tar.gz` + wheel_outfile=`basename ${wheel%.whl}-build-${GITHUB_RUN_NUMBER}.whl` + + s3_prefix="s3://staging.artifacts.opendistroforelasticsearch.amazon.com/snapshot/elasticsearch-clients/sql-cli/" + + echo "Copying ${tarball} to ${s3_prefix}${tarball_outfile}" + aws s3 cp --quiet $tarball ${s3_prefix}${tarball_outfile} - # aws cloudfront create-invalidation --distribution-id ${{ secrets.DISTRIBUTION_ID }} --paths "/downloads/*" + echo "Copying ${wheel} to ${s3_prefix}${wheel_outfile}" + aws s3 cp --quiet $wheel ${s3_prefix}${wheel_outfile} - # TODO: Publish to PyPI + # TODO: Publish to PyPI + # - name: Publish to PyPI # env: # TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} # TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} diff --git a/.github/workflows/sql-jdbc-release-workflow.yml b/.github/workflows/sql-jdbc-release-workflow.yml index 9b24a33805..19783b087e 100644 --- a/.github/workflows/sql-jdbc-release-workflow.yml +++ b/.github/workflows/sql-jdbc-release-workflow.yml @@ -20,22 +20,32 @@ jobs: steps: - name: Checkout SQL-JDBC uses: actions/checkout@v1 - + - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v1 with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws-region: us-east-1 - + - name: Setup Java ${{ matrix.java }} uses: actions/setup-java@v1 with: java-version: ${{ matrix.java }} - - - name: Run build + + - name: Build run: | ./gradlew publishShadowPublicationToInternal-releasesRepository '-Dorg.gradle.jvmargs=--add-modules java.xml.bind' -Dbuild.snapshot=false - artifact=`ls -1t build/libs/*.jar | grep -v "SNAPSHOT.jar" | grep -v "sources.jar" | head -1` - aws s3 cp $artifact s3://artifacts.opendistroforelasticsearch.amazon.com/downloads/elasticsearch-clients/opendistro-sql-jdbc/ - aws cloudfront create-invalidation --distribution-id E2S86LO5GZAJQM --paths "/downloads/*" + + - name: Publish to S3 + shell: bash + run: | + jar=`ls -1t build/libs/*.jar | grep -v "SNAPSHOT.jar" | grep -v "sources.jar" | head -1` + + # Inject the build number before the suffix + jar_outfile=`basename ${jar%.jar}-build-${GITHUB_RUN_NUMBER}.jar` + + s3_prefix="s3://staging.artifacts.opendistroforelasticsearch.amazon.com/snapshot/elasticsearch-clients/sql-jdbc/" + + echo "Copying ${jar} to ${s3_prefix}${jar_outfile}" + aws s3 cp --quiet $jar ${s3_prefix}${jar_outfile} diff --git a/.github/workflows/sql-odbc-release-workflow.yml b/.github/workflows/sql-odbc-release-workflow.yml index 6d4a5ef353..1bb1783989 100644 --- a/.github/workflows/sql-odbc-release-workflow.yml +++ b/.github/workflows/sql-odbc-release-workflow.yml @@ -79,9 +79,16 @@ jobs: if: success() run: | cd installer - mac_installer=`ls -1t *.pkg | grep "Open Distro for Elasticsearch SQL ODBC Driver" | head -1` - echo $mac_installer - aws s3 cp "$mac_installer" s3://artifacts.opendistroforelasticsearch.amazon.com/downloads/elasticsearch-clients/opendistro-sql-odbc/mac/ + pkg=`ls -1t *.pkg | grep "Open Distro for Elasticsearch SQL ODBC Driver" | head -1` + + # Inject the build number before the suffix + pkg_outfile=`basename ${pkg%.pkg}-build-${GITHUB_RUN_NUMBER}.pkg` + + s3_prefix="s3://staging.artifacts.opendistroforelasticsearch.amazon.com/snapshot/elasticsearch-clients/sql-odbc/" + + echo "Copying ${pkg} to ${s3_prefix}${pkg_outfile}" + aws s3 cp --quiet $pkg ${s3_prefix}${pkg_outfile} + build-windows32: runs-on: windows-latest defaults: @@ -125,9 +132,16 @@ jobs: shell: bash run: | cd ci-output/installer - windows_installer=`ls -1t *.msi | grep "Open Distro for Elasticsearch SQL ODBC Driver" | head -1` - echo $windows_installer - aws s3 cp "$windows_installer" s3://artifacts.opendistroforelasticsearch.amazon.com/downloads/elasticsearch-clients/opendistro-sql-odbc/windows/ + msi=`ls -1t *.msi | grep "Open Distro for Elasticsearch SQL ODBC Driver" | head -1` + + # Inject the build number before the suffix + msi_outfile=`basename ${msi%.msi}-build-${GITHUB_RUN_NUMBER}.msi` + + s3_prefix="s3://staging.artifacts.opendistroforelasticsearch.amazon.com/snapshot/elasticsearch-clients/sql-odbc/" + + echo "Copying ${msi} to ${s3_prefix}${msi_outfile}" + aws s3 cp --quiet $msi ${s3_prefix}${msi_outfile} + build-windows64: runs-on: windows-latest defaults: @@ -171,6 +185,12 @@ jobs: shell: bash run: | cd ci-output/installer - windows_installer=`ls -1t *.msi | grep "Open Distro for Elasticsearch SQL ODBC Driver" | head -1` - echo $windows_installer - aws s3 cp "$windows_installer" s3://artifacts.opendistroforelasticsearch.amazon.com/downloads/elasticsearch-clients/opendistro-sql-odbc/windows/ + msi=`ls -1t *.msi | grep "Open Distro for Elasticsearch SQL ODBC Driver" | head -1` + + # Inject the build number before the suffix + msi_outfile=`basename ${msi%.msi}-build-${GITHUB_RUN_NUMBER}.msi` + + s3_prefix="s3://staging.artifacts.opendistroforelasticsearch.amazon.com/snapshot/elasticsearch-clients/sql-odbc/" + + echo "Copying ${msi} to ${s3_prefix}${msi_outfile}" + aws s3 cp --quiet $msi ${s3_prefix}${msi_outfile} diff --git a/.github/workflows/sql-workbench-release-workflow.yml b/.github/workflows/sql-workbench-release-workflow.yml index 1f15c53ad2..781cb487e3 100644 --- a/.github/workflows/sql-workbench-release-workflow.yml +++ b/.github/workflows/sql-workbench-release-workflow.yml @@ -48,11 +48,20 @@ jobs: max_attempts: 3 command: cd kibana/plugins/sql-workbench; yarn kbn bootstrap - - name: Build Artifact + - name: Build run: | cd kibana/plugins/sql-workbench yarn build - artifact=`ls ./build/*.zip` - aws s3 cp $artifact s3://artifacts.opendistroforelasticsearch.amazon.com/downloads/kibana-plugins/opendistro-sql-workbench/ - aws cloudfront create-invalidation --distribution-id ${{ secrets.DISTRIBUTION_ID }} --paths "/downloads/*" + - name: Upload to S3 + shell: bash + run: | + zip=`ls ./build/*.zip` + + # Inject the build number before the suffix + zip_outfile=`basename ${zip%.zip}-build-${GITHUB_RUN_NUMBER}.zip` + + s3_prefix="s3://staging.artifacts.opendistroforelasticsearch.amazon.com/snapshot/kibana-plugins/sql-workbench/" + + echo "Copying ${zip} to ${s3_prefix}${zip_outfile}" + aws s3 cp --quiet $zip ${s3_prefix}${zip_outfile} From 1a13922df86338aa5446cb19e459964e9ecdf2dd Mon Sep 17 00:00:00 2001 From: Sayali Gaikawad Date: Fri, 8 Jan 2021 13:28:58 -0800 Subject: [PATCH 2/3] Change AWS creds and rename staging folder --- .github/workflows/sql-cli-release-workflow.yml | 6 +++--- .../workflows/sql-jdbc-release-workflow.yml | 6 +++--- .../workflows/sql-odbc-release-workflow.yml | 18 +++++++++--------- .../sql-workbench-release-workflow.yml | 6 +++--- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/.github/workflows/sql-cli-release-workflow.yml b/.github/workflows/sql-cli-release-workflow.yml index aba99da4ef..1afc480280 100644 --- a/.github/workflows/sql-cli-release-workflow.yml +++ b/.github/workflows/sql-cli-release-workflow.yml @@ -28,8 +28,8 @@ jobs: - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v1 with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-access-key-id: ${{ secrets.AWS_STAGING_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_STAGING_SECRET_ACCESS_KEY }} aws-region: us-east-1 - name: Install Dependencies @@ -51,7 +51,7 @@ jobs: tarball_outfile=`basename ${tarball%.tar.gz}-build-${GITHUB_RUN_NUMBER}.tar.gz` wheel_outfile=`basename ${wheel%.whl}-build-${GITHUB_RUN_NUMBER}.whl` - s3_prefix="s3://staging.artifacts.opendistroforelasticsearch.amazon.com/snapshot/elasticsearch-clients/sql-cli/" + s3_prefix="s3://staging.artifacts.opendistroforelasticsearch.amazon.com/snapshots/elasticsearch-clients/sql-cli/" echo "Copying ${tarball} to ${s3_prefix}${tarball_outfile}" aws s3 cp --quiet $tarball ${s3_prefix}${tarball_outfile} diff --git a/.github/workflows/sql-jdbc-release-workflow.yml b/.github/workflows/sql-jdbc-release-workflow.yml index 19783b087e..3fb9cab896 100644 --- a/.github/workflows/sql-jdbc-release-workflow.yml +++ b/.github/workflows/sql-jdbc-release-workflow.yml @@ -24,8 +24,8 @@ jobs: - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v1 with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-access-key-id: ${{ secrets.AWS_STAGING_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_STAGING_SECRET_ACCESS_KEY }} aws-region: us-east-1 - name: Setup Java ${{ matrix.java }} @@ -45,7 +45,7 @@ jobs: # Inject the build number before the suffix jar_outfile=`basename ${jar%.jar}-build-${GITHUB_RUN_NUMBER}.jar` - s3_prefix="s3://staging.artifacts.opendistroforelasticsearch.amazon.com/snapshot/elasticsearch-clients/sql-jdbc/" + s3_prefix="s3://staging.artifacts.opendistroforelasticsearch.amazon.com/snapshots/elasticsearch-clients/sql-jdbc/" echo "Copying ${jar} to ${s3_prefix}${jar_outfile}" aws s3 cp --quiet $jar ${s3_prefix}${jar_outfile} diff --git a/.github/workflows/sql-odbc-release-workflow.yml b/.github/workflows/sql-odbc-release-workflow.yml index 1bb1783989..2bcf473e66 100644 --- a/.github/workflows/sql-odbc-release-workflow.yml +++ b/.github/workflows/sql-odbc-release-workflow.yml @@ -27,8 +27,8 @@ jobs: - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v1 with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-access-key-id: ${{ secrets.AWS_STAGING_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_STAGING_SECRET_ACCESS_KEY }} aws-region: us-east-1 - name: upload-cppcheck-results if: failure() @@ -84,7 +84,7 @@ jobs: # Inject the build number before the suffix pkg_outfile=`basename ${pkg%.pkg}-build-${GITHUB_RUN_NUMBER}.pkg` - s3_prefix="s3://staging.artifacts.opendistroforelasticsearch.amazon.com/snapshot/elasticsearch-clients/sql-odbc/" + s3_prefix="s3://staging.artifacts.opendistroforelasticsearch.amazon.com/snapshots/elasticsearch-clients/sql-odbc/" echo "Copying ${pkg} to ${s3_prefix}${pkg_outfile}" aws s3 cp --quiet $pkg ${s3_prefix}${pkg_outfile} @@ -101,8 +101,8 @@ jobs: - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v1 with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-access-key-id: ${{ secrets.AWS_STAGING_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_STAGING_SECRET_ACCESS_KEY }} aws-region: us-east-1 - name: configure-and-build-driver run: | @@ -137,7 +137,7 @@ jobs: # Inject the build number before the suffix msi_outfile=`basename ${msi%.msi}-build-${GITHUB_RUN_NUMBER}.msi` - s3_prefix="s3://staging.artifacts.opendistroforelasticsearch.amazon.com/snapshot/elasticsearch-clients/sql-odbc/" + s3_prefix="s3://staging.artifacts.opendistroforelasticsearch.amazon.com/snapshots/elasticsearch-clients/sql-odbc/" echo "Copying ${msi} to ${s3_prefix}${msi_outfile}" aws s3 cp --quiet $msi ${s3_prefix}${msi_outfile} @@ -154,8 +154,8 @@ jobs: - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v1 with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-access-key-id: ${{ secrets.AWS_STAGING_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_STAGING_SECRET_ACCESS_KEY }} aws-region: us-east-1 - name: configure-and-build-driver run: | @@ -190,7 +190,7 @@ jobs: # Inject the build number before the suffix msi_outfile=`basename ${msi%.msi}-build-${GITHUB_RUN_NUMBER}.msi` - s3_prefix="s3://staging.artifacts.opendistroforelasticsearch.amazon.com/snapshot/elasticsearch-clients/sql-odbc/" + s3_prefix="s3://staging.artifacts.opendistroforelasticsearch.amazon.com/snapshots/elasticsearch-clients/sql-odbc/" echo "Copying ${msi} to ${s3_prefix}${msi_outfile}" aws s3 cp --quiet $msi ${s3_prefix}${msi_outfile} diff --git a/.github/workflows/sql-workbench-release-workflow.yml b/.github/workflows/sql-workbench-release-workflow.yml index 781cb487e3..9b3a5006e9 100644 --- a/.github/workflows/sql-workbench-release-workflow.yml +++ b/.github/workflows/sql-workbench-release-workflow.yml @@ -16,8 +16,8 @@ jobs: - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v1 with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-access-key-id: ${{ secrets.AWS_STAGING_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_STAGING_SECRET_ACCESS_KEY }} aws-region: us-east-1 - name: Checkout Plugin @@ -61,7 +61,7 @@ jobs: # Inject the build number before the suffix zip_outfile=`basename ${zip%.zip}-build-${GITHUB_RUN_NUMBER}.zip` - s3_prefix="s3://staging.artifacts.opendistroforelasticsearch.amazon.com/snapshot/kibana-plugins/sql-workbench/" + s3_prefix="s3://staging.artifacts.opendistroforelasticsearch.amazon.com/snapshots/kibana-plugins/sql-workbench/" echo "Copying ${zip} to ${s3_prefix}${zip_outfile}" aws s3 cp --quiet $zip ${s3_prefix}${zip_outfile} From ab089e4a22f472b438f4c8fdd34b53963cf52fd0 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Thu, 28 Jan 2021 18:19:41 -0500 Subject: [PATCH 3/3] Update sql-odbc-release-workflow.yml --- .github/workflows/sql-odbc-release-workflow.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/sql-odbc-release-workflow.yml b/.github/workflows/sql-odbc-release-workflow.yml index 120e95200c..f0dba6c00b 100644 --- a/.github/workflows/sql-odbc-release-workflow.yml +++ b/.github/workflows/sql-odbc-release-workflow.yml @@ -79,7 +79,7 @@ jobs: if: success() run: | cd installer - pkg=`ls -1t *.pkg | grep "Open Distro for Elasticsearch SQL ODBC Driver" | head -1` + pkg=`ls -1t *.pkg | grep "macos" | head -1` # Inject the build number before the suffix pkg_outfile=`basename ${pkg%.pkg}-build-${GITHUB_RUN_NUMBER}.pkg` @@ -134,7 +134,7 @@ jobs: shell: bash run: | cd ci-output/installer - msi=`ls -1t *.msi | grep "Open Distro for Elasticsearch SQL ODBC Driver" | head -1` + msi=`ls -1t *.msi | grep "x86" | head -1` # Inject the build number before the suffix msi_outfile=`basename ${msi%.msi}-build-${GITHUB_RUN_NUMBER}.msi` @@ -189,7 +189,7 @@ jobs: shell: bash run: | cd ci-output/installer - msi=`ls -1t *.msi | grep "Open Distro for Elasticsearch SQL ODBC Driver" | head -1` + msi=`ls -1t *.msi | grep "x64" | head -1` # Inject the build number before the suffix msi_outfile=`basename ${msi%.msi}-build-${GITHUB_RUN_NUMBER}.msi`