From 2aca52333f301116dea3255c9bf53b24e008ac3f Mon Sep 17 00:00:00 2001 From: ptiurin Date: Mon, 6 Mar 2023 16:52:29 +0000 Subject: [PATCH 1/8] ci: Support two parallel branches --- .github/workflows/code-check.yml | 9 ++- .github/workflows/integration-tests.yml | 7 ++ .github/workflows/nightly-v0.yml | 10 +++ .github/workflows/nightly-workflow.yml | 91 +++++++++++++++++++++++++ .github/workflows/nightly.yml | 76 +-------------------- .github/workflows/pull-request.yml | 2 +- .github/workflows/security-scan.yml | 2 + .github/workflows/unit-tests.yml | 9 ++- 8 files changed, 130 insertions(+), 76 deletions(-) create mode 100644 .github/workflows/nightly-v0.yml create mode 100644 .github/workflows/nightly-workflow.yml diff --git a/.github/workflows/code-check.yml b/.github/workflows/code-check.yml index e6d6f614a02..08f9d594b08 100644 --- a/.github/workflows/code-check.yml +++ b/.github/workflows/code-check.yml @@ -2,8 +2,13 @@ name: Code quality checks on: workflow_call: + inputs: + branch: + required: false + type: string + description: 'Branch to run on' push: - branches: [ main ] + branches: [ main, 0.x ] jobs: check-code: @@ -11,6 +16,8 @@ jobs: steps: - name: Check out code uses: actions/checkout@v2 + with: + ref: ${{ github.event.inputs.branch }} - name: Set up Python 3.8 uses: actions/setup-python@v2 diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 4e661706b73..b9ae7f712c7 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -2,6 +2,11 @@ name: Integration tests on: workflow_dispatch: workflow_call: + inputs: + branch: + required: false + type: string + description: 'Branch to run on' secrets: FIREBOLT_USERNAME: required: true @@ -17,6 +22,8 @@ jobs: steps: - name: Check out code uses: actions/checkout@v2 + with: + ref: ${{ inputs.branch }} - name: Set up Python 3.7 uses: actions/setup-python@v2 diff --git a/.github/workflows/nightly-v0.yml b/.github/workflows/nightly-v0.yml new file mode 100644 index 00000000000..fd23cbb971d --- /dev/null +++ b/.github/workflows/nightly-v0.yml @@ -0,0 +1,10 @@ +name: v0.x Nightly code check +on: + workflow_dispatch: + schedule: + - cron: '0 3 * * *' # 3 am UTC every day +jobs: + code-check: + uses: ./.github/workflows/nightly-workflow.yml + with: + branch: 0.x diff --git a/.github/workflows/nightly-workflow.yml b/.github/workflows/nightly-workflow.yml new file mode 100644 index 00000000000..b241a6020ac --- /dev/null +++ b/.github/workflows/nightly-workflow.yml @@ -0,0 +1,91 @@ +name: Nightly workflow +on: + workflow_call: + inputs: + branch: + required: true + type: string + description: 'Branch to run on' +jobs: + code-check: + uses: ./.github/workflows/code-check.yml + with: + branch: ${{ inputs.branch }} + unit-tests: + uses: ./.github/workflows/unit-tests.yml + with: + branch: ${{ inputs.branch }} + secrets: + GIST_PAT: ${{ secrets.GIST_PAT }} + security-scan: + needs: [unit-tests] + uses: ./.github/workflows/security-scan.yml + with: + branch: ${{ inputs.branch }} + secrets: + FOSSA_TOKEN: ${{ secrets.FOSSA_TOKEN }} + SONARCLOUD_TOKEN: ${{ secrets.SONARCLOUD_TOKEN }} + tests: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false # finish all jobs even if one fails + max-parallel: 2 + matrix: + os: ['ubuntu-latest', 'macos-latest', 'windows-latest'] + python-version: ['3.7', '3.8', '3.9', '3.10'] + steps: + - name: Check out code + uses: actions/checkout@v2 + with: + ref: ${{ inputs.branch }} + + - name: Set up Python 3.7 + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install ".[dev]" + + - name: Unit test with pytest + run: | + pytest tests/unit + + - name: Setup database and engine + id: setup + uses: firebolt-db/integration-testing-setup@master + with: + firebolt-username: ${{ secrets.FIREBOLT_STG_USERNAME }} + firebolt-password: ${{ secrets.FIREBOLT_STG_PASSWORD }} + api-endpoint: "api.staging.firebolt.io" + region: "us-east-1" + db_suffix: ${{ format('{0}_{1}', matrix.os, matrix.python-version) }} + + - name: Run integration tests + env: + USER_NAME: ${{ secrets.FIREBOLT_STG_USERNAME }} + PASSWORD: ${{ secrets.FIREBOLT_STG_PASSWORD }} + SERVICE_ID: ${{ secrets.SERVICE_ID_STG }} + SERVICE_SECRET: ${{ secrets.SERVICE_SECRET_STG }} + DATABASE_NAME: ${{ steps.setup.outputs.database_name }} + ENGINE_NAME: ${{ steps.setup.outputs.engine_name }} + ENGINE_URL: ${{ steps.setup.outputs.engine_url }} + STOPPED_ENGINE_NAME: ${{ steps.setup.outputs.stopped_engine_name }} + STOPPED_ENGINE_URL: ${{ steps.setup.outputs.stopped_engine_url }} + ACCOUNT_NAME: "firebolt" + API_ENDPOINT: "api.staging.firebolt.io" + run: | + pytest --timeout_method "thread" -o log_cli=true -o log_cli_level=INFO tests/integration + + - name: Slack Notify of failure + if: failure() + id: slack + uses: firebolt-db/action-slack-nightly-notify@v1 + with: + os: ${{ matrix.os }} + programming-language: Python + language-version: ${{ matrix.python-version }} + notifications-channel: 'ecosystem-ci-notifications' + slack-api-key: ${{ secrets.SLACK_BOT_TOKEN }} diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 86ff292d336..992dfb8ab09 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -5,76 +5,6 @@ on: - cron: '0 3 * * *' # 3 am UTC every day jobs: code-check: - uses: ./.github/workflows/code-check.yml - unit-tests: - uses: ./.github/workflows/unit-tests.yml - secrets: - GIST_PAT: ${{ secrets.GIST_PAT }} - security-scan: - needs: [unit-tests] - uses: ./.github/workflows/security-scan.yml - secrets: - FOSSA_TOKEN: ${{ secrets.FOSSA_TOKEN }} - SONARCLOUD_TOKEN: ${{ secrets.SONARCLOUD_TOKEN }} - tests: - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false # finish all jobs even if one fails - max-parallel: 2 - matrix: - os: ['ubuntu-latest', 'macos-latest', 'windows-latest'] - python-version: ['3.7', '3.8', '3.9', '3.10'] - steps: - - name: Check out code - uses: actions/checkout@v2 - - - name: Set up Python 3.7 - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - python -m pip install ".[dev]" - - - name: Unit test with pytest - run: | - pytest tests/unit - - - name: Setup database and engine - id: setup - uses: firebolt-db/integration-testing-setup@master - with: - firebolt-username: ${{ secrets.FIREBOLT_STG_USERNAME }} - firebolt-password: ${{ secrets.FIREBOLT_STG_PASSWORD }} - api-endpoint: "api.staging.firebolt.io" - region: "us-east-1" - db_suffix: ${{ format('{0}_{1}', matrix.os, matrix.python-version) }} - - - name: Run integration tests - env: - USER_NAME: ${{ secrets.FIREBOLT_STG_USERNAME }} - PASSWORD: ${{ secrets.FIREBOLT_STG_PASSWORD }} - SERVICE_ID: ${{ secrets.SERVICE_ID_STG }} - SERVICE_SECRET: ${{ secrets.SERVICE_SECRET_STG }} - DATABASE_NAME: ${{ steps.setup.outputs.database_name }} - ENGINE_NAME: ${{ steps.setup.outputs.engine_name }} - ENGINE_URL: ${{ steps.setup.outputs.engine_url }} - STOPPED_ENGINE_NAME: ${{ steps.setup.outputs.stopped_engine_name }} - STOPPED_ENGINE_URL: ${{ steps.setup.outputs.stopped_engine_url }} - ACCOUNT_NAME: "firebolt" - API_ENDPOINT: "api.staging.firebolt.io" - run: | - pytest --timeout_method "thread" -o log_cli=true -o log_cli_level=INFO tests/integration - - - name: Slack Notify of failure - if: failure() - id: slack - uses: firebolt-db/action-slack-nightly-notify@v1 - with: - os: ${{ matrix.os }} - programming-language: Python - language-version: ${{ matrix.python-version }} - notifications-channel: 'ecosystem-ci-notifications' - slack-api-key: ${{ secrets.SLACK_BOT_TOKEN }} + uses: ./.github/workflows/nightly-workflow.yml + with: + branch: main diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml index d19da4c30ab..9eab77e742d 100644 --- a/.github/workflows/pull-request.yml +++ b/.github/workflows/pull-request.yml @@ -2,7 +2,7 @@ name: Pull request on: pull_request: - branches: [ main ] + branches: [ main, 0.x ] jobs: check-title: diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index fb8706108e3..d5053df9cc9 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -15,6 +15,8 @@ jobs: steps: - name: "Checkout Code" uses: actions/checkout@v2 + with: + ref: ${{ github.event.inputs.branch || github.ref }} - name: "Download coverage report" uses: actions/download-artifact@v2 diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index feed92fcbc7..e65fc7fb265 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -5,11 +5,16 @@ name: Unit tests on: workflow_call: + inputs: + branch: + required: false + type: string + description: 'Branch to run on' secrets: GIST_PAT: required: true push: - branches: [ main ] + branches: [ main, 0.x ] jobs: unit-tests: @@ -19,6 +24,8 @@ jobs: steps: - name: Check out code uses: actions/checkout@v2 + with: + ref: ${{ github.event.inputs.branch }} - name: Set up Python 3.7 uses: actions/setup-python@v2 From 8bc65be560e4b38a228c33d1d13a0477b7cf9f97 Mon Sep 17 00:00:00 2001 From: ptiurin Date: Mon, 6 Mar 2023 16:54:47 +0000 Subject: [PATCH 2/8] syntax --- .github/workflows/code-check.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/code-check.yml b/.github/workflows/code-check.yml index 08f9d594b08..94cecfe8d66 100644 --- a/.github/workflows/code-check.yml +++ b/.github/workflows/code-check.yml @@ -17,7 +17,7 @@ jobs: - name: Check out code uses: actions/checkout@v2 with: - ref: ${{ github.event.inputs.branch }} + ref: ${{ inputs.branch }} - name: Set up Python 3.8 uses: actions/setup-python@v2 From d6b3c8e833946e16b0f0a08384331d016910d856 Mon Sep 17 00:00:00 2001 From: ptiurin Date: Mon, 6 Mar 2023 16:55:30 +0000 Subject: [PATCH 3/8] clean --- .github/workflows/security-scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index d5053df9cc9..02e5e0d1940 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -16,7 +16,7 @@ jobs: - name: "Checkout Code" uses: actions/checkout@v2 with: - ref: ${{ github.event.inputs.branch || github.ref }} + ref: ${{ github.event.inputs.branch }} - name: "Download coverage report" uses: actions/download-artifact@v2 From 3441aeb3f84c7bc154079da65a28d6c3181a5688 Mon Sep 17 00:00:00 2001 From: ptiurin Date: Mon, 6 Mar 2023 17:10:10 +0000 Subject: [PATCH 4/8] add security scan input --- .github/workflows/security-scan.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index 02e5e0d1940..3b3ac274482 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -3,6 +3,11 @@ name: Firebolt Security Scan on: workflow_dispatch: workflow_call: + inputs: + branch: + required: false + type: string + description: 'Branch to run on' secrets: FOSSA_TOKEN: required: true From ab6d1f663d3e571809372f2ea5f4c02b616e389c Mon Sep 17 00:00:00 2001 From: ptiurin Date: Mon, 6 Mar 2023 17:32:36 +0000 Subject: [PATCH 5/8] Release 0.x --- .github/workflows/release-v0.x.yml | 44 ++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 .github/workflows/release-v0.x.yml diff --git a/.github/workflows/release-v0.x.yml b/.github/workflows/release-v0.x.yml new file mode 100644 index 00000000000..351d3ac1589 --- /dev/null +++ b/.github/workflows/release-v0.x.yml @@ -0,0 +1,44 @@ +name: v0.x Release new version + +on: + workflow_dispatch: + inputs: + pre-release-tag: + required: false + description: 'Tag for pre-release (optional)' + major-release: + required: false + description: 'Trigger a major release (optional). Leave empty for regular release.' + +jobs: + integration-tests: + uses: ./.github/workflows/integration-tests.yml + with: + branch: 0.x + secrets: + FIREBOLT_USERNAME: ${{ secrets.FIREBOLT_USERNAME }} + FIREBOLT_PASSWORD: ${{ secrets.FIREBOLT_PASSWORD }} + SERVICE_ID: ${{ secrets.SERVICE_ID }} + SERVICE_SECRET: ${{ secrets.SERVICE_SECRET }} + + publish: + runs-on: ubuntu-latest + permissions: + contents: write + needs: integration-tests + steps: + - name: Check out code + uses: actions/checkout@v2 + with: + fetch-depth: 0 + ref: 0.x + token: ${{ secrets.RELEASE_PAT }} + + - name: 'Publish action' + uses: firebolt-db/action-python-release@main + with: + pre-release-tag: ${{ inputs.pre-release-tag }} + major-release: ${{ inputs.major-release }} + pypi-username: ${{ secrets.PYPI_USERNAME }} + pypi-password: ${{ secrets.PYPI_PASSWORD }} + version-file-path: "src/firebolt/__init__.py" From c86a032d1814506d104ba92d2367e5c3ad005460 Mon Sep 17 00:00:00 2001 From: ptiurin Date: Mon, 6 Mar 2023 17:43:46 +0000 Subject: [PATCH 6/8] fix naming --- .github/workflows/{release-v0.x.yml => release-v0.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{release-v0.x.yml => release-v0.yml} (100%) diff --git a/.github/workflows/release-v0.x.yml b/.github/workflows/release-v0.yml similarity index 100% rename from .github/workflows/release-v0.x.yml rename to .github/workflows/release-v0.yml From 6a47831094308c507df44c345634d3fb02340742 Mon Sep 17 00:00:00 2001 From: ptiurin Date: Mon, 6 Mar 2023 17:44:46 +0000 Subject: [PATCH 7/8] fix inputs in security scan --- .github/workflows/security-scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index 3b3ac274482..0fdfabc3a8e 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -21,7 +21,7 @@ jobs: - name: "Checkout Code" uses: actions/checkout@v2 with: - ref: ${{ github.event.inputs.branch }} + ref: ${{ inputs.branch }} - name: "Download coverage report" uses: actions/download-artifact@v2 From b4c46fa084b5cac786592deddb7fa6999206fc14 Mon Sep 17 00:00:00 2001 From: ptiurin Date: Tue, 7 Mar 2023 14:51:58 +0000 Subject: [PATCH 8/8] override main branch for v0 release --- .github/workflows/release-v0.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/release-v0.yml b/.github/workflows/release-v0.yml index 351d3ac1589..74ff77a8019 100644 --- a/.github/workflows/release-v0.yml +++ b/.github/workflows/release-v0.yml @@ -42,3 +42,4 @@ jobs: pypi-username: ${{ secrets.PYPI_USERNAME }} pypi-password: ${{ secrets.PYPI_PASSWORD }} version-file-path: "src/firebolt/__init__.py" + main-branch: 0.x