From d241422db3a3fd48547962dbb154ae68f9aaae03 Mon Sep 17 00:00:00 2001 From: Stepan Burlakov Date: Wed, 1 Dec 2021 11:33:50 +0200 Subject: [PATCH 01/18] make python versions string --- .github/workflows/nightly.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 1ba287b4e67..52384d08cd5 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -9,7 +9,7 @@ jobs: strategy: matrix: os: ['ubuntu-latest', 'macos-latest', 'windows-latest'] - python-version: [3.7, 3.8, 3.9, 3.10] + python-version: ['3.7', '3.8', '3.9', '3.10'] steps: - name: Check out code uses: actions/checkout@v2 From d4fc6f454c16806526d1846d6f4802b94c97aede Mon Sep 17 00:00:00 2001 From: Stepan Burlakov Date: Wed, 1 Dec 2021 11:36:48 +0200 Subject: [PATCH 02/18] remove fail-fast strategy --- .github/workflows/nightly.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 52384d08cd5..1d695b3f2c6 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -7,6 +7,7 @@ jobs: code-check: runs-on: ${{ matrix.os }} strategy: + fail-fast: false # finish all jobs even if one fails matrix: os: ['ubuntu-latest', 'macos-latest', 'windows-latest'] python-version: ['3.7', '3.8', '3.9', '3.10'] From 7df21f4ea413be88b3ade6db43e6a770a4d0d435 Mon Sep 17 00:00:00 2001 From: Stepan Burlakov Date: Wed, 1 Dec 2021 12:26:22 +0200 Subject: [PATCH 03/18] refactor workflows --- .github/workflows/code-check.yml | 29 +++++++++++++++++++ .github/workflows/integration-tests.yml | 4 +-- .github/workflows/nightly.yml | 38 ++++++------------------- .github/workflows/python-app.yml | 10 ++----- 4 files changed, 41 insertions(+), 40 deletions(-) create mode 100644 .github/workflows/code-check.yml diff --git a/.github/workflows/code-check.yml b/.github/workflows/code-check.yml new file mode 100644 index 00000000000..fa2c33d2f0f --- /dev/null +++ b/.github/workflows/code-check.yml @@ -0,0 +1,29 @@ +name: Python code checks + +on: + workflow_call: + +jobs: + check-code: + steps: + - name: Check out code + uses: actions/checkout@v2 + + - name: Set up Python 3.7 + uses: actions/setup-python@v2 + with: + python-version: 3.7 + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install ".[dev]" + + - name: Run pre-commit checks + uses: pre-commit/action@v2.0.3 + with: + extra_args: --all-files + + - name: Type check with mypy + run: | + mypy src diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index b79b54a4b19..d6875e97ae5 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -9,7 +9,7 @@ on: required: true jobs: tests: - runs-on: ubuntu-latest + runs-on: ${{ inputs.os }} steps: - name: Check out code uses: actions/checkout@v2 @@ -17,7 +17,7 @@ jobs: - name: Set up Python 3.7 uses: actions/setup-python@v2 with: - python-version: 3.7 + python-version: ${{ inputs.python-version}} - name: Install dependencies run: | diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 1d695b3f2c6..5074f6682fd 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -5,6 +5,12 @@ on: - cron: '0 3 * * *' # 3 am UTC every day jobs: code-check: + runs-on: ubuntu-latest + steps: + - name: Run code checks + using: .github/workflows/code-checks + + tests: runs-on: ${{ matrix.os }} strategy: fail-fast: false # finish all jobs even if one fails @@ -25,38 +31,10 @@ jobs: python -m pip install --upgrade pip pip install ".[dev]" - - name: Run pre-commit checks - uses: pre-commit/action@v2.0.3 - with: - extra_args: --all-files - - - name: Type check with mypy - run: | - mypy src - - name: Test with pytest run: | pytest tests/unit - - name: Setup database and engine - id: setup - uses: firebolt-db/integration-testing-setup@master - with: - firebolt-username: ${{ secrets.FIREBOLT_USERNAME }} - firebolt-password: ${{ secrets.FIREBOLT_PASSWORD }} - api-endpoint: "api.dev.firebolt.io" - region: "us-east-1" - - - name: Run integration tests - env: - USERNAME: ${{ secrets.FIREBOLT_USERNAME }} - PASSWORD: ${{ secrets.FIREBOLT_PASSWORD }} - DATABASE_NAME: ${{ steps.setup.outputs.database_name }} - ENGINE_NAME: ${{ steps.setup.outputs.engine_name }} - ENGINE_URL: ${{ steps.setup.outputs.engine_url }} - STOPPED_ENGINE_NAME: ${{ steps.setup.outputs.stopped_engine_name }} - STOPPED_ENGINE_URL: ${{ steps.setup.outputs.stopped_engine_url }} - API_ENDPOINT: "api.dev.firebolt.io" - run: | - pytest -o log_cli=true -o log_cli_level=INFO tests/integration + - name: Run integration tests: + uses: .github/workflows/integration-tests diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index 0cf4133c70d..49e57e0ec1c 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -28,14 +28,8 @@ jobs: python -m pip install --upgrade pip pip install ".[dev]" - - name: Run pre-commit checks - uses: pre-commit/action@v2.0.3 - with: - extra_args: --all-files - - - name: Type check with mypy - run: | - mypy src + - name: Run code checks + uses: .github/workflows/type-check - name: Test with pytest run: | From 4d40ee05bd359fec61e2c41b3ac4772759d22cb5 Mon Sep 17 00:00:00 2001 From: Stepan Burlakov Date: Wed, 1 Dec 2021 12:27:59 +0200 Subject: [PATCH 04/18] fix workflow paths --- .github/workflows/nightly.yml | 4 ++-- .github/workflows/python-app.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 5074f6682fd..a0f16b41c4c 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -8,7 +8,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Run code checks - using: .github/workflows/code-checks + using: .github/workflows/code-checks.yml tests: runs-on: ${{ matrix.os }} @@ -36,5 +36,5 @@ jobs: pytest tests/unit - name: Run integration tests: - uses: .github/workflows/integration-tests + uses: .github/workflows/integration-tests.yml diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index 49e57e0ec1c..2609769103f 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -29,7 +29,7 @@ jobs: pip install ".[dev]" - name: Run code checks - uses: .github/workflows/type-check + uses: .github/workflows/type-check.yml - name: Test with pytest run: | From b56b2061678719e2189a90a22aab9152a07f574f Mon Sep 17 00:00:00 2001 From: Stepan Burlakov Date: Wed, 1 Dec 2021 12:28:26 +0200 Subject: [PATCH 05/18] fix typo --- .github/workflows/nightly.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index a0f16b41c4c..088df632933 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -35,6 +35,6 @@ jobs: run: | pytest tests/unit - - name: Run integration tests: + - name: Run integration tests uses: .github/workflows/integration-tests.yml From 3fb142863a3de11f5785915a59ef7ae9fbb988d9 Mon Sep 17 00:00:00 2001 From: Stepan Burlakov Date: Wed, 1 Dec 2021 12:40:09 +0200 Subject: [PATCH 06/18] revert seprating code-check --- .github/workflows/nightly.yml | 25 ++++++++++++++++++++++--- .github/workflows/python-app.yml | 10 ++++++++-- 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 088df632933..f88c1812a81 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -7,8 +7,27 @@ jobs: code-check: runs-on: ubuntu-latest steps: - - name: Run code checks - using: .github/workflows/code-checks.yml + - name: Check out code + uses: actions/checkout@v2 + + - name: Set up Python 3.7 + uses: actions/setup-python@v2 + with: + python-version: 3.7 + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install ".[dev]" + + - name: Run pre-commit checks + uses: pre-commit/action@v2.0.3 + with: + extra_args: --all-files + + - name: Type check with mypy + run: | + mypy src tests: runs-on: ${{ matrix.os }} @@ -36,5 +55,5 @@ jobs: pytest tests/unit - name: Run integration tests - uses: .github/workflows/integration-tests.yml + uses: .github/workflows/integration-tests.yml@main diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index 2609769103f..0cf4133c70d 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -28,8 +28,14 @@ jobs: python -m pip install --upgrade pip pip install ".[dev]" - - name: Run code checks - uses: .github/workflows/type-check.yml + - name: Run pre-commit checks + uses: pre-commit/action@v2.0.3 + with: + extra_args: --all-files + + - name: Type check with mypy + run: | + mypy src - name: Test with pytest run: | From 520ad4a184e587e41f8f0b33bbda05574b70dea4 Mon Sep 17 00:00:00 2001 From: Stepan Burlakov Date: Wed, 1 Dec 2021 12:41:42 +0200 Subject: [PATCH 07/18] fix integration tests reference --- .github/workflows/nightly.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index f88c1812a81..15a24566a66 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -55,5 +55,5 @@ jobs: pytest tests/unit - name: Run integration tests - uses: .github/workflows/integration-tests.yml@main + uses: firebolt-db/firebolt-python-sdk/.github/workflows/integration-tests.yml@main From 0af369c49e986515463e53469a12b214182de015 Mon Sep 17 00:00:00 2001 From: Stepan Burlakov Date: Wed, 1 Dec 2021 12:43:37 +0200 Subject: [PATCH 08/18] revert reusing integration tests --- .github/workflows/nightly.yml | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 15a24566a66..ab303997b0c 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -54,6 +54,25 @@ jobs: run: | pytest tests/unit + - name: Setup database and engine + id: setup + uses: firebolt-db/integration-testing-setup@master + with: + firebolt-username: ${{ secrets.FIREBOLT_USERNAME }} + firebolt-password: ${{ secrets.FIREBOLT_PASSWORD }} + api-endpoint: "api.dev.firebolt.io" + region: "us-east-1" + - name: Run integration tests - uses: firebolt-db/firebolt-python-sdk/.github/workflows/integration-tests.yml@main + env: + USERNAME: ${{ secrets.FIREBOLT_USERNAME }} + PASSWORD: ${{ secrets.FIREBOLT_PASSWORD }} + DATABASE_NAME: ${{ steps.setup.outputs.database_name }} + ENGINE_NAME: ${{ steps.setup.outputs.engine_name }} + ENGINE_URL: ${{ steps.setup.outputs.engine_url }} + STOPPED_ENGINE_NAME: ${{ steps.setup.outputs.stopped_engine_name }} + STOPPED_ENGINE_URL: ${{ steps.setup.outputs.stopped_engine_url }} + API_ENDPOINT: "api.dev.firebolt.io" + run: | + pytest -o log_cli=true -o log_cli_level=INFO tests/integration From 725151bf7d4b860179e45487ec8af875a240ffa3 Mon Sep 17 00:00:00 2001 From: Stepan Burlakov Date: Wed, 1 Dec 2021 16:12:31 +0200 Subject: [PATCH 09/18] remove workflow call, use js action --- .github/workflows/nightly.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index ab303997b0c..79d327c1578 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -56,7 +56,7 @@ jobs: - name: Setup database and engine id: setup - uses: firebolt-db/integration-testing-setup@master + uses: firebolt-db/integration-testing-setup@convert_to_js with: firebolt-username: ${{ secrets.FIREBOLT_USERNAME }} firebolt-password: ${{ secrets.FIREBOLT_PASSWORD }} From 455e1f9d3b518200c87694ed46d91560c6699b3d Mon Sep 17 00:00:00 2001 From: Stepan Burlakov Date: Wed, 1 Dec 2021 18:36:10 +0200 Subject: [PATCH 10/18] improve response error handling --- .github/workflows/nightly.yml | 2 +- src/firebolt/client/resource_manager_hooks.py | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 79d327c1578..c0c46e96661 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -48,7 +48,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install ".[dev]" + python -m pip install ".[dev]" - name: Test with pytest run: | diff --git a/src/firebolt/client/resource_manager_hooks.py b/src/firebolt/client/resource_manager_hooks.py index 2e718342422..f2afdde336e 100644 --- a/src/firebolt/client/resource_manager_hooks.py +++ b/src/firebolt/client/resource_manager_hooks.py @@ -1,3 +1,4 @@ +from json import JSONDecodeError from logging import getLogger from httpx import HTTPStatusError, Request, RequestError, Response @@ -34,7 +35,10 @@ def raise_on_4xx_5xx(response: Response) -> None: raise exc except HTTPStatusError as exc: response.read() # without this, you can get a ResponseNotRead error - parsed_response = exc.response.json() + try: + parsed_response = exc.response.json() + except JSONDecodeError: + parsed_response = {"_raw": exc.response.text} debug_message = ( f"Error response {exc.response.status_code} " f"while requesting {exc.request.url!r}. " From bd92ee4ffe231e6ac161cfc49c978e5c662bd306 Mon Sep 17 00:00:00 2001 From: Stepan Burlakov Date: Wed, 1 Dec 2021 19:00:05 +0200 Subject: [PATCH 11/18] pass db suffix --- .github/workflows/nightly.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index c0c46e96661..a49c9106f50 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -62,6 +62,7 @@ jobs: firebolt-password: ${{ secrets.FIREBOLT_PASSWORD }} api-endpoint: "api.dev.firebolt.io" region: "us-east-1" + db_suffix: "${{ matrix.os }}_${{ matrix.python-version}}" - name: Run integration tests env: From b5db2d5418109e47ceebb8319b251345f8792bc6 Mon Sep 17 00:00:00 2001 From: Stepan Burlakov Date: Wed, 1 Dec 2021 19:07:51 +0200 Subject: [PATCH 12/18] fix db suffix --- .github/workflows/nightly.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index a49c9106f50..7031dc9f0a4 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -62,7 +62,7 @@ jobs: firebolt-password: ${{ secrets.FIREBOLT_PASSWORD }} api-endpoint: "api.dev.firebolt.io" region: "us-east-1" - db_suffix: "${{ matrix.os }}_${{ matrix.python-version}}" + db_suffix: ${{ format('{0}_{1}', matrix.os, matrix.python-version) }} - name: Run integration tests env: From bc1597aabe9941eabb5001e284e56e5db228f39a Mon Sep 17 00:00:00 2001 From: Stepan Burlakov Date: Thu, 2 Dec 2021 10:31:57 +0200 Subject: [PATCH 13/18] use js action in integration tests workflow --- .github/workflows/integration-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index d6875e97ae5..02f92e0e6d2 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -26,7 +26,7 @@ jobs: - name: Setup database and engine id: setup - uses: firebolt-db/integration-testing-setup@master + uses: firebolt-db/integration-testing-setup@convert_to_js with: firebolt-username: ${{ secrets.FIREBOLT_USERNAME }} firebolt-password: ${{ secrets.FIREBOLT_PASSWORD }} From fc4066962fed25d014305895c3bef8f62f0f33c1 Mon Sep 17 00:00:00 2001 From: Stepan Burlakov Date: Thu, 2 Dec 2021 10:32:05 +0200 Subject: [PATCH 14/18] reduce max parallel to 4 --- .github/workflows/nightly.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 7031dc9f0a4..473f3cfe707 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -33,6 +33,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: fail-fast: false # finish all jobs even if one fails + max-parallel: 4 matrix: os: ['ubuntu-latest', 'macos-latest', 'windows-latest'] python-version: ['3.7', '3.8', '3.9', '3.10'] From c34a753ffb77672880dd38d45161a37332ec2830 Mon Sep 17 00:00:00 2001 From: Stepan Burlakov Date: Thu, 2 Dec 2021 10:35:41 +0200 Subject: [PATCH 15/18] fix integration tests action --- .github/workflows/integration-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 02f92e0e6d2..cb5f6adb55e 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -9,7 +9,7 @@ on: required: true jobs: tests: - runs-on: ${{ inputs.os }} + runs-on: ubuntu-latest steps: - name: Check out code uses: actions/checkout@v2 @@ -17,7 +17,7 @@ jobs: - name: Set up Python 3.7 uses: actions/setup-python@v2 with: - python-version: ${{ inputs.python-version}} + python-version: 3.7 - name: Install dependencies run: | From c1a38aa94cc03140ab52f4a003293e92d54e2400 Mon Sep 17 00:00:00 2001 From: Stepan Burlakov Date: Fri, 3 Dec 2021 17:38:04 +0200 Subject: [PATCH 16/18] reduce parallelism to 3 --- .github/workflows/nightly.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 473f3cfe707..99cc6cfa035 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -33,7 +33,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: fail-fast: false # finish all jobs even if one fails - max-parallel: 4 + max-parallel: 3 matrix: os: ['ubuntu-latest', 'macos-latest', 'windows-latest'] python-version: ['3.7', '3.8', '3.9', '3.10'] From 89eca09da6a52b1686abe8e1d445aceb55f0317a Mon Sep 17 00:00:00 2001 From: Stepan Burlakov Date: Fri, 10 Dec 2021 14:23:36 +0200 Subject: [PATCH 17/18] reduce parallelism to 2 --- .github/workflows/nightly.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 99cc6cfa035..356c90c0660 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -33,7 +33,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: fail-fast: false # finish all jobs even if one fails - max-parallel: 3 + max-parallel: 2 matrix: os: ['ubuntu-latest', 'macos-latest', 'windows-latest'] python-version: ['3.7', '3.8', '3.9', '3.10'] From b5d44ac8d636f7b565ae720e7e475db0e949c410 Mon Sep 17 00:00:00 2001 From: Stepan Burlakov Date: Fri, 10 Dec 2021 14:26:21 +0200 Subject: [PATCH 18/18] use master for setup db action --- .github/workflows/integration-tests.yml | 2 +- .github/workflows/nightly.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index cb5f6adb55e..b79b54a4b19 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -26,7 +26,7 @@ jobs: - name: Setup database and engine id: setup - uses: firebolt-db/integration-testing-setup@convert_to_js + uses: firebolt-db/integration-testing-setup@master with: firebolt-username: ${{ secrets.FIREBOLT_USERNAME }} firebolt-password: ${{ secrets.FIREBOLT_PASSWORD }} diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 356c90c0660..96b735156e7 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -57,7 +57,7 @@ jobs: - name: Setup database and engine id: setup - uses: firebolt-db/integration-testing-setup@convert_to_js + uses: firebolt-db/integration-testing-setup@master with: firebolt-username: ${{ secrets.FIREBOLT_USERNAME }} firebolt-password: ${{ secrets.FIREBOLT_PASSWORD }}