From 4bf1dd9c63a56fde63b52c8243dd56118ddc79f4 Mon Sep 17 00:00:00 2001 From: ptiurin Date: Mon, 5 Feb 2024 14:56:59 +0000 Subject: [PATCH 1/3] ci: Remove dev environment --- .github/workflows/integration-tests-v1.yml | 42 +++--------------- .github/workflows/integration-tests-v2.yml | 44 ++++--------------- .github/workflows/nightly-v2.yml | 4 +- .../workflows/python-integration-tests.yml | 8 ---- 4 files changed, 17 insertions(+), 81 deletions(-) diff --git a/.github/workflows/integration-tests-v1.yml b/.github/workflows/integration-tests-v1.yml index d955d7b..9dd218a 100644 --- a/.github/workflows/integration-tests-v1.yml +++ b/.github/workflows/integration-tests-v1.yml @@ -2,26 +2,8 @@ name: v1 Integration tests on: workflow_dispatch: - inputs: - environment: - description: 'Environment to run the tests against' - type: choice - required: true - default: 'dev' - options: - - dev - - staging workflow_call: - inputs: - environment: - default: 'staging' - required: false - type: string secrets: - FIREBOLT_USERNAME: - required: true - FIREBOLT_PASSWORD: - required: true FIREBOLT_STG_USERNAME: required: true FIREBOLT_STG_PASSWORD: @@ -44,23 +26,13 @@ jobs: python -m pip install --upgrade pip pip install ".[dev]" - - name: Determine env variables - run: | - if [ "${{ inputs.environment }}" == 'staging' ]; then - echo "USERNAME=${{ secrets.FIREBOLT_STG_USERNAME }}" >> "$GITHUB_ENV" - echo "PASSWORD=${{ secrets.FIREBOLT_STG_PASSWORD }}" >> "$GITHUB_ENV" - else - echo "USERNAME=${{ secrets.FIREBOLT_USERNAME }}" >> "$GITHUB_ENV" - echo "PASSWORD=${{ secrets.FIREBOLT_PASSWORD }}" >> "$GITHUB_ENV" - fi - - name: Setup database and engine id: setup uses: firebolt-db/integration-testing-setup@v1 with: - firebolt-username: ${{ env.USERNAME }} - firebolt-password: ${{ env.PASSWORD }} - api-endpoint: "api.${{ inputs.environment }}.firebolt.io" + firebolt-username: ${{ secrets.FIREBOLT_STG_USERNAME }} + firebolt-password: ${{ secrets.FIREBOLT_STG_PASSWORD }} + api-endpoint: "api.staging.firebolt.io" region: "us-east-1" - name: Restore cached failed tests @@ -74,14 +46,14 @@ jobs: - name: Run integration tests env: # the distinction is on the SDK side so just use CLIENT_* for username password - CLIENT_ID: ${{ env.USERNAME }} - CLIENT_SECRET: ${{ env.PASSWORD }} + CLIENT_ID: ${{ secrets.FIREBOLT_STG_USERNAME }} + CLIENT_SECRET: ${{ secrets.FIREBOLT_STG_PASSWORD }} DATABASE_NAME: ${{ steps.setup.outputs.database_name }} ENGINE_NAME: ${{ steps.setup.outputs.engine_name }} ENGINE_URL: ${{ steps.setup.outputs.engine_url }} STOPPED_ENGINE_NAME: ${{ steps.setup.outputs.stopped_engine_name }} STOPPED_ENGINE_URL: ${{ steps.setup.outputs.stopped_engine_url }} - FIREBOLT_BASE_URL: "api.${{ inputs.environment }}.firebolt.io" + FIREBOLT_BASE_URL: "api.staging.firebolt.io" ACCOUNT_NAME: "firebolt" run: | pytest --last-failed -o log_cli=true -o log_cli_level=INFO tests/integration @@ -93,4 +65,4 @@ jobs: with: path: | .pytest_cache/v/cache/lastfailed - key: ${{ steps.cache-tests-restore.outputs.cache-primary-key }} \ No newline at end of file + key: ${{ steps.cache-tests-restore.outputs.cache-primary-key }} diff --git a/.github/workflows/integration-tests-v2.yml b/.github/workflows/integration-tests-v2.yml index d816e83..2f25e11 100644 --- a/.github/workflows/integration-tests-v2.yml +++ b/.github/workflows/integration-tests-v2.yml @@ -2,30 +2,12 @@ name: v2 Integration tests on: workflow_dispatch: - inputs: - environment: - description: 'Environment to run the tests against' - type: choice - required: true - default: 'dev' - options: - - dev - - staging workflow_call: - inputs: - environment: - default: 'staging' - required: false - type: string secrets: FIREBOLT_CLIENT_ID_STG_NEW_IDN: required: true FIREBOLT_CLIENT_SECRET_STG_NEW_IDN: required: true - FIREBOLT_CLIENT_ID_NEW_IDN: - required: true - FIREBOLT_CLIENT_SECRET_NEW_IDN: - required: true jobs: integration-tests: @@ -44,24 +26,14 @@ jobs: python -m pip install --upgrade pip pip install ".[dev]" - - name: Determine env variables - run: | - if [ "${{ inputs.environment }}" == 'staging' ]; then - echo "CLIENT_ID=${{ secrets.FIREBOLT_CLIENT_ID_STG_NEW_IDN }}" >> "$GITHUB_ENV" - echo "CLIENT_SECRET=${{ secrets.FIREBOLT_CLIENT_SECRET_STG_NEW_IDN }}" >> "$GITHUB_ENV" - else - echo "CLIENT_ID=${{ secrets.FIREBOLT_CLIENT_ID_NEW_IDN }}" >> "$GITHUB_ENV" - echo "CLIENT_SECRET=${{ secrets.FIREBOLT_CLIENT_SECRET_NEW_IDN }}" >> "$GITHUB_ENV" - fi - - name: Setup database and engine id: setup uses: firebolt-db/integration-testing-setup@v2 with: - firebolt-client-id: ${{ env.CLIENT_ID }} - firebolt-client-secret: ${{ env.CLIENT_SECRET }} - account: "developer" - api-endpoint: "api.${{ inputs.environment }}.firebolt.io" + firebolt-client-id: ${{ secrets.FIREBOLT_CLIENT_ID_STG_NEW_IDN }} + firebolt-client-secret: ${{ secrets.FIREBOLT_CLIENT_SECRET_STG_NEW_IDN }} + account: ${{ vars.FIREBOLT_ACCOUNT }} + api-endpoint: "api.staging.firebolt.io" - name: Restore cached failed tests id: cache-tests-restore @@ -73,13 +45,13 @@ jobs: - name: Run integration tests env: - CLIENT_ID: ${{ env.CLIENT_ID }} - CLIENT_SECRET: ${{ env.CLIENT_SECRET }} + CLIENT_ID: ${{ secrets.FIREBOLT_CLIENT_ID_STG_NEW_IDN }} + CLIENT_SECRET: ${{ secrets.FIREBOLT_CLIENT_SECRET_STG_NEW_IDN }} DATABASE_NAME: ${{ steps.setup.outputs.database_name }} ENGINE_NAME: ${{ steps.setup.outputs.engine_name }} STOPPED_ENGINE_NAME: ${{ steps.setup.outputs.stopped_engine_name }} - FIREBOLT_BASE_URL: "api.${{ inputs.environment }}.firebolt.io" - ACCOUNT_NAME: "developer" + FIREBOLT_BASE_URL: "api.staging.firebolt.io" + ACCOUNT_NAME: ${{ vars.FIREBOLT_ACCOUNT }} run: | pytest --last-failed -o log_cli=true -o log_cli_level=INFO tests/integration diff --git a/.github/workflows/nightly-v2.yml b/.github/workflows/nightly-v2.yml index 38d63f7..869490d 100644 --- a/.github/workflows/nightly-v2.yml +++ b/.github/workflows/nightly-v2.yml @@ -47,7 +47,7 @@ jobs: with: firebolt-client-id: ${{ secrets.FIREBOLT_CLIENT_ID_STG_NEW_IDN }} firebolt-client-secret: ${{ secrets.FIREBOLT_CLIENT_SECRET_STG_NEW_IDN }} - account: "developer" + account: ${{ vars.FIREBOLT_ACCOUNT }} api-endpoint: "api.staging.firebolt.io" db_suffix: ${{ format('{0}_{1}', matrix.os, matrix.python-version) }} @@ -67,7 +67,7 @@ jobs: ENGINE_NAME: ${{ steps.setup.outputs.engine_name }} STOPPED_ENGINE_NAME: ${{ steps.setup.outputs.stopped_engine_name }} FIREBOLT_BASE_URL: "api.staging.firebolt.io" - ACCOUNT_NAME: "developer" + ACCOUNT_NAME: ${{ vars.FIREBOLT_ACCOUNT }} run: | pytest --last-failed -o log_cli=true -o log_cli_level=INFO --junit-xml=report/junit.xml tests/integration diff --git a/.github/workflows/python-integration-tests.yml b/.github/workflows/python-integration-tests.yml index c1e1bc5..11b9430 100644 --- a/.github/workflows/python-integration-tests.yml +++ b/.github/workflows/python-integration-tests.yml @@ -17,10 +17,6 @@ on: required: false type: string secrets: - FIREBOLT_USERNAME: - required: true - FIREBOLT_PASSWORD: - required: true FIREBOLT_STG_USERNAME: required: true FIREBOLT_STG_PASSWORD: @@ -30,10 +26,6 @@ on: required: true FIREBOLT_CLIENT_SECRET_STG_NEW_IDN: required: true - FIREBOLT_CLIENT_ID_NEW_IDN: - required: true - FIREBOLT_CLIENT_SECRET_NEW_IDN: - required: true jobs: integration-test-v1: uses: ./.github/workflows/integration-tests-v1.yml From 347b1e284a21bd013b7a6e1114018b6b1bc2b33d Mon Sep 17 00:00:00 2001 From: ptiurin Date: Mon, 5 Feb 2024 14:59:09 +0000 Subject: [PATCH 2/3] remove from parent wf --- .github/workflows/python-integration-tests.yml | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/.github/workflows/python-integration-tests.yml b/.github/workflows/python-integration-tests.yml index 11b9430..d91548d 100644 --- a/.github/workflows/python-integration-tests.yml +++ b/.github/workflows/python-integration-tests.yml @@ -1,21 +1,7 @@ name: Integration tests on: workflow_dispatch: - inputs: - environment: - description: 'Environment to run the tests against' - type: choice - required: true - default: 'dev' - options: - - dev - - staging workflow_call: - inputs: - environment: - default: 'staging' - required: false - type: string secrets: FIREBOLT_STG_USERNAME: required: true @@ -29,11 +15,7 @@ on: jobs: integration-test-v1: uses: ./.github/workflows/integration-tests-v1.yml - with: - environment: ${{ inputs.environment }} secrets: inherit integration-test-v2: uses: ./.github/workflows/integration-tests-v2.yml - with: - environment: ${{ inputs.environment }} secrets: inherit From 3711f29f1b6a23ade16c7bb9273105fe04d3d1b3 Mon Sep 17 00:00:00 2001 From: ptiurin Date: Mon, 5 Feb 2024 15:19:12 +0000 Subject: [PATCH 3/3] remove join index test --- tests/integration/test_sqlalchemy_integration.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/tests/integration/test_sqlalchemy_integration.py b/tests/integration/test_sqlalchemy_integration.py index af747ff..6ec6574 100644 --- a/tests/integration/test_sqlalchemy_integration.py +++ b/tests/integration/test_sqlalchemy_integration.py @@ -74,21 +74,6 @@ def test_agg_index(self, connection: Connection, fact_table_name: str): ) connection.execute(text(f"DROP AGGREGATING INDEX {agg_index}")) - def test_join_index(self, connection: Connection, dimension_table_name: str): - # Test if sql parsing allows it - join_index = "idx_join" - connection.execute( - text( - f""" - CREATE JOIN INDEX {join_index} ON {dimension_table_name} ( - idx, - dummy - ); - """ - ) - ) - connection.execute(text(f"DROP JOIN INDEX {join_index}")) - def test_get_schema_names(self, engine: Engine, database_name: str): results = engine.dialect.get_schema_names(engine) assert "public" in results