From 9fcd99766a8e96fcc712ef0027ca1957e350a910 Mon Sep 17 00:00:00 2001 From: Neville Samuell Date: Fri, 19 Nov 2021 12:57:02 -0500 Subject: [PATCH 1/3] Refactor CI checks to run as a single job This allows the docker build cache to be shared for each check, which is the majority of the build time. This will likely reduce CI usage by a factor of 7, until we have a solution that allows running in parallel with a shared docker cache. --- .github/workflows/safe_pr_checks.yml | 75 ++++++------------- .github/workflows/unsafe_pr_checks.yml | 7 +- Makefile | 8 +- .../test_external_database_connections.py | 4 +- 4 files changed, 30 insertions(+), 64 deletions(-) diff --git a/.github/workflows/safe_pr_checks.yml b/.github/workflows/safe_pr_checks.yml index 6225371fc..7e5beaa78 100644 --- a/.github/workflows/safe_pr_checks.yml +++ b/.github/workflows/safe_pr_checks.yml @@ -1,69 +1,36 @@ name: CI Checks - Safe on: [ + push, pull_request ] jobs: - Autoformat: + Check-All: runs-on: ubuntu-latest steps: - - run: echo "Running CI for branch ${{ github.ref }}." - - name: Check out repository code + - name: Checkout uses: actions/checkout@v2 - - run: echo "💡 The ${{ github.repository }} repository has been cloned to the runner." - - run: make black-ci - Check-Migrations: - runs-on: ubuntu-latest - steps: - - run: echo "Running CI for branch ${{ github.ref }}." - - name: Check out repository code - uses: actions/checkout@v2 - - run: echo "💡 The ${{ github.repository }} repository has been cloned to the runner." - - run: make init-db - - run: make check-migrations + - name: Format + run: make black-ci - Lint: - runs-on: ubuntu-latest - steps: - - run: echo "Running CI for branch ${{ github.ref }}." - - name: Check out repository code - uses: actions/checkout@v2 - - run: echo "💡 The ${{ github.repository }} repository has been cloned to the runner." - - run: make pylint + - name: Lint + run: make pylint - Check-Types: - runs-on: ubuntu-latest - steps: - - run: echo "Running CI for branch ${{ github.ref }}." - - name: Check out repository code - uses: actions/checkout@v2 - - run: echo "💡 The ${{ github.repository }} repository has been cloned to the runner." - - run: make mypy + - name: Check Types + run: make mypy - Unit-Tests: - runs-on: ubuntu-latest - steps: - - run: echo "Running CI for branch ${{ github.ref }}." - - name: Check out repository code - uses: actions/checkout@v2 - - run: echo "💡 The ${{ github.repository }} repository has been cloned to the runner." - - run: make pytest + - name: Init DB + run: make init-db - Integration-Tests-Access: - runs-on: ubuntu-latest - steps: - - run: echo "Running CI for branch ${{ github.ref }}." - - name: Check out repository code - uses: actions/checkout@v2 - - run: echo "💡 The ${{ github.repository }} repository has been cloned to the runner." - - run: make pytest-integration-access + - name: Check Migrations + run: make check-migrations - Integration-Tests-Erasure: - runs-on: ubuntu-latest - steps: - - run: echo "Running CI for branch ${{ github.ref }}." - - name: Check out repository code - uses: actions/checkout@v2 - - run: echo "💡 The ${{ github.repository }} repository has been cloned to the runner." - - run: make pytest-integration-erasure + - name: Unit Tests + run: make pytest + + - name: Integration Tests (Access) + run: make pytest-integration-access + + - name: Integration Tests (Erasure) + run: make pytest-integration-erasure \ No newline at end of file diff --git a/.github/workflows/unsafe_pr_checks.yml b/.github/workflows/unsafe_pr_checks.yml index 6f12affe3..4fa7066fc 100644 --- a/.github/workflows/unsafe_pr_checks.yml +++ b/.github/workflows/unsafe_pr_checks.yml @@ -9,11 +9,10 @@ jobs: runs-on: ubuntu-latest if: contains(github.event.pull_request.labels.*.name, 'run unsafe ci checks') steps: - - run: echo "Running CI for branch ${{ github.ref }}." - - name: Check out repository code + - name: Checkout uses: actions/checkout@v2 - - run: echo "💡 The ${{ github.repository }} repository has been cloned to the runner." - - name: Run unit tests that connect to an external db + + - name: Integration Tests (External) env: REDSHIFT_TEST_URI: ${{ secrets.REDSHIFT_TEST_URI }} SNOWFLAKE_TEST_URI: ${{ secrets.SNOWFLAKE_TEST_URI }} diff --git a/Makefile b/Makefile index 90ee02831..2e59c9177 100644 --- a/Makefile +++ b/Makefile @@ -70,7 +70,7 @@ docker-push: # CI #################### -check-all: black-ci pylint mypy pytest pytest-integration check-migrations +check-all: black-ci pylint mypy check-migrations pytest pytest-integration-access pytest-integration-erasure black-ci: compose-build @echo "Running black checks..." @@ -99,7 +99,7 @@ mypy: compose-build pytest: compose-build @echo "Running pytest unit tests..." @docker-compose run $(IMAGE_NAME) \ - pytest $(pytestpath) -m "not integration and not integration_erasure and not external_integration" + pytest $(pytestpath) -m "not integration and not integration_erasure and not integration_external" # Run the pytest integration tests. pytest-integration-access: compose-build @@ -124,10 +124,10 @@ pytest-integration-erasure: compose-build pytest $(pytestpath) -m "integration_erasure" # These tests connect to external third-party test databases -pytest-external-integration: compose-build +pytest-integration-external: compose-build @echo "Running tests that connect to external third party test databases" @docker-compose run -e REDSHIFT_TEST_URI -e SNOWFLAKE_TEST_URI $(IMAGE_NAME) \ - pytest $(pytestpath) -m "external_integration" + pytest $(pytestpath) -m "integration_external" #################### diff --git a/tests/integration_tests/test_external_database_connections.py b/tests/integration_tests/test_external_database_connections.py index 46e3ac90b..c1d96467d 100644 --- a/tests/integration_tests/test_external_database_connections.py +++ b/tests/integration_tests/test_external_database_connections.py @@ -59,7 +59,7 @@ def snowflake_test_engine() -> Generator: engine.dispose() -@pytest.mark.external_integration +@pytest.mark.integration_external def test_redshift_example_data(redshift_test_engine): """Confirm that we can connect to the redshift test db and get table names""" inspector = inspect(redshift_test_engine) @@ -78,7 +78,7 @@ def test_redshift_example_data(redshift_test_engine): ] -@pytest.mark.external_integration +@pytest.mark.integration_external def test_snowflake_example_data(snowflake_test_engine): """Confirm that we can connect to the snowflake test db and get table names""" inspector = inspect(snowflake_test_engine) From 614585c3f5fe04657fd2f2e2eae884c50c62d321 Mon Sep 17 00:00:00 2001 From: Neville Samuell Date: Fri, 19 Nov 2021 14:16:36 -0500 Subject: [PATCH 2/3] Fixes --- .github/workflows/safe_pr_checks.yml | 9 +++++---- .github/workflows/unsafe_pr_checks.yml | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/safe_pr_checks.yml b/.github/workflows/safe_pr_checks.yml index 7e5beaa78..7f952fca7 100644 --- a/.github/workflows/safe_pr_checks.yml +++ b/.github/workflows/safe_pr_checks.yml @@ -1,8 +1,9 @@ name: CI Checks - Safe -on: [ - push, - pull_request -] +on: + push: + branches: + - main + pull_request: jobs: Check-All: diff --git a/.github/workflows/unsafe_pr_checks.yml b/.github/workflows/unsafe_pr_checks.yml index 4fa7066fc..9082ebff3 100644 --- a/.github/workflows/unsafe_pr_checks.yml +++ b/.github/workflows/unsafe_pr_checks.yml @@ -16,4 +16,4 @@ jobs: env: REDSHIFT_TEST_URI: ${{ secrets.REDSHIFT_TEST_URI }} SNOWFLAKE_TEST_URI: ${{ secrets.SNOWFLAKE_TEST_URI }} - run: make pytest-external-integration + run: make pytest-integration-external From 480fc01bb75666071b244a2f21fb98d2ec3ab827 Mon Sep 17 00:00:00 2001 From: Neville Samuell Date: Fri, 19 Nov 2021 14:24:18 -0500 Subject: [PATCH 3/3] Polish tweaks --- .github/workflows/safe_pr_checks.yml | 3 +++ .github/workflows/unsafe_pr_checks.yml | 2 ++ 2 files changed, 5 insertions(+) diff --git a/.github/workflows/safe_pr_checks.yml b/.github/workflows/safe_pr_checks.yml index 7f952fca7..0d7db876c 100644 --- a/.github/workflows/safe_pr_checks.yml +++ b/.github/workflows/safe_pr_checks.yml @@ -12,6 +12,9 @@ jobs: - name: Checkout uses: actions/checkout@v2 + - name: Docker Compose Build + run: make compose-build + - name: Format run: make black-ci diff --git a/.github/workflows/unsafe_pr_checks.yml b/.github/workflows/unsafe_pr_checks.yml index 9082ebff3..9e6b347ec 100644 --- a/.github/workflows/unsafe_pr_checks.yml +++ b/.github/workflows/unsafe_pr_checks.yml @@ -1,6 +1,8 @@ name: CI Checks - Unsafe on: push: + branches: + - main pull_request: types: [labeled]