migrating integration tests

databricks · Aug 25, 2023 · cda601e · cda601e
1 parent 9bc6abc
commit cda601e
Show file tree

Hide file tree

Showing 8 changed files with 191 additions and 150 deletions.
diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml
@@ -1,7 +1,7 @@
 name: Integration Tests
 on: push
 jobs:
-  run-tox-tests-uc:
+  run-integration-tests-uc:
     runs-on: ubuntu-latest
     environment: azure-prod
     env:
@@ -10,45 +10,91 @@ jobs:
       DBT_DATABRICKS_HTTP_PATH: ${{ secrets.TEST_PECO_WAREHOUSE_HTTP_PATH }}
       DBT_DATABRICKS_UC_INITIAL_CATALOG: peco
       TEST_PECO_CLUSTER_ID: ${{ secrets.TEST_PECO_UC_CLUSTER_ID }}
+
     steps:
       - name: Check out repository
         uses: actions/checkout@v3
+
       - name: Set up python
         id: setup-python
         uses: actions/setup-python@v4
         with:
           python-version: "3.10"
+
       - name: Get http path from environment
         run: python .github/workflows/build_cluster_http_path.py
         shell: sh
-      - name: Install tox
-        id: install-dependencies
-        run: pip install tox
-      - name: Run integration-databricks-uc-sql-endpoint
-        run: DBT_TEST_USER=notnecessaryformosttests@example.com DBT_DATABRICKS_HOST_NAME=$DBT_DATABRICKS_HOST_NAME DBT_DATABRICKS_HTTP_PATH=$DBT_DATABRICKS_HTTP_PATH DBT_DATABRICKS_TOKEN=$DBT_DATABRICKS_TOKEN tox -e integration-databricks-uc-sql-endpoint
-      - name: Run integration-uc-databricks-cluster
-        run: DBT_TEST_USER=notnecessaryformosttests@example.com DBT_DATABRICKS_HOST_NAME=$DBT_DATABRICKS_HOST_NAME DBT_DATABRICKS_HTTP_PATH=$DBT_DATABRICKS_CLUSTER_HTTP_PATH DBT_DATABRICKS_TOKEN=$DBT_DATABRICKS_TOKEN tox -e integration-databricks-uc-cluster
-  run-tox-tests-non-uc:
+
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+        with:
+          virtualenvs-create: true
+          virtualenvs-in-project: true
+          installer-parallel: true
+
+      - name: Load cached venv
+        id: cached-poetry-dependencies
+        uses: actions/cache@v3
+        with:
+          path: .venv
+          key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}
+
+      - name: Install dependencies
+        if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
+        run: poetry install --no-interaction --no-root
+
+      - name: Install project
+        run: poetry install --no-interaction
+
+      - name: Run UC cluster integration tests
+        run: poetry run nox -t uc_cluster
+
+      - name: Run UC SQL Warehouse integration tests
+        run: poetry run nox -t uc_sql_endpoint
+
+  run-integration-tests-non-uc:
     runs-on: ubuntu-latest
     environment: azure-prod
     env:
       DBT_DATABRICKS_HOST_NAME: ${{ secrets.DATABRICKS_HOST }}
       DBT_DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }}
       DBT_DATABRICKS_UC_INITIAL_CATALOG: peco
       TEST_PECO_CLUSTER_ID: ${{ secrets.TEST_PECO_CLUSTER_ID }}
+
     steps:
       - name: Check out repository
         uses: actions/checkout@v3
+
       - name: Set up python
         id: setup-python
         uses: actions/setup-python@v4
         with:
           python-version: "3.10"
+
       - name: Get http path from environment
         run: python .github/workflows/build_cluster_http_path.py
         shell: sh
-      - name: Install tox
-        id: install-dependencies
-        run: pip install tox
-      - name: Run integration-databricks-cluster
-        run: DBT_TEST_USER=notnecessaryformosttests@example.com DBT_DATABRICKS_HOST_NAME=$DBT_DATABRICKS_HOST_NAME DBT_DATABRICKS_HTTP_PATH=$DBT_DATABRICKS_CLUSTER_HTTP_PATH DBT_DATABRICKS_TOKEN=$DBT_DATABRICKS_TOKEN tox -e integration-databricks-cluster
+
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+        with:
+          virtualenvs-create: true
+          virtualenvs-in-project: true
+          installer-parallel: true
+
+      - name: Load cached venv
+        id: cached-poetry-dependencies
+        uses: actions/cache@v3
+        with:
+          path: .venv
+          key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}
+
+      - name: Install dependencies
+        if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
+        run: poetry install --no-interaction --no-root
+
+      - name: Install project
+        run: poetry install --no-interaction
+
+      - name: Run UC cluster integration tests
+        run: poetry run nox -t cluster
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -51,17 +51,31 @@ jobs:
           persist-credentials: false
 
       - name: Set up Python
+        id: setup-python
         uses: actions/setup-python@v4.3.0
         with:
-          python-version: "3.8"
+          python-version: "3.10"
 
-      - name: Install python dependencies
-        run: |
-          python3 -m pip install --user pipx
-          ./install_tools.sh
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+        with:
+          virtualenvs-create: true
+          virtualenvs-in-project: true
+          installer-parallel: true
+
+      - name: Load cached venv
+        id: cached-poetry-dependencies
+        uses: actions/cache@v3
+        with:
+          path: .venv
+          key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}
+
+      - name: Install dependencies
+        if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
+        run: poetry install --no-interaction --no-root
 
       - name: Run linting
-        run: nox -t lint
+        run: poetry run nox -t lint
 
   unit:
     name: unit test / python ${{ matrix.python-version }}
@@ -84,17 +98,31 @@ jobs:
           persist-credentials: false
 
       - name: Set up Python ${{ matrix.python-version }}
+        id: setup-python
         uses: actions/setup-python@v4.3.0
         with:
           python-version: ${{ matrix.python-version }}
 
-      - name: Install python dependencies
-        run: |
-          python3 -m pip install --user pipx
-          ./install_tools.sh
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+        with:
+          virtualenvs-create: true
+          virtualenvs-in-project: true
+          installer-parallel: true
+
+      - name: Load cached venv
+        id: cached-poetry-dependencies
+        uses: actions/cache@v3
+        with:
+          path: .venv
+          key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}
+
+      - name: Install dependencies
+        if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
+        run: poetry install --no-interaction --no-root
 
       - name: Run unit tests
-        run: nox -s unit-${{ matrix.python-version }}
+        run: poetry run nox -s unit-${{ matrix.python-version }}
 
       - name: Get current date
         if: always()

diff --git a/install_tools.sh b/install_tools.sh
diff --git a/noxfile.py b/noxfile.py
@@ -31,3 +31,81 @@ def mypy(session):
 def unit(session):
     session.run_always("poetry", "install", external=True)
     session.run("pytest", "--color=yes", "-v", "tests/unit")
+
+
+@session(tags=["cluster", "integration"])
+def cluster_integration(session):
+    session.run_always("poetry", "install", external=True)
+    session.run(
+        "pytest",
+        "-v",
+        "-m",
+        "profile_databricks_cluster",
+        "-n4",
+        "tests/integration",
+    )
+
+
+@session(tags=["cluster", "functional"])
+def cluster_functional(session):
+    session.run_always("poetry", "install", external=True)
+    session.run(
+        "pytest",
+        "-v",
+        "--profile",
+        "databricks_cluster",
+        "-n4",
+        "tests/functional",
+    )
+
+
+@session(tags=["uc_cluster", "integration"])
+def uc_cluster_integration(session):
+    session.run_always("poetry", "install", external=True)
+    session.run(
+        "pytest",
+        "-v",
+        "-m",
+        "profile_databricks_uc_cluster",
+        "-n4",
+        "tests/integration",
+    )
+
+
+@session(tags=["uc_cluster", "functional"])
+def uc_cluster_functional(session):
+    session.run_always("poetry", "install", external=True)
+    session.run(
+        "pytest",
+        "-v",
+        "--profile",
+        "databricks_uc_cluster",
+        "-n4",
+        "tests/functional",
+    )
+
+
+@session(tags=["uc_sql_endpoint", "integration"])
+def uc_sql_integration(session):
+    session.run_always("poetry", "install", external=True)
+    session.run(
+        "pytest",
+        "-v",
+        "-m",
+        "profile_databricks_uc_sql_endpoint",
+        "-n4",
+        "tests/integration",
+    )
+
+
+@session(tags=["uc_sql_endpoint", "functional"])
+def uc_sql_functional(session):
+    session.run_always("poetry", "install", external=True)
+    session.run(
+        "pytest",
+        "-v",
+        "--profile",
+        "databricks_uc_sql_endpoint",
+        "-n4",
+        "tests/functional",
+    )
diff --git a/poetry.lock b/poetry.lock