diff --git a/.github/workflows/hdfs-tests.yml b/.github/workflows/hdfs-tests.yml new file mode 100644 index 00000000..28c91225 --- /dev/null +++ b/.github/workflows/hdfs-tests.yml @@ -0,0 +1,88 @@ +name: HDFS Tests +on: + push: + branches: + - develop + pull_request: + branches-ignore: + - master + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} + cancel-in-progress: true + +env: + DEFAULT_PYTHON: '3.11' + +jobs: + tests: + name: Run HDFS tests + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@main + with: + # this might remove tools that are actually needed, + # if set to "true" but frees about 6 GB + tool-cache: false + + # all of these default to true, but feel free to set to + # "false" if necessary for your workflow + large-packages: true + docker-images: true + swap-storage: true + + - name: Cache jars + uses: actions/cache@v4 + with: + path: ./cached_jars + key: ${{ runner.os }}-python-${{ env.DEFAULT_PYTHON }}-test-hdfs-jars + restore-keys: | + ${{ runner.os }}-python-${{ env.DEFAULT_PYTHON }}-test-hdfs-jars + ${{ runner.os }}-python + ${{ runner.os }} + + - name: Build Worker Image + uses: docker/build-push-action@v5 + with: + context: . + tags: syncmaster_worker:${{ github.sha }} + target: test + file: docker/worker.dockerfile + load: true + cache-to: type=gha,mode=max + cache-from: type=gha + + - name: Docker compose up + run: | + docker compose -f docker-compose.test.yml down -v --remove-orphans + docker compose -f docker-compose.test.yml up -d db worker rabbitmq test-hive test-postgres --wait --wait-timeout 200 + env: + WORKER_IMAGE_TAG: ${{ github.sha }} + COMPOSE_PROJECT_NAME: ${{ github.run_id }}-syncmaster + + - name: Run HDFS Tests. + # Tests for the backend are run on the worker. + # Backend and worker on the same container. + run: | + docker compose -f ./docker-compose.test.yml exec -T worker pytest -vvv -s -m hdfs + env: + COMPOSE_PROJECT_NAME: ${{ github.run_id }}-syncmaster + + - name: Shutdown + if: always() + run: | + docker compose -f docker-compose.test.yml down -v --remove-orphans + env: + COMPOSE_PROJECT_NAME: ${{ github.run_id }}-syncmaster diff --git a/.github/workflows/hive-tests.yml b/.github/workflows/hive-tests.yml index a0aed7e5..2ae2a905 100644 --- a/.github/workflows/hive-tests.yml +++ b/.github/workflows/hive-tests.yml @@ -58,6 +58,7 @@ jobs: with: context: . tags: syncmaster_worker:${{ github.sha }} + target: test file: docker/worker.dockerfile load: true cache-to: type=gha,mode=max diff --git a/.github/workflows/oracle-tests.yml b/.github/workflows/oracle-tests.yml index e2588912..77b96507 100644 --- a/.github/workflows/oracle-tests.yml +++ b/.github/workflows/oracle-tests.yml @@ -58,6 +58,7 @@ jobs: with: context: . tags: syncmaster_worker:${{ github.sha }} + target: test file: docker/worker.dockerfile load: true cache-to: type=gha,mode=max diff --git a/.github/workflows/s3-tests.yml b/.github/workflows/s3-tests.yml index 6af07d4c..9aa01352 100644 --- a/.github/workflows/s3-tests.yml +++ b/.github/workflows/s3-tests.yml @@ -59,6 +59,7 @@ jobs: context: . tags: syncmaster_worker:${{ github.sha }} file: docker/worker.dockerfile + target: test load: true cache-to: type=gha,mode=max cache-from: type=gha diff --git a/syncmaster/tests/test_integration/test_run_transfer/conftest.py b/syncmaster/tests/test_integration/test_run_transfer/conftest.py index 32cd1cd5..6a0075fd 100644 --- a/syncmaster/tests/test_integration/test_run_transfer/conftest.py +++ b/syncmaster/tests/test_integration/test_run_transfer/conftest.py @@ -313,7 +313,7 @@ def hdfs_file_df_connection(spark, hdfs_server): from onetl.connection import SparkHDFS return SparkHDFS( - cluster="rnd-dwh", + cluster="test-hive", host=hdfs_server.host, ipc_port=hdfs_server.ipc_port, spark=spark,