refactor(test): consolidate testing infrastructure with Docker Compose

.github/workflows/build_and_test.yml

@@ -1,4 +1,4 @@
-name: Build and Test
+name: Build and Unit Tests
 
 on:
   push:
@@ -18,33 +18,23 @@ jobs:
     - name: Checkout code
       uses: actions/checkout@v4
 
-    - name: Build all services
+    - name: Check BUILD files are up to date
+      run: |
+        echo "Running Gazelle to check BUILD files..."
+        make gazelle
+        if ! git diff --quiet; then
+          echo "❌ BUILD files are out of date!"
+          echo ""
+          echo "The following files were modified by Gazelle:"
+          git diff --name-only
+          echo ""
+          echo "Please run 'make gazelle' locally and commit the changes."
+          exit 1
+        fi
+        echo "✅ BUILD files are up to date"
+
+    - name: Build project
       run: make build
 
     - name: Run unit tests
-      run: make test || echo "No tests found yet"
-
-    - name: Start all servers
-      run: make start-servers
-
-    - name: Run service integration tests
-      run: make integration-test
-
-    - name: Run end-to-end tests
-      run: make e2e-test
-
-    - name: Stop servers
-      if: always()
-      run: make stop-servers
-
-    - name: Display server logs on failure
-      if: failure()
-      run: |
-        echo "=== Gateway logs ==="
-        cat /tmp/gateway.log || echo "No gateway logs found"
-        echo ""
-        echo "=== Orchestrator logs ==="
-        cat /tmp/orchestrator.log || echo "No orchestrator logs found"
-        echo ""
-        echo "=== Speculator logs ==="
-        cat /tmp/speculator.log || echo "No speculator logs found"
+      run: make test || echo "No unit tests found"

.github/workflows/e2e_test.yml

@@ -0,0 +1,34 @@
+name: E2E Tests
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    types:
+      - opened
+      - reopened
+      - synchronize
+
+jobs:
+  e2e:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Run E2E tests
+        run: make e2e-test
+
+      - name: Display container logs on failure
+        if: failure()
+        run: |
+          echo "=== Listing all Docker containers ==="
+          docker ps -a --filter "name=sq-test-" || true
+          echo ""
+          echo "=== Dumping container logs ==="
+          for container in $(docker ps -a --filter "name=sq-test-" --format "{{.Names}}"); do
+            echo ">>> Logs for $container <<<"
+            docker logs "$container" 2>&1 || true
+            echo ""
+          done

.github/workflows/integration_test_extension.yml

@@ -0,0 +1,117 @@
+name: Integration Tests - Extensions
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    types:
+      - opened
+      - reopened
+      - synchronize
+
+permissions:
+  contents: read
+  pull-requests: read
+
+jobs:
+  # Detect which extensions changed
+  detect-changes:
+    runs-on: ubuntu-latest
+    outputs:
+      counter: ${{ steps.filter.outputs.counter }}
+      queue: ${{ steps.filter.outputs.queue }}
+      storage: ${{ steps.filter.outputs.storage }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dorny/paths-filter@v3
+        id: filter
+        with:
+          filters: |
+            counter:
+              - 'extension/counter/**'
+              - 'test/integration/extension/counter/**'
+              - 'entity/**'
+              - '.github/workflows/integration_test_extension.yml'
+            queue:
+              - 'extension/queue/**'
+              - 'test/integration/extension/queue/**'
+              - 'entity/**'
+              - '.github/workflows/integration_test_extension.yml'
+            storage:
+              - 'extension/storage/**'
+              - 'test/integration/extension/storage/**'
+              - 'entity/**'
+              - '.github/workflows/integration_test_extension.yml'
+
+  # Counter extension tests
+  test-counter:
+    needs: detect-changes
+    if: needs.detect-changes.outputs.counter == 'true'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Run counter extension tests
+        run: ./tool/bazel test //test/integration/extension/counter/... --test_output=errors
+
+      - name: Display container logs on failure
+        if: failure()
+        run: |
+          echo "=== Listing all Docker containers ==="
+          docker ps -a --filter "name=sq-test-" || true
+          echo ""
+          echo "=== Dumping container logs ==="
+          for container in $(docker ps -a --filter "name=sq-test-" --format "{{.Names}}"); do
+            echo ">>> Logs for $container <<<"
+            docker logs "$container" 2>&1 || true
+            echo ""
+          done
+
+  # Queue extension tests
+  test-queue:
+    needs: detect-changes
+    if: needs.detect-changes.outputs.queue == 'true'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Run queue extension tests
+        run: ./tool/bazel test //test/integration/extension/queue/... --test_output=errors
+
+      - name: Display container logs on failure
+        if: failure()
+        run: |
+          echo "=== Listing all Docker containers ==="
+          docker ps -a --filter "name=sq-test-" || true
+          echo ""
+          echo "=== Dumping container logs ==="
+          for container in $(docker ps -a --filter "name=sq-test-" --format "{{.Names}}"); do
+            echo ">>> Logs for $container <<<"
+            docker logs "$container" 2>&1 || true
+            echo ""
+          done
+
+  # Storage extension tests
+  test-storage:
+    needs: detect-changes
+    if: needs.detect-changes.outputs.storage == 'true'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Run storage extension tests
+        run: ./tool/bazel test //test/integration/extension/storage/... --test_output=errors
+
+      - name: Display container logs on failure
+        if: failure()
+        run: |
+          echo "=== Listing all Docker containers ==="
+          docker ps -a --filter "name=sq-test-" || true
+          echo ""
+          echo "=== Dumping container logs ==="
+          for container in $(docker ps -a --filter "name=sq-test-" --format "{{.Names}}"); do
+            echo ">>> Logs for $container <<<"
+            docker logs "$container" 2>&1 || true
+            echo ""
+          done

.github/workflows/integration_test_gateway.yml

@@ -0,0 +1,34 @@
+name: Integration Tests - Gateway Service
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    types:
+      - opened
+      - reopened
+      - synchronize
+
+jobs:
+  test-gateway:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Run Gateway integration tests
+        run: make integration-test-gateway
+
+      - name: Display container logs on failure
+        if: failure()
+        run: |
+          echo "=== Listing all Docker containers ==="
+          docker ps -a --filter "name=sq-test-" || true
+          echo ""
+          echo "=== Dumping container logs ==="
+          for container in $(docker ps -a --filter "name=sq-test-" --format "{{.Names}}"); do
+            echo ">>> Logs for $container <<<"
+            docker logs "$container" 2>&1 || true
+            echo ""
+          done

.github/workflows/integration_test_orchestrator.yml

@@ -0,0 +1,34 @@
+name: Integration Tests - Orchestrator Service
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    types:
+      - opened
+      - reopened
+      - synchronize
+
+jobs:
+  test-orchestrator:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Run Orchestrator integration tests
+        run: make integration-test-orchestrator
+
+      - name: Display container logs on failure
+        if: failure()
+        run: |
+          echo "=== Listing all Docker containers ==="
+          docker ps -a --filter "name=sq-test-" || true
+          echo ""
+          echo "=== Dumping container logs ==="
+          for container in $(docker ps -a --filter "name=sq-test-" --format "{{.Names}}"); do
+            echo ">>> Logs for $container <<<"
+            docker logs "$container" 2>&1 || true
+            echo ""
+          done

.gitignore

@@ -10,3 +10,7 @@ MODULE.bazel.lock
 
 # Built binaries
 bin/
+.docker-bin/
+
+# Make completion cache
+.make_targets_cache

BUILD.bazel

@@ -7,4 +7,13 @@ load("@gazelle//:def.bzl", "gazelle")
 # gazelle:resolve go github.com/uber/submitqueue/orchestrator/protopb //orchestrator/protopb
 # gazelle:resolve go github.com/uber/submitqueue/speculator/protopb //speculator/protopb
 
+# Export marker files for test data dependencies (used by FindRepoRoot in tests)
+exports_files(
+    [
+        "MODULE.bazel",
+        "go.mod",
+    ],
+    visibility = ["//visibility:public"],
+)
+
 gazelle(name = "gazelle")