refactor(test): consolidate testing infrastructure with Docker Compose

.github/actions/logs/action.yml

@@ -0,0 +1,19 @@
+name: Logs
+description: Display container logs on test failure
+
+runs:
+  using: composite
+  steps:
+    - name: Display container logs on failure
+      if: failure()
+      shell: bash
+      run: |
+        echo "=== Listing all Docker containers ==="
+        docker ps -a --filter "name=sq-test-" || true
+        echo ""
+        echo "=== Dumping container logs ==="
+        for container in $(docker ps -a --filter "name=sq-test-" --format "{{.Names}}"); do
+          echo ">>> Logs for $container <<<"
+          docker logs "$container" 2>&1 || true
+          echo ""
+        done

.github/actions/run-bazel-test/action.yml

@@ -0,0 +1,17 @@
+name: Run Bazel Test
+description: Run Bazel test and show logs on failure
+
+inputs:
+  target:
+    description: Bazel test target pattern
+    required: true
+
+runs:
+  using: composite
+  steps:
+    - name: Run Bazel test
+      shell: bash
+      run: ./tool/bazel test ${{ inputs.target }} --test_output=errors
+
+    - uses: ./.github/actions/logs
+      if: always()

.github/actions/setup/action.yml

@@ -0,0 +1,15 @@
+name: Setup
+description: Setup Bazel cache for CI jobs
+
+runs:
+  using: composite
+  steps:
+    - name: Setup Bazel cache
+      uses: actions/cache@v4
+      with:
+        path: |
+          ~/.cache/bazel
+          ~/.cache/bazelisk
+        key: bazel-${{ runner.os }}-${{ hashFiles('MODULE.bazel', 'go.mod', '.bazelversion') }}
+        restore-keys: |
+          bazel-${{ runner.os }}-

.github/workflows/ci.yml

@@ -0,0 +1,138 @@
+name: CI
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    types:
+      - opened
+      - reopened
+      - synchronize
+
+permissions:
+  contents: read
+  pull-requests: read
+
+jobs:
+  # ---------------------------------------------------------------------------
+  # BUILD AND UNIT TESTS (special case - Gazelle + build + unit tests)
+  # ---------------------------------------------------------------------------
+  build-and-unit-test:
+    name: Build and Unit Test
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/setup
+
+      - name: Check BUILD files are up to date
+        run: |
+          echo "Running Gazelle to check BUILD files..." >&2
+          make gazelle
+          if ! git diff --quiet; then
+            echo "BUILD files are out of date!" >&2
+            echo "" >&2
+            echo "The following files were modified by Gazelle:" >&2
+            git diff --name-only >&2
+            echo "" >&2
+            echo "Please run 'make gazelle' locally and commit the changes." >&2
+            exit 1
+          fi
+          echo "BUILD files are up to date" >&2
+
+      - name: Build project
+        run: make build
+
+      - name: Run unit tests
+        run: make test || echo "No unit tests found"
+
+  # ---------------------------------------------------------------------------
+  # INTEGRATION TESTS (e2e, gateway, orchestrator)
+  # ---------------------------------------------------------------------------
+  e2e:
+    name: E2E Integration Test
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/setup
+
+      - name: Run E2E tests
+        run: make e2e-test
+
+      - uses: ./.github/actions/logs
+
+  gateway-integration-test:
+    name: Gateway Integration Test
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/setup
+
+      - name: Run Gateway integration tests
+        run: make integration-test-gateway
+
+      - uses: ./.github/actions/logs
+
+  orchestrator-integration-test:
+    name: Orchestrator Integration Test
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/setup
+
+      - name: Run Orchestrator integration tests
+        run: make integration-test-orchestrator
+
+      - uses: ./.github/actions/logs
+
+  # ---------------------------------------------------------------------------
+  # EXTENSION TESTS
+  # ---------------------------------------------------------------------------
+  counter-integration-test:
+    name: Counter Extension Test
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/setup
+      - uses: ./.github/actions/run-bazel-test
+        with:
+          target: //test/integration/extension/counter/...
+
+  queue-integration-test:
+    name: Queue Extension Test
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/setup
+      - uses: ./.github/actions/run-bazel-test
+        with:
+          target: //test/integration/extension/queue/...
+
+  storage-integration-test:
+    name: Storage Extension Test
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/setup
+      - uses: ./.github/actions/run-bazel-test
+        with:
+          target: //test/integration/extension/storage/...
+
+  # ---------------------------------------------------------------------------
+  # REQUIRED CHECKS GATE
+  # ---------------------------------------------------------------------------
+  required-checks:
+    name: Required Checks
+    runs-on: ubuntu-latest
+    needs:
+      - build-and-unit-test
+      - e2e
+      - gateway-integration-test
+      - orchestrator-integration-test
+      - counter-integration-test
+      - queue-integration-test
+      - storage-integration-test
+    steps:
+      - name: All required checks passed
+        run: |
+          echo "All required checks passed!" >&2

.gitignore

@@ -10,3 +10,7 @@ MODULE.bazel.lock
 
 # Built binaries
 bin/
+.docker-bin/
+
+# Make completion cache
+.make_targets_cache

BUILD.bazel

@@ -7,4 +7,13 @@ load("@gazelle//:def.bzl", "gazelle")
 # gazelle:resolve go github.com/uber/submitqueue/orchestrator/protopb //orchestrator/protopb
 # gazelle:resolve go github.com/uber/submitqueue/speculator/protopb //speculator/protopb
 
+# Export marker files for test data dependencies (used by FindRepoRoot in tests)
+exports_files(
+    [
+        "MODULE.bazel",
+        "go.mod",
+    ],
+    visibility = ["//visibility:public"],
+)
+
 gazelle(name = "gazelle")

CLAUDE.md

@@ -53,8 +53,7 @@ Three services, each following the same layout:
 │       ├── {step}.go    # Step in workflow
 │       └── {step}_test.go
 ├── proto/               # Proto definitions (.proto files)
-├── protopb/             # Generated proto code (committed to repo)
-└── integration_test/
+└── protopb/             # Generated proto code (committed to repo)
 ```
 
 ### Controllers
@@ -141,27 +140,7 @@ extension/
 
 ### Directory Structure
 
-```
-submitqueue/
-├── MODULE.bazel              # Bzlmod dependencies
-├── go.mod                    # Go module dependencies
-├── BUILD.bazel               # Root build configuration
-├── Makefile                  # Build automation
-├── .bazelversion             # Pinned Bazel version
-├── .envrc                    # direnv configuration
-├── tool/bazel                # Bazelisk wrapper
-├── gateway/                  # Gateway service
-├── orchestrator/             # Orchestrator service
-├── speculator/               # Speculator service
-├── extension/                # Pluggable backend implementations
-├── entity/                   # Domain entities
-├── example/                  # Server and client examples
-│   ├── server/{service}/
-│   └── client/{service}/
-├── e2e_test/                 # Cross-service hermetic tests (Testcontainers)
-├── doc/                      # Documentation
-└── bin/                      # Compiled binaries (gitignored)
-```
+See [doc/PROJECT_STRUCTURE.md](doc/PROJECT_STRUCTURE.md) for detailed project organization and architecture.
 
 ### Build System
 
@@ -173,6 +152,9 @@ This repository uses **Bazel with Bzlmod** (NOT WORKSPACE) for dependency manage
 - **Bazel wrapper**: `./tool/bazel` (Bazelisk wrapper). With direnv (`.envrc`), use `bazel` directly.
 - **External dependencies**: Must be added to both `go.mod` AND `MODULE.bazel`
 - **BUILD files**: Every Go package must have a `BUILD.bazel` file
+- **Gazelle**: Run `make gazelle` after adding/removing Go files to update BUILD files
+  - CI enforces BUILD files are in sync - will fail if `make gazelle` generates changes
+  - Always run `make gazelle` before committing
 
 ### Proto Generation
 
@@ -196,20 +178,49 @@ All generated proto files are **committed to the repository**. When modifying `.
 - Use **singular** names for directories (e.g., `mock/` not `mocks/`, `entity/` not `entities/`)
 - This applies to all folders including test mocks, extensions, entities, and service directories
 
+### Makefile Convention
+
+The `Makefile` follows strict conventions for maintainability:
+
+**Alphabetical ordering:**
+- **Targets are alphabetically sorted** — makes it easy to find specific targets
+- **`.PHONY` declaration** — lists all targets in alphabetical order
+- **`help` target is always last** — exception to alphabetical ordering for discoverability
+- When adding new targets, insert them in alphabetical order (not at the end)
+
+**Help text documentation:**
+- **Add `## Description` after each target** — enables auto-generated help and shell completion
+- Format: `target: ## Short description of what this target does`
+- Example: `build: ## Build all services and examples`
+- Run `make help` to see all documented targets with descriptions
+- Shell completion (zsh) shows these descriptions when you press `<TAB>`
+
+**Example target with help text:**
+```makefile
+integration-test: build-all-linux ## Run all integration tests (auto-builds binaries)
+	@echo "Running all integration tests..."
+	@$(BAZEL) test //test/integration/... --test_output=errors
+```
+
+This convention makes the Makefile self-documenting and enables powerful shell completion.
+
 ### Common Make Targets
 
 ```bash
 make build                    # Build all services
 make proto                    # Regenerate proto files
 make test                     # Run unit tests
-make integration-test         # Run service integration tests
-make e2e-test                 # Run hermetic tests with Testcontainers
-make run-gateway              # Run gateway (port 8081)
-make run-orchestrator         # Run orchestrator (port 8082)
-make run-speculator           # Run speculator (port 8083)
-make run-client-gateway       # Run gateway client
+make integration-test         # Run all integration tests (Docker-based)
+make integration-test-gateway # Test Gateway service
+make e2e-test                 # Run end-to-end tests
+make local-start              # Start full stack with Docker Compose
+make local-ps                 # Show running containers and ports
+make local-logs               # View logs from all services
+make local-stop               # Stop all services
+make run-client-gateway       # Test Gateway client (SERVER_ADDR, MESSAGE)
+make run-client-orchestrator  # Test Orchestrator client
 make gazelle                  # Update BUILD.bazel files
-make clean                    # Remove binaries and Bazel cache
+make clean                    # Clean Bazel cache
 make clean-proto              # Remove generated proto files
 ```
 
@@ -241,6 +252,23 @@ make clean-proto              # Remove generated proto files
 2. **Avoid blocking operations for synchronization** — do not use `time.Sleep`. Design the tested routine to signal back (channels, callbacks, condition variables).
 3. **Use testify assertions** — use `stretchr/assert` or `require` instead of `t.Fatal()`.
 
+**Integration Test Conventions:**
+
+1. **Package naming** — use folder name as package name (NOT `*_test` suffix):
+   - `test/integration/gateway/` → `package gateway`
+   - `test/integration/extension/counter/mysql/` → `package mysql`
+   - This matches Uber's go-code integration test pattern
+
+2. **Bazel target naming** — use Gazelle-generated names and add `tags = ["integration"]`:
+   - Target name matches folder: `name = "gateway_test"`, `name = "mysql_test"`
+   - Always include `tags = ["integration"]` to exclude from unit tests
+   - Include `data = [...]` for docker-compose and schema files
+
+3. **Docker Compose-based** — all integration tests use Docker Compose:
+   - Use `testutil.NewComposeStack()` for hermetic setup
+   - Provide meaningful test context (e.g., "ext-storage-mysql", "svc-gateway")
+   - Use `stack.ConnectMySQLService()` or `stack.MySQLServiceDSN()` for DB connections
+
 ### Code Style Guidelines
 
 1. **Use SugaredLogger for structured logging** — always use `zap.SugaredLogger` with structured logging methods: