diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 2dffbca8..442fbe60 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -1,4 +1,4 @@ -name: Build and Test +name: Build and Unit Tests on: push: @@ -18,33 +18,23 @@ jobs: - name: Checkout code uses: actions/checkout@v4 - - name: Build all services + - name: Check BUILD files are up to date + run: | + echo "Running Gazelle to check BUILD files..." + make gazelle + if ! git diff --quiet; then + echo "❌ BUILD files are out of date!" + echo "" + echo "The following files were modified by Gazelle:" + git diff --name-only + echo "" + echo "Please run 'make gazelle' locally and commit the changes." + exit 1 + fi + echo "✅ BUILD files are up to date" + + - name: Build project run: make build - name: Run unit tests - run: make test || echo "No tests found yet" - - - name: Start all servers - run: make start-servers - - - name: Run service integration tests - run: make integration-test - - - name: Run end-to-end tests - run: make e2e-test - - - name: Stop servers - if: always() - run: make stop-servers - - - name: Display server logs on failure - if: failure() - run: | - echo "=== Gateway logs ===" - cat /tmp/gateway.log || echo "No gateway logs found" - echo "" - echo "=== Orchestrator logs ===" - cat /tmp/orchestrator.log || echo "No orchestrator logs found" - echo "" - echo "=== Speculator logs ===" - cat /tmp/speculator.log || echo "No speculator logs found" + run: make test || echo "No unit tests found" diff --git a/.github/workflows/e2e_test.yml b/.github/workflows/e2e_test.yml new file mode 100644 index 00000000..a69b0708 --- /dev/null +++ b/.github/workflows/e2e_test.yml @@ -0,0 +1,34 @@ +name: E2E Tests + +on: + push: + branches: + - main + pull_request: + types: + - opened + - reopened + - synchronize + +jobs: + e2e: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Run E2E tests + run: make e2e-test + + - name: Display container logs on failure + if: failure() + run: | + echo "=== Listing all Docker containers ===" + docker ps -a --filter "name=sq-test-" || true + echo "" + echo "=== Dumping container logs ===" + for container in $(docker ps -a --filter "name=sq-test-" --format "{{.Names}}"); do + echo ">>> Logs for $container <<<" + docker logs "$container" 2>&1 || true + echo "" + done diff --git a/.github/workflows/integration_test_extension.yml b/.github/workflows/integration_test_extension.yml new file mode 100644 index 00000000..096f4453 --- /dev/null +++ b/.github/workflows/integration_test_extension.yml @@ -0,0 +1,117 @@ +name: Integration Tests - Extensions + +on: + push: + branches: + - main + pull_request: + types: + - opened + - reopened + - synchronize + +permissions: + contents: read + pull-requests: read + +jobs: + # Detect which extensions changed + detect-changes: + runs-on: ubuntu-latest + outputs: + counter: ${{ steps.filter.outputs.counter }} + queue: ${{ steps.filter.outputs.queue }} + storage: ${{ steps.filter.outputs.storage }} + steps: + - uses: actions/checkout@v4 + - uses: dorny/paths-filter@v3 + id: filter + with: + filters: | + counter: + - 'extension/counter/**' + - 'test/integration/extension/counter/**' + - 'entity/**' + - '.github/workflows/integration_test_extension.yml' + queue: + - 'extension/queue/**' + - 'test/integration/extension/queue/**' + - 'entity/**' + - '.github/workflows/integration_test_extension.yml' + storage: + - 'extension/storage/**' + - 'test/integration/extension/storage/**' + - 'entity/**' + - '.github/workflows/integration_test_extension.yml' + + # Counter extension tests + test-counter: + needs: detect-changes + if: needs.detect-changes.outputs.counter == 'true' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Run counter extension tests + run: ./tool/bazel test //test/integration/extension/counter/... --test_output=errors + + - name: Display container logs on failure + if: failure() + run: | + echo "=== Listing all Docker containers ===" + docker ps -a --filter "name=sq-test-" || true + echo "" + echo "=== Dumping container logs ===" + for container in $(docker ps -a --filter "name=sq-test-" --format "{{.Names}}"); do + echo ">>> Logs for $container <<<" + docker logs "$container" 2>&1 || true + echo "" + done + + # Queue extension tests + test-queue: + needs: detect-changes + if: needs.detect-changes.outputs.queue == 'true' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Run queue extension tests + run: ./tool/bazel test //test/integration/extension/queue/... --test_output=errors + + - name: Display container logs on failure + if: failure() + run: | + echo "=== Listing all Docker containers ===" + docker ps -a --filter "name=sq-test-" || true + echo "" + echo "=== Dumping container logs ===" + for container in $(docker ps -a --filter "name=sq-test-" --format "{{.Names}}"); do + echo ">>> Logs for $container <<<" + docker logs "$container" 2>&1 || true + echo "" + done + + # Storage extension tests + test-storage: + needs: detect-changes + if: needs.detect-changes.outputs.storage == 'true' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Run storage extension tests + run: ./tool/bazel test //test/integration/extension/storage/... --test_output=errors + + - name: Display container logs on failure + if: failure() + run: | + echo "=== Listing all Docker containers ===" + docker ps -a --filter "name=sq-test-" || true + echo "" + echo "=== Dumping container logs ===" + for container in $(docker ps -a --filter "name=sq-test-" --format "{{.Names}}"); do + echo ">>> Logs for $container <<<" + docker logs "$container" 2>&1 || true + echo "" + done diff --git a/.github/workflows/integration_test_gateway.yml b/.github/workflows/integration_test_gateway.yml new file mode 100644 index 00000000..89479ee4 --- /dev/null +++ b/.github/workflows/integration_test_gateway.yml @@ -0,0 +1,34 @@ +name: Integration Tests - Gateway Service + +on: + push: + branches: + - main + pull_request: + types: + - opened + - reopened + - synchronize + +jobs: + test-gateway: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Run Gateway integration tests + run: make integration-test-gateway + + - name: Display container logs on failure + if: failure() + run: | + echo "=== Listing all Docker containers ===" + docker ps -a --filter "name=sq-test-" || true + echo "" + echo "=== Dumping container logs ===" + for container in $(docker ps -a --filter "name=sq-test-" --format "{{.Names}}"); do + echo ">>> Logs for $container <<<" + docker logs "$container" 2>&1 || true + echo "" + done diff --git a/.github/workflows/integration_test_orchestrator.yml b/.github/workflows/integration_test_orchestrator.yml new file mode 100644 index 00000000..6f5642e0 --- /dev/null +++ b/.github/workflows/integration_test_orchestrator.yml @@ -0,0 +1,34 @@ +name: Integration Tests - Orchestrator Service + +on: + push: + branches: + - main + pull_request: + types: + - opened + - reopened + - synchronize + +jobs: + test-orchestrator: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Run Orchestrator integration tests + run: make integration-test-orchestrator + + - name: Display container logs on failure + if: failure() + run: | + echo "=== Listing all Docker containers ===" + docker ps -a --filter "name=sq-test-" || true + echo "" + echo "=== Dumping container logs ===" + for container in $(docker ps -a --filter "name=sq-test-" --format "{{.Names}}"); do + echo ">>> Logs for $container <<<" + docker logs "$container" 2>&1 || true + echo "" + done diff --git a/.gitignore b/.gitignore index 22b0c568..d9451591 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,7 @@ MODULE.bazel.lock # Built binaries bin/ +.docker-bin/ + +# Make completion cache +.make_targets_cache diff --git a/BUILD.bazel b/BUILD.bazel index ca41deec..052c08d9 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -7,4 +7,13 @@ load("@gazelle//:def.bzl", "gazelle") # gazelle:resolve go github.com/uber/submitqueue/orchestrator/protopb //orchestrator/protopb # gazelle:resolve go github.com/uber/submitqueue/speculator/protopb //speculator/protopb +# Export marker files for test data dependencies (used by FindRepoRoot in tests) +exports_files( + [ + "MODULE.bazel", + "go.mod", + ], + visibility = ["//visibility:public"], +) + gazelle(name = "gazelle") diff --git a/CLAUDE.md b/CLAUDE.md index 24b737d3..b9528d5c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -53,8 +53,7 @@ Three services, each following the same layout: │ ├── {step}.go # Step in workflow │ └── {step}_test.go ├── proto/ # Proto definitions (.proto files) -├── protopb/ # Generated proto code (committed to repo) -└── integration_test/ +└── protopb/ # Generated proto code (committed to repo) ``` ### Controllers @@ -141,27 +140,7 @@ extension/ ### Directory Structure -``` -submitqueue/ -├── MODULE.bazel # Bzlmod dependencies -├── go.mod # Go module dependencies -├── BUILD.bazel # Root build configuration -├── Makefile # Build automation -├── .bazelversion # Pinned Bazel version -├── .envrc # direnv configuration -├── tool/bazel # Bazelisk wrapper -├── gateway/ # Gateway service -├── orchestrator/ # Orchestrator service -├── speculator/ # Speculator service -├── extension/ # Pluggable backend implementations -├── entity/ # Domain entities -├── example/ # Server and client examples -│ ├── server/{service}/ -│ └── client/{service}/ -├── e2e_test/ # Cross-service hermetic tests (Testcontainers) -├── doc/ # Documentation -└── bin/ # Compiled binaries (gitignored) -``` +See [doc/PROJECT_STRUCTURE.md](doc/PROJECT_STRUCTURE.md) for detailed project organization and architecture. ### Build System @@ -173,6 +152,9 @@ This repository uses **Bazel with Bzlmod** (NOT WORKSPACE) for dependency manage - **Bazel wrapper**: `./tool/bazel` (Bazelisk wrapper). With direnv (`.envrc`), use `bazel` directly. - **External dependencies**: Must be added to both `go.mod` AND `MODULE.bazel` - **BUILD files**: Every Go package must have a `BUILD.bazel` file +- **Gazelle**: Run `make gazelle` after adding/removing Go files to update BUILD files + - CI enforces BUILD files are in sync - will fail if `make gazelle` generates changes + - Always run `make gazelle` before committing ### Proto Generation @@ -196,20 +178,49 @@ All generated proto files are **committed to the repository**. When modifying `. - Use **singular** names for directories (e.g., `mock/` not `mocks/`, `entity/` not `entities/`) - This applies to all folders including test mocks, extensions, entities, and service directories +### Makefile Convention + +The `Makefile` follows strict conventions for maintainability: + +**Alphabetical ordering:** +- **Targets are alphabetically sorted** — makes it easy to find specific targets +- **`.PHONY` declaration** — lists all targets in alphabetical order +- **`help` target is always last** — exception to alphabetical ordering for discoverability +- When adding new targets, insert them in alphabetical order (not at the end) + +**Help text documentation:** +- **Add `## Description` after each target** — enables auto-generated help and shell completion +- Format: `target: ## Short description of what this target does` +- Example: `build: ## Build all services and examples` +- Run `make help` to see all documented targets with descriptions +- Shell completion (zsh) shows these descriptions when you press `` + +**Example target with help text:** +```makefile +integration-test: build-all-linux ## Run all integration tests (auto-builds binaries) + @echo "Running all integration tests..." + @$(BAZEL) test //test/integration/... --test_output=errors +``` + +This convention makes the Makefile self-documenting and enables powerful shell completion. + ### Common Make Targets ```bash make build # Build all services make proto # Regenerate proto files make test # Run unit tests -make integration-test # Run service integration tests -make e2e-test # Run hermetic tests with Testcontainers -make run-gateway # Run gateway (port 8081) -make run-orchestrator # Run orchestrator (port 8082) -make run-speculator # Run speculator (port 8083) -make run-client-gateway # Run gateway client +make integration-test # Run all integration tests (Docker-based) +make integration-test-gateway # Test Gateway service +make e2e-test # Run end-to-end tests +make local-start # Start full stack with Docker Compose +make local-ps # Show running containers and ports +make local-logs # View logs from all services +make local-stop # Stop all services +make run-client-gateway # Test Gateway client (SERVER_ADDR, MESSAGE) +make run-client-orchestrator # Test Orchestrator client make gazelle # Update BUILD.bazel files -make clean # Remove binaries and Bazel cache +make clean # Clean Bazel cache make clean-proto # Remove generated proto files ``` @@ -241,6 +252,23 @@ make clean-proto # Remove generated proto files 2. **Avoid blocking operations for synchronization** — do not use `time.Sleep`. Design the tested routine to signal back (channels, callbacks, condition variables). 3. **Use testify assertions** — use `stretchr/assert` or `require` instead of `t.Fatal()`. +**Integration Test Conventions:** + +1. **Package naming** — use folder name as package name (NOT `*_test` suffix): + - `test/integration/gateway/` → `package gateway` + - `test/integration/extension/counter/mysql/` → `package mysql` + - This matches Uber's go-code integration test pattern + +2. **Bazel target naming** — use Gazelle-generated names and add `tags = ["integration"]`: + - Target name matches folder: `name = "gateway_test"`, `name = "mysql_test"` + - Always include `tags = ["integration"]` to exclude from unit tests + - Include `data = [...]` for docker-compose and schema files + +3. **Docker Compose-based** — all integration tests use Docker Compose: + - Use `testutil.NewComposeStack()` for hermetic setup + - Provide meaningful test context (e.g., "ext-storage-mysql", "svc-gateway") + - Use `stack.ConnectMySQLService()` or `stack.MySQLServiceDSN()` for DB connections + ### Code Style Guidelines 1. **Use SugaredLogger for structured logging** — always use `zap.SugaredLogger` with structured logging methods: diff --git a/MODULE.bazel b/MODULE.bazel index 70ce092f..ad92bd90 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -34,8 +34,6 @@ use_repo( "com_github_go_sql_driver_mysql", "com_github_gogo_protobuf", "com_github_stretchr_testify", - "com_github_testcontainers_testcontainers_go", - "com_github_testcontainers_testcontainers_go_modules_mysql", "com_github_uber_go_tally_v4", "org_golang_google_grpc", "org_golang_google_protobuf", diff --git a/Makefile b/Makefile index cc537a70..9b25664c 100644 --- a/Makefile +++ b/Makefile @@ -1,210 +1,242 @@ -.PHONY: proto build test integration-test integration-test-gateway integration-test-orchestrator integration-test-speculator e2e-test gazelle clean run-all start-servers stop-servers run-gateway run-orchestrator run-speculator run-client-gateway run-client-orchestrator run-client-speculator - # Bazel wrapper BAZEL = ./tool/bazel -# Generate protobuf files for all services using protoc -proto: - @echo "Generating protobuf files with protoc..." - @protoc --go_out=gateway/protopb --go_opt=paths=source_relative \ - --go-grpc_out=gateway/protopb --go-grpc_opt=paths=source_relative \ - --yarpc-go_out=gateway/protopb --yarpc-go_opt=paths=source_relative \ - --proto_path=gateway/proto gateway/proto/gateway.proto - @protoc --go_out=orchestrator/protopb --go_opt=paths=source_relative \ - --go-grpc_out=orchestrator/protopb --go-grpc_opt=paths=source_relative \ - --yarpc-go_out=orchestrator/protopb --yarpc-go_opt=paths=source_relative \ - --proto_path=orchestrator/proto orchestrator/proto/orchestrator.proto - @protoc --go_out=speculator/protopb --go_opt=paths=source_relative \ - --go-grpc_out=speculator/protopb --go-grpc_opt=paths=source_relative \ - --yarpc-go_out=speculator/protopb --yarpc-go_opt=paths=source_relative \ - --proto_path=speculator/proto speculator/proto/speculator.proto - @echo "Protobuf files generated successfully!" - -# Build everything in the project using Bazel -build: - @echo "Building all targets with Bazel..." - @$(BAZEL) build //... - @echo "Copying binaries to ./bin/..." - @mkdir -p bin - @cp -f bazel-bin/example/server/gateway/gateway_/gateway bin/gateway_server 2>/dev/null || \ - cp -f bazel-bin/example/server/gateway/gateway bin/gateway_server 2>/dev/null || true - @cp -f bazel-bin/example/server/orchestrator/orchestrator_/orchestrator bin/orchestrator_server 2>/dev/null || \ - cp -f bazel-bin/example/server/orchestrator/orchestrator bin/orchestrator_server 2>/dev/null || true - @cp -f bazel-bin/example/server/speculator/speculator_/speculator bin/speculator_server 2>/dev/null || \ - cp -f bazel-bin/example/server/speculator/speculator bin/speculator_server 2>/dev/null || true - @cp -f bazel-bin/example/client/gateway/gateway_/gateway bin/gateway_client 2>/dev/null || \ - cp -f bazel-bin/example/client/gateway/gateway bin/gateway_client 2>/dev/null || true - @cp -f bazel-bin/example/client/orchestrator/orchestrator_/orchestrator bin/orchestrator_client 2>/dev/null || \ - cp -f bazel-bin/example/client/orchestrator/orchestrator bin/orchestrator_client 2>/dev/null || true - @cp -f bazel-bin/example/client/speculator/speculator_/speculator bin/speculator_client 2>/dev/null || \ - cp -f bazel-bin/example/client/speculator/speculator bin/speculator_client 2>/dev/null || true - @echo "Build complete! Binaries are in ./bin/" - -# Run unit tests using Bazel (excludes integration tests which require running servers) -test: - @echo "Running unit tests..." - @$(BAZEL) test //... --test_tag_filters=-manual,-integration || echo "No unit tests found (only integration tests exist)" - -# Generate/update BUILD.bazel files using Gazelle -gazelle: - @echo "Running Gazelle to update BUILD files..." - @$(BAZEL) run //:gazelle - -# Run integration tests for a specific service (requires that service to be running) -integration-test-gateway: - @echo "Running Gateway integration tests..." - @$(BAZEL) test //gateway/integration_test:integration_test_test --test_output=all +# Docker Compose wrapper +COMPOSE = docker-compose +COMPOSE_FILE = example/server/docker-compose.yml +GATEWAY_COMPOSE_FILE = example/server/gateway/docker-compose.yml +ORCHESTRATOR_COMPOSE_FILE = example/server/orchestrator/docker-compose.yml -integration-test-orchestrator: - @echo "Running Orchestrator integration tests..." - @$(BAZEL) test //orchestrator/integration_test:integration_test_test --test_output=all +# Fixed project name for local manual testing (tests use unique random names) +LOCAL_PROJECT = submitqueue -integration-test-speculator: - @echo "Running Speculator integration tests..." - @$(BAZEL) test //speculator/integration_test:integration_test_test --test_output=all +# Set REPO_ROOT for docker-compose +export REPO_ROOT := $(shell pwd) -# Run all service integration tests (requires all services to be running) -integration-test: - @echo "Running all service integration tests..." - @$(BAZEL) test //gateway/integration_test:integration_test_test //orchestrator/integration_test:integration_test_test //speculator/integration_test:integration_test_test --test_output=all +.PHONY: build build-all-linux build-gateway-linux build-orchestrator-linux clean clean-proto deps e2e-test gazelle integration-test integration-test-extensions integration-test-gateway integration-test-orchestrator local-clean local-gateway-start local-gateway-stop local-init-schemas local-logs local-orchestrator-start local-orchestrator-stop local-ps local-restart local-start local-stop proto query-deps query-targets run-client-gateway run-client-orchestrator run-client-speculator test test-no-cache help -# Run end-to-end integration tests (hermetic, no manual server setup needed) -e2e-test: - @echo "Running integration tests..." - @$(BAZEL) test //e2e_test:e2e_test --test_output=all -# Clean generated files and binaries -clean: +build: ## Build all services and examples + @echo "Building all targets with Bazel..." + @$(BAZEL) build //... + @echo "Build complete!" + +# Build Linux binaries required for Docker containers +build-all-linux: build-gateway-linux build-orchestrator-linux ## Build all Linux binaries for Docker + @echo "All Linux binaries ready for Docker" + +build-gateway-linux: ## Build Gateway Linux binary for Docker + @echo "Building Gateway Linux binary for Docker..." + @$(BAZEL) build --platforms=@rules_go//go/toolchain:linux_amd64 //example/server/gateway:gateway + @mkdir -p .docker-bin + @cp -f bazel-bin/example/server/gateway/gateway_/gateway .docker-bin/gateway 2>/dev/null || \ + cp -f bazel-bin/example/server/gateway/gateway .docker-bin/gateway + @echo "Gateway Linux binary ready at .docker-bin/gateway" + +build-orchestrator-linux: ## Build Orchestrator Linux binary for Docker + @echo "Building Orchestrator Linux binary for Docker..." + @$(BAZEL) build --platforms=@rules_go//go/toolchain:linux_amd64 //example/server/orchestrator:orchestrator + @mkdir -p .docker-bin + @cp -f bazel-bin/example/server/orchestrator/orchestrator_/orchestrator .docker-bin/orchestrator 2>/dev/null || \ + cp -f bazel-bin/example/server/orchestrator/orchestrator .docker-bin/orchestrator + @echo "Orchestrator Linux binary ready at .docker-bin/orchestrator" + +clean: ## Clean generated files and binaries @echo "Cleaning with Bazel..." @$(BAZEL) clean @rm -rf bin/ @echo "Clean complete!" -# Clean generated proto files (normally not needed as they are checked in) -clean-proto: +clean-proto: ## Clean generated proto files @echo "Cleaning generated proto files..." @rm -rf gateway/protopb/*.pb.go @rm -rf orchestrator/protopb/*.pb.go @rm -rf speculator/protopb/*.pb.go @echo "Proto clean complete!" -# Start all servers in background (for testing) -start-servers: - @echo "Starting all servers in background..." - @./bin/gateway_server > /tmp/gateway.log 2>&1 & echo $$! > /tmp/gateway.pid - @./bin/orchestrator_server > /tmp/orchestrator.log 2>&1 & echo $$! > /tmp/orchestrator.pid - @./bin/speculator_server > /tmp/speculator.log 2>&1 & echo $$! > /tmp/speculator.pid - @sleep 2 - @echo "All servers started:" - @echo " Gateway (PID: $$(cat /tmp/gateway.pid)) - http://localhost:8081" - @echo " Orchestrator (PID: $$(cat /tmp/orchestrator.pid)) - http://localhost:8082" - @echo " Speculator (PID: $$(cat /tmp/speculator.pid)) - http://localhost:8083" +deps: ## Install Go dependencies + @echo "Installing Go dependencies..." + @go mod download + @go mod tidy + @echo "Dependencies installed!" + +e2e-test: build-all-linux ## Run end-to-end tests (hermetic, auto-builds binaries) + @echo "Running end-to-end tests..." + @$(BAZEL) test //test/e2e:e2e_test --test_output=all + +gazelle: ## Update BUILD.bazel files + @echo "Running Gazelle to update BUILD files..." + @$(BAZEL) run //:gazelle + +integration-test: build-all-linux ## Run all integration tests (auto-builds binaries) + @echo "Running all integration tests..." + @$(BAZEL) test //test/integration/... --test_output=errors + +integration-test-extensions: ## Run extension integration tests + @echo "Running extension integration tests..." + @$(BAZEL) test //test/integration/extension/... --test_output=errors + +integration-test-gateway: build-gateway-linux ## Run Gateway integration tests (auto-builds binary) + @echo "Running Gateway integration tests..." + @$(BAZEL) test //test/integration/gateway:gateway_test --test_output=errors + +integration-test-orchestrator: build-orchestrator-linux ## Run Orchestrator integration tests (auto-builds binary) + @echo "Running Orchestrator integration tests..." + @$(BAZEL) test //test/integration/orchestrator:orchestrator_test --test_output=errors + +local-clean: ## Stop and remove all local services, volumes, and images + @echo "Cleaning all services and data..." + @$(COMPOSE) -f $(COMPOSE_FILE) -p $(LOCAL_PROJECT) down -v --rmi local + @echo "All services, volumes, and images removed." + +local-gateway-start: build-gateway-linux ## Start Gateway service locally (Gateway + 2 MySQL databases) + @echo "Starting Gateway with docker-compose..." + @$(COMPOSE) -f $(GATEWAY_COMPOSE_FILE) -p $(LOCAL_PROJECT) up -d --build --wait + @echo "Applying database schemas..." + @$(MAKE) -s local-init-schemas + @echo "" + @echo "✅ Gateway is running!" + @echo "" + @$(COMPOSE) -f $(GATEWAY_COMPOSE_FILE) -p $(LOCAL_PROJECT) ps + @echo "" + @echo "Gateway gRPC port: $$(docker port $(LOCAL_PROJECT)-gateway-service-1 8080 2>/dev/null | cut -d: -f2 || echo 'unknown')" + @echo "MySQL App port: $$(docker port $(LOCAL_PROJECT)-mysql-app-1 3306 2>/dev/null | cut -d: -f2 || echo 'unknown')" + @echo "MySQL Queue port: $$(docker port $(LOCAL_PROJECT)-mysql-queue-1 3306 2>/dev/null | cut -d: -f2 || echo 'unknown')" + +local-gateway-stop: ## Stop Gateway service + @echo "Stopping Gateway services..." + @$(COMPOSE) -f $(GATEWAY_COMPOSE_FILE) -p $(LOCAL_PROJECT) down + @echo "Gateway services stopped." + +local-init-schemas: ## Manually apply all database schemas + @echo "Applying storage schema to mysql-app..." + @for file in extension/storage/mysql/schema/*.sql; do \ + echo " - Applying $$(basename $$file)..."; \ + docker exec -i $(LOCAL_PROJECT)-mysql-app-1 mysql -uroot -proot submitqueue < $$file 2>&1 | grep -v "Using a password" || true; \ + done + @echo "Applying counter schema to mysql-app..." + @for file in extension/counter/mysql/schema/*.sql; do \ + echo " - Applying $$(basename $$file)..."; \ + docker exec -i $(LOCAL_PROJECT)-mysql-app-1 mysql -uroot -proot submitqueue < $$file 2>&1 | grep -v "Using a password" || true; \ + done + @echo "Applying queue schema to mysql-queue..." + @for file in extension/queue/sql/schema/*.sql; do \ + echo " - Applying $$(basename $$file)..."; \ + docker exec -i $(LOCAL_PROJECT)-mysql-queue-1 mysql -uroot -proot submitqueue < $$file 2>&1 | grep -v "Using a password" || true; \ + done + @echo "✅ All schemas applied successfully" + +local-logs: ## View logs from all running services + @$(COMPOSE) -f $(COMPOSE_FILE) -p $(LOCAL_PROJECT) logs -f + +local-orchestrator-start: build-orchestrator-linux ## Start Orchestrator service locally (Orchestrator + 2 MySQL databases) + @echo "Starting Orchestrator with docker-compose..." + @$(COMPOSE) -f $(ORCHESTRATOR_COMPOSE_FILE) -p $(LOCAL_PROJECT) up -d --build --wait + @echo "Applying database schemas..." + @$(MAKE) -s local-init-schemas + @echo "" + @echo "✅ Orchestrator is running!" + @echo "" + @$(COMPOSE) -f $(ORCHESTRATOR_COMPOSE_FILE) -p $(LOCAL_PROJECT) ps + @echo "" + @echo "Orchestrator gRPC port: $$(docker port $(LOCAL_PROJECT)-orchestrator-service-1 8080 2>/dev/null | cut -d: -f2 || echo 'unknown')" + @echo "MySQL App port: $$(docker port $(LOCAL_PROJECT)-mysql-app-1 3306 2>/dev/null | cut -d: -f2 || echo 'unknown')" + @echo "MySQL Queue port: $$(docker port $(LOCAL_PROJECT)-mysql-queue-1 3306 2>/dev/null | cut -d: -f2 || echo 'unknown')" + +local-orchestrator-stop: ## Stop Orchestrator service + @echo "Stopping Orchestrator services..." + @$(COMPOSE) -f $(ORCHESTRATOR_COMPOSE_FILE) -p $(LOCAL_PROJECT) down + @echo "Orchestrator services stopped." + +local-ps: ## Show running containers and their ports + @echo "Running containers and ports:" + @echo "" + @$(COMPOSE) -f $(COMPOSE_FILE) -p $(LOCAL_PROJECT) ps + @echo "" + @echo "📡 Service Endpoints:" + @echo " Gateway gRPC: localhost:$$(docker port $(LOCAL_PROJECT)-gateway-service-1 8080 2>/dev/null | cut -d: -f2 || echo 'not running')" + @echo " Orchestrator gRPC: localhost:$$(docker port $(LOCAL_PROJECT)-orchestrator-service-1 8080 2>/dev/null | cut -d: -f2 || echo 'not running')" + @echo "" + @echo "🗄️ Database Endpoints:" + @echo " MySQL App: localhost:$$(docker port $(LOCAL_PROJECT)-mysql-app-1 3306 2>/dev/null | cut -d: -f2 || echo 'not running')" + @echo " MySQL Queue: localhost:$$(docker port $(LOCAL_PROJECT)-mysql-queue-1 3306 2>/dev/null | cut -d: -f2 || echo 'not running')" + @echo "" + @echo "💡 Usage:" + @echo " # Connect to MySQL App DB" + @echo " mysql -h127.0.0.1 -P$$(docker port $(LOCAL_PROJECT)-mysql-app-1 3306 2>/dev/null | cut -d: -f2 || echo 'PORT') -uroot -proot submitqueue" @echo "" - @echo "Logs:" - @echo " tail -f /tmp/gateway.log" - @echo " tail -f /tmp/orchestrator.log" - @echo " tail -f /tmp/speculator.log" + @echo " # Call Gateway gRPC" + @echo " grpcurl -plaintext -d '{\"message\":\"test\"}' localhost:$$(docker port $(LOCAL_PROJECT)-gateway-service-1 8080 2>/dev/null | cut -d: -f2 || echo 'PORT') submitqueue.SubmitQueueGateway/Ping" @echo "" - @echo "To stop: make stop-servers" - -# Stop all background servers -stop-servers: - @echo "Stopping all servers..." - @if [ -f /tmp/gateway.pid ]; then kill $$(cat /tmp/gateway.pid) 2>/dev/null || true; rm -f /tmp/gateway.pid; fi - @if [ -f /tmp/orchestrator.pid ]; then kill $$(cat /tmp/orchestrator.pid) 2>/dev/null || true; rm -f /tmp/orchestrator.pid; fi - @if [ -f /tmp/speculator.pid ]; then kill $$(cat /tmp/speculator.pid) 2>/dev/null || true; rm -f /tmp/speculator.pid; fi - @echo "All servers stopped" - -# Run all servers (for testing) - starts servers, waits for Ctrl+C, then stops -run-all: start-servers + @echo " # View logs" + @echo " make local-logs" + +local-restart: build-all-linux ## Restart all services (rebuild and restart) + @echo "Restarting all services..." + @$(COMPOSE) -f $(COMPOSE_FILE) -p $(LOCAL_PROJECT) restart + @echo "Services restarted!" + @make local-ps + +local-start: build-all-linux ## Start full stack (Gateway + Orchestrator + MySQL) + @echo "Starting full stack with docker-compose..." + @$(COMPOSE) -f $(COMPOSE_FILE) -p $(LOCAL_PROJECT) up -d --build --wait + @echo "Applying database schemas..." + @$(MAKE) -s local-init-schemas @echo "" - @echo "Press Ctrl+C to stop all servers..." - @trap 'make stop-servers' INT; while true; do sleep 1; done + @echo "✅ Full stack is running!" + @echo "" + @make local-ps + +local-stop: ## Stop all services (keep data) + @echo "Stopping all services..." + @$(COMPOSE) -f $(COMPOSE_FILE) -p $(LOCAL_PROJECT) down + @echo "Services stopped. Data volumes preserved." -# Run gateway server using Bazel -run-gateway: - @echo "Starting gateway server on port 8081..." - @$(BAZEL) run //example/server/gateway:gateway +proto: ## Generate protobuf files from .proto definitions + @echo "Generating protobuf files with protoc..." + @protoc --go_out=gateway/protopb --go_opt=paths=source_relative \ + --go-grpc_out=gateway/protopb --go-grpc_opt=paths=source_relative \ + --yarpc-go_out=gateway/protopb --yarpc-go_opt=paths=source_relative \ + --proto_path=gateway/proto gateway/proto/gateway.proto + @protoc --go_out=orchestrator/protopb --go_opt=paths=source_relative \ + --go-grpc_out=orchestrator/protopb --go-grpc_opt=paths=source_relative \ + --yarpc-go_out=orchestrator/protopb --yarpc-go_opt=paths=source_relative \ + --proto_path=orchestrator/proto orchestrator/proto/orchestrator.proto + @protoc --go_out=speculator/protopb --go_opt=paths=source_relative \ + --go-grpc_out=speculator/protopb --go-grpc_opt=paths=source_relative \ + --yarpc-go_out=speculator/protopb --yarpc-go_opt=paths=source_relative \ + --proto_path=speculator/proto speculator/proto/speculator.proto + @echo "Protobuf files generated successfully!" -# Run orchestrator server using Bazel -run-orchestrator: - @echo "Starting orchestrator server on port 8082..." - @$(BAZEL) run //example/server/orchestrator:orchestrator +# Bazel query helpers +query-deps: + @$(BAZEL) query 'deps(//example/server/gateway:gateway)' -# Run speculator server using Bazel -run-speculator: - @echo "Starting speculator server on port 8083..." - @$(BAZEL) run //example/server/speculator:speculator +query-targets: + @$(BAZEL) query //... -# Run gateway client using Bazel +# Run gateway client (connects to any running gateway service) run-client-gateway: @$(BAZEL) run //example/client/gateway:gateway -- -addr $(or $(SERVER_ADDR),localhost:8081) -message "$(or $(MESSAGE),ping)" -# Run orchestrator client using Bazel +# Run orchestrator client (connects to any running orchestrator service) run-client-orchestrator: @$(BAZEL) run //example/client/orchestrator:orchestrator -- -addr $(or $(SERVER_ADDR),localhost:8082) -message "$(or $(MESSAGE),ping)" -# Run speculator client using Bazel +# Run speculator client (connects to any running speculator service) run-client-speculator: @$(BAZEL) run //example/client/speculator:speculator -- -addr $(or $(SERVER_ADDR),localhost:8083) -message "$(or $(MESSAGE),ping)" -# Install dependencies (for go mod users) -deps: - @echo "Installing Go dependencies..." - @go mod download - @go mod tidy - @echo "Dependencies installed!" - -# Bazel query helpers -query-targets: - @$(BAZEL) query //... +test: ## Run unit tests + @echo "Running unit tests..." + @$(BAZEL) test //... --test_tag_filters=-manual,-integration || echo "No unit tests found (only integration tests exist)" -query-deps: - @$(BAZEL) query 'deps(//example/server/gateway:gateway)' +test-no-cache: ## Run unit tests without cache (force re-run) + @echo "Running unit tests (no cache)..." + @$(BAZEL) test //... --test_tag_filters=-manual,-integration --nocache_test_results -# Help -help: +help: ## Show this help message @echo "Available targets:" @echo "" - @echo "Build & Test:" - @echo " make proto - Generate protobuf files" - @echo " make build - Build all services and examples" - @echo " make test - Run unit tests" - @echo " make gazelle - Update BUILD.bazel files" - @echo " make clean - Clean generated files and binaries" - @echo "" - @echo "Run Servers:" - @echo " make run-all - Run all servers (Ctrl+C to stop)" - @echo " make start-servers - Start all servers in background" - @echo " make stop-servers - Stop all background servers" - @echo " make run-gateway - Run gateway server (port 8081)" - @echo " make run-orchestrator - Run orchestrator server (port 8082)" - @echo " make run-speculator - Run speculator server (port 8083)" - @echo "" - @echo "Integration Tests (requires servers to be running):" - @echo " make integration-test-gateway - Test Gateway service" - @echo " make integration-test-orchestrator - Test Orchestrator service" - @echo " make integration-test-speculator - Test Speculator service" - @echo " make integration-test - Test all services" - @echo "" - @echo "End-to-End Tests (hermetic, no setup needed):" - @echo " make e2e-test - Run integration tests with Testcontainers" - @echo "" - @echo "Run Clients:" - @echo " make run-client-gateway - Run gateway client" - @echo " make run-client-orchestrator - Run orchestrator client" - @echo " make run-client-speculator - Run speculator client" - @echo "" - @echo "Other:" - @echo " make deps - Install Go dependencies" - @echo " make query-targets - List all Bazel targets" - @echo "" - @echo "Examples:" - @echo " # Start all servers and run integration tests" - @echo " make build && make start-servers && make integration-test && make stop-servers" - @echo "" - @echo " # Run a single server" - @echo " make run-gateway" - @echo "" - @echo " # Test with custom message" - @echo " make run-client-gateway MESSAGE='hello gateway'" + @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf " \033[36m%-30s\033[0m %s\n", $$1, $$2}' + diff --git a/README.md b/README.md index 85cd4380..c72e5c80 100644 --- a/README.md +++ b/README.md @@ -94,6 +94,70 @@ direnv allow With direnv enabled, you can use `bazel` directly instead of `./tool/bazel`. +### Shell Configuration (Optional) + +#### Make Target Auto-Completion (zsh) + +Enable tab-completion for Makefile targets with descriptions: + +1. **Add to `~/.zshrc`:** + ```bash + # Initialize completion system + autoload -Uz compinit + compinit + + # Makefile target completion with caching and help text + function _make_targets() { + local -a targets + local makefile_cache=".make_targets_cache" + + if [[ -f Makefile ]]; then + # Regenerate cache if Makefile is newer + if [[ ! -f $makefile_cache ]] || [[ Makefile -nt $makefile_cache ]]; then + awk -F':.*?## ' '/^[a-zA-Z0-9_-]+:.*?## / {printf "%s:%s\n", $1, $2}' Makefile > $makefile_cache + fi + + # Read cache into targets array + targets=(${(f)"$(<$makefile_cache)"}) + + # If cache has descriptions, use them; otherwise fallback to simple list + if [[ -s $makefile_cache ]] && grep -q ':' $makefile_cache 2>/dev/null; then + _describe 'make targets' targets + else + # Fallback: just target names + awk -F: '/^[a-zA-Z0-9_-]+:/ {print $1}' Makefile > $makefile_cache + targets=(${(f)"$(<$makefile_cache)"}) + _describe 'make targets' targets + fi + fi + } + + compdef _make_targets make + ``` + +2. **Reload your shell:** + ```bash + source ~/.zshrc + ``` + +3. **Try it out:** + ```bash + make # Shows all targets with descriptions + make local- # Shows all local-* targets + make integration # Shows integration test options + ``` + + You'll see: + ``` + build -- Build all services and examples + test -- Run unit tests + local-start -- Start full stack (Gateway + Orchestrator + MySQL) + integration-test -- Run all integration tests (auto-builds binaries) + # ... and more! + ``` + +**Note**: The completion cache (`.make_targets_cache`) is gitignored and automatically regenerates when the Makefile changes. + Install optional tools: ```bash # macOS diff --git a/doc/PROJECT_STRUCTURE.md b/doc/PROJECT_STRUCTURE.md new file mode 100644 index 00000000..dfb33a06 --- /dev/null +++ b/doc/PROJECT_STRUCTURE.md @@ -0,0 +1,178 @@ +# Project Structure + +This document describes the organization of the SubmitQueue repository. + +## Directory Layout + +``` +submitqueue/ +├── .bazelversion # Pins Bazel version +├── .envrc # direnv configuration +├── .docker-bin/ # Linux binaries for Docker (gitignored) +├── MODULE.bazel # Bzlmod dependency management +├── go.mod # Go module dependencies +├── Makefile # Build automation +├── BUILD.bazel # Root build file +│ +├── tool/ # Bazel tooling (Bazelisk wrapper) +│ +├── gateway/ # Gateway service - entry point for external requests +│ ├── controller/ # Business logic (land, ping) +│ ├── proto/ # Proto definitions +│ └── protopb/ # Generated proto code (*.pb.go, *_grpc.pb.go, *.pb.yarpc.go) +│ +├── orchestrator/ # Orchestrator service - processes requests via queues +│ ├── controller/ # Business logic (request, ping) +│ ├── proto/ # Proto definitions +│ └── protopb/ # Generated proto code +│ +├── entity/ # Domain entities (Request, Change, enums) +│ └── queue/ # Queue-specific entities (Message) +│ +├── extension/ # Pluggable backend implementations +│ ├── counter/ # Sequential number generation interface +│ │ └── mysql/ # MySQL implementation + schema +│ │ +│ ├── queue/ # Messaging queue abstraction +│ │ └── sql/ # SQL (MySQL) implementation + schema +│ │ +│ └── storage/ # Storage abstraction +│ └── mysql/ # MySQL implementation + schema +│ +├── consumer/ # Reusable queue consumer infrastructure +│ # (Handler interface, Consumer) +│ +├── example/ # Runnable examples +│ ├── server/ # Server implementations with Docker Compose +│ │ ├── gateway/ # Gateway server + Dockerfile +│ │ └── orchestrator/ # Orchestrator server + Dockerfile +│ └── client/ # Client examples (gateway, orchestrator) +│ +├── test/ # All tests +│ ├── e2e/ # End-to-end tests (full stack) +│ ├── integration/ # Integration tests +│ │ ├── gateway/ # Gateway service tests +│ │ ├── orchestrator/ # Orchestrator service tests +│ │ └── extension/ # Extension implementation tests +│ │ ├── counter/mysql/ +│ │ ├── queue/sql/ +│ │ └── storage/mysql/ +│ └── testutil/ # Test utilities (Docker Compose, MySQL, servers) +│ +└── doc/ # Documentation + ├── CLAUDE.md # Development guidelines + ├── PROJECT_STRUCTURE.md # This file + ├── howto/ # How-to guides + │ └── TESTING.md # Testing guide + └── rfc/ # Design documents and proposals +``` + +## Key Design Principles + +### 1. Clean Architecture with Interface-Driven Extensions + +**Extensions** are pluggable, vendor-agnostic interfaces: +- `extension/{extension}/` - Interface definitions +- `extension/{extension}/{impl}/` - Implementations (e.g., `mysql/`) +- Each extension has its own schema files + +**Examples:** +- `extension/storage/` - Storage interface, MySQL implementation +- `extension/queue/` - Queue interface, SQL implementation +- `extension/counter/` - Counter interface, MySQL implementation + +### 2. Service Structure + +Each service follows a consistent layout: +- `controller/` - Pure business logic (transport-agnostic) +- `proto/` - Proto definitions (`.proto` files) +- `protopb/` - Generated proto code (committed to repo) + +**Controllers** contain pure business logic, independent of gRPC/YARPC transport layer. + +### 3. Separate `proto/` and `protopb/` Directories + +Each service has: +- `proto/` - Contains the `.proto` file(s) +- `protopb/` - Contains all generated files (`.pb.go`, `_grpc.pb.go`, `.pb.yarpc.go`) + +This separation makes it clear what is source vs. generated. **All generated files are committed** to the repository. + +### 4. YARPC Support + +All proto files generate three types of files: +- `*.pb.go` - Standard protobuf code +- `*_grpc.pb.go` - gRPC service code +- `*.pb.yarpc.go` - YARPC service code for Uber's RPC framework + +This allows services to support both gRPC and YARPC clients. + +### 5. Entity-Driven Design + +Domain entities live in `entity/`: +- Pure, framework-agnostic value types +- Use `int64` for timestamps (Unix milliseconds) +- Reference other entities by ID, not directly +- String enums with clear names + +### 6. Consumer Infrastructure + +The `consumer/` package provides reusable queue consumer infrastructure: +- `Handler` interface - Business logic for processing messages +- `Manager` - Orchestrates multiple consumers across different topics +- Services register handlers and the manager handles subscriptions, polling, ack/nack + +### 7. Docker-Based Testing + +All integration and e2e tests use Docker Compose: +- Tests in `test/integration/` for services and extensions +- Tests in `test/e2e/` for full stack +- `test/testutil/` provides Docker Compose helpers +- Hermetic, parallel-safe, auto cleanup + +### 8. Python-Based Bazel Wrapper + +The `tool/bazel` script is a Python implementation of Bazelisk that: +- Reads `.bazelversion` to determine which Bazel version to use +- Downloads and caches the appropriate Bazel binary +- Delegates to the correct version automatically + +### 9. Two Separate Databases + +SubmitQueue demonstrates proper architectural separation: +- **Application DB** (port 3306) - Business data (requests, counters) +- **Queue DB** (port 3307) - Messaging infrastructure (messages, offsets, leases) + +This allows the queue to be swapped for other technologies (Kafka, SQS, etc.) in production. + +## Build System + +- **Bazel with Bzlmod** (NOT WORKSPACE) for dependency management +- **Version pinning**: `.bazelversion` pins the Bazel version +- **Go version**: Defined in `go.mod`, read by `MODULE.bazel` via `go_sdk.from_file()` +- **External dependencies**: Must be added to both `go.mod` AND `MODULE.bazel` +- **BUILD files**: Every Go package must have a `BUILD.bazel` file + +## Services + +### Gateway +Entry point for external requests. Receives land requests, stores in DB, publishes to queue. + +### Orchestrator +The engine that processes requests through a multi-stage pipeline via queues. + +## Testing + +See [howto/TESTING.md](howto/TESTING.md) for comprehensive testing guide. + +**Quick overview:** +- **Unit tests** - Co-located with code, fast, no Docker +- **Integration tests** - `test/integration/`, Docker-based, hermetic +- **E2E tests** - `test/e2e/`, full stack, Docker-based + +All automated tests use Docker Compose with unique container prefixes for parallel execution. + +## See Also + +- [CLAUDE.md](../CLAUDE.md) - Development guidelines and coding conventions +- [howto/TESTING.md](howto/TESTING.md) - Comprehensive testing documentation diff --git a/doc/architecture/STRUCTURE.md b/doc/architecture/STRUCTURE.md deleted file mode 100644 index f71ba358..00000000 --- a/doc/architecture/STRUCTURE.md +++ /dev/null @@ -1,125 +0,0 @@ -# Project Structure - -This document describes the structure of the submitqueue project, which follows the same Bazel and proto organization as the tango repository. - -## Directory Layout - -``` -submitqueue/ -├── .bazelversion # Pins Bazel version to 8.4.1 -├── .envrc # direnv configuration -├── MODULE.bazel # Bzlmod dependency management -├── go.mod # Go module with YARPC dependencies -├── Makefile # Build automation -├── BUILD.bazel # Root build file -│ -├── tool/ # Bazel tooling -│ ├── bazel # Python-based Bazelisk wrapper -│ ├── BUILD.bazel -│ └── README.md -│ -├── gateway/ # Gateway service -│ ├── BUILD.bazel -│ ├── core/ -│ │ └── controller/ -│ │ ├── BUILD.bazel -│ │ └── ping.go # Service implementation -│ ├── proto/ -│ │ ├── BUILD.bazel -│ │ └── gateway.proto # Proto definition -│ └── protopb/ # Generated proto files -│ ├── BUILD.bazel -│ ├── gateway.pb.go # Protobuf generated code -│ ├── gateway_grpc.pb.go # gRPC generated code -│ └── gateway.pb.yarpc.go # YARPC generated code -│ -├── orchestrator/ # Orchestrator service -│ ├── BUILD.bazel -│ ├── core/ -│ │ └── controller/ -│ │ ├── BUILD.bazel -│ │ └── ping.go -│ ├── proto/ -│ │ ├── BUILD.bazel -│ │ └── orchestrator.proto -│ └── protopb/ -│ ├── BUILD.bazel -│ ├── orchestrator.pb.go -│ ├── orchestrator_grpc.pb.go -│ └── orchestrator.pb.yarpc.go -│ -├── speculator/ # Speculator service -│ ├── BUILD.bazel -│ ├── core/ -│ │ └── controller/ -│ │ ├── BUILD.bazel -│ │ └── ping.go -│ ├── proto/ -│ │ ├── BUILD.bazel -│ │ └── speculator.proto -│ └── protopb/ -│ ├── BUILD.bazel -│ ├── speculator.pb.go -│ ├── speculator_grpc.pb.go -│ └── speculator.pb.yarpc.go -│ -└── example/ # Examples (like tango/example) - ├── README.md - ├── server/ # Server examples - │ ├── gateway/ - │ ├── orchestrator/ - │ └── speculator/ - └── client/ # Client examples - ├── gateway/ - ├── orchestrator/ - └── speculator/ -``` - -## Key Design Principles - -This structure follows the tango repository's conventions: - -### 1. **Separate `proto/` and `protopb/` Directories** - -Each service has: -- `proto/` - Contains the `.proto` file(s) -- `protopb/` - Contains all generated files (`.pb.go`, `_grpc.pb.go`, `.pb.yarpc.go`) -- `core/controller/` - Contains service implementation - -This separation makes it clear what is source vs. generated, and all generated files are committed to the repository. - -### 2. **YARPC Support** - -All proto files generate three types of files: -- `*.pb.go` - Standard protobuf code -- `*_grpc.pb.go` - gRPC service code -- `*.pb.yarpc.go` - YARPC service code for Uber's RPC framework - -This allows services to support both gRPC and YARPC clients. - -### 3. **Python-Based Bazel Wrapper** - -The `tool/bazel` script is a Python implementation of Bazelisk that: -- Reads `.bazelversion` to determine which Bazel version to use -- Downloads and caches the appropriate Bazel binary -- Delegates to the correct version automatically - -### 4. **Committed Generated Files** - -All `*pb/` generated files are committed to the repository because: -- This is a library that will be consumed by other services -- Consumers can import and use the proto packages without needing protoc -- Ensures consistent generated code across builds - -## Comparison with Tango - -| Aspect | Tango | Submit Queue | -|--------|-------|--------------| -| Proto location | `proto/` (root) | `/proto/` | -| Generated files | `tangopb/` | `/protopb/` | -| Bazel tool | Python script | Python script (copied) | -| Dependency mgmt | Bzlmod | Bzlmod | -| YARPC | Yes | Yes | -| Generated committed | Yes | Yes | -| Examples dir | `example/` | `example/server/` and `example/client/` | -| Bazel config | No `.bazelrc` | No `.bazelrc` | diff --git a/doc/design/README.md b/doc/design/README.md deleted file mode 100644 index c7e20063..00000000 --- a/doc/design/README.md +++ /dev/null @@ -1 +0,0 @@ -This folder contains designs diff --git a/doc/howto/TESTING.md b/doc/howto/TESTING.md new file mode 100644 index 00000000..5dcf7d11 --- /dev/null +++ b/doc/howto/TESTING.md @@ -0,0 +1,534 @@ +# Testing + +All testing (automated and manual) uses **containerized environments** for consistency and reproducibility. + +## Prerequisites + +**Docker must be running** for all integration, e2e, and manual testing. + +```bash +# Check Docker is running +docker ps + +# If not running: +# - macOS: Start Docker Desktop +# - Linux: sudo systemctl start docker +``` + +--- + +## Database Architecture + +SubmitQueue uses **two separate databases** to demonstrate proper architectural separation: + +### 1. Application Database (port 3306) +- **Purpose**: Business data (requests, counters, batches) +- **Schema**: `extension/storage/mysql/schema`, `extension/counter/mysql/schema` +- **Used by**: Gateway (stores requests), Orchestrator (reads/updates request state) +- **Connection**: `MYSQL_DSN` + +### 2. Queue Database (port 3307) +- **Purpose**: Messaging infrastructure (queue messages, offsets, partition leases) +- **Schema**: `extension/queue/sql/schema` +- **Used by**: Gateway (publishes), Orchestrator (consumes) +- **Connection**: `QUEUE_MYSQL_DSN` + +**Why separate?** +- Queue is **pluggable infrastructure** - you can swap MySQL queue for Kafka, SQS, etc. +- Application data and messaging concerns scale independently +- Clear architectural boundary between business logic and infrastructure +- In production, queue infrastructure often runs separately (e.g., managed Kafka cluster) + +**Note:** Both use MySQL in examples for simplicity, but in production the queue could use a different technology entirely. + +--- + +## Automated Testing + +### Quick Reference + +```bash +# Unit tests (no Docker required) +make test + +# Integration tests (Docker required) +make integration-test-gateway # Gateway in isolation +make integration-test-orchestrator # Orchestrator in isolation +make integration-test-queue # Queue infrastructure +make integration-test # All integration tests + +# E2E tests (Docker required) +make e2e-test + +# Everything (Docker required) +make test-all +``` + +### Testing Levels + +**1. Unit Tests** - Fast, no containers +- Location: Co-located with code (`{package}/*_test.go`) +- Run: `make test` +- Speed: Fast (< 1s typically) + +**2. Integration Tests** - Service in isolation with real dependencies +- Location: `test/integration/{service}/` +- Run: `make integration-test-{service}` +- Containers: MySQL + one service +- Tests one service isolated from others + +**3. E2E Tests** - Complete workflows across all services +- Location: `test/e2e/` +- Run: `make e2e-test` +- Containers: MySQL + all services +- Tests cross-service communication + +### How Automated Tests Work + +Tests use **docker-compose** to spin up containers automatically: + +1. `SetupSuite()` - Creates MySQL + service containers **once** per test suite +2. All tests run against those containers +3. `TearDownSuite()` - Cleans up containers automatically + +**Benefits:** +- ✅ Hermetic (isolated environment per suite) +- ✅ Fast (containers reused across tests) +- ✅ No manual setup required +- ✅ Reproducible (same behavior everywhere) +- ✅ Parallel execution (tests don't conflict) + +--- + +## Container Naming + +Test containers use **meaningful, context-rich names** for easy debugging and correlation. + +### Naming Format + +``` +{project-name}-{service-name}-{instance} + └─┬─┘ └────┬────┘ └──┬──┘ + From test From compose Docker adds +``` + +Project name format: +``` +sq-test-{context}-{shortid} +│ │ │ +│ │ └─ 6-char hex timestamp (unique per test run) +│ └─────────── Test context (storage, gateway, e2e, etc.) +└─────────────────── Namespace prefix +``` + +### Real Examples + +**Extension Test - Storage:** +```bash +Container: sq-test-storage-2ce1d0-mysql-storage-1 + │ │ │ │ │ + │ │ │ │ └─ Instance number + │ │ │ └─ Service from docker-compose.yml + │ │ └─ Short unique ID + │ └─ Test context + └─ Namespace prefix + +# From: +NewComposeStack(t, log, ctx, "docker-compose.yml", "storage") + └─────┘ +# docker-compose.yml: +services: + mysql-storage: # ← Service name +``` + +**Service Test - Gateway (Multiple Containers):** +```bash +# All share same project prefix, different services: +sq-test-gateway-abc123-mysql-app-1 +sq-test-gateway-abc123-mysql-queue-1 +sq-test-gateway-abc123-gateway-1 +└──────────┬───────────┘ + Same project (same test run) + +# From: +NewComposeStack(t, log, ctx, composeFile, "gateway") + +# docker-compose.yml: +services: + mysql-app: # ← Service name + mysql-queue: # ← Service name + gateway: # ← Service name +``` + +**E2E Test - Full Stack (4 Containers):** +```bash +sq-test-e2e-def456-mysql-app-1 +sq-test-e2e-def456-mysql-queue-1 +sq-test-e2e-def456-gateway-1 +sq-test-e2e-def456-orchestrator-1 +└────┬─────┘ + All same project +``` + +### Container Name Reference + +| Test Type | Context | Service Names | Example Container | +|-----------|---------|---------------|-------------------| +| Storage extension | `storage` | `mysql-storage` | `sq-test-storage-2ce1d0-mysql-storage-1` | +| Counter extension | `counter` | `mysql-counter` | `sq-test-counter-ecff68-mysql-counter-1` | +| Queue extension | `queue` | `mysql-queue` | `sq-test-queue-a1b2c3-mysql-queue-1` | +| Gateway service | `gateway` | `mysql-app`, `mysql-queue`, `gateway` | `sq-test-gateway-abc123-gateway-1` | +| Orchestrator service | `orchestrator` | `mysql-app`, `mysql-queue`, `orchestrator` | `sq-test-orchestrator-xyz789-orchestrator-1` | +| E2E full stack | `e2e` | `mysql-app`, `mysql-queue`, `gateway`, `orchestrator` | `sq-test-e2e-def456-gateway-1` | + +### Why This Naming? + +**Before** (opaque timestamps): +```bash +$ docker ps +test-1771786794254426000-mysql-storage-1 # What test is this? +test-1771786795123456789-gateway-1 # What's being tested? +``` + +**After** (context-rich): +```bash +$ docker ps +sq-test-storage-2ce1d0-mysql-storage-1 # Storage extension test +sq-test-gateway-abc123-mysql-app-1 # Gateway test - app database +sq-test-gateway-abc123-gateway-1 # Gateway test - gateway service +``` + +**Benefits:** +- ✅ **Easy correlation** - Instantly know which test created each container +- ✅ **Meaningful context** - Shows what's being tested (`storage`, `gateway`, `e2e`) +- ✅ **Parallel-safe** - Unique ID prevents conflicts when tests run simultaneously +- ✅ **Grouped containers** - Same project prefix = related containers from one test +- ✅ **Debugging** - Quickly identify containers in `docker ps` or logs + +### Debugging with Container Names + +```bash +# See what tests are currently running +docker ps --format "table {{.Names}}\t{{.Status}}" | grep sq-test + +# Find all containers from gateway test +docker ps | grep sq-test-gateway + +# View logs from specific test container +docker logs sq-test-gateway-abc123-mysql-app-1 + +# Inspect a specific test's MySQL +docker exec -it sq-test-storage-2ce1d0-mysql-storage-1 \ + mysql -uroot -proot submitqueue -e "SHOW TABLES;" +``` + +--- + +## Manual Testing + +### Quick Start + +```bash +# Start all services (Gateway + Orchestrator + 2 MySQL DBs) +make start-all-services + +# Test with client +make client-gateway + +# View logs +make logs + +# Stop all services +make stop-all-services +``` + +### Testing Individual Services + +**Gateway Only:** +```bash +# Start Gateway in isolation (Gateway + 2 MySQL DBs) +make start-gateway + +# Test Ping API +grpcurl -plaintext -d '{"message": "hello"}' localhost:8081 submitqueue.SubmitQueueGateway/Ping + +# Test Land API +grpcurl -plaintext -d '{ + "queue": "test-queue", + "change": {"source": "github", "ids": ["PR-123"]}, + "strategy": "REBASE" +}' localhost:8081 submitqueue.SubmitQueueGateway/Land + +# Stop +docker-compose -f example/server/gateway/docker-compose.yml down +``` + +**Orchestrator Only:** +```bash +# Start Orchestrator in isolation (Orchestrator + 2 MySQL DBs) +make start-orchestrator + +# Test Ping API +grpcurl -plaintext -d '{"message": "hello"}' localhost:8082 submitqueue.SubmitQueueOrchestrator/Ping + +# View consumer logs +docker-compose -f example/server/orchestrator/docker-compose.yml logs -f orchestrator + +# Stop +docker-compose -f example/server/orchestrator/docker-compose.yml down +``` + +### After Code Changes + +```bash +# Rebuild and restart all services +make rebuild-services + +# Or rebuild specific service +docker-compose -f example/server/docker-compose.yml build gateway +docker-compose -f example/server/docker-compose.yml up -d gateway +``` + +### Inspecting the Databases + +**Application Database** (requests, counters): +```bash +# Connect to application DB (port 3306) +docker exec -it submitqueue-mysql-app mysql -uroot -proot submitqueue + +# Show tables +SHOW TABLES; + +# View requests +SELECT * FROM requests; + +# Exit +exit +``` + +**Queue Database** (messages, offsets): +```bash +# Connect to queue DB (port 3307) +docker exec -it submitqueue-mysql-queue mysql -uroot -proot submitqueue + +# Show tables +SHOW TABLES; + +# View queue messages +SELECT * FROM queue_messages; + +# View partition offsets +SELECT * FROM queue_offsets; + +# View partition leases +SELECT * FROM queue_partition_leases; + +# Exit +exit +``` + +### Publishing Test Messages + +Manually publish a message to test Orchestrator consumer: + +```bash +# Insert message into queue database +docker exec -it submitqueue-mysql-queue mysql -uroot -proot submitqueue -e " +INSERT INTO queue_messages (id, topic, partition_key, payload, created_at, invisible_until) +VALUES ( + 'test-msg-123', + 'land_request', + 'test-queue', + '{\"id\":\"test-msg-123\",\"queue\":\"test-queue\",\"state\":\"pending\"}', + NOW(), + NOW() +); +" + +# Watch Orchestrator process it +docker-compose -f example/server/docker-compose.yml logs -f orchestrator +``` +``` + +### Using grpcurl + +```bash +# Install grpcurl if not already installed +brew install grpcurl # macOS +# OR: go install github.com/fullstorydev/grpcurl/cmd/grpcurl@latest + +# List services +grpcurl -plaintext localhost:8081 list + +# Describe a service +grpcurl -plaintext localhost:8081 describe submitqueue.SubmitQueueGateway + +# Call Ping +grpcurl -plaintext -d '{"message": "test"}' \ + localhost:8081 submitqueue.SubmitQueueGateway/Ping + +# Call Land +grpcurl -plaintext -d '{ + "queue": "my-queue", + "change": {"source": "github", "ids": ["PR-456"]}, + "strategy": "REBASE" +}' localhost:8081 submitqueue.SubmitQueueGateway/Land +``` + +### Available Commands + +| Command | Description | +|---------|-------------| +| `make start-all-services` | Start all services (full stack) | +| `make start-gateway` | Start Gateway in isolation | +| `make start-orchestrator` | Start Orchestrator in isolation | +| `make logs` | Follow logs from all services | +| `make stop-all-services` | Stop all services | +| `make rebuild-services` | Rebuild after code changes | +| `make clean-services` | Stop and remove volumes (fresh start) | + +--- + +## Troubleshooting + +### Docker Not Running +```bash +# Error: "Cannot connect to the Docker daemon" +# Solution: Start Docker Desktop or Docker daemon +docker ps # Should not error +``` + +### Services Not Starting +```bash +# Check logs +docker-compose logs + +# Check specific service +docker-compose logs gateway + +# Rebuild from scratch +make dev-clean +make dev-up +``` + +### Port Already in Use +```bash +# Error: "port is already allocated" +# Find conflicting container +docker ps | grep 8081 # (or 8082, 8083, 3306) + +# Stop it +docker stop + +# Or stop all +make dev-down +``` + +### Database Schema Not Applied +```bash +# Recreate database with fresh schema +make clean-services # Removes volumes +make start-all-services # Recreates everything +``` + +### Tests Timing Out +```bash +# Clean Docker cache +docker system prune -a + +# Clean Bazel cache +make clean + +# Re-run tests +make test-all +``` + +### Containers Not Cleaning Up +```bash +# List all test containers +docker ps -a | grep sq-test + +# Remove all test containers +docker ps -a | grep sq-test | awk '{print $1}' | xargs docker rm -f + +# Remove all test networks +docker network ls | grep sq-test | awk '{print $1}' | xargs docker network rm + +# Or use docker-compose to clean up (from any test directory) +cd test/integration/gateway +docker-compose down -v + +# Nuclear option (removes everything) +docker system prune -af --volumes +``` + +--- + +## Architecture + +### Consistency Across Testing + +**Both automated and manual testing use the same setup:** + +1. **Docker Compose** defines services (`docker-compose.yml`) +2. **Automated tests** use docker-compose programmatically via `testutil.ComposeStack` +3. **Manual testing** uses `make start-all-services` (same docker-compose files) + +**Result:** Tests and manual testing are identical - no surprises! + +### Test vs Manual + +| Aspect | Automated Tests | Manual Testing | +|--------|----------------|----------------| +| **Setup** | docker-compose (auto) | `make start-all-services` | +| **Containers** | Per test suite | Persistent | +| **Cleanup** | Automatic | `make stop-all-services` | +| **Container names** | `sq-test-{context}-{id}` | `submitqueue-{service}` | +| **Use case** | CI, pre-commit | Debugging, exploration | +| **Speed** | Fast (parallel) | Fast (persistent) | + +--- + +## Writing New Tests + +### Adding Unit Tests + +1. Create `{file}_test.go` next to production code +2. Use table-driven tests +3. Run: `make test` + +### Adding Integration Tests + +1. Add test to `test/integration/{service}/suite_test.go` +2. Use suite's resources (`s.client`, `s.db`) +3. Run: `make integration-test-{service}` + +Example: +```go +func (s *GatewayIntegrationSuite) TestNewFeature() { + resp, err := s.client.NewAPI(s.ctx, &pb.Request{...}) + require.NoError(s.T(), err) + + // Verify in database + var result string + s.db.QueryRow("SELECT ...").Scan(&result) + assert.Equal(s.T(), "expected", result) +} +``` + +### Adding E2E Tests + +1. Add test to `test/e2e/suite_test.go` +2. Use all service clients +3. Use `require.Eventually()` for async operations +4. Run: `make e2e-test` + +--- + +## See Also + +- [PROJECT_STRUCTURE.md](PROJECT_STRUCTURE.md) - Project organization +- [CLAUDE.md](../CLAUDE.md) - Development guidelines +- [example/server/docker-compose.yml](../example/server/docker-compose.yml) - Full stack service definitions +- [example/server/gateway/docker-compose.yml](../example/server/gateway/docker-compose.yml) - Gateway isolation +- [example/server/orchestrator/docker-compose.yml](../example/server/orchestrator/docker-compose.yml) - Orchestrator isolation diff --git a/doc/rfc/index.md b/doc/rfc/index.md new file mode 100644 index 00000000..3f1597d2 --- /dev/null +++ b/doc/rfc/index.md @@ -0,0 +1,7 @@ +# RFCs (Request for Comments) + +Design documents and technical proposals for SubmitQueue. + +## Index + +- [SQL-Based Distributed Queue](sql-queue-rfc.md) - MySQL-based distributed message queue with partition leasing and at-least-once delivery diff --git a/doc/design/sql-queue-rfc.md b/doc/rfc/sql-queue-rfc.md similarity index 100% rename from doc/design/sql-queue-rfc.md rename to doc/rfc/sql-queue-rfc.md diff --git a/e2e_test/BUILD.bazel b/e2e_test/BUILD.bazel deleted file mode 100644 index aa7b2ad0..00000000 --- a/e2e_test/BUILD.bazel +++ /dev/null @@ -1,35 +0,0 @@ -# gazelle:ignore - -load("@rules_go//go:def.bzl", "go_test") - -go_test( - name = "e2e_test", - srcs = [ - "servers.go", - "suite_test.go", - ], - data = [ - "//extension/counter/mysql/schema", - "//extension/storage/mysql/schema", - "//example/server/gateway", - "//example/server/orchestrator", - "//example/server/speculator", - ], - tags = ["integration"], - deps = [ - "//e2e_test/testutil", - "//gateway/protopb", - "//orchestrator/protopb", - "//speculator/protopb", - "@com_github_go_sql_driver_mysql//:mysql", - "@com_github_stretchr_testify//assert", - "@com_github_stretchr_testify//require", - "@com_github_stretchr_testify//suite", - "@com_github_testcontainers_testcontainers_go//:testcontainers-go", - "@com_github_testcontainers_testcontainers_go//network", - "@com_github_testcontainers_testcontainers_go//wait", - "@com_github_testcontainers_testcontainers_go_modules_mysql//:mysql", - "@org_golang_google_grpc//:grpc", - "@org_golang_google_grpc//credentials/insecure", - ], -) diff --git a/e2e_test/README.md b/e2e_test/README.md deleted file mode 100644 index 327f4922..00000000 --- a/e2e_test/README.md +++ /dev/null @@ -1,110 +0,0 @@ -# End-to-End (E2E) Tests - -This directory contains hermetic end-to-end tests for the SubmitQueue system. All infrastructure (MySQL, gRPC servers) is managed automatically via [Testcontainers-Go](https://golang.testcontainers.org/) — no manual setup required. - -## Architecture - -Tests run as a `testify/suite` that manages the full lifecycle: - -1. **Docker network** is created for inter-container communication -2. **MySQL container** starts on the network (alias `mysql`), schema is applied -3. **Server containers** (gateway, orchestrator, speculator) are built from the actual `go_binary` targets in `example/server/` and started on the network -4. **gRPC clients** connect to the mapped host ports -5. **Tests execute** against the real server binaries -6. **Cleanup** tears down all containers and the network - -All servers listen on port `8080` inside their containers. Docker maps each to a random host port, so there are no port conflicts even when tests run in parallel. The fixed internal port also simplifies inter-service communication on the Docker network — services reach each other at `:8080` (e.g., `orchestrator:8080`). - -## Structure - -| Path | Purpose | -|------|---------| -| `suite_test.go` | Test suite with `SetupSuite`/`TearDownSuite` and all test methods | -| `servers.go` | Helpers to build Docker images from server binaries and start containers | -| `testutil/docker.go` | Docker environment setup (network creation, Ryuk/HOME workarounds) | -| `testutil/mysql.go` | MySQL container setup, schema application, and test logger | -| `queue/queue_test.go` | SQL queue integration tests (publish, subscribe, partitioning, DLQ) | - -## Running Tests - -```bash -# Run all e2e tests with Bazel -bazel test //e2e_test/... --test_output=all - -# Run only the service-level suite -bazel test //e2e_test:e2e_test --test_output=all - -# Run only the queue tests -bazel test //e2e_test/queue:queue_test --test_output=all - -# Run with verbose output -bazel test //e2e_test/... --test_output=all --test_arg=-test.v - -# Run with Go (from repo root) -go test ./e2e_test/... -v -``` - -The test targets are tagged `integration` (not `manual`), so they are discovered by `bazel test //e2e_test/...`. - -## Test Cases - -### Service suite (`suite_test.go`) - -- `TestPingGateway` — Ping gateway, assert `service_name="gateway"` -- `TestPingOrchestrator` — Ping orchestrator, assert `service_name="orchestrator"` -- `TestPingSpeculator` — Ping speculator, assert `service_name="speculator"` -- `TestLandRequest` — Send `LandRequest` through gateway gRPC, assert `sqid` is returned - -### Queue suite (`queue/queue_test.go`) - -- `TestPublishAndSubscribe` — Basic publish/subscribe round-trip -- `TestMultiplePartitions` — Messages distribute across partitions -- `TestVisibilityTimeoutAndRetry` — Un-acked messages become visible again -- `TestNackWithDelay` — Nack redelivers with configurable delay -- `TestIdempotentPublish` — Duplicate message IDs are deduplicated -- `TestConcurrentPublishers` — Multiple publishers write safely -- `TestCrashRecovery` — Subscriber resumes from last committed offset -- `TestMultipleConsumerGroups` — Independent consumer groups each get all messages -- `TestMultipleWorkersInConsumerGroup` — Workers share partitions within a group -- `TestConcurrentSubscribers` — Concurrent subscribers process without duplication -- `TestDeadLetterQueue` — Failed messages move to DLQ after max retries -- `TestMessageOrderingWithinPartition` — Order preserved within a partition -- `TestLateSubscriber` — New subscriber reads existing messages -- `TestEmptyTopicSubscribe` — Subscribing to an empty topic blocks gracefully -- `TestGracefulShutdownDuringProcessing` — Close mid-processing without data loss - -## Adding New Tests - -Add a method to `IntegrationSuite` in `suite_test.go`: - -```go -func (s *IntegrationSuite) TestNewEndpoint() { - ctx := context.Background() - resp, err := s.gatewayClient.NewMethod(ctx, &gatewaypb.NewRequest{...}) - require.NoError(s.T(), err) - assert.Equal(s.T(), "expected", resp.Field) -} -``` - -If the servers need to communicate with each other, pass addresses via environment variables in `servers.go`: - -```go -_, addr := startServerContainer(ctx, t, log, "gateway", map[string]string{ - "MYSQL_DSN": "root:root@tcp(mysql:3306)/submitqueue?parseTime=true", - "ORCHESTRATOR_ADDR": "orchestrator:8080", - "SPECULATOR_ADDR": "speculator:8080", -}, nw) -``` - -## Troubleshooting - -**`$HOME is not defined`** — The Bazel sandbox doesn't set `HOME`. This is handled in `SetupSuite` by setting it to a temp directory. - -**Ryuk reaper failure** — The Testcontainers reaper container may fail in Docker-in-Docker environments. This is handled by setting `TESTCONTAINERS_RYUK_DISABLED=true` in `SetupSuite`. - -**Binary not found** — Ensure the `data` attribute in `BUILD.bazel` includes the server binary targets. Bazel places them in runfiles at `example/server//_/`. - -## TODO - -- [ ] Speed up container setup (pre-built images, parallel container starts, image caching) -- [ ] Support Tracetest/Jaeger for trace-based assertions diff --git a/e2e_test/servers.go b/e2e_test/servers.go deleted file mode 100644 index d7fb87c2..00000000 --- a/e2e_test/servers.go +++ /dev/null @@ -1,125 +0,0 @@ -package e2etest - -import ( - "context" - "fmt" - "io" - "os" - "path/filepath" - "testing" - - "github.com/stretchr/testify/require" - "github.com/testcontainers/testcontainers-go" - "github.com/testcontainers/testcontainers-go/network" - "github.com/testcontainers/testcontainers-go/wait" - "github.com/uber/submitqueue/e2e_test/testutil" -) - -const serverPort = "8080" - -// serverBinaryPath returns the path to a Bazel-built server binary. -func serverBinaryPath(name string) string { - if dir := os.Getenv("TEST_SRCDIR"); dir != "" { - workspace := os.Getenv("TEST_WORKSPACE") - return filepath.Join(dir, workspace, "example/server", name, name+"_", name) - } - return filepath.Join("example/server", name, name) -} - -// startServerContainer builds a Docker image from the server binary and starts it. -func startServerContainer( - ctx context.Context, - t *testing.T, - log *testutil.TestLogger, - name string, - env map[string]string, - nw *testcontainers.DockerNetwork, -) (testcontainers.Container, string) { - t.Helper() - - binaryPath := serverBinaryPath(name) - log.Logf("Resolved %s binary: %s", name, binaryPath) - - // Create temp build context with binary and Dockerfile. - tmpDir := t.TempDir() - copyBinary(t, binaryPath, filepath.Join(tmpDir, "server")) - - dockerfile := "FROM debian:bookworm-slim\nCOPY server /usr/local/bin/server\nCMD [\"/usr/local/bin/server\"]\n" - os.WriteFile(filepath.Join(tmpDir, "Dockerfile"), []byte(dockerfile), 0o644) - - env["PORT"] = ":" + serverPort - - log.Logf("Starting %s container", name) - ctr, err := testcontainers.Run(ctx, "", - testcontainers.WithDockerfile(testcontainers.FromDockerfile{ - Context: tmpDir, - Dockerfile: "Dockerfile", - }), - testcontainers.WithExposedPorts(serverPort+"/tcp"), - testcontainers.WithEnv(env), - testcontainers.WithWaitStrategy(wait.ForLog("gRPC server is running")), - network.WithNetwork([]string{name}, nw), - ) - if err != nil { - // Print container logs on failure if container was created - if ctr != nil { - if logs, logErr := ctr.Logs(ctx); logErr == nil { - logBytes, _ := io.ReadAll(logs) - log.Logf("%s container logs:\n%s", name, string(logBytes)) - } - } - require.NoError(t, err, "failed to start %s container", name) - } - t.Cleanup(func() { - log.Logf("Terminating %s container", name) - if err := ctr.Terminate(ctx); err != nil { - t.Logf("failed to terminate %s container: %v", name, err) - } - log.Logf("%s container terminated", name) - }) - - mappedPort, err := ctr.MappedPort(ctx, serverPort+"/tcp") - require.NoError(t, err, "failed to get mapped port for %s", name) - host, err := ctr.Host(ctx) - require.NoError(t, err, "failed to get host for %s", name) - addr := fmt.Sprintf("%s:%s", host, mappedPort.Port()) - log.Logf("%s container started on %s", name, addr) - return ctr, addr -} - - -func startGatewayContainer(ctx context.Context, t *testing.T, log *testutil.TestLogger, nw *testcontainers.DockerNetwork) string { - t.Helper() - _, addr := startServerContainer(ctx, t, log, "gateway", map[string]string{ - "MYSQL_DSN": "root:root@tcp(mysql:3306)/submitqueue?parseTime=true", - }, nw) - return addr -} - -func startOrchestratorContainer(ctx context.Context, t *testing.T, log *testutil.TestLogger, nw *testcontainers.DockerNetwork) string { - t.Helper() - _, addr := startServerContainer(ctx, t, log, "orchestrator", map[string]string{}, nw) - return addr -} - -func startSpeculatorContainer(ctx context.Context, t *testing.T, log *testutil.TestLogger, nw *testcontainers.DockerNetwork) string { - t.Helper() - _, addr := startServerContainer(ctx, t, log, "speculator", map[string]string{}, nw) - return addr -} - -// copyBinary copies a file from src to dst preserving executable permissions. -func copyBinary(t *testing.T, src, dst string) { - t.Helper() - - in, err := os.Open(src) - require.NoError(t, err, "failed to open binary %s", src) - defer in.Close() - - out, err := os.OpenFile(dst, os.O_CREATE|os.O_WRONLY, 0o755) - require.NoError(t, err, "failed to create binary copy %s", dst) - defer out.Close() - - _, err = io.Copy(out, in) - require.NoError(t, err, "failed to copy binary from %s to %s", src, dst) -} diff --git a/e2e_test/suite_test.go b/e2e_test/suite_test.go deleted file mode 100644 index 863222c7..00000000 --- a/e2e_test/suite_test.go +++ /dev/null @@ -1,115 +0,0 @@ -package e2etest - -import ( - "context" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/stretchr/testify/suite" - "github.com/testcontainers/testcontainers-go" - gatewaypb "github.com/uber/submitqueue/gateway/protopb" - "github.com/uber/submitqueue/e2e_test/testutil" - orchestratorpb "github.com/uber/submitqueue/orchestrator/protopb" - speculatorpb "github.com/uber/submitqueue/speculator/protopb" - "google.golang.org/grpc" - "google.golang.org/grpc/credentials/insecure" -) - -type IntegrationSuite struct { - suite.Suite - log *testutil.TestLogger - - nw *testcontainers.DockerNetwork - - gatewayClient gatewaypb.SubmitQueueGatewayClient - orchestratorClient orchestratorpb.SubmitQueueOrchestratorClient - speculatorClient speculatorpb.SubmitQueueSpeculatorClient - - cleanups []func() -} - -func TestIntegration(t *testing.T) { - suite.Run(t, new(IntegrationSuite)) -} - -func (s *IntegrationSuite) SetupSuite() { - t := s.T() - ctx := context.Background() - s.log = testutil.NewTestLogger(t) - - // Setup Docker environment and network - s.nw, ctx = testutil.SetupDockerEnv(t, s.log, ctx) - - // Start MySQL container on the network and apply schemas. - mysqlContainer, db, _ := testutil.SetupMySQL(t, s.log, s.nw, "extension/storage/mysql/schema") - testutil.ApplySchema(t, s.log, db, testutil.SchemaDir("extension/counter/mysql/schema")) - - // Register MySQL cleanup - s.addCleanup(func() { - s.log.Logf("Closing MySQL connection") - if err := db.Close(); err != nil { - s.log.Logf("Failed to close MySQL connection: %v", err) - } - s.log.Logf("Terminating MySQL container") - if err := mysqlContainer.Terminate(context.Background()); err != nil { - s.log.Logf("Failed to terminate MySQL container: %v", err) - } - s.log.Logf("MySQL container terminated") - }) - - // Start all server containers. - gatewayAddr := startGatewayContainer(ctx, t, s.log, s.nw) - orchestratorAddr := startOrchestratorContainer(ctx, t, s.log, s.nw) - speculatorAddr := startSpeculatorContainer(ctx, t, s.log, s.nw) - - // Create gRPC client connections. - opts := grpc.WithTransportCredentials(insecure.NewCredentials()) - s.gatewayClient = gatewaypb.NewSubmitQueueGatewayClient(s.dial(gatewayAddr, opts)) - s.orchestratorClient = orchestratorpb.NewSubmitQueueOrchestratorClient(s.dial(orchestratorAddr, opts)) - s.speculatorClient = speculatorpb.NewSubmitQueueSpeculatorClient(s.dial(speculatorAddr, opts)) - - s.log.Logf("All containers started and clients connected") -} - -func (s *IntegrationSuite) TearDownSuite() { - for i := len(s.cleanups) - 1; i >= 0; i-- { - s.cleanups[i]() - } -} - -func (s *IntegrationSuite) addCleanup(fn func()) { - s.cleanups = append(s.cleanups, fn) -} - -func (s *IntegrationSuite) dial(addr string, opts ...grpc.DialOption) *grpc.ClientConn { - conn, err := grpc.NewClient(addr, opts...) - require.NoError(s.T(), err, "failed to connect to %s", addr) - s.addCleanup(func() { conn.Close() }) - return conn -} - -func (s *IntegrationSuite) TestPingGateway() { - ctx := context.Background() - resp, err := s.gatewayClient.Ping(ctx, &gatewaypb.PingRequest{Message: "integration test"}) - require.NoError(s.T(), err, "Gateway Ping failed") - assert.Equal(s.T(), "gateway", resp.ServiceName) - s.log.Logf("Gateway ping: %s", resp.Message) -} - -func (s *IntegrationSuite) TestPingOrchestrator() { - ctx := context.Background() - resp, err := s.orchestratorClient.Ping(ctx, &orchestratorpb.PingRequest{Message: "integration test"}) - require.NoError(s.T(), err, "Orchestrator Ping failed") - assert.Equal(s.T(), "orchestrator", resp.ServiceName) - s.log.Logf("Orchestrator ping: %s", resp.Message) -} - -func (s *IntegrationSuite) TestPingSpeculator() { - ctx := context.Background() - resp, err := s.speculatorClient.Ping(ctx, &speculatorpb.PingRequest{Message: "integration test"}) - require.NoError(s.T(), err, "Speculator Ping failed") - assert.Equal(s.T(), "speculator", resp.ServiceName) - s.log.Logf("Speculator ping: %s", resp.Message) -} - diff --git a/e2e_test/testutil/BUILD.bazel b/e2e_test/testutil/BUILD.bazel deleted file mode 100644 index d6112052..00000000 --- a/e2e_test/testutil/BUILD.bazel +++ /dev/null @@ -1,18 +0,0 @@ -load("@rules_go//go:def.bzl", "go_library") - -go_library( - name = "testutil", - srcs = [ - "docker.go", - "mysql.go", - ], - importpath = "github.com/uber/submitqueue/e2e_test/testutil", - visibility = ["//visibility:public"], - deps = [ - "@com_github_go_sql_driver_mysql//:mysql", - "@com_github_stretchr_testify//require", - "@com_github_testcontainers_testcontainers_go//:testcontainers-go", - "@com_github_testcontainers_testcontainers_go//network", - "@com_github_testcontainers_testcontainers_go_modules_mysql//:mysql", - ], -) diff --git a/e2e_test/testutil/docker.go b/e2e_test/testutil/docker.go deleted file mode 100644 index e5c4460e..00000000 --- a/e2e_test/testutil/docker.go +++ /dev/null @@ -1,41 +0,0 @@ -package testutil - -import ( - "context" - "os" - "testing" - - "github.com/stretchr/testify/require" - "github.com/testcontainers/testcontainers-go" - "github.com/testcontainers/testcontainers-go/network" -) - -// SetupDockerEnv configures Docker environment for testcontainers and creates a network. -// Automatically registers cleanup to remove the network on test completion. -// Returns the Docker network and the context to use for container operations. -func SetupDockerEnv(t *testing.T, log *TestLogger, ctx context.Context) (*testcontainers.DockerNetwork, context.Context) { - t.Helper() - - // Disable Ryuk reaper for Docker-in-Docker environments - t.Setenv("TESTCONTAINERS_RYUK_DISABLED", "true") - - // Ensure HOME is set for Docker config - if os.Getenv("HOME") == "" { - t.Setenv("HOME", t.TempDir()) - } - - // Create Docker network - nw, err := network.New(ctx) - require.NoError(t, err, "failed to create Docker network") - - log.Logf("Docker network created: %s", nw.Name) - - // Register cleanup - t.Cleanup(func() { - log.Logf("Removing Docker network") - require.NoError(t, nw.Remove(ctx), "failed to remove Docker network") - log.Logf("Docker network removed") - }) - - return nw, ctx -} diff --git a/e2e_test/testutil/mysql.go b/e2e_test/testutil/mysql.go deleted file mode 100644 index 5c51e29d..00000000 --- a/e2e_test/testutil/mysql.go +++ /dev/null @@ -1,112 +0,0 @@ -package testutil - -import ( - "context" - "database/sql" - "os" - "path/filepath" - "sort" - "testing" - "time" - - _ "github.com/go-sql-driver/mysql" - "github.com/stretchr/testify/require" - "github.com/testcontainers/testcontainers-go" - "github.com/testcontainers/testcontainers-go/modules/mysql" - "github.com/testcontainers/testcontainers-go/network" -) - -// TestLogger is a simple test-aware logger that records elapsed time between logs. -type TestLogger struct { - t *testing.T // The testing object to report logs to. - last time.Time // Timestamp of the last log, for elapsed calculation. -} - -// NewTestLogger creates a TestLogger for the current test. -func NewTestLogger(t *testing.T) *TestLogger { - t.Helper() - return &TestLogger{t: t} -} - -// Logf prints a formatted log message with timestamp and elapsed time since last log. -func (l *TestLogger) Logf(format string, args ...any) { - l.t.Helper() - now := time.Now() - delta := "" - if !l.last.IsZero() { - delta = " +" + now.Sub(l.last).Truncate(time.Millisecond).String() - } - l.last = now - l.t.Logf("[%s%s] "+format, append([]any{now.Format(time.RFC3339Nano), delta}, args...)...) -} - -// SchemaDir returns the path to a schema directory. -// It checks for both Bazel runfiles and direct go test paths. -// relativePath should be like "extension/storage/mysql/schema" or "extension/queue/sql/schema" -func SchemaDir(relativePath string) string { - // Bazel runfiles path - if dir := os.Getenv("TEST_SRCDIR"); dir != "" { - return filepath.Join(dir, os.Getenv("TEST_WORKSPACE"), relativePath) - } - // Direct go test path (run from repo root) - return relativePath -} - -// ApplySchema reads all .sql files from the schema directory and executes them on the database. -func ApplySchema(t *testing.T, log *TestLogger, db *sql.DB, schemaDirectory string) { - t.Helper() - - files, err := filepath.Glob(filepath.Join(schemaDirectory, "*.sql")) - require.NoError(t, err, "failed to glob schema files") - require.NotEmpty(t, files, "no .sql schema files found in %s", schemaDirectory) - - // Sort files to ensure deterministic schema application order. - sort.Strings(files) - - for _, f := range files { - name := filepath.Base(f) - log.Logf("Applying schema: %s", name) - - content, err := os.ReadFile(f) - require.NoError(t, err, "failed to read schema file %s", name) - - _, err = db.ExecContext(context.Background(), string(content)) - require.NoError(t, err, "failed to execute schema file %s", name) - - log.Logf("Schema applied: %s", name) - } -} - -// SetupMySQL starts a MySQL container on the given Docker network, applies the schema, -// and returns the container, db connection, and DSN for use in tests. -// The caller is responsible for cleanup (closing db, terminating container). -// schemaPath is the relative path to the schema directory (e.g., "extension/storage/mysql/schema"). -func SetupMySQL(t *testing.T, log *TestLogger, nw *testcontainers.DockerNetwork, schemaPath string) (*mysql.MySQLContainer, *sql.DB, string) { - t.Helper() - - ctx := context.Background() - - log.Logf("Starting MySQL container") - mysqlContainer, err := mysql.Run(ctx, "mysql:8.0", - mysql.WithDatabase("submitqueue"), - mysql.WithUsername("root"), - mysql.WithPassword("root"), - network.WithNetwork([]string{"mysql"}, nw), - ) - require.NoError(t, err, "failed to start MySQL container") - log.Logf("MySQL container started") - - dsn, err := mysqlContainer.ConnectionString(ctx, "parseTime=true") - require.NoError(t, err, "failed to get MySQL connection string") - log.Logf("MySQL DSN obtained: %s", dsn) - - log.Logf("Opening MySQL connection") - db, err := sql.Open("mysql", dsn) - require.NoError(t, err, "failed to open MySQL connection") - log.Logf("MySQL connection opened") - - dir := SchemaDir(schemaPath) - ApplySchema(t, log, db, dir) - - return mysqlContainer, db, dsn -} diff --git a/example/client/speculator/BUILD.bazel b/example/client/speculator/BUILD.bazel deleted file mode 100644 index fd253303..00000000 --- a/example/client/speculator/BUILD.bazel +++ /dev/null @@ -1,19 +0,0 @@ -load("@rules_go//go:def.bzl", "go_binary", "go_library") - -go_library( - name = "speculator_lib", - srcs = ["main.go"], - importpath = "github.com/uber/submitqueue/example/client/speculator", - visibility = ["//visibility:private"], - deps = [ - "//speculator/protopb", - "@org_golang_google_grpc//:grpc", - "@org_golang_google_grpc//credentials/insecure", - ], -) - -go_binary( - name = "speculator", - embed = [":speculator_lib"], - visibility = ["//visibility:public"], -) diff --git a/example/client/speculator/main.go b/example/client/speculator/main.go deleted file mode 100644 index 67c5e005..00000000 --- a/example/client/speculator/main.go +++ /dev/null @@ -1,64 +0,0 @@ -package main - -import ( - "context" - "flag" - "fmt" - "os" - "time" - - pb "github.com/uber/submitqueue/speculator/protopb" - "google.golang.org/grpc" - "google.golang.org/grpc/credentials/insecure" -) - -func main() { - addr := flag.String("addr", "localhost:8083", "speculator server address") - message := flag.String("message", "", "message to send in ping request") - timeout := flag.Duration("timeout", 5*time.Second, "request timeout") - flag.Parse() - - if err := run(*addr, *message, *timeout); err != nil { - fmt.Fprintf(os.Stderr, "Error: %v\n", err) - os.Exit(1) - } -} - -func run(addr, message string, timeout time.Duration) error { - // Create a gRPC connection - conn, err := grpc.NewClient( - addr, - grpc.WithTransportCredentials(insecure.NewCredentials()), - ) - if err != nil { - return fmt.Errorf("failed to connect: %w", err) - } - defer conn.Close() - - // Create a client - client := pb.NewSubmitQueueSpeculatorClient(conn) - - // Create context with timeout - ctx, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() - - // Make the ping request - req := &pb.PingRequest{ - Message: message, - } - - fmt.Printf("Sending ping to speculator at %s...\n", addr) - resp, err := client.Ping(ctx, req) - if err != nil { - return fmt.Errorf("ping failed: %w", err) - } - - // Print the response - fmt.Printf("\nResponse:\n") - fmt.Printf(" Message: %s\n", resp.Message) - fmt.Printf(" Service Name: %s\n", resp.ServiceName) - fmt.Printf(" Timestamp: %d (%s)\n", resp.Timestamp, time.Unix(resp.Timestamp, 0)) - fmt.Printf(" Hostname: %s\n", resp.Hostname) - - return nil -} diff --git a/example/server/BUILD.bazel b/example/server/BUILD.bazel new file mode 100644 index 00000000..cb1f7a27 --- /dev/null +++ b/example/server/BUILD.bazel @@ -0,0 +1,4 @@ +exports_files( + ["docker-compose.yml"], + visibility = ["//visibility:public"], +) diff --git a/example/server/docker-compose.yml b/example/server/docker-compose.yml new file mode 100644 index 00000000..01b8d266 --- /dev/null +++ b/example/server/docker-compose.yml @@ -0,0 +1,76 @@ +# Docker Compose for Full Stack (Gateway + Orchestrator + MySQL) +# +# IMPORTANT: Before running docker-compose, build the Linux binaries: +# make build-gateway-linux build-orchestrator-linux +# OR +# bazel build --platforms=@rules_go//go/toolchain:linux_amd64 //example/server/gateway //example/server/orchestrator +# +# Quick start: +# make e2e-test + +services: + # Application Database - Stores business data (requests, counters, etc.) + mysql-app: + image: mysql:8.0 + environment: + MYSQL_ROOT_PASSWORD: root + MYSQL_DATABASE: submitqueue + ports: + - "3306" # Random ephemeral port to avoid conflicts + healthcheck: + test: ["CMD", "mysqladmin", "ping", "-h", "localhost", "-proot"] + interval: 5s + timeout: 5s + retries: 10 + + # Queue Database - Messaging infrastructure (messages, offsets, partition leases) + # Separate from app DB to demonstrate queue is pluggable infrastructure + mysql-queue: + image: mysql:8.0 + environment: + MYSQL_ROOT_PASSWORD: root + MYSQL_DATABASE: submitqueue + ports: + - "3306" # Random ephemeral port to avoid conflicts + healthcheck: + test: ["CMD", "mysqladmin", "ping", "-h", "localhost", "-proot"] + interval: 5s + timeout: 5s + retries: 10 + + gateway-service: + build: + context: ${REPO_ROOT} + dockerfile: example/server/gateway/Dockerfile + ports: + - "8080" # Random ephemeral port to avoid conflicts + environment: + - PORT=:8080 + # Application database connection + - MYSQL_DSN=root:root@tcp(mysql-app:3306)/submitqueue?parseTime=true + # Queue infrastructure connection (separate database) + - QUEUE_MYSQL_DSN=root:root@tcp(mysql-queue:3306)/submitqueue?parseTime=true + depends_on: + mysql-app: + condition: service_healthy + mysql-queue: + condition: service_healthy + + orchestrator-service: + build: + context: ${REPO_ROOT} + dockerfile: example/server/orchestrator/Dockerfile + ports: + - "8080" # Random ephemeral port to avoid conflicts + environment: + - PORT=:8080 + # Application database connection (for request state, batches, etc.) + - MYSQL_DSN=root:root@tcp(mysql-app:3306)/submitqueue?parseTime=true + # Queue infrastructure connection (separate database) + - QUEUE_MYSQL_DSN=root:root@tcp(mysql-queue:3306)/submitqueue?parseTime=true + - HOSTNAME=orchestrator-dev + depends_on: + mysql-app: + condition: service_healthy + mysql-queue: + condition: service_healthy diff --git a/example/server/gateway/BUILD.bazel b/example/server/gateway/BUILD.bazel index 7c5d03ff..8e40705b 100644 --- a/example/server/gateway/BUILD.bazel +++ b/example/server/gateway/BUILD.bazel @@ -1,5 +1,10 @@ load("@rules_go//go:def.bzl", "go_binary", "go_library") +exports_files( + ["docker-compose.yml"], + visibility = ["//visibility:public"], +) + go_library( name = "gateway_lib", srcs = ["main.go"], @@ -7,7 +12,9 @@ go_library( visibility = ["//visibility:private"], deps = [ "//extension/counter/mysql", + "//extension/queue", "//extension/queue/sql", + "//extension/storage", "//extension/storage/mysql", "//gateway/controller", "//gateway/protopb", diff --git a/example/server/gateway/Dockerfile b/example/server/gateway/Dockerfile new file mode 100644 index 00000000..4a581628 --- /dev/null +++ b/example/server/gateway/Dockerfile @@ -0,0 +1,12 @@ +FROM debian:bookworm-slim + +RUN apt-get update && apt-get install -y ca-certificates && rm -rf /var/lib/apt/lists/* +WORKDIR /root/ + +# Copy pre-built Linux binary +# Built via: make build-gateway-linux +COPY .docker-bin/gateway ./gateway + +EXPOSE 8080 + +CMD ["./gateway"] diff --git a/example/server/gateway/docker-compose.yml b/example/server/gateway/docker-compose.yml new file mode 100644 index 00000000..292f65d1 --- /dev/null +++ b/example/server/gateway/docker-compose.yml @@ -0,0 +1,57 @@ +# Docker Compose for Gateway Manual Testing +# +# IMPORTANT: Before running docker-compose, build the Linux binary: +# make build-gateway-linux +# OR +# bazel build --platforms=@rules_go//go/toolchain:linux_amd64 //example/server/gateway +# +# Quick start: +# make docker-gateway (builds binary + starts compose) + +services: + # Application Database - Stores business data (requests, counters, etc.) + mysql-app: + image: mysql:8.0 + environment: + MYSQL_ROOT_PASSWORD: root + MYSQL_DATABASE: submitqueue + ports: + - "3306" # Random ephemeral port to avoid conflicts + healthcheck: + test: ["CMD", "mysqladmin", "ping", "-h", "localhost", "-proot"] + interval: 5s + timeout: 5s + retries: 10 + + # Queue Database - Messaging infrastructure (messages, offsets, partition leases) + # Separate from app DB to demonstrate queue is pluggable infrastructure + mysql-queue: + image: mysql:8.0 + environment: + MYSQL_ROOT_PASSWORD: root + MYSQL_DATABASE: submitqueue + ports: + - "3306" # Random ephemeral port to avoid conflicts + healthcheck: + test: ["CMD", "mysqladmin", "ping", "-h", "localhost", "-proot"] + interval: 5s + timeout: 5s + retries: 10 + + gateway-service: + build: + context: ${REPO_ROOT} + dockerfile: example/server/gateway/Dockerfile + ports: + - "8080" # Random ephemeral port to avoid conflicts + environment: + - PORT=:8080 + # Application database connection + - MYSQL_DSN=root:root@tcp(mysql-app:3306)/submitqueue?parseTime=true + # Queue infrastructure connection (separate database) + - QUEUE_MYSQL_DSN=root:root@tcp(mysql-queue:3306)/submitqueue?parseTime=true + depends_on: + mysql-app: + condition: service_healthy + mysql-queue: + condition: service_healthy diff --git a/example/server/gateway/main.go b/example/server/gateway/main.go index d5e40f59..3cd36d29 100644 --- a/example/server/gateway/main.go +++ b/example/server/gateway/main.go @@ -14,7 +14,9 @@ import ( _ "github.com/go-sql-driver/mysql" "github.com/uber-go/tally/v4" mysqlcounter "github.com/uber/submitqueue/extension/counter/mysql" + "github.com/uber/submitqueue/extension/queue" queueSQL "github.com/uber/submitqueue/extension/queue/sql" + "github.com/uber/submitqueue/extension/storage" "github.com/uber/submitqueue/extension/storage/mysql" "github.com/uber/submitqueue/gateway/controller" pb "github.com/uber/submitqueue/gateway/protopb" @@ -89,19 +91,35 @@ func run() error { metricsWgDone.Wait() }() - // Initialize MySQL storage + // Initialize MySQL storage (with retries for MySQL startup) mysqlDSN := os.Getenv("MYSQL_DSN") if mysqlDSN == "" { mysqlDSN = "root:root@tcp(localhost:3306)/submitqueue?parseTime=true" } - store, err := mysql.NewStorage(mysql.MySQLParameters{ - DSN: mysqlDSN, - MaxOpenConns: 10, - MaxIdleConns: 5, - ConnMaxLifetime: 5 * time.Minute, - }) + var store storage.Storage + maxRetries := 10 + retryInterval := time.Second + for i := 0; i < maxRetries; i++ { + store, err = mysql.NewStorage(mysql.MySQLParameters{ + DSN: mysqlDSN, + MaxOpenConns: 10, + MaxIdleConns: 5, + ConnMaxLifetime: 5 * time.Minute, + }) + if err == nil { + break + } + if i < maxRetries-1 { + logger.Warn("failed to create MySQL storage, retrying...", + zap.Int("attempt", i+1), + zap.Int("max_retries", maxRetries), + zap.Error(err), + ) + time.Sleep(retryInterval) + } + } if err != nil { - return fmt.Errorf("failed to create MySQL storage: %w", err) + return fmt.Errorf("failed to create MySQL storage after %d retries: %w", maxRetries, err) } defer store.Close() @@ -132,13 +150,30 @@ func run() error { } defer queueDB.Close() - q, err := queueSQL.NewQueue(queueSQL.Params{ - DB: queueDB, - Logger: logger, - MetricsScope: scope.SubScope("queue"), - }) + // Retry queue creation (with retries for MySQL startup) + var q queue.Queue + maxRetries := 10 + retryInterval := time.Second + for i := 0; i < maxRetries; i++ { + q, err = queueSQL.NewQueue(queueSQL.Params{ + DB: queueDB, + Logger: logger, + MetricsScope: scope.SubScope("queue"), + }) + if err == nil { + break + } + if i < maxRetries-1 { + logger.Warn("failed to create queue, retrying...", + zap.Int("attempt", i+1), + zap.Int("max_retries", maxRetries), + zap.Error(err), + ) + time.Sleep(retryInterval) + } + } if err != nil { - return fmt.Errorf("failed to create queue: %w", err) + return fmt.Errorf("failed to create queue after %d retries: %w", maxRetries, err) } defer q.Close() diff --git a/example/server/orchestrator/BUILD.bazel b/example/server/orchestrator/BUILD.bazel index 9662373e..f006cb63 100644 --- a/example/server/orchestrator/BUILD.bazel +++ b/example/server/orchestrator/BUILD.bazel @@ -1,5 +1,10 @@ load("@rules_go//go:def.bzl", "go_binary", "go_library") +exports_files( + ["docker-compose.yml"], + visibility = ["//visibility:public"], +) + go_library( name = "orchestrator_lib", srcs = ["main.go"], @@ -7,6 +12,7 @@ go_library( visibility = ["//visibility:private"], deps = [ "//consumer", + "//extension/queue", "//extension/queue/sql", "//orchestrator/controller", "//orchestrator/controller/request", diff --git a/example/server/orchestrator/Dockerfile b/example/server/orchestrator/Dockerfile new file mode 100644 index 00000000..fb1a4626 --- /dev/null +++ b/example/server/orchestrator/Dockerfile @@ -0,0 +1,12 @@ +FROM debian:bookworm-slim + +RUN apt-get update && apt-get install -y ca-certificates && rm -rf /var/lib/apt/lists/* +WORKDIR /root/ + +# Copy pre-built Linux binary +# Built via: make build-orchestrator-linux +COPY .docker-bin/orchestrator ./orchestrator + +EXPOSE 8080 + +CMD ["./orchestrator"] diff --git a/example/server/orchestrator/docker-compose.yml b/example/server/orchestrator/docker-compose.yml new file mode 100644 index 00000000..36dd44d1 --- /dev/null +++ b/example/server/orchestrator/docker-compose.yml @@ -0,0 +1,58 @@ +# Docker Compose for Orchestrator Manual Testing +# +# IMPORTANT: Before running docker-compose, build the Linux binary: +# make build-orchestrator-linux +# OR +# bazel build --platforms=@rules_go//go/toolchain:linux_amd64 //example/server/orchestrator +# +# Quick start: +# make docker-orchestrator (builds binary + starts compose) + +services: + # Application Database - Stores business data (requests, batches, etc.) + mysql-app: + image: mysql:8.0 + environment: + MYSQL_ROOT_PASSWORD: root + MYSQL_DATABASE: submitqueue + ports: + - "3306" # Random ephemeral port to avoid conflicts + healthcheck: + test: ["CMD", "mysqladmin", "ping", "-h", "localhost", "-proot"] + interval: 5s + timeout: 5s + retries: 10 + + # Queue Database - Messaging infrastructure (messages, offsets, partition leases) + # Separate from app DB to demonstrate queue is pluggable infrastructure + mysql-queue: + image: mysql:8.0 + environment: + MYSQL_ROOT_PASSWORD: root + MYSQL_DATABASE: submitqueue + ports: + - "3306" # Random ephemeral port to avoid conflicts + healthcheck: + test: ["CMD", "mysqladmin", "ping", "-h", "localhost", "-proot"] + interval: 5s + timeout: 5s + retries: 10 + + orchestrator-service: + build: + context: ${REPO_ROOT} + dockerfile: example/server/orchestrator/Dockerfile + ports: + - "8080" # Random ephemeral port to avoid conflicts + environment: + - PORT=:8080 + # Application database connection (for request state, batches, etc.) + - MYSQL_DSN=root:root@tcp(mysql-app:3306)/submitqueue?parseTime=true + # Queue infrastructure connection (separate database) + - QUEUE_MYSQL_DSN=root:root@tcp(mysql-queue:3306)/submitqueue?parseTime=true + - HOSTNAME=orchestrator-dev + depends_on: + mysql-app: + condition: service_healthy + mysql-queue: + condition: service_healthy diff --git a/example/server/orchestrator/main.go b/example/server/orchestrator/main.go index cc1cead1..cb006c13 100644 --- a/example/server/orchestrator/main.go +++ b/example/server/orchestrator/main.go @@ -14,6 +14,7 @@ import ( _ "github.com/go-sql-driver/mysql" "github.com/uber-go/tally/v4" "github.com/uber/submitqueue/consumer" + "github.com/uber/submitqueue/extension/queue" queueSQL "github.com/uber/submitqueue/extension/queue/sql" "github.com/uber/submitqueue/orchestrator/controller" "github.com/uber/submitqueue/orchestrator/controller/request" @@ -91,13 +92,30 @@ func run() error { } defer queueDB.Close() - q, err := queueSQL.NewQueue(queueSQL.Params{ - DB: queueDB, - Logger: logger, - MetricsScope: scope.SubScope("queue"), - }) + // Retry queue creation (with retries for MySQL startup) + var q queue.Queue + maxRetries := 10 + retryInterval := time.Second + for i := 0; i < maxRetries; i++ { + q, err = queueSQL.NewQueue(queueSQL.Params{ + DB: queueDB, + Logger: logger, + MetricsScope: scope.SubScope("queue"), + }) + if err == nil { + break + } + if i < maxRetries-1 { + logger.Warn("failed to create queue, retrying...", + zap.Int("attempt", i+1), + zap.Int("max_retries", maxRetries), + zap.Error(err), + ) + time.Sleep(retryInterval) + } + } if err != nil { - return fmt.Errorf("failed to create queue: %w", err) + return fmt.Errorf("failed to create queue after %d retries: %w", maxRetries, err) } defer q.Close() diff --git a/example/server/speculator/BUILD.bazel b/example/server/speculator/BUILD.bazel deleted file mode 100644 index dd14ae66..00000000 --- a/example/server/speculator/BUILD.bazel +++ /dev/null @@ -1,22 +0,0 @@ -load("@rules_go//go:def.bzl", "go_binary", "go_library") - -go_library( - name = "speculator_lib", - srcs = ["main.go"], - importpath = "github.com/uber/submitqueue/example/server/speculator", - visibility = ["//visibility:private"], - deps = [ - "//speculator/controller", - "//speculator/protopb", - "@com_github_uber_go_tally_v4//:tally", - "@org_golang_google_grpc//:grpc", - "@org_golang_google_grpc//reflection", - "@org_uber_go_zap//:zap", - ], -) - -go_binary( - name = "speculator", - embed = [":speculator_lib"], - visibility = ["//visibility:public"], -) diff --git a/example/server/speculator/main.go b/example/server/speculator/main.go deleted file mode 100644 index 71bf416d..00000000 --- a/example/server/speculator/main.go +++ /dev/null @@ -1,126 +0,0 @@ -package main - -import ( - "context" - "fmt" - "net" - "os" - "os/signal" - "sync" - "syscall" - "time" - - "github.com/uber-go/tally/v4" - "github.com/uber/submitqueue/speculator/controller" - pb "github.com/uber/submitqueue/speculator/protopb" - "go.uber.org/zap" - "google.golang.org/grpc" - "google.golang.org/grpc/reflection" -) - -// SpeculatorServer wraps the controller and implements the gRPC service interface -type SpeculatorServer struct { - pb.UnimplementedSubmitQueueSpeculatorServer - controller *controller.PingController -} - -// Ping delegates to the controller -func (s *SpeculatorServer) Ping(ctx context.Context, req *pb.PingRequest) (*pb.PingResponse, error) { - return s.controller.Ping(ctx, req) -} - -func main() { - if err := run(); err != nil { - fmt.Fprintf(os.Stderr, "Speculator server failure: %v\n", err) - os.Exit(1) - } -} - -func run() error { - // Initialize development logger (human-readable console output) - logger, err := zap.NewDevelopment() - if err != nil { - return fmt.Errorf("failed to create logger: %w", err) - } - defer logger.Sync() - - // Initialize metrics scope - scope := tally.NewTestScope("speculator", nil) - metricsStopCh := make(chan interface{}, 1) - metricsWgDone := sync.WaitGroup{} - metricsWgDone.Add(1) - go func() { - defer metricsWgDone.Done() - - ticker := time.NewTicker(10 * time.Second) - defer ticker.Stop() - - for { - select { - case <-metricsStopCh: - return - case <-ticker.C: - snapshot := scope.Snapshot() - logger.Info("metrics snapshot", - zap.Any("counters", snapshot.Counters()), - zap.Any("gauges", snapshot.Gauges()), - zap.Any("timers", snapshot.Timers()), - ) - } - } - }() - - defer func() { - close(metricsStopCh) - metricsWgDone.Wait() - }() - - // Create gRPC server - grpcServer := grpc.NewServer() - - // Create ping controller and wrap it for gRPC - pingController := controller.NewPingController(logger, scope) - speculatorServer := &SpeculatorServer{ - controller: pingController, - } - pb.RegisterSubmitQueueSpeculatorServer(grpcServer, speculatorServer) - - // Register reflection service for debugging with grpcurl - reflection.Register(grpcServer) - - // Listen on configurable port - port := os.Getenv("PORT") - if port == "" { - port = ":8083" - } - listener, err := net.Listen("tcp", port) - if err != nil { - return fmt.Errorf("failed to listen on port %s: %w", port, err) - } - - fmt.Printf("Speculator gRPC server is running on %s\n", port) - fmt.Println("Press Ctrl+C to stop.") - - // Start server in a goroutine and wait for it to finish - serverErrCh := make(chan error, 1) - go func() { - serverErrCh <- grpcServer.Serve(listener) - }() - - // Wait for interrupt signal or server exit - sigCh := make(chan os.Signal, 1) - signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM) - - select { - case <-sigCh: - fmt.Println("\nShutting down speculator server...") - grpcServer.GracefulStop() - _ = <-serverErrCh // Wait for the server to exit and ignore the error - case errCh := <-serverErrCh: - if errCh != nil { - err = fmt.Errorf("\nServer exited with error: %w\n", errCh) - } - } - - return err -} diff --git a/gateway/integration_test/BUILD.bazel b/gateway/integration_test/BUILD.bazel deleted file mode 100644 index 19d8d845..00000000 --- a/gateway/integration_test/BUILD.bazel +++ /dev/null @@ -1,19 +0,0 @@ -# gazelle:ignore - -load("@rules_go//go:def.bzl", "go_test") - -go_test( - name = "integration_test_test", - srcs = ["ping_test.go"], - tags = [ - "integration", - "manual", - ], - deps = [ - "//gateway/protopb", - "@com_github_stretchr_testify//assert", - "@com_github_stretchr_testify//require", - "@org_golang_google_grpc//:grpc", - "@org_golang_google_grpc//credentials/insecure", - ], -) diff --git a/gateway/integration_test/ping_test.go b/gateway/integration_test/ping_test.go deleted file mode 100644 index bfa0517a..00000000 --- a/gateway/integration_test/ping_test.go +++ /dev/null @@ -1,92 +0,0 @@ -package gateway_test - -import ( - "context" - "fmt" - "os" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - pb "github.com/uber/submitqueue/gateway/protopb" - "google.golang.org/grpc" - "google.golang.org/grpc/credentials/insecure" -) - -const ( - defaultTimeout = 5 * time.Second - serverReadyTimeout = 30 * time.Second - retryInterval = 500 * time.Millisecond -) - -// TestPingAPI tests the Gateway service Ping API -func TestPingAPI(t *testing.T) { - addr := getEnvOrDefault("GATEWAY_ADDR", "localhost:8081") - - // Wait for server to be ready - conn, err := waitForServer(t, addr, serverReadyTimeout) - require.NoError(t, err, "Gateway server not ready") - defer conn.Close() - - client := pb.NewSubmitQueueGatewayClient(conn) - ctx, cancel := context.WithTimeout(context.Background(), defaultTimeout) - defer cancel() - - // Test Ping API - req := &pb.PingRequest{ - Message: "integration test", - } - - resp, err := client.Ping(ctx, req) - require.NoError(t, err, "Ping failed") - - // Validate response - assert.NotEmpty(t, resp.Message, "Response message should not be empty") - assert.Equal(t, "gateway", resp.ServiceName) - assert.NotZero(t, resp.Timestamp, "Timestamp should not be zero") - assert.NotEmpty(t, resp.Hostname, "Hostname should not be empty") - - t.Logf("Gateway Ping test passed:") - t.Logf(" Message: %s", resp.Message) - t.Logf(" Service: %s", resp.ServiceName) - t.Logf(" Timestamp: %d", resp.Timestamp) - t.Logf(" Hostname: %s", resp.Hostname) -} - -// waitForServer waits for a gRPC server to become ready -func waitForServer(t *testing.T, addr string, timeout time.Duration) (*grpc.ClientConn, error) { - t.Helper() - - deadline := time.Now().Add(timeout) - var lastErr error - - for time.Now().Before(deadline) { - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) - conn, err := grpc.DialContext( - ctx, - addr, - grpc.WithTransportCredentials(insecure.NewCredentials()), - grpc.WithBlock(), - ) - cancel() - - if err == nil { - t.Logf("Server at %s is ready", addr) - return conn, nil - } - - lastErr = err - time.Sleep(retryInterval) - } - - return nil, fmt.Errorf("server at %s not ready after %v: %w", addr, timeout, lastErr) -} - -// getEnvOrDefault returns the value of an environment variable or a default value -func getEnvOrDefault(key, defaultValue string) string { - if value := os.Getenv(key); value != "" { - return value - } - return defaultValue -} diff --git a/go.mod b/go.mod index e6eb7e7a..51255d21 100644 --- a/go.mod +++ b/go.mod @@ -7,8 +7,6 @@ require ( github.com/go-sql-driver/mysql v1.9.3 github.com/gogo/protobuf v1.3.2 github.com/stretchr/testify v1.11.1 - github.com/testcontainers/testcontainers-go v0.40.0 - github.com/testcontainers/testcontainers-go/modules/mysql v0.40.0 github.com/uber-go/tally/v4 v4.1.17 go.uber.org/fx v1.22.0 go.uber.org/mock v0.6.0 @@ -19,76 +17,32 @@ require ( ) require ( - dario.cat/mergo v1.0.2 // indirect filippo.io/edwards25519 v1.1.0 // indirect - github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect github.com/BurntSushi/toml v1.2.1 // indirect - github.com/Microsoft/go-winio v0.6.2 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/containerd/errdefs v1.0.0 // indirect - github.com/containerd/errdefs/pkg v0.3.0 // indirect - github.com/containerd/log v0.1.0 // indirect - github.com/containerd/platforms v0.2.1 // indirect - github.com/cpuguy83/dockercfg v0.3.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/distribution/reference v0.6.0 // indirect - github.com/docker/docker v28.5.1+incompatible // indirect - github.com/docker/go-connections v0.6.0 // indirect - github.com/docker/go-units v0.5.0 // indirect - github.com/ebitengine/purego v0.8.4 // indirect - github.com/felixge/httpsnoop v1.0.4 // indirect - github.com/go-logr/logr v1.4.3 // indirect - github.com/go-logr/stdr v1.2.2 // indirect - github.com/go-ole/go-ole v1.2.6 // indirect github.com/gogo/googleapis v1.4.1 // indirect github.com/gogo/status v1.1.0 // indirect github.com/golang/mock v1.7.0-rc.1 // indirect github.com/golang/protobuf v1.5.4 // indirect - github.com/google/uuid v1.6.0 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0 // indirect - github.com/klauspost/compress v1.18.0 // indirect - github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect - github.com/magiconair/properties v1.8.10 // indirect + github.com/google/go-cmp v0.7.0 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect - github.com/moby/docker-image-spec v1.3.1 // indirect - github.com/moby/go-archive v0.1.0 // indirect - github.com/moby/patternmatcher v0.6.0 // indirect - github.com/moby/sys/sequential v0.6.0 // indirect - github.com/moby/sys/user v0.4.0 // indirect - github.com/moby/sys/userns v0.1.0 // indirect - github.com/moby/term v0.5.0 // indirect - github.com/morikuni/aec v1.0.0 // indirect - github.com/opencontainers/go-digest v1.0.0 // indirect - github.com/opencontainers/image-spec v1.1.1 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect - github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/prometheus/client_golang v1.11.1 // indirect github.com/prometheus/client_model v0.2.0 // indirect github.com/prometheus/common v0.26.0 // indirect github.com/prometheus/procfs v0.6.0 // indirect - github.com/shirou/gopsutil/v4 v4.25.6 // indirect - github.com/sirupsen/logrus v1.9.3 // indirect - github.com/tklauser/go-sysconf v0.3.12 // indirect - github.com/tklauser/numcpus v0.6.1 // indirect + github.com/rogpeppe/go-internal v1.14.1 // indirect github.com/twmb/murmur3 v1.1.8 // indirect github.com/uber-go/tally v3.5.8+incompatible // indirect github.com/uber/tchannel-go v1.34.4 // indirect - github.com/yusufpapurcu/wmi v1.2.4 // indirect - go.opentelemetry.io/auto/sdk v1.2.1 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect - go.opentelemetry.io/otel v1.39.0 // indirect - go.opentelemetry.io/otel/metric v1.39.0 // indirect - go.opentelemetry.io/otel/trace v1.39.0 // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/dig v1.17.1 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/net/metrics v1.4.0 // indirect go.uber.org/thriftrw v1.32.0 // indirect - golang.org/x/crypto v0.47.0 // indirect golang.org/x/exp/typeparams v0.0.0-20221208152030-732eee02a75a // indirect golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 // indirect golang.org/x/mod v0.32.0 // indirect @@ -98,8 +52,8 @@ require ( golang.org/x/text v0.34.0 // indirect golang.org/x/tools v0.41.0 // indirect golang.org/x/tools/go/expect v0.1.1-deprecated // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20241230172942-26aa7a208def // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20241230172942-26aa7a208def // indirect + gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/yaml.v3 v3.0.1 // indirect honnef.co/go/tools v0.4.3 // indirect ) diff --git a/go.sum b/go.sum index 6a49fdc7..60c09798 100644 --- a/go.sum +++ b/go.sum @@ -1,19 +1,11 @@ cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= -dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= -github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk= -github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= -github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= -github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak= github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU= github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU= -github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= -github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -27,53 +19,20 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bmizerany/perks v0.0.0-20141205001514-d9a9656a3a4b h1:AP/Y7sqYicnjGDfD5VcY4CIfh1hRXBUavxrvELjTiOE= github.com/bmizerany/perks v0.0.0-20141205001514-d9a9656a3a4b/go.mod h1:ac9efd0D1fsDb3EJvhqgXRbFx7bs2wqZ10HQPeU8U/Q= -github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= -github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= -github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= -github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= -github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= -github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= -github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= -github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A= -github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw= -github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GKorA= -github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= -github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= -github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= -github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= -github.com/docker/docker v28.5.1+incompatible h1:Bm8DchhSD2J6PsFzxC35TZo4TLGR2PdW/E69rU45NhM= -github.com/docker/docker v28.5.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/go-connections v0.6.0 h1:LlMG9azAe1TqfR7sO+NJttz1gy6KO7VJBh+pMmjSD94= -github.com/docker/go-connections v0.6.0/go.mod h1:AahvXYshr6JgfUJGdDCs2b5EZG/vmaMAntpSFH5BFKE= -github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= -github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/ebitengine/purego v0.8.4 h1:CF7LEKg5FFOsASUj0+QwaXf8Ht6TlFxg09+S9wz0omw= -github.com/ebitengine/purego v0.8.4/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= github.com/fatih/structtag v1.2.0 h1:/OdNE99OxoI/PqaW/SuSK9uxxT3f/tcSZgon/ssNSx4= github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94= -github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= -github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= -github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= -github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= -github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= -github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= -github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-sql-driver/mysql v1.9.3 h1:U/N249h2WzJ3Ukj8SowVFjdtZKfu9vlLZxjPXV1aweo= github.com/go-sql-driver/mysql v1.9.3/go.mod h1:qn46aNg1333BRMNU69Lq93t8du/dwxI64Gl8i5p1WMU= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= @@ -106,15 +65,10 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= -github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0 h1:TmHmbvxPmaegwhDubVz0lICL0J5Ka2vwTzhoePEXsGE= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0/go.mod h1:qztMSjm835F2bXf+5HKAPIS5qsmQDqZna/PgVt4rWtI= github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LFvc= github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= @@ -129,63 +83,33 @@ github.com/kisielk/errcheck v1.7.0 h1:+SbscKmWJ5mOK/bO1zS60F5I9WwZDWOfRsC4RwfwRV github.com/kisielk/errcheck v1.7.0/go.mod h1:1kLL+jV4e+CFfueBmI1dSK2ADDyQnlrnrY/FqKluHJQ= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kisielk/sqlstruct v0.0.0-20201105191214-5f3e10d3ab46/go.mod h1:yyMNCyc/Ib3bDTKd379tNMpB/7/H5TjM2Y9QJ5THLbE= -github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= -github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= -github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= -github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8SYxI99mE= -github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= -github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= -github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= -github.com/moby/go-archive v0.1.0 h1:Kk/5rdW/g+H8NHdJW2gsXyZ7UnzvJNOy6VKJqueWdcQ= -github.com/moby/go-archive v0.1.0/go.mod h1:G9B+YoujNohJmrIYFBpSd54GTUB4lt9S+xVQvsJyFuo= -github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk= -github.com/moby/patternmatcher v0.6.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= -github.com/moby/sys/atomicwriter v0.1.0 h1:kw5D/EqkBwsBFi0ss9v1VG3wIkVhzGvLklJ+w3A14Sw= -github.com/moby/sys/atomicwriter v0.1.0/go.mod h1:Ul8oqv2ZMNHOceF643P6FKPXeCmYtlQMvpizfsSoaWs= -github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU= -github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko= -github.com/moby/sys/user v0.4.0 h1:jhcMKit7SA80hivmFJcbB1vqmw//wU61Zdui2eQXuMs= -github.com/moby/sys/user v0.4.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs= -github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g= -github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28= -github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= -github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= -github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= -github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= -github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= -github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/prashantv/protectmem v0.0.0-20171002184600-e20412882b3a h1:AA9vgIBDjMHPC2McaGPojgV2dcI78ZC0TLNhYCXEKH8= github.com/prashantv/protectmem v0.0.0-20171002184600-e20412882b3a/go.mod h1:lzZQ3Noex5pfAy7mkAeCjcBDteYU85uWWnJ/y6gKU8k= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= @@ -216,33 +140,18 @@ github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0t github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/samuel/go-thrift v0.0.0-20191111193933-5165175b40af h1:EiWVfh8mr40yFZEui2oF0d45KgH48PkB2H0Z0GANvSI= github.com/samuel/go-thrift v0.0.0-20191111193933-5165175b40af/go.mod h1:Vrkh1pnjV9Bl8c3P9zH0/D4NlOHWP5d4/hF4YTULaec= -github.com/shirou/gopsutil/v4 v4.25.6 h1:kLysI2JsKorfaFPcYmcJqbzROzsBWEOAtw6A7dIfqXs= -github.com/shirou/gopsutil/v4 v4.25.6/go.mod h1:PfybzyydfZcN+JMMjkF6Zb8Mq1A/VcogFFg7hj50W9c= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= -github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/streadway/quantile v0.0.0-20220407130108-4246515d968d h1:X4+kt6zM/OVO6gbJdAfJR60MGPsqCzbtXNnjoGqdfAs= github.com/streadway/quantile v0.0.0-20220407130108-4246515d968d/go.mod h1:lbP8tGiBjZ5YWIc2fzuRpTaz0b/53vT6PEs3QuAWzuU= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= -github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= -github.com/testcontainers/testcontainers-go v0.40.0 h1:pSdJYLOVgLE8YdUY2FHQ1Fxu+aMnb6JfVz1mxk7OeMU= -github.com/testcontainers/testcontainers-go v0.40.0/go.mod h1:FSXV5KQtX2HAMlm7U3APNyLkkap35zNLxukw9oBi/MY= -github.com/testcontainers/testcontainers-go/modules/mysql v0.40.0 h1:P9Txfy5Jothx2wFdcus0QoSmX/PKSIXZxrTbZPVJswA= -github.com/testcontainers/testcontainers-go/modules/mysql v0.40.0/go.mod h1:oZPHHqJqXG7FD8OB/yWH7gLnDvZUlFHAVJNrGftL+eg= -github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= -github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= -github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= -github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= github.com/twmb/murmur3 v1.1.8 h1:8Yt9taO/WN3l08xErzjeschgZU2QSrwm1kclYq+0aRg= github.com/twmb/murmur3 v1.1.8/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/uber-go/mapdecode v1.0.0 h1:euUEFM9KnuCa1OBixz1xM+FIXmpixyay5DLymceOVrU= @@ -261,26 +170,6 @@ github.com/uber/tchannel-go v1.34.4/go.mod h1:ERHDsQa50nNJxV8Mm6V4nxPWyGvGxiV+T/ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= -github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= -go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= -go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw= -go.opentelemetry.io/otel v1.39.0 h1:8yPrr/S0ND9QEfTfdP9V+SiwT4E0G7Y5MO7p85nis48= -go.opentelemetry.io/otel v1.39.0/go.mod h1:kLlFTywNWrFyEdH0oj2xK0bFYZtHRYUdv1NklR/tgc8= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 h1:Mne5On7VWdx7omSrSSZvM4Kw7cS7NQkOOmLcgscI51U= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0/go.mod h1:IPtUMKL4O3tH5y+iXVyAXqpAwMuzC1IrxVS81rummfE= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0 h1:IeMeyr1aBvBiPVYihXIaeIZba6b8E1bYp7lbdxK8CQg= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0/go.mod h1:oVdCUtjq9MK9BlS7TtucsQwUcXcymNiEDjgDD2jMtZU= -go.opentelemetry.io/otel/metric v1.39.0 h1:d1UzonvEZriVfpNKEVmHXbdf909uGTOQjA0HF0Ls5Q0= -go.opentelemetry.io/otel/metric v1.39.0/go.mod h1:jrZSWL33sD7bBxg1xjrqyDjnuzTUB0x1nBERXd7Ftcs= -go.opentelemetry.io/otel/sdk v1.19.0 h1:6USY6zH+L8uMH8L3t1enZPR3WFEmSTADlqldyHtJi3o= -go.opentelemetry.io/otel/sdk v1.19.0/go.mod h1:NedEbbS4w3C6zElbLdPJKOpJQOrGUJ+GfzpjUvI0v1A= -go.opentelemetry.io/otel/trace v1.39.0 h1:2d2vfpEDmCJ5zVYz7ijaJdOF59xLomrvj7bjt6/qCJI= -go.opentelemetry.io/otel/trace v1.39.0/go.mod h1:88w4/PnZSazkGzz/w84VHpQafiU4EtqqlVdxWy+rNOA= -go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= -go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= go.uber.org/atomic v1.5.1/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= @@ -309,8 +198,6 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.47.0 h1:V6e3FRj+n4dbpw86FJ8Fv7XVOql7TEwpHapKoMJ/GO8= -golang.org/x/crypto v0.47.0/go.mod h1:ff3Y9VzzKbwSSEzWqJsJVBnWmRwRSHt/6Op5n9bQc4A= golang.org/x/exp/typeparams v0.0.0-20221208152030-732eee02a75a h1:Jw5wfR+h9mnIYH+OtGT2im5wV1YGGDora5vTv/aa5bE= golang.org/x/exp/typeparams v0.0.0-20221208152030-732eee02a75a/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= @@ -353,7 +240,6 @@ golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200117145432-59e60aa80a0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -362,20 +248,13 @@ golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ= golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.39.0 h1:RclSuaJf32jOqZz74CkPA9qFuVTX7vhLlpfj/IGWlqY= -golang.org/x/term v0.39.0/go.mod h1:yxzUCTP/U+FzoxfdKmLaA0RV1WgE0VY7hXBwKtY/4ww= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -383,8 +262,6 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk= golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA= -golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 h1:vVKdlvoWBphwdxWKrFZEuM0kGgGLxUOYcY4U/2Vjg44= -golang.org/x/time v0.0.0-20220210224613-90d013bbcef8/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= @@ -406,8 +283,6 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180518175338-11a468237815/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto/googleapis/api v0.0.0-20241230172942-26aa7a208def h1:0Km0hi+g2KXbXL0+riZzSCKz23f4MmwicuEb00JeonI= -google.golang.org/genproto/googleapis/api v0.0.0-20241230172942-26aa7a208def/go.mod h1:u2DoMSpCXjrzqLdobRccQMc9wrnMAJ1DLng0a2yqM2Q= google.golang.org/genproto/googleapis/rpc v0.0.0-20241230172942-26aa7a208def h1:4P81qv5JXI/sDNae2ClVx88cgDDA6DPilADkG9tYKz8= google.golang.org/genproto/googleapis/rpc v0.0.0-20241230172942-26aa7a208def/go.mod h1:bdAgzvd4kFrpykc5/AC2eLUiegK9T/qxZHD4hXYf/ho= google.golang.org/grpc v1.12.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= @@ -436,11 +311,8 @@ gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= -gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.4.3 h1:o/n5/K5gXqk8Gozvs2cnL0F2S1/g1vcGCAx2vETjITw= honnef.co/go/tools v0.4.3/go.mod h1:36ZgoUOrqOk1GxwHhyryEkq8FQWkUO2xGuSMhUCcdvA= diff --git a/orchestrator/integration_test/BUILD.bazel b/orchestrator/integration_test/BUILD.bazel deleted file mode 100644 index 545fa60c..00000000 --- a/orchestrator/integration_test/BUILD.bazel +++ /dev/null @@ -1,19 +0,0 @@ -# gazelle:ignore - -load("@rules_go//go:def.bzl", "go_test") - -go_test( - name = "integration_test_test", - srcs = ["ping_test.go"], - tags = [ - "integration", - "manual", - ], - deps = [ - "//orchestrator/protopb", - "@com_github_stretchr_testify//assert", - "@com_github_stretchr_testify//require", - "@org_golang_google_grpc//:grpc", - "@org_golang_google_grpc//credentials/insecure", - ], -) diff --git a/orchestrator/integration_test/ping_test.go b/orchestrator/integration_test/ping_test.go deleted file mode 100644 index 19126c93..00000000 --- a/orchestrator/integration_test/ping_test.go +++ /dev/null @@ -1,92 +0,0 @@ -package orchestrator_test - -import ( - "context" - "fmt" - "os" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - pb "github.com/uber/submitqueue/orchestrator/protopb" - "google.golang.org/grpc" - "google.golang.org/grpc/credentials/insecure" -) - -const ( - defaultTimeout = 5 * time.Second - serverReadyTimeout = 30 * time.Second - retryInterval = 500 * time.Millisecond -) - -// TestPingAPI tests the Orchestrator service Ping API -func TestPingAPI(t *testing.T) { - addr := getEnvOrDefault("ORCHESTRATOR_ADDR", "localhost:8082") - - // Wait for server to be ready - conn, err := waitForServer(t, addr, serverReadyTimeout) - require.NoError(t, err, "Orchestrator server not ready") - defer conn.Close() - - client := pb.NewSubmitQueueOrchestratorClient(conn) - ctx, cancel := context.WithTimeout(context.Background(), defaultTimeout) - defer cancel() - - // Test Ping API - req := &pb.PingRequest{ - Message: "integration test", - } - - resp, err := client.Ping(ctx, req) - require.NoError(t, err, "Ping failed") - - // Validate response - assert.NotEmpty(t, resp.Message, "Response message should not be empty") - assert.Equal(t, "orchestrator", resp.ServiceName) - assert.NotZero(t, resp.Timestamp, "Timestamp should not be zero") - assert.NotEmpty(t, resp.Hostname, "Hostname should not be empty") - - t.Logf("Orchestrator Ping test passed:") - t.Logf(" Message: %s", resp.Message) - t.Logf(" Service: %s", resp.ServiceName) - t.Logf(" Timestamp: %d", resp.Timestamp) - t.Logf(" Hostname: %s", resp.Hostname) -} - -// waitForServer waits for a gRPC server to become ready -func waitForServer(t *testing.T, addr string, timeout time.Duration) (*grpc.ClientConn, error) { - t.Helper() - - deadline := time.Now().Add(timeout) - var lastErr error - - for time.Now().Before(deadline) { - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) - conn, err := grpc.DialContext( - ctx, - addr, - grpc.WithTransportCredentials(insecure.NewCredentials()), - grpc.WithBlock(), - ) - cancel() - - if err == nil { - t.Logf("Server at %s is ready", addr) - return conn, nil - } - - lastErr = err - time.Sleep(retryInterval) - } - - return nil, fmt.Errorf("server at %s not ready after %v: %w", addr, timeout, lastErr) -} - -// getEnvOrDefault returns the value of an environment variable or a default value -func getEnvOrDefault(key, defaultValue string) string { - if value := os.Getenv(key); value != "" { - return value - } - return defaultValue -} diff --git a/speculator/README.md b/speculator/README.md deleted file mode 100644 index dfec33fc..00000000 --- a/speculator/README.md +++ /dev/null @@ -1 +0,0 @@ -SubmitQueue Speculator diff --git a/speculator/controller/BUILD.bazel b/speculator/controller/BUILD.bazel deleted file mode 100644 index 1884867b..00000000 --- a/speculator/controller/BUILD.bazel +++ /dev/null @@ -1,26 +0,0 @@ -load("@rules_go//go:def.bzl", "go_library", "go_test") - -go_library( - name = "controller", - srcs = ["ping.go"], - importpath = "github.com/uber/submitqueue/speculator/controller", - visibility = ["//visibility:public"], - deps = [ - "//speculator/protopb", - "@com_github_uber_go_tally_v4//:tally", - "@org_uber_go_zap//:zap", - ], -) - -go_test( - name = "controller_test", - srcs = ["ping_test.go"], - embed = [":controller"], - deps = [ - "//speculator/protopb", - "@com_github_stretchr_testify//assert", - "@com_github_stretchr_testify//require", - "@com_github_uber_go_tally_v4//:tally", - "@org_uber_go_zap//:zap", - ], -) diff --git a/speculator/controller/ping.go b/speculator/controller/ping.go deleted file mode 100644 index 6f91b58b..00000000 --- a/speculator/controller/ping.go +++ /dev/null @@ -1,58 +0,0 @@ -package controller - -import ( - "context" - "os" - "time" - - "github.com/uber-go/tally/v4" - pb "github.com/uber/submitqueue/speculator/protopb" - "go.uber.org/zap" -) - -// PingController handles ping business logic for the speculator -type PingController struct { - logger *zap.Logger - metricsScope tally.Scope -} - -// NewPingController creates a new instance of the speculator ping controller -func NewPingController(logger *zap.Logger, scope tally.Scope) *PingController { - return &PingController{ - logger: logger, - metricsScope: scope, - } -} - -// Ping handles the ping request and returns a response -func (c *PingController) Ping(ctx context.Context, req *pb.PingRequest) (*pb.PingResponse, error) { - start := time.Now() - defer func() { - c.metricsScope.Timer("ping_latency").Record(time.Since(start)) - }() - - c.metricsScope.Counter("ping_requests_total").Inc(1) - - message := "pong" - isEcho := false - if req.Message != "" { - message = "echo: " + req.Message - isEcho = true - c.metricsScope.Counter("echo_requests_total").Inc(1) - } - - hostname, _ := os.Hostname() - - c.logger.Info("ping request received", - zap.String("message", req.Message), - zap.Bool("is_echo", isEcho), - zap.String("hostname", hostname), - ) - - return &pb.PingResponse{ - Message: message, - ServiceName: "speculator", - Timestamp: time.Now().Unix(), - Hostname: hostname, - }, nil -} diff --git a/speculator/controller/ping_test.go b/speculator/controller/ping_test.go deleted file mode 100644 index 80e60f01..00000000 --- a/speculator/controller/ping_test.go +++ /dev/null @@ -1,90 +0,0 @@ -package controller - -import ( - "context" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/uber-go/tally/v4" - pb "github.com/uber/submitqueue/speculator/protopb" - "go.uber.org/zap" -) - -func TestNewPingController(t *testing.T) { - controller := NewPingController(zap.NewNop(), tally.NoopScope) - require.NotNil(t, controller) -} - -func TestPing_DefaultMessage(t *testing.T) { - controller := NewPingController(zap.NewNop(), tally.NoopScope) - ctx := context.Background() - - req := &pb.PingRequest{} - resp, err := controller.Ping(ctx, req) - - require.NoError(t, err) - assert.Equal(t, "pong", resp.Message) -} - -func TestPing_CustomMessage(t *testing.T) { - controller := NewPingController(zap.NewNop(), tally.NoopScope) - ctx := context.Background() - - testCases := []struct { - name string - input string - expected string - }{ - {"simple message", "hello", "echo: hello"}, - {"message with spaces", "hello world", "echo: hello world"}, - {"special characters", "test!@#", "echo: test!@#"}, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - req := &pb.PingRequest{Message: tc.input} - resp, err := controller.Ping(ctx, req) - - require.NoError(t, err) - assert.Equal(t, tc.expected, resp.Message) - }) - } -} - -func TestPing_ServiceName(t *testing.T) { - controller := NewPingController(zap.NewNop(), tally.NoopScope) - ctx := context.Background() - - req := &pb.PingRequest{} - resp, err := controller.Ping(ctx, req) - - require.NoError(t, err) - assert.Equal(t, "speculator", resp.ServiceName) -} - -func TestPing_Timestamp(t *testing.T) { - controller := NewPingController(zap.NewNop(), tally.NoopScope) - ctx := context.Background() - - before := time.Now().Unix() - req := &pb.PingRequest{} - resp, err := controller.Ping(ctx, req) - after := time.Now().Unix() - - require.NoError(t, err) - assert.GreaterOrEqual(t, resp.Timestamp, before) - assert.LessOrEqual(t, resp.Timestamp, after) -} - -func TestPing_Hostname(t *testing.T) { - controller := NewPingController(zap.NewNop(), tally.NoopScope) - ctx := context.Background() - - req := &pb.PingRequest{} - resp, err := controller.Ping(ctx, req) - - require.NoError(t, err) - assert.NotEmpty(t, resp.Hostname) -} diff --git a/speculator/integration_test/BUILD.bazel b/speculator/integration_test/BUILD.bazel deleted file mode 100644 index bc9b096d..00000000 --- a/speculator/integration_test/BUILD.bazel +++ /dev/null @@ -1,19 +0,0 @@ -# gazelle:ignore - -load("@rules_go//go:def.bzl", "go_test") - -go_test( - name = "integration_test_test", - srcs = ["ping_test.go"], - tags = [ - "integration", - "manual", - ], - deps = [ - "//speculator/protopb", - "@com_github_stretchr_testify//assert", - "@com_github_stretchr_testify//require", - "@org_golang_google_grpc//:grpc", - "@org_golang_google_grpc//credentials/insecure", - ], -) diff --git a/speculator/integration_test/ping_test.go b/speculator/integration_test/ping_test.go deleted file mode 100644 index 3e550826..00000000 --- a/speculator/integration_test/ping_test.go +++ /dev/null @@ -1,92 +0,0 @@ -package speculator_test - -import ( - "context" - "fmt" - "os" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - pb "github.com/uber/submitqueue/speculator/protopb" - "google.golang.org/grpc" - "google.golang.org/grpc/credentials/insecure" -) - -const ( - defaultTimeout = 5 * time.Second - serverReadyTimeout = 30 * time.Second - retryInterval = 500 * time.Millisecond -) - -// TestPingAPI tests the Speculator service Ping API -func TestPingAPI(t *testing.T) { - addr := getEnvOrDefault("SPECULATOR_ADDR", "localhost:8083") - - // Wait for server to be ready - conn, err := waitForServer(t, addr, serverReadyTimeout) - require.NoError(t, err, "Speculator server not ready") - defer conn.Close() - - client := pb.NewSubmitQueueSpeculatorClient(conn) - ctx, cancel := context.WithTimeout(context.Background(), defaultTimeout) - defer cancel() - - // Test Ping API - req := &pb.PingRequest{ - Message: "integration test", - } - - resp, err := client.Ping(ctx, req) - require.NoError(t, err, "Ping failed") - - // Validate response - assert.NotEmpty(t, resp.Message, "Response message should not be empty") - assert.Equal(t, "speculator", resp.ServiceName) - assert.NotZero(t, resp.Timestamp, "Timestamp should not be zero") - assert.NotEmpty(t, resp.Hostname, "Hostname should not be empty") - - t.Logf("Speculator Ping test passed:") - t.Logf(" Message: %s", resp.Message) - t.Logf(" Service: %s", resp.ServiceName) - t.Logf(" Timestamp: %d", resp.Timestamp) - t.Logf(" Hostname: %s", resp.Hostname) -} - -// waitForServer waits for a gRPC server to become ready -func waitForServer(t *testing.T, addr string, timeout time.Duration) (*grpc.ClientConn, error) { - t.Helper() - - deadline := time.Now().Add(timeout) - var lastErr error - - for time.Now().Before(deadline) { - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) - conn, err := grpc.DialContext( - ctx, - addr, - grpc.WithTransportCredentials(insecure.NewCredentials()), - grpc.WithBlock(), - ) - cancel() - - if err == nil { - t.Logf("Server at %s is ready", addr) - return conn, nil - } - - lastErr = err - time.Sleep(retryInterval) - } - - return nil, fmt.Errorf("server at %s not ready after %v: %w", addr, timeout, lastErr) -} - -// getEnvOrDefault returns the value of an environment variable or a default value -func getEnvOrDefault(key, defaultValue string) string { - if value := os.Getenv(key); value != "" { - return value - } - return defaultValue -} diff --git a/speculator/proto/BUILD.bazel b/speculator/proto/BUILD.bazel deleted file mode 100644 index 65de9c86..00000000 --- a/speculator/proto/BUILD.bazel +++ /dev/null @@ -1,35 +0,0 @@ -load("@rules_go//go:def.bzl", "go_library") -load("@rules_go//proto:def.bzl", "go_proto_library") -load("@rules_proto//proto:defs.bzl", "proto_library") - -proto_library( - name = "speculatorpb_proto", - srcs = ["speculator.proto"], - visibility = ["//visibility:public"], -) - -# keep -go_proto_library( - name = "speculatorpb_go_proto", - compilers = [ - "@rules_go//proto:go_proto", - "@rules_go//proto:go_grpc_v2", - ], - importpath = "github.com/uber/submitqueue/speculator/proto", - proto = ":speculatorpb_proto", - visibility = ["//visibility:public"], -) - -go_library( - name = "proto", - embed = [":speculatorpb_go_proto"], - importpath = "github.com/uber/submitqueue/speculator/proto", - visibility = ["//visibility:public"], -) - -go_library( - name = "protopb", - embed = [":speculatorpb_go_proto"], - importpath = "github.com/uber/submitqueue/speculator/protopb", - visibility = ["//visibility:public"], -) diff --git a/speculator/proto/speculator.proto b/speculator/proto/speculator.proto deleted file mode 100644 index 1746aa3c..00000000 --- a/speculator/proto/speculator.proto +++ /dev/null @@ -1,32 +0,0 @@ -syntax = "proto3"; - -package uber.devexp.submitqueue.speculator; - -option go_package = "github.com/uber/submitqueue/speculator/protopb"; -option java_multiple_files = true; -option java_outer_classname = "SpeculatorProto"; -option java_package = "com.uber.devexp.submitqueue.speculator"; - -// PingRequest is the request for the Ping method -message PingRequest { - // Optional message to include in the ping - string message = 1; -} - -// PingResponse is the response for the Ping method -message PingResponse { - // The response message - string message = 1; - // The service name that handled the request - string service_name = 2; - // Timestamp of when the ping was received - int64 timestamp = 3; - // Hostname of the server that handled the request - string hostname = 4; -} - -// SubmitQueueSpeculator provides the speculator API -service SubmitQueueSpeculator { - // Ping returns a response indicating the service is alive - rpc Ping(PingRequest) returns (PingResponse) {} -} diff --git a/speculator/protopb/BUILD.bazel b/speculator/protopb/BUILD.bazel deleted file mode 100644 index f4bfd2b1..00000000 --- a/speculator/protopb/BUILD.bazel +++ /dev/null @@ -1,27 +0,0 @@ -load("@rules_go//go:def.bzl", "go_library") - -go_library( - name = "protopb", - srcs = [ - "speculator.pb.go", - "speculator.pb.yarpc.go", - "speculator_grpc.pb.go", - ], - importpath = "github.com/uber/submitqueue/speculator/protopb", - visibility = ["//visibility:public"], - deps = [ - "@com_github_gogo_protobuf//jsonpb", - "@com_github_gogo_protobuf//proto", - "@org_golang_google_grpc//:grpc", - "@org_golang_google_grpc//codes", - "@org_golang_google_grpc//status", - "@org_golang_google_protobuf//reflect/protoreflect", - "@org_golang_google_protobuf//runtime/protoimpl", - "@org_uber_go_fx//:fx", - "@org_uber_go_yarpc//:yarpc", - "@org_uber_go_yarpc//api/transport", - "@org_uber_go_yarpc//api/x/restriction", - "@org_uber_go_yarpc//encoding/protobuf", - "@org_uber_go_yarpc//encoding/protobuf/reflection", - ], -) diff --git a/speculator/protopb/speculator.pb.go b/speculator/protopb/speculator.pb.go deleted file mode 100644 index 479fb8fd..00000000 --- a/speculator/protopb/speculator.pb.go +++ /dev/null @@ -1,208 +0,0 @@ -// Code generated by protoc-gen-go. DO NOT EDIT. -// versions: -// protoc-gen-go v1.36.11 -// protoc v3.21.12 -// source: speculator.proto - -package protopb - -import ( - protoreflect "google.golang.org/protobuf/reflect/protoreflect" - protoimpl "google.golang.org/protobuf/runtime/protoimpl" - reflect "reflect" - sync "sync" - unsafe "unsafe" -) - -const ( - // Verify that this generated code is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) - // Verify that runtime/protoimpl is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) -) - -// PingRequest is the request for the Ping method -type PingRequest struct { - state protoimpl.MessageState `protogen:"open.v1"` - // Optional message to include in the ping - Message string `protobuf:"bytes,1,opt,name=message,proto3" json:"message,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache -} - -func (x *PingRequest) Reset() { - *x = PingRequest{} - mi := &file_speculator_proto_msgTypes[0] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) -} - -func (x *PingRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*PingRequest) ProtoMessage() {} - -func (x *PingRequest) ProtoReflect() protoreflect.Message { - mi := &file_speculator_proto_msgTypes[0] - if x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use PingRequest.ProtoReflect.Descriptor instead. -func (*PingRequest) Descriptor() ([]byte, []int) { - return file_speculator_proto_rawDescGZIP(), []int{0} -} - -func (x *PingRequest) GetMessage() string { - if x != nil { - return x.Message - } - return "" -} - -// PingResponse is the response for the Ping method -type PingResponse struct { - state protoimpl.MessageState `protogen:"open.v1"` - // The response message - Message string `protobuf:"bytes,1,opt,name=message,proto3" json:"message,omitempty"` - // The service name that handled the request - ServiceName string `protobuf:"bytes,2,opt,name=service_name,json=serviceName,proto3" json:"service_name,omitempty"` - // Timestamp of when the ping was received - Timestamp int64 `protobuf:"varint,3,opt,name=timestamp,proto3" json:"timestamp,omitempty"` - // Hostname of the server that handled the request - Hostname string `protobuf:"bytes,4,opt,name=hostname,proto3" json:"hostname,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache -} - -func (x *PingResponse) Reset() { - *x = PingResponse{} - mi := &file_speculator_proto_msgTypes[1] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) -} - -func (x *PingResponse) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*PingResponse) ProtoMessage() {} - -func (x *PingResponse) ProtoReflect() protoreflect.Message { - mi := &file_speculator_proto_msgTypes[1] - if x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use PingResponse.ProtoReflect.Descriptor instead. -func (*PingResponse) Descriptor() ([]byte, []int) { - return file_speculator_proto_rawDescGZIP(), []int{1} -} - -func (x *PingResponse) GetMessage() string { - if x != nil { - return x.Message - } - return "" -} - -func (x *PingResponse) GetServiceName() string { - if x != nil { - return x.ServiceName - } - return "" -} - -func (x *PingResponse) GetTimestamp() int64 { - if x != nil { - return x.Timestamp - } - return 0 -} - -func (x *PingResponse) GetHostname() string { - if x != nil { - return x.Hostname - } - return "" -} - -var File_speculator_proto protoreflect.FileDescriptor - -const file_speculator_proto_rawDesc = "" + - "\n" + - "\x10speculator.proto\x12\"uber.devexp.submitqueue.speculator\"'\n" + - "\vPingRequest\x12\x18\n" + - "\amessage\x18\x01 \x01(\tR\amessage\"\x85\x01\n" + - "\fPingResponse\x12\x18\n" + - "\amessage\x18\x01 \x01(\tR\amessage\x12!\n" + - "\fservice_name\x18\x02 \x01(\tR\vserviceName\x12\x1c\n" + - "\ttimestamp\x18\x03 \x01(\x03R\ttimestamp\x12\x1a\n" + - "\bhostname\x18\x04 \x01(\tR\bhostname2\x84\x01\n" + - "\x15SubmitQueueSpeculator\x12k\n" + - "\x04Ping\x12/.uber.devexp.submitqueue.speculator.PingRequest\x1a0.uber.devexp.submitqueue.speculator.PingResponse\"\x00Bk\n" + - "&com.uber.devexp.submitqueue.speculatorB\x0fSpeculatorProtoP\x01Z.github.com/uber/submitqueue/speculator/protopbb\x06proto3" - -var ( - file_speculator_proto_rawDescOnce sync.Once - file_speculator_proto_rawDescData []byte -) - -func file_speculator_proto_rawDescGZIP() []byte { - file_speculator_proto_rawDescOnce.Do(func() { - file_speculator_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_speculator_proto_rawDesc), len(file_speculator_proto_rawDesc))) - }) - return file_speculator_proto_rawDescData -} - -var file_speculator_proto_msgTypes = make([]protoimpl.MessageInfo, 2) -var file_speculator_proto_goTypes = []any{ - (*PingRequest)(nil), // 0: uber.devexp.submitqueue.speculator.PingRequest - (*PingResponse)(nil), // 1: uber.devexp.submitqueue.speculator.PingResponse -} -var file_speculator_proto_depIdxs = []int32{ - 0, // 0: uber.devexp.submitqueue.speculator.SubmitQueueSpeculator.Ping:input_type -> uber.devexp.submitqueue.speculator.PingRequest - 1, // 1: uber.devexp.submitqueue.speculator.SubmitQueueSpeculator.Ping:output_type -> uber.devexp.submitqueue.speculator.PingResponse - 1, // [1:2] is the sub-list for method output_type - 0, // [0:1] is the sub-list for method input_type - 0, // [0:0] is the sub-list for extension type_name - 0, // [0:0] is the sub-list for extension extendee - 0, // [0:0] is the sub-list for field type_name -} - -func init() { file_speculator_proto_init() } -func file_speculator_proto_init() { - if File_speculator_proto != nil { - return - } - type x struct{} - out := protoimpl.TypeBuilder{ - File: protoimpl.DescBuilder{ - GoPackagePath: reflect.TypeOf(x{}).PkgPath(), - RawDescriptor: unsafe.Slice(unsafe.StringData(file_speculator_proto_rawDesc), len(file_speculator_proto_rawDesc)), - NumEnums: 0, - NumMessages: 2, - NumExtensions: 0, - NumServices: 1, - }, - GoTypes: file_speculator_proto_goTypes, - DependencyIndexes: file_speculator_proto_depIdxs, - MessageInfos: file_speculator_proto_msgTypes, - }.Build() - File_speculator_proto = out.File - file_speculator_proto_goTypes = nil - file_speculator_proto_depIdxs = nil -} diff --git a/speculator/protopb/speculator.pb.yarpc.go b/speculator/protopb/speculator.pb.yarpc.go deleted file mode 100644 index 2c273555..00000000 --- a/speculator/protopb/speculator.pb.yarpc.go +++ /dev/null @@ -1,262 +0,0 @@ -// Code generated by protoc-gen-yarpc-go. DO NOT EDIT. -// source: speculator.proto - -package protopb - -import ( - "context" - "io/ioutil" - "reflect" - - "github.com/gogo/protobuf/jsonpb" - "github.com/gogo/protobuf/proto" - "go.uber.org/fx" - "go.uber.org/yarpc" - "go.uber.org/yarpc/api/transport" - "go.uber.org/yarpc/api/x/restriction" - "go.uber.org/yarpc/encoding/protobuf" - "go.uber.org/yarpc/encoding/protobuf/reflection" -) - -var _ = ioutil.NopCloser - -// SubmitQueueSpeculatorYARPCClient is the YARPC client-side interface for the SubmitQueueSpeculator service. -type SubmitQueueSpeculatorYARPCClient interface { - Ping(context.Context, *PingRequest, ...yarpc.CallOption) (*PingResponse, error) -} - -func newSubmitQueueSpeculatorYARPCClient(clientConfig transport.ClientConfig, anyResolver jsonpb.AnyResolver, options ...protobuf.ClientOption) SubmitQueueSpeculatorYARPCClient { - return &_SubmitQueueSpeculatorYARPCCaller{protobuf.NewStreamClient( - protobuf.ClientParams{ - ServiceName: "uber.devexp.submitqueue.speculator.SubmitQueueSpeculator", - ClientConfig: clientConfig, - AnyResolver: anyResolver, - Options: options, - }, - )} -} - -// NewSubmitQueueSpeculatorYARPCClient builds a new YARPC client for the SubmitQueueSpeculator service. -func NewSubmitQueueSpeculatorYARPCClient(clientConfig transport.ClientConfig, options ...protobuf.ClientOption) SubmitQueueSpeculatorYARPCClient { - return newSubmitQueueSpeculatorYARPCClient(clientConfig, nil, options...) -} - -// SubmitQueueSpeculatorYARPCServer is the YARPC server-side interface for the SubmitQueueSpeculator service. -type SubmitQueueSpeculatorYARPCServer interface { - Ping(context.Context, *PingRequest) (*PingResponse, error) -} - -type buildSubmitQueueSpeculatorYARPCProceduresParams struct { - Server SubmitQueueSpeculatorYARPCServer - AnyResolver jsonpb.AnyResolver -} - -func buildSubmitQueueSpeculatorYARPCProcedures(params buildSubmitQueueSpeculatorYARPCProceduresParams) []transport.Procedure { - handler := &_SubmitQueueSpeculatorYARPCHandler{params.Server} - return protobuf.BuildProcedures( - protobuf.BuildProceduresParams{ - ServiceName: "uber.devexp.submitqueue.speculator.SubmitQueueSpeculator", - UnaryHandlerParams: []protobuf.BuildProceduresUnaryHandlerParams{ - { - MethodName: "Ping", - Handler: protobuf.NewUnaryHandler( - protobuf.UnaryHandlerParams{ - Handle: handler.Ping, - NewRequest: newSubmitQueueSpeculatorServicePingYARPCRequest, - AnyResolver: params.AnyResolver, - }, - ), - }, - }, - OnewayHandlerParams: []protobuf.BuildProceduresOnewayHandlerParams{}, - StreamHandlerParams: []protobuf.BuildProceduresStreamHandlerParams{}, - }, - ) -} - -// BuildSubmitQueueSpeculatorYARPCProcedures prepares an implementation of the SubmitQueueSpeculator service for YARPC registration. -func BuildSubmitQueueSpeculatorYARPCProcedures(server SubmitQueueSpeculatorYARPCServer) []transport.Procedure { - return buildSubmitQueueSpeculatorYARPCProcedures(buildSubmitQueueSpeculatorYARPCProceduresParams{Server: server}) -} - -// FxSubmitQueueSpeculatorYARPCClientParams defines the input -// for NewFxSubmitQueueSpeculatorYARPCClient. It provides the -// paramaters to get a SubmitQueueSpeculatorYARPCClient in an -// Fx application. -type FxSubmitQueueSpeculatorYARPCClientParams struct { - fx.In - - Provider yarpc.ClientConfig - AnyResolver jsonpb.AnyResolver `name:"yarpcfx" optional:"true"` - Restriction restriction.Checker `optional:"true"` -} - -// FxSubmitQueueSpeculatorYARPCClientResult defines the output -// of NewFxSubmitQueueSpeculatorYARPCClient. It provides a -// SubmitQueueSpeculatorYARPCClient to an Fx application. -type FxSubmitQueueSpeculatorYARPCClientResult struct { - fx.Out - - Client SubmitQueueSpeculatorYARPCClient - - // We are using an fx.Out struct here instead of just returning a client - // so that we can add more values or add named versions of the client in - // the future without breaking any existing code. -} - -// NewFxSubmitQueueSpeculatorYARPCClient provides a SubmitQueueSpeculatorYARPCClient -// to an Fx application using the given name for routing. -// -// fx.Provide( -// protopb.NewFxSubmitQueueSpeculatorYARPCClient("service-name"), -// ... -// ) -func NewFxSubmitQueueSpeculatorYARPCClient(name string, options ...protobuf.ClientOption) interface{} { - return func(params FxSubmitQueueSpeculatorYARPCClientParams) FxSubmitQueueSpeculatorYARPCClientResult { - cc := params.Provider.ClientConfig(name) - - if params.Restriction != nil { - if namer, ok := cc.GetUnaryOutbound().(transport.Namer); ok { - if err := params.Restriction.Check(protobuf.Encoding, namer.TransportName()); err != nil { - panic(err.Error()) - } - } - } - - return FxSubmitQueueSpeculatorYARPCClientResult{ - Client: newSubmitQueueSpeculatorYARPCClient(cc, params.AnyResolver, options...), - } - } -} - -// FxSubmitQueueSpeculatorYARPCProceduresParams defines the input -// for NewFxSubmitQueueSpeculatorYARPCProcedures. It provides the -// paramaters to get SubmitQueueSpeculatorYARPCServer procedures in an -// Fx application. -type FxSubmitQueueSpeculatorYARPCProceduresParams struct { - fx.In - - Server SubmitQueueSpeculatorYARPCServer - AnyResolver jsonpb.AnyResolver `name:"yarpcfx" optional:"true"` -} - -// FxSubmitQueueSpeculatorYARPCProceduresResult defines the output -// of NewFxSubmitQueueSpeculatorYARPCProcedures. It provides -// SubmitQueueSpeculatorYARPCServer procedures to an Fx application. -// -// The procedures are provided to the "yarpcfx" value group. -// Dig 1.2 or newer must be used for this feature to work. -type FxSubmitQueueSpeculatorYARPCProceduresResult struct { - fx.Out - - Procedures []transport.Procedure `group:"yarpcfx"` - ReflectionMeta reflection.ServerMeta `group:"yarpcfx"` -} - -// NewFxSubmitQueueSpeculatorYARPCProcedures provides SubmitQueueSpeculatorYARPCServer procedures to an Fx application. -// It expects a SubmitQueueSpeculatorYARPCServer to be present in the container. -// -// fx.Provide( -// protopb.NewFxSubmitQueueSpeculatorYARPCProcedures(), -// ... -// ) -func NewFxSubmitQueueSpeculatorYARPCProcedures() interface{} { - return func(params FxSubmitQueueSpeculatorYARPCProceduresParams) FxSubmitQueueSpeculatorYARPCProceduresResult { - return FxSubmitQueueSpeculatorYARPCProceduresResult{ - Procedures: buildSubmitQueueSpeculatorYARPCProcedures(buildSubmitQueueSpeculatorYARPCProceduresParams{ - Server: params.Server, - AnyResolver: params.AnyResolver, - }), - ReflectionMeta: SubmitQueueSpeculatorReflectionMeta, - } - } -} - -// SubmitQueueSpeculatorReflectionMeta is the reflection server metadata -// required for using the gRPC reflection protocol with YARPC. -// -// See https://github.com/grpc/grpc/blob/master/doc/server-reflection.md. -var SubmitQueueSpeculatorReflectionMeta = reflection.ServerMeta{ - ServiceName: "uber.devexp.submitqueue.speculator.SubmitQueueSpeculator", - FileDescriptors: yarpcFileDescriptorClosure4a6246af296c2143, -} - -type _SubmitQueueSpeculatorYARPCCaller struct { - streamClient protobuf.StreamClient -} - -func (c *_SubmitQueueSpeculatorYARPCCaller) Ping(ctx context.Context, request *PingRequest, options ...yarpc.CallOption) (*PingResponse, error) { - responseMessage, err := c.streamClient.Call(ctx, "Ping", request, newSubmitQueueSpeculatorServicePingYARPCResponse, options...) - if responseMessage == nil { - return nil, err - } - response, ok := responseMessage.(*PingResponse) - if !ok { - return nil, protobuf.CastError(emptySubmitQueueSpeculatorServicePingYARPCResponse, responseMessage) - } - return response, err -} - -type _SubmitQueueSpeculatorYARPCHandler struct { - server SubmitQueueSpeculatorYARPCServer -} - -func (h *_SubmitQueueSpeculatorYARPCHandler) Ping(ctx context.Context, requestMessage proto.Message) (proto.Message, error) { - var request *PingRequest - var ok bool - if requestMessage != nil { - request, ok = requestMessage.(*PingRequest) - if !ok { - return nil, protobuf.CastError(emptySubmitQueueSpeculatorServicePingYARPCRequest, requestMessage) - } - } - response, err := h.server.Ping(ctx, request) - if response == nil { - return nil, err - } - return response, err -} - -func newSubmitQueueSpeculatorServicePingYARPCRequest() proto.Message { - return &PingRequest{} -} - -func newSubmitQueueSpeculatorServicePingYARPCResponse() proto.Message { - return &PingResponse{} -} - -var ( - emptySubmitQueueSpeculatorServicePingYARPCRequest = &PingRequest{} - emptySubmitQueueSpeculatorServicePingYARPCResponse = &PingResponse{} -) - -var yarpcFileDescriptorClosure4a6246af296c2143 = [][]byte{ - // speculator.proto - []byte{ - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x91, 0xc1, 0x4a, 0x33, 0x31, - 0x10, 0xc7, 0xbf, 0x7c, 0x2d, 0x6a, 0xa7, 0x05, 0x25, 0x20, 0x2c, 0xc5, 0x43, 0xdd, 0x83, 0xf6, - 0x94, 0x88, 0xbe, 0x41, 0x1f, 0x40, 0xd6, 0xed, 0xcd, 0x8b, 0x24, 0xeb, 0xb0, 0x0d, 0x35, 0x9b, - 0x74, 0x27, 0x29, 0x3e, 0x80, 0xbe, 0xb7, 0x6c, 0xaa, 0xdd, 0x5e, 0xc4, 0xde, 0x32, 0x93, 0xff, - 0x6f, 0x98, 0x1f, 0x03, 0x17, 0xe4, 0xb1, 0x8a, 0x6f, 0x2a, 0xb8, 0x56, 0xf8, 0xd6, 0x05, 0xc7, - 0xf3, 0xa8, 0xb1, 0x15, 0xaf, 0xb8, 0xc5, 0x77, 0x2f, 0x28, 0x6a, 0x6b, 0xc2, 0x26, 0x62, 0x44, - 0xd1, 0x27, 0xf3, 0x5b, 0x18, 0x17, 0xa6, 0xa9, 0x4b, 0xdc, 0x44, 0xa4, 0xc0, 0x33, 0x38, 0xb5, - 0x48, 0xa4, 0x6a, 0xcc, 0xd8, 0x8c, 0xcd, 0x47, 0xe5, 0x4f, 0x99, 0x7f, 0x32, 0x98, 0xec, 0x92, - 0xe4, 0x5d, 0x43, 0xf8, 0x7b, 0x94, 0x5f, 0xc3, 0x84, 0xb0, 0xdd, 0x9a, 0x0a, 0x5f, 0x1a, 0x65, - 0x31, 0xfb, 0x9f, 0xbe, 0xc7, 0xdf, 0xbd, 0x47, 0x65, 0x91, 0x5f, 0xc1, 0x28, 0x18, 0x8b, 0x14, - 0x94, 0xf5, 0xd9, 0x60, 0xc6, 0xe6, 0x83, 0xb2, 0x6f, 0xf0, 0x29, 0x9c, 0xad, 0x1c, 0x85, 0x04, - 0x0f, 0x13, 0xbc, 0xaf, 0xef, 0x3f, 0x18, 0x5c, 0x2e, 0x93, 0xcb, 0x53, 0xe7, 0xb2, 0xdc, 0xab, - 0xf0, 0x35, 0x0c, 0xbb, 0x05, 0xb9, 0x14, 0x7f, 0x7b, 0x8b, 0x03, 0xe9, 0xe9, 0xdd, 0xf1, 0xc0, - 0xce, 0x3d, 0xff, 0xb7, 0x58, 0xc3, 0x4d, 0xe5, 0xec, 0x11, 0xe0, 0xe2, 0xbc, 0x5f, 0xb1, 0xe8, - 0xce, 0x52, 0xb0, 0x67, 0x51, 0x9b, 0xb0, 0x8a, 0x5a, 0x54, 0xce, 0xca, 0x6e, 0x82, 0x3c, 0x40, - 0x65, 0x8f, 0xca, 0x74, 0x46, 0xaf, 0xf5, 0x49, 0x7a, 0x3c, 0x7c, 0x05, 0x00, 0x00, 0xff, 0xff, - 0x84, 0xb6, 0x23, 0xbd, 0xe3, 0x01, 0x00, 0x00, - }, -} - -func init() { - yarpc.RegisterClientBuilder( - func(clientConfig transport.ClientConfig, structField reflect.StructField) SubmitQueueSpeculatorYARPCClient { - return NewSubmitQueueSpeculatorYARPCClient(clientConfig, protobuf.ClientBuilderOptions(clientConfig, structField)...) - }, - ) -} diff --git a/speculator/protopb/speculator_grpc.pb.go b/speculator/protopb/speculator_grpc.pb.go deleted file mode 100644 index 4cca0f63..00000000 --- a/speculator/protopb/speculator_grpc.pb.go +++ /dev/null @@ -1,127 +0,0 @@ -// Code generated by protoc-gen-go-grpc. DO NOT EDIT. -// versions: -// - protoc-gen-go-grpc v1.6.1 -// - protoc v3.21.12 -// source: speculator.proto - -package protopb - -import ( - context "context" - grpc "google.golang.org/grpc" - codes "google.golang.org/grpc/codes" - status "google.golang.org/grpc/status" -) - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the grpc package it is being compiled against. -// Requires gRPC-Go v1.64.0 or later. -const _ = grpc.SupportPackageIsVersion9 - -const ( - SubmitQueueSpeculator_Ping_FullMethodName = "/uber.devexp.submitqueue.speculator.SubmitQueueSpeculator/Ping" -) - -// SubmitQueueSpeculatorClient is the client API for SubmitQueueSpeculator service. -// -// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. -// -// SubmitQueueSpeculator provides the speculator API -type SubmitQueueSpeculatorClient interface { - // Ping returns a response indicating the service is alive - Ping(ctx context.Context, in *PingRequest, opts ...grpc.CallOption) (*PingResponse, error) -} - -type submitQueueSpeculatorClient struct { - cc grpc.ClientConnInterface -} - -func NewSubmitQueueSpeculatorClient(cc grpc.ClientConnInterface) SubmitQueueSpeculatorClient { - return &submitQueueSpeculatorClient{cc} -} - -func (c *submitQueueSpeculatorClient) Ping(ctx context.Context, in *PingRequest, opts ...grpc.CallOption) (*PingResponse, error) { - cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) - out := new(PingResponse) - err := c.cc.Invoke(ctx, SubmitQueueSpeculator_Ping_FullMethodName, in, out, cOpts...) - if err != nil { - return nil, err - } - return out, nil -} - -// SubmitQueueSpeculatorServer is the server API for SubmitQueueSpeculator service. -// All implementations must embed UnimplementedSubmitQueueSpeculatorServer -// for forward compatibility. -// -// SubmitQueueSpeculator provides the speculator API -type SubmitQueueSpeculatorServer interface { - // Ping returns a response indicating the service is alive - Ping(context.Context, *PingRequest) (*PingResponse, error) - mustEmbedUnimplementedSubmitQueueSpeculatorServer() -} - -// UnimplementedSubmitQueueSpeculatorServer must be embedded to have -// forward compatible implementations. -// -// NOTE: this should be embedded by value instead of pointer to avoid a nil -// pointer dereference when methods are called. -type UnimplementedSubmitQueueSpeculatorServer struct{} - -func (UnimplementedSubmitQueueSpeculatorServer) Ping(context.Context, *PingRequest) (*PingResponse, error) { - return nil, status.Error(codes.Unimplemented, "method Ping not implemented") -} -func (UnimplementedSubmitQueueSpeculatorServer) mustEmbedUnimplementedSubmitQueueSpeculatorServer() {} -func (UnimplementedSubmitQueueSpeculatorServer) testEmbeddedByValue() {} - -// UnsafeSubmitQueueSpeculatorServer may be embedded to opt out of forward compatibility for this service. -// Use of this interface is not recommended, as added methods to SubmitQueueSpeculatorServer will -// result in compilation errors. -type UnsafeSubmitQueueSpeculatorServer interface { - mustEmbedUnimplementedSubmitQueueSpeculatorServer() -} - -func RegisterSubmitQueueSpeculatorServer(s grpc.ServiceRegistrar, srv SubmitQueueSpeculatorServer) { - // If the following call panics, it indicates UnimplementedSubmitQueueSpeculatorServer was - // embedded by pointer and is nil. This will cause panics if an - // unimplemented method is ever invoked, so we test this at initialization - // time to prevent it from happening at runtime later due to I/O. - if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { - t.testEmbeddedByValue() - } - s.RegisterService(&SubmitQueueSpeculator_ServiceDesc, srv) -} - -func _SubmitQueueSpeculator_Ping_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(PingRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(SubmitQueueSpeculatorServer).Ping(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: SubmitQueueSpeculator_Ping_FullMethodName, - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(SubmitQueueSpeculatorServer).Ping(ctx, req.(*PingRequest)) - } - return interceptor(ctx, in, info, handler) -} - -// SubmitQueueSpeculator_ServiceDesc is the grpc.ServiceDesc for SubmitQueueSpeculator service. -// It's only intended for direct use with grpc.RegisterService, -// and not to be introspected or modified (even as a copy) -var SubmitQueueSpeculator_ServiceDesc = grpc.ServiceDesc{ - ServiceName: "uber.devexp.submitqueue.speculator.SubmitQueueSpeculator", - HandlerType: (*SubmitQueueSpeculatorServer)(nil), - Methods: []grpc.MethodDesc{ - { - MethodName: "Ping", - Handler: _SubmitQueueSpeculator_Ping_Handler, - }, - }, - Streams: []grpc.StreamDesc{}, - Metadata: "speculator.proto", -} diff --git a/test/e2e/BUILD.bazel b/test/e2e/BUILD.bazel new file mode 100644 index 00000000..7c20d409 --- /dev/null +++ b/test/e2e/BUILD.bazel @@ -0,0 +1,27 @@ +load("@rules_go//go:def.bzl", "go_test") + +go_test( + name = "e2e_test", + srcs = ["suite_test.go"], + data = [ + "//:MODULE.bazel", + "//:go.mod", + "//example/server:docker-compose.yml", + "//extension/counter/mysql/schema", + "//extension/queue/sql/schema", + "//extension/storage/mysql/schema", + ], + tags = [ + "e2e", + "integration", + ], + deps = [ + "//gateway/protopb", + "//orchestrator/protopb", + "//test/testutil", + "@com_github_stretchr_testify//assert", + "@com_github_stretchr_testify//require", + "@com_github_stretchr_testify//suite", + "@org_golang_google_grpc//:grpc", + ], +) diff --git a/test/e2e/suite_test.go b/test/e2e/suite_test.go new file mode 100644 index 00000000..f05b541f --- /dev/null +++ b/test/e2e/suite_test.go @@ -0,0 +1,134 @@ +package e2e_test + +// E2E Integration Tests +// +// These tests use docker-compose from example/server/docker-compose.yml +// which requires pre-built Linux binaries. +// +// Run with make target (builds binaries + runs test): +// make e2e-test + +import ( + "context" + "database/sql" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + gatewaypb "github.com/uber/submitqueue/gateway/protopb" + orchestratorpb "github.com/uber/submitqueue/orchestrator/protopb" + "github.com/uber/submitqueue/test/testutil" + "google.golang.org/grpc" +) + +type E2EIntegrationSuite struct { + suite.Suite + ctx context.Context + log *testutil.TestLogger + stack *testutil.ComposeStack + gatewayClient gatewaypb.SubmitQueueGatewayClient + orchestratorClient orchestratorpb.SubmitQueueOrchestratorClient + db *sql.DB // App database + queueDB *sql.DB // Queue database +} + +func TestE2EIntegration(t *testing.T) { + suite.Run(t, new(E2EIntegrationSuite)) +} + +func (s *E2EIntegrationSuite) SetupSuite() { + t := s.T() + s.ctx = context.Background() + s.log = testutil.NewTestLogger(t) + + s.log.Logf("Starting E2E integration test suite using docker-compose") + + // Set REPO_ROOT for docker-compose volume mounts and build context + repoRoot := testutil.FindRepoRoot(t) + t.Setenv("REPO_ROOT", repoRoot) + + // Use docker-compose from example/server (full stack) + // NOTE: Assumes Linux binaries are pre-built via make target + composeFile := filepath.Join(repoRoot, "example/server/docker-compose.yml") + s.stack = testutil.NewComposeStack(t, s.log, s.ctx, composeFile, "e2e") + + // Start the compose stack (Gateway + Orchestrator + 2 MySQL DBs) + err := s.stack.Up() + require.NoError(t, err, "failed to start compose stack") + + s.log.Logf("Compose stack started successfully") + + // Connect to application database + s.db, err = s.stack.ConnectMySQLService("mysql-app") + require.NoError(t, err, "failed to connect to MySQL") + + // Connect to queue database + s.queueDB, err = s.stack.ConnectMySQLService("mysql-queue") + require.NoError(t, err, "failed to connect to queue MySQL") + + // Apply schemas programmatically to application database + testutil.ApplySchema(t, s.log, s.db, testutil.SchemaDir("extension/storage/mysql/schema")) + testutil.ApplySchema(t, s.log, s.db, testutil.SchemaDir("extension/counter/mysql/schema")) + + // Apply schemas programmatically to queue database + testutil.ApplySchema(t, s.log, s.queueDB, testutil.SchemaDir("extension/queue/sql/schema")) + + s.log.Logf("Schemas applied successfully") + + // Connect to Gateway gRPC service + var gatewayConn *grpc.ClientConn + gatewayConn, err = s.stack.ConnectGRPC("gateway-service", 8080) + require.NoError(t, err, "failed to connect to gateway") + s.gatewayClient = gatewaypb.NewSubmitQueueGatewayClient(gatewayConn) + + // Connect to Orchestrator gRPC service + var orchestratorConn *grpc.ClientConn + orchestratorConn, err = s.stack.ConnectGRPC("orchestrator-service", 8080) + require.NoError(t, err, "failed to connect to orchestrator") + s.orchestratorClient = orchestratorpb.NewSubmitQueueOrchestratorClient(orchestratorConn) + + s.log.Logf("E2E integration test suite ready") +} + +func (s *E2EIntegrationSuite) TearDownSuite() { + s.log.Logf("Tearing down E2E integration test suite") + // Cleanup handled automatically by testutil.ComposeStack +} + +func (s *E2EIntegrationSuite) TestPingGateway() { + resp, err := s.gatewayClient.Ping(s.ctx, &gatewaypb.PingRequest{Message: "e2e test"}) + require.NoError(s.T(), err, "Gateway Ping failed") + assert.Equal(s.T(), "gateway", resp.ServiceName) + s.log.Logf("Gateway ping: %s", resp.Message) +} + +func (s *E2EIntegrationSuite) TestPingOrchestrator() { + resp, err := s.orchestratorClient.Ping(s.ctx, &orchestratorpb.PingRequest{Message: "e2e test"}) + require.NoError(s.T(), err, "Orchestrator Ping failed") + assert.Equal(s.T(), "orchestrator", resp.ServiceName) + s.log.Logf("Orchestrator ping: %s", resp.Message) +} + +func (s *E2EIntegrationSuite) TestLandRequest() { + req := &gatewaypb.LandRequest{ + Queue: "e2e-test-queue", + Change: &gatewaypb.Change{Source: "github", Ids: []string{"pr-100", "pr-101"}}, + Strategy: gatewaypb.Strategy_REBASE, + } + + s.log.Logf("Sending Land request for queue=%s", req.Queue) + resp, err := s.gatewayClient.Land(s.ctx, req) + require.NoError(s.T(), err, "Land request failed") + require.NotEmpty(s.T(), resp.Sqid, "SQID should not be empty") + s.log.Logf("Land request succeeded: sqid=%s", resp.Sqid) + + // Verify request stored in database + var state string + err = s.db.QueryRow("SELECT state FROM request WHERE id = ?", resp.Sqid).Scan(&state) + require.NoError(s.T(), err, "failed to query request from database") + assert.Equal(s.T(), "new", state, "request should be in 'new' state") + + s.log.Logf("Verified request %s is in database with state=%s", resp.Sqid, state) +} diff --git a/test/integration/extension/counter/BUILD.bazel b/test/integration/extension/counter/BUILD.bazel new file mode 100644 index 00000000..3e502f51 --- /dev/null +++ b/test/integration/extension/counter/BUILD.bazel @@ -0,0 +1,15 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "counter", + srcs = ["suite.go"], + importpath = "github.com/uber/submitqueue/test/integration/extension/counter", + visibility = ["//visibility:public"], + deps = [ + "//extension/counter", + "//test/testutil", + "@com_github_stretchr_testify//assert", + "@com_github_stretchr_testify//require", + "@com_github_stretchr_testify//suite", + ], +) diff --git a/test/integration/extension/counter/mysql/BUILD.bazel b/test/integration/extension/counter/mysql/BUILD.bazel new file mode 100644 index 00000000..fb475cc1 --- /dev/null +++ b/test/integration/extension/counter/mysql/BUILD.bazel @@ -0,0 +1,19 @@ +load("@rules_go//go:def.bzl", "go_test") + +go_test( + name = "mysql_test", + srcs = ["counter_test.go"], + data = [ + "docker-compose.yml", + "//extension/counter/mysql/schema", + ], + tags = ["integration"], + deps = [ + "//extension/counter/mysql", + "//test/integration/extension/counter", + "//test/testutil", + "@com_github_go_sql_driver_mysql//:mysql", + "@com_github_stretchr_testify//require", + "@com_github_stretchr_testify//suite", + ], +) diff --git a/test/integration/extension/counter/mysql/counter_test.go b/test/integration/extension/counter/mysql/counter_test.go new file mode 100644 index 00000000..bdb6532b --- /dev/null +++ b/test/integration/extension/counter/mysql/counter_test.go @@ -0,0 +1,82 @@ +package mysql + +import ( + "context" + "database/sql" + "testing" + + _ "github.com/go-sql-driver/mysql" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + mysqlcounter "github.com/uber/submitqueue/extension/counter/mysql" + countersuite "github.com/uber/submitqueue/test/integration/extension/counter" + "github.com/uber/submitqueue/test/testutil" +) + +// MySQLCounterIntegrationSuite tests the MySQL counter implementation +// by embedding the shared contract suite. +type MySQLCounterIntegrationSuite struct { + countersuite.CounterContractSuite + stack *testutil.ComposeStack + db *sql.DB + log *testutil.TestLogger +} + +func TestMySQLCounterIntegration(t *testing.T) { + suite.Run(t, new(MySQLCounterIntegrationSuite)) +} + +func (s *MySQLCounterIntegrationSuite) SetupSuite() { + t := s.T() + ctx := context.Background() + s.log = testutil.NewTestLogger(t) + + s.log.Logf("Starting MySQL Counter integration test suite using docker-compose") + + // Use docker-compose to start MySQL (schema applied programmatically) + s.stack = testutil.NewComposeStack( + t, + s.log, + ctx, + "docker-compose.yml", + "ext-counter-mysql", // Test context for meaningful container names + ) + + // Start the compose stack (MySQL only, no schema) + err := s.stack.Up() + require.NoError(t, err, "failed to start compose stack") + + s.log.Logf("Compose stack started successfully") + + // Connect to MySQL using utility + s.db, err = s.stack.ConnectMySQLService("mysql") + require.NoError(t, err, "failed to connect to MySQL") + + // Apply schemas programmatically from directory + schemaDir := testutil.SchemaDir("extension/counter/mysql/schema") + testutil.ApplySchema(t, s.log, s.db, schemaDir) + + s.log.Logf("Schemas applied successfully") + + // Create counter instance + cnt := mysqlcounter.NewCounter(s.db) + + // Provide the counter instance to the contract suite + s.SetContext(ctx) + s.SetCounter(cnt) + s.SetLogger(s.log) + + t.Cleanup(func() { + if s.db != nil { + s.log.Logf("Closing MySQL connection") + s.db.Close() + } + }) + + s.log.Logf("MySQL Counter integration test suite ready") +} + +func (s *MySQLCounterIntegrationSuite) TearDownSuite() { + s.log.Logf("Tearing down MySQL Counter integration test suite") + // Cleanup handled automatically by testutil.ComposeStack +} diff --git a/test/integration/extension/counter/mysql/docker-compose.yml b/test/integration/extension/counter/mysql/docker-compose.yml new file mode 100644 index 00000000..82be3b01 --- /dev/null +++ b/test/integration/extension/counter/mysql/docker-compose.yml @@ -0,0 +1,17 @@ +# Docker Compose for MySQL Counter Library Tests +# Tests the counter library's MySQL implementation in isolation + +services: + # MySQL database for counter infrastructure + mysql: + image: mysql:8.0 + environment: + MYSQL_ROOT_PASSWORD: root + MYSQL_DATABASE: submitqueue + ports: + - "3306" # Random ephemeral port to avoid conflicts + healthcheck: + test: ["CMD", "mysqladmin", "ping", "-h", "localhost", "-proot"] + interval: 5s + timeout: 5s + retries: 10 diff --git a/test/integration/extension/counter/suite.go b/test/integration/extension/counter/suite.go new file mode 100644 index 00000000..9aebade7 --- /dev/null +++ b/test/integration/extension/counter/suite.go @@ -0,0 +1,184 @@ +package counter + +import ( + "context" + "sync" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + "github.com/uber/submitqueue/extension/counter" + "github.com/uber/submitqueue/test/testutil" +) + +// CounterContractSuite defines the contract tests for the counter.Counter interface. +// All counter implementations must pass these tests. +// Implementation-specific tests should embed this suite and call SetCounter(). +type CounterContractSuite struct { + suite.Suite + ctx context.Context + counter counter.Counter + log *testutil.TestLogger +} + +// SetContext sets the context for tests +func (s *CounterContractSuite) SetContext(ctx context.Context) { + s.ctx = ctx +} + +// SetCounter is called by implementation tests to provide the concrete counter instance +func (s *CounterContractSuite) SetCounter(c counter.Counter) { + s.counter = c +} + +// SetLogger sets the logger for tests +func (s *CounterContractSuite) SetLogger(log *testutil.TestLogger) { + s.log = log +} + +// TestCounter_Next tests getting the next sequence number +func (s *CounterContractSuite) TestCounter_Next() { + t := s.T() + ctx := s.ctx + + domain := "test-counter-next" + + // Get first sequence number + seq1, err := s.counter.Next(ctx, domain) + require.NoError(t, err, "failed to get next sequence") + assert.Greater(t, seq1, int64(0), "sequence should be positive") + + // Get next sequence number + seq2, err := s.counter.Next(ctx, domain) + require.NoError(t, err, "failed to get next sequence") + assert.Equal(t, seq1+1, seq2, "sequence should increment by 1") + + // Get another + seq3, err := s.counter.Next(ctx, domain) + require.NoError(t, err) + assert.Equal(t, seq2+1, seq3, "sequence should continue incrementing") + + s.log.Logf("Next test passed: %d → %d → %d", seq1, seq2, seq3) +} + +// TestCounter_MultipleDomains tests independent counters +func (s *CounterContractSuite) TestCounter_MultipleDomains() { + t := s.T() + ctx := s.ctx + + domain1 := "test-counter-1" + domain2 := "test-counter-2" + + // Get sequences from both domains + seq1a, err := s.counter.Next(ctx, domain1) + require.NoError(t, err) + + seq2a, err := s.counter.Next(ctx, domain2) + require.NoError(t, err) + + seq1b, err := s.counter.Next(ctx, domain1) + require.NoError(t, err) + + seq2b, err := s.counter.Next(ctx, domain2) + require.NoError(t, err) + + // Each domain should increment independently + assert.Equal(t, seq1a+1, seq1b, "domain1 should increment") + assert.Equal(t, seq2a+1, seq2b, "domain2 should increment") + + s.log.Logf("Multiple domains test passed: domain1=%d→%d, domain2=%d→%d", + seq1a, seq1b, seq2a, seq2b) +} + +// TestCounter_Concurrency tests concurrent access to the same counter +func (s *CounterContractSuite) TestCounter_Concurrency() { + t := s.T() + ctx := s.ctx + + domain := "test-counter-concurrent" + numGoroutines := 10 + numIterations := 10 + totalExpected := numGoroutines * numIterations + + // Collect all sequence numbers + sequences := make([]int64, 0, totalExpected) + var mu sync.Mutex + var wg sync.WaitGroup + + // Launch concurrent goroutines + for i := 0; i < numGoroutines; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for j := 0; j < numIterations; j++ { + seq, err := s.counter.Next(ctx, domain) + require.NoError(t, err) + + mu.Lock() + sequences = append(sequences, seq) + mu.Unlock() + } + }() + } + + wg.Wait() + + // Verify we got the expected number of sequences + assert.Len(t, sequences, totalExpected, "should have all sequences") + + // Verify all sequences are unique (no duplicates due to race conditions) + seqMap := make(map[int64]bool) + for _, seq := range sequences { + assert.False(t, seqMap[seq], "sequence %d should be unique", seq) + seqMap[seq] = true + } + + s.log.Logf("Concurrency test passed: %d goroutines generated %d unique sequences", + numGoroutines, len(sequences)) +} + +// TestCounter_Atomicity tests that counter increments are atomic +func (s *CounterContractSuite) TestCounter_Atomicity() { + t := s.T() + ctx := s.ctx + + domain := "test-counter-atomic" + + // Get initial value + initial, err := s.counter.Next(ctx, domain) + require.NoError(t, err) + + // Launch two goroutines that both try to get next at the same time + var seq1, seq2 int64 + var wg sync.WaitGroup + + wg.Add(2) + go func() { + defer wg.Done() + seq1, _ = s.counter.Next(ctx, domain) + }() + go func() { + defer wg.Done() + seq2, _ = s.counter.Next(ctx, domain) + }() + + wg.Wait() + + // Both should get different values + assert.NotEqual(t, seq1, seq2, "concurrent Next calls should return different values") + + // Both should be greater than initial + assert.Greater(t, seq1, initial) + assert.Greater(t, seq2, initial) + + // Both results should be initial+1 and initial+2 (in any order) + // Verify we got exactly those two values + results := []int64{seq1, seq2} + expected := []int64{initial + 1, initial + 2} + + assert.Contains(t, results, expected[0], "should have initial+1") + assert.Contains(t, results, expected[1], "should have initial+2") + + s.log.Logf("Atomicity test passed: initial=%d, concurrent results=%d and %d", + initial, seq1, seq2) +} diff --git a/e2e_test/queue/BUILD.bazel b/test/integration/extension/queue/sql/BUILD.bazel similarity index 72% rename from e2e_test/queue/BUILD.bazel rename to test/integration/extension/queue/sql/BUILD.bazel index 7fe4a1e0..63a71d10 100644 --- a/e2e_test/queue/BUILD.bazel +++ b/test/integration/extension/queue/sql/BUILD.bazel @@ -1,23 +1,22 @@ load("@rules_go//go:def.bzl", "go_test") go_test( - name = "queue_test", + name = "sql_test", srcs = ["queue_test.go"], data = [ + "docker-compose.yml", "//extension/queue/sql/schema", ], tags = ["integration"], deps = [ - "//e2e_test/testutil", "//entity/queue", "//extension/queue", "//extension/queue/sql", + "//test/testutil", "@com_github_go_sql_driver_mysql//:mysql", "@com_github_stretchr_testify//assert", "@com_github_stretchr_testify//require", "@com_github_stretchr_testify//suite", - "@com_github_testcontainers_testcontainers_go//:testcontainers-go", - "@com_github_testcontainers_testcontainers_go_modules_mysql//:mysql", "@com_github_uber_go_tally_v4//:tally", "@org_uber_go_zap//zaptest", ], diff --git a/test/integration/extension/queue/sql/docker-compose.yml b/test/integration/extension/queue/sql/docker-compose.yml new file mode 100644 index 00000000..487587b9 --- /dev/null +++ b/test/integration/extension/queue/sql/docker-compose.yml @@ -0,0 +1,17 @@ +# Docker Compose for SQL Queue Library Tests +# Tests the queue library's SQL implementation in isolation + +services: + # MySQL database for queue infrastructure + mysql: + image: mysql:8.0 + environment: + MYSQL_ROOT_PASSWORD: root + MYSQL_DATABASE: submitqueue + ports: + - "3306" # Random ephemeral port to avoid conflicts + healthcheck: + test: ["CMD", "mysqladmin", "ping", "-h", "localhost", "-proot"] + interval: 5s + timeout: 5s + retries: 10 diff --git a/e2e_test/queue/queue_test.go b/test/integration/extension/queue/sql/queue_test.go similarity index 93% rename from e2e_test/queue/queue_test.go rename to test/integration/extension/queue/sql/queue_test.go index 10f121fa..999edf10 100644 --- a/e2e_test/queue/queue_test.go +++ b/test/integration/extension/queue/sql/queue_test.go @@ -1,4 +1,4 @@ -package queue_test +package sql import ( "context" @@ -13,50 +13,74 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" - "github.com/testcontainers/testcontainers-go" - "github.com/testcontainers/testcontainers-go/modules/mysql" "github.com/uber-go/tally/v4" "go.uber.org/zap/zaptest" "github.com/uber/submitqueue/entity/queue" extqueue "github.com/uber/submitqueue/extension/queue" queueSQL "github.com/uber/submitqueue/extension/queue/sql" - "github.com/uber/submitqueue/e2e_test/testutil" + "github.com/uber/submitqueue/test/testutil" ) -type QueueIntegrationSuite struct { +type SQLQueueIntegrationSuite struct { suite.Suite - ctx context.Context - db *sql.DB - container *mysql.MySQLContainer - network *testcontainers.DockerNetwork - dsn string - log *testutil.TestLogger + ctx context.Context + stack *testutil.ComposeStack + db *sql.DB + log *testutil.TestLogger } -func TestQueueIntegration(t *testing.T) { - suite.Run(t, new(QueueIntegrationSuite)) +func TestSQLQueueIntegration(t *testing.T) { + suite.Run(t, new(SQLQueueIntegrationSuite)) } -func (s *QueueIntegrationSuite) SetupSuite() { +func (s *SQLQueueIntegrationSuite) SetupSuite() { t := s.T() s.ctx = context.Background() s.log = testutil.NewTestLogger(t) - // Setup Docker environment and network - s.network, s.ctx = testutil.SetupDockerEnv(t, s.log, s.ctx) + s.log.Logf("Starting SQL Queue integration test suite using docker-compose") - // Setup MySQL using shared helper - s.container, s.db, s.dsn = testutil.SetupMySQL(t, s.log, s.network, "extension/queue/sql/schema") + // Use docker-compose to start MySQL (schema applied programmatically) + s.stack = testutil.NewComposeStack( + t, + s.log, + s.ctx, + "docker-compose.yml", + "ext-queue-sql", // Test context for meaningful container names + ) + + // Start the compose stack (MySQL only, no schema) + err := s.stack.Up() + require.NoError(t, err, "failed to start compose stack") + + s.log.Logf("Compose stack started successfully") + + // Connect to MySQL using utility + s.db, err = s.stack.ConnectMySQLService("mysql") + require.NoError(t, err, "failed to connect to MySQL") + + s.log.Logf("Connected to MySQL for queue testing") + + // Apply schemas programmatically from directory (queue has 3 schema files) + schemaDir := testutil.SchemaDir("extension/queue/sql/schema") + testutil.ApplySchema(t, s.log, s.db, schemaDir) + + s.log.Logf("Schemas applied successfully") + + t.Cleanup(func() { + if s.db != nil { + s.log.Logf("Closing MySQL connection") + s.db.Close() + } + }) + + s.log.Logf("SQL Queue integration test suite ready") } -func (s *QueueIntegrationSuite) TearDownSuite() { - if s.db != nil { - s.db.Close() - } - if s.container != nil { - require.NoError(s.T(), s.container.Terminate(s.ctx)) - } +func (s *SQLQueueIntegrationSuite) TearDownSuite() { + s.log.Logf("Tearing down SQL Queue integration test suite") + // Cleanup handled automatically by testutil.ComposeStack } // receiveWithTimeout receives a single delivery from the channel with a timeout. @@ -93,7 +117,7 @@ func receiveNWithTimeout( } } -func (s *QueueIntegrationSuite) TestPublishAndSubscribe() { +func (s *SQLQueueIntegrationSuite) TestPublishAndSubscribe() { t := s.T() // Create queue @@ -164,7 +188,7 @@ func (s *QueueIntegrationSuite) TestPublishAndSubscribe() { t.Logf("Successfully received and acked 2 messages with metadata verified") } -func (s *QueueIntegrationSuite) TestMultiplePartitions() { +func (s *SQLQueueIntegrationSuite) TestMultiplePartitions() { t := s.T() q, err := queueSQL.NewQueue(queueSQL.Params{ @@ -209,7 +233,7 @@ func (s *QueueIntegrationSuite) TestMultiplePartitions() { t.Logf("Successfully processed all %d messages", expectedCount) } -func (s *QueueIntegrationSuite) TestVisibilityTimeoutAndRetry() { +func (s *SQLQueueIntegrationSuite) TestVisibilityTimeoutAndRetry() { t := s.T() q, err := queueSQL.NewQueue(queueSQL.Params{ @@ -296,7 +320,7 @@ func (s *QueueIntegrationSuite) TestVisibilityTimeoutAndRetry() { t.Logf("Successfully tested ExtendVisibilityTimeout and visibility timeout retry") } -func (s *QueueIntegrationSuite) TestNackWithDelay() { +func (s *SQLQueueIntegrationSuite) TestNackWithDelay() { t := s.T() q, err := queueSQL.NewQueue(queueSQL.Params{ @@ -348,7 +372,7 @@ func (s *QueueIntegrationSuite) TestNackWithDelay() { require.NoError(t, delivery2.Ack(s.ctx)) } -func (s *QueueIntegrationSuite) TestIdempotentPublish() { +func (s *SQLQueueIntegrationSuite) TestIdempotentPublish() { t := s.T() q, err := queueSQL.NewQueue(queueSQL.Params{ @@ -395,7 +419,7 @@ func (s *QueueIntegrationSuite) TestIdempotentPublish() { } } -func (s *QueueIntegrationSuite) TestConcurrentPublishers() { +func (s *SQLQueueIntegrationSuite) TestConcurrentPublishers() { t := s.T() q, err := queueSQL.NewQueue(queueSQL.Params{ @@ -451,7 +475,7 @@ func (s *QueueIntegrationSuite) TestConcurrentPublishers() { t.Logf("Received all %d concurrent messages", totalMessages) } -func (s *QueueIntegrationSuite) TestCrashRecovery() { +func (s *SQLQueueIntegrationSuite) TestCrashRecovery() { t := s.T() q1, err := queueSQL.NewQueue(queueSQL.Params{ @@ -526,7 +550,7 @@ func (s *QueueIntegrationSuite) TestCrashRecovery() { t.Logf("Crash recovery successful: message processed by worker 2") } -func (s *QueueIntegrationSuite) TestMultipleConsumerGroups() { +func (s *SQLQueueIntegrationSuite) TestMultipleConsumerGroups() { t := s.T() topic := "multi_group_topic" @@ -604,7 +628,7 @@ func (s *QueueIntegrationSuite) TestMultipleConsumerGroups() { t.Logf("Both consumer groups independently received all %d messages", numMessages) } -func (s *QueueIntegrationSuite) TestMultipleWorkersInConsumerGroup() { +func (s *SQLQueueIntegrationSuite) TestMultipleWorkersInConsumerGroup() { t := s.T() topic := "multi_worker_topic" @@ -718,7 +742,7 @@ func (s *QueueIntegrationSuite) TestMultipleWorkersInConsumerGroup() { t.Logf("Load balanced: %d messages distributed across 2 workers with no duplicates", numMessages) } -func (s *QueueIntegrationSuite) TestConcurrentSubscribers() { +func (s *SQLQueueIntegrationSuite) TestConcurrentSubscribers() { t := s.T() topic := "concurrent_subscribers_topic" @@ -828,7 +852,7 @@ func (s *QueueIntegrationSuite) TestConcurrentSubscribers() { t.Logf("Concurrent subscribers test: %d messages processed by %d workers with no duplicates", totalMessages, numSubscribers) } -func (s *QueueIntegrationSuite) TestDeadLetterQueue() { +func (s *SQLQueueIntegrationSuite) TestDeadLetterQueue() { t := s.T() topic := "dlq_topic" @@ -923,7 +947,7 @@ func (s *QueueIntegrationSuite) TestDeadLetterQueue() { t.Logf("DLQ test successful: poison message consumed from DLQ topic '%s' with metadata: %+v", dlqTopic, metadata) } -func (s *QueueIntegrationSuite) TestMessageOrderingWithinPartition() { +func (s *SQLQueueIntegrationSuite) TestMessageOrderingWithinPartition() { t := s.T() topic := "ordering_topic" @@ -975,7 +999,7 @@ func (s *QueueIntegrationSuite) TestMessageOrderingWithinPartition() { t.Logf("FIFO ordering verified: all %d messages received in exact publish order", numMessages) } -func (s *QueueIntegrationSuite) TestLateSubscriber() { +func (s *SQLQueueIntegrationSuite) TestLateSubscriber() { t := s.T() topic := "late_subscriber_topic" @@ -1026,7 +1050,7 @@ func (s *QueueIntegrationSuite) TestLateSubscriber() { t.Logf("Late subscriber successfully received all %d pre-published messages", numMessages) } -func (s *QueueIntegrationSuite) TestEmptyTopicSubscribe() { +func (s *SQLQueueIntegrationSuite) TestEmptyTopicSubscribe() { t := s.T() topic := "empty_topic" @@ -1070,7 +1094,7 @@ func (s *QueueIntegrationSuite) TestEmptyTopicSubscribe() { t.Logf("Successfully received message published after subscription to empty topic") } -func (s *QueueIntegrationSuite) TestGracefulShutdownDuringProcessing() { +func (s *SQLQueueIntegrationSuite) TestGracefulShutdownDuringProcessing() { t := s.T() topic := "shutdown_topic" diff --git a/test/integration/extension/storage/BUILD.bazel b/test/integration/extension/storage/BUILD.bazel new file mode 100644 index 00000000..b8ea794f --- /dev/null +++ b/test/integration/extension/storage/BUILD.bazel @@ -0,0 +1,16 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "storage", + srcs = ["suite.go"], + importpath = "github.com/uber/submitqueue/test/integration/extension/storage", + visibility = ["//visibility:public"], + deps = [ + "//entity", + "//extension/storage", + "//test/testutil", + "@com_github_stretchr_testify//assert", + "@com_github_stretchr_testify//require", + "@com_github_stretchr_testify//suite", + ], +) diff --git a/test/integration/extension/storage/mysql/BUILD.bazel b/test/integration/extension/storage/mysql/BUILD.bazel new file mode 100644 index 00000000..00f704d0 --- /dev/null +++ b/test/integration/extension/storage/mysql/BUILD.bazel @@ -0,0 +1,19 @@ +load("@rules_go//go:def.bzl", "go_test") + +go_test( + name = "mysql_test", + srcs = ["storage_test.go"], + data = [ + "docker-compose.yml", + "//extension/storage/mysql/schema", + ], + tags = ["integration"], + deps = [ + "//extension/storage/mysql", + "//test/integration/extension/storage", + "//test/testutil", + "@com_github_go_sql_driver_mysql//:mysql", + "@com_github_stretchr_testify//require", + "@com_github_stretchr_testify//suite", + ], +) diff --git a/test/integration/extension/storage/mysql/docker-compose.yml b/test/integration/extension/storage/mysql/docker-compose.yml new file mode 100644 index 00000000..79be09e0 --- /dev/null +++ b/test/integration/extension/storage/mysql/docker-compose.yml @@ -0,0 +1,17 @@ +# Docker Compose for MySQL Storage Library Tests +# Tests the storage library's MySQL implementation in isolation + +services: + # MySQL database for storage infrastructure + mysql: + image: mysql:8.0 + environment: + MYSQL_ROOT_PASSWORD: root + MYSQL_DATABASE: submitqueue + ports: + - "3306" # Random ephemeral port to avoid conflicts + healthcheck: + test: ["CMD", "mysqladmin", "ping", "-h", "localhost", "-proot"] + interval: 5s + timeout: 5s + retries: 10 diff --git a/test/integration/extension/storage/mysql/storage_test.go b/test/integration/extension/storage/mysql/storage_test.go new file mode 100644 index 00000000..9dd3444c --- /dev/null +++ b/test/integration/extension/storage/mysql/storage_test.go @@ -0,0 +1,89 @@ +package mysql + +import ( + "context" + "database/sql" + "testing" + + _ "github.com/go-sql-driver/mysql" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + mysqlstorage "github.com/uber/submitqueue/extension/storage/mysql" + storagesuite "github.com/uber/submitqueue/test/integration/extension/storage" + "github.com/uber/submitqueue/test/testutil" +) + +// MySQLStorageIntegrationSuite tests the MySQL storage implementation +// by embedding the shared contract suite. +type MySQLStorageIntegrationSuite struct { + storagesuite.StorageContractSuite + stack *testutil.ComposeStack + db *sql.DB + log *testutil.TestLogger +} + +func TestMySQLStorageIntegration(t *testing.T) { + suite.Run(t, new(MySQLStorageIntegrationSuite)) +} + +func (s *MySQLStorageIntegrationSuite) SetupSuite() { + t := s.T() + ctx := context.Background() + s.log = testutil.NewTestLogger(t) + + s.log.Logf("Starting MySQL Storage integration test suite using docker-compose") + + // Use docker-compose to start MySQL (schema applied programmatically) + s.stack = testutil.NewComposeStack( + t, + s.log, + ctx, + "docker-compose.yml", + "ext-storage-mysql", // Test context for meaningful container names + ) + + // Start the compose stack (MySQL only, no schema) + err := s.stack.Up() + require.NoError(t, err, "failed to start compose stack") + + s.log.Logf("Compose stack started successfully") + + // Get MySQL DSN for storage initialization + dsn, err := s.stack.MySQLServiceDSN("mysql") + require.NoError(t, err, "failed to get MySQL DSN") + + // Connect to MySQL for schema application + s.db, err = s.stack.ConnectMySQLService("mysql") + require.NoError(t, err, "failed to connect to MySQL") + + // Apply schemas programmatically from directory + schemaDir := testutil.SchemaDir("extension/storage/mysql/schema") + testutil.ApplySchema(t, s.log, s.db, schemaDir) + + s.log.Logf("Schemas applied successfully") + + // Create storage instance + store, err := mysqlstorage.NewStorage(mysqlstorage.MySQLParameters{ + DSN: dsn, + }) + require.NoError(t, err, "failed to create storage") + + // Provide the storage instance to the contract suite + s.SetContext(ctx) + s.SetStorage(store) + s.SetLogger(s.log) + + t.Cleanup(func() { + if s.db != nil { + s.log.Logf("Closing MySQL connection") + s.db.Close() + } + }) + + s.log.Logf("MySQL Storage integration test suite ready") +} + +func (s *MySQLStorageIntegrationSuite) TearDownSuite() { + s.log.Logf("Tearing down MySQL Storage integration test suite") + // Cleanup handled automatically by testutil.ComposeStack +} diff --git a/test/integration/extension/storage/suite.go b/test/integration/extension/storage/suite.go new file mode 100644 index 00000000..4a7df35b --- /dev/null +++ b/test/integration/extension/storage/suite.go @@ -0,0 +1,177 @@ +package storage + +import ( + "context" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + "github.com/uber/submitqueue/entity" + "github.com/uber/submitqueue/extension/storage" + "github.com/uber/submitqueue/test/testutil" +) + +// StorageContractSuite defines the contract tests for the storage.Storage interface. +// All storage implementations must pass these tests. +// Implementation-specific tests should embed this suite and call SetStorage(). +type StorageContractSuite struct { + suite.Suite + ctx context.Context + storage storage.Storage + log *testutil.TestLogger +} + +// SetContext sets the context for tests +func (s *StorageContractSuite) SetContext(ctx context.Context) { + s.ctx = ctx +} + +// SetStorage is called by implementation tests to provide the concrete storage instance +func (s *StorageContractSuite) SetStorage(store storage.Storage) { + s.storage = store +} + +// SetLogger sets the logger for tests +func (s *StorageContractSuite) SetLogger(log *testutil.TestLogger) { + s.log = log +} + +// TestStorage_CreateAndGet tests creating and retrieving a request +func (s *StorageContractSuite) TestStorage_CreateAndGet() { + t := s.T() + ctx := s.ctx + + request := entity.Request{ + ID: "test/create-get", + Queue: "test-queue", + State: entity.RequestStateNew, + Change: entity.Change{ + Source: "github", + IDs: []string{"123"}, + }, + LandStrategy: entity.RequestLandStrategyMerge, + Version: 1, + } + + // Create request + err := s.storage.GetRequestStore().Create(ctx, request) + require.NoError(t, err, "failed to create request") + + // Get request back + retrieved, err := s.storage.GetRequestStore().Get(ctx, request.ID) + require.NoError(t, err, "failed to get request") + + // Verify fields + assert.Equal(t, request.ID, retrieved.ID) + assert.Equal(t, request.Queue, retrieved.Queue) + assert.Equal(t, request.State, retrieved.State) + assert.Equal(t, request.Change.Source, retrieved.Change.Source) + assert.Equal(t, request.Change.IDs, retrieved.Change.IDs) + assert.Equal(t, request.LandStrategy, retrieved.LandStrategy) + assert.Equal(t, request.Version, retrieved.Version) + + s.log.Logf("CreateAndGet test passed: created and retrieved request %s", request.ID) +} + +// TestStorage_UpdateState tests updating request state +func (s *StorageContractSuite) TestStorage_UpdateState() { + t := s.T() + ctx := s.ctx + + request := entity.Request{ + ID: "test/update", + Queue: "test-queue", + State: entity.RequestStateNew, + LandStrategy: entity.RequestLandStrategyMerge, + Version: 1, + } + + // Create initial request + err := s.storage.GetRequestStore().Create(ctx, request) + require.NoError(t, err) + + // Update state + err = s.storage.GetRequestStore().UpdateState(ctx, request.ID, request.Version, entity.RequestStateProcessing) + require.NoError(t, err, "failed to update request state") + + // Verify update + retrieved, err := s.storage.GetRequestStore().Get(ctx, request.ID) + require.NoError(t, err) + assert.Equal(t, entity.RequestStateProcessing, retrieved.State) + assert.Equal(t, int32(2), retrieved.Version, "version should increment after update") + + s.log.Logf("UpdateState test passed: updated request %s to state %s", request.ID, retrieved.State) +} + +// TestStorage_OptimisticLocking tests version-based optimistic locking +func (s *StorageContractSuite) TestStorage_OptimisticLocking() { + t := s.T() + ctx := s.ctx + + request := entity.Request{ + ID: "test/optimistic-lock", + Queue: "test-queue", + State: entity.RequestStateNew, + LandStrategy: entity.RequestLandStrategyMerge, + Version: 1, + } + + // Create request + err := s.storage.GetRequestStore().Create(ctx, request) + require.NoError(t, err) + + // Update with correct version + err = s.storage.GetRequestStore().UpdateState(ctx, request.ID, 1, entity.RequestStateProcessing) + require.NoError(t, err, "update with correct version should succeed") + + // Try to update with stale version (should fail) + err = s.storage.GetRequestStore().UpdateState(ctx, request.ID, 1, entity.RequestStateLanded) + assert.Error(t, err, "update with stale version should fail") + assert.ErrorIs(t, err, storage.ErrVersionMismatch, "should return ErrVersionMismatch") + + // Verify state wasn't changed by stale update + retrieved, err := s.storage.GetRequestStore().Get(ctx, request.ID) + require.NoError(t, err) + assert.Equal(t, entity.RequestStateProcessing, retrieved.State, "stale update should not modify state") + assert.Equal(t, int32(2), retrieved.Version) + + s.log.Logf("Optimistic locking test passed: prevented stale update for request %s", request.ID) +} + +// TestStorage_NotFound tests getting a non-existent request +func (s *StorageContractSuite) TestStorage_NotFound() { + t := s.T() + ctx := s.ctx + + // Try to get non-existent request + _, err := s.storage.GetRequestStore().Get(ctx, "test/nonexistent") + assert.Error(t, err, "getting non-existent request should return error") + assert.ErrorIs(t, err, storage.ErrNotFound, "should return ErrNotFound") + + s.log.Logf("NotFound test passed: correctly returned ErrNotFound") +} + +// TestStorage_CreateDuplicate tests creating a request with duplicate ID +func (s *StorageContractSuite) TestStorage_CreateDuplicate() { + t := s.T() + ctx := s.ctx + + request := entity.Request{ + ID: "test/duplicate", + Queue: "test-queue", + State: entity.RequestStateNew, + LandStrategy: entity.RequestLandStrategyMerge, + Version: 1, + } + + // Create request + err := s.storage.GetRequestStore().Create(ctx, request) + require.NoError(t, err) + + // Try to create duplicate + err = s.storage.GetRequestStore().Create(ctx, request) + assert.Error(t, err, "creating duplicate request should return error") + assert.ErrorIs(t, err, storage.ErrAlreadyExists, "should return ErrAlreadyExists") + + s.log.Logf("CreateDuplicate test passed: prevented duplicate creation") +} diff --git a/test/integration/gateway/BUILD.bazel b/test/integration/gateway/BUILD.bazel new file mode 100644 index 00000000..72f5c2a6 --- /dev/null +++ b/test/integration/gateway/BUILD.bazel @@ -0,0 +1,23 @@ +load("@rules_go//go:def.bzl", "go_test") + +go_test( + name = "gateway_test", + srcs = ["suite_test.go"], + data = [ + "//:MODULE.bazel", + "//:go.mod", + "//example/server/gateway:docker-compose.yml", + "//extension/counter/mysql/schema", + "//extension/queue/sql/schema", + "//extension/storage/mysql/schema", + ], + tags = ["integration"], + deps = [ + "//gateway/protopb", + "//test/testutil", + "@com_github_stretchr_testify//assert", + "@com_github_stretchr_testify//require", + "@com_github_stretchr_testify//suite", + "@org_golang_google_grpc//:grpc", + ], +) diff --git a/test/integration/gateway/suite_test.go b/test/integration/gateway/suite_test.go new file mode 100644 index 00000000..740433ce --- /dev/null +++ b/test/integration/gateway/suite_test.go @@ -0,0 +1,138 @@ +package gateway + +// Gateway Integration Tests +// +// These tests use docker-compose from example/server/gateway/docker-compose.yml +// which requires pre-built Linux binaries. +// +// Run with make target (builds binary + runs test): +// make integration-test-gateway +// +// For manual testing with docker-compose: +// make docker-gateway + +import ( + "context" + "database/sql" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + pb "github.com/uber/submitqueue/gateway/protopb" + "github.com/uber/submitqueue/test/testutil" + "google.golang.org/grpc" +) + +type GatewayIntegrationSuite struct { + suite.Suite + ctx context.Context + log *testutil.TestLogger + stack *testutil.ComposeStack + client pb.SubmitQueueGatewayClient + db *sql.DB // App database + queueDB *sql.DB // Queue database +} + +func TestGatewayIntegration(t *testing.T) { + suite.Run(t, new(GatewayIntegrationSuite)) +} + +func (s *GatewayIntegrationSuite) SetupSuite() { + t := s.T() + s.ctx = context.Background() + s.log = testutil.NewTestLogger(t) + + s.log.Logf("Starting Gateway integration test suite using docker-compose") + + // Set REPO_ROOT for docker-compose volume mounts and build context + repoRoot := testutil.FindRepoRoot(t) + t.Setenv("REPO_ROOT", repoRoot) + + // Use docker-compose from example/server/gateway + // NOTE: Assumes Linux binary is pre-built via make target + composeFile := filepath.Join(repoRoot, "example/server/gateway/docker-compose.yml") + s.stack = testutil.NewComposeStack(t, s.log, s.ctx, composeFile, "svc-gateway") + + // Start the compose stack (Gateway + 2 MySQL DBs) + err := s.stack.Up() + require.NoError(t, err, "failed to start compose stack") + + s.log.Logf("Compose stack started successfully") + + // Connect to application database + s.db, err = s.stack.ConnectMySQLService("mysql-app") + require.NoError(t, err, "failed to connect to MySQL") + + // Connect to queue database + s.queueDB, err = s.stack.ConnectMySQLService("mysql-queue") + require.NoError(t, err, "failed to connect to queue MySQL") + + // Apply schemas programmatically to application database + testutil.ApplySchema(t, s.log, s.db, testutil.SchemaDir("extension/storage/mysql/schema")) + testutil.ApplySchema(t, s.log, s.db, testutil.SchemaDir("extension/counter/mysql/schema")) + + // Apply schemas programmatically to queue database + testutil.ApplySchema(t, s.log, s.queueDB, testutil.SchemaDir("extension/queue/sql/schema")) + + s.log.Logf("Schemas applied successfully") + + // Connect to Gateway gRPC service + var conn *grpc.ClientConn + conn, err = s.stack.ConnectGRPC("gateway-service", 8080) + require.NoError(t, err, "failed to connect to gateway") + s.client = pb.NewSubmitQueueGatewayClient(conn) + + s.log.Logf("Gateway integration test suite ready") +} + +func (s *GatewayIntegrationSuite) TearDownSuite() { + s.log.Logf("Tearing down Gateway integration test suite") + // Cleanup handled automatically by testutil.ComposeStack +} + +// TestPingAPI tests the Gateway Ping API +func (s *GatewayIntegrationSuite) TestPingAPI() { + t := s.T() + + resp, err := s.client.Ping(s.ctx, &pb.PingRequest{Message: "integration test"}) + require.NoError(t, err, "Gateway Ping failed") + assert.Equal(t, "gateway", resp.ServiceName) + assert.NotEmpty(t, resp.Message) + assert.NotZero(t, resp.Timestamp) + + s.log.Logf("Gateway Ping test passed: %s", resp.Message) +} + +// TestLandAPI tests the Gateway Land API with queue publishing +func (s *GatewayIntegrationSuite) TestLandAPI() { + t := s.T() + + req := &pb.LandRequest{ + Queue: "test-queue", + Change: &pb.Change{Source: "github", Ids: []string{"PR-123"}}, + Strategy: pb.Strategy_REBASE, + } + + s.log.Logf("Sending Land request for queue=%s", req.Queue) + resp, err := s.client.Land(s.ctx, req) + require.NoError(t, err, "Land request failed") + require.NotEmpty(t, resp.Sqid, "SQID should not be empty") + + s.log.Logf("Land request succeeded: sqid=%s", resp.Sqid) + + // Verify request stored in database + var state string + err = s.db.QueryRow("SELECT state FROM request WHERE id = ?", resp.Sqid).Scan(&state) + require.NoError(t, err, "failed to query request from database") + assert.Equal(t, "new", state, "request state should be new") + + // Verify message published to queue + var msgCount int + err = s.queueDB.QueryRow("SELECT COUNT(*) FROM queue_messages WHERE id = ?", resp.Sqid).Scan(&msgCount) + require.NoError(t, err, "failed to query queue messages") + assert.Equal(t, 1, msgCount, "should have 1 message in queue") + + s.log.Logf("Land API test passed: request stored and message published") +} diff --git a/test/integration/orchestrator/BUILD.bazel b/test/integration/orchestrator/BUILD.bazel new file mode 100644 index 00000000..86103413 --- /dev/null +++ b/test/integration/orchestrator/BUILD.bazel @@ -0,0 +1,23 @@ +load("@rules_go//go:def.bzl", "go_test") + +go_test( + name = "orchestrator_test", + srcs = ["suite_test.go"], + data = [ + "//:MODULE.bazel", + "//:go.mod", + "//example/server/orchestrator:docker-compose.yml", + "//extension/counter/mysql/schema", + "//extension/queue/sql/schema", + "//extension/storage/mysql/schema", + ], + tags = ["integration"], + deps = [ + "//orchestrator/protopb", + "//test/testutil", + "@com_github_stretchr_testify//assert", + "@com_github_stretchr_testify//require", + "@com_github_stretchr_testify//suite", + "@org_golang_google_grpc//:grpc", + ], +) diff --git a/test/integration/orchestrator/suite_test.go b/test/integration/orchestrator/suite_test.go new file mode 100644 index 00000000..fd59a686 --- /dev/null +++ b/test/integration/orchestrator/suite_test.go @@ -0,0 +1,107 @@ +package orchestrator + +// Orchestrator Integration Tests +// +// These tests use docker-compose from example/server/orchestrator/docker-compose.yml +// which requires pre-built Linux binaries. +// +// Run with make target (builds binary + runs test): +// make integration-test-orchestrator +// +// For manual testing with docker-compose: +// make docker-orchestrator + +import ( + "context" + "database/sql" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + pb "github.com/uber/submitqueue/orchestrator/protopb" + "github.com/uber/submitqueue/test/testutil" + "google.golang.org/grpc" +) + +type OrchestratorIntegrationSuite struct { + suite.Suite + ctx context.Context + log *testutil.TestLogger + stack *testutil.ComposeStack + client pb.SubmitQueueOrchestratorClient + db *sql.DB // App database + queueDB *sql.DB // Queue database +} + +func TestOrchestratorIntegration(t *testing.T) { + suite.Run(t, new(OrchestratorIntegrationSuite)) +} + +func (s *OrchestratorIntegrationSuite) SetupSuite() { + t := s.T() + s.ctx = context.Background() + s.log = testutil.NewTestLogger(t) + + s.log.Logf("Starting Orchestrator integration test suite using docker-compose") + + // Set REPO_ROOT for docker-compose volume mounts and build context + repoRoot := testutil.FindRepoRoot(t) + t.Setenv("REPO_ROOT", repoRoot) + + // Use docker-compose from example/server/orchestrator + // NOTE: Assumes Linux binary is pre-built via make target + composeFile := filepath.Join(repoRoot, "example/server/orchestrator/docker-compose.yml") + s.stack = testutil.NewComposeStack(t, s.log, s.ctx, composeFile, "svc-orchestrator") + + // Start the compose stack (Orchestrator + 2 MySQL DBs) + err := s.stack.Up() + require.NoError(t, err, "failed to start compose stack") + + s.log.Logf("Compose stack started successfully") + + // Connect to application database + s.db, err = s.stack.ConnectMySQLService("mysql-app") + require.NoError(t, err, "failed to connect to MySQL") + + // Connect to queue database + s.queueDB, err = s.stack.ConnectMySQLService("mysql-queue") + require.NoError(t, err, "failed to connect to queue MySQL") + + // Apply schemas programmatically to application database + testutil.ApplySchema(t, s.log, s.db, testutil.SchemaDir("extension/storage/mysql/schema")) + testutil.ApplySchema(t, s.log, s.db, testutil.SchemaDir("extension/counter/mysql/schema")) + + // Apply schemas programmatically to queue database + testutil.ApplySchema(t, s.log, s.queueDB, testutil.SchemaDir("extension/queue/sql/schema")) + + s.log.Logf("Schemas applied successfully") + + // Connect to Orchestrator gRPC service + var conn *grpc.ClientConn + conn, err = s.stack.ConnectGRPC("orchestrator-service", 8080) + require.NoError(t, err, "failed to connect to orchestrator") + s.client = pb.NewSubmitQueueOrchestratorClient(conn) + + s.log.Logf("Orchestrator integration test suite ready") +} + +func (s *OrchestratorIntegrationSuite) TearDownSuite() { + s.log.Logf("Tearing down Orchestrator integration test suite") + // Cleanup handled automatically by testutil.ComposeStack +} + +// TestPingAPI tests the Orchestrator Ping API +func (s *OrchestratorIntegrationSuite) TestPingAPI() { + t := s.T() + + resp, err := s.client.Ping(s.ctx, &pb.PingRequest{Message: "integration test"}) + require.NoError(t, err, "Orchestrator Ping failed") + assert.Equal(t, "orchestrator", resp.ServiceName) + assert.NotEmpty(t, resp.Message) + assert.NotZero(t, resp.Timestamp) + + s.log.Logf("Orchestrator Ping test passed: %s", resp.Message) +} + diff --git a/test/testutil/BUILD.bazel b/test/testutil/BUILD.bazel new file mode 100644 index 00000000..1b9cf6f3 --- /dev/null +++ b/test/testutil/BUILD.bazel @@ -0,0 +1,18 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "testutil", + srcs = [ + "compose.go", + "logger.go", + "schema.go", + ], + importpath = "github.com/uber/submitqueue/test/testutil", + visibility = ["//test:__subpackages__"], + deps = [ + "@com_github_go_sql_driver_mysql//:mysql", + "@com_github_stretchr_testify//require", + "@org_golang_google_grpc//:grpc", + "@org_golang_google_grpc//credentials/insecure", + ], +) diff --git a/test/testutil/compose.go b/test/testutil/compose.go new file mode 100644 index 00000000..6eafad46 --- /dev/null +++ b/test/testutil/compose.go @@ -0,0 +1,316 @@ +package testutil + +import ( + "context" + "database/sql" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + _ "github.com/go-sql-driver/mysql" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" +) + +// ComposeStack manages a docker-compose stack for testing. +type ComposeStack struct { + composeFile string + projectName string + t *testing.T + log *TestLogger + ctx context.Context + composeCmd []string // docker-compose command (either ["docker-compose"] or ["docker", "compose"]) +} + +// getDockerComposeCommand returns the docker-compose command to use. +// Tries "docker-compose" first (V1), falls back to "docker compose" (V2). +func getDockerComposeCommand() []string { + // Try docker-compose (V1) + if _, err := exec.LookPath("docker-compose"); err == nil { + return []string{"docker-compose"} + } + + // Fall back to docker compose (V2) + return []string{"docker", "compose"} +} + +// NewComposeStack creates a new compose stack from the given docker-compose file. +// Automatically registers cleanup to tear down the stack. +// testContext should describe what's being tested (e.g., "gateway", "storage", "e2e"). +func NewComposeStack(t *testing.T, log *TestLogger, ctx context.Context, composeFile, testContext string) *ComposeStack { + t.Helper() + + // Setup Docker environment + setupDockerEnv(t) + + // Get absolute path to compose file + absPath, err := filepath.Abs(composeFile) + require.NoError(t, err, "failed to get absolute path to compose file") + + // Generate meaningful project name: sq-test-{context}-{short-timestamp} + // Results in container names like: sq-test-gateway-a1b2c3d-mysql-app-1 + timestamp := fmt.Sprintf("%x", time.Now().UnixNano()&0xFFFFFF) // Last 6 hex digits + projectName := fmt.Sprintf("sq-test-%s-%s", testContext, timestamp) + + stack := &ComposeStack{ + composeFile: absPath, + projectName: projectName, + t: t, + log: log, + ctx: ctx, + composeCmd: getDockerComposeCommand(), + } + + // Register cleanup + t.Cleanup(func() { + // Skip cleanup if test failed (for debugging) or SKIP_CLEANUP env var is set + if t.Failed() { + log.Logf("Test FAILED - keeping containers for debugging") + log.Logf("Container prefix: %s", projectName) + log.Logf("List containers: docker ps -a | grep %s", projectName) + log.Logf("View logs: docker logs %s--1", projectName) + composeCmd := strings.Join(stack.composeCmd, " ") + log.Logf("Clean up manually: %s -f %s -p %s down -v --rmi local", composeCmd, absPath, projectName) + return + } + + if os.Getenv("SKIP_CLEANUP") == "true" { + log.Logf("SKIP_CLEANUP=true - keeping containers for inspection") + log.Logf("Container prefix: %s", projectName) + composeCmd := strings.Join(stack.composeCmd, " ") + log.Logf("Clean up manually: %s -f %s -p %s down -v --rmi local", composeCmd, absPath, projectName) + return + } + + log.Logf("Tearing down compose stack") + stack.down() + }) + + return stack +} + +// Up starts all services in the compose stack. +func (s *ComposeStack) Up() error { + s.t.Helper() + s.log.Logf("Starting compose stack from %s", s.composeFile) + + args := append(s.composeCmd[1:], "-f", s.composeFile, "-p", s.projectName, "up", "-d", "--build") + cmd := exec.CommandContext(s.ctx, s.composeCmd[0], args...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + if err := cmd.Run(); err != nil { + return fmt.Errorf("failed to start compose stack: %w", err) + } + + // Wait for services to be healthy + s.log.Logf("Waiting for services to be healthy...") + time.Sleep(5 * time.Second) // Simple wait for now + + s.log.Logf("Compose stack started successfully") + return nil +} + +// down stops and removes all services in the compose stack. +// Also removes locally built images to prevent accumulation. +func (s *ComposeStack) down() { + s.log.Logf("Stopping compose stack and removing images") + + args := append(s.composeCmd[1:], "-f", s.composeFile, "-p", s.projectName, "down", "-v", "--rmi", "local") + cmd := exec.CommandContext(s.ctx, s.composeCmd[0], args...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + if err := cmd.Run(); err != nil { + s.log.Logf("Warning: failed to stop compose stack: %v", err) + } +} + +// ServicePort returns the mapped host port for a service's container port. +func (s *ComposeStack) ServicePort(serviceName string, containerPort int) (int, error) { + s.t.Helper() + + args := append(s.composeCmd[1:], "-f", s.composeFile, "-p", s.projectName, "port", serviceName, fmt.Sprintf("%d", containerPort)) + cmd := exec.CommandContext(s.ctx, s.composeCmd[0], args...) + + output, err := cmd.Output() + if err != nil { + return 0, fmt.Errorf("failed to get port for service %s: %w", serviceName, err) + } + + // Parse output like "0.0.0.0:49153\n" + // Strip whitespace and split on colon + outputStr := strings.TrimSpace(string(output)) + + // Find the last colon (port separator) + colonIdx := strings.LastIndex(outputStr, ":") + if colonIdx < 0 { + return 0, fmt.Errorf("failed to parse port output: no colon found in %q", outputStr) + } + + portStr := outputStr[colonIdx+1:] + var port int + _, err = fmt.Sscanf(portStr, "%d", &port) + if err != nil { + return 0, fmt.Errorf("failed to parse port number from %q: %w", portStr, err) + } + + return port, nil +} + +// ServiceHost returns the host:port address for a service. +func (s *ComposeStack) ServiceHost(serviceName string, containerPort int) (string, error) { + s.t.Helper() + + port, err := s.ServicePort(serviceName, containerPort) + if err != nil { + return "", err + } + + return fmt.Sprintf("localhost:%d", port), nil +} + +// ConnectMySQLService connects to a MySQL service by name in the compose stack. +// Retries the connection and registers cleanup automatically. +func (s *ComposeStack) ConnectMySQLService(serviceName string) (*sql.DB, error) { + s.t.Helper() + + dsn, err := s.MySQLServiceDSN(serviceName) + if err != nil { + return nil, err + } + + // Retry connection a few times as MySQL might still be initializing + var db *sql.DB + for i := 0; i < 10; i++ { + db, err = sql.Open("mysql", dsn) + if err != nil { + return nil, fmt.Errorf("failed to open mysql connection: %w", err) + } + + if err = db.Ping(); err == nil { + break + } + + db.Close() + time.Sleep(1 * time.Second) + } + + if err != nil { + return nil, fmt.Errorf("failed to connect to %s mysql after retries: %w", serviceName, err) + } + + port, _ := s.ServicePort(serviceName, 3306) // We already got the port successfully + s.log.Logf("Connected to %s MySQL at localhost:%d", serviceName, port) + + // Register cleanup + s.t.Cleanup(func() { + s.log.Logf("Closing %s MySQL connection", serviceName) + db.Close() + }) + + return db, nil +} + +// MySQLServiceDSN returns the DSN string for a MySQL service. +// Useful when the implementation manages its own database connection. +func (s *ComposeStack) MySQLServiceDSN(serviceName string) (string, error) { + s.t.Helper() + + port, err := s.ServicePort(serviceName, 3306) + if err != nil { + return "", err + } + + return fmt.Sprintf("root:root@tcp(localhost:%d)/submitqueue?parseTime=true", port), nil +} + +// ConnectGRPC creates a gRPC client connection to a service. +func (s *ComposeStack) ConnectGRPC(serviceName string, containerPort int) (*grpc.ClientConn, error) { + s.t.Helper() + + addr, err := s.ServiceHost(serviceName, containerPort) + if err != nil { + return nil, err + } + + // Retry connection a few times as service might still be starting + var conn *grpc.ClientConn + for i := 0; i < 10; i++ { + conn, err = grpc.NewClient(addr, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err == nil { + break + } + time.Sleep(1 * time.Second) + } + + if err != nil { + return nil, fmt.Errorf("failed to connect to %s after retries: %w", serviceName, err) + } + + s.log.Logf("Connected to %s at %s", serviceName, addr) + + // Register cleanup + s.t.Cleanup(func() { + s.log.Logf("Closing gRPC connection to %s", serviceName) + conn.Close() + }) + + return conn, nil +} + +// setupDockerEnv configures Docker environment for docker-compose. +func setupDockerEnv(t *testing.T) { + t.Helper() + + // Ensure HOME is set for Docker config + if os.Getenv("HOME") == "" { + t.Setenv("HOME", t.TempDir()) + } +} + +// FindRepoRoot finds the repository root. +// Checks REPO_ROOT env var, then git, then walks up to find marker files. +func FindRepoRoot(t *testing.T) string { + t.Helper() + + // Check if REPO_ROOT is set (from .envrc or test environment) + if repoRoot := os.Getenv("REPO_ROOT"); repoRoot != "" { + return repoRoot + } + + // Try git (works outside Bazel sandbox) + cmd := exec.Command("git", "rev-parse", "--show-toplevel") + if output, err := cmd.Output(); err == nil { + if repoRoot := strings.TrimSpace(string(output)); repoRoot != "" { + return repoRoot + } + } + + // Walk up from current directory to find marker files + // In Bazel sandbox, marker files are symlinks - resolve them to get source location + dir, err := os.Getwd() + require.NoError(t, err, "failed to get working directory") + + for { + // Try to find and resolve marker file symlinks + for _, marker := range []string{"MODULE.bazel", "go.mod"} { + markerPath := filepath.Join(dir, marker) + if realMarker, err := filepath.EvalSymlinks(markerPath); err == nil { + return filepath.Dir(realMarker) + } + } + + // Move up one directory + parent := filepath.Dir(dir) + if parent == dir { + t.Fatalf("repository root not found") + } + dir = parent + } +} diff --git a/test/testutil/logger.go b/test/testutil/logger.go new file mode 100644 index 00000000..33a15cf6 --- /dev/null +++ b/test/testutil/logger.go @@ -0,0 +1,30 @@ +package testutil + +import ( + "testing" + "time" +) + +// TestLogger is a simple test-aware logger that records elapsed time between logs. +type TestLogger struct { + t *testing.T // The testing object to report logs to. + last time.Time // Timestamp of the last log, for elapsed calculation. +} + +// NewTestLogger creates a TestLogger for the current test. +func NewTestLogger(t *testing.T) *TestLogger { + t.Helper() + return &TestLogger{t: t} +} + +// Logf prints a formatted log message with timestamp and elapsed time since last log. +func (l *TestLogger) Logf(format string, args ...any) { + l.t.Helper() + now := time.Now() + delta := "" + if !l.last.IsZero() { + delta = " +" + now.Sub(l.last).Truncate(time.Millisecond).String() + } + l.last = now + l.t.Logf("[%s%s] "+format, append([]any{now.Format(time.RFC3339Nano), delta}, args...)...) +} diff --git a/test/testutil/schema.go b/test/testutil/schema.go new file mode 100644 index 00000000..595220b0 --- /dev/null +++ b/test/testutil/schema.go @@ -0,0 +1,50 @@ +package testutil + +import ( + "context" + "database/sql" + "os" + "path/filepath" + "sort" + "testing" + + _ "github.com/go-sql-driver/mysql" + "github.com/stretchr/testify/require" +) + +// SchemaDir returns the path to a schema directory. +// It checks for both Bazel runfiles and direct go test paths. +// relativePath should be like "extension/storage/mysql/schema" or "extension/queue/sql/schema" +func SchemaDir(relativePath string) string { + // Bazel runfiles path + if dir := os.Getenv("TEST_SRCDIR"); dir != "" { + return filepath.Join(dir, os.Getenv("TEST_WORKSPACE"), relativePath) + } + // Direct go test path (run from repo root) + return relativePath +} + +// ApplySchema reads all .sql files from the schema directory and executes them on the database. +func ApplySchema(t *testing.T, log *TestLogger, db *sql.DB, schemaDirectory string) { + t.Helper() + + files, err := filepath.Glob(filepath.Join(schemaDirectory, "*.sql")) + require.NoError(t, err, "failed to glob schema files") + require.NotEmpty(t, files, "no .sql schema files found in %s", schemaDirectory) + + // Sort files to ensure deterministic schema application order. + sort.Strings(files) + + for _, f := range files { + name := filepath.Base(f) + log.Logf("Applying schema: %s", name) + + content, err := os.ReadFile(f) + require.NoError(t, err, "failed to read schema file %s", name) + + _, err = db.ExecContext(context.Background(), string(content)) + require.NoError(t, err, "failed to execute schema file %s", name) + + log.Logf("Schema applied: %s", name) + } +}