From 3e949b2ae8aadc39c3b684b67bf858cf34e66eaf Mon Sep 17 00:00:00 2001 From: tduhamel42 Date: Wed, 22 Oct 2025 11:45:04 +0200 Subject: [PATCH] ci: add worker validation and Docker build checks Add automated validation to prevent worker-related issues: **Worker Validation Script:** - New script: .github/scripts/validate-workers.sh - Validates all workers in docker-compose.yml exist - Checks required files: Dockerfile, requirements.txt, worker.py - Verifies files are tracked by git (not gitignored) - Detects gitignore issues that could hide workers **CI Workflow Updates:** - Added validate-workers job (runs on every PR) - Added build-workers job (runs if workers/ modified) - Uses Docker Buildx for caching - Validates Docker images build successfully - Updated test-summary to check validation results **PR Template:** - New pull request template with comprehensive checklist - Specific section for worker-related changes - Reminds contributors to validate worker files - Includes documentation and changelog reminders These checks would have caught the secrets worker gitignore issue. Implements Phase 1 improvements from CI/CD quality assessment. --- .github/pull_request_template.md | 79 ++++++++++++++++++++++++ .github/scripts/validate-workers.sh | 95 +++++++++++++++++++++++++++++ .github/workflows/test.yml | 36 ++++++++++- 3 files changed, 209 insertions(+), 1 deletion(-) create mode 100644 .github/pull_request_template.md create mode 100755 .github/scripts/validate-workers.sh diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 00000000..04ece70a --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,79 @@ +## Description + + + +## Type of Change + + + +- [ ] 🐛 Bug fix (non-breaking change which fixes an issue) +- [ ] ✨ New feature (non-breaking change which adds functionality) +- [ ] 💥 Breaking change (fix or feature that would cause existing functionality to not work as expected) +- [ ] 📝 Documentation update +- [ ] 🔧 Configuration change +- [ ] ♻️ Refactoring (no functional changes) +- [ ] 🎨 Style/formatting changes +- [ ] ✅ Test additions or updates + +## Related Issues + + + + +## Changes Made + + + +- +- +- + +## Testing + + + +### Tested Locally + +- [ ] All tests pass (`pytest`, `uv build`, etc.) +- [ ] Linting passes (`ruff check`) +- [ ] Code builds successfully + +### Worker Changes (if applicable) + +- [ ] Docker images build successfully (`docker compose build`) +- [ ] Worker containers start correctly +- [ ] Tested with actual workflow execution + +### Documentation + +- [ ] Documentation updated (if needed) +- [ ] README updated (if needed) +- [ ] CHANGELOG.md updated (if user-facing changes) + +## Pre-Merge Checklist + + + +- [ ] My code follows the project's coding standards +- [ ] I have performed a self-review of my code +- [ ] I have commented my code, particularly in hard-to-understand areas +- [ ] I have made corresponding changes to the documentation +- [ ] My changes generate no new warnings +- [ ] I have added tests that prove my fix is effective or that my feature works +- [ ] New and existing unit tests pass locally with my changes +- [ ] Any dependent changes have been merged and published + +### Worker-Specific Checks (if workers/ modified) + +- [ ] All worker files properly tracked by git (not gitignored) +- [ ] Worker validation script passes (`.github/scripts/validate-workers.sh`) +- [ ] Docker images build without errors +- [ ] Worker configuration updated in `docker-compose.yml` (if needed) + +## Screenshots (if applicable) + + + +## Additional Notes + + diff --git a/.github/scripts/validate-workers.sh b/.github/scripts/validate-workers.sh new file mode 100755 index 00000000..8170181f --- /dev/null +++ b/.github/scripts/validate-workers.sh @@ -0,0 +1,95 @@ +#!/bin/bash +# Worker Validation Script +# Ensures all workers defined in docker-compose.yml exist in the repository +# and are properly tracked by git. + +set -e + +echo "🔍 Validating worker completeness..." + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +ERRORS=0 +WARNINGS=0 + +# Extract worker service names from docker-compose.yml +echo "" +echo "📋 Checking workers defined in docker-compose.yml..." +WORKERS=$(grep -E "^\s+worker-" docker-compose.yml | grep -v "#" | cut -d: -f1 | tr -d ' ' | sort -u) + +if [ -z "$WORKERS" ]; then + echo -e "${RED}❌ No workers found in docker-compose.yml${NC}" + exit 1 +fi + +echo "Found workers:" +for worker in $WORKERS; do + echo " - $worker" +done + +# Check each worker +echo "" +echo "🔎 Validating worker files..." +for worker in $WORKERS; do + WORKER_DIR="workers/${worker#worker-}" + + echo "" + echo "Checking $worker ($WORKER_DIR)..." + + # Check if directory exists + if [ ! -d "$WORKER_DIR" ]; then + echo -e "${RED} ❌ Directory not found: $WORKER_DIR${NC}" + ERRORS=$((ERRORS + 1)) + continue + fi + + # Check required files + REQUIRED_FILES=("Dockerfile" "requirements.txt" "worker.py") + for file in "${REQUIRED_FILES[@]}"; do + FILE_PATH="$WORKER_DIR/$file" + + if [ ! -f "$FILE_PATH" ]; then + echo -e "${RED} ❌ Missing file: $FILE_PATH${NC}" + ERRORS=$((ERRORS + 1)) + else + # Check if file is tracked by git + if ! git ls-files --error-unmatch "$FILE_PATH" &> /dev/null; then + echo -e "${RED} ❌ File not tracked by git: $FILE_PATH${NC}" + echo -e "${YELLOW} Check .gitignore patterns!${NC}" + ERRORS=$((ERRORS + 1)) + else + echo -e "${GREEN} ✓ $file (tracked)${NC}" + fi + fi + done +done + +# Check for any ignored worker files +echo "" +echo "🚫 Checking for gitignored worker files..." +IGNORED_FILES=$(git check-ignore workers/*/* 2>/dev/null || true) +if [ -n "$IGNORED_FILES" ]; then + echo -e "${YELLOW}⚠️ Warning: Some worker files are being ignored:${NC}" + echo "$IGNORED_FILES" | while read -r file; do + echo -e "${YELLOW} - $file${NC}" + done + WARNINGS=$((WARNINGS + 1)) +fi + +# Summary +echo "" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +if [ $ERRORS -eq 0 ] && [ $WARNINGS -eq 0 ]; then + echo -e "${GREEN}✅ All workers validated successfully!${NC}" + exit 0 +elif [ $ERRORS -eq 0 ]; then + echo -e "${YELLOW}⚠️ Validation passed with $WARNINGS warning(s)${NC}" + exit 0 +else + echo -e "${RED}❌ Validation failed with $ERRORS error(s) and $WARNINGS warning(s)${NC}" + exit 1 +fi diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 03581ef5..7f2f6bf2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -7,6 +7,36 @@ on: branches: [ main, master, develop ] jobs: + validate-workers: + name: Validate Workers + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Run worker validation + run: | + chmod +x .github/scripts/validate-workers.sh + .github/scripts/validate-workers.sh + + build-workers: + name: Build Worker Docker Images + runs-on: ubuntu-latest + # Only run if workers directory is modified + if: | + github.event_name == 'pull_request' && + contains(github.event.pull_request.changed_files, 'workers/') + steps: + - uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build worker images + run: | + echo "Building worker Docker images..." + docker compose build worker-python worker-secrets worker-rust worker-android worker-ossfuzz --no-cache + continue-on-error: false + lint: name: Lint runs-on: ubuntu-latest @@ -143,11 +173,15 @@ jobs: test-summary: name: Test Summary runs-on: ubuntu-latest - needs: [lint, unit-tests] + needs: [validate-workers, lint, unit-tests] if: always() steps: - name: Check test results run: | + if [ "${{ needs.validate-workers.result }}" != "success" ]; then + echo "Worker validation failed" + exit 1 + fi if [ "${{ needs.unit-tests.result }}" != "success" ]; then echo "Unit tests failed" exit 1