diff --git a/.github/workflows/scheduled-test.yml b/.github/workflows/scheduled-test.yml new file mode 100644 index 0000000..2b57734 --- /dev/null +++ b/.github/workflows/scheduled-test.yml @@ -0,0 +1,184 @@ +name: "Scheduled - Test All Features" + +on: + schedule: + - cron: '0 0 * * 0' # Every Sunday at midnight UTC + workflow_dispatch: + +permissions: + contents: read + +jobs: + get-all-features: + runs-on: ubuntu-latest + outputs: + features: ${{ steps.list.outputs.features }} + steps: + - uses: actions/checkout@v4 + + - name: "List all features" + id: list + run: | + FEATURES=$(ls src/ | sort | jq -R -s -c 'split("\n") | map(select(length > 0))') + echo "features=$FEATURES" >> $GITHUB_OUTPUT + + test-autogenerated: + needs: get-all-features + runs-on: ubuntu-latest + timeout-minutes: 60 + continue-on-error: true + strategy: + fail-fast: false + max-parallel: 10 + matrix: + feature: ${{ fromJson(needs.get-all-features.outputs.features) }} + baseImage: + - debian:latest + - ubuntu:latest + - mcr.microsoft.com/devcontainers/base:ubuntu + steps: + - uses: actions/checkout@v4 + + - name: "Sanitize base image name" + run: | + BASE_IMAGE_SAFE=$(echo "${{ matrix.baseImage }}" | sed 's/[/: ]/-/g') + echo "BASE_IMAGE_SAFE=$BASE_IMAGE_SAFE" >> $GITHUB_ENV + + - name: "Install latest devcontainer CLI" + run: npm install -g @devcontainers/cli + + - name: "Test '${{ matrix.feature }}' against '${{ matrix.baseImage }}' (with retry)" + run: | + LOG_FILE="${{ runner.temp }}/test-log.txt" + max_attempts=3 + attempt=1 + while true; do + echo "=== Attempt $attempt of $max_attempts ===" | tee -a "$LOG_FILE" + if devcontainer features test \ + --skip-scenarios \ + -f "${{ matrix.feature }}" \ + -i "${{ matrix.baseImage }}" \ + . 2>&1 | tee -a "$LOG_FILE"; then + echo "Tests passed on attempt $attempt" + exit 0 + fi + echo "Attempt $attempt failed" | tee -a "$LOG_FILE" + if [ "$attempt" -ge "$max_attempts" ]; then + echo "All $max_attempts attempts failed" | tee -a "$LOG_FILE" + exit 1 + fi + attempt=$((attempt + 1)) + echo "Retrying in 30 seconds..." | tee -a "$LOG_FILE" + sleep 30 + done + + - name: "Upload failure log" + if: failure() + uses: actions/upload-artifact@v4 + with: + name: failure-autogen-${{ matrix.feature }}-${{ env.BASE_IMAGE_SAFE }} + path: ${{ runner.temp }}/test-log.txt + + test-scenarios: + needs: get-all-features + runs-on: ubuntu-latest + timeout-minutes: 60 + continue-on-error: true + strategy: + fail-fast: false + max-parallel: 10 + matrix: + feature: ${{ fromJson(needs.get-all-features.outputs.features) }} + steps: + - uses: actions/checkout@v4 + + - name: "Install latest devcontainer CLI" + run: npm install -g @devcontainers/cli + + - name: "Test '${{ matrix.feature }}' scenarios (with retry)" + run: | + LOG_FILE="${{ runner.temp }}/test-log.txt" + max_attempts=3 + attempt=1 + while true; do + echo "=== Attempt $attempt of $max_attempts ===" | tee -a "$LOG_FILE" + if devcontainer features test \ + -f "${{ matrix.feature }}" \ + --skip-autogenerated \ + --skip-duplicated \ + . 2>&1 | tee -a "$LOG_FILE"; then + echo "Tests passed on attempt $attempt" + exit 0 + fi + echo "Attempt $attempt failed" | tee -a "$LOG_FILE" + if [ "$attempt" -ge "$max_attempts" ]; then + echo "All $max_attempts attempts failed" | tee -a "$LOG_FILE" + exit 1 + fi + attempt=$((attempt + 1)) + echo "Retrying in 30 seconds..." | tee -a "$LOG_FILE" + sleep 30 + done + + - name: "Upload failure log" + if: failure() + uses: actions/upload-artifact@v4 + with: + name: failure-scenarios-${{ matrix.feature }} + path: ${{ runner.temp }}/test-log.txt + + create-issue: + needs: [test-autogenerated, test-scenarios] + if: always() + runs-on: ubuntu-latest + permissions: + issues: write + steps: + - name: "Download all failure artifacts" + id: download + continue-on-error: true + uses: actions/download-artifact@v4 + with: + pattern: failure-* + path: ${{ runner.temp }}/failures + + - name: "Create issue for failures" + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + FAILURES_DIR="${{ runner.temp }}/failures" + if [ ! -d "$FAILURES_DIR" ] || [ -z "$(ls -A "$FAILURES_DIR" 2>/dev/null)" ]; then + echo "No test failures found. Skipping issue creation." + exit 0 + fi + + # Build issue body + { + echo "## Scheduled Feature Test Failures" + echo "" + echo "The weekly scheduled test detected failures in one or more features after **3 retry attempts**." + echo "" + echo "**Workflow run:** ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" + echo "" + echo "---" + echo "" + for dir in "$FAILURES_DIR"/*/; do + artifact_name=$(basename "$dir") + echo "### \`$artifact_name\`" + echo "" + echo "
" + echo "Expand log" + echo "" + echo '```' + cat "$dir/test-log.txt" 2>/dev/null || echo "No log available" + echo '```' + echo "" + echo "
" + echo "" + done + } > "${{ runner.temp }}/issue-body.md" + + gh issue create \ + --repo "${{ github.repository }}" \ + --title "Scheduled Test Failures - ${{ github.run_started_at }}" \ + --body-file "${{ runner.temp }}/issue-body.md"