diff --git a/.github/workflows/daily-semgrep-scan.lock.yml b/.github/workflows/daily-semgrep-scan.lock.yml index f1376e24a23..0308aa9880d 100644 --- a/.github/workflows/daily-semgrep-scan.lock.yml +++ b/.github/workflows/daily-semgrep-scan.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v3","frontmatter_hash":"740e4fcbce5e8e0b46f69f916e2d7e2bd23709871efc09eb1d00747115a5f3a6","strict":true,"agent_id":"copilot"} +# gh-aw-metadata: {"schema_version":"v3","frontmatter_hash":"c908a89f2ca11c9e9a11f432ee55fd19afe84ec9448e253846cd683f5eff2316","strict":true,"agent_id":"copilot"} # gh-aw-manifest: {"version":1,"secrets":["GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GH_AW_OTEL_GRAFANA_AUTHORIZATION","GH_AW_OTEL_GRAFANA_ENDPOINT","GH_AW_OTEL_SENTRY_AUTHORIZATION","GH_AW_OTEL_SENTRY_ENDPOINT","GITHUB_TOKEN"],"actions":[{"repo":"actions/checkout","sha":"de0fac2e4500dabe0009e67214ff5f5447ce83dd","version":"v6.0.2"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e","version":"v6.4.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"github/codeql-action/upload-sarif","sha":"68bde559dea0fdcac2102bfdf6230c5f70eb485e","version":"v4.35.4"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.25.46"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.25.46"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.25.46"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.3.9","digest":"sha256:64828b42a4482f58fab16509d7f8f495a6d97c972a98a68aff20543531ac0388","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.3.9@sha256:64828b42a4482f58fab16509d7f8f495a6d97c972a98a68aff20543531ac0388"},{"image":"ghcr.io/github/github-mcp-server:v1.0.4"},{"image":"node:lts-alpine","digest":"sha256:d1b3b4da11eefd5941e7f0b9cf17783fc99d9c6fc34884a665f40a06dbdfc94f","pinned_image":"node:lts-alpine@sha256:d1b3b4da11eefd5941e7f0b9cf17783fc99d9c6fc34884a665f40a06dbdfc94f"},{"image":"semgrep/semgrep:latest","digest":"sha256:17d89ddd91a7729bbd5de09402f7f79a70204289e2a94635086e9db532a495f2","pinned_image":"semgrep/semgrep:latest@sha256:17d89ddd91a7729bbd5de09402f7f79a70204289e2a94635086e9db532a495f2"}]} # ___ _ _ # / _ \ | | (_) @@ -94,6 +94,7 @@ jobs: comment_id: "" comment_repo: "" engine_id: ${{ steps.generate_aw_info.outputs.engine_id }} + experiments: ${{ steps.pick-experiment.outputs.experiments }} lockdown_check_failed: ${{ steps.generate_aw_info.outputs.lockdown_check_failed == 'true' }} model: ${{ steps.generate_aw_info.outputs.model }} setup-parent-span-id: ${{ steps.setup.outputs.parent-span-id || steps.setup.outputs.span-id }} @@ -181,10 +182,45 @@ jobs: setupGlobals(core, github, context, exec, io, getOctokit); const { main } = require('${{ runner.temp }}/gh-aw/actions/check_workflow_timestamp_api.cjs'); await main(); + - name: Restore experiment state from git + id: restore-experiment-state + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_EXPERIMENT_STATE_FILE: /tmp/gh-aw/experiments/state.json + GH_AW_EXPERIMENT_STATE_DIR: /tmp/gh-aw/experiments + GH_AW_EXPERIMENT_BRANCH: experiments/dailysemgrepscan + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/load_experiment_state_from_repo.cjs'); + await main(); + - name: Pick experiment variants + id: pick-experiment + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_EXPERIMENT_SPEC: '{"semgrep_output_format":{"variants":["bullet_list","structured_sections","prose"],"description":"Tests whether the structure of Semgrep findings output (bullet list vs. grouped sections vs. prose) affects code scanning alert creation rate and output completeness.","hypothesis":"H0: no change in alert creation rate across formats. H1: structured_sections produces ≥15% more alerts successfully created vs. baseline bullet_list.","metric":"alert_creation_rate","secondary_metrics":["run_duration_ms","output_length_chars","findings_reported"],"guardrail_metrics":[{"name":"run_success_rate","threshold":"\u003e=0.85"}],"min_samples":30,"weight":[34,33,33],"issue":32795,"start_date":"2026-05-17","analysis_type":"proportion_test","tags":["security","output-quality","semgrep"]}}' + GH_AW_EXPERIMENT_STATE_FILE: /tmp/gh-aw/experiments/state.json + GH_AW_EXPERIMENT_STATE_DIR: /tmp/gh-aw/experiments + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/pick_experiment.cjs'); + await main(); + - name: Upload experiment artifact + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: dailysemgrepscan-experiment + path: /tmp/gh-aw/experiments + if-no-files-found: ignore + retention-days: 30 - name: Create prompt with built-in context env: GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt GH_AW_SAFE_OUTPUTS: ${{ runner.temp }}/gh-aw/safeoutputs/outputs.jsonl + GH_AW_EXPERIMENTS_SEMGREP_OUTPUT_FORMAT: ${{ steps.pick-experiment.outputs.semgrep_output_format }} GH_AW_EXPR_1A3A194A: ${{ github.event.discussion.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'discussion' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }} GH_AW_EXPR_463A214A: ${{ github.event.pull_request.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'pull_request' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }} GH_AW_EXPR_802A9F6A: ${{ github.event.issue.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'issue' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }} @@ -197,20 +233,20 @@ jobs: run: | bash "${RUNNER_TEMP}/gh-aw/actions/create_prompt_first.sh" { - cat << 'GH_AW_PROMPT_b13ab3d5a7d8d3c5_EOF' + cat << 'GH_AW_PROMPT_b869e23287de95d2_EOF' - GH_AW_PROMPT_b13ab3d5a7d8d3c5_EOF + GH_AW_PROMPT_b869e23287de95d2_EOF cat "${RUNNER_TEMP}/gh-aw/prompts/xpia.md" cat "${RUNNER_TEMP}/gh-aw/prompts/temp_folder_prompt.md" cat "${RUNNER_TEMP}/gh-aw/prompts/markdown.md" cat "${RUNNER_TEMP}/gh-aw/prompts/safe_outputs_prompt.md" - cat << 'GH_AW_PROMPT_b13ab3d5a7d8d3c5_EOF' + cat << 'GH_AW_PROMPT_b869e23287de95d2_EOF' Tools: create_code_scanning_alert, missing_tool, missing_data, noop - GH_AW_PROMPT_b13ab3d5a7d8d3c5_EOF + GH_AW_PROMPT_b869e23287de95d2_EOF cat "${RUNNER_TEMP}/gh-aw/prompts/mcp_cli_tools_prompt.md" - cat << 'GH_AW_PROMPT_b13ab3d5a7d8d3c5_EOF' + cat << 'GH_AW_PROMPT_b869e23287de95d2_EOF' The following GitHub context information is available for this workflow: {{#if github.actor}} @@ -239,22 +275,23 @@ jobs: {{/if}} - GH_AW_PROMPT_b13ab3d5a7d8d3c5_EOF + GH_AW_PROMPT_b869e23287de95d2_EOF cat "${RUNNER_TEMP}/gh-aw/prompts/github_mcp_tools_with_safeoutputs_prompt.md" - cat << 'GH_AW_PROMPT_b13ab3d5a7d8d3c5_EOF' + cat << 'GH_AW_PROMPT_b869e23287de95d2_EOF' {{#runtime-import .github/workflows/shared/security-analysis-base.md}} {{#runtime-import .github/workflows/shared/mcp/semgrep.md}} {{#runtime-import .github/workflows/shared/observability-otlp.md}} {{#runtime-import .github/workflows/shared/noop-reminder.md}} {{#runtime-import .github/workflows/daily-semgrep-scan.md}} - GH_AW_PROMPT_b13ab3d5a7d8d3c5_EOF + GH_AW_PROMPT_b869e23287de95d2_EOF } > "$GH_AW_PROMPT" - name: Interpolate variables and render templates uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt GH_AW_ENGINE_ID: "copilot" + GH_AW_EXPERIMENTS_SEMGREP_OUTPUT_FORMAT: ${{ steps.pick-experiment.outputs.semgrep_output_format }} with: script: | const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); @@ -265,6 +302,7 @@ jobs: uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_EXPERIMENTS_SEMGREP_OUTPUT_FORMAT: ${{ steps.pick-experiment.outputs.semgrep_output_format }} GH_AW_EXPR_1A3A194A: ${{ github.event.discussion.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'discussion' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }} GH_AW_EXPR_463A214A: ${{ github.event.pull_request.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'pull_request' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }} GH_AW_EXPR_802A9F6A: ${{ github.event.issue.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'issue' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }} @@ -285,6 +323,7 @@ jobs: return await substitutePlaceholders({ file: process.env.GH_AW_PROMPT, substitutions: { + GH_AW_EXPERIMENTS_SEMGREP_OUTPUT_FORMAT: process.env.GH_AW_EXPERIMENTS_SEMGREP_OUTPUT_FORMAT, GH_AW_EXPR_1A3A194A: process.env.GH_AW_EXPR_1A3A194A, GH_AW_EXPR_463A214A: process.env.GH_AW_EXPR_463A214A, GH_AW_EXPR_802A9F6A: process.env.GH_AW_EXPR_802A9F6A, @@ -463,9 +502,9 @@ jobs: mkdir -p "${RUNNER_TEMP}/gh-aw/safeoutputs" mkdir -p /tmp/gh-aw/safeoutputs mkdir -p /tmp/gh-aw/mcp-logs/safeoutputs - cat > "${RUNNER_TEMP}/gh-aw/safeoutputs/config.json" << 'GH_AW_SAFE_OUTPUTS_CONFIG_c043754d50548469_EOF' + cat > "${RUNNER_TEMP}/gh-aw/safeoutputs/config.json" << 'GH_AW_SAFE_OUTPUTS_CONFIG_e80200e022fbfd44_EOF' {"create_code_scanning_alert":{"driver":"Semgrep Security Scanner"},"create_report_incomplete_issue":{},"missing_data":{},"missing_tool":{},"noop":{"max":1,"report-as-issue":"true"},"report_incomplete":{}} - GH_AW_SAFE_OUTPUTS_CONFIG_c043754d50548469_EOF + GH_AW_SAFE_OUTPUTS_CONFIG_e80200e022fbfd44_EOF - name: Generate Safe Outputs Tools env: GH_AW_TOOLS_META_JSON: | @@ -678,7 +717,7 @@ jobs: mkdir -p /home/runner/.copilot GH_AW_NODE=$(which node 2>/dev/null || command -v node 2>/dev/null || echo node) - cat << GH_AW_MCP_CONFIG_02817521c009714f_EOF | "$GH_AW_NODE" "${RUNNER_TEMP}/gh-aw/actions/start_mcp_gateway.cjs" + cat << GH_AW_MCP_CONFIG_d4ba50035aa20c37_EOF | "$GH_AW_NODE" "${RUNNER_TEMP}/gh-aw/actions/start_mcp_gateway.cjs" { "mcpServers": { "github": { @@ -744,7 +783,7 @@ jobs: } } } - GH_AW_MCP_CONFIG_02817521c009714f_EOF + GH_AW_MCP_CONFIG_d4ba50035aa20c37_EOF - name: Mount MCP servers as CLIs id: mount-mcp-clis continue-on-error: true @@ -985,6 +1024,7 @@ jobs: - activation - agent - detection + - push_experiments_state - safe_outputs - upload_code_scanning_sarif if: > @@ -1184,6 +1224,12 @@ jobs: mkdir -p /tmp/gh-aw/ find "/tmp/gh-aw/" -type f -print echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/agent_output.json" >> "$GITHUB_OUTPUT" + - name: Download experiment artifact + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: dailysemgrepscan-experiment + path: /tmp/gh-aw/experiments/ - name: Checkout repository for patch context if: needs.agent.outputs.has_patch == 'true' uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -1343,6 +1389,83 @@ jobs: } } + push_experiments_state: + needs: activation + if: always() && (!cancelled()) && needs.activation.result == 'success' + runs-on: ubuntu-slim + permissions: + contents: write + steps: + - name: Checkout actions folder + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + repository: github/gh-aw + sparse-checkout: | + actions + persist-credentials: false + - name: Setup Scripts + id: setup + uses: ./actions/setup + with: + destination: ${{ runner.temp }}/gh-aw/actions + job-name: ${{ github.job }} + trace-id: ${{ needs.activation.outputs.setup-trace-id }} + parent-span-id: ${{ needs.activation.outputs.setup-parent-span-id || needs.activation.outputs.setup-span-id }} + env: + GH_AW_SETUP_WORKFLOW_NAME: "Daily Semgrep Scan" + GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/daily-semgrep-scan.lock.yml@${{ github.ref }} + GH_AW_INFO_VERSION: "1.0.48" + GH_AW_INFO_ENGINE_ID: "copilot" + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + sparse-checkout: . + - name: Configure Git credentials + env: + REPO_NAME: ${{ github.repository }} + SERVER_URL: ${{ github.server_url }} + GITHUB_TOKEN: ${{ github.token }} + run: | + git config --global user.email "github-actions[bot]@users.noreply.github.com" + git config --global user.name "github-actions[bot]" + git config --global am.keepcr true + # Re-authenticate git with GitHub token + SERVER_URL_STRIPPED="${SERVER_URL#https://}" + git remote set-url origin "https://x-access-token:${GITHUB_TOKEN}@${SERVER_URL_STRIPPED}/${REPO_NAME}.git" + echo "Git configured with standard GitHub Actions identity" + - name: Download experiment artifact + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + continue-on-error: true + with: + name: dailysemgrepscan-experiment + path: /tmp/gh-aw/experiments + - name: Push experiment state to git + id: push_experiments_state + if: always() + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_TOKEN: ${{ github.token }} + GITHUB_RUN_ID: ${{ github.run_id }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GH_AW_EXPERIMENT_STATE_DIR: /tmp/gh-aw/experiments + GH_AW_EXPERIMENT_BRANCH: experiments/dailysemgrepscan + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/push_experiment_state.cjs'); + await main(); + - name: Restore actions folder + if: always() + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + repository: github/gh-aw + sparse-checkout: | + actions/setup + sparse-checkout-cone-mode: true + persist-credentials: false + safe_outputs: needs: - activation diff --git a/.github/workflows/daily-semgrep-scan.md b/.github/workflows/daily-semgrep-scan.md index 5fe5f318503..e7c5f95a282 100644 --- a/.github/workflows/daily-semgrep-scan.md +++ b/.github/workflows/daily-semgrep-scan.md @@ -21,8 +21,45 @@ safe-outputs: tools: cli-proxy: true +experiments: + semgrep_output_format: + variants: [bullet_list, structured_sections, prose] + description: "Tests whether the structure of Semgrep findings output (bullet list vs. grouped sections vs. prose) affects code scanning alert creation rate and output completeness." + hypothesis: "H0: no change in alert creation rate across formats. H1: structured_sections produces ≥15% more alerts successfully created vs. baseline bullet_list." + metric: alert_creation_rate + secondary_metrics: [run_duration_ms, output_length_chars, findings_reported] + guardrail_metrics: + - name: run_success_rate + threshold: ">=0.85" + min_samples: 30 + weight: [34, 33, 33] + start_date: "2026-05-17" + analysis_type: proportion_test + tags: [security, output-quality, semgrep] + issue: 32795 + --- Scan the repository for SQL injection vulnerabilities using Semgrep. +{{#if experiments.semgrep_output_format == 'bullet_list' }} +Report each finding as a flat bullet point in this format: +- **[SEVERITY]** `:` — Rule: `` — + +Create one code scanning alert per finding. +{{/if}} +{{#if experiments.semgrep_output_format == 'structured_sections' }} +Structure your findings report with: +1. A summary table: | Severity | Count | +2. Sections grouped by severity (Critical, High, Medium, Low), then by rule ID +3. For each finding: file path, line number, rule, and recommended fix + +Create one code scanning alert per finding. +{{/if}} +{{#if experiments.semgrep_output_format == 'prose' }} +Write a narrative security assessment describing the vulnerability patterns found. Embed specific findings (file, line, rule) within the prose. Conclude with a prioritized remediation list. + +Create one code scanning alert per finding. +{{/if}} + {{#runtime-import shared/noop-reminder.md}}