diff --git a/.github/instructions/github-agentic-workflows.instructions.md b/.github/instructions/github-agentic-workflows.instructions.md index 577558da6cb..c54708f9a22 100644 --- a/.github/instructions/github-agentic-workflows.instructions.md +++ b/.github/instructions/github-agentic-workflows.instructions.md @@ -69,13 +69,15 @@ The YAML frontmatter supports these fields: version: beta # Optional: version of the action (has sensible default) model: gpt-5 # Optional: LLM model to use (has sensible default) max-turns: 5 # Optional: maximum chat iterations per run (has sensible default) + max-concurrency: 3 # Optional: max concurrent workflows across all workflows (default: 3) ``` - - **Note**: The `version`, `model`, and `max-turns` fields have sensible defaults and can typically be omitted unless you need specific customization. + - **Note**: The `version`, `model`, `max-turns`, and `max-concurrency` fields have sensible defaults and can typically be omitted unless you need specific customization. - **Custom engine format** (⚠️ experimental): ```yaml engine: id: custom # Required: custom engine identifier max-turns: 10 # Optional: maximum iterations (for consistency) + max-concurrency: 5 # Optional: max concurrent workflows (for consistency) steps: # Required: array of custom GitHub Actions steps - name: Setup Node.js uses: actions/setup-node@v4 diff --git a/.github/workflows/artifacts-summary.lock.yml b/.github/workflows/artifacts-summary.lock.yml index 5c0bd9ef6cf..16d4f473cc1 100644 --- a/.github/workflows/artifacts-summary.lock.yml +++ b/.github/workflows/artifacts-summary.lock.yml @@ -119,6 +119,8 @@ jobs: permissions: actions: read contents: read + concurrency: + group: "gh-aw-copilot-${{ github.run_id % 3 }}" env: GITHUB_AW_SAFE_OUTPUTS: /tmp/safe-outputs/outputs.jsonl GITHUB_AW_SAFE_OUTPUTS_CONFIG: "{\"create-issue\":{\"max\":1},\"missing-tool\":{}}" diff --git a/.github/workflows/brave.lock.yml b/.github/workflows/brave.lock.yml index 367df886b47..4f4487fdfc6 100644 --- a/.github/workflows/brave.lock.yml +++ b/.github/workflows/brave.lock.yml @@ -428,6 +428,8 @@ jobs: permissions: actions: read contents: read + concurrency: + group: "gh-aw-copilot-${{ github.run_id % 3 }}" env: GITHUB_AW_SAFE_OUTPUTS: /tmp/safe-outputs/outputs.jsonl GITHUB_AW_SAFE_OUTPUTS_CONFIG: "{\"add-comment\":{\"max\":1},\"missing-tool\":{}}" diff --git a/.github/workflows/ci-doctor.lock.yml b/.github/workflows/ci-doctor.lock.yml index b7ae259cf05..e11e80d8f68 100644 --- a/.github/workflows/ci-doctor.lock.yml +++ b/.github/workflows/ci-doctor.lock.yml @@ -70,6 +70,8 @@ jobs: needs: activation runs-on: ubuntu-latest permissions: read-all + concurrency: + group: "gh-aw-copilot-${{ github.run_id % 3 }}" env: GITHUB_AW_SAFE_OUTPUTS: /tmp/safe-outputs/outputs.jsonl GITHUB_AW_SAFE_OUTPUTS_CONFIG: "{\"add-comment\":{\"max\":1},\"create-issue\":{\"max\":1},\"missing-tool\":{}}" diff --git a/.github/workflows/dev.lock.yml b/.github/workflows/dev.lock.yml index 6103c1a9b29..370228e7dad 100644 --- a/.github/workflows/dev.lock.yml +++ b/.github/workflows/dev.lock.yml @@ -130,6 +130,8 @@ jobs: needs: activation runs-on: ubuntu-latest permissions: read-all + concurrency: + group: "gh-aw-claude-${{ github.run_id % 3 }}" env: GITHUB_AW_SAFE_OUTPUTS: /tmp/safe-outputs/outputs.jsonl GITHUB_AW_SAFE_OUTPUTS_CONFIG: "{\"create-issue\":{\"max\":1},\"missing-tool\":{}}" diff --git a/.github/workflows/duplicate-code-detector.lock.yml b/.github/workflows/duplicate-code-detector.lock.yml index e0aaa4ab20d..ecab985a456 100644 --- a/.github/workflows/duplicate-code-detector.lock.yml +++ b/.github/workflows/duplicate-code-detector.lock.yml @@ -122,6 +122,8 @@ jobs: permissions: actions: read contents: read + concurrency: + group: "gh-aw-copilot-${{ github.run_id % 3 }}" env: GITHUB_AW_SAFE_OUTPUTS: /tmp/safe-outputs/outputs.jsonl GITHUB_AW_SAFE_OUTPUTS_CONFIG: "{\"create-issue\":{\"max\":1},\"missing-tool\":{}}" @@ -819,11 +821,11 @@ jobs: "--rm", "-i", "-e", - "SERENA_DOCKER", - "-e", "SERENA_PORT", "-e", "SERENA_DASHBOARD_PORT", + "-e", + "SERENA_DOCKER", "-v", "${{ github.workspace }}:/workspace:ro", "-w", diff --git a/.github/workflows/go-pattern-detector.lock.yml b/.github/workflows/go-pattern-detector.lock.yml index b5431ba2571..029dfbd5044 100644 --- a/.github/workflows/go-pattern-detector.lock.yml +++ b/.github/workflows/go-pattern-detector.lock.yml @@ -126,6 +126,8 @@ jobs: permissions: actions: read contents: read + concurrency: + group: "gh-aw-claude-${{ github.run_id % 3 }}" env: GITHUB_AW_SAFE_OUTPUTS: /tmp/safe-outputs/outputs.jsonl GITHUB_AW_SAFE_OUTPUTS_CONFIG: "{\"create-issue\":{\"max\":1},\"missing-tool\":{}}" diff --git a/.github/workflows/issue-classifier.lock.yml b/.github/workflows/issue-classifier.lock.yml index 6e90a140eca..9dedff8be5b 100644 --- a/.github/workflows/issue-classifier.lock.yml +++ b/.github/workflows/issue-classifier.lock.yml @@ -425,6 +425,8 @@ jobs: actions: read contents: read models: read + concurrency: + group: "gh-aw-custom-${{ github.run_id % 3 }}" env: GITHUB_AW_SAFE_OUTPUTS: /tmp/safe-outputs/outputs.jsonl GITHUB_AW_SAFE_OUTPUTS_CONFIG: "{\"add-labels\":{\"allowed\":[\"bug\",\"feature\",\"enhancement\",\"documentation\"],\"max\":1},\"missing-tool\":{}}" diff --git a/.github/workflows/pdf-summary.lock.yml b/.github/workflows/pdf-summary.lock.yml index dfb0c7cb29b..5cc21cd8e93 100644 --- a/.github/workflows/pdf-summary.lock.yml +++ b/.github/workflows/pdf-summary.lock.yml @@ -453,6 +453,8 @@ jobs: permissions: actions: read contents: read + concurrency: + group: "gh-aw-copilot-${{ github.run_id % 3 }}" env: GITHUB_AW_SAFE_OUTPUTS: /tmp/safe-outputs/outputs.jsonl GITHUB_AW_SAFE_OUTPUTS_CONFIG: "{\"add-comment\":{\"max\":1},\"missing-tool\":{}}" diff --git a/.github/workflows/poem-bot.lock.yml b/.github/workflows/poem-bot.lock.yml index aa588ebf779..b323c0de487 100644 --- a/.github/workflows/poem-bot.lock.yml +++ b/.github/workflows/poem-bot.lock.yml @@ -436,6 +436,8 @@ jobs: permissions: actions: read contents: read + concurrency: + group: "gh-aw-copilot-${{ github.run_id % 3 }}" env: GITHUB_AW_SAFE_OUTPUTS: /tmp/safe-outputs/outputs.jsonl GITHUB_AW_SAFE_OUTPUTS_CONFIG: "{\"add-comment\":{\"max\":3,\"target\":\"*\"},\"add-labels\":{\"allowed\":[\"poetry\",\"creative\",\"automation\",\"ai-generated\",\"epic\",\"haiku\",\"sonnet\",\"limerick\"],\"max\":5},\"create-issue\":{\"max\":2},\"create-pull-request\":{},\"create-pull-request-review-comment\":{\"max\":2},\"missing-tool\":{},\"push-to-pull-request-branch\":{},\"update-issue\":{\"max\":2},\"upload-asset\":{}}" diff --git a/.github/workflows/scout.lock.yml b/.github/workflows/scout.lock.yml index 6c8474c4826..3f4e593c229 100644 --- a/.github/workflows/scout.lock.yml +++ b/.github/workflows/scout.lock.yml @@ -467,6 +467,8 @@ jobs: permissions: actions: read contents: read + concurrency: + group: "gh-aw-copilot-${{ github.run_id % 3 }}" env: GITHUB_AW_SAFE_OUTPUTS: /tmp/safe-outputs/outputs.jsonl GITHUB_AW_SAFE_OUTPUTS_CONFIG: "{\"add-comment\":{\"max\":1},\"missing-tool\":{}}" diff --git a/.github/workflows/technical-doc-writer.lock.yml b/.github/workflows/technical-doc-writer.lock.yml index 4fa33d0fd74..a41bd071739 100644 --- a/.github/workflows/technical-doc-writer.lock.yml +++ b/.github/workflows/technical-doc-writer.lock.yml @@ -120,6 +120,8 @@ jobs: needs: activation runs-on: ubuntu-latest permissions: read-all + concurrency: + group: "gh-aw-claude-${{ github.run_id % 3 }}" env: GITHUB_AW_SAFE_OUTPUTS: /tmp/safe-outputs/outputs.jsonl GITHUB_AW_SAFE_OUTPUTS_CONFIG: "{\"add-comment\":{\"max\":1},\"create-pull-request\":{},\"missing-tool\":{},\"upload-asset\":{}}" diff --git a/.github/workflows/tidy.lock.yml b/.github/workflows/tidy.lock.yml index a9ae16be2a1..ce6cc33fa1b 100644 --- a/.github/workflows/tidy.lock.yml +++ b/.github/workflows/tidy.lock.yml @@ -289,6 +289,8 @@ jobs: permissions: actions: read contents: read + concurrency: + group: "gh-aw-copilot-${{ github.run_id % 3 }}" env: GITHUB_AW_SAFE_OUTPUTS: /tmp/safe-outputs/outputs.jsonl GITHUB_AW_SAFE_OUTPUTS_CONFIG: "{\"create-pull-request\":{},\"missing-tool\":{},\"push-to-pull-request-branch\":{}}" diff --git a/docs/src/content/docs/reference/concurrency.md b/docs/src/content/docs/reference/concurrency.md new file mode 100644 index 00000000000..f95d255396f --- /dev/null +++ b/docs/src/content/docs/reference/concurrency.md @@ -0,0 +1,361 @@ +--- +title: Concurrency Control +description: Complete guide to concurrency control in GitHub Agentic Workflows, including max-concurrency configuration, global locks, and engine isolation. +sidebar: + order: 6 +--- + +GitHub Agentic Workflows provides sophisticated concurrency control to manage how many AI-powered workflows can run simultaneously. This helps prevent resource exhaustion, control costs, and ensure predictable workflow execution. + +## Overview + +Concurrency control in GitHub Agentic Workflows uses a dual-level approach with different strategies at each level: +- **Workflow-level concurrency**: Context-specific limiting based on workflow type (issue, PR, branch, etc.) +- **Agent concurrency (max-concurrency)**: Global limiting across all workflows using the same engine + +This dual-level approach provides both fine-grained control per workflow and global resource management across all workflows. + +## Max Concurrency Configuration + +The `max-concurrency` option is configured under the `engine` section and controls how many agentic jobs can run concurrently across **all workflows** in your repository: + +```yaml +engine: + id: claude + max-concurrency: 5 +``` + +### Default Value + +- **Default**: 3 concurrent slots (when not specified or set to 0) +- **Minimum**: 1 (sequential execution) +- **Disabled**: -1 (no agent concurrency limiting) +- **No maximum**: Set to any positive integer based on your needs + +### Configuration Examples + +**Sequential execution (one at a time):** +```yaml +engine: + id: copilot + max-concurrency: 1 +``` + +**Moderate parallelism (default):** +```yaml +engine: + id: claude + # max-concurrency not specified, defaults to 3 +``` + +**High parallelism for busy repositories:** +```yaml +engine: + id: claude + max-concurrency: 10 +``` + +**Disable agent concurrency limiting:** +```yaml +engine: + id: claude + max-concurrency: -1 # No global limiting, only workflow-level concurrency applies +``` + +## How It Works + +### Workflow-Level Concurrency + +The workflow-level concurrency uses context-specific keys based on the trigger type: + +**For issue workflows:** +```yaml +concurrency: + group: "gh-aw-${{ github.workflow }}-${{ github.event.issue.number }}" +``` + +**For pull request workflows:** +```yaml +concurrency: + group: "gh-aw-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}" + cancel-in-progress: true +``` + +**For push workflows:** +```yaml +concurrency: + group: "gh-aw-${{ github.workflow }}-${{ github.ref }}" +``` + +**For schedule/other workflows:** +```yaml +concurrency: + group: "gh-aw-${{ github.workflow }}" +``` + +This ensures workflows operating on different issues, PRs, or branches can run concurrently without interfering with each other. + +### Agent Concurrency (Max-Concurrency) + +The agent concurrency uses **only** the engine ID and slot number for global limiting: + +```yaml +jobs: + agent: + concurrency: + group: "gh-aw-{engine-id}-${{ github.run_id % max-concurrency }}" +``` + +**Example for Claude with max-concurrency of 5:** +```yaml +jobs: + agent: + concurrency: + group: "gh-aw-claude-${{ github.run_id % 5 }}" +``` + +This creates a global lock across **all workflows and refs** for each engine, preventing resource exhaustion from too many concurrent AI executions. + +### Complete Example + +Here's how both levels work together in a generated workflow: + +```yaml +name: "Issue Responder" +on: + issues: + types: [opened] + +permissions: {} + +# Workflow-level: Context-specific concurrency +concurrency: + group: "gh-aw-${{ github.workflow }}-${{ github.event.issue.number }}" + +jobs: + agent: + runs-on: ubuntu-latest + permissions: read-all + # Agent concurrency: Global max-concurrency limiting + concurrency: + group: "gh-aw-claude-${{ github.run_id % 5 }}" + steps: + - name: Execute workflow + ... +``` + +### Slot Distribution + +Workflows are distributed across available slots using modulo arithmetic: +- `github.run_id % max-concurrency` calculates the slot number (0 to max-concurrency-1) +- Each slot can only run one workflow at a time +- Workflows are automatically assigned to the next available slot + +**Example with max-concurrency: 3** +- Run ID 1001 → Slot 2 (`1001 % 3 = 2`) +- Run ID 1002 → Slot 0 (`1002 % 3 = 0`) +- Run ID 1003 → Slot 1 (`1003 % 3 = 1`) +- Run ID 1004 → Slot 2 (`1004 % 3 = 2`) + +### Dual-Level Application + +The dual-level concurrency provides complementary control: + +1. **Workflow-level**: Prevents conflicts between runs of the same workflow on different contexts (e.g., different issues or PRs) +2. **Agent concurrency**: Prevents resource exhaustion by limiting total concurrent AI executions across all workflows + +**Example scenario:** +- 5 different issues trigger the same workflow +- Workflow-level concurrency allows all 5 to start (different issue numbers) +- Agent concurrency with max-concurrency (e.g., 3) ensures only 3 AI jobs run simultaneously +- The other 2 workflows queue until slots become available + +This approach balances: +- **Workflow isolation**: Different contexts don't block each other at the workflow level +- **Global resource management**: Total AI resource usage is controlled at the agent level + +## Global Lock Behavior + +The **agent concurrency** (max-concurrency) uses **only** engine ID and slot number, creating a true global lock: + +### What's Included in Agent Concurrency +- ✅ Engine ID (`copilot`, `claude`, `codex`) +- ✅ Slot number (from `run_id % max-concurrency`) +- ✅ `gh-aw-` prefix + +### What's NOT Included in Agent Concurrency +- ❌ Workflow name +- ❌ Issue number +- ❌ Pull request number +- ❌ Branch/ref name +- ❌ Event type + +This ensures the max-concurrency limit applies **repository-wide** across all workflows and refs for each engine. + +**Disabling Agent Concurrency**: +Set `max-concurrency: -1` to disable agent concurrency limiting entirely. When disabled, only workflow-level concurrency applies, and there is no global limit on concurrent AI executions across workflows. + +### Workflow-Level Concurrency Includes Context + +The **workflow-level** concurrency includes context-specific information: +- ✅ Workflow name +- ✅ Issue/PR/discussion number (when applicable) +- ✅ Branch ref (for push workflows) +- ✅ `gh-aw-` prefix + +This allows different contexts to run concurrently while preventing conflicts within the same context. + +## Engine Isolation + +Different engines can run concurrently without interfering with each other: + +```yaml +# Workflow A uses Copilot +engine: + id: copilot + max-concurrency: 3 + +# Workflow B uses Claude +engine: + id: claude + max-concurrency: 5 +``` + +- Copilot workflows have their own 3-slot concurrency pool +- Claude workflows have their own 5-slot concurrency pool +- Both can run simultaneously without conflict + +## Cancellation Behavior + +Concurrency cancellation varies by workflow trigger type: + +| Trigger Type | Cancel-in-Progress | Reason | +|--------------|-------------------|--------| +| `pull_request` | ✅ Enabled | New commits should cancel outdated PR runs | +| All other triggers | ❌ Disabled | Issue/discussion workflows should run to completion | + +**Example for pull request workflow:** +```yaml +concurrency: + group: "gh-aw-copilot-${{ github.run_id % 3 }}" + cancel-in-progress: true +``` + +## Benefits + +### Cost Control +- **Prevents runaway costs**: Limits the number of concurrent AI executions +- **Predictable spending**: Maximum concurrent workflows are known in advance +- **Flexible budgeting**: Adjust limits based on repository needs + +### Resource Management +- **Prevents resource exhaustion**: Ensures system stability +- **Fair resource distribution**: Workflows queue when slots are full +- **Maintains throughput**: Multiple workflows can still run concurrently + +### Engine Isolation +- **Independent limits**: Each engine has its own concurrency pool +- **No cross-engine interference**: Copilot workflows don't block Claude workflows +- **Flexible configuration**: Different limits for different engines + +### Simplicity +- **Global lock**: Same limit across all workflows and refs +- **Automatic distribution**: No manual slot assignment needed +- **Consistent behavior**: Predictable execution patterns + +## Custom Concurrency + +You can override the automatic concurrency generation by specifying your own `concurrency` section in the frontmatter: + +```yaml +--- +on: push +concurrency: + group: custom-group-${{ github.ref }} + cancel-in-progress: true +tools: + github: + allowed: [list_issues] +--- +``` + +**Note**: Custom concurrency bypasses the max-concurrency limit and engine isolation features. + +## Best Practices + +### Setting Max Concurrency + +**Start conservative:** +```yaml +engine: + id: claude + max-concurrency: 1 # Start with sequential execution +``` + +**Increase as needed:** +```yaml +engine: + id: claude + max-concurrency: 5 # Increase after monitoring costs and performance +``` + +### Different Limits for Different Engines + +**Cost-sensitive engine:** +```yaml +engine: + id: claude # Expensive model + max-concurrency: 2 +``` + +**Budget-friendly engine:** +```yaml +engine: + id: copilot # More affordable + max-concurrency: 5 +``` + +### Monitoring and Adjustment + +1. **Monitor workflow execution**: Use GitHub Actions insights +2. **Track costs**: Review AI model usage and expenses +3. **Adjust limits**: Increase or decrease based on needs +4. **Test changes**: Validate new limits with test workflows + +## Troubleshooting + +### Workflows Queuing + +**Symptom**: Workflows wait in queue instead of running + +**Cause**: All concurrency slots are full + +**Solution**: +- Increase `max-concurrency` value +- Check for long-running workflows +- Consider using different engines for different workflows + +### Too Many Concurrent Runs + +**Symptom**: High costs or resource usage + +**Cause**: `max-concurrency` set too high + +**Solution**: +- Decrease `max-concurrency` value +- Monitor usage patterns +- Set appropriate limits per engine + +### Workflows Not Canceling + +**Symptom**: Old pull request workflows continue running after new commits + +**Cause**: Custom concurrency without `cancel-in-progress` + +**Solution**: Ensure pull request workflows have `cancel-in-progress: true` in custom concurrency configuration + +## Related Documentation + +- [AI Engines](/gh-aw/reference/engines/) - Engine configuration and capabilities +- [Frontmatter Options](/gh-aw/reference/frontmatter/) - Complete frontmatter reference +- [Workflow Structure](/gh-aw/reference/workflow-structure/) - Overall workflow organization diff --git a/docs/src/content/docs/reference/frontmatter.md b/docs/src/content/docs/reference/frontmatter.md index 43cee25d0e9..5d57903f20d 100644 --- a/docs/src/content/docs/reference/frontmatter.md +++ b/docs/src/content/docs/reference/frontmatter.md @@ -348,6 +348,7 @@ engine: version: latest # Optional: version of the action model: gpt-5 # Optional: specific LLM model (for copilot) max-turns: 5 # Optional: maximum chat iterations per run (for claude) + max-concurrency: 3 # Optional: max concurrent workflows across all workflows (default: 3) env: # Optional: custom environment variables AWS_REGION: us-west-2 CUSTOM_API_ENDPOINT: https://api.example.com @@ -363,6 +364,7 @@ engine: - **`version`** (optional): Action version (`beta`, `stable`) - **`model`** (optional): Specific LLM model to use - **`max-turns`** (optional): Maximum number of chat iterations per run (cost-control option) +- **`max-concurrency`** (optional): Maximum number of concurrent workflows across all workflows (default: 3) - **`env`** (optional): Custom environment variables to pass to the agentic engine as key-value pairs - **`config`** (optional): Additional TOML configuration text appended to generated config.toml (codex engine only) @@ -463,6 +465,71 @@ engine: 3. Helps prevent runaway chat loops and control costs 4. Only applies to engines that support turn limiting (currently Claude) +### Concurrency Limiting + +The `max-concurrency` option limits how many agentic jobs can run concurrently across **all workflows** in your repository: + +```yaml +engine: + id: claude + max-concurrency: 5 +``` + +**Default Value:** 3 (if not specified) + +**How it works:** +- Uses GitHub Actions concurrency groups with slot distribution for global limiting +- Workflows are distributed across available slots using `github.run_id % max-concurrency` +- Each slot can only run one workflow at a time across ALL workflows and refs +- Includes engine ID in concurrency group for isolation between different engines +- Prevents resource exhaustion from too many concurrent AI executions + +**Example configurations:** + +```yaml +# Allow up to 10 concurrent Claude workflows +engine: + id: claude + max-concurrency: 10 +``` + +```yaml +# Restrict to 1 workflow at a time (sequential execution) +engine: + id: copilot + max-concurrency: 1 +``` + +```yaml +# Use default of 3 concurrent workflows +engine: + id: claude + # max-concurrency not specified, defaults to 3 +``` + +**Generated concurrency group pattern:** +```yaml +# At workflow level +concurrency: + group: "gh-aw-{engine-id}-${{ github.run_id % max-concurrency }}" + +# At job level (agentic job) +jobs: + agent: + concurrency: + group: "gh-aw-{engine-id}-${{ github.run_id % max-concurrency }}" +``` + +Example for claude with max-concurrency of 5: +```yaml +concurrency: + group: "gh-aw-claude-${{ github.run_id % 5 }}" +``` + +The concurrency group uses **only** the engine ID and slot number (prefixed with "gh-aw-"), creating a global lock across all workflows and refs for that engine. This ensures the max-concurrency limit applies repository-wide. + +The concurrency is applied at **both workflow and job level** for maximum control. + ## Tools Configuration (`tools:`) The `tools:` section specifies which tools and MCP (Model Context Protocol) servers are available to the AI engine. This enables integration with GitHub APIs, browser automation, and other external services. @@ -510,32 +577,30 @@ timeout_minutes: 30 # Defaults to 15 minutes ## Concurrency Control (`concurrency:`) -GitHub Agentic Workflows automatically generates enhanced concurrency policies based on workflow trigger types to provide better isolation and resource management. For example, most workflows produce this: +GitHub Agentic Workflows automatically generates concurrency policies to limit concurrent execution across all workflows using the same engine. -```yaml -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true -``` +See [Concurrency Control](/gh-aw/reference/concurrency/) for complete documentation on max-concurrency configuration, global locks, and engine isolation. -Different workflow types receive different concurrency groups and cancellation behavior: +**Quick reference:** +- Configure via `engine.max-concurrency` (default: 3) +- Creates global lock across all workflows and refs for each engine +- Applied at both workflow and job levels +- Different engines can run concurrently without interfering -| Trigger Type | Concurrency Group | Cancellation | Description | -|--------------|-------------------|--------------|-------------| -| `issues` | `gh-aw-${{ github.workflow }}-${{ github.event.issue.number }}` | ❌ | Issue workflows include issue number for isolation | -| `pull_request` | `gh-aw-${{ github.workflow }}-${{ github.event.pull_request.number \|\| github.ref }}` | ✅ | PR workflows include PR number with cancellation | -| `discussion` | `gh-aw-${{ github.workflow }}-${{ github.event.discussion.number }}` | ❌ | Discussion workflows include discussion number | -| Mixed issue/PR | `gh-aw-${{ github.workflow }}-${{ github.event.issue.number \|\| github.event.pull_request.number }}` | ✅ | Mixed workflows handle both contexts with cancellation | -| Alias workflows | `gh-aw-${{ github.workflow }}-${{ github.event.issue.number \|\| github.event.pull_request.number }}` | ❌ | Alias workflows handle both contexts without cancellation | -| Other triggers | `gh-aw-${{ github.workflow }}` | ❌ | Default behavior for schedule, push, etc. | +**Example:** +```yaml +engine: + id: claude + max-concurrency: 5 +``` -**Benefits:** -- **Better Isolation**: Workflows operating on different issues/PRs can run concurrently -- **Conflict Prevention**: No interference between unrelated workflow executions -- **Resource Management**: Pull request workflows can cancel previous runs when updated -- **Predictable Behavior**: Consistent concurrency rules based on trigger type +Generates: +```yaml +concurrency: + group: "gh-aw-claude-${{ github.run_id % 5 }}" +``` -If you need custom concurrency behavior, you can override the automatic generation by specifying your own `concurrency` section in the frontmatter. +You can override automatic concurrency by specifying a custom `concurrency` section in the frontmatter. ## Environment Variables (`env:`) diff --git a/pkg/cli/templates/instructions.md b/pkg/cli/templates/instructions.md index 577558da6cb..c54708f9a22 100644 --- a/pkg/cli/templates/instructions.md +++ b/pkg/cli/templates/instructions.md @@ -69,13 +69,15 @@ The YAML frontmatter supports these fields: version: beta # Optional: version of the action (has sensible default) model: gpt-5 # Optional: LLM model to use (has sensible default) max-turns: 5 # Optional: maximum chat iterations per run (has sensible default) + max-concurrency: 3 # Optional: max concurrent workflows across all workflows (default: 3) ``` - - **Note**: The `version`, `model`, and `max-turns` fields have sensible defaults and can typically be omitted unless you need specific customization. + - **Note**: The `version`, `model`, `max-turns`, and `max-concurrency` fields have sensible defaults and can typically be omitted unless you need specific customization. - **Custom engine format** (⚠️ experimental): ```yaml engine: id: custom # Required: custom engine identifier max-turns: 10 # Optional: maximum iterations (for consistency) + max-concurrency: 5 # Optional: max concurrent workflows (for consistency) steps: # Required: array of custom GitHub Actions steps - name: Setup Node.js uses: actions/setup-node@v4 diff --git a/pkg/parser/schemas/main_workflow_schema.json b/pkg/parser/schemas/main_workflow_schema.json index ef25ca88b12..e541c7cc260 100644 --- a/pkg/parser/schemas/main_workflow_schema.json +++ b/pkg/parser/schemas/main_workflow_schema.json @@ -2264,6 +2264,11 @@ "type": "integer", "description": "Maximum number of chat iterations per run. Helps prevent runaway loops and control costs. Has sensible defaults and can typically be omitted." }, + "max-concurrency": { + "type": "integer", + "minimum": -1, + "description": "Maximum number of agentic jobs that can run concurrently across all workflows using this engine. Defaults to 3. Set to -1 to disable agent concurrency limiting. Uses GitHub Actions concurrency controls with slot-based distribution." + }, "user-agent": { "type": "string", "description": "Custom user agent string for GitHub MCP server configuration (codex engine only)" diff --git a/pkg/workflow/compiler.go b/pkg/workflow/compiler.go index 8dcdbf85e8b..1f29028d391 100644 --- a/pkg/workflow/compiler.go +++ b/pkg/workflow/compiler.go @@ -526,6 +526,12 @@ func (c *Compiler) ParseWorkflowFile(markdownPath string) (*WorkflowData, error) if c.verbose { fmt.Println(console.FormatInfoMessage(fmt.Sprintf("NOTE: No 'engine:' setting found, defaulting to: %s", engineSetting))) } + // Create a default EngineConfig with the default engine ID if not already set + if engineConfig == nil { + engineConfig = &EngineConfig{ID: engineSetting} + } else if engineConfig.ID == "" { + engineConfig.ID = engineSetting + } } // Validate the engine setting @@ -1952,6 +1958,9 @@ func (c *Compiler) buildMainJob(data *WorkflowData, activationJobCreated bool) ( } } + // Generate agent concurrency for max-concurrency feature + agentConcurrency := GenerateJobConcurrencyConfig(data) + job := &Job{ Name: constants.AgentJobName, If: jobCondition, @@ -1960,6 +1969,7 @@ func (c *Compiler) buildMainJob(data *WorkflowData, activationJobCreated bool) ( Container: c.indentYAMLLines(data.Container, " "), Services: c.indentYAMLLines(data.Services, " "), Permissions: c.indentYAMLLines(data.Permissions, " "), + Concurrency: c.indentYAMLLines(agentConcurrency, " "), Env: env, Steps: steps, Needs: depends, diff --git a/pkg/workflow/concurrency.go b/pkg/workflow/concurrency.go index b2b1c5c2703..4a858383c5d 100644 --- a/pkg/workflow/concurrency.go +++ b/pkg/workflow/concurrency.go @@ -13,7 +13,7 @@ func GenerateConcurrencyConfig(workflowData *WorkflowData, isCommandTrigger bool return workflowData.Concurrency } - // Build concurrency group keys + // Build concurrency group keys using the original workflow-specific logic keys := buildConcurrencyGroupKeys(workflowData, isCommandTrigger) groupValue := strings.Join(keys, "-") @@ -28,6 +28,47 @@ func GenerateConcurrencyConfig(workflowData *WorkflowData, isCommandTrigger bool return concurrencyConfig } +// GenerateJobConcurrencyConfig generates the agent concurrency configuration +// for max-concurrency limiting across all workflows using the same engine +func GenerateJobConcurrencyConfig(workflowData *WorkflowData) string { + // Check if max-concurrency is -1 (disabled) + if workflowData.EngineConfig != nil && workflowData.EngineConfig.MaxConcurrency == -1 { + return "" // Don't emit agent concurrency when disabled + } + + // Build agent concurrency for max-concurrency feature + // This uses ONLY engine ID (or custom concurrency-group) and run_id slot for global limiting + var keys []string + + // Prepend with gh-aw- prefix + keys = append(keys, "gh-aw") + + // Use custom concurrency-group if provided, otherwise use engine ID + if workflowData.EngineConfig != nil && workflowData.EngineConfig.ConcurrencyGroup != "" { + keys = append(keys, workflowData.EngineConfig.ConcurrencyGroup) + } else if workflowData.EngineConfig != nil && workflowData.EngineConfig.ID != "" { + keys = append(keys, workflowData.EngineConfig.ID) + } + + // Add max-concurrency slot to the group + maxConcurrency := 3 // default value + if workflowData.EngineConfig != nil && workflowData.EngineConfig.MaxConcurrency > 0 { + maxConcurrency = workflowData.EngineConfig.MaxConcurrency + } + + // Add a slot number based on run_id to distribute workflows across concurrency slots + // This implements a simple round-robin distribution using modulo + slotKey := fmt.Sprintf("${{ github.run_id %% %d }}", maxConcurrency) + keys = append(keys, slotKey) + + groupValue := strings.Join(keys, "-") + + // Build the concurrency configuration (no cancel-in-progress at agent level) + concurrencyConfig := fmt.Sprintf("concurrency:\n group: \"%s\"", groupValue) + + return concurrencyConfig +} + // isPullRequestWorkflow checks if a workflow's "on" section contains pull_request triggers func isPullRequestWorkflow(on string) bool { return strings.Contains(on, "pull_request") diff --git a/pkg/workflow/concurrency_test.go b/pkg/workflow/concurrency_test.go index 280f5288787..71ca797fd51 100644 --- a/pkg/workflow/concurrency_test.go +++ b/pkg/workflow/concurrency_test.go @@ -104,7 +104,7 @@ tools: expectedConcurrency: `concurrency: group: "gh-aw-${{ github.workflow }}-${{ github.event.issue.number }}"`, shouldHaveCancel: false, - description: "Issue workflows should use dynamic concurrency with issue number but no cancellation", + description: "Issue workflows use global concurrency with engine ID and slot", }, } @@ -145,35 +145,6 @@ This is a test workflow for concurrency behavior. } else if !tt.shouldHaveCancel && hasCancel { t.Errorf("Did not expect cancel-in-progress: true for %s workflow, but found in: %s", tt.name, workflowData.Concurrency) } - - // For PR workflows, check for PR number inclusion; for alias workflows, check for issue/PR numbers; for issue workflows, check for issue number; for push workflows, check for github.ref - isPRWorkflow := strings.Contains(tt.name, "PR workflow") - isAliasWorkflow := strings.Contains(tt.name, "alias workflow") - isIssueWorkflow := strings.Contains(tt.name, "issue workflow") - isPushWorkflow := strings.Contains(tt.name, "push workflow") - - if isPRWorkflow { - if !strings.Contains(workflowData.Concurrency, "github.event.pull_request.number") { - t.Errorf("Expected concurrency to include github.event.pull_request.number for %s workflow, got: %s", tt.name, workflowData.Concurrency) - } - } else if isAliasWorkflow { - if !strings.Contains(workflowData.Concurrency, "github.event.issue.number || github.event.pull_request.number") { - t.Errorf("Expected concurrency to include issue/PR numbers for %s workflow, got: %s", tt.name, workflowData.Concurrency) - } - } else if isIssueWorkflow { - if !strings.Contains(workflowData.Concurrency, "github.event.issue.number") { - t.Errorf("Expected concurrency to include github.event.issue.number for %s workflow, got: %s", tt.name, workflowData.Concurrency) - } - } else if isPushWorkflow { - if !strings.Contains(workflowData.Concurrency, "github.ref") { - t.Errorf("Expected concurrency to include github.ref for %s workflow, got: %s", tt.name, workflowData.Concurrency) - } - } else { - // For regular workflows (like schedule), don't expect github.ref unless it's also a push workflow - if strings.Contains(workflowData.Concurrency, "github.ref") && !isPushWorkflow { - t.Errorf("Did not expect concurrency to include github.ref for %s workflow, got: %s", tt.name, workflowData.Concurrency) - } - } }) } } @@ -336,6 +307,71 @@ func TestGenerateConcurrencyConfig(t *testing.T) { } } +// TestGenerateJobConcurrencyConfig tests the job-level concurrency configuration for max-concurrency +func TestGenerateJobConcurrencyConfig(t *testing.T) { + tests := []struct { + name string + workflowData *WorkflowData + expected string + description string + }{ + { + name: "Default max-concurrency (3) with copilot engine", + workflowData: &WorkflowData{ + EngineConfig: &EngineConfig{ID: "copilot"}, + }, + expected: `concurrency: + group: "gh-aw-copilot-${{ github.run_id % 3 }}"`, + description: "Should use default max-concurrency of 3 with copilot engine ID", + }, + { + name: "Custom max-concurrency value should be used", + workflowData: &WorkflowData{ + EngineConfig: &EngineConfig{ID: "claude", MaxConcurrency: 5}, + }, + expected: `concurrency: + group: "gh-aw-claude-${{ github.run_id % 5 }}"`, + description: "Custom max-concurrency should use specified value instead of default", + }, + { + name: "Zero max-concurrency should use default (3)", + workflowData: &WorkflowData{ + EngineConfig: &EngineConfig{ID: "copilot", MaxConcurrency: 0}, // 0 means use default + }, + expected: `concurrency: + group: "gh-aw-copilot-${{ github.run_id % 3 }}"`, + description: "Zero max-concurrency should default to 3", + }, + { + name: "Different engine ID should be included in concurrency group", + workflowData: &WorkflowData{ + EngineConfig: &EngineConfig{ID: "codex"}, + }, + expected: `concurrency: + group: "gh-aw-codex-${{ github.run_id % 3 }}"`, + description: "Different engine IDs should be included in concurrency group for isolation", + }, + { + name: "Max-concurrency -1 should disable agent concurrency", + workflowData: &WorkflowData{ + EngineConfig: &EngineConfig{ID: "claude", MaxConcurrency: -1}, + }, + expected: "", + description: "Max-concurrency -1 should return empty string (no agent concurrency)", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := GenerateJobConcurrencyConfig(tt.workflowData) + + if result != tt.expected { + t.Errorf("GenerateJobConcurrencyConfig() failed for %s\nExpected:\n%s\nGot:\n%s", tt.description, tt.expected, result) + } + }) + } +} + func TestIsPullRequestWorkflow(t *testing.T) { tests := []struct { name string diff --git a/pkg/workflow/engine.go b/pkg/workflow/engine.go index 67f8f8ac840..90f15e7ea42 100644 --- a/pkg/workflow/engine.go +++ b/pkg/workflow/engine.go @@ -7,15 +7,17 @@ import ( // EngineConfig represents the parsed engine configuration type EngineConfig struct { - ID string - Version string - Model string - MaxTurns string - UserAgent string - Env map[string]string - Steps []map[string]any - ErrorPatterns []ErrorPattern - Config string + ID string + Version string + Model string + MaxTurns string + MaxConcurrency int + ConcurrencyGroup string + UserAgent string + Env map[string]string + Steps []map[string]any + ErrorPatterns []ErrorPattern + Config string } // NetworkPermissions represents network access permissions @@ -74,6 +76,24 @@ func (c *Compiler) ExtractEngineConfig(frontmatter map[string]any) (string, *Eng } } + // Extract optional 'max-concurrency' field + if maxConcurrency, hasMaxConcurrency := engineObj["max-concurrency"]; hasMaxConcurrency { + if maxConcurrencyInt, ok := maxConcurrency.(int); ok { + config.MaxConcurrency = maxConcurrencyInt + } else if maxConcurrencyFloat, ok := maxConcurrency.(float64); ok { + config.MaxConcurrency = int(maxConcurrencyFloat) + } else if maxConcurrencyUint64, ok := maxConcurrency.(uint64); ok { + config.MaxConcurrency = int(maxConcurrencyUint64) + } + } + + // Extract optional 'concurrency-group' field + if concurrencyGroup, hasConcurrencyGroup := engineObj["concurrency-group"]; hasConcurrencyGroup { + if concurrencyGroupStr, ok := concurrencyGroup.(string); ok { + config.ConcurrencyGroup = concurrencyGroupStr + } + } + // Extract optional 'user-agent' field if userAgent, hasUserAgent := engineObj["user-agent"]; hasUserAgent { if userAgentStr, ok := userAgent.(string); ok { diff --git a/pkg/workflow/jobs.go b/pkg/workflow/jobs.go index e1579660925..dde6001e63b 100644 --- a/pkg/workflow/jobs.go +++ b/pkg/workflow/jobs.go @@ -16,6 +16,7 @@ type Job struct { If string Permissions string TimeoutMinutes int + Concurrency string // Job-level concurrency configuration Environment string // Job environment configuration Container string // Job container configuration Services string // Job services configuration @@ -222,6 +223,11 @@ func (jm *JobManager) renderJob(job *Job) string { yaml.WriteString(fmt.Sprintf(" %s\n", job.Permissions)) } + // Add concurrency section + if job.Concurrency != "" { + yaml.WriteString(fmt.Sprintf(" %s\n", job.Concurrency)) + } + // Add timeout_minutes if specified if job.TimeoutMinutes > 0 { yaml.WriteString(fmt.Sprintf(" timeout-minutes: %d\n", job.TimeoutMinutes))