Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/workflows/agent-authorship-label.yml
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ permissions:
jobs:
label:
runs-on: ${{ inputs.runner_label }}
timeout-minutes: 10
steps:
- name: Checkout org workflow helpers
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
Expand Down
24 changes: 22 additions & 2 deletions .github/workflows/codex-rails-check.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,28 +12,35 @@ on:
- ".github/workflows/**"
- ".github/workflow-templates/**"
- "profile/**"
- "test/**"
workflow_call:
inputs:
require_agents:
description: "Fail when the repository does not have AGENTS.md"
required: false
type: boolean
default: false
runner_label:
description: "Runner label used for the validation job"
required: false
type: string
default: blacksmith-4vcpu-ubuntu-2404

permissions:
contents: read

jobs:
validate:
runs-on: ubuntu-latest
runs-on: ${{ inputs.runner_label || 'blacksmith-4vcpu-ubuntu-2404' }}
timeout-minutes: 10
steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5

- name: Validate issue template YAML
shell: bash
run: |
set -euo pipefail
shopt -s nullglob
shopt -s globstar nullglob
files=(.github/ISSUE_TEMPLATE/*.yml .github/ISSUE_TEMPLATE/*.yaml)
if [ "${#files[@]}" -eq 0 ]; then
echo "No issue template YAML files found."
Expand Down Expand Up @@ -132,3 +139,16 @@ jobs:
end
' "${skill}"
done

- name: Run repo Ruby tests
shell: bash
run: |
set -euo pipefail
shopt -s globstar nullglob
tests=(test/**/*_test.rb test/*_test.rb)
Comment thread
haasonsaas marked this conversation as resolved.
if [ "${#tests[@]}" -eq 0 ]; then
echo "No Ruby tests found."
exit 0
fi

ruby -Itest "${tests[@]}"
Comment thread
haasonsaas marked this conversation as resolved.
4 changes: 4 additions & 0 deletions profile/AGENT_AUTHORSHIP.md
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,10 @@ authorship labels, and applies the label that matches the current PR commit set.
It only mutates labels when the desired label set changed, so repeated
`synchronize` events do not remove and re-add the same label.

For production repositories, pin the reusable workflow to an immutable
`evalops/.github` commit SHA. When pinning, pass the same SHA as `helper_ref` so
the workflow and helper scripts are resolved from the same reviewed revision.

## Audit Indexing

Audit ingestion should parse trailers from every commit merged to protected
Expand Down
83 changes: 83 additions & 0 deletions test/classify_agent_authorship_test.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
# frozen_string_literal: true

require "json"
require "minitest/autorun"
require "open3"
require "tempfile"

class ClassifyAgentAuthorshipTest < Minitest::Test
ROOT = File.expand_path("..", __dir__)
SCRIPT = File.join(ROOT, ".github/scripts/classify-agent-authorship.rb")

def test_untrailered_commits_are_agent_assisted
outputs = classify([{ "sha" => "abc", "message" => "fix: regular change" }])

assert_equal "agent-assisted", outputs.fetch("label")
assert_equal "1", outputs.fetch("total_commits")
assert_equal "0", outputs.fetch("agent_commits")
assert_equal "1", outputs.fetch("untrailered_commits")
assert_equal "0", outputs.fetch("incomplete_agent_commits")
end

def test_complete_maestro_trailers_are_agent_authored
outputs = classify([{ "sha" => "abc", "message" => <<~MSG }])
feat: ship change

Co-Authored-By: Maestro <maestro@evalops.dev>
Maestro-Version: 2026.04.28 / gpt-5
Maestro-Prompt-Id: prompt-123
Maestro-Approvals-Id: approval-456
MSG

assert_equal "agent-authored", outputs.fetch("label")
assert_equal "1", outputs.fetch("agent_commits")
assert_equal "0", outputs.fetch("untrailered_commits")
assert_equal "0", outputs.fetch("incomplete_agent_commits")
end

def test_mixed_authorship_and_incomplete_trailers_are_reported
outputs = classify(
[
{ "sha" => "abc", "message" => <<~MSG },
feat: partial agent change

Co-Authored-By: Maestro <maestro@evalops.dev>
Maestro-Version: 2026.04.28 / gpt-5
MSG
{ "sha" => "def", "message" => "docs: human follow-up" },
],
)

assert_equal "mixed-authorship", outputs.fetch("label")
assert_equal "1", outputs.fetch("agent_commits")
assert_equal "1", outputs.fetch("untrailered_commits")
assert_equal "1", outputs.fetch("incomplete_agent_commits")
end

def test_github_output_file_gets_same_outputs
Tempfile.create("github-output") do |file|
outputs = classify(
[{ "sha" => "abc", "message" => "fix: regular change" }],
github_output: file.path,
)
file_outputs = parse_outputs(File.read(file.path))

assert_equal outputs, file_outputs
end
end

private

def classify(commits, github_output: nil)
input = commits.map(&:to_json).join("\n")
args = ["ruby", SCRIPT]
args += ["--github-output", github_output] if github_output
stdout, stderr, status = Open3.capture3(*args, stdin_data: input)
assert status.success?, stderr
parse_outputs(stdout)
end

def parse_outputs(text)
text.each_line(chomp: true).to_h { |line| line.split("=", 2) }
end
end
Loading