diff --git a/provider/github/github.go b/provider/github/github.go index 350b7fe..a052468 100644 --- a/provider/github/github.go +++ b/provider/github/github.go @@ -18,6 +18,7 @@ import ( "time" "github.com/gkanitz/coderepute/provider" + "github.com/gkanitz/coderepute/provider/recognition" ) const defaultBaseURL = "https://api.github.com" @@ -208,6 +209,7 @@ type githubFilesResponse struct { type apiUser struct { Login string `json:"login"` ID int64 `json:"id"` + Type string `json:"type,omitempty"` } type apiPull struct { @@ -259,6 +261,7 @@ func (a *Adapter) FetchActivity(ctx context.Context, opts provider.FetchOptions) } } as.AccessManifest = a.counting.Manifest(provider.GitHubNeverRequested(), "All requests are to the GitHub REST API and GraphQL API. No repository contents, file contents, diffs, branch names, colleague profiles, or commit data are ever requested.") + as.AccessManifest.AIRecognitionVersion = recognition.Version() return as, nil } @@ -268,6 +271,7 @@ type pendingReview struct { prNumber int64 submittedAt time.Time state string + authorClass string // classification of the reviewed PR's author } // fetchRepoActivity collects one repo's activity into the set. Everything @@ -318,6 +322,9 @@ func (a *Adapter) fetchRepoActivity(ctx context.Context, repo string, subjectID continue } // Someone else's PR: only the subject's in-window reviews matter. + // Classify the PR author and record only the class string -- + // the colleague's identity never leaves the adapter. + authorClass := recognition.Classify(p.User.Login, p.User.Type) for _, rv := range reviews { if rv.User.ID != subjectID || !inWindow(rv.SubmittedAt, window) { continue @@ -326,6 +333,7 @@ func (a *Adapter) fetchRepoActivity(ctx context.Context, repo string, subjectID prNumber: p.Number, submittedAt: rv.SubmittedAt, state: rv.State, + authorClass: authorClass, }) } } @@ -343,6 +351,7 @@ func (a *Adapter) fetchRepoActivity(ctx context.Context, repo string, subjectID SubmittedAt: rv.submittedAt, State: rv.state, CommentCount: commentCounts[rv.prNumber], + AuthorClass: rv.authorClass, }) } return nil diff --git a/provider/github/github_test.go b/provider/github/github_test.go index 2641139..c1e53f7 100644 --- a/provider/github/github_test.go +++ b/provider/github/github_test.go @@ -577,3 +577,131 @@ func TestFetchActivityWidenedReviewAllTimeWindow(t *testing.T) { t.Errorf("got %d PRs, want 1", len(as.PullRequests)) } } + +// TestFetchActivityAuthorClassification verifies that the author of a reviewed +// PR is classified against the recognition ruleset and the class string is +// recorded on the Review, without leaking the colleague's identity. +func TestFetchActivityAuthorClassification(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/users/octocat": + w.Header().Set("X-OAuth-Scopes", "repo") + w.Header().Set("Content-Type", "application/json") + w.Write([]byte(`{"login":"octocat","id":583231}`)) + case "/repos/acme/widgets/pulls": + w.Header().Set("Content-Type", "application/json") + w.Write([]byte(`[ + {"number":1,"user":{"login":"copilot[bot]","id":999001,"type":"Bot"},"created_at":"2026-02-15T08:00:00Z","updated_at":"2026-02-20T08:00:00Z"}, + {"number":2,"user":{"login":"human-colleague","id":888001},"created_at":"2026-02-15T08:00:00Z","updated_at":"2026-02-20T08:00:00Z"} + ]`)) + case "/repos/acme/widgets/pulls/1/reviews": + w.Header().Set("Content-Type", "application/json") + w.Write([]byte(`[ + {"user":{"login":"octocat","id":583231},"state":"APPROVED","submitted_at":"2026-02-20T09:00:00Z"} + ]`)) + case "/repos/acme/widgets/pulls/2/reviews": + w.Header().Set("Content-Type", "application/json") + w.Write([]byte(`[ + {"user":{"login":"octocat","id":583231},"state":"COMMENTED","submitted_at":"2026-02-20T10:00:00Z"} + ]`)) + case "/repos/acme/widgets/pulls/comments": + w.Header().Set("Content-Type", "application/json") + w.Write([]byte(`[]`)) + default: + http.Error(w, "not found", http.StatusNotFound) + } + })) + t.Cleanup(srv.Close) + + adapter := github.New("test-token", github.WithBaseURL(srv.URL)) + as, err := adapter.FetchActivity(context.Background(), provider.FetchOptions{ + Repos: []string{"acme/widgets"}, + Subject: "octocat", + Window: provider.Window{ + Since: time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC), + Until: time.Date(2026, 6, 1, 0, 0, 0, 0, time.UTC), + }, + }) + if err != nil { + t.Fatalf("FetchActivity: %v", err) + } + + // Expect two reviews: one on a copilot-authored PR, one on a human-authored PR. + if len(as.ReviewsGiven) != 2 { + t.Fatalf("got %d reviews given, want 2", len(as.ReviewsGiven)) + } + + // Review on copilot-authored PR should have AuthorClass = "copilot". + var copilotReview, humanReview *provider.Review + for i, rv := range as.ReviewsGiven { + if rv.AuthorClass == "copilot" { + copilotReview = &as.ReviewsGiven[i] + } else if rv.AuthorClass == "" { + humanReview = &as.ReviewsGiven[i] + } + } + + if copilotReview == nil { + t.Errorf("no review with AuthorClass 'copilot' found; all reviews: %+v", as.ReviewsGiven) + } + if humanReview == nil { + t.Errorf("no review with AuthorClass '' (human) found; all reviews: %+v", as.ReviewsGiven) + } + + // Verify that no colleague identity (login, ID) appears in ReviewsGiven. + dump := fmt.Sprintf("%+v", as.ReviewsGiven) + for _, forbidden := range []string{"copilot[bot]", "999001", "888001"} { + if strings.Contains(dump, forbidden) { + t.Errorf("Review carries prohibited colleague identity %q", forbidden) + } + } +} + +// TestFetchActivityBotAuthorClassification verifies that a PR authored by an +// unknown bot (matched only by type:"Bot" or *[bot] login) gets AuthorClass "bot". +func TestFetchActivityBotAuthorClassification(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/users/octocat": + w.Header().Set("X-OAuth-Scopes", "repo") + w.Header().Set("Content-Type", "application/json") + w.Write([]byte(`{"login":"octocat","id":583231}`)) + case "/repos/acme/widgets/pulls": + w.Header().Set("Content-Type", "application/json") + w.Write([]byte(`[ + {"number":1,"user":{"login":"some-unknown-tool[bot]","id":999002,"type":"Bot"},"created_at":"2026-02-15T08:00:00Z","updated_at":"2026-02-20T08:00:00Z"} + ]`)) + case "/repos/acme/widgets/pulls/1/reviews": + w.Header().Set("Content-Type", "application/json") + w.Write([]byte(`[ + {"user":{"login":"octocat","id":583231},"state":"APPROVED","submitted_at":"2026-02-20T09:00:00Z"} + ]`)) + case "/repos/acme/widgets/pulls/comments": + w.Header().Set("Content-Type", "application/json") + w.Write([]byte(`[]`)) + default: + http.Error(w, "not found", http.StatusNotFound) + } + })) + t.Cleanup(srv.Close) + + adapter := github.New("test-token", github.WithBaseURL(srv.URL)) + as, err := adapter.FetchActivity(context.Background(), provider.FetchOptions{ + Repos: []string{"acme/widgets"}, + Subject: "octocat", + Window: provider.Window{ + Since: time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC), + Until: time.Date(2026, 6, 1, 0, 0, 0, 0, time.UTC), + }, + }) + if err != nil { + t.Fatalf("FetchActivity: %v", err) + } + + if len(as.ReviewsGiven) != 1 { + t.Fatalf("got %d reviews given, want 1", len(as.ReviewsGiven)) + } + if as.ReviewsGiven[0].AuthorClass != "bot" { + t.Errorf("AuthorClass = %q, want %q (unknown bot should get 'bot')", as.ReviewsGiven[0].AuthorClass, "bot") + } +} diff --git a/provider/manifest.go b/provider/manifest.go index 5497bf5..c44ec90 100644 --- a/provider/manifest.go +++ b/provider/manifest.go @@ -27,6 +27,10 @@ type Manifest struct { Endpoints []EndpointCount `json:"endpoints"` NeverRequested []string `json:"never_requested"` Notes string `json:"notes"` + // AIRecognitionVersion is the version of the embedded recognition ruleset + // (airuleset.json) used during this fetch, or zero if no classification + // was performed. + AIRecognitionVersion int `json:"ai_recognition_version,omitempty"` } // RouteEntry maps one URL pattern to its route class. The Pattern is a diff --git a/provider/provider.go b/provider/provider.go index 569d7f7..ffc971c 100644 --- a/provider/provider.go +++ b/provider/provider.go @@ -84,6 +84,12 @@ type Review struct { // populated from diff-shape data when available. Zero means "unknown / // no diff data" and triggers the fallback deep-review threshold. PRLines int + // AuthorClass is the classification of the reviewed PR's author: a + // canonical agent id ("copilot", "devin") when the author matches the + // curated recognition ruleset, "bot" when matched only by structural + // bot-type indicators, or "" for a human author. Populated inside the + // adapter; the colleague's identity never leaves the adapter. + AuthorClass string } // ReviewComment is a single review comment written or received by the subject. diff --git a/provider/recognition/airuleset.json b/provider/recognition/airuleset.json new file mode 100644 index 0000000..b79925b --- /dev/null +++ b/provider/recognition/airuleset.json @@ -0,0 +1,45 @@ +{ + "version": 1, + "entries": [ + { + "login": "copilot[bot]", + "agent": "copilot" + }, + { + "login": "devin[bot]", + "agent": "devin" + }, + { + "login": "codeium[bot]", + "agent": "codeium" + }, + { + "login": "tabnine[bot]", + "agent": "tabnine" + }, + { + "login": "amazon-q-developer[bot]", + "agent": "amazon-q" + }, + { + "login": "coderabbit[bot]", + "agent": "coderabbit" + }, + { + "login": "codesee[bot]", + "agent": "codesee" + }, + { + "login": "github-actions[bot]", + "agent": "github-actions" + }, + { + "login": "dependabot[bot]", + "agent": "dependabot" + }, + { + "login": "github-merge-queue[bot]", + "agent": "github-merge-queue" + } + ] +} diff --git a/provider/recognition/recognition.go b/provider/recognition/recognition.go new file mode 100644 index 0000000..95d97db --- /dev/null +++ b/provider/recognition/recognition.go @@ -0,0 +1,90 @@ +// Package recognition provides an embedded, versioned ruleset for classifying +// PR/MR authors as human, bot, or a recognized AI agent. The ruleset follows +// the same go:embed pattern as metrics/bands/bands.json. +package recognition + +import ( + "embed" + "encoding/json" + "fmt" + "strings" + "sync" +) + +//go:embed airuleset.json +var rulesetFS embed.FS + +// Entry maps a known agent login to its canonical agent id. +type Entry struct { + Login string `json:"login"` + Agent string `json:"agent"` +} + +// Ruleset is the top-level structure of the embedded airuleset.json. +type Ruleset struct { + Version int `json:"version"` + Entries []Entry `json:"entries"` +} + +var ( + once sync.Once + loginMap map[string]string // login -> canonical agent id + version int +) + +// load reads and parses the embedded airuleset.json once. +func load() { + raw, err := rulesetFS.ReadFile("airuleset.json") + if err != nil { + panic(fmt.Sprintf("recognition: embedded airuleset.json: %v", err)) + } + var rs Ruleset + if err := json.Unmarshal(raw, &rs); err != nil { + panic(fmt.Sprintf("recognition: parse embedded airuleset.json: %v", err)) + } + version = rs.Version + loginMap = make(map[string]string, len(rs.Entries)) + for _, e := range rs.Entries { + loginMap[strings.ToLower(e.Login)] = e.Agent + } +} + +// agentIDForLogin returns the canonical agent id for a known agent login, or +// false if the login is not in the curated ruleset. +func agentIDForLogin(login string) (string, bool) { + once.Do(load) + aid, ok := loginMap[strings.ToLower(login)] + return aid, ok +} + +// Classify returns the canonical agent id when the given login+type matches a +// curated ruleset entry, "bot" when the match is structural (GitHub type:"Bot" +// or *[bot] login pattern), or "" for a human author. The login comparison is +// case-insensitive. +func Classify(login, userType string) string { + if login == "" { + return "" + } + // Layer 1: curated agent ruleset. + if aid, ok := agentIDForLogin(login); ok { + return aid + } + // Layer 2: structural bot-type. + // GitHub API returns type:"Bot" for bot accounts. + if userType == "Bot" { + return "bot" + } + // GitHub bot logins follow the *[bot] pattern (e.g. "copilot[bot]"). + // This also catches ruleset entries that didn't match above, though + // that's redundant — ruleset entries use [bot] logins. + if strings.HasSuffix(login, "[bot]") { + return "bot" + } + return "" +} + +// Version returns the embedded airuleset.json version number. +func Version() int { + once.Do(load) + return version +} diff --git a/provider/recognition/recognition_test.go b/provider/recognition/recognition_test.go new file mode 100644 index 0000000..93ba8c2 --- /dev/null +++ b/provider/recognition/recognition_test.go @@ -0,0 +1,74 @@ +package recognition_test + +import ( + "testing" + + "github.com/gkanitz/coderepute/provider/recognition" +) + +func TestClassifyCuratedAgentReturnsAgentID(t *testing.T) { + tests := []struct { + login string + want string + }{ + {"copilot[bot]", "copilot"}, + {"devin[bot]", "devin"}, + {"dependabot[bot]", "dependabot"}, + {"github-actions[bot]", "github-actions"}, + {"coderabbit[bot]", "coderabbit"}, + {"CODEIUM[bot]", "codeium"}, // case-insensitive + {"GitHub-Actions[bot]", "github-actions"}, // case-insensitive + } + for _, tc := range tests { + got := recognition.Classify(tc.login, "") + if got != tc.want { + t.Errorf("Classify(%q, \"\") = %q, want %q", tc.login, got, tc.want) + } + } +} + +func TestClassifyStructuralBotByType(t *testing.T) { + // Author type:"Bot" without a curated ruleset entry should return "bot". + got := recognition.Classify("some-unknown-bot", "Bot") + if got != "bot" { + t.Errorf("Classify(%q, \"Bot\") = %q, want %q", "some-unknown-bot", got, "bot") + } +} + +func TestClassifyStructuralBotByLoginPattern(t *testing.T) { + // Login matching *[bot] pattern without a curated ruleset entry + // should return "bot". + got := recognition.Classify("some-unknown-bot[bot]", "") + if got != "bot" { + t.Errorf("Classify(%q, \"\") = %q, want %q", "some-unknown-bot[bot]", got, "bot") + } +} + +func TestClassifyHumanReturnsEmpty(t *testing.T) { + // A regular user (no [bot] login, no Bot type) should return "". + got := recognition.Classify("octocat", "") + if got != "" { + t.Errorf("Classify(%q, \"\") = %q, want %q", "octocat", got, "") + } +} + +func TestClassifyEmptyLoginReturnsEmpty(t *testing.T) { + got := recognition.Classify("", "") + if got != "" { + t.Errorf("Classify(%q, \"\") = %q, want %q", "", got, "") + } +} + +func TestVersionNonZero(t *testing.T) { + if v := recognition.Version(); v == 0 { + t.Error("Version() = 0, want > 0") + } +} + +func TestVersionConsistent(t *testing.T) { + v1 := recognition.Version() + v2 := recognition.Version() + if v1 != v2 { + t.Errorf("Version() returned different values: %d then %d", v1, v2) + } +} diff --git a/report/manifest_test.go b/report/manifest_test.go index 892984b..8c20879 100644 --- a/report/manifest_test.go +++ b/report/manifest_test.go @@ -645,6 +645,81 @@ func TestNoScoreDeclarationInBuiltReport(t *testing.T) { } } +// TestManifestAIRecognitionVersion verifies that the AIRecognitionVersion +// field is propagated from the provider manifest into the report's +// AccessManifest block. (Issue #121) +func TestManifestAIRecognitionVersion(t *testing.T) { + now := time.Date(2026, 7, 1, 12, 0, 0, 0, time.UTC) + as := activityFixture() + as.AccessManifest = provider.Manifest{ + Endpoints: []provider.EndpointCount{ + {Class: "rest:users_show", Count: 1}, + }, + NeverRequested: []string{"file contents"}, + Notes: "test manifest", + AIRecognitionVersion: 1, + } + + r := report.Build(as, nil, nil, now) + + if r.AccessManifest == nil { + t.Fatal("Build() produced nil AccessManifest") + } + if r.AccessManifest.AIRecognitionVersion != 1 { + t.Errorf("AIRecognitionVersion = %d, want 1", r.AccessManifest.AIRecognitionVersion) + } + + // Round-trip through JSON to verify the field survives. + raw, err := json.Marshal(r) + if err != nil { + t.Fatalf("marshal: %v", err) + } + parsed, err := report.Parse(raw) + if err != nil { + t.Fatalf("parse: %v", err) + } + if parsed.AccessManifest == nil { + t.Fatal("round-trip lost AccessManifest") + } + if parsed.AccessManifest.AIRecognitionVersion != 1 { + t.Errorf("round-trip AIRecognitionVersion = %d, want 1", parsed.AccessManifest.AIRecognitionVersion) + } + + // Verify the version is non-zero when built from a live GitHub fetch. + // The GitHub adapter's FetchActivity sets AIRecognitionVersion from the + // embedded ruleset. A zero value means the field was not propagated. + if parsed.AccessManifest.AIRecognitionVersion == 0 { + t.Error("AIRecognitionVersion is 0 after round-trip, want > 0") + } +} + +// TestManifestAIRecognitionVersionZeroWhenNotSet verifies that when the +// provider manifest has AIRecognitionVersion = 0, it's omitted from the +// JSON output (omitempty). +func TestManifestAIRecognitionVersionZeroWhenNotSet(t *testing.T) { + now := time.Date(2026, 7, 1, 12, 0, 0, 0, time.UTC) + as := activityFixture() + as.AccessManifest = provider.Manifest{ + Endpoints: []provider.EndpointCount{ + {Class: "rest:users_show", Count: 1}, + }, + NeverRequested: []string{"file contents"}, + Notes: "test manifest", + // AIRecognitionVersion defaults to 0 + } + + r := report.Build(as, nil, nil, now) + + raw, err := json.Marshal(r) + if err != nil { + t.Fatalf("marshal: %v", err) + } + + if strings.Contains(string(raw), "ai_recognition_version") { + t.Error("JSON output contains ai_recognition_version field despite zero value (expected omitempty)") + } +} + // findRepoRoot walks up from the current directory to find go.mod. func findRepoRoot(t *testing.T) string { t.Helper() diff --git a/report/report.go b/report/report.go index 6394fa0..324437d 100644 --- a/report/report.go +++ b/report/report.go @@ -242,6 +242,10 @@ type AccessManifest struct { Notes string `json:"notes"` Omissions []OmissionEntry `json:"omissions,omitempty"` NoScoreDeclaration string `json:"no_score_declaration,omitempty"` + // AIRecognitionVersion records the version of the embedded recognition + // ruleset (airuleset.json) used to classify PR authors during the + // fetch, or zero if no classification was performed. + AIRecognitionVersion int `json:"ai_recognition_version,omitempty"` } // NoScoreDeclarationText is the plain-language statement declaring that the @@ -464,11 +468,12 @@ func buildAccessManifest(m provider.Manifest) *AccessManifest { never = []string{} } return &AccessManifest{ - Endpoints: endpoints, - NeverRequested: never, - Notes: m.Notes, - Omissions: defaultOmissions(), - NoScoreDeclaration: NoScoreDeclarationText, + Endpoints: endpoints, + NeverRequested: never, + Notes: m.Notes, + Omissions: defaultOmissions(), + NoScoreDeclaration: NoScoreDeclarationText, + AIRecognitionVersion: m.AIRecognitionVersion, } }