diff --git a/.github/workflows/metrics-collector.lock.yml b/.github/workflows/metrics-collector.lock.yml index acb1d123834..c6bc3f9da20 100644 --- a/.github/workflows/metrics-collector.lock.yml +++ b/.github/workflows/metrics-collector.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"0d8a2330ef831c4b5579917c5d07289e9b92d5d76c3fcf7437d2da8176741a25","body_hash":"0d3b29ba886f1664e7ee0b27605e13f6c4b6004d7f785e83065a2dee01a3ce3f","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.77","copilot-sdk":"1.0.8"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"ab804d1e801d8a6b4ebf78ea5840ff4a70a76cdf018f436c4491edbb40a3fc7e","body_hash":"1715b47b0ad218aba319dc769c70a7485ea13965d28c52295850e5aad501591f","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.77","copilot-sdk":"1.0.8"}} # gh-aw-manifest: {"version":1,"secrets":["COPILOT_GITHUB_TOKEN","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GH_AW_OTEL_GRAFANA_AUTHORIZATION","GH_AW_OTEL_GRAFANA_ENDPOINT","GH_AW_OTEL_SENTRY_AUTHORIZATION","GH_AW_OTEL_SENTRY_ENDPOINT","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache/restore","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/save","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-go","sha":"b7ad1dad31e06c5925ef5d2fc7ad053ef454303e","version":"v7.0.0"},{"repo":"actions/setup-node","sha":"820762786026740c76f36085b0efc47a31fe5020","version":"v7.0.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"docker/build-push-action","sha":"53b7df96c91f9c12dcc8a07bcb9ccacbed38856a","version":"v7.3.0"},{"repo":"docker/setup-buildx-action","sha":"bb05f3f5519dd87d3ba754cc423b652a5edd6d2c","version":"v4.2.0"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.27.43","digest":"sha256:04e2d1987a565000a8f114b89d806ae7a3864dd4f944be65275b28c93d8690e6","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.27.43@sha256:04e2d1987a565000a8f114b89d806ae7a3864dd4f944be65275b28c93d8690e6"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.43","digest":"sha256:d85f57975af5ea23af4996e41ed73fbc8f5b4a47402472bfe82e508f352cb0c1","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.43@sha256:d85f57975af5ea23af4996e41ed73fbc8f5b4a47402472bfe82e508f352cb0c1"},{"image":"ghcr.io/github/gh-aw-firewall/cli-proxy:0.27.43","digest":"sha256:65c45ea2967984d0024f3df61bc71335658a77ede96c8d9665da7a5f33a795ab","pinned_image":"ghcr.io/github/gh-aw-firewall/cli-proxy:0.27.43@sha256:65c45ea2967984d0024f3df61bc71335658a77ede96c8d9665da7a5f33a795ab"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.27.43","digest":"sha256:26be5e0b8c8f4c41c8a59126b29bb5d80b07253597472ded2a16bdd75abcbf9d","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.27.43@sha256:26be5e0b8c8f4c41c8a59126b29bb5d80b07253597472ded2a16bdd75abcbf9d"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.7","digest":"sha256:7545220a9aca134b71e51193ee0eaf4c50756ebf8fbd25a63ae7556e62815c00","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.7@sha256:7545220a9aca134b71e51193ee0eaf4c50756ebf8fbd25a63ae7556e62815c00"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:a8082161d7dceda14b68f32eb39d0eaa96b825d07f5895b096afab9d9e0c7748","pinned_image":"ghcr.io/github/gh-aw-node@sha256:a8082161d7dceda14b68f32eb39d0eaa96b825d07f5895b096afab9d9e0c7748"},{"image":"ghcr.io/github/github-mcp-server:v1.8.0","digest":"sha256:d5a18c04b92714c309eb46a2305087e91a4dbd80420f6e462656699f95093520","pinned_image":"ghcr.io/github/github-mcp-server:v1.8.0@sha256:d5a18c04b92714c309eb46a2305087e91a4dbd80420f6e462656699f95093520"}]} # This file was automatically generated by gh-aw. DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # diff --git a/.github/workflows/metrics-collector.md b/.github/workflows/metrics-collector.md index 50ee73dfb09..4251efa1823 100644 --- a/.github/workflows/metrics-collector.md +++ b/.github/workflows/metrics-collector.md @@ -1,5 +1,6 @@ --- emoji: "📊" +name: "Metrics Collector" description: Collects daily performance metrics for the agent ecosystem and stores them in repo-memory on: daily permissions: @@ -58,6 +59,19 @@ As an infrastructure agent, you collect and persist performance data that enable - **Collection Time**: $(date +%H:%M:%S) UTC - **Storage Path**: `/tmp/gh-aw/repo-memory/default/metrics/` +#### Pre-flight: Clean Memory Directory + +**MANDATORY FIRST STEP** — Run this bash command before doing anything else. It removes any non-metrics files that may be present from previous failed runs and ensures only `metrics/**` files exist in your working directory: + +```bash +# Remove any files at the root level that are NOT under metrics/ +find /tmp/gh-aw/repo-memory/default -maxdepth 1 -type f -delete 2>/dev/null || true +# List what remains to confirm only metrics/ content is present +ls -la /tmp/gh-aw/repo-memory/default/ +``` + +If you see any `.md` files at the root (e.g. `agent-performance-latest.md`, `shared-alerts.md`) — **delete them now**. You are NOT the agent-performance or alerts workflow. You are the Metrics Collector. Your ONLY job is to write JSON files under `metrics/`. + #### Metrics Collection Process ### 1. Use Agentic Workflows Tool to Collect Workflow Metrics @@ -148,10 +162,12 @@ Create a JSON object following this schema: ### 3. Store Metrics in Repo Memory -> ⚠️ **CRITICAL**: You MUST write ONLY JSON files inside the `metrics/` subdirectory. The repo-memory +> ⚠️ **CRITICAL — FILE PATH CONSTRAINT**: You MUST write ONLY JSON files inside the `metrics/` subdirectory. The repo-memory > glob filter is set to `metrics/**`, which means **any file written outside this subdirectory will be > silently dropped and no data will be persisted**. Do NOT write files to the root of > `/tmp/gh-aw/repo-memory/default/` — they will be ignored. +> +> **If you write any file to a path that does NOT start with `/tmp/gh-aw/repo-memory/default/metrics/`, that file will NOT be saved. Do not waste effort writing to any other path.** **Daily Storage**: - Write metrics to: `/tmp/gh-aw/repo-memory/default/metrics/daily/YYYY-MM-DD.json` @@ -162,7 +178,20 @@ Create a JSON object following this schema: - This provides quick access to most recent data without date calculations **Create Directory Structure**: -- Ensure the directory exists: `mkdir -p /tmp/gh-aw/repo-memory/default/metrics/daily/` +```bash +mkdir -p /tmp/gh-aw/repo-memory/default/metrics/daily/ +``` + +**Write and validate each file**: +```bash +# Write the daily metrics file (replace DATE and JSON_DATA with actual values) +echo "$JSON_DATA" | jq . > /tmp/gh-aw/repo-memory/default/metrics/daily/$(date +%Y-%m-%d).json +# Copy to latest +cp /tmp/gh-aw/repo-memory/default/metrics/daily/$(date +%Y-%m-%d).json \ + /tmp/gh-aw/repo-memory/default/metrics/latest.json +# Validate JSON is correct +jq . /tmp/gh-aw/repo-memory/default/metrics/latest.json >/dev/null && echo "JSON valid" || echo "JSON INVALID" +``` **File Constraint Summary** (glob filter: `metrics/**`): - ✅ `/tmp/gh-aw/repo-memory/default/metrics/latest.json` — allowed @@ -220,6 +249,25 @@ find /tmp/gh-aw/repo-memory/default/metrics/daily/ -name "*.json" -mtime +30 -de - If a workflow has no safe outputs, set all safe output counts to 0 - If token/cost data is unavailable, omit or set to null - Always include workflows in the metrics even if they have no activity (helps detect stalled workflows) +- **If the agentic-workflows `logs` tool is unavailable**, collect what you can from the GitHub API directly (workflow runs via `list_workflow_runs`) and set `"data_source": "github_api_fallback"` in the JSON +- **NEVER write a partial stub file like `{"date": "...", "status": "no-data"}`** — if you can't collect data, write a minimal valid metrics JSON with zeros instead: + ```json + { + "timestamp": "YYYY-MM-DDTHH:MM:SSZ", + "period": "daily", + "collection_status": "partial", + "collection_note": "Description of what could not be collected", + "workflows": {}, + "ecosystem": { + "total_workflows": 0, + "active_workflows": 0, + "total_safe_outputs": 0, + "overall_success_rate": 0, + "total_tokens": 0, + "total_cost_usd": 0 + } + } + ``` ### Workflow Name Extraction @@ -269,7 +317,8 @@ At the end of collection: #### Important Notes - **PRIMARY TOOL**: Use the agentic-workflows tool (`status`, `logs`) for all workflow run metrics -- **SECONDARY TOOL**: Use GitHub MCP server only for engagement metrics (reactions, comments) +- **SECONDARY TOOL**: Use GitHub MCP server for engagement metrics (reactions, comments) +- **YOU ARE THE METRICS COLLECTOR, NOT THE AGENT-PERFORMANCE WORKFLOW** — do NOT update `agent-performance-latest.md`, `shared-alerts.md`, or any other `.md` file in the repo-memory root. Those files belong to other workflows. - **DO NOT** create issues, PRs, or comments - this is a data collection agent only - **DO NOT** analyze or interpret the metrics - that's the job of meta-orchestrators - **DO NOT** write any files to the root of `/tmp/gh-aw/repo-memory/default/` — the glob filter `metrics/**` will silently discard them @@ -292,6 +341,43 @@ At the end of collection: ✅ Collection completed within timeout ✅ No errors or warnings in execution log +#### Pre-noop Validation (MANDATORY) + +Before calling `noop`, you MUST run this validation. If it fails, you must fix the files before proceeding: + +```bash +# Step 1: Verify no non-metrics files remain in the memory root +ROOT_FILES=$(find /tmp/gh-aw/repo-memory/default -maxdepth 1 -type f 2>/dev/null) +if [ -n "$ROOT_FILES" ]; then + echo "ERROR: Non-metrics files found at root level: $ROOT_FILES" + echo "Deleting them now..." + find /tmp/gh-aw/repo-memory/default -maxdepth 1 -type f -delete +fi + +# Step 2: Verify metrics/latest.json exists and has valid JSON with today's timestamp +if [ ! -f /tmp/gh-aw/repo-memory/default/metrics/latest.json ]; then + echo "ERROR: metrics/latest.json does not exist. You must write this file before calling noop." + exit 1 +fi + +# Step 3: Validate JSON syntax +jq . /tmp/gh-aw/repo-memory/default/metrics/latest.json >/dev/null || { + echo "ERROR: metrics/latest.json contains invalid JSON" + exit 1 +} + +# Step 4: Confirm the timestamp is recent (today) +STORED_DATE=$(jq -r '.timestamp' /tmp/gh-aw/repo-memory/default/metrics/latest.json | cut -c1-10) +TODAY=$(date +%Y-%m-%d) +echo "Stored date: $STORED_DATE | Today: $TODAY" + +# Step 5: List all metrics files that will be committed +echo "Files to be committed:" +find /tmp/gh-aw/repo-memory/default/metrics -type f | sort +``` + +If the validation passes, proceed with the `noop` call. + After successfully collecting and storing all metrics data, you **MUST** call `noop` with a brief collection summary — this is a data-collection workflow that persists results to repo-memory, so `noop` is the expected safe-output for every successful run. ```json