diff --git a/.gitignore b/.gitignore index 17484422..aad55269 100644 --- a/.gitignore +++ b/.gitignore @@ -49,3 +49,6 @@ prisma/dev.db-journal # IDE .idea/ .vscode/ + +# claude code (worktrees, local session state) +.claude/ diff --git a/Dockerfile.sync b/Dockerfile.sync new file mode 100644 index 00000000..2048c9c9 --- /dev/null +++ b/Dockerfile.sync @@ -0,0 +1,38 @@ +FROM node:20-slim + +# Install Chrome for Puppeteer +RUN apt-get update && apt-get install -y \ + chromium \ + fonts-liberation \ + libasound2 \ + libatk-bridge2.0-0 \ + libatk1.0-0 \ + libcups2 \ + libdbus-1-3 \ + libdrm2 \ + libgbm1 \ + libgtk-3-0 \ + libnspr4 \ + libnss3 \ + libxcomposite1 \ + libxdamage1 \ + libxrandr2 \ + xdg-utils \ + --no-install-recommends \ + && rm -rf /var/lib/apt/lists/* + +ENV CHROME_PATH=/usr/bin/chromium +ENV PUPPETEER_SKIP_DOWNLOAD=true + +WORKDIR /app + +COPY package*.json ./ +COPY prisma ./prisma/ +COPY scripts/daily-sync.ts ./scripts/ + +# tsx is not in package.json deps; install it into the image so the +# runtime `npx tsx` never has to fetch it. prisma generate runs against +# the copied (main) schema → enum-aware client (fixes the P2032 drift). +RUN npm ci && npm install --no-save tsx && npx prisma generate + +CMD ["npx", "tsx", "scripts/daily-sync.ts"] diff --git a/cloudbuild.sync.yaml b/cloudbuild.sync.yaml new file mode 100644 index 00000000..fb15b5be --- /dev/null +++ b/cloudbuild.sync.yaml @@ -0,0 +1,20 @@ +# Builds the daily-sync job image from Dockerfile.sync (not the web Dockerfile). +# The Prisma client is generated from this branch's enum schema, fixing the +# String-vs-enum P2032 drift that broke takoapi-daily-sync. +steps: + - name: 'gcr.io/cloud-builders/docker' + args: + - 'build' + - '-f' + - 'Dockerfile.sync' + - '-t' + - 'us-central1-docker.pkg.dev/$PROJECT_ID/takoapi-repo/takoapi-sync:latest' + - '.' + - name: 'gcr.io/cloud-builders/docker' + args: + - 'push' + - 'us-central1-docker.pkg.dev/$PROJECT_ID/takoapi-repo/takoapi-sync:latest' +images: + - 'us-central1-docker.pkg.dev/$PROJECT_ID/takoapi-repo/takoapi-sync:latest' +options: + logging: CLOUD_LOGGING_ONLY diff --git a/docs/05-automation-ingestion-review.md b/docs/05-automation-ingestion-review.md new file mode 100644 index 00000000..e29e2c8d --- /dev/null +++ b/docs/05-automation-ingestion-review.md @@ -0,0 +1,265 @@ +# 自动化内容摄取机制 — 全面 Review 与优化方案 + +> 范围:所有"自动往库里新增/刷新内容"的路径 —— GitHub 项目抓取、托管 A2A 代理导入、 +> 用户提交、技能抓取,以及配套的健康探测、场景分类、AgentCard 校验、cron 鉴权。 +> 每条问题都经过代码级复现验证,并附**正反方辩论**与**裁决**。 +> 生成日期:2026-07-11。 + +--- + +## 0. 机制全景 + +内容进入库有三条自动/半自动管道,核心逻辑收敛在 `src/lib/` 由 cron 端点与 CLI 脚本共享: + +| 管道 | 触发入口 | 共享逻辑 | 内容 | 上线策略 | +|------|----------|----------|------|----------| +| GitHub 项目抓取 | `api/cron/scrape-agents` · `scripts/scrape-github-agents.ts` | `lib/scrape-agents.ts` | Agent `kind=PROJECT` | **直接 APPROVED** | +| 托管 A2A 导入 | `api/cron/import-hosted` · `scripts/import-hosted-agents.ts` | `lib/import-hosted.ts` | Agent `kind=HOSTED` | 默认 PENDING(待审) | +| 用户提交 | `api/agents/submit` | — | Agent `HOSTED` | 普通用户 PENDING / admin key APPROVED | +| 技能抓取 | `scripts/scrape-github-skills.ts` 等(手动) | — | Skill `source=GITHUB_SCRAPE` | **直接 APPROVED** | + +配套:`api/cron/health`(探活 HOSTED)、`lib/scenarios.ts`(确定性关键词场景分类)、 +`lib/agentcard.ts`(AgentCard 拉取 + zod 校验 + 基础 SSRF 防护)、`lib/cron-auth.ts` +(`CRON_SECRET` 鉴权,未配置即 fail-closed)。 + +**结构性优点(保留)**:CLI/cron 共用同一 lib、全链路幂等 upsert、AgentCard 有 +大小/超时/schema 三重校验、GitHub 抓取带限流退避、star-farm 启发式过滤、第三方 +hosted agent 默认进审核队列、cron 鉴权 fail-closed。架构底子是健康的,以下是需要 +收口的具体缺陷。 + +--- + +## 1. 🔴 严重 · import-hosted 可劫持已上架条目(slug 碰撞覆盖) + +### 结论:CONFIRMED + +`lib/import-hosted.ts` 的 upsert 按 `slugify(card.name)` 定位。**update 分支不校验 +现有条目的 `kind` / `publisherId`,不把 `status` 重置为 PENDING**,却会覆盖 +`name / description / endpointUrl / cardUrl / securitySchemes`。"默认 PENDING 待审" +只对 **create** 生效,对 **update** 完全失效。 + +### 证据与复现 + +- `lib/import-hosted.ts:103-133`:update 分支写入 endpointUrl/cardUrl 等,无任何身份/状态守卫。 +- 默认注册表 `DEFAULT_REGISTRY = prassanna-ravishankar/a2a-registry`(第三方仓库,任何人可 PR)。 +- slugify 一致性验证:`lib/utils.ts` 的 `slugify`(用户提交用)与 `import-hosted.ts` 内联 + `slugify`(≤80 字符时)对同一 name 产出**完全相同**的 slug。 +- `api/registry` 对 `status=APPROVED` 的条目直接输出 `endpoint`(`registry/route.ts:69`)。 + +**复现路径**:某用户提交 HOSTED "Acme Assistant"(slug `acme-assistant`,admin 审核通过 → +APPROVED)。攻击者向社区注册表 PR 一张 name="Acme Assistant" 的 card → 下次 `import-hosted` +cron:slug 命中该已上架条目 → 走 update → `endpointUrl`/`cardUrl` 被静默替换为攻击者服务, +**且保持 APPROVED 直接对外**。同理可撞 `kind=PROJECT` 条目(card 命名 "vercel next.js" → +`vercel-next-js`),把"仅发现"的开源项目条目翻成一个可调用的恶意 endpoint。 + +### 正方(应修) + +- 这是**内容完整性 + 供应链**双重风险:外部一个 PR 就能改写我们已背书条目的执行端点。 +- registry 是产品的核心承诺("one API to discover all agents"),被投毒直接摧毁信任。 +- 修复面很小(集中在一个 update 分支加守卫),收益/风险比极高。 + +### 反方(可不修 / 风险) + +- 需攻击者知道目标 slug 且社区注册表 PR 被合并 —— 有一定门槛,非零日。 +- 加 publisher/origin 守卫可能挡掉**合法的域名迁移**(agent 换了托管域名),导致刷新失败。 +- 目前生产上 import-hosted 是否在定时跑、注册表是否可写,取决于外部调度(见 #7), + 未跑时风险是潜伏的而非活跃的。 + +### 裁决:修,采用"所有权 + 同源"双守卫 + +在 upsert 前 `findUnique(slug)`,按以下规则: +1. `existing.publisherId !== pub.id` → **跳过**(绝不触碰用户/他人拥有的行)。 +2. `existing.kind !== "HOSTED"` → **跳过**(hosted card 不得覆盖 PROJECT)。 +3. 同源(`origin(existing.cardUrl) === origin(new.cardUrl)`)→ 正常刷新,保留 status。 +4. 异源(域名变更)→ 允许更新字段但 **status 降级为 PENDING** 并 log,让审核台重新裁决 —— + 既不阻断合法迁移(反方顾虑),又不让换域后的端点继承旧背书。 +- 反方的"迁移被挡"由规则 4 化解;门槛低不构成不修理由。 + +--- + +## 2. 🔴 严重 · scrape-agents 撤销人工治理(强制翻回 APPROVED) + +### 结论:CONFIRMED(比初判更重 —— 删除也会被撤销) + +`lib/scrape-agents.ts:161` 的 update 分支**无条件写 `status: "APPROVED"`**;create 分支 +同样以 APPROVED 建行。于是 admin 的两种治理动作都不持久: + +- **REJECT**:admin 把垃圾/低质 repo 标 REJECTED(`admin/agents/[id]` PATCH,schema 允许该值)→ + 下次 cron 走 update → 被翻回 APPROVED。 +- **DELETE**:admin 删除该行 → slug 不存在 → 下次 cron 走 create → 以 APPROVED **重新建出来**。 + +### 证据与复现 + +- `lib/scrape-agents.ts:161` update `{ status: "APPROVED", ... }`;`:164` create `status: "APPROVED"`。 +- `schemas.ts:85` `adminAgentUpdateSchema.status` 含 `REJECTED / DISABLED`;`admin/agents/[id]/route.ts` PATCH 落库。 +- 与 memory 记录直接相关:prod `/api/registry` 顶部混入 star-farm/flagged 条目 —— 人工清理会被自动化持续覆盖。 +- 触发条件:该 repo 仍在 GitHub 搜索结果内且通过 `minStars` + `looksStarFarmed` 过滤(即"看起来正常但被人工判低质"的那类,恰恰最容易复发)。 + +### 正方(应修) + +- 自动化**不应有权撤销人类的显式裁决**,这是治理系统的底线。 +- 直接解释了 memory 里长期存在的"registry 顶部混垃圾"现象,是根因而非表象。 + +### 反方(可不修 / 风险) + +- 若某 repo 被误拒,"自动复活"反而是一种自愈。 +- 保留 REJECTED 墓碑意味着爬虫要多查一次状态(2000 行 × findUnique),有成本。 +- 只改 update、不动 create 的话,删除仍会复活 —— 要彻底需要墓碑机制,复杂度上升。 + +### 裁决:修,墓碑集合 + update 不碰 status + +1. 循环前一次性查出 `status ∈ {REJECTED, DISABLED}` 的 slug 集合(单查询,规避 2000 次往返 —— 化解反方成本顾虑)。 +2. 命中墓碑的 repo **跳过**(尊重 REJECT 与 DELETE-后-被拒 的人工意志)。 +3. update 分支**移除 `status`**,只刷 `stars / description / scenarios`;新发现仍以 create=APPROVED 上线。 +- "误拒自愈"由 admin 手动改回 APPROVED 即可,不该由爬虫代劳。 + +--- + +## 3. 🟠 中 · cron 抓取无耗时可观测 + 超时留下半写(初判"必然超时"已更正) + +### 结论:CONFIRMED(降级 —— 非"默认必然超时",而是"不可观测 + 非事务半写") + +初版说"300s 几乎必然超时"**高估了**。实测口径:默认 `pages=1`、20 条 query、 +`reqDelayMs=900` → 约 18s 搜索 + 最多 2000 次串行 upsert(prod ~15ms/次 ≈ 30s), +合计约 ~50s,**默认配置下并不必然超时**。真正的缺陷是另外两点: + +1. `export const maxDuration = 300` 在 Cloud Run `output:"standalone"` 下**大概率是空操作** —— + 实际上限是 Cloud Run 服务的 request timeout(默认 300s),`maxDuration` 是 Vercel 语义。 + 即代码里那行给不了它声称的保护。 +2. `pages=2~3` 或触发 GitHub 限流(每次等待上限 120s)时,总耗时可逼近/超过服务超时; + 一旦被切断,由于**逐条 upsert 非事务**,会留下部分写入,且响应无耗时/进度字段,**无法判断某次 run 是否只完成了一半**。 + +### 证据与复现 + +- `api/cron/scrape-agents/route.ts:31` `reqDelayMs: 900`;`:11` `maxDuration = 300`。 +- `next.config.ts` `output: "standalone"`(Cloud Run),无 route-timeout 处理 → maxDuration 非有效上限。 +- `lib/scrape-agents.ts` 逐条 `await db.agent.upsert`,无事务、无批处理、响应无 `durationMs`。 + +### 正方(应修) + +- 静默半失败最坏:每天定时任务悄悄只跑一半,没人发现。 +- 加耗时/计数遥测成本极低,收益是可观测性。 + +### 反方(可不修 / 风险) + +- 幂等 upsert 意味着"半写"下次会补齐,数据不会损坏,只是当次不全。 +- 真正的时限治理应在 Cloud Run 服务配置层(调 timeout),代码层改动有限。 + +### 裁决:低成本收口(不做大重构) + +- 三个 cron 响应统一加 `durationMs`(和已有 `ranAt` 配套),让调度日志能看出耗时趋势与截断。 +- 保留 `maxDuration`(无害),但在注释里点明"Cloud Run 下有效上限来自服务 timeout",避免误导。 +- 事务化/批处理留作后续(反方合理:幂等已兜底数据安全,不紧急)。 + +--- + +## 4. 🟠 高 · AgentCard SSRF:重定向 + DNS 两个绕过口 + +### 结论:CONFIRMED + +`lib/agentcard.ts` 的 `assertSafeUrl` 只对**初始 URL 字符串**做静态私网黑名单,存在两个现实绕过: + +1. **重定向**:`fetchOne` 用 `redirect: "follow"`(`agentcard.ts:134`),302 目标**不再过校验** → + 攻击者用一个公网 URL 302 到 `169.254.169.254`(云元数据)即可。 +2. **DNS**:只比对 hostname 字面量,不解析 IP → `http://x.attacker.com` 解析到内网地址即绕过(DNS rebinding)。 + +该链路对**公开的用户提交端点**(`api/agents/submit` → `fetchAgentCard`)同样生效,不止 cron。 +代码 §11 注释已自认是 best-effort TODO,但端点是公开可达的,风险是现实的。 + +### 证据与复现 + +- `agentcard.ts:62-86` `assertSafeUrl` 仅字符串匹配;`:131-135` `redirect: "follow"`。 +- `api/agents/submit/route.ts:55` 未认证前置的 `fetchAgentCard(input.cardUrl)`(有速率限制,但任意登录用户可打)。 + +### 正方(应修) + +- 云元数据端点泄露 = 潜在凭证泄露,SSRF 属高危类。 +- 修法成熟(手动逐跳重定向 + 解析后 IP 校验),不影响正常 card 抓取。 + +### 反方(可不修 / 风险) + +- 彻底防护需 socket 级(connect 时校验 IP)才能杜绝 check-then-connect 的 TOCTOU,纯应用层是"大幅缓解"非"根治"。 +- 严格 DNS 校验可能误伤某些走内网 DNS 但合法的自托管 agent(生产环境少见)。 +- 生产 egress 若已有网络层限制(VPC/防火墙),应用层是纵深防御而非唯一防线。 + +### 裁决:修,实现"手动逐跳 + 每跳 DNS/IP 私网校验" + +- `redirect: "manual"`,自己跟随(上限 3 跳),每跳 `Location` 都过 `assertSafeUrl` + DNS 解析校验。 +- 新增 `assertPublicHost(hostname)`:解析所有 A/AAAA,任一落在私网/环回/链路本地即拒。 +- 明确记录**残留风险**:connect 级 TOCTOU 未根治(反方点),留作 socket-level 后续;当前修复关闭了两个可被单请求利用的现实口子,是纵深防御的应用层一环。 + +--- + +## 5. 🟡 轻 · star-farm 阈值与实测星农 band 不一致(初判"正则重编译"已撤回) + +### 结论:大部分 RETRACTED,仅保留一个可调参数 + +- **撤回**:初版称 `classifyScenarios` "每次调用重编译 20×N 正则" —— **错误**。`scenarios.ts:294` + `MATCHERS` 是**模块级 const,只编译一次**;每次调用只做 `.test()`。无性能缺陷,收回该条。 +- **保留(轻)**:`looksStarFarmed`(`scrape-agents.ts:52-56`)对 `stars < 150` 直接放行不判 fork 比; + 而 memory 记录实测星农 band 在 ~340★ 附近。150 的地板可能仍放进一部分星农 repo。 + +### 正反方 + +- 正方:调高地板或对 150–400★ 区间也看 fork 比,能少放一点垃圾进顶部。 +- 反方:阈值本质是经验值,收紧有**误伤真实小项目**的风险;且 #2 修好后,人工 REJECT 会持久生效, + 自动阈值的重要性下降 —— 用"人工兜底"替代"参数精调"更稳。 + +### 裁决:本轮不改参数,依赖 #2 的人工治理持久化 + +仅在代码注释里补一行,提示 150 地板与观测到的 ~340 band 的关系,留给后续按数据调。不动逻辑,避免误伤。 + +--- + +## 6. 🟡 低 · health cron 无时限 + 静默吞错 + +### 结论:CONFIRMED(低) + +- `api/cron/health/route.ts` **无 `maxDuration`**;HOSTED 增多后 `PROBE_CONCURRENCY=8` × 8s 超时可能撞默认时限。 +- `lib/health.ts:53` `.catch(() => {})` 把写库失败完全吞掉 → 排障时看不到失败。 + +### 正反方 + +- 正方:探测量会随 hosted agent 增长,现在加时限 + 错误计数几乎零成本。 +- 反方:当前 hosted 数量少,离撞限很远;吞错只影响可观测性,不影响正确性。 + +### 裁决:顺手修(零风险) + +加 `maxDuration=120`;`.catch` 改为累加 `writeErrors` 计数并计入 summary,不再静默。 + +--- + +## 7. 🟠 中(流程) · 调度器配置不在代码库 + +### 结论:CONFIRMED + +三个 cron 全靠外部 Cloud Scheduler 触发(HANDOFF 提到 `tako-cron-secret`),但仓库内 +**无任何调度配置**(无 `vercel.json`;`cloudbuild.yaml` 按 memory 是 stale 模板)。 +"哪些 job 在跑、频率、参数(尤其 `?pages=`/`?minStars=`/`?max=`)"完全是带外知识, +无法 review、无法版本化、迁移/重装极易漏配或配错。 + +### 正反方 + +- 正方:调度是自动化的"启动开关",不进版本库等于生产行为不可复现。 +- 反方:Cloud Scheduler 配置含项目 ID 等环境细节,直接入库需注意不泄密;且它属于基础设施而非应用代码。 + +### 裁决:补一份可复现的调度说明 + 幂等建置脚本 + +新增 `docs/06-cron-schedule.md`:列出每个 job 的 URL/频率/参数/鉴权,并给出**幂等的 `gcloud scheduler` 建置命令** +(密钥用占位符从 Secret Manager 取,不硬编码)。不碰 stale 的 cloudbuild.yaml。 + +--- + +## 8. 实施优先级与顺序 + +| 优先级 | 问题 | 动作 | 风险 | +|--------|------|------|------| +| P0 | #1 import 劫持 | 所有权 + 同源守卫,异源降级 PENDING | 低(只加守卫) | +| P0 | #2 撤销治理 | 墓碑集合跳过 + update 不碰 status | 低 | +| P1 | #4 SSRF | 手动逐跳重定向 + DNS/IP 私网校验 | 中(需测正常 card 仍可拉) | +| P2 | #3 可观测 | 三 cron 加 `durationMs`;注释澄清 maxDuration | 极低 | +| P2 | #6 health | `maxDuration` + 错误计数不吞 | 极低 | +| P3 | #7 调度 | 新增 cron 调度文档 + gcloud 幂等脚本 | 无(纯文档) | +| — | #5 阈值 | 仅补注释,不改逻辑 | 无 | + +验证:每步后 `npx tsc --noEmit` + `npm run lint`;SSRF 改动额外跑一次真实 card 拉取回归。 diff --git a/docs/06-cron-schedule.md b/docs/06-cron-schedule.md new file mode 100644 index 00000000..91dea8ec --- /dev/null +++ b/docs/06-cron-schedule.md @@ -0,0 +1,76 @@ +# Cron 调度 — 权威清单与幂等建置 + +> 三个自动摄取端点全靠 **Cloud Scheduler** 触发,但调度配置此前不在代码库里 +> (无 `vercel.json`,`cloudbuild.yaml` 是 stale 模板)。本文件是它们的**唯一权威来源**: +> 频率、参数、鉴权,以及可复现的 `gcloud` 建置命令。改调度请改这里并同步执行。 +> 关联:`docs/05-automation-ingestion-review.md` §7。 + +## 端点一览 + +| Job | 端点 | 建议频率 | 参数 | 落库策略 | +|-----|------|----------|------|----------| +| `tako-scrape-agents` | `GET /api/cron/scrape-agents` | 每日 04:00 UTC | `?pages=1&minStars=200&max=2000` | PROJECT 直接 APPROVED;admin REJECT/DISABLE 的 slug 会被跳过 | +| `tako-import-hosted` | `GET /api/cron/import-hosted` | 每日 04:30 UTC | `?max=0`(不限) | HOSTED 落 PENDING,待 `/admin/agents` 审核;跨域变更也降级 PENDING | +| `tako-health` | `GET /api/cron/health` | 每 6 小时 | —— | 更新 `healthStatus`/`healthCheckedAt` | + +所有端点鉴权:`Authorization: Bearer `(见 `src/lib/cron-auth.ts`; +`CRON_SECRET` 未配置时端点 fail-closed 返回 401)。响应含 `durationMs` + `ranAt`, +调度日志据此判断是否接近超时 / 被截断。 + +## 建置(幂等 create-or-update) + +```bash +set -euo pipefail +PROJECT=takoapi-491505 +REGION=us-central1 +LOCATION=us-central1 # Cloud Scheduler location +BASE=https://takoapi.com # 或 run URL: https://takoapi-429522911261.us-central1.run.app + +# CRON_SECRET 来源(二选一,取决于是否启用 Secret Manager — 见 docs/00-infrastructure.md): +# a) Secret Manager: +SECRET=$(gcloud secrets versions access latest --secret=tako-cron-secret --project="$PROJECT") +# b) 若 Secret Manager 未启用,从 Cloud Run 服务 env 读取当前值: +# SECRET=$(gcloud run services describe takoapi --region "$REGION" --project="$PROJECT" \ +# --format='value(spec.template.spec.containers[0].env)' | tr ',' '\n' | grep -A1 CRON_SECRET | tail -1) + +# 幂等 upsert:create 失败(已存在)即改 update。 +upsert_job () { + local name="$1" schedule="$2" uri="$3" + local verb=create + gcloud scheduler jobs describe "$name" --location="$LOCATION" --project="$PROJECT" >/dev/null 2>&1 && verb=update + gcloud scheduler jobs "$verb" http "$name" \ + --location="$LOCATION" --project="$PROJECT" \ + --schedule="$schedule" --time-zone="Etc/UTC" \ + --uri="$uri" --http-method=GET \ + --headers="Authorization=Bearer ${SECRET}" \ + --attempt-deadline=320s \ + --max-retry-attempts=1 +} + +upsert_job tako-scrape-agents "0 4 * * *" "${BASE}/api/cron/scrape-agents?pages=1&minStars=200&max=2000" +upsert_job tako-import-hosted "30 4 * * *" "${BASE}/api/cron/import-hosted?max=0" +upsert_job tako-health "0 */6 * * *" "${BASE}/api/cron/health" +``` + +> `--attempt-deadline=320s` 略高于 scrape-agents 的 `maxDuration=300`,以免 Scheduler 先于端点超时。 +> **Cloud Run 服务的 request timeout 必须 ≥ 320s**(否则那才是真正的截断点);核对: +> `gcloud run services describe takoapi --region us-central1 --project=takoapi-491505 --format='value(spec.template.spec.timeoutSeconds)'`。 + +## 运维 + +```bash +# 列出 / 查看 +gcloud scheduler jobs list --location=us-central1 --project=takoapi-491505 +# 手动触发一次(冒烟) +gcloud scheduler jobs run tako-health --location=us-central1 --project=takoapi-491505 +# 轮换 CRON_SECRET 后,重跑上面的 upsert_job 即可刷新 header +# 暂停 / 恢复 +gcloud scheduler jobs pause tako-scrape-agents --location=us-central1 --project=takoapi-491505 +gcloud scheduler jobs resume tako-scrape-agents --location=us-central1 --project=takoapi-491505 +``` + +## 安全备注 + +- header 里带的是明文 bearer secret;`gcloud scheduler jobs describe` 可见,权限同项目其他密钥。 + 若日后启用 OIDC(service account),可改用 `--oidc-service-account-email` 并把 `cron-auth.ts` 换成校验 OIDC token。 +- **别把 `SECRET` 硬编码进本文件或提交历史**;上面的命令都从 Secret Manager / 运行时 env 现取。 diff --git a/package-lock.json b/package-lock.json index acc07d9c..b3ace79a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -21,6 +21,7 @@ "prisma": "^6.19.2", "react": "19.2.4", "react-dom": "19.2.4", + "resend": "^6.10.0", "tailwind-merge": "^3.5.0", "zod": "^4.3.6" }, @@ -1755,6 +1756,12 @@ "integrity": "sha512-bXHSaW5jRTmke9Vd0h5P7BtWZG9Znqb8gSDxZnxaGSJnGwPLDPfS+3g0BKzeWqzgZPsIVZkM7m2tbo18cm5HBw==", "license": "MIT" }, + "node_modules/@stablelib/base64": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@stablelib/base64/-/base64-1.0.1.tgz", + "integrity": "sha512-1bnPQqSxSuc3Ii6MhBysoWCg58j97aUjuCSZrGSmDxNqtytIi0k8utUenAwTZN4V5mXXYGsVUI9zeBqy+jBOSQ==", + "license": "MIT" + }, "node_modules/@standard-schema/spec": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz", @@ -4884,6 +4891,12 @@ "dev": true, "license": "MIT" }, + "node_modules/fast-sha256": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/fast-sha256/-/fast-sha256-1.3.0.tgz", + "integrity": "sha512-n11RGP/lrWEFI/bWdygLxhI+pVeo1ZYIVwvvPkW7azl/rOy+F3HYRZ2K5zeE9mmkhQppyv9sQFx0JM9UabnpPQ==", + "license": "Unlicense" + }, "node_modules/fastq": { "version": "1.20.1", "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.20.1.tgz", @@ -6801,17 +6814,6 @@ } } }, - "node_modules/next-intl/node_modules/@swc/helpers": { - "version": "0.5.23", - "resolved": "https://registry.npmjs.org/@swc/helpers/-/helpers-0.5.23.tgz", - "integrity": "sha512-5lSsMOTXURePglDfvuAQUqkGek9Hg2kksOYay2m0+XR++b2NWYL/4sWyuvVBIs8oKnJaxkdi9whaL/sqN13afw==", - "license": "Apache-2.0", - "optional": true, - "peer": true, - "dependencies": { - "tslib": "^2.8.0" - } - }, "node_modules/next/node_modules/postcss": { "version": "8.4.31", "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.31.tgz", @@ -7256,6 +7258,12 @@ "node": ">= 0.4" } }, + "node_modules/postal-mime": { + "version": "2.7.4", + "resolved": "https://registry.npmjs.org/postal-mime/-/postal-mime-2.7.4.tgz", + "integrity": "sha512-0WdnFQYUrPGGTFu1uOqD2s7omwua8xaeYGdO6rb88oD5yJ/4pPHDA4sdWqfD8wQVfCny563n/HQS7zTFft+f/g==", + "license": "MIT-0" + }, "node_modules/postcss": { "version": "8.5.8", "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.8.tgz", @@ -7580,6 +7588,27 @@ "node": ">=0.10.0" } }, + "node_modules/resend": { + "version": "6.17.1", + "resolved": "https://registry.npmjs.org/resend/-/resend-6.17.1.tgz", + "integrity": "sha512-QhHHIRPPM9Tq2uilWmNF8C8kd6nLkUmiFgDQCt1v4eR4o+6p86qrK+Vn12jnAs0jR8Gf6ABdxI18/90LGQ7GVA==", + "license": "MIT", + "dependencies": { + "postal-mime": "2.7.4", + "standardwebhooks": "1.0.0" + }, + "engines": { + "node": ">=20" + }, + "peerDependencies": { + "@react-email/render": "*" + }, + "peerDependenciesMeta": { + "@react-email/render": { + "optional": true + } + } + }, "node_modules/resolve": { "version": "1.22.11", "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.11.tgz", @@ -8021,6 +8050,16 @@ "dev": true, "license": "MIT" }, + "node_modules/standardwebhooks": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/standardwebhooks/-/standardwebhooks-1.0.0.tgz", + "integrity": "sha512-BbHGOQK9olHPMvQNHWul6MYlrRTAOKn03rOe4A8O3CLWhNf4YHBqq2HJKKC+sfqpxiBY52pNeesD6jIiLDz8jg==", + "license": "MIT", + "dependencies": { + "@stablelib/base64": "^1.0.0", + "fast-sha256": "^1.3.0" + } + }, "node_modules/stop-iteration-iterator": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/stop-iteration-iterator/-/stop-iteration-iterator-1.1.0.tgz", diff --git a/package.json b/package.json index 6910add5..c4e5f460 100644 --- a/package.json +++ b/package.json @@ -23,6 +23,7 @@ "prisma": "^6.19.2", "react": "19.2.4", "react-dom": "19.2.4", + "resend": "^6.10.0", "tailwind-merge": "^3.5.0", "zod": "^4.3.6" }, diff --git a/scripts/check-github-status.ts b/scripts/check-github-status.ts new file mode 100644 index 00000000..abd134b0 --- /dev/null +++ b/scripts/check-github-status.ts @@ -0,0 +1,65 @@ +import { PrismaClient } from "@prisma/client"; + +const prisma = new PrismaClient(); +const CONCURRENCY = 8; + +async function checkOne(skill: { id: string; githubUrl: string }) { + try { + const res = await fetch(skill.githubUrl, { method: "HEAD", redirect: "follow" }); + const status = res.status === 404 ? "404" : res.ok ? "ok" : "error"; + await prisma.skill.update({ + where: { id: skill.id }, + data: { ghStatus: status, ghCheckedAt: new Date() }, + }); + return status; + } catch { + await prisma.skill.update({ + where: { id: skill.id }, + data: { ghStatus: "error", ghCheckedAt: new Date() }, + }); + return "error"; + } +} + +async function worker(queue: { id: string; githubUrl: string }[], stats: Record) { + while (queue.length > 0) { + const skill = queue.shift(); + if (!skill) break; + const status = await checkOne(skill); + stats[status] = (stats[status] || 0) + 1; + // Parenthesize the sum: `%` binds tighter than `+`, so without these the + // condition was `ok + 404 + (error % 100) === 0` and only ever fired at zero. + if (((stats.ok || 0) + (stats["404"] || 0) + (stats.error || 0)) % 100 === 0) { + console.log(` Progress: ok=${stats.ok || 0}, 404=${stats["404"] || 0}, error=${stats.error || 0}`); + } + } +} + +async function main() { + const oneWeekAgo = new Date(Date.now() - 7 * 24 * 60 * 60 * 1000); + + const skills = await prisma.skill.findMany({ + where: { + githubUrl: { not: null }, + OR: [{ ghCheckedAt: null }, { ghCheckedAt: { lt: oneWeekAgo } }], + }, + select: { id: true, githubUrl: true }, + take: 1000, + }); + + console.log(`Checking ${skills.length} GitHub URLs (concurrency=${CONCURRENCY})...`); + + const queue = skills.filter((s): s is { id: string; githubUrl: string } => !!s.githubUrl); + const stats: Record = {}; + + await Promise.all(Array.from({ length: CONCURRENCY }, () => worker(queue, stats))); + + console.log(`\nDone:`); + console.log(` ok: ${stats.ok || 0}`); + console.log(` 404: ${stats["404"] || 0}`); + console.log(` error: ${stats.error || 0}`); +} + +main() + .catch(console.error) + .finally(() => prisma.$disconnect()); diff --git a/scripts/daily-sync.ts b/scripts/daily-sync.ts new file mode 100644 index 00000000..99abcc3d --- /dev/null +++ b/scripts/daily-sync.ts @@ -0,0 +1,307 @@ +/** + * Daily Sync Script for TakoAPI + * 1. Incremental update from ClawSkills.sh (new skills + updated downloads/stars) + * 2. Deduplicate skills (keep highest downloads per skill name) + * 3. Recount category skill counts + * + * Run: DATABASE_URL=... npx tsx scripts/daily-sync.ts + */ + +import puppeteer from "puppeteer-core"; +import { PrismaClient } from "@prisma/client"; + +const CHROME_PATH = + process.env.CHROME_PATH || + "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"; + +const prisma = new PrismaClient(); + +function slugify(text: string): string { + return text + .toLowerCase() + .replace(/&/g, "and") + .replace(/[^a-z0-9]+/g, "-") + .replace(/(^-|-$)/g, ""); +} + +function parseDownloads(dl: string): number { + if (!dl || dl === "0") return 0; + dl = dl.replace(/,/g, ""); + if (dl.endsWith("k")) return Math.round(parseFloat(dl) * 1000); + if (dl.endsWith("M")) return Math.round(parseFloat(dl) * 1000000); + return parseInt(dl) || 0; +} + +// ============================================ +// STEP 1: Incremental sync from ClawSkills.sh +// ============================================ +async function syncFromClawSkills() { + console.log("\n=== Step 1: Syncing from ClawSkills.sh ==="); + + const browser = await puppeteer.launch({ + executablePath: CHROME_PATH, + headless: true, + args: ["--no-sandbox", "--disable-setuid-sandbox"], + }); + + const page = await browser.newPage(); + await page.goto("https://clawskills.sh/", { + waitUntil: "networkidle0", + timeout: 60000, + }); + await page.waitForSelector('a[href*="/skills/"]', { timeout: 30000 }); + + // Get all category buttons + const catNames: string[] = await page.evaluate(() => { + const buttons = Array.from(document.querySelectorAll("button")); + return buttons + .filter((b) => { + const text = b.textContent?.trim() || ""; + return ( + /^[A-Z].*\d+$/.test(text) && text.length < 60 && !text.includes("5147") + ); + }) + .map((b) => { + const text = b.textContent?.trim() || ""; + const match = text.match(/^(.+?)(\d+)$/); + return match ? match[1].trim() : text; + }); + }); + + // Get existing slugs from DB + const existingSlugs = new Set( + (await prisma.skill.findMany({ select: { slug: true } })).map((s) => s.slug) + ); + + // Get existing category map + const categoryMap = new Map(); + const categories = await prisma.category.findMany(); + for (const cat of categories) { + categoryMap.set(cat.name, cat.id); + } + + let newCount = 0; + let updatedCount = 0; + + // Process each category + for (const catName of catNames) { + let categoryId = categoryMap.get(catName); + if (!categoryId) { + // Create new category + const cat = await prisma.category.create({ + data: { name: catName, slug: slugify(catName), skillCount: 0 }, + }); + categoryId = cat.id; + categoryMap.set(catName, categoryId); + console.log(` New category: ${catName}`); + } + + // Click category button + const clicked = await page.evaluate((name: string) => { + const buttons = Array.from(document.querySelectorAll("button")); + const btn = buttons.find((b) => + (b.textContent?.trim() || "").startsWith(name) + ); + if (btn) { + btn.click(); + return true; + } + return false; + }, catName); + + if (!clicked) continue; + await new Promise((r) => setTimeout(r, 500)); + + // Extract skills + const skills = await page.evaluate(() => { + const links = Array.from(document.querySelectorAll("a")).filter( + (a) => + a.href && + a.href.includes("/skills/") && + !a.href.endsWith("/skills/") + ); + return links.map((a) => { + const slug = a.href.split("/skills/")[1]; + const children = Array.from(a.querySelectorAll("*")).filter( + (c) => c.children.length === 0 + ); + const texts = children + .map((c) => c.textContent?.trim() || "") + .filter((t) => t.length > 0); + return { + slug, + name: texts[1] || "", + author: (texts[2] || "").replace("/skills", ""), + description: texts[3] || "", + downloads: texts[4] || "0", + stars: texts[5] || "0", + }; + }); + }); + + for (const sk of skills) { + if (!sk.slug || !sk.name) continue; + + const downloads = parseDownloads(sk.downloads); + const stars = parseInt(sk.stars) || 0; + + if (existingSlugs.has(sk.slug)) { + // Update existing: downloads + stars only + await prisma.skill.update({ + where: { slug: sk.slug }, + data: { downloads, stars }, + }); + updatedCount++; + } else { + // Insert new skill + try { + await prisma.skill.create({ + data: { + name: sk.name, + slug: sk.slug, + brief: sk.description, + description: sk.description, + clawSkillsUrl: `https://clawskills.sh/skills/${sk.slug}`, + clawHubUrl: `https://clawskills.sh/skills/${sk.slug}`, + installCmd: `clawhub install ${sk.name}`, + author: sk.author, + categoryId, + downloads, + stars, + // Enum schema (main): new skills default to PENDING (hidden). + // ClawSkills is a curated bulk import → publish + tag source. + status: "APPROVED", + source: "CURATED", + }, + }); + existingSlugs.add(sk.slug); + newCount++; + } catch { + // Slug collision - skip + } + } + } + } + + await browser.close(); + console.log(` New skills: ${newCount}`); + console.log(` Updated skills: ${updatedCount}`); + return { newCount, updatedCount }; +} + +// ============================================ +// STEP 2: Deduplicate skills +// ============================================ +async function deduplicateSkills() { + console.log("\n=== Step 2: Deduplicating skills ==="); + + // Find duplicate names (case-insensitive) + const allSkills = await prisma.skill.findMany({ + select: { id: true, name: true, slug: true, downloads: true, likesCount: true, viewsCount: true }, + orderBy: { downloads: "desc" }, + }); + + // Group by lowercase name + const groups = new Map(); + for (const skill of allSkills) { + const key = skill.name.toLowerCase().trim(); + if (!groups.has(key)) { + groups.set(key, []); + } + groups.get(key)!.push(skill); + } + + let removedCount = 0; + const toDelete: string[] = []; + + for (const [name, skills] of groups) { + if (skills.length <= 1) continue; + + // Sort by downloads desc, then likesCount, then viewsCount + skills.sort((a, b) => { + if (b.downloads !== a.downloads) return b.downloads - a.downloads; + if (b.likesCount !== a.likesCount) return b.likesCount - a.likesCount; + return b.viewsCount - a.viewsCount; + }); + + // Keep the first (highest downloads), delete the rest + const keep = skills[0]; + const dupes = skills.slice(1); + + if (dupes.length > 0) { + console.log( + ` "${name}": keeping ${keep.slug} (${keep.downloads} dl), removing ${dupes.length} dupes` + ); + } + + for (const dupe of dupes) { + toDelete.push(dupe.id); + } + removedCount += dupes.length; + } + + // Delete duplicates (and their likes) + if (toDelete.length > 0) { + await prisma.like.deleteMany({ + where: { skillId: { in: toDelete } }, + }); + await prisma.skill.deleteMany({ + where: { id: { in: toDelete } }, + }); + } + + console.log(` Removed ${removedCount} duplicate skills`); + return removedCount; +} + +// ============================================ +// STEP 3: Recount categories +// ============================================ +async function recountCategories() { + console.log("\n=== Step 3: Recounting categories ==="); + + const categories = await prisma.category.findMany(); + for (const cat of categories) { + const count = await prisma.skill.count({ + where: { categoryId: cat.id }, + }); + if (count !== cat.skillCount) { + await prisma.category.update({ + where: { id: cat.id }, + data: { skillCount: count }, + }); + } + } + console.log(` Recounted ${categories.length} categories`); +} + +// ============================================ +// Main +// ============================================ +async function main() { + const start = Date.now(); + console.log(`TakoAPI Daily Sync - ${new Date().toISOString()}`); + + try { + const syncResult = await syncFromClawSkills(); + const removedCount = await deduplicateSkills(); + await recountCategories(); + + const totalSkills = await prisma.skill.count(); + const duration = ((Date.now() - start) / 1000).toFixed(1); + + console.log(`\n=== Summary ===`); + console.log(` New skills: ${syncResult.newCount}`); + console.log(` Updated: ${syncResult.updatedCount}`); + console.log(` Deduped: ${removedCount}`); + console.log(` Total skills: ${totalSkills}`); + console.log(` Duration: ${duration}s`); + } catch (error) { + console.error("Sync failed:", error); + process.exit(1); + } +} + +main() + .catch(console.error) + .finally(() => prisma.$disconnect()); diff --git a/scripts/deploy-sync-job.sh b/scripts/deploy-sync-job.sh new file mode 100644 index 00000000..75da8f1f --- /dev/null +++ b/scripts/deploy-sync-job.sh @@ -0,0 +1,67 @@ +#!/bin/bash +# Deploy the daily sync job to Cloud Run + Cloud Scheduler +# Usage: ./scripts/deploy-sync-job.sh + +set -e + +PROJECT=takoapi-491505 +REGION=us-central1 +JOB_NAME=takoapi-daily-sync +IMAGE=us-central1-docker.pkg.dev/$PROJECT/takoapi-repo/takoapi-sync:latest + +echo "=== Building sync Docker image ===" +gcloud builds submit \ + --project=$PROJECT \ + --tag=$IMAGE \ + --dockerfile=Dockerfile.sync \ + --timeout=600 + +echo "=== Creating/Updating Cloud Run Job ===" +gcloud run jobs create $JOB_NAME \ + --project=$PROJECT \ + --region=$REGION \ + --image=$IMAGE \ + --set-env-vars="DATABASE_URL=postgresql://postgres:TakoAPI2026Secure@localhost/takoapi?host=/cloudsql/$PROJECT:$REGION:takoapi-db,CHROME_PATH=/usr/bin/chromium" \ + --set-cloudsql-instances=$PROJECT:$REGION:takoapi-db \ + --memory=1Gi \ + --cpu=1 \ + --task-timeout=1800 \ + --max-retries=1 \ + --quiet 2>/dev/null || \ +gcloud run jobs update $JOB_NAME \ + --project=$PROJECT \ + --region=$REGION \ + --image=$IMAGE \ + --set-env-vars="DATABASE_URL=postgresql://postgres:TakoAPI2026Secure@localhost/takoapi?host=/cloudsql/$PROJECT:$REGION:takoapi-db,CHROME_PATH=/usr/bin/chromium" \ + --set-cloudsql-instances=$PROJECT:$REGION:takoapi-db \ + --memory=1Gi \ + --cpu=1 \ + --task-timeout=1800 \ + --max-retries=1 \ + --quiet + +echo "=== Enabling Cloud Scheduler API ===" +gcloud services enable cloudscheduler.googleapis.com --project=$PROJECT --quiet + +echo "=== Creating Cloud Scheduler (daily at 03:00 UTC) ===" +gcloud scheduler jobs create http $JOB_NAME-schedule \ + --project=$PROJECT \ + --location=$REGION \ + --schedule="0 3 * * *" \ + --uri="https://$REGION-run.googleapis.com/apis/run.googleapis.com/v1/namespaces/$PROJECT/jobs/$JOB_NAME:run" \ + --http-method=POST \ + --oauth-service-account-email=$(gcloud iam service-accounts list --project=$PROJECT --format="value(email)" --filter="displayName:Default compute service account" | head -1) \ + --quiet 2>/dev/null || \ +gcloud scheduler jobs update http $JOB_NAME-schedule \ + --project=$PROJECT \ + --location=$REGION \ + --schedule="0 3 * * *" \ + --uri="https://$REGION-run.googleapis.com/apis/run.googleapis.com/v1/namespaces/$PROJECT/jobs/$JOB_NAME:run" \ + --http-method=POST \ + --quiet + +echo "" +echo "=== Done! ===" +echo "Job: $JOB_NAME" +echo "Schedule: Daily at 03:00 UTC" +echo "Manual run: gcloud run jobs execute $JOB_NAME --region=$REGION --project=$PROJECT" diff --git a/scripts/scrape-skill-details.ts b/scripts/scrape-skill-details.ts index 983c015f..36bbf7c1 100644 --- a/scripts/scrape-skill-details.ts +++ b/scripts/scrape-skill-details.ts @@ -10,7 +10,6 @@ interface SkillDetail { whenToUseIt: string[] | null; exampleWorkflow: string | null; requirements: string[] | null; - githubUrl: string | null; installCmd: string | null; version: string | null; } @@ -89,12 +88,16 @@ async function scrapeSkillPage(page: puppeteer.Page, slug: string): Promise a.href && a.href.includes("github.com/openclaw/skills") - ); - if (ghLink) githubUrl = ghLink.href; + // NOTE: intentionally NOT scraping a githubUrl here. + // clawskills.sh skill pages do not expose a per-skill source repo — the + // only GitHub link on the page is the site-wide "Star" button pointing at + // the aggregate list github.com/VoltAgent/awesome-openclaw-skills, and the + // per-skill "original" link the page used to render was + // github.com/openclaw/skills/tree/main/skills//, a repo that + // does not exist (HTTP 404). A previous version of this scraper captured + // that dead link and polluted 5028 Skill.githubUrl rows. The canonical + // per-skill link is the ClawHub page, already stored as clawHubUrl / + // clawSkillsUrl, so githubUrl is left untouched (NULL for these skills). // Extract install command let installCmd: string | null = null; @@ -106,7 +109,7 @@ async function scrapeSkillPage(page: puppeteer.Page, slug: string): Promise = {}; if (descParts.length > 0) updateData.description = descParts.join(""); if (readmeParts.length > 0) updateData.readme = readmeParts.join("\n\n"); - if (detail.githubUrl) updateData.githubUrl = detail.githubUrl; if (detail.installCmd) updateData.installCmd = detail.installCmd; if (Object.keys(updateData).length > 0) { diff --git a/src/app/api/cron/check-github/route.ts b/src/app/api/cron/check-github/route.ts new file mode 100644 index 00000000..dd431366 --- /dev/null +++ b/src/app/api/cron/check-github/route.ts @@ -0,0 +1,65 @@ +import { NextRequest, NextResponse } from "next/server"; +import { prisma } from "@/lib/prisma"; +import { isAuthorizedCron } from "@/lib/cron-auth"; + +// Cron-only endpoint: probe skill GitHub URLs for dead links (404) and record +// ghStatus + ghCheckedAt. Batched (take 200, oldest-checked first) to stay under +// the request budget. Wire to Cloud Scheduler weekly with +// `Authorization: Bearer `. +export const dynamic = "force-dynamic"; +export const maxDuration = 300; + +async function handle(req: NextRequest) { + if (!isAuthorizedCron(req)) { + return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + } + + const oneWeekAgo = new Date(Date.now() - 7 * 24 * 60 * 60 * 1000); + const skills = await prisma.skill.findMany({ + where: { + githubUrl: { not: null }, + OR: [{ ghCheckedAt: null }, { ghCheckedAt: { lt: oneWeekAgo } }], + }, + select: { id: true, githubUrl: true }, + orderBy: { ghCheckedAt: { sort: "asc", nulls: "first" } }, + take: 200, + }); + + let ok = 0; + let notFound = 0; + let errored = 0; + + for (const skill of skills) { + if (!skill.githubUrl) continue; + try { + const res = await fetch(skill.githubUrl, { method: "HEAD", redirect: "follow" }); + const status = res.status === 404 ? "404" : res.ok ? "ok" : "error"; + await prisma.skill.update({ + where: { id: skill.id }, + data: { ghStatus: status, ghCheckedAt: new Date() }, + }); + if (status === "ok") ok++; + else if (status === "404") notFound++; + else errored++; + } catch { + await prisma.skill.update({ + where: { id: skill.id }, + data: { ghStatus: "error", ghCheckedAt: new Date() }, + }); + errored++; + } + // Be polite to GitHub between probes. + await new Promise((r) => setTimeout(r, 100)); + } + + return NextResponse.json({ + checked: skills.length, + ok, + notFound, + errored, + ranAt: new Date().toISOString(), + }); +} + +export const GET = handle; +export const POST = handle; diff --git a/src/app/api/cron/health/route.ts b/src/app/api/cron/health/route.ts index 5610c5a0..7f755a70 100644 --- a/src/app/api/cron/health/route.ts +++ b/src/app/api/cron/health/route.ts @@ -6,13 +6,16 @@ import { isAuthorizedCron } from "@/lib/cron-auth"; // healthStatus / healthCheckedAt. Wire to Cloud Scheduler, which sends // `Authorization: Bearer ` (the tako-cron-secret value). export const dynamic = "force-dynamic"; +// Bounded probes (8-wide, 8s each) — give the run headroom as HOSTED count grows. +export const maxDuration = 120; async function handle(req: NextRequest) { if (!isAuthorizedCron(req)) { return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); } + const startedAt = Date.now(); const summary = await runHealthChecks(); - return NextResponse.json(summary); + return NextResponse.json({ ...summary, durationMs: Date.now() - startedAt }); } export const GET = handle; diff --git a/src/app/api/cron/import-hosted/route.ts b/src/app/api/cron/import-hosted/route.ts index 3d5c650a..7d083041 100644 --- a/src/app/api/cron/import-hosted/route.ts +++ b/src/app/api/cron/import-hosted/route.ts @@ -21,13 +21,14 @@ async function handle(req: NextRequest) { if (!isAuthorizedCron(req)) { return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); } + const startedAt = Date.now(); const sp = new URL(req.url).searchParams; // 0 = no cap. Bounded to protect the request from an over-large registry. const max = intParam(sp.get("max"), 0, 0, 1000); try { // Default status PENDING — third-party agents await admin review. const result = await importHostedAgents({ max, status: "PENDING" }, prisma); - return NextResponse.json({ ...result, max, ranAt: new Date().toISOString() }); + return NextResponse.json({ ...result, max, durationMs: Date.now() - startedAt, ranAt: new Date().toISOString() }); } catch (e) { return NextResponse.json({ error: e instanceof Error ? e.message : "import failed" }, { status: 500 }); } diff --git a/src/app/api/cron/scrape-agents/route.ts b/src/app/api/cron/scrape-agents/route.ts index a25f5a43..39de644d 100644 --- a/src/app/api/cron/scrape-agents/route.ts +++ b/src/app/api/cron/scrape-agents/route.ts @@ -8,6 +8,10 @@ import { scrapeGithubAgents } from "@/lib/scrape-agents"; // catalog fresh and adds newly-popular repos. Wire to Cloud Scheduler with // `Authorization: Bearer `. Tunable via ?pages=&minStars=&max=. export const dynamic = "force-dynamic"; +// NOTE: on Cloud Run (output: "standalone") the effective ceiling is the +// service's request timeout, not this Vercel-style export — keep the Cloud Run +// timeout ≥ this. `durationMs` in the response lets the scheduler log spot runs +// that approach the limit (a truncated run leaves partial, non-transactional writes). export const maxDuration = 300; function intParam(v: string | null, def: number, min: number, max: number): number { @@ -20,6 +24,7 @@ async function handle(req: NextRequest) { if (!isAuthorizedCron(req)) { return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); } + const startedAt = Date.now(); const sp = new URL(req.url).searchParams; // Defaults match the live catalog (PAGES=1 keeps it fast enough for a sync // request; raise pages for a deeper backfill). Bounded to protect the request. @@ -31,7 +36,14 @@ async function handle(req: NextRequest) { { token: process.env.GITHUB_TOKEN, pages, minStars, maxAgents, reqDelayMs: 900 }, prisma ); - return NextResponse.json({ ...result, pages, minStars, maxAgents, ranAt: new Date().toISOString() }); + return NextResponse.json({ + ...result, + pages, + minStars, + maxAgents, + durationMs: Date.now() - startedAt, + ranAt: new Date().toISOString(), + }); } catch (e) { return NextResponse.json({ error: e instanceof Error ? e.message : "scrape failed" }, { status: 500 }); } diff --git a/src/app/api/cron/weekly-digest/route.ts b/src/app/api/cron/weekly-digest/route.ts new file mode 100644 index 00000000..f38fa715 --- /dev/null +++ b/src/app/api/cron/weekly-digest/route.ts @@ -0,0 +1,62 @@ +import { NextRequest, NextResponse } from "next/server"; +import { prisma } from "@/lib/prisma"; +import { isAuthorizedCron } from "@/lib/cron-auth"; +import { sendWeeklyDigest } from "@/lib/email"; + +// Cron-only endpoint: email the weekly digest (new-skill count, top skills, +// total) to every verified subscriber. Sends are throttled to ~2/sec to stay +// within Resend limits. No verified subscribers => no-op. Wire to Cloud +// Scheduler weekly with `Authorization: Bearer `. +export const dynamic = "force-dynamic"; +export const maxDuration = 300; + +async function handle(req: NextRequest) { + if (!isAuthorizedCron(req)) { + return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + } + + try { + const oneWeekAgo = new Date(Date.now() - 7 * 24 * 60 * 60 * 1000); + + const [newSkillsCount, topSkills, totalSkills, subscribers] = await Promise.all([ + prisma.skill.count({ where: { createdAt: { gte: oneWeekAgo } } }), + prisma.skill.findMany({ + orderBy: { downloads: "desc" }, + take: 10, + select: { name: true, slug: true, downloads: true }, + }), + prisma.skill.count(), + prisma.subscriber.findMany({ + where: { verified: true }, + select: { email: true }, + }), + ]); + + let sent = 0; + let failed = 0; + for (const sub of subscribers) { + try { + await sendWeeklyDigest(sub.email, { newSkillsCount, topSkills, totalSkills }); + sent++; + } catch { + failed++; + } + // Rate limit: ~2/sec to stay within Resend limits. + await new Promise((r) => setTimeout(r, 500)); + } + + return NextResponse.json({ + sent, + failed, + total: subscribers.length, + newSkillsCount, + ranAt: new Date().toISOString(), + }); + } catch (error) { + console.error("Weekly digest error:", error); + return NextResponse.json({ error: "Internal error" }, { status: 500 }); + } +} + +export const GET = handle; +export const POST = handle; diff --git a/src/lib/agentcard.ts b/src/lib/agentcard.ts index da109906..5c1e96e8 100644 --- a/src/lib/agentcard.ts +++ b/src/lib/agentcard.ts @@ -1,4 +1,5 @@ import { z } from "zod"; +import { lookup } from "node:dns/promises"; // ── A2A AgentCard parsing ──────────────────────────────────────────────── // Fetches and validates an A2A AgentCard (the canonical machine-readable @@ -13,6 +14,7 @@ const WELL_KNOWN_LEGACY = "/.well-known/agent.json"; // pre-v1 path, still in th const MAX_BYTES = 256 * 1024; const TIMEOUT_MS = 8000; +const MAX_REDIRECTS = 3; const AgentSkillSchema = z.object({ id: z.string().min(1).max(200), @@ -57,8 +59,12 @@ export type ParsedAgentCard = { }[]; }; -// Best-effort SSRF guard. Full protection needs DNS-resolution checks -// (a hardening TODO, see docs/agent-marketplace/03-technical-architecture.md §11). +// SSRF guard — first line: reject non-http(s) and literal private/loopback +// hosts from the URL string. Paired with assertPublicHost() (DNS/IP resolution) +// and manual per-hop redirect revalidation in fetchOne(), this closes the +// redirect-to-metadata and DNS-rebinding vectors. Residual: a connect-level +// TOCTOU (rebind between resolve and connect) still needs socket-level pinning — +// see docs/agent-marketplace/03-technical-architecture.md §11. function assertSafeUrl(raw: string): URL { let u: URL; try { @@ -85,6 +91,55 @@ function assertSafeUrl(raw: string): URL { return u; } +// True if an IPv4 literal is in a private / loopback / link-local / CGNAT range. +// Unparseable input is treated as unsafe (fail closed). +function ipv4IsPrivate(ip: string): boolean { + const p = ip.split(".").map((n) => Number(n)); + if (p.length !== 4 || p.some((n) => !Number.isInteger(n) || n < 0 || n > 255)) return true; + const [a, b] = p; + if (a === 0 || a === 127) return true; // 0.0.0.0/8, loopback + if (a === 10) return true; // 10/8 + if (a === 172 && b >= 16 && b <= 31) return true; // 172.16/12 + if (a === 192 && b === 168) return true; // 192.168/16 + if (a === 169 && b === 254) return true; // 169.254/16 link-local (cloud metadata) + if (a === 100 && b >= 64 && b <= 127) return true; // 100.64/10 CGNAT + if (a === 198 && (b === 18 || b === 19)) return true; // 198.18/15 benchmarking, non-routable + return false; +} + +// True if a resolved address (v4 or v6) is one we must not fetch from. +function ipIsPrivate(address: string, family: number): boolean { + if (family === 4) return ipv4IsPrivate(address); + const ip = address.toLowerCase(); + // IPv4-mapped / -embedded (e.g. ::ffff:169.254.169.254) → check the v4 part. + const embedded = ip.match(/((?:\d{1,3}\.){3}\d{1,3})$/); + if (embedded) return ipv4IsPrivate(embedded[1]); + if (ip === "::1" || ip === "::") return true; // loopback / unspecified + if (ip.startsWith("fc") || ip.startsWith("fd")) return true; // fc00::/7 unique-local + if (/^fe[89ab]/.test(ip)) return true; // fe80::/10 link-local + return false; +} + +// Resolve a hostname and refuse if ANY resolved address is private/loopback/ +// link-local. Closes the DNS-rebinding gap that the string-only host check in +// assertSafeUrl can't catch (e.g. a public name pointing at 169.254.169.254). +// Prod-only, matching assertSafeUrl's guard, so localhost cards still work in dev. +async function assertPublicHost(hostname: string): Promise { + if (process.env.NODE_ENV !== "production") return; + let addrs: { address: string; family: number }[]; + try { + addrs = await lookup(hostname, { all: true }); + } catch { + throw new AgentCardError(`Could not resolve ${hostname}`); + } + if (!addrs.length) throw new AgentCardError(`Host ${hostname} did not resolve`); + for (const a of addrs) { + if (ipIsPrivate(a.address, a.family)) { + throw new AgentCardError("Refusing to fetch a private/loopback address"); + } + } +} + function originOf(u: string): string { try { const x = new URL(u); @@ -123,33 +178,55 @@ function normalize(card: z.infer, cardUrl: string): Pars } async function fetchOne(cardUrl: string): Promise { - assertSafeUrl(cardUrl); + // Store the URL we were asked to fetch as the card's canonical URL, even if a + // redirect serves the bytes from elsewhere (keeps stored cardUrl stable). + const requested = assertSafeUrl(cardUrl).toString(); + let current = requested; + // One timer bounds the whole redirect chain, not each hop. const ctrl = new AbortController(); const timer = setTimeout(() => ctrl.abort(), TIMEOUT_MS); - let res: Response; try { - res = await fetch(cardUrl, { - signal: ctrl.signal, - headers: { Accept: "application/json" }, - redirect: "follow", - }); - } catch { - throw new AgentCardError(`Could not reach ${cardUrl}`); + for (let hop = 0; ; hop++) { + // Revalidate every hop: string checks + DNS/IP resolution. A "follow" + // fetch would let a public URL redirect into a private address unchecked. + const u = assertSafeUrl(current); + await assertPublicHost(u.hostname); + + let res: Response; + try { + res = await fetch(current, { + signal: ctrl.signal, + headers: { Accept: "application/json" }, + redirect: "manual", + }); + } catch { + throw new AgentCardError(`Could not reach ${current}`); + } + + if (res.status >= 300 && res.status < 400) { + const loc = res.headers.get("location"); + if (!loc) throw new AgentCardError("Redirect without a Location header"); + if (hop >= MAX_REDIRECTS) throw new AgentCardError("Too many redirects"); + current = new URL(loc, current).toString(); // resolve relative redirects + continue; + } + + if (!res.ok) throw new AgentCardError(`AgentCard fetch returned HTTP ${res.status}`); + const text = await res.text(); + if (text.length > MAX_BYTES) throw new AgentCardError("AgentCard response too large"); + let json: unknown; + try { + json = JSON.parse(text); + } catch { + throw new AgentCardError("AgentCard is not valid JSON"); + } + const parsed = AgentCardSchema.safeParse(json); + if (!parsed.success) throw new AgentCardError("Not a valid A2A AgentCard"); + return normalize(parsed.data, requested); + } } finally { clearTimeout(timer); } - if (!res.ok) throw new AgentCardError(`AgentCard fetch returned HTTP ${res.status}`); - const text = await res.text(); - if (text.length > MAX_BYTES) throw new AgentCardError("AgentCard response too large"); - let json: unknown; - try { - json = JSON.parse(text); - } catch { - throw new AgentCardError("AgentCard is not valid JSON"); - } - const parsed = AgentCardSchema.safeParse(json); - if (!parsed.success) throw new AgentCardError("Not a valid A2A AgentCard"); - return normalize(parsed.data, cardUrl); } /** diff --git a/src/lib/email.ts b/src/lib/email.ts new file mode 100644 index 00000000..e253b473 --- /dev/null +++ b/src/lib/email.ts @@ -0,0 +1,166 @@ +import { Resend } from "resend"; + +let _resend: Resend | null = null; +function getResend(): Resend | null { + if (!process.env.RESEND_API_KEY) { + console.warn("RESEND_API_KEY is not set — skipping email send"); + return null; + } + if (!_resend) { + _resend = new Resend(process.env.RESEND_API_KEY); + } + return _resend; +} + +const FROM_EMAIL = "TakoAPI "; +const REPLY_TO = "wangharp@gmail.com"; + +export async function sendWelcomeEmail(to: string, name?: string) { + const resend = getResend(); + if (!resend) return null; + const displayName = name || to.split("@")[0]; + + return resend.emails.send({ + from: FROM_EMAIL, + replyTo: REPLY_TO, + to, + subject: `Welcome to TakoAPI, ${displayName}!`, + html: ` + + + + + + + + +
+
🐙
+

Welcome to TakoAPI

+

All in One OpenClaw Skills Marketplace

+
+

Hi ${displayName},

+

Thanks for joining TakoAPI! You now have access to 5,000+ OpenClaw skills across 30+ categories.

+ +

Get Started

+ + +
+

1. Browse trending skills

+ takoapi.com/trending → +
+ + +
+

2. Install the official TakoAPI skill

+ clawhub install takoapi +
+ + +
+

3. Submit your own skill

+ takoapi.com/submit → +
+ + + +

Have questions? Just reply to this email. We'd love to hear from you.

+
+

TakoAPI · takoapi.com · GitHub

+
+ +`, + }); +} + +export async function sendWeeklyDigest( + to: string, + data: { + newSkillsCount: number; + topSkills: { name: string; slug: string; downloads: number }[]; + totalSkills: number; + } +) { + const skillRows = data.topSkills + .map( + (s, i) => + `${i + 1}. ${s.name}${s.downloads.toLocaleString()} downloads` + ) + .join(""); + + const resend = getResend(); + if (!resend) return null; + + return resend.emails.send({ + from: FROM_EMAIL, + replyTo: REPLY_TO, + to, + subject: `TakoAPI Weekly: ${data.newSkillsCount} new skills this week`, + html: ` + + + + + + + + +
+

🐙 TakoAPI Weekly Digest

+
+
+
+
${data.newSkillsCount}
+
New This Week
+
+
+
${data.totalSkills.toLocaleString()}
+
Total Skills
+
+
+

Top Skills This Week

+ ${skillRows}
+ +
+

TakoAPI · takoapi.com · Unsubscribe

+
+ +`, + }); +} + +export async function sendKolOutreach( + to: string, + data: { name: string; skillName?: string; projectName?: string } +) { + const resend = getResend(); + if (!resend) return null; + + return resend.emails.send({ + from: FROM_EMAIL, + replyTo: REPLY_TO, + to, + subject: `Your ${data.skillName || data.projectName || "project"} on TakoAPI`, + html: ` + + + + +
+

Hi ${data.name},

+

Love your ${data.skillName || data.projectName || "work"}! We featured it on TakoAPI — the OpenClaw skills marketplace with 5,000+ skills.

+

Would you like a "Featured Developer" badge on your skill page? We can also provide an "Install via TakoAPI" badge for your README.

+

No commitment needed — just thought you'd appreciate the exposure to our growing community.

+

Best,
TakoAPI Team

+
+

🐙 takoapi.com · GitHub

+
+
+ +`, + }); +} diff --git a/src/lib/health.ts b/src/lib/health.ts index b9a7dcc3..7881bf7a 100644 --- a/src/lib/health.ts +++ b/src/lib/health.ts @@ -30,7 +30,15 @@ export async function probeAgentHealth(agent: { cardUrl: string | null; endpoint } } -export type HealthSummary = { checked: number; ok: number; degraded: number; down: number; checkedAt: string }; +export type HealthSummary = { + checked: number; + ok: number; + degraded: number; + down: number; + /** Persist failures — surfaced instead of being silently swallowed. */ + writeErrors: number; + checkedAt: string; +}; /** Probe all APPROVED HOSTED agents (bounded concurrency) and persist their health. */ export async function runHealthChecks(): Promise { @@ -40,7 +48,7 @@ export async function runHealthChecks(): Promise { }); const now = new Date(); - const summary: HealthSummary = { checked: agents.length, ok: 0, degraded: 0, down: 0, checkedAt: now.toISOString() }; + const summary: HealthSummary = { checked: agents.length, ok: 0, degraded: 0, down: 0, writeErrors: 0, checkedAt: now.toISOString() }; for (let i = 0; i < agents.length; i += PROBE_CONCURRENCY) { const batch = agents.slice(i, i + PROBE_CONCURRENCY); @@ -48,9 +56,13 @@ export async function runHealthChecks(): Promise { await Promise.all( results.map((r) => { summary[r.health]++; + // Don't let one failed write abort the batch, but do count it so a + // systemic DB problem shows up in the summary rather than vanishing. return prisma.agent .update({ where: { id: r.id }, data: { healthStatus: r.health, healthCheckedAt: now } }) - .catch(() => {}); + .catch(() => { + summary.writeErrors++; + }); }) ); } diff --git a/src/lib/import-hosted.ts b/src/lib/import-hosted.ts index d0ed5d03..f7acec3c 100644 --- a/src/lib/import-hosted.ts +++ b/src/lib/import-hosted.ts @@ -20,6 +20,18 @@ function slugify(s: string): string { return s.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/(^-|-$)/g, "").slice(0, 80); } +// Origin (scheme://host) of a URL, for comparing an existing card's host with an +// incoming one. Returns null when the input can't be parsed. +function originOf(u: string | null | undefined): string | null { + if (!u) return null; + try { + const x = new URL(u); + return `${x.protocol}//${x.host}`; + } catch { + return null; + } +} + const dedupe = (a: string[]) => Array.from(new Set(a)); // Resolve the list of AgentCard URLs from a local file, a registry index URL, @@ -94,6 +106,35 @@ export async function importHostedAgents( failed++; continue; } + + // Ownership + same-origin guards. The registry importer upserts by + // slugify(card.name); without these, a card whose name collides with an + // existing entry would silently overwrite that entry's endpoint (and keep + // its APPROVED status). Since the default registry is a community-editable + // list, that's a takeover vector. Rules: + // 1. Only refresh rows this importer owns (publisherId === pub.id) — never + // touch a user's submitted agent. + // 2. A hosted card must not overwrite a non-HOSTED (PROJECT) entry. + // 3. If the card's host differs from the stored one, treat it as a + // possible takeover / migration: refresh fields but drop to PENDING so + // an admin re-approves the new endpoint before it serves traffic. + const existing = await db.agent.findUnique({ + where: { slug }, + select: { kind: true, publisherId: true, cardUrl: true }, + }); + if (existing && existing.publisherId !== pub.id) { + log(`↷ ${slug} — owned by another publisher; skipping`); + failed++; + continue; + } + if (existing && existing.kind !== "HOSTED") { + log(`↷ ${slug} — existing ${existing.kind} entry; a hosted card must not overwrite it; skipping`); + failed++; + continue; + } + const existingOrigin = originOf(existing?.cardUrl); + const crossOrigin = !!existingOrigin && existingOrigin !== originOf(card.cardUrl); + const scenarios = classifyScenarios( [card.name, card.description, card.skills.map((s) => s.name).join(" ")].join(" ") ); @@ -112,6 +153,8 @@ export async function importHostedAgents( securitySchemes: security, cardFetchedAt: new Date(), scenarios, + // Endpoint moved to a new host → force re-review before it's public. + ...(crossOrigin ? { status: "PENDING" as AgentStatus } : {}), }, create: { slug, @@ -151,7 +194,10 @@ export async function importHostedAgents( } } - log(`✓ ${slug} — ${card.skills.length} skill(s), scenarios: ${scenarios.join("/") || "none"}`); + log( + `✓ ${slug} — ${card.skills.length} skill(s), scenarios: ${scenarios.join("/") || "none"}` + + (crossOrigin ? " [host changed → status reset to PENDING for re-review]" : "") + ); ok++; } catch (e) { const msg = e instanceof AgentCardError ? e.message : (e as Error).message; diff --git a/src/lib/scrape-agents.ts b/src/lib/scrape-agents.ts index 87e32360..bf5cf062 100644 --- a/src/lib/scrape-agents.ts +++ b/src/lib/scrape-agents.ts @@ -1,4 +1,4 @@ -import type { PrismaClient } from "@prisma/client"; +import type { PrismaClient, AgentStatus } from "@prisma/client"; import { classifyScenarios } from "./scenarios"; // Import top open-source agent PROJECTS from GitHub (by stars) into the registry @@ -51,6 +51,11 @@ type Repo = { // the star-farmed band polluting the newest-by-recency slice of the catalog. function looksStarFarmed(r: Repo): boolean { const stars = r.stargazers_count; + // The 150 floor is deliberately permissive to avoid false positives on small + // real projects. Note: the observed star-farm band sits higher (~340★), so + // some farmed repos in 150–400★ can slip past the fork-ratio test. That's an + // accepted trade-off — the durable backstop is admin REJECT/DISABLE, which a + // scheduled re-scrape now honors (see scenarios loop below), not this heuristic. if (stars < 150) return false; // small repos: don't judge on fork ratio return (r.forks_count ?? 0) / stars < 0.01; // < 1 fork per 100 stars → suspicious } @@ -85,6 +90,14 @@ async function search(q: string, page: number, token: string | undefined, log: ( await sleep(Math.min(waitMs, 120_000)); continue; } + // 401 = the token is missing/invalid/expired. Fail loudly rather than + // returning []: a swallowed 401 makes the run look successful (HTTP 200, + // 0 results) while the catalog quietly stops growing — exactly how a PAT + // expiry went unnoticed for 19 days. Throwing surfaces a 500 that the + // scheduler records as a failed job. + if (res.status === 401) { + throw new Error("GitHub auth failed (HTTP 401) — GITHUB_TOKEN is missing, invalid, or expired"); + } if (!res.ok) { log(`search "${q}" p${page} -> HTTP ${res.status}`); return []; @@ -144,21 +157,45 @@ export async function scrapeGithubAgents(opts: ScrapeOpts, db: PrismaClient): Pr } const all = [...byName.values()].sort((a, b) => b.stargazers_count - a.stargazers_count); + + // Backstop for the silent-no-op failure mode: a healthy run always finds repos + // across 20 topic queries, so zero means every search failed (rate-limit + // exhaustion, an API change, a revoked token). Report it as an error instead of + // a "successful" run that imported nothing. + if (all.length === 0) { + throw new Error("GitHub search returned no repositories across all queries — check GITHUB_TOKEN and rate limits"); + } const starred = minStars ? all.filter((r) => r.stargazers_count >= minStars) : all; const eligible = starred.filter((r) => !looksStarFarmed(r)); const skipped = starred.length - eligible.length; const repos = eligible.slice(0, max); - log(`found ${all.length} unique repos${minStars ? `, ${starred.length} with >=${minStars} stars` : ""}${skipped ? `, skipped ${skipped} likely star-farmed` : ""}; importing top ${repos.length} by stars`); + + // Respect human governance: never resurrect a repo an admin has REJECTED or + // DISABLED. Load those slugs up front (one query) so the loop can skip them — + // otherwise the update path below would flip them back to APPROVED, and a prior + // deletion would be re-created as APPROVED, silently undoing the moderation. + const repoSlugs = repos.map((r) => slugify(`${r.owner.login}-${r.name}`)); + const governed = await db.agent.findMany({ + where: { slug: { in: repoSlugs }, status: { in: ["REJECTED", "DISABLED"] as AgentStatus[] } }, + select: { slug: true }, + }); + const tombstoned = new Set(governed.map((g) => g.slug)); + + log(`found ${all.length} unique repos${minStars ? `, ${starred.length} with >=${minStars} stars` : ""}${skipped ? `, skipped ${skipped} likely star-farmed` : ""}${tombstoned.size ? `, skipping ${tombstoned.size} admin-rejected/disabled` : ""}; importing top ${repos.length} by stars`); let imported = 0; for (const r of repos) { const slug = slugify(`${r.owner.login}-${r.name}`); + if (tombstoned.has(slug)) continue; // admin REJECTED/DISABLED — leave it alone const description = (r.description || r.name).slice(0, 500); // Classify by name + description + GitHub topics into use-case scenarios. const scenarios = classifyScenarios([r.name, r.description ?? "", (r.topics ?? []).join(" ")].join(" ")); + // NOTE: the update branch deliberately does NOT set `status` — a scheduled + // refresh must not override an admin's moderation decision. Only newly + // discovered repos (the create branch) land as APPROVED. await db.agent.upsert({ where: { slug }, - update: { stars: r.stargazers_count, description, status: "APPROVED", scenarios }, + update: { stars: r.stargazers_count, description, scenarios }, create: { slug, name: r.name,