Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -46,3 +46,6 @@ packages/cli/scene/**/outputs/

# Environment variables (sensitive data)
.env

# Local scratch / plan drafts (never commit)
.scratch/
9 changes: 6 additions & 3 deletions packages/cli/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,8 @@
},
"files": [
"dist",
"README.zh.md"
"README.zh.md",
"postinstall.js"
],
"type": "module",
"exports": {
Expand All @@ -45,12 +46,14 @@
"build": "vp pack",
"dev": "tsx src/main.ts",
"test": "vp test",
"check": "vp check"
"check": "vp check",
"postinstall": "node postinstall.js"
},
"dependencies": {
"bailian-cli-commands": "workspace:*",
"bailian-cli-core": "workspace:*",
"bailian-cli-runtime": "workspace:*"
"bailian-cli-runtime": "workspace:*",
"tar-stream": "catalog:"
},
"devDependencies": {
"@clack/prompts": "^0.7.0",
Expand Down
213 changes: 213 additions & 0 deletions packages/cli/postinstall.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,213 @@
/**
* postinstall.js — Wiki data sync (layer 1: triggered by npm install)
*
* Runs automatically after npm/pnpm installs bailian-cli: unconditionally downloads the full Wiki data
* package and overwrites the local directory, ensuring data is in place the first time the user runs
* `bl advisor recommend`.
*
* Flow (unified skill publishing protocol: skills/index.json + one content-addressed object per skill):
* 1. Download skills/index.json from public-read OSS, get the bailian-docs-llm-wiki entry
* 2. Download skills/bailian-docs-llm-wiki/<entry.object> (sha256-<hex>.tar.br, brotli q6, ~2.3MB);
* legacy fallback to skill.tar.br when the entry has no valid object field
* 3. Node built-in brotli decompress + tar-stream extract (per-entry path safety check) to same-volume temp dir
* 4. renameSync atomic swap into ~/.bailian/skills/bailian-docs-llm-wiki/
* 5. Write ~/.bailian/wiki-sync-state.json
* 6. Write ~/.bailian/skills/skill-lock.json record (same ledger as bl skill)
*
* Design constraints:
* - Unconditional overwrite: every install fully replaces, no version comparison
* - Silent failure: any step failure → console.warn → process.exit(0), never blocks install
* - Standalone implementation: does not import bailian-cli-core, avoiding ESM path issues after bundling
* - Depends on Node built-in modules + tar-stream (consistent with sync.ts / publisher skills-publish.mjs)
*/
import {
createWriteStream,
existsSync,
mkdirSync,
readFileSync,
renameSync,
rmSync,
writeFileSync,
} from "node:fs";
import { homedir } from "node:os";
import { dirname, join } from "node:path";
import { Readable } from "node:stream";
import { pipeline } from "node:stream/promises";
import { createBrotliDecompress } from "node:zlib";
import tar from "tar-stream";

const REGISTRY_BASE_URL = "https://bailian-wiki.oss-cn-hangzhou.aliyuncs.com/skills";
const WIKI_SKILL_NAME = "bailian-docs-llm-wiki";
const CONFIG_DIR_NAME = ".bailian";
const SKILL_DIR_NAME = "skills/bailian-docs-llm-wiki";
const STATE_FILE_NAME = "wiki-sync-state.json";
const INDEX_KEY = "index.json";
/** Legacy fixed asset key (entries without a valid content-addressed object field) */
const LEGACY_ASSET_NAME = "skill.tar.br";
/** Same strict shape check as core registry.ts: only a valid object name may enter the URL */
const OBJECT_FILE_RE = /^sha256-[0-9a-f]{64}\.tar\.br$/;

const INDEX_TIMEOUT_MS = 3000;
const DOWNLOAD_TIMEOUT_MS = 30000;

function getConfigDir() {
if (process.env.BAILIAN_CONFIG_DIR) return process.env.BAILIAN_CONFIG_DIR;
return join(homedir(), CONFIG_DIR_NAME);
}

function getCatalogDir() {
return join(getConfigDir(), SKILL_DIR_NAME);
}

function getStatePath() {
return join(getConfigDir(), STATE_FILE_NAME);
}

function getSkillLockPath() {
return join(getConfigDir(), "skills", "skill-lock.json");
}

/**
* Record this sync in skill-lock.json (same ledger as bl skill; list shows installed).
* Semantics aligned with upsertSkillLockEntry in core/src/skills/lock.ts: shallow-merge with the existing
* entry, preserving fields like links written by bl skill add; rebuild as empty table if lock is corrupted/unrecognized.
* best-effort: failure does not affect data sync results.
*/
function upsertSkillLock(name, entry) {
try {
let lock = { version: 1, skills: {} };
try {
const parsed = JSON.parse(readFileSync(getSkillLockPath(), "utf-8"));
if (parsed?.version === 1 && parsed.skills && typeof parsed.skills === "object") {
lock = parsed;
}
} catch {
/* absent/corrupted → empty table */
}
lock.skills[name] = { ...lock.skills[name], ...entry };
mkdirSync(dirname(getSkillLockPath()), { recursive: true });
writeFileSync(getSkillLockPath(), JSON.stringify(lock, null, 2) + "\n");
} catch {
/* Bookkeeping failure does not block install; advisor-side sync will backfill */
}
}

async function fetchJson(url, timeoutMs) {
const res = await fetch(url, { signal: AbortSignal.timeout(timeoutMs) });
if (!res.ok) throw new Error(`HTTP ${res.status}`);
return res.json();
}

async function downloadBuffer(url) {
const res = await fetch(url, { signal: AbortSignal.timeout(DOWNLOAD_TIMEOUT_MS) });
if (!res.ok) throw new Error(`HTTP ${res.status}`);
return Buffer.from(await res.arrayBuffer());
}

/** tar 条目路径必须是相对路径且不含 ..,防止 tar-slip 逃逸解包目录 */
function isSafeEntryName(name) {
if (name.startsWith("/") || /^[a-zA-Z]:[\\/]/.test(name)) return false;
return !name.split("/").includes("..");
}

/** Brotli decompress + tar-stream extract into destDir (symmetric with publisher tar.pack()). */
async function extractTarBr(tarBrBuffer, destDir) {
const extract = tar.extract();

extract.on("entry", (header, stream, next) => {
if (!isSafeEntryName(header.name)) {
// Same semantics as core skills/extract.ts: destroy so the pipeline rejects with this
// error; silence the entry stream to avoid its companion error becoming unhandled
stream.on("error", () => {});
stream.resume();
extract.destroy(new Error(`unsafe tar entry: ${header.name}`));
return;
}
const filePath = join(destDir, header.name);
if (header.type === "directory") {
mkdirSync(filePath, { recursive: true });
stream.resume();
stream.on("end", next);
return;
}
mkdirSync(dirname(filePath), { recursive: true });
const ws = createWriteStream(filePath);
stream.pipe(ws);
ws.on("finish", next);
ws.on("error", next);
});

await pipeline(Readable.from(tarBrBuffer), createBrotliDecompress(), extract);
}

/** Atomic swap: tmpDir (same volume) → catalogDir. */
function atomicSwap(tmpDir, catalogDir) {
mkdirSync(dirname(catalogDir), { recursive: true });
const backup = `${catalogDir}.old-${Date.now()}`;
if (existsSync(catalogDir)) renameSync(catalogDir, backup);
try {
renameSync(tmpDir, catalogDir);
} catch (err) {
if (existsSync(backup) && !existsSync(catalogDir)) renameSync(backup, catalogDir);
throw err;
}
if (existsSync(backup)) rmSync(backup, { recursive: true, force: true });
}

async function main() {
// 1. Download skills/index.json and get the wiki entry
const index = await fetchJson(`${REGISTRY_BASE_URL}/${INDEX_KEY}`, INDEX_TIMEOUT_MS);
const entry = index?.skills?.[WIKI_SKILL_NAME];
if (!entry?.contentHash)
throw new Error("no bailian-docs-llm-wiki entry (or contentHash) in index.json");

// 2. Download the skill archive: content-addressed object first, legacy fixed key as fallback
const assetName =
entry.object && OBJECT_FILE_RE.test(entry.object) ? entry.object : LEGACY_ASSET_NAME;
const tarBuf = await downloadBuffer(`${REGISTRY_BASE_URL}/${WIKI_SKILL_NAME}/${assetName}`);

// 3. Extract to same-volume temp dir + atomic swap
const catalogDir = getCatalogDir();
const tmpDir = `${catalogDir}.tmp-${process.pid}-${Date.now()}`;
try {
mkdirSync(tmpDir, { recursive: true });
await extractTarBr(tarBuf, tmpDir);
atomicSwap(tmpDir, catalogDir);
} catch (err) {
if (existsSync(tmpDir)) rmSync(tmpDir, { recursive: true, force: true });
throw err;
}

// 4. Write state
try {
writeFileSync(
getStatePath(),
JSON.stringify({ lastChecked: Date.now(), contentHash: entry.contentHash }),
);
} catch {
/* state write failure has no impact: first recommend will re-check */
}

// 5. skill-lock.json record: wiki shares the same ledger as bl skill
upsertSkillLock(WIKI_SKILL_NAME, {
contentHash: entry.contentHash,
...(entry.publishedAt ? { publishedAt: entry.publishedAt } : {}),
installedAt: new Date().toISOString(),
sourceType: "oss",
...(entry.description ? { description: entry.description } : {}),
});

process.stdout.write(`bailian-cli: wiki data ready (${entry.publishedAt ?? "latest"})\n`);
}

main().catch((err) => {
// Unconditional pass-through: install-time network/permission issues should not block npm install;
// sync.ts will fall back to syncing on the first `bl advisor recommend`.
const msg = err instanceof Error ? err.message : String(err);
process.stderr.write(
`bailian-cli: wiki data pre-download skipped (${msg}); will sync automatically on first use.\n`,
);
// Force a success exit code so a download failure never fails `npm install`.
// eslint-disable-next-line unicorn/no-process-exit
process.exit(0);
});
8 changes: 8 additions & 0 deletions packages/cli/src/commands.ts
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,10 @@ import {
pluginLink,
pluginList,
pluginRemove,
skillAdd,
skillUpdate,
skillRemove,
skillList,
managedAgentInit,
managedAgentValidate,
managedAgentPlan,
Expand Down Expand Up @@ -203,6 +207,10 @@ export const commands: Record<string, AnyCommand> = {
"plugin link": pluginLink,
"plugin list": pluginList,
"plugin remove": pluginRemove,
"skill add": skillAdd,
"skill update": skillUpdate,
"skill remove": skillRemove,
"skill list": skillList,
"managed-agent init": managedAgentInit,
"managed-agent validate": managedAgentValidate,
"managed-agent plan": managedAgentPlan,
Expand Down
7 changes: 7 additions & 0 deletions packages/commands/src/commands/advisor/recommend.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import {
type GetModelsOptions,
getModels,
type IntentProfile,
maybeSyncWikiData,
type PipelineStep,
type RecommendedModel,
type RecommendResult,
Expand Down Expand Up @@ -248,6 +249,12 @@ export default defineCommand({
const { settings, flags } = ctx;
const userInput = flags.message;
const top = 3;

// Keep the local wiki catalog fresh: throttled (12h) version check against
// the remote manifest, silently replaces data when a newer version exists.
// Never throws — a sync failure must not block recommendation.
await maybeSyncWikiData();

// Default to JSON for structured output; render boxen cards only when the
// user explicitly asked for text output.
const format = settings.outputExplicit ? detectOutputFormat(settings.output) : "json";
Expand Down
3 changes: 2 additions & 1 deletion packages/commands/src/commands/dataset/delete.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { defineCommand, detectOutputFormat, deleteDataset, type FlagsDef } from "bailian-cli-core";
import { emitResult, emitBare } from "bailian-cli-runtime";
import { emitResult, emitBare, emitRequestId } from "bailian-cli-runtime";

const DELETE_FLAGS = {
fileId: {
Expand Down Expand Up @@ -30,6 +30,7 @@ export default defineCommand({

if (settings.quiet || format === "text") {
emitBare(`Deleted ${fileId}.`);
emitRequestId(response.request_id, settings.quiet);
} else {
emitResult(response, format);
}
Expand Down
5 changes: 3 additions & 2 deletions packages/commands/src/commands/dataset/get.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { defineCommand, detectOutputFormat, getDataset, type FlagsDef } from "bailian-cli-core";
import { emitResult, emitBare } from "bailian-cli-runtime";
import { emitResult, emitBare, emitRequestId } from "bailian-cli-runtime";

const GET_FLAGS = {
fileId: {
Expand Down Expand Up @@ -46,7 +46,7 @@ export default defineCommand({
};

if (format === "json") {
emitResult(item, format);
emitResult({ ...item, request_id: response.request_id }, format);
return;
}

Expand All @@ -58,5 +58,6 @@ export default defineCommand({
if (item.purpose) emitBare(`purpose: ${item.purpose}`);
if (item.created_at) emitBare(`created_at: ${item.created_at}`);
if (item.description) emitBare(`description: ${item.description}`);
emitRequestId(response.request_id, settings.quiet);
},
});
5 changes: 3 additions & 2 deletions packages/commands/src/commands/dataset/list.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { defineCommand, detectOutputFormat, listDatasets, type FlagsDef } from "bailian-cli-core";
import { emitResult, emitBare, formatTable } from "bailian-cli-runtime";
import { emitResult, emitBare, emitRequestId, formatTable } from "bailian-cli-runtime";

const LIST_FLAGS = {
page: { type: "number", valueHint: "<n>", description: "Page number (default: 1)" },
Expand Down Expand Up @@ -55,7 +55,7 @@ export default defineCommand({
}));

if (format === "json") {
emitResult({ items, total }, format);
emitResult({ items, total, request_id: response.request_id }, format);
return;
}

Expand All @@ -68,5 +68,6 @@ export default defineCommand({
const rows = items.map((i) => [i.file_id, i.name, i.size, i.purpose]);
for (const line of formatTable(headers, rows)) emitBare(line);
if (total !== undefined) emitBare(`\nTotal: ${total}`);
emitRequestId(response.request_id, settings.quiet);
},
});
13 changes: 7 additions & 6 deletions packages/commands/src/commands/dataset/upload.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,9 @@ import {
MAX_MEDIA_ZIP_BYTES,
BailianError,
ExitCode,
type DatasetFile,
type FlagsDef,
} from "bailian-cli-core";
import { emitResult, emitBare } from "bailian-cli-runtime";
import { emitResult, emitBare, emitRequestId } from "bailian-cli-runtime";

const UPLOAD_FLAGS = {
file: {
Expand Down Expand Up @@ -135,17 +134,19 @@ export default defineCommand({
return;
}

const uploaded: DatasetFile = await uploadDataset(ctx.client, {
const uploaded = await uploadDataset(ctx.client, {
filePath,
purpose,
});
const { request_id, ...file } = uploaded;

if (settings.quiet) {
emitBare(uploaded.file_id);
emitBare(file.file_id);
} else if (format === "text") {
emitBare(`Uploaded ${uploaded.name} → file_id=${uploaded.file_id}`);
emitBare(`Uploaded ${file.name} → file_id=${file.file_id}`);
emitRequestId(request_id, settings.quiet);
} else {
emitResult(uploaded, format);
emitResult({ ...file, request_id }, format);
}
},
});
Loading