From 4a74f5ebff068787df26d2ffb51393c0c618d728 Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Sun, 6 Sep 2026 09:33:55 +0000 Subject: [PATCH] feat(crawlproof): what the fleet costs and what it returns, as a command MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The dashboard is @profullstack/crawlproof, published from crawlproof.com. Nothing here reimplements it; this is the part that has to exist so `crawlproof` is a command on a server like every other one in this repo. Five screens over three feeds that are not otherwise in the same place: the tracker for who arrived, the ad network for what was delivered, and CoinPay for what the bank actually did. Bare `crawlproof` opens the dashboard, because the reason to type it on a box is to look at it. Installed into a private prefix on first use, and here that matters more than it does for hqtui: upstream's executable is called `crawlproof` and so is this wrapper. A global install would put two of them on PATH, and whichever came first would win — with a real chance of the command exec'ing itself. The private prefix means the name exists exactly once, and resolveRunner refuses to follow a PATH entry that resolves back into this repository's bin/ for the same reason. That refusal is load-bearing rather than defensive, so it has its own test. A missing API token is reported before handing over rather than after, because the failure it prevents is a 401 from inside a TUI, where there is nowhere good to explain anything. The CoinPay session is probed separately and never required: four of the five screens work without it. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01HvWJ4336pxTFRdRbvsTQeD --- README.md | 50 ++++++++++ bin/crawlproof.ts | 126 +++++++++++++++++++++++++ src/crawlproof.ts | 200 ++++++++++++++++++++++++++++++++++++++++ src/registry.ts | 1 + test/crawlproof.test.ts | 162 ++++++++++++++++++++++++++++++++ 5 files changed, 539 insertions(+) create mode 100755 bin/crawlproof.ts create mode 100644 src/crawlproof.ts create mode 100644 test/crawlproof.test.ts diff --git a/README.md b/README.md index 54aae8e..4277cc8 100644 --- a/README.md +++ b/README.md @@ -1081,6 +1081,56 @@ watch directory (`--watch`, `$TORLINK_WATCH`) as the offline handoff. How long it seeds for is a torlnk daemon setting (`--seed-time`), not a per-torrent one; left alone, it seeds indefinitely. +### `crawlproof` + +What the fleet costs and what it returns — +[CrawlProof](https://crawlproof.com)'s dashboard, wrapped so it is a command: + +```sh +crawlproof # the live dashboard, last day, humans +crawlproof dashboard --range=1m +crawlproof stats [site] # who arrived and from where, as text +crawlproof dashboard --json # the same snapshot, for a script +crawlproof --help # it is upstream's CLI: upstream's flags +``` + +Five screens over three feeds that are not otherwise in the same place: the +tracker for who arrived, the ad network for what was delivered, and CoinPay for +what the bank actually did. **ROI** is monthly burn against revenue, cost per +reader and break-even; **Traffic** ranks every site on the account with its +share of the cost; **Ads** is delivery as advertiser and as publisher; **Money** +is earnings, bank position and invoices; **Spend** is who you pay, largest +first. + +Two rules run through the arithmetic. Where an account advertises on its own +slots, ad spend and ad earnings are one dollar moving between two pockets, so +they are shown under *Internal* and counted as neither cost nor revenue. And a +bank feed carries groceries next to servers, so cost is the business scope +only. It also reports what it cannot know: a site that did not answer is +missing rather than zero, and a fleet whose visits run far above its pageviews +says so next to the number. + +It needs a CrawlProof API token — `CRAWLPROOF_TOKEN`, or the `token` field of +`~/.crawlproof.json`. The money screens additionally want a CoinPay merchant +session (`~/.coinpay.json`, which `coinpay auth login` writes); without one the +other four screens still work and the money panels say what is missing. + +Two flags are ours, spelled `--self-*` because every plain word belongs to the +dashboard: + +```sh +crawlproof --self-update # reinstall the latest release +crawlproof --self-where # which copy runs, and from where +``` + +**The first run installs it**, with `pnpm` and with `npm` when pnpm is absent +or fails. It lands in `~/.local/share/cli-tools/vendor/crawlproof`, not +globally, and the reason is sharper here than for `hqtui`: upstream's +executable is called `crawlproof` and so is this wrapper, so a global install +would put two of them on PATH and the command could end up running itself. A +private prefix means the name exists exactly once. `CRAWLPROOF_BIN` points at +a checkout instead, and `CRAWLPROOF_SPEC` pins what gets installed. + ### `hqtui` Every server's vitals, in the terminal — diff --git a/bin/crawlproof.ts b/bin/crawlproof.ts new file mode 100755 index 0000000..382376c --- /dev/null +++ b/bin/crawlproof.ts @@ -0,0 +1,126 @@ +#!/usr/bin/env node +/** + * crawlproof — what the fleet costs and what it returns. + * + * crawlproof the live dashboard, last day, humans + * crawlproof dashboard --range=1m + * crawlproof stats [site] who arrived and from where, as text + * crawlproof dashboard --json the same snapshot, for a script + * crawlproof --help upstream's CLI, so upstream's flags + * crawlproof --self-update refresh the installed dashboard + * crawlproof --self-where which copy runs, and from where + * + * Five screens: ROI, Traffic, Ads, Money, Spend — traffic across every site on + * the account, ad delivery, and the bank feed behind it. The money screens + * need a CoinPay session; without one the rest still works. + * + * The dashboard is @profullstack/crawlproof, installed on first use into a + * private prefix rather than globally. src/crawlproof.ts says why that matters + * more here than elsewhere: upstream's executable has the same name as this + * wrapper. + */ + +import { + MIN_NODE, + PACKAGE, + hasToken, + install, + meetsNodeFloor, + resolveRunner, + vendorBin, +} from '../src/crawlproof.ts'; +import { spawnInherit } from '../src/codeburn.ts'; +import { isMain } from '../src/is-main.ts'; + +/** + * The only two flags this wrapper keeps for itself. + * + * Spelled `--self-*` because every plain word belongs to the dashboard: it has + * its own --help, --range, --who and --json, and intercepting any of them + * would mean this file drifting out of step with a tool it does not own. + */ +const OURS = new Set(['--self-update', '--self-where']); + +async function main(argv: string[]): Promise { + const flags = new Set(argv.filter((argument) => OURS.has(argument))); + const rest = argv.filter((argument) => !OURS.has(argument)); + + if (!meetsNodeFloor(process.versions.node)) { + process.stderr.write( + `crawlproof: needs Node ${MIN_NODE} or newer (found ${process.version}).\n`, + ); + return 1; + } + + if (flags.has('--self-update')) { + const spec = process.env.CRAWLPROOF_SPEC || `${PACKAGE}@latest`; + process.stdout.write(`crawlproof: installing ${spec}\n`); + const result = await install(spec); + if (!result.ok) { + process.stderr.write('crawlproof: could not install the dashboard.\n'); + return 1; + } + process.stdout.write(`crawlproof: installed with ${result.manager}\n`); + return 0; + } + + let runner = resolveRunner(); + + if (flags.has('--self-where')) { + process.stdout.write(`${runner.file ?? '(not installed)'}\n`); + return runner.file ? 0 : 1; + } + + // First run on a box: install it, then run it. A dashboard that says "not + // found" on the machine you are trying to look at is not much use. + if (runner.kind === 'missing') { + const spec = process.env.CRAWLPROOF_SPEC || `${PACKAGE}@latest`; + process.stderr.write(`crawlproof: first run, installing ${spec}\n`); + const result = await install(spec); + if (!result.ok) { + process.stderr.write( + 'crawlproof: could not install the dashboard. Check the network, or run:\n' + + ` npm install -g ${PACKAGE}\n`, + ); + return 1; + } + runner = { kind: 'vendor', file: vendorBin() }; + } + + if (!runner.file) { + process.stderr.write('crawlproof: nothing to run.\n'); + return 1; + } + + // Said once, before handing over, because the failure it prevents is a 401 + // from inside a TUI — where there is no good place to explain anything. + if (!hasToken()) { + process.stderr.write( + 'crawlproof: no API token. Set CRAWLPROOF_TOKEN, or put {"token":"crp_…"}\n' + + ' in ~/.crawlproof.json. Mint one at crawlproof.com under Social → API tokens.\n', + ); + } + + // No subcommand is the dashboard: the reason to type this on a box is to + // look at it, and `crawlproof` alone printing usage would be a step in the + // way of the only thing most people want. + const args = rest.length === 0 ? ['dashboard'] : rest; + + const code = await spawnInherit(runner.file, args); + if (code === null) { + process.stderr.write(`crawlproof: could not start ${runner.file}\n`); + return 1; + } + return code; +} + +if (isMain(import.meta.url)) { + main(process.argv.slice(2)) + .then((code) => { + process.exitCode = code; + }) + .catch((error) => { + process.stderr.write(`crawlproof: ${(error as Error).message}\n`); + process.exitCode = 1; + }); +} diff --git a/src/crawlproof.ts b/src/crawlproof.ts new file mode 100644 index 0000000..e113e23 --- /dev/null +++ b/src/crawlproof.ts @@ -0,0 +1,200 @@ +/** + * crawlproof — what the fleet costs and what it returns, on every box. + * + * The dashboard itself is `@profullstack/crawlproof`, published from + * profullstack/crawlproof.com. Nothing here reimplements it; this is the part + * that has to exist so `crawlproof` is a command on a server like every other + * one in this repo. + * + * INSTALLED rather than run through npx, for the same reason as hqtui: a + * dashboard is opened many times a day and dlx hits the registry for metadata + * on every one of those, which is the wrong dependency to have on a box you + * are SSHed into because something is wrong. Installed once, refreshed with + * --self-update. + * + * Into a PRIVATE PREFIX, and here the reason is sharper than it is for hqtui: + * upstream's executable is called `crawlproof` and so is this wrapper. A + * global install would put a second `crawlproof` on PATH, and whichever came + * first would win — with a real chance of this command exec'ing itself. The + * private prefix means the name exists exactly once on PATH, and + * `resolveRunner` refuses to follow a PATH entry that resolves back into this + * repository's bin/ for the same reason. + * + * CRAWLPROOF_BIN run this executable instead — a checkout, or a global install + * CRAWLPROOF_SPEC what gets installed, when you want a pinned version + */ + +import { existsSync, mkdirSync, writeFileSync } from 'node:fs'; +import { homedir } from 'node:os'; +import { join } from 'node:path'; +import { onPath, resolveCommand } from './registry.ts'; +import { spawnInherit } from './codeburn.ts'; + +/** The published package, and the executable it installs. */ +export const PACKAGE = '@profullstack/crawlproof'; +export const EXECUTABLE = 'crawlproof'; + +/** The floor the package itself declares. Its TUI needs it; so does hqtui. */ +export const MIN_NODE = '22.6.0'; + +/** Where XDG says durable, non-config state goes. */ +export function dataHome(env: NodeJS.ProcessEnv = process.env): string { + return env.XDG_DATA_HOME || join(env.HOME ?? homedir(), '.local', 'share'); +} + +/** The private prefix: a directory whose entire job is to hold one package. */ +export function vendorRoot(env: NodeJS.ProcessEnv = process.env): string { + return join(dataHome(env), 'cli-tools', 'vendor', 'crawlproof'); +} + +/** The installed executable, whether or not it exists yet. */ +export function vendorBin(env: NodeJS.ProcessEnv = process.env): string { + return join(vendorRoot(env), 'node_modules', '.bin', EXECUTABLE); +} + +export type PackageManager = 'pnpm' | 'npm'; + +export interface InstallPlan { + file: string; + args: string[]; +} + +/** + * How to install with each manager. + * + * `--ignore-workspace` is not decoration: pnpm walks up from the install + * directory looking for a workspace root, and ~/.local/share is inside + * somebody's home directory. + */ +export function installPlan(manager: PackageManager, spec = `${PACKAGE}@latest`): InstallPlan { + if (manager === 'pnpm') { + return { file: 'pnpm', args: ['add', '--ignore-workspace', '--reporter=silent', spec] }; + } + return { file: 'npm', args: ['install', '--no-audit', '--no-fund', '--silent', spec] }; +} + +/** The managers to try, in order. pnpm is the intent, npm is what a bare box has. */ +export function managers(env: NodeJS.ProcessEnv = process.env): PackageManager[] { + return onPath('pnpm', env) ? ['pnpm', 'npm'] : ['npm']; +} + +export type RunnerKind = 'env' | 'vendor' | 'path' | 'missing'; + +export interface Runner { + kind: RunnerKind; + file: string | null; +} + +export interface ResolveDeps { + env?: NodeJS.ProcessEnv; + exists?: (path: string) => boolean; + onPathStatus?: () => 'ours' | 'other' | 'missing'; + onPathTarget?: () => string | null; +} + +/** + * Which dashboard to run. + * + * The `ours` check is load-bearing rather than defensive: the wrapper and the + * package install the same name, so a PATH hit that resolves back into this + * repository's bin/ is this file, and following it would be an exec loop. + */ +export function resolveRunner(deps: ResolveDeps = {}): Runner { + const env = deps.env ?? process.env; + const exists = deps.exists ?? existsSync; + const status = deps.onPathStatus ?? (() => resolveCommand(EXECUTABLE, undefined, env).status); + const target = deps.onPathTarget ?? (() => resolveCommand(EXECUTABLE, undefined, env).target); + + const override = env.CRAWLPROOF_BIN; + if (override) return { kind: 'env', file: override }; + + const vendored = vendorBin(env); + if (exists(vendored)) return { kind: 'vendor', file: vendored }; + + if (status() === 'other') return { kind: 'path', file: target() }; + + return { kind: 'missing', file: null }; +} + +/** Give the private prefix the package.json both managers insist on. */ +export function prepareVendorDir(root: string): void { + mkdirSync(root, { recursive: true }); + const manifest = join(root, 'package.json'); + if (existsSync(manifest)) return; + + writeFileSync( + manifest, + `${JSON.stringify( + { + name: 'cli-tools-vendor-crawlproof', + version: '0.0.0', + private: true, + description: 'Prefix owned by profullstack/cli-tools. Managed by the crawlproof command.', + }, + null, + 2, + )}\n`, + ); +} + +/** Is this Node new enough? Prerelease and build suffixes are dropped. */ +export function meetsNodeFloor(version: string, floor: string = MIN_NODE): boolean { + const parse = (v: string): number[] => + v + .replace(/^v/, '') + .split(/[-+]/)[0]! + .split('.') + .map((part) => Number.parseInt(part, 10) || 0); + + const got = parse(version); + const want = parse(floor); + + for (let i = 0; i < 3; i += 1) { + const a = got[i] ?? 0; + const b = want[i] ?? 0; + if (a !== b) return a > b; + } + return true; +} + +export interface InstallResult { + ok: boolean; + manager?: PackageManager; + code?: number | null; +} + +/** Install (or refresh) the dashboard in the private prefix. */ +export async function install( + spec: string = `${PACKAGE}@latest`, + env: NodeJS.ProcessEnv = process.env, + run: typeof spawnInherit = spawnInherit, +): Promise { + const root = vendorRoot(env); + prepareVendorDir(root); + + for (const manager of managers(env)) { + const plan = installPlan(manager, spec); + const code = await run(plan.file, plan.args, root); + if (code === 0) return { ok: true, manager, code }; + } + return { ok: false }; +} + +/** + * Whether this box can answer the money half at all. + * + * Reported rather than enforced: the traffic and ads screens work without a + * CoinPay session, and the dashboard says which panels are missing. A wrapper + * that refused to start would be hiding four working screens behind one + * absent credential. + */ +export function hasCoinpaySession(env: NodeJS.ProcessEnv = process.env): boolean { + if (env.COINPAY_SESSION_TOKEN?.trim()) return true; + return existsSync(join(env.HOME ?? homedir(), '.coinpay.json')); +} + +/** Whether a CrawlProof API token is reachable without the caller exporting one. */ +export function hasToken(env: NodeJS.ProcessEnv = process.env): boolean { + if (env.CRAWLPROOF_TOKEN?.trim()) return true; + return existsSync(env.CRAWLPROOF_CONFIG ?? join(env.HOME ?? homedir(), '.crawlproof.json')); +} diff --git a/src/registry.ts b/src/registry.ts index 263622c..fc5fa96 100644 --- a/src/registry.ts +++ b/src/registry.ts @@ -37,6 +37,7 @@ const SUMMARIES: Record = { domainfree: 'Which of these domains can you actually register', domainjson: 'whois-style, JSON-first name lookup', favicon: 'Every icon a site links, rendered from one SVG', + crawlproof: 'What the fleet costs and what it returns: traffic, ads and the bank behind them', 'free-names': 'Name ideas nobody has registered yet, in one command', 'generate-names': 'Turn a sentence about a product into a thousand candidate names', genrewatch: 'What is coming out, and whether it exists at all', diff --git a/test/crawlproof.test.ts b/test/crawlproof.test.ts new file mode 100644 index 0000000..e969104 --- /dev/null +++ b/test/crawlproof.test.ts @@ -0,0 +1,162 @@ +import { describe, expect, it } from 'vitest'; +import { + EXECUTABLE, + MIN_NODE, + PACKAGE, + hasCoinpaySession, + hasToken, + installPlan, + managers, + meetsNodeFloor, + prepareVendorDir, + resolveRunner, + vendorBin, + vendorRoot, +} from '../src/crawlproof.ts'; +import { mkdtempSync, readFileSync, rmSync, existsSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +describe('vendorRoot', () => { + it('follows XDG_DATA_HOME when it is set', () => { + expect(vendorRoot({ XDG_DATA_HOME: '/data' })).toBe('/data/cli-tools/vendor/crawlproof'); + }); + + it('falls back to ~/.local/share', () => { + expect(vendorRoot({ HOME: '/home/x' })).toBe('/home/x/.local/share/cli-tools/vendor/crawlproof'); + }); + + it('installs into a private prefix, because the package owns our own name', () => { + // Unlike hqtui, upstream's executable is called `crawlproof` and so is + // this wrapper. A global install would put two of them on PATH. + expect(vendorBin({ XDG_DATA_HOME: '/data' })).toBe( + '/data/cli-tools/vendor/crawlproof/node_modules/.bin/crawlproof', + ); + expect(EXECUTABLE).toBe('crawlproof'); + expect(PACKAGE).toBe('@profullstack/crawlproof'); + }); +}); + +describe('installPlan', () => { + it('keeps pnpm out of a workspace it happens to be standing in', () => { + const plan = installPlan('pnpm'); + expect(plan.file).toBe('pnpm'); + expect(plan.args).toContain('--ignore-workspace'); + expect(plan.args.at(-1)).toBe(`${PACKAGE}@latest`); + }); + + it('installs a pinned spec when one is given', () => { + expect(installPlan('npm', `${PACKAGE}@0.1.0`).args.at(-1)).toBe(`${PACKAGE}@0.1.0`); + }); +}); + +describe('managers', () => { + it('is npm alone on a box without pnpm', () => { + expect(managers({ PATH: '' })).toEqual(['npm']); + }); +}); + +describe('resolveRunner', () => { + it('lets CRAWLPROOF_BIN win over everything', () => { + const runner = resolveRunner({ + env: { CRAWLPROOF_BIN: '/opt/crawlproof' }, + exists: () => true, + onPathStatus: () => 'other', + onPathTarget: () => '/usr/bin/crawlproof', + }); + expect(runner).toEqual({ kind: 'env', file: '/opt/crawlproof' }); + }); + + it('prefers the vendored copy to anything on PATH', () => { + const runner = resolveRunner({ + env: { XDG_DATA_HOME: '/data' }, + exists: (p) => p === '/data/cli-tools/vendor/crawlproof/node_modules/.bin/crawlproof', + onPathStatus: () => 'other', + onPathTarget: () => '/usr/bin/crawlproof', + }); + expect(runner.kind).toBe('vendor'); + }); + + it('follows a PATH copy that is somebody else', () => { + const runner = resolveRunner({ + env: {}, + exists: () => false, + onPathStatus: () => 'other', + onPathTarget: () => '/usr/bin/crawlproof', + }); + expect(runner).toEqual({ kind: 'path', file: '/usr/bin/crawlproof' }); + }); + + it('refuses to follow itself, which would be an exec loop', () => { + // The wrapper and the package share a name, so a PATH hit that resolves + // into this repo's bin/ is this file. + const runner = resolveRunner({ + env: {}, + exists: () => false, + onPathStatus: () => 'ours', + onPathTarget: () => '/home/x/.local/bin/crawlproof', + }); + expect(runner).toEqual({ kind: 'missing', file: null }); + }); +}); + +describe('prepareVendorDir', () => { + it('writes the manifest both package managers insist on, once', () => { + const root = join(mkdtempSync(join(tmpdir(), 'cp-vendor-')), 'nested'); + try { + prepareVendorDir(root); + const manifest = join(root, 'package.json'); + expect(existsSync(manifest)).toBe(true); + const first = readFileSync(manifest, 'utf8'); + expect(JSON.parse(first).private).toBe(true); + + prepareVendorDir(root); + expect(readFileSync(manifest, 'utf8')).toBe(first); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); + +describe('meetsNodeFloor', () => { + it('accepts the floor and anything above it', () => { + expect(meetsNodeFloor('22.6.0')).toBe(true); + expect(meetsNodeFloor('v24.18.1')).toBe(true); + }); + + it('rejects anything below, including a same-major older minor', () => { + expect(meetsNodeFloor('22.5.9')).toBe(false); + expect(meetsNodeFloor('20.19.0')).toBe(false); + }); + + it('ignores prerelease and build suffixes', () => { + expect(meetsNodeFloor('23.0.0-nightly')).toBe(true); + expect(MIN_NODE).toBe('22.6.0'); + }); +}); + +describe('credential probes', () => { + it('sees a token in the environment', () => { + expect(hasToken({ CRAWLPROOF_TOKEN: 'crp_x' })).toBe(true); + }); + + it('sees a token in the config file', () => { + const dir = mkdtempSync(join(tmpdir(), 'cp-cfg-')); + const file = join(dir, 'crawlproof.json'); + writeFileSync(file, '{"token":"crp_x"}'); + try { + expect(hasToken({ CRAWLPROOF_CONFIG: file })).toBe(true); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it('reports no token rather than guessing one', () => { + expect(hasToken({ HOME: join(tmpdir(), 'definitely-not-a-home') })).toBe(false); + }); + + it('reports the CoinPay session separately, since four screens work without it', () => { + expect(hasCoinpaySession({ COINPAY_SESSION_TOKEN: 'jwt' })).toBe(true); + expect(hasCoinpaySession({ HOME: join(tmpdir(), 'definitely-not-a-home') })).toBe(false); + }); +});