feat(crawlproof): what the fleet costs and what it returns, as a command

README.md

@@ -1081,6 +1081,56 @@ watch directory (`--watch`, `$TORLINK_WATCH`) as the offline handoff. How long
 it seeds for is a torlnk daemon setting (`--seed-time`), not a per-torrent one;
 left alone, it seeds indefinitely.
 
+### `crawlproof`
+
+What the fleet costs and what it returns —
+[CrawlProof](https://crawlproof.com)'s dashboard, wrapped so it is a command:
+
+```sh
+crawlproof                     # the live dashboard, last day, humans
+crawlproof dashboard --range=1m
+crawlproof stats [site]        # who arrived and from where, as text
+crawlproof dashboard --json    # the same snapshot, for a script
+crawlproof --help              # it is upstream's CLI: upstream's flags
+```
+
+Five screens over three feeds that are not otherwise in the same place: the
+tracker for who arrived, the ad network for what was delivered, and CoinPay for
+what the bank actually did. **ROI** is monthly burn against revenue, cost per
+reader and break-even; **Traffic** ranks every site on the account with its
+share of the cost; **Ads** is delivery as advertiser and as publisher; **Money**
+is earnings, bank position and invoices; **Spend** is who you pay, largest
+first.
+
+Two rules run through the arithmetic. Where an account advertises on its own
+slots, ad spend and ad earnings are one dollar moving between two pockets, so
+they are shown under *Internal* and counted as neither cost nor revenue. And a
+bank feed carries groceries next to servers, so cost is the business scope
+only. It also reports what it cannot know: a site that did not answer is
+missing rather than zero, and a fleet whose visits run far above its pageviews
+says so next to the number.
+
+It needs a CrawlProof API token — `CRAWLPROOF_TOKEN`, or the `token` field of
+`~/.crawlproof.json`. The money screens additionally want a CoinPay merchant
+session (`~/.coinpay.json`, which `coinpay auth login` writes); without one the
+other four screens still work and the money panels say what is missing.
+
+Two flags are ours, spelled `--self-*` because every plain word belongs to the
+dashboard:
+
+```sh
+crawlproof --self-update       # reinstall the latest release
+crawlproof --self-where        # which copy runs, and from where
+```
+
+**The first run installs it**, with `pnpm` and with `npm` when pnpm is absent
+or fails. It lands in `~/.local/share/cli-tools/vendor/crawlproof`, not
+globally, and the reason is sharper here than for `hqtui`: upstream's
+executable is called `crawlproof` and so is this wrapper, so a global install
+would put two of them on PATH and the command could end up running itself. A
+private prefix means the name exists exactly once. `CRAWLPROOF_BIN` points at
+a checkout instead, and `CRAWLPROOF_SPEC` pins what gets installed.
+
 ### `hqtui`
 
 Every server's vitals, in the terminal —

bin/crawlproof.ts

@@ -0,0 +1,126 @@
+#!/usr/bin/env node
+/**
+ * crawlproof — what the fleet costs and what it returns.
+ *
+ *   crawlproof                     the live dashboard, last day, humans
+ *   crawlproof dashboard --range=1m
+ *   crawlproof stats [site]        who arrived and from where, as text
+ *   crawlproof dashboard --json    the same snapshot, for a script
+ *   crawlproof --help              upstream's CLI, so upstream's flags
+ *   crawlproof --self-update       refresh the installed dashboard
+ *   crawlproof --self-where        which copy runs, and from where
+ *
+ * Five screens: ROI, Traffic, Ads, Money, Spend — traffic across every site on
+ * the account, ad delivery, and the bank feed behind it. The money screens
+ * need a CoinPay session; without one the rest still works.
+ *
+ * The dashboard is @profullstack/crawlproof, installed on first use into a
+ * private prefix rather than globally. src/crawlproof.ts says why that matters
+ * more here than elsewhere: upstream's executable has the same name as this
+ * wrapper.
+ */
+
+import {
+  MIN_NODE,
+  PACKAGE,
+  hasToken,
+  install,
+  meetsNodeFloor,
+  resolveRunner,
+  vendorBin,
+} from '../src/crawlproof.ts';
+import { spawnInherit } from '../src/codeburn.ts';
+import { isMain } from '../src/is-main.ts';
+
+/**
+ * The only two flags this wrapper keeps for itself.
+ *
+ * Spelled `--self-*` because every plain word belongs to the dashboard: it has
+ * its own --help, --range, --who and --json, and intercepting any of them
+ * would mean this file drifting out of step with a tool it does not own.
+ */
+const OURS = new Set(['--self-update', '--self-where']);
+
+async function main(argv: string[]): Promise<number> {
+  const flags = new Set(argv.filter((argument) => OURS.has(argument)));
+  const rest = argv.filter((argument) => !OURS.has(argument));
+
+  if (!meetsNodeFloor(process.versions.node)) {
+    process.stderr.write(
+      `crawlproof: needs Node ${MIN_NODE} or newer (found ${process.version}).\n`,
+    );
+    return 1;
+  }
+
+  if (flags.has('--self-update')) {
+    const spec = process.env.CRAWLPROOF_SPEC || `${PACKAGE}@latest`;
+    process.stdout.write(`crawlproof: installing ${spec}\n`);
+    const result = await install(spec);
+    if (!result.ok) {
+      process.stderr.write('crawlproof: could not install the dashboard.\n');
+      return 1;
+    }
+    process.stdout.write(`crawlproof: installed with ${result.manager}\n`);
+    return 0;
+  }
+
+  let runner = resolveRunner();
+
+  if (flags.has('--self-where')) {
+    process.stdout.write(`${runner.file ?? '(not installed)'}\n`);
+    return runner.file ? 0 : 1;
+  }
+
+  // First run on a box: install it, then run it. A dashboard that says "not
+  // found" on the machine you are trying to look at is not much use.
+  if (runner.kind === 'missing') {
+    const spec = process.env.CRAWLPROOF_SPEC || `${PACKAGE}@latest`;
+    process.stderr.write(`crawlproof: first run, installing ${spec}\n`);
+    const result = await install(spec);
+    if (!result.ok) {
+      process.stderr.write(
+        'crawlproof: could not install the dashboard. Check the network, or run:\n' +
+          `  npm install -g ${PACKAGE}\n`,
+      );
+      return 1;
+    }
+    runner = { kind: 'vendor', file: vendorBin() };
+  }
+
+  if (!runner.file) {
+    process.stderr.write('crawlproof: nothing to run.\n');
+    return 1;
+  }
+
+  // Said once, before handing over, because the failure it prevents is a 401
+  // from inside a TUI — where there is no good place to explain anything.
+  if (!hasToken()) {
+    process.stderr.write(
+      'crawlproof: no API token. Set CRAWLPROOF_TOKEN, or put {"token":"crp_…"}\n' +
+        '  in ~/.crawlproof.json. Mint one at crawlproof.com under Social → API tokens.\n',
+    );
+  }
+
+  // No subcommand is the dashboard: the reason to type this on a box is to
+  // look at it, and `crawlproof` alone printing usage would be a step in the
+  // way of the only thing most people want.
+  const args = rest.length === 0 ? ['dashboard'] : rest;
+
+  const code = await spawnInherit(runner.file, args);
+  if (code === null) {
+    process.stderr.write(`crawlproof: could not start ${runner.file}\n`);
+    return 1;
+  }
+  return code;
+}
+
+if (isMain(import.meta.url)) {
+  main(process.argv.slice(2))
+    .then((code) => {
+      process.exitCode = code;
+    })
+    .catch((error) => {
+      process.stderr.write(`crawlproof: ${(error as Error).message}\n`);
+      process.exitCode = 1;
+    });
+}

src/crawlproof.ts

@@ -0,0 +1,200 @@
+/**
+ * crawlproof — what the fleet costs and what it returns, on every box.
+ *
+ * The dashboard itself is `@profullstack/crawlproof`, published from
+ * profullstack/crawlproof.com. Nothing here reimplements it; this is the part
+ * that has to exist so `crawlproof` is a command on a server like every other
+ * one in this repo.
+ *
+ * INSTALLED rather than run through npx, for the same reason as hqtui: a
+ * dashboard is opened many times a day and dlx hits the registry for metadata
+ * on every one of those, which is the wrong dependency to have on a box you
+ * are SSHed into because something is wrong. Installed once, refreshed with
+ * --self-update.
+ *
+ * Into a PRIVATE PREFIX, and here the reason is sharper than it is for hqtui:
+ * upstream's executable is called `crawlproof` and so is this wrapper. A
+ * global install would put a second `crawlproof` on PATH, and whichever came
+ * first would win — with a real chance of this command exec'ing itself. The
+ * private prefix means the name exists exactly once on PATH, and
+ * `resolveRunner` refuses to follow a PATH entry that resolves back into this
+ * repository's bin/ for the same reason.
+ *
+ *   CRAWLPROOF_BIN    run this executable instead — a checkout, or a global install
+ *   CRAWLPROOF_SPEC   what gets installed, when you want a pinned version
+ */
+
+import { existsSync, mkdirSync, writeFileSync } from 'node:fs';
+import { homedir } from 'node:os';
+import { join } from 'node:path';
+import { onPath, resolveCommand } from './registry.ts';
+import { spawnInherit } from './codeburn.ts';
+
+/** The published package, and the executable it installs. */
+export const PACKAGE = '@profullstack/crawlproof';
+export const EXECUTABLE = 'crawlproof';
+
+/** The floor the package itself declares. Its TUI needs it; so does hqtui. */
+export const MIN_NODE = '22.6.0';
+
+/** Where XDG says durable, non-config state goes. */
+export function dataHome(env: NodeJS.ProcessEnv = process.env): string {
+  return env.XDG_DATA_HOME || join(env.HOME ?? homedir(), '.local', 'share');
+}
+
+/** The private prefix: a directory whose entire job is to hold one package. */
+export function vendorRoot(env: NodeJS.ProcessEnv = process.env): string {
+  return join(dataHome(env), 'cli-tools', 'vendor', 'crawlproof');
+}
+
+/** The installed executable, whether or not it exists yet. */
+export function vendorBin(env: NodeJS.ProcessEnv = process.env): string {
+  return join(vendorRoot(env), 'node_modules', '.bin', EXECUTABLE);
+}
+
+export type PackageManager = 'pnpm' | 'npm';
+
+export interface InstallPlan {
+  file: string;
+  args: string[];
+}
+
+/**
+ * How to install with each manager.
+ *
+ * `--ignore-workspace` is not decoration: pnpm walks up from the install
+ * directory looking for a workspace root, and ~/.local/share is inside
+ * somebody's home directory.
+ */
+export function installPlan(manager: PackageManager, spec = `${PACKAGE}@latest`): InstallPlan {
+  if (manager === 'pnpm') {
+    return { file: 'pnpm', args: ['add', '--ignore-workspace', '--reporter=silent', spec] };
+  }
+  return { file: 'npm', args: ['install', '--no-audit', '--no-fund', '--silent', spec] };
+}
+
+/** The managers to try, in order. pnpm is the intent, npm is what a bare box has. */
+export function managers(env: NodeJS.ProcessEnv = process.env): PackageManager[] {
+  return onPath('pnpm', env) ? ['pnpm', 'npm'] : ['npm'];
+}
+
+export type RunnerKind = 'env' | 'vendor' | 'path' | 'missing';
+
+export interface Runner {
+  kind: RunnerKind;
+  file: string | null;
+}
+
+export interface ResolveDeps {
+  env?: NodeJS.ProcessEnv;
+  exists?: (path: string) => boolean;
+  onPathStatus?: () => 'ours' | 'other' | 'missing';
+  onPathTarget?: () => string | null;
+}
+
+/**
+ * Which dashboard to run.
+ *
+ * The `ours` check is load-bearing rather than defensive: the wrapper and the
+ * package install the same name, so a PATH hit that resolves back into this
+ * repository's bin/ is this file, and following it would be an exec loop.
+ */
+export function resolveRunner(deps: ResolveDeps = {}): Runner {
+  const env = deps.env ?? process.env;
+  const exists = deps.exists ?? existsSync;
+  const status = deps.onPathStatus ?? (() => resolveCommand(EXECUTABLE, undefined, env).status);
+  const target = deps.onPathTarget ?? (() => resolveCommand(EXECUTABLE, undefined, env).target);
+
+  const override = env.CRAWLPROOF_BIN;
+  if (override) return { kind: 'env', file: override };
+
+  const vendored = vendorBin(env);
+  if (exists(vendored)) return { kind: 'vendor', file: vendored };
+
+  if (status() === 'other') return { kind: 'path', file: target() };
+
+  return { kind: 'missing', file: null };
+}
+
+/** Give the private prefix the package.json both managers insist on. */
+export function prepareVendorDir(root: string): void {
+  mkdirSync(root, { recursive: true });
+  const manifest = join(root, 'package.json');
+  if (existsSync(manifest)) return;
+
+  writeFileSync(
+    manifest,
+    `${JSON.stringify(
+      {
+        name: 'cli-tools-vendor-crawlproof',
+        version: '0.0.0',
+        private: true,
+        description: 'Prefix owned by profullstack/cli-tools. Managed by the crawlproof command.',
+      },
+      null,
+      2,
+    )}\n`,
+  );
+}
+
+/** Is this Node new enough? Prerelease and build suffixes are dropped. */
+export function meetsNodeFloor(version: string, floor: string = MIN_NODE): boolean {
+  const parse = (v: string): number[] =>
+    v
+      .replace(/^v/, '')
+      .split(/[-+]/)[0]!
+      .split('.')
+      .map((part) => Number.parseInt(part, 10) || 0);
+
+  const got = parse(version);
+  const want = parse(floor);
+
+  for (let i = 0; i < 3; i += 1) {
+    const a = got[i] ?? 0;
+    const b = want[i] ?? 0;
+    if (a !== b) return a > b;
+  }
+  return true;
+}
+
+export interface InstallResult {
+  ok: boolean;
+  manager?: PackageManager;
+  code?: number | null;
+}
+
+/** Install (or refresh) the dashboard in the private prefix. */
+export async function install(
+  spec: string = `${PACKAGE}@latest`,
+  env: NodeJS.ProcessEnv = process.env,
+  run: typeof spawnInherit = spawnInherit,
+): Promise<InstallResult> {
+  const root = vendorRoot(env);
+  prepareVendorDir(root);
+
+  for (const manager of managers(env)) {
+    const plan = installPlan(manager, spec);
+    const code = await run(plan.file, plan.args, root);
+    if (code === 0) return { ok: true, manager, code };
+  }
+  return { ok: false };
+}
+
+/**
+ * Whether this box can answer the money half at all.
+ *
+ * Reported rather than enforced: the traffic and ads screens work without a
+ * CoinPay session, and the dashboard says which panels are missing. A wrapper
+ * that refused to start would be hiding four working screens behind one
+ * absent credential.
+ */
+export function hasCoinpaySession(env: NodeJS.ProcessEnv = process.env): boolean {
+  if (env.COINPAY_SESSION_TOKEN?.trim()) return true;
+  return existsSync(join(env.HOME ?? homedir(), '.coinpay.json'));
+}
+
+/** Whether a CrawlProof API token is reachable without the caller exporting one. */
+export function hasToken(env: NodeJS.ProcessEnv = process.env): boolean {
+  if (env.CRAWLPROOF_TOKEN?.trim()) return true;
+  return existsSync(env.CRAWLPROOF_CONFIG ?? join(env.HOME ?? homedir(), '.crawlproof.json'));
+}

src/registry.ts

@@ -37,6 +37,7 @@ const SUMMARIES: Record<string, string> = {
   domainfree: 'Which of these domains can you actually register',
   domainjson: 'whois-style, JSON-first name lookup',
   favicon: 'Every icon a site links, rendered from one SVG',
+  crawlproof: 'What the fleet costs and what it returns: traffic, ads and the bank behind them',
   'free-names': 'Name ideas nobody has registered yet, in one command',
   'generate-names': 'Turn a sentence about a product into a thousand candidate names',
   genrewatch: 'What is coming out, and whether it exists at all',