feat(pit): make /n/ crawlable

apps/pwa/src/moshpit.mjs

@@ -250,6 +250,17 @@ export async function countNames(tld) {
   return Number(row?.n ?? 0);
 }
 
+/**
+ * Every registered name, for the sitemap.
+ *
+ * Bounded because a sitemap has a hard 50k-URL ceiling and this has to stay one
+ * file; past the limit the tail is dropped rather than paged, which is the
+ * right trade while the namespace is far below it.
+ */
+export async function listAllNames(limit = 20_000) {
+  return all(`SELECT tld, label FROM moshpit_names ORDER BY tld, label LIMIT ?`, [limit]);
+}
+
 export async function listNamesForUser(userId) {
   return all(`SELECT ${NAME_COLS} FROM moshpit_names WHERE user_id = ? ORDER BY tld, label`, [userId]);
 }

apps/pwa/src/routes/moshpit.mjs

@@ -35,6 +35,7 @@ import {
   getName,
   getTld,
   getTldWithPrice,
+  listAllNames,
   listExempt,
   listNames,
   listPins,
@@ -252,6 +253,77 @@ moshpitRouter.delete("/api/moshpit/tlds/:tld/names", async (req, res) => {
 
 /* ---- serving a name over the clearnet ---- */
 
+/** The canonical clearnet URL for a name. One name, one indexable address. */
+const nameUrl = (name) => `${config.origin}/n/${encodeURIComponent(name)}`;
+
+/**
+ * Head tags for a name's page.
+ *
+ * These pages are the network's public surface — a name nobody holds is a page
+ * somebody should be able to *find*, which is the whole pitch. So they get a
+ * canonical URL and a description rather than being left to whatever a crawler
+ * infers from a directory listing.
+ *
+ * An aliased name canonicalises to what it resolves to: `.agentic` pointing at
+ * `.agent` means one page, reachable by two names, and saying so keeps the two
+ * from competing as duplicates.
+ */
+function nameHead(resolution) {
+  const canonical = nameUrl(resolution.resolved || resolution.name);
+  const description = resolution.name_registered
+    ? `${resolution.name} is registered on the Moshpit network.`
+    : `${resolution.name} is unclaimed on the Moshpit network — take it in the pit.`;
+  return `<link rel="canonical" href="${esc(canonical)}">
+<meta name="description" content="${esc(description)}">
+<meta property="og:type" content="website">
+<meta property="og:title" content="${esc(resolution.name)}">
+<meta property="og:url" content="${esc(canonical)}">
+<meta property="og:description" content="${esc(description)}">`;
+}
+
+/**
+ * Crawlers get an explicit invitation rather than an inferred one.
+ *
+ * `/n/` is the point of the network being on the clearnet at all, so it is
+ * named as allowed. The proxied half of a name (`/n/<name>/<path>`) is somebody
+ * else's site reached through us and is not ours to get indexed under this
+ * host, so only the name's own page is offered.
+ */
+moshpitRouter.get("/robots.txt", (_req, res) => {
+  res.type("text/plain").send(`User-agent: *
+Allow: /$
+Allow: /pit
+Allow: /n/
+Disallow: /api/
+Disallow: /app
+Disallow: /settings
+Disallow: /sessions
+
+Sitemap: ${config.origin}/sitemap.xml
+`);
+});
+
+/**
+ * Every name and ending in the pit, as one file.
+ *
+ * Generated rather than stored: the namespace changes whenever somebody claims
+ * something, and a sitemap that lags the registry is worse than none — it
+ * advertises URLs that did not exist yet and omits the ones that do.
+ */
+moshpitRouter.get("/sitemap.xml", async (_req, res) => {
+  const names = await listAllNames();
+
+  // Only whole names. `/n/<ending>` is not a name — it 400s — so listing
+  // endings here would advertise URLs that do not resolve.
+  const urls = [`${config.origin}/pit`, ...names.map((n) => nameUrl(`${n.label}.${n.tld}`))];
+
+  res.type("application/xml").send(`<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+${urls.map((u) => `  <url><loc>${esc(u)}</loc></url>`).join("\n")}
+</urlset>
+`);
+});
+
 /**
  * GET /n/:name — what a Moshpit name actually shows.
  *
@@ -304,6 +376,7 @@ moshpitRouter.get("/n/:name", async (req, res) => {
   // to a link checker, and to anything that treats the status before the body.
   res.status(200).send(page({
     title: resolution.name,
+    head: nameHead(resolution),
     body: directory({ resolution, tld, owner, names, tlds, quote, user: req.user, req }),
   }));
 });

apps/pwa/test/moshpit-crawlable.test.mjs

@@ -0,0 +1,115 @@
+// Whether a crawler can find the network.
+//
+// `/n/<name>` is the whole point of the pit being on the clearnet: a name
+// nobody holds is a page somebody should be able to *find*. Before this there
+// was no robots.txt, no sitemap and no canonical — the pages existed and
+// nothing advertised them.
+//
+// Same harness as moshpit-pit-page.test.mjs: the real router against a
+// throwaway libsql file, skipped cleanly when the PWA deps are not installed.
+import assert from "node:assert/strict";
+import fs from "node:fs";
+import { mkdtempSync } from "node:fs";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import { createRequire } from "node:module";
+import test from "node:test";
+
+const require = createRequire(import.meta.url);
+let deps = null;
+try {
+  deps = { express: require("express") };
+} catch {
+  deps = null;
+}
+
+const workdir = mkdtempSync(path.join(tmpdir(), "moshcode-crawl-test-"));
+process.env.DATABASE_URL = `file:${path.join(workdir, "test.db")}`;
+process.env.SESSION_SECRET = "test-secret";
+process.env.PUBLIC_ORIGIN = "https://pit.example.test";
+
+const ORIGIN = "https://pit.example.test";
+
+async function boot() {
+  const { migrate } = await import("../src/migrate.mjs");
+  await migrate();
+  const { run, db } = await import("../src/db.mjs");
+  const { moshpitRouter } = await import("../src/routes/moshpit.mjs");
+
+  await run(`INSERT OR REPLACE INTO users (id,email,created_at) VALUES ('u1','a@b.c',1)`);
+  await run(`INSERT INTO moshpit_tlds (tld,user_id,owner_email,created_at) VALUES ('eggs','u1','a@b.c',1)`);
+  await run(`INSERT INTO moshpit_names (tld,label,user_id,target,created_at) VALUES ('eggs','scrambled','u1',NULL,1)`);
+  await run(`INSERT INTO moshpit_names (tld,label,user_id,target,created_at) VALUES ('eggs','poached','u1',NULL,1)`);
+
+  const app = deps.express();
+  app.use((req, _res, next) => { req.csrfToken = () => "csrf"; next(); });
+  app.use(moshpitRouter);
+  const server = await new Promise((resolve) => {
+    const s = app.listen(0, "127.0.0.1", () => resolve(s));
+  });
+  const base = `http://127.0.0.1:${server.address().port}`;
+  const get = async (p) => {
+    const res = await fetch(`${base}${p}`);
+    return { status: res.status, type: res.headers.get("content-type") || "", body: await res.text() };
+  };
+  return { server, db, get };
+}
+
+let booted = null;
+const app = () => (booted ||= boot());
+
+test.after(() => {
+  if (!booted) return;
+  booted.then(({ server, db }) => { server.close(); db.close?.(); })
+    .finally(() => { try { fs.rmSync(workdir, { recursive: true, force: true }); } catch { /* noop */ } });
+});
+
+const skip = { skip: !deps && "apps/pwa deps not installed" };
+
+test("robots.txt invites crawlers to /n/ and points at the sitemap", skip, async () => {
+  const { get } = await app();
+  const { status, type, body } = await get("/robots.txt");
+
+  assert.equal(status, 200);
+  assert.match(type, /text\/plain/);
+  assert.match(body, /^User-agent: \*/m);
+  assert.match(body, /^Allow: \/n\//m);
+  assert.match(body, new RegExp(`^Sitemap: ${ORIGIN}/sitemap\\.xml$`, "m"));
+  // The private half of the app has no business being indexed.
+  assert.match(body, /^Disallow: \/api\//m);
+});
+
+test("the sitemap lists every registered name, and nothing that 400s", skip, async () => {
+  const { get } = await app();
+  const { status, type, body } = await get("/sitemap.xml");
+
+  assert.equal(status, 200);
+  assert.match(type, /xml/);
+  assert.ok(body.includes(`<loc>${ORIGIN}/n/scrambled.eggs</loc>`), body);
+  assert.ok(body.includes(`<loc>${ORIGIN}/n/poached.eggs</loc>`), body);
+  assert.ok(body.includes(`<loc>${ORIGIN}/pit</loc>`), body);
+
+  // `/n/eggs` is an ending, not a name — it 400s, so it must not be advertised.
+  assert.ok(!body.includes(`<loc>${ORIGIN}/n/eggs</loc>`), "an ending is not a name");
+});
+
+test("a name's page canonicalises to itself and describes itself", skip, async () => {
+  const { get } = await app();
+  const { status, body } = await get("/n/scrambled.eggs");
+
+  assert.equal(status, 200);
+  assert.ok(body.includes(`<link rel="canonical" href="${ORIGIN}/n/scrambled.eggs">`), body.slice(0, 600));
+  assert.match(body, /<meta name="description" content="scrambled\.eggs [^"]+">/);
+  assert.ok(body.includes(`<meta property="og:url" content="${ORIGIN}/n/scrambled.eggs">`));
+});
+
+test("an unclaimed name still gets indexable head tags", skip, async () => {
+  const { get } = await app();
+  // Nobody holds `.chicken`, so this name is unregistered — and still a page
+  // worth finding, which is the entire pitch.
+  const { status, body } = await get("/n/hawaiian.chicken");
+
+  assert.equal(status, 200);
+  assert.ok(body.includes(`<link rel="canonical" href="${ORIGIN}/n/hawaiian.chicken">`));
+  assert.match(body, /content="hawaiian\.chicken is unclaimed[^"]*"/);
+});