Pay x402 gateways when the crawler meets one

packages/feed/package.json

@@ -16,6 +16,7 @@
   },
   "dependencies": {
     "@mozilla/readability": "^0.6.0",
+    "@profullstack/x402-client": "0.2.0",
     "fast-xml-parser": "^5.2.5",
     "linkedom": "^0.18.13"
   }

packages/feed/src/fetch.js

@@ -2,6 +2,7 @@ import dns from 'node:dns/promises';
 import net from 'node:net';
 
 import { looksLikeFeed, normalizeUrl, findFeedLinks, guessFeedUrls } from './discover.js';
+import { paidFetch } from './paid.js';
 import { parseFeed } from './parse.js';
 import { findPlaylistLinks } from './playlist.js';
 
@@ -145,7 +146,7 @@ export async function safeFetch(url, conditional = {}) {
       headers[name] = String(value);
     }
 
-    const res = await fetch(normalized, {
+    const res = await paidFetch(normalized, {
       headers,
       redirect: 'follow',
       signal: controller.signal,
@@ -254,7 +255,7 @@ export async function safeFetchBytes(url, maxBytes = 64 * 1024) {
   const timer = setTimeout(() => controller.abort(), TIMEOUT_MS);
 
   try {
-    const res = await fetch(normalized, {
+    const res = await paidFetch(normalized, {
       headers: {
         'user-agent': USER_AGENT,
         accept: 'image/*,*/*',

packages/feed/src/frameable.js

@@ -1,5 +1,6 @@
 import { normalizeUrl } from './discover.js';
 import { isPublicHost } from './fetch.js';
+import { paidFetch } from './paid.js';
 
 /**
  * Whether a page will load inside our reader's iframe.
@@ -271,7 +272,7 @@ async function attempt(normalized, { origin, wantHtml, timeout }) {
   let timer = setTimeout(() => controller.abort(), timeout);
 
   try {
-    const res = await fetch(normalized, {
+    const res = await paidFetch(normalized, {
       method: 'GET',
       headers: { 'user-agent': USER_AGENT, accept: 'text/html,*/*' },
       redirect: 'follow',

packages/feed/src/paid.js

@@ -0,0 +1,24 @@
+/**
+ * The fetch the crawler reaches other people's sites with — paying when asked.
+ *
+ * A site behind an x402 gateway answers a crawler with 402 and an offer. With
+ * `X402_PRIVATE_KEY` set, the client signs the offer with the shared crawler
+ * wallet, buys the pass, files it by origin and presents it on every later
+ * request to that site, so a gated site costs a dollar a day rather than
+ * being silently empty. Without the key this is the global fetch, unchanged.
+ *
+ * The ceiling is five dollars a payment: a hostile or misconfigured offer
+ * cannot drain the wallet by asking. Read through a non-literal accessor
+ * because Next inlines `process.env.NAME` at build time, and this module is
+ * shared by the poller and the web reader.
+ */
+
+import { createClient } from '@profullstack/x402-client';
+
+const key = globalThis.process?.env?.[['X402', 'PRIVATE_KEY'].join('_')];
+
+/** The paying client, or null when no key is configured. */
+export const x402 = key ? createClient({ key, maxUsd: 5 }) : null;
+
+/** @type {typeof fetch} */
+export const paidFetch = x402 ? (input, init) => x402.fetch(input, init) : (input, init) => globalThis.fetch(input, init);

packages/feed/test/paid.test.js

@@ -0,0 +1,23 @@
+import assert from 'node:assert/strict';
+import { createServer } from 'node:http';
+import { test } from 'node:test';
+
+import { paidFetch, x402 } from '../src/paid.js';
+
+test('without X402_PRIVATE_KEY the crawler fetch is the plain fetch', async () => {
+  // The test process has no key; a process with one gets the paying client.
+  assert.equal(x402, null);
+
+  const server = createServer((req, res) => {
+    res.writeHead(200, { 'content-type': 'text/plain' });
+    res.end(`hello ${req.headers['user-agent'] ?? ''}`);
+  });
+  await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve));
+  try {
+    const res = await paidFetch(`http://127.0.0.1:${server.address().port}/`, { headers: { 'user-agent': 'bot/1' } });
+    assert.equal(res.status, 200);
+    assert.equal(await res.text(), 'hello bot/1');
+  } finally {
+    await new Promise((resolve) => server.close(resolve));
+  }
+});

pnpm-workspace.yaml

@@ -14,3 +14,4 @@ allowBuilds:
 minimumReleaseAgeExclude:
   - '@profullstack/player@0.2.0 || 0.3.1'
   - '@profullstack/x402-gateway@0.1.0 || 0.2.1 || 0.3.0'
+  - '@profullstack/x402-client@0.2.0'