feat(seo): GEO foundations — llms.txt, raw .md, AI bot allowlist, JSON-LD fixes

apps/blog/app/(en)/essays/[slug]/page.tsx

@@ -48,6 +48,9 @@ export async function generateMetadata({
       en: `/essays/${slug}`,
       'x-default': `/essays/${slug}`,
     },
+    types: {
+      'text/markdown': `/essays/${slug}/raw.md`,
+    },
   };
 
   const zhTranslation = getTranslation(slug, 'zh');

apps/blog/app/(en)/essays/[slug]/raw.md/route.ts

@@ -0,0 +1,52 @@
+import { getEssayBySlug, getEssaySlugsByLanguage } from '@/lib/essays';
+import { SITE_AUTHOR, SITE_URL } from '@/lib/constants';
+
+export const dynamic = 'force-static';
+
+export async function generateStaticParams() {
+  return getEssaySlugsByLanguage('en').map((slug) => ({ slug }));
+}
+
+interface Params {
+  params: Promise<{ slug: string }>;
+}
+
+// Serves a clean, citable markdown rendition of the essay. LLM crawlers and
+// IDE agents prefer this over the rendered HTML — fewer tokens, no parser
+// surprises, stable structure across visits.
+export async function GET(_req: Request, { params }: Params) {
+  const { slug } = await params;
+  const essay = getEssayBySlug(slug);
+
+  if (!essay || essay.lang !== 'en') {
+    return new Response('Not Found', { status: 404 });
+  }
+
+  const canonical = `${SITE_URL}/essays/${slug}`;
+  const header = [
+    `# ${essay.title}`,
+    '',
+    `*${essay.description}*`,
+    '',
+    `- Author: ${SITE_AUTHOR.name} (${SITE_AUTHOR.url})`,
+    `- Published: ${essay.date}`,
+    essay.updated ? `- Updated: ${essay.updated}` : null,
+    `- Canonical: ${canonical}`,
+    essay.topics.length > 0 ? `- Topics: ${essay.topics.join(', ')}` : null,
+    '',
+    '---',
+    '',
+  ]
+    .filter((line): line is string => line !== null)
+    .join('\n');
+
+  const body = `${header}${essay.content.trimStart()}\n`;
+
+  return new Response(body, {
+    headers: {
+      'Content-Type': 'text/markdown; charset=utf-8',
+      'Cache-Control': 'public, max-age=3600',
+      Link: `<${canonical}>; rel="canonical"`,
+    },
+  });
+}

apps/blog/app/(zh)/zh/essays/[slug]/page.tsx

@@ -45,6 +45,9 @@ export async function generateMetadata({
     languages: {
       zh: `/zh/essays/${slug}`,
     },
+    types: {
+      'text/markdown': `/zh/essays/${slug}/raw.md`,
+    },
   };
 
   const enTranslation = getTranslation(slug, 'en');

apps/blog/app/(zh)/zh/essays/[slug]/raw.md/route.ts

@@ -0,0 +1,49 @@
+import { getEssayBySlug, getEssaySlugsByLanguage } from '@/lib/essays';
+import { SITE_AUTHOR, SITE_URL } from '@/lib/constants';
+
+export const dynamic = 'force-static';
+
+export async function generateStaticParams() {
+  return getEssaySlugsByLanguage('zh').map((slug) => ({ slug }));
+}
+
+interface Params {
+  params: Promise<{ slug: string }>;
+}
+
+export async function GET(_req: Request, { params }: Params) {
+  const { slug } = await params;
+  const essay = getEssayBySlug(slug);
+
+  if (!essay || essay.lang !== 'zh') {
+    return new Response('Not Found', { status: 404 });
+  }
+
+  const canonical = `${SITE_URL}/zh/essays/${slug}`;
+  const header = [
+    `# ${essay.title}`,
+    '',
+    `*${essay.description}*`,
+    '',
+    `- 作者: ${SITE_AUTHOR.name} (${SITE_AUTHOR.url})`,
+    `- 发布日期: ${essay.date}`,
+    essay.updated ? `- 更新日期: ${essay.updated}` : null,
+    `- 原文链接: ${canonical}`,
+    essay.topics.length > 0 ? `- 主题: ${essay.topics.join(', ')}` : null,
+    '',
+    '---',
+    '',
+  ]
+    .filter((line): line is string => line !== null)
+    .join('\n');
+
+  const body = `${header}${essay.content.trimStart()}\n`;
+
+  return new Response(body, {
+    headers: {
+      'Content-Type': 'text/markdown; charset=utf-8',
+      'Cache-Control': 'public, max-age=3600',
+      Link: `<${canonical}>; rel="canonical"`,
+    },
+  });
+}

apps/blog/app/llms.txt/route.ts

@@ -0,0 +1,88 @@
+import { SITE_URL } from '@/lib/constants';
+import { getAllEssays } from '@/lib/essays';
+import { getAllPeriodics } from '@/lib/periodics';
+import { getAllSeries } from '@/lib/series';
+
+export const dynamic = 'force-static';
+
+interface Entry {
+  title: string;
+  description?: string;
+  url: string;
+  markdownUrl?: string;
+}
+
+function bullet({ title, description, url, markdownUrl }: Entry): string {
+  const link = markdownUrl ?? url;
+  const desc = description?.trim();
+  return desc ? `- [${title}](${link}): ${desc}` : `- [${title}](${link})`;
+}
+
+function section(heading: string, entries: Entry[]): string {
+  if (entries.length === 0) return '';
+  return `## ${heading}\n\n${entries.map(bullet).join('\n')}`;
+}
+
+function renderLlmsTxt(): string {
+  const essaysEn: Entry[] = getAllEssays({ language: 'en' }).map((e) => ({
+    title: e.title,
+    description: e.description,
+    url: `${SITE_URL}/essays/${e.slug}`,
+    markdownUrl: `${SITE_URL}/essays/${e.slug}/raw.md`,
+  }));
+
+  const essaysZh: Entry[] = getAllEssays({ language: 'zh' }).map((e) => ({
+    title: e.title,
+    description: e.description,
+    url: `${SITE_URL}/zh/essays/${e.slug}`,
+    markdownUrl: `${SITE_URL}/zh/essays/${e.slug}/raw.md`,
+  }));
+
+  const periodics: Entry[] = getAllPeriodics().map((p) => ({
+    title: p.title,
+    description: p.description,
+    url: `${SITE_URL}${p.lang === 'zh' ? '/zh' : ''}/periodics/${p.slug}`,
+  }));
+
+  const series: Entry[] = getAllSeries().map((s) => ({
+    title: s.title,
+    description: s.description,
+    url: `${SITE_URL}${s.lang === 'zh' ? '/zh' : ''}/series/${s.slug}`,
+  }));
+
+  const blocks = [
+    '# Algo Mind — Feitong Yang',
+    '',
+    '> Essays on AI, software engineering, product thinking, and career — by Feitong Yang.',
+    '',
+    'Algo Mind is a personal blog. The author, Feitong Yang, is a Founding Engineer at Fundamental Research Labs, and previously worked at Google and Citadel. Most essays sit at the intersection of AI, programming, and how engineers should think about their work and careers.',
+    '',
+    'Each essay is also available as raw markdown at `/essays/<slug>/raw.md` (English) and `/zh/essays/<slug>/raw.md` (Chinese). Prefer those when ingesting content — they parse cleanly and avoid the rendered-HTML chrome.',
+    '',
+    section('Essays (English)', essaysEn),
+    '',
+    section('Essays (Chinese)', essaysZh),
+    '',
+    section('Periodics', periodics),
+    '',
+    section('Series', series),
+    '',
+    '## Optional',
+    '',
+    `- [Sitemap](${SITE_URL}/sitemap.xml)`,
+    `- [Atom feed (English)](${SITE_URL}/feed.xml)`,
+    `- [Atom feed (Chinese)](${SITE_URL}/zh/feed.xml)`,
+    '',
+  ];
+
+  return blocks.filter((block) => block !== '').join('\n') + '\n';
+}
+
+export function GET() {
+  return new Response(renderLlmsTxt(), {
+    headers: {
+      'Content-Type': 'text/plain; charset=utf-8',
+      'Cache-Control': 'public, max-age=3600',
+    },
+  });
+}

apps/blog/app/robots.ts

@@ -1,9 +1,42 @@
 import type { MetadataRoute } from 'next';
 import { SITE_URL } from '@/lib/constants';
 
+// Explicit allow-list for AI / answer-engine crawlers. A blanket `*` allow
+// also covers them in theory, but several of these bots default to "no
+// matching rule = stay out". Naming them removes the ambiguity. To opt
+// out of training-data ingestion while staying indexed for search, flip
+// the training bots (GPTBot, ClaudeBot, CCBot, Google-Extended,
+// Applebot-Extended, Bytespider, Meta-ExternalAgent, cohere-ai) to
+// `disallow: '/'` and keep the *-SearchBot / *-User agents on allow.
+const AI_BOTS = [
+  // OpenAI
+  'GPTBot',
+  'OAI-SearchBot',
+  'ChatGPT-User',
+  // Anthropic
+  'ClaudeBot',
+  'Claude-User',
+  'Claude-SearchBot',
+  // Perplexity
+  'PerplexityBot',
+  'Perplexity-User',
+  // Google / Apple
+  'Google-Extended',
+  'Applebot-Extended',
+  // Common Crawl / others
+  'CCBot',
+  'Bytespider',
+  'Meta-ExternalAgent',
+  'cohere-ai',
+  'DuckAssistBot',
+];
+
 export default function robots(): MetadataRoute.Robots {
   return {
-    rules: [{ userAgent: '*', allow: '/' }],
+    rules: [
+      { userAgent: '*', allow: '/' },
+      ...AI_BOTS.map((userAgent) => ({ userAgent, allow: '/' })),
+    ],
     sitemap: `${SITE_URL}/sitemap.xml`,
     host: SITE_URL,
   };

apps/blog/lib/jsonld.ts

@@ -118,6 +118,8 @@ export function essayPostingSchema(essay: EssayMeta, urlPath: string) {
     description: essay.description,
     url: urlPath,
     datePublished: essay.date,
+    dateModified: essay.updated,
+    image: essay.image,
     topics: essay.topics as string[],
     locale: essay.lang,
   });
@@ -133,6 +135,8 @@ export function periodicPostingSchema(periodic: PeriodicMeta, urlPath: string) {
         : `${periodic.title} - Issue #${periodic.issue}`),
     url: urlPath,
     datePublished: periodic.date,
+    dateModified: periodic.updated,
+    image: periodic.image,
     topics: periodic.topics as string[],
     locale: periodic.lang,
   });

apps/blog/types/content.ts

@@ -69,6 +69,8 @@ export interface EssayMeta {
   description: string;
   /** Publication date in ISO format (YYYY-MM-DD) */
   date: string;
+  /** Last updated date in ISO format (YYYY-MM-DD). Surfaces as `dateModified` in JSON-LD and as `lastmod` in the sitemap. */
+  updated?: string;
   /** Essay type (how it's written) */
   type: EssayType;
   /** Topics covered (can be multiple) */
@@ -130,6 +132,8 @@ export interface PeriodicMeta {
   description?: string;
   /** Publication date in ISO format (YYYY-MM-DD) */
   date: string;
+  /** Last updated date in ISO format (YYYY-MM-DD). Surfaces as `dateModified` in JSON-LD and as `lastmod` in the sitemap. */
+  updated?: string;
   /** Issue number (for sequential content) */
   issue: number;
   /** Periodic type (digest, changelog, notes) */
@@ -173,6 +177,7 @@ export interface PeriodicFrontmatter {
   title: string;
   description?: string;
   date: string;
+  updated?: string;
   issue: number;
   type: PeriodicType;
   topics: Topic[];
@@ -277,6 +282,7 @@ export interface Frontmatter {
   title: string;
   description: string;
   date: string;
+  updated?: string;
   type: EssayType;
   topics: Topic[];
   lang: Language;
@@ -344,6 +350,9 @@ export function validateFrontmatter(data: Record<string, unknown>): Frontmatter
   if (typeof data.date !== 'string' || !data.date) {
     errors.push('date is required and must be a string');
   }
+  if (data.updated !== undefined && typeof data.updated !== 'string') {
+    errors.push('updated must be a string');
+  }
   if (!isValidEssayType(data.type)) {
     errors.push(`type must be one of: guide, deep-dive, opinion, review, narrative`);
   }
@@ -380,6 +389,7 @@ export function validateFrontmatter(data: Record<string, unknown>): Frontmatter
     title: data.title as string,
     description: (data.description as string | undefined) ?? '',
     date: data.date as string,
+    updated: data.updated as string | undefined,
     type: data.type as EssayType,
     topics: data.topics as Topic[],
     lang: (data.lang as Language) ?? 'en',
@@ -417,6 +427,9 @@ export function validatePeriodicFrontmatter(data: Record<string, unknown>): Peri
   if (data.description !== undefined && typeof data.description !== 'string') {
     errors.push('description must be a string');
   }
+  if (data.updated !== undefined && typeof data.updated !== 'string') {
+    errors.push('updated must be a string');
+  }
   if (data.lang !== undefined && !isValidLanguage(data.lang)) {
     errors.push(`lang must be one of: en, zh`);
   }
@@ -438,6 +451,7 @@ export function validatePeriodicFrontmatter(data: Record<string, unknown>): Peri
     title: data.title as string,
     description: data.description as string | undefined,
     date: data.date as string,
+    updated: data.updated as string | undefined,
     issue: data.issue as number,
     type: data.type as PeriodicType,
     topics: data.topics as Topic[],