feat: generate markdown static files for LLM agent token optimization

.circleci/config.yml

@@ -81,6 +81,17 @@ jobs:
           name: Validate llms.txt
           command: yarn validate-llms-txt
 
+  validate-markdown:
+    executor:
+      name: default
+    steps:
+      - checkout
+      - attach_workspace:
+          at: .
+      - run:
+          name: Validate markdown files
+          command: yarn validate-markdown
+
   test-nginx:
     docker:
       - image: heroku/heroku:24-build
@@ -155,3 +166,6 @@ workflows:
       - validate-llms-txt:
           requires:
             - build
+      - validate-markdown:
+          requires:
+            - build

README.md

@@ -22,6 +22,41 @@ To run the docs site locally, run `bin/dev` from the root directory. Alternative
 
 View the [contribution guide](CONTRIBUTING.md) for information on how to write content and contribute to Ably docs.
 
+## Markdown Static Files
+
+The build process generates both HTML and Markdown versions of each documentation page. This provides a more token-efficient format for LLM crawlers and API clients.
+
+### Content Negotiation
+
+The site supports content negotiation via the `Accept` header:
+
+```bash
+# Request markdown version
+curl -H "Accept: text/markdown" https://ably.com/docs/channels
+
+# Request HTML version (default)
+curl https://ably.com/docs/channels
+```
+
+Markdown files are located at `/docs/{page-path}/index.md` alongside their HTML counterparts at `/docs/{page-path}/index.html`.
+
+### Build Process
+
+1. **Source**: Content is written in Textile or MDX format
+2. **HTML Generation**: Gatsby converts source files to static HTML
+3. **Markdown Generation**: The `generateMarkdown` post-build hook converts HTML to clean Markdown
+4. **Compression**: Both HTML and Markdown files are gzip compressed
+
+### Validation
+
+Validate markdown generation after building:
+
+```bash
+yarn validate-markdown
+```
+
+This ensures all HTML pages have corresponding Markdown files and reports any issues.
+
 ## Support
 
 If you have any questions or suggestions, please [raise an issue](https://github.com/ably/docs/issues).

bin/validate-markdown.ts

@@ -0,0 +1,119 @@
+#!/usr/bin/env node
+
+/**
+ * Validates that markdown files exist for all HTML pages in the public directory.
+ * This script ensures the markdown generation process completed successfully.
+ */
+
+import * as fs from 'fs';
+import * as path from 'path';
+import fastGlob from 'fast-glob';
+
+const publicDir = path.join(process.cwd(), 'public', 'docs');
+
+// Constants for content validation (must match generateMarkdown.ts)
+const REDIRECT_PAGE_MAX_SIZE = 1000; // Maximum size in bytes for redirect pages
+
+interface ValidationResult {
+  totalPages: number;
+  markdownFound: number;
+  markdownMissing: number;
+  redirectPages: number;
+  missingFiles: string[];
+}
+
+const validateMarkdownFiles = async (): Promise<ValidationResult> => {
+  // Find all index.html files in the docs directory
+  const htmlFiles = await fastGlob('**/index.html', {
+    cwd: publicDir,
+    absolute: false,
+  });
+
+  const result: ValidationResult = {
+    totalPages: htmlFiles.length,
+    markdownFound: 0,
+    markdownMissing: 0,
+    redirectPages: 0,
+    missingFiles: [],
+  };
+
+  for (const htmlFile of htmlFiles) {
+    // Get the directory of the HTML file
+    const dir = path.dirname(htmlFile);
+
+    // Check if this is a redirect page (skip validation for these)
+    const htmlPath = path.join(publicDir, htmlFile);
+    const htmlContent = fs.readFileSync(htmlPath, 'utf8');
+
+    if (htmlContent.length < REDIRECT_PAGE_MAX_SIZE && /<script>window\.location\.href=/.test(htmlContent)) {
+      result.redirectPages++;
+      continue; // Skip redirect pages
+    }
+
+    // Check if corresponding markdown file exists
+    const markdownFile = dir === '.'
+      ? path.join(publicDir, 'index.md')
+      : path.join(publicDir, `${dir}.md`);
+
+    if (fs.existsSync(markdownFile)) {
+      result.markdownFound++;
+
+      // Verify the markdown file has content
+      const stats = fs.statSync(markdownFile);
+      if (stats.size === 0) {
+        console.warn(`⚠️  Warning: ${markdownFile} is empty`);
+      }
+    } else {
+      result.markdownMissing++;
+      result.missingFiles.push(dir);
+    }
+  }
+
+  return result;
+};
+
+const main = async () => {
+  console.log('🔍 Validating markdown files...\n');
+
+  if (!fs.existsSync(publicDir)) {
+    console.error(`❌ Error: Public docs directory not found: ${publicDir}`);
+    console.error('   Make sure to run this script after the build process.');
+    process.exit(1);
+  }
+
+  try {
+    const result = await validateMarkdownFiles();
+
+    console.log(`📊 Validation Results:`);
+    console.log(`   Total HTML pages: ${result.totalPages}`);
+    console.log(`   🔀 Redirect pages (skipped): ${result.redirectPages}`);
+    console.log(`   📄 Content pages: ${result.totalPages - result.redirectPages}`);
+    console.log(`   ✅ Markdown files found: ${result.markdownFound}`);
+    console.log(`   ❌ Markdown files missing: ${result.markdownMissing}`);
+
+    if (result.markdownMissing > 0) {
+      console.log('\n⚠️  Missing markdown files:');
+      result.missingFiles.slice(0, 10).forEach((dir) => {
+        const mdPath = dir === '.' ? 'index.md' : `${dir}.md`;
+        console.log(`   - ${mdPath}`);
+      });
+
+      if (result.missingFiles.length > 10) {
+        console.log(`   ... and ${result.missingFiles.length - 10} more`);
+      }
+
+      console.log('\n❌ Validation failed: Some markdown files are missing.');
+      console.log('   This may indicate an issue with the markdown generation process.');
+      process.exit(1);
+    }
+
+    // Calculate coverage percentage
+    const coverage = (result.markdownFound / result.totalPages) * 100;
+    console.log(`\n✅ Validation passed! Markdown coverage: ${coverage.toFixed(1)}%`);
+  } catch (error) {
+    console.error('❌ Error during validation:', error);
+    process.exit(1);
+  }
+};
+
+main();

config/mime.types

@@ -11,6 +11,7 @@ types {
 
     text/mathml                                      mml;
     text/plain                                       txt;
+    text/markdown                                    md markdown;
     text/vnd.sun.j2me.app-descriptor                 jad;
     text/vnd.wap.wml                                 wml;
     text/x-component                                 htc;

config/nginx.conf.erb

@@ -18,7 +18,7 @@ http {
   gzip on;
   gzip_comp_level 6;
   gzip_min_length 512;
-  gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss font/woff font/woff2 image/svg+xml;
+  gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss font/woff font/woff2 image/svg+xml text/markdown;
   gzip_vary on;
   gzip_proxied any; # Heroku router sends Via header
 
@@ -126,6 +126,15 @@ http {
 
   # / PROTECTED CONTENT REQUESTS
 
+  ##
+  # CONTENT NEGOTIATION FOR MARKDOWN
+  # Check if the client accepts markdown by looking for text/markdown in the Accept header
+  map $http_accept $prefers_markdown {
+    default "no";
+    "~*text/markdown" "yes";
+  }
+  # / CONTENT NEGOTIATION FOR MARKDOWN
+
   server {
     listen <%= ENV["PORT"] %>;
     charset UTF-8;
@@ -230,13 +239,42 @@ http {
       <% if content_request_protected %>
       # Serve the file if it exists, otherwise try to authenticate
       # (.html requests won't match here, they'll go to the @html_auth location)
+      # If client prefers markdown and is authenticated, serve markdown
+      if ($prefers_markdown = "yes") {
+        rewrite ^ @markdown_request last;
+      }
       try_files $request_uri @html_auth;
       <% else %>
+      # If client prefers markdown, serve markdown instead of HTML
+      if ($prefers_markdown = "yes") {
+        rewrite ^ @markdown_request last;
+      }
       # Serve the file if it exists, try index.html for paths without a trailing slash, otherwise 404
       try_files $request_uri $request_uri/index.html $request_uri/ =404;
       <% end %>
     }
 
+    # Serve markdown files with content negotiation
+    location @markdown_request {
+      <% if content_request_protected %>
+      # Check authentication for markdown requests
+      if ($token_auth_status != "allowed") {
+        <% if host = ENV['CONTENT_REQUEST_CANONICAL_HOST'] %>
+        return 301 <%= ENV['SKIP_HTTPS'] == 'true' ? '$scheme' : 'https' %>://<%= host %>$request_uri;
+        <% else %>
+        return 404;
+        <% end %>
+      }
+      <% end %>
+
+      # Set proper content type for markdown
+      more_set_headers 'Content-Type: text/markdown; charset=utf-8';
+      more_set_headers 'Vary: Accept';
+
+      # Try to serve the markdown file, fall back to HTML if not available
+      try_files $request_uri/index.md $request_uri.md $request_uri/index.html $request_uri/ =404;
+    }
+
     <% if content_request_protected %>
     # Authenticate .html requests by checking the token_auth_status variable
     # which is set in the map block earlier in this file.