fix: harden lastmod validation and support gzipped XML sources (#637)

Author: harlan-zw

src/runtime/server/sitemap/urlset/gzip.ts

@@ -0,0 +1,20 @@
+// gzip magic bytes: 1f 8b. Catches both `.xml.gz` sources and servers that respond
+// with a gzip body for a plain `.xml` URL without a Content-Encoding header (so the
+// runtime fetch layer never gets a chance to auto-decompress it).
+export function looksLikeGzipBytes(bytes: Uint8Array): boolean {
+  return bytes.length >= 2 && bytes[0] === 0x1F && bytes[1] === 0x8B
+}
+
+export async function decodeSitemapResponseBytes(bytes: Uint8Array): Promise<string> {
+  if (!looksLikeGzipBytes(bytes))
+    return new TextDecoder('utf-8').decode(bytes)
+
+  // Blob only accepts an ArrayBuffer-backed view, not the wider ArrayBufferLike
+  // (e.g. SharedArrayBuffer) that a generic Uint8Array parameter admits.
+  const buffer = new Uint8Array(bytes.byteLength)
+  buffer.set(bytes)
+
+  const stream = new Blob([buffer]).stream().pipeThrough(new DecompressionStream('gzip'))
+  const decompressed = await new Response(stream).arrayBuffer()
+  return new TextDecoder('utf-8').decode(decompressed)
+}

src/runtime/server/sitemap/urlset/normalise.ts

@@ -195,13 +195,24 @@ export function normaliseEntry(_e: ResolvedSitemapUrl, defaults?: Omit<SitemapUr
 }
 
 const IS_VALID_W3C_DATE = [
-  /(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d\.\d+([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))/,
+  /^\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d\.\d+([+-][0-2]\d:[0-5]\d|Z)$/,
+  /^\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z)$/,
+  /^\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z)$/,
   /^\d{4}-[01]\d-[0-3]\d$/,
   /^\d{4}-[01]\d$/,
   /^\d{4}$/,
 ]
 export function isValidW3CDate(d: string) {
-  return IS_VALID_W3C_DATE.some(r => r.test(d))
+  if (!IS_VALID_W3C_DATE.some(r => r.test(d)))
+    return false
+  // The shape regex alone accepts impossible dates (e.g. month 13, Feb 30);
+  // reject those explicitly rather than silently normalising garbage.
+  const [year, month, day] = d.slice(0, 10).split('-').map(Number)
+  if (month !== undefined && (month < 1 || month > 12))
+    return false
+  if (day !== undefined && (day < 1 || day > new Date(year!, month!, 0).getDate()))
+    return false
+  return true
 }
 
 export function normaliseDate(date: string | Date): string

src/runtime/server/sitemap/urlset/sources.ts

@@ -12,6 +12,7 @@ import { defu } from 'defu'
 import { getRequestHost } from 'h3'
 import { parseURL } from 'ufo'
 import { logger } from '../../../utils-pure'
+import { decodeSitemapResponseBytes } from './gzip'
 
 export function normalizeSourceInput(source: SitemapSourceInput): SitemapSourceBase | SitemapSourceResolved {
   // string -> { fetch: string, context: { name: 'hook' } }
@@ -73,7 +74,11 @@ export async function fetchDataSource(input: SitemapSourceBase | SitemapSourceRe
 
   try {
     let isMaybeErrorResponse = false
-    const isXmlRequest = parseURL(url).pathname.endsWith('.xml')
+    const pathname = parseURL(url).pathname.toLowerCase()
+    // A `.gz` source (or a `.xml.gz` one) is still an XML sitemap once decompressed;
+    // fetch it as bytes rather than assuming JSON.
+    const isGzUrl = pathname.endsWith('.gz')
+    const isXmlRequest = pathname.endsWith('.xml') || isGzUrl
 
     // Merge external source headers with request headers
     const mergedHeaders = defu(
@@ -86,7 +91,10 @@ export async function fetchDataSource(input: SitemapSourceBase | SitemapSourceRe
 
     const fetchOptions = {
       ...options,
-      responseType: isXmlRequest ? 'text' : 'json',
+      // Fetch XML sources as raw bytes so we can detect and decompress a gzip body
+      // (either a `.gz` URL, or a server that serves gzip without Content-Encoding)
+      // before it's mangled by a UTF-8 text decode.
+      responseType: isXmlRequest ? 'arrayBuffer' : 'json',
       signal: timeoutController.signal,
       headers: mergedHeaders,
       // Use ofetch's built-in retry for external sources
@@ -114,13 +122,24 @@ export async function fetchDataSource(input: SitemapSourceBase | SitemapSourceRe
       }
     }
     let urls = []
-    if (typeof res === 'object') {
-      urls = res.urls || res
-    }
-    else if (typeof res === 'string' && isXmlRequest) {
-      const result = await parseSitemapXml(res)
+    if (isXmlRequest) {
+      const bytes = res instanceof Uint8Array ? res : new Uint8Array(res as ArrayBuffer)
+      const text = await decodeSitemapResponseBytes(bytes)
+      if (text.startsWith('<!DOCTYPE html>')) {
+        return {
+          ...input,
+          context,
+          urls: [],
+          timeTakenMs,
+          error: 'Received HTML response instead of XML',
+        }
+      }
+      const result = await parseSitemapXml(text)
       urls = result.urls
     }
+    else if (typeof res === 'object') {
+      urls = res.urls || res
+    }
     return {
       ...input,
       context,

test/unit/fetchDataSourceGzip.test.ts

@@ -0,0 +1,36 @@
+import { gzipSync } from 'node:zlib'
+import { describe, expect, it } from 'vitest'
+import { decodeSitemapResponseBytes } from '../../src/runtime/server/sitemap/urlset/gzip'
+
+describe('decodeSitemapResponseBytes', () => {
+  it('decompresses a gzip-compressed sitemap body (as served from a .xml.gz source)', async () => {
+    const xml = '<?xml version="1.0" encoding="UTF-8"?><urlset><url><loc>https://example.com/</loc></url></urlset>'
+    const compressed = new Uint8Array(gzipSync(Buffer.from(xml, 'utf-8')))
+
+    const result = await decodeSitemapResponseBytes(compressed)
+    expect(result).toBe(xml)
+  })
+
+  it('decompresses a gzip body served without a .gz extension (detected via magic bytes)', async () => {
+    // Some servers serve a gzip-compressed body for a plain .xml URL without setting
+    // Content-Encoding; the only signal is the 1f 8b magic bytes at the start.
+    const xml = '<?xml version="1.0" encoding="UTF-8"?><urlset><url><loc>https://example.com/a</loc></url></urlset>'
+    const compressed = new Uint8Array(gzipSync(Buffer.from(xml, 'utf-8')))
+
+    const result = await decodeSitemapResponseBytes(compressed)
+    expect(result).toBe(xml)
+  })
+
+  it('passes plain-text XML through untouched', async () => {
+    const xml = '<?xml version="1.0" encoding="UTF-8"?><urlset><url><loc>https://example.com/</loc></url></urlset>'
+    const bytes = new TextEncoder().encode(xml)
+
+    const result = await decodeSitemapResponseBytes(bytes)
+    expect(result).toBe(xml)
+  })
+
+  it('passes through empty input without throwing', async () => {
+    const result = await decodeSitemapResponseBytes(new Uint8Array())
+    expect(result).toBe('')
+  })
+})

test/unit/lastmod.test.ts

@@ -15,4 +15,20 @@ describe('lastmod', () => {
     // time without timezone
     expect(normaliseDate('2023-12-21T13:49:27.963745')).toMatchInlineSnapshot(`"2023-12-21T13:49:27Z"`)
   })
+
+  it('rejects semantically impossible dates that match the shape regex', () => {
+    // Feb 30 doesn't exist; 2024 is a leap year so Feb has 29 days max.
+    expect(isValidW3CDate('2024-02-30')).toBeFalsy()
+    // April has 30 days.
+    expect(isValidW3CDate('2024-04-31')).toBeFalsy()
+    // 2023 is not a leap year.
+    expect(isValidW3CDate('2023-02-29')).toBeFalsy()
+    // month 13 matches the [01]\d shape but isn't a real month.
+    expect(isValidW3CDate('2024-13-01')).toBeFalsy()
+  })
+
+  it('rejects garbage surrounding an otherwise-valid date (regex must be anchored)', () => {
+    expect(isValidW3CDate('garbage2024-01-15T09:30:00Z')).toBeFalsy()
+    expect(isValidW3CDate('2024-01-15T09:30:00Zjunk')).toBeFalsy()
+  })
 })

test/unit/parseSitemapXml.test.ts

@@ -936,4 +936,14 @@ describe('parseSitemapXml', () => {
       expect(warningTypes).toContain('validation')
     })
   })
+
+  it('tolerates a leading UTF-8 byte-order mark before the XML declaration', async () => {
+    const BOM = String.fromCharCode(0xFEFF)
+    const xml = `${BOM}<?xml version="1.0" encoding="UTF-8"?>
+<urlset>
+  <url><loc>http://example.com/</loc></url>
+</urlset>`
+    const result = await parseSitemapXml(xml)
+    expect(result.urls).toEqual([{ loc: 'http://example.com/' }])
+  })
 })

test/unit/sitemapIndex.test.ts

@@ -147,4 +147,18 @@ describe('parseSitemapIndex', () => {
 
     await expect(parseSitemapIndex(xml)).rejects.toThrow('XML does not contain a valid sitemapindex element')
   })
+
+  it('tolerates a leading UTF-8 byte-order mark before the XML declaration', async () => {
+    const BOM = String.fromCharCode(0xFEFF)
+    const xml = `${BOM}<?xml version="1.0" encoding="UTF-8"?>
+<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+  <sitemap>
+    <loc>https://example.com/sitemap.xml</loc>
+  </sitemap>
+</sitemapindex>`
+
+    const { entries, warnings } = await parseSitemapIndex(xml)
+    expect(entries).toEqual([{ loc: 'https://example.com/sitemap.xml' }])
+    expect(warnings).toEqual([])
+  })
 })