fix(validator): handle Content-Usage/Content-Signal with spaces after commas

Fixes validation for comma-separated preferences like "search=y, train-ai=n"
which were incorrectly treated as path-based rules due to whitespace splitting.

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

Author: harlan-zw

src/util.ts

@@ -215,20 +215,17 @@ function validateGroupRules(group: ParsedRobotsTxt['groups'][number], errors: st
         })
       }
       else if (parts.length >= 2) {
-        // Path-specific preference like "/path train-ai=n"
-        const path = parts[0]
-        const preference = parts.slice(1).join(' ')
-
-        if (!path?.startsWith('/')) {
-          errors.push(`Content-Usage path "${path}" must start with a \`/\`.`)
-        }
-        if (!preference.includes('=')) {
-          errors.push(`Content-Usage preference "${preference}" must contain an assignment (e.g., "train-ai=n").`)
-        }
-        else {
-          // Validate category and value in path-specific rules
-          const preferences = preference.split(',').map(p => p.trim())
-          preferences.forEach((pref) => {
+        const firstPart = parts[0]
+        // Check if first part is a preference (contains =) vs a path (starts with /)
+        // This handles comma-separated values with spaces like "search=y, train-ai=n"
+        if (firstPart?.includes('=')) {
+          // Global preferences with spaces around commas - validate each preference
+          const allPreferences = rule.split(',').map(p => p.trim())
+          allPreferences.forEach((pref) => {
+            if (!pref.includes('=')) {
+              errors.push(`Content-Usage rule "${pref}" must contain a preference assignment (e.g., "train-ai=n").`)
+              return
+            }
             const [category, value] = pref.split('=').map(s => s.trim())
             if (!validCategories.includes(category || '')) {
               errors.push(`Content-Usage category "${category}" is invalid. Valid categories: ${validCategories.join(', ')}.`)
@@ -238,6 +235,31 @@ function validateGroupRules(group: ParsedRobotsTxt['groups'][number], errors: st
             }
           })
         }
+        else {
+          // Path-specific preference like "/path train-ai=n"
+          const path = firstPart
+          const preference = parts.slice(1).join(' ')
+
+          if (!path?.startsWith('/')) {
+            errors.push(`Content-Usage path "${path}" must start with a \`/\`.`)
+          }
+          if (!preference.includes('=')) {
+            errors.push(`Content-Usage preference "${preference}" must contain an assignment (e.g., "train-ai=n").`)
+          }
+          else {
+            // Validate category and value in path-specific rules
+            const preferences = preference.split(',').map(p => p.trim())
+            preferences.forEach((pref) => {
+              const [category, value] = pref.split('=').map(s => s.trim())
+              if (!validCategories.includes(category || '')) {
+                errors.push(`Content-Usage category "${category}" is invalid. Valid categories: ${validCategories.join(', ')}.`)
+              }
+              if (!validValues.includes(value || '')) {
+                errors.push(`Content-Usage value "${value}" for "${category}" is invalid. Valid values: y, n.`)
+              }
+            })
+          }
+        }
       }
     })
   }
@@ -276,20 +298,17 @@ function validateGroupRules(group: ParsedRobotsTxt['groups'][number], errors: st
         })
       }
       else if (parts.length >= 2) {
-        // Path-specific preference like "/path ai-train=no"
-        const path = parts[0]
-        const preference = parts.slice(1).join(' ')
-
-        if (!path?.startsWith('/')) {
-          errors.push(`Content-Signal path "${path}" must start with a \`/\`.`)
-        }
-        if (!preference.includes('=')) {
-          errors.push(`Content-Signal preference "${preference}" must contain an assignment (e.g., "ai-train=no").`)
-        }
-        else {
-          // Validate category and value in path-specific rules
-          const preferences = preference.split(',').map(p => p.trim())
-          preferences.forEach((pref) => {
+        const firstPart = parts[0]
+        // Check if first part is a preference (contains =) vs a path (starts with /)
+        // This handles comma-separated values with spaces like "search=yes, ai-train=no"
+        if (firstPart?.includes('=')) {
+          // Global preferences with spaces around commas - validate each preference
+          const allPreferences = rule.split(',').map(p => p.trim())
+          allPreferences.forEach((pref) => {
+            if (!pref.includes('=')) {
+              errors.push(`Content-Signal rule "${pref}" must contain a preference assignment (e.g., "ai-train=no").`)
+              return
+            }
             const [category, value] = pref.split('=').map(s => s.trim())
             if (!validCategories.includes(category || '')) {
               errors.push(`Content-Signal category "${category}" is invalid. Valid categories: ${validCategories.join(', ')}.`)
@@ -299,6 +318,31 @@ function validateGroupRules(group: ParsedRobotsTxt['groups'][number], errors: st
             }
           })
         }
+        else {
+          // Path-specific preference like "/path ai-train=no"
+          const path = firstPart
+          const preference = parts.slice(1).join(' ')
+
+          if (!path?.startsWith('/')) {
+            errors.push(`Content-Signal path "${path}" must start with a \`/\`.`)
+          }
+          if (!preference.includes('=')) {
+            errors.push(`Content-Signal preference "${preference}" must contain an assignment (e.g., "ai-train=no").`)
+          }
+          else {
+            // Validate category and value in path-specific rules
+            const preferences = preference.split(',').map(p => p.trim())
+            preferences.forEach((pref) => {
+              const [category, value] = pref.split('=').map(s => s.trim())
+              if (!validCategories.includes(category || '')) {
+                errors.push(`Content-Signal category "${category}" is invalid. Valid categories: ${validCategories.join(', ')}.`)
+              }
+              if (!validValues.includes(value || '')) {
+                errors.push(`Content-Signal value "${value}" for "${category}" is invalid. Valid values: yes, no.`)
+              }
+            })
+          }
+        }
       }
     })
   }

test/unit/robotsTxtValidator.test.ts

@@ -114,4 +114,48 @@ describe('robotsTxtValidator', () => {
       ]
     `)
   })
+
+  it('content-usage with comma-separated values and spaces', () => {
+    // Regression test: "search=y, train-ai=y" should not be treated as path-based
+    const { errors } = validateRobots({
+      errors: [],
+      sitemaps: [],
+      groups: [
+        {
+          allow: ['/'],
+          comment: [],
+          disallow: [],
+          userAgent: ['*'],
+          contentUsage: [
+            'search=y, train-ai=y', // space after comma - should be valid
+            'search=y,train-ai=n', // no space - should be valid
+            'bots=y, ai-output=n, search=y', // multiple with spaces
+          ],
+        },
+      ],
+    })
+    expect(errors).toEqual([])
+  })
+
+  it('content-signal with comma-separated values and spaces', () => {
+    // Regression test: "search=yes, ai-train=no" should not be treated as path-based
+    const { errors } = validateRobots({
+      errors: [],
+      sitemaps: [],
+      groups: [
+        {
+          allow: ['/'],
+          comment: [],
+          disallow: [],
+          userAgent: ['*'],
+          contentSignal: [
+            'search=yes, ai-train=no', // space after comma - should be valid
+            'search=yes,ai-input=no', // no space - should be valid
+            'search=yes, ai-input=no, ai-train=yes', // multiple with spaces
+          ],
+        },
+      ],
+    })
+    expect(errors).toEqual([])
+  })
 })