fix: --sitemaps option will override robots.txt sitemaps

Fixes #169
harlan-zw · Mar 3, 2024 · 8f01db0 · 8f01db0
1 parent 3ec2531
commit 8f01db0
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 3 deletions.
diff --git a/packages/cli/src/createCli.ts b/packages/cli/src/createCli.ts
@@ -21,7 +21,7 @@ export default function createCli() {
 
   cli.option('--site <site>', 'Host URL to scan.')
   cli.option('--router-prefix <site>', 'The URL path prefix for the client and API to run from.')
-  cli.option('--sitemaps <sitemaps>', 'Comma separated list of sitemaps to use for scanning.')
+  cli.option('--sitemaps <sitemaps>', 'Comma separated list of sitemaps to use for scanning. Providing these will override any in robots.txt.')
   cli.option('--samples <samples>', 'Specify the amount of samples to run.')
   cli.option('--throttle', 'Enable the throttling')
   cli.option('--enable-javascript', 'When inspecting the HTML wait for the javascript to execute. Useful for SPAs.')

diff --git a/packages/core/src/discovery/robotsTxt.ts b/packages/core/src/discovery/robotsTxt.ts
@@ -80,6 +80,9 @@ export function mergeRobotsTxtConfig(config: ResolvedUserConfig, { groups, sitem
       ...normalisedGroups.flatMap(group => group.allow),
     ])].filter(isValidRegex)
   }
-  if (config.scanner.sitemap !== false && sitemaps.length)
-    config.scanner.sitemap = [...new Set([...(Array.isArray(config.scanner.sitemap) ? config.scanner.sitemap : []), ...sitemaps])]
+  if (config.scanner.sitemap !== false && sitemaps.length) {
+    // allow overriding the robots.txt sitemaps with your own
+    if (!Array.isArray(config.scanner.sitemap) || !config.scanner.sitemap.length)
+      config.scanner.sitemap = [...new Set([...(Array.isArray(config.scanner.sitemap) ? config.scanner.sitemap : []), ...sitemaps])]
+  }
 }