feat: log robots.txt disallow to console

viasite · Mar 27, 2020 · 4ef6a8e · 4ef6a8e
1 parent 49180b9
commit 4ef6a8e
Showing 1 changed file with 3 additions and 0 deletions.
diff --git a/src/scrap-site.js b/src/scrap-site.js
@@ -200,6 +200,9 @@ module.exports = async (baseUrl, options = {}) => {
   crawler.on('requestfailed', error => {
     console.error(`Failed: ${error.options.url}`);
   });
+  crawler.on('requestdisallowed', options => {
+    console.error(`Disallowed in robots.txt: ${options.url}`);
+  });
   crawler.on('maxdepthreached', options => {
     console.log(`Max depth reached`);
   });