Merge pull request #400 from robinparisi/master

Fix wrong meta robots tag attribute
simplecrawler · Sep 29, 2017 · 1cb0492 · 1cb0492
2 parents 9c70222 + 6fad8c8
commit 1cb0492
Show file tree

Hide file tree

Showing 3 changed files with 4 additions and 4 deletions.
diff --git a/lib/crawler.js b/lib/crawler.js
@@ -903,7 +903,7 @@ Crawler.prototype.discoverResources = function(resourceText) {
     }
 
     if (crawler.respectRobotsTxt && /<meta(?:\s[^>]*)?\sname\s*=\s*["']?robots["']?[^>]*>/i.test(resourceText)) {
-        var robotsValue = /<meta(?:\s[^>]*)?\svalue\s*=\s*["']?([\w\s,]+)["']?[^>]*>/i.exec(resourceText.toLowerCase());
+        var robotsValue = /<meta(?:\s[^>]*)?\scontent\s*=\s*["']?([\w\s,]+)["']?[^>]*>/i.exec(resourceText.toLowerCase());
 
         if (Array.isArray(robotsValue) && /nofollow/i.test(robotsValue[1])) {
             return [];

diff --git a/test/discovery.js b/test/discovery.js
@@ -203,10 +203,10 @@ describe("Crawler link discovery", function() {
 
     it("should respect nofollow values in robots meta tags", function() {
 
-        discover("<meta name='robots' value='nofollow'><a href='/stage2'>Don't follow me!</a>")
+        discover("<meta name='robots' content='nofollow'><a href='/stage2'>Don't follow me!</a>")
             .should.eql([]);
 
-        discover("<meta name='robots' value='nofollow, noindex'><a href='/stage2'>Don't follow me!</a>")
+        discover("<meta name='robots' content='nofollow, noindex'><a href='/stage2'>Don't follow me!</a>")
             .should.eql([]);
     });
 });
diff --git a/test/lib/routes.js b/test/lib/routes.js
@@ -50,7 +50,7 @@ module.exports = {
     },
 
     "/nofollow": function(write) {
-        write(200, "<meta name='robots' value='nofollow'><a href='/stage7'>Don't go here!</a>");
+        write(200, "<meta name='robots' content='nofollow'><a href='/stage7'>Don't go here!</a>");
     },
 
     "/cookie": function(write) {