Merge 4f78425 into 9ad4f40

chrisakroyd · Feb 1, 2023 · 3a5c9ca · 3a5c9ca
2 parents 9ad4f40 + 4f78425
commit 3a5c9ca
Show file tree

Hide file tree

Showing 23 changed files with 61 additions and 53 deletions.
diff --git a/package.json b/package.json
@@ -18,12 +18,13 @@
     "test": "test"
   },
   "scripts": {
-    "test": "mocha test/**/*.js",
+    "test": "npm run lint && mocha test/**/*.js",
     "test-robots": "mocha test/robots/*.js",
     "test-parser": "mocha test/parser/*.js",
     "test-util": "mocha test/util/*.js",
     "coverage": "nyc --reporter=lcov mocha test/**/*.js --timeout 10s",
-    "local-coverage": "nyc mocha test/**/*.js"
+    "local-coverage": "nyc mocha test/**/*.js",
+    "lint": "eslint src test"
   },
   "repository": {
     "type": "git",

diff --git a/src/index.js b/src/index.js
@@ -1,3 +1,3 @@
-const Robots = require('./robots.js');
+const Robots = require('./robots');
 
 module.exports = (opts) => new Robots(opts);
diff --git a/src/parser.js b/src/parser.js
@@ -19,9 +19,7 @@ const cleanSpaces = (rawString) => rawString.replace(whitespace, '').trim();
 
 const splitOnLines = (string) => string.split(lineEndings);
 
-const robustSplit = (string) => {
-  return !string.includes('<html>') ? [...string.match(recordSlices)].map(cleanSpaces) : [];
-};
+const robustSplit = (string) => (!string.includes('<html>') ? [...string.match(recordSlices)].map(cleanSpaces) : []);
 
 const parseRecord = (line) => {
   // Find first colon and assume is the field delimiter.
@@ -70,7 +68,7 @@ const parser = (rawString) => {
   lines.forEach((line) => {
     const record = parseRecord(line);
     switch (record.field) {
-      case USER_AGENT:
+      case USER_AGENT: {
         const recordValue = record.value.toLowerCase();
         if (recordValue !== agent && recordValue.length > 0) {
           // Bot names are non-case sensitive.
@@ -84,6 +82,7 @@ const parser = (rawString) => {
           agent = '';
         }
         break;
+      }
       // https://developers.google.com/webmasters/control-crawl-index/docs/robots_txt#order-of-precedence-for-group-member-records
       case ALLOW:
         if (agent.length > 0 && record.value.length > 0) {

diff --git a/src/robots.js b/src/robots.js
@@ -1,6 +1,6 @@
-const get = require('./get.js');
-const parser = require('./parser.js');
-const util = require('./util.js');
+const get = require('./get');
+const parser = require('./parser');
+const util = require('./util');
 
 const DFLT_OPTS = {
   userAgent: '*',
@@ -25,7 +25,7 @@ class Robots {
     const otherBots = '*' in domainBots;
     if (ourBotInBots) {
       return domainBots[this.opts.userAgent];
-    } else if (otherBots) {
+    } if (otherBots) {
       return domainBots['*'];
     }
     return false;
@@ -44,7 +44,7 @@ class Robots {
 
     if (noDisallows || (allow.maxSpecificity > disallow.maxSpecificity)) {
       return true;
-    } else if (noAllows || (allow.maxSpecificity < disallow.maxSpecificity)) {
+    } if (noAllows || (allow.maxSpecificity < disallow.maxSpecificity)) {
       return false;
     }
     return this.opts.allowOnNeutral;

diff --git a/test/.eslintrc b/test/.eslintrc
@@ -0,0 +1,8 @@
+{
+    "env": {
+        "mocha": true
+    },
+    "rules": {
+        "no-unused-expressions": 0
+    }
+}
diff --git a/test/parser/can-parse-allow.js b/test/parser/can-parse-allow.js
@@ -1,6 +1,6 @@
 const chai = require('chai');
-const allowRobots = require('../test-data/example-robots-txt-allow.js');
-const parse = require('../../src/parser.js');
+const allowRobots = require('../test-data/example-robots-txt-allow');
+const parse = require('../../src/parser');
 
 const { expect } = chai;
 

diff --git a/test/parser/can-parse-crawl-delay.js b/test/parser/can-parse-crawl-delay.js
@@ -1,6 +1,6 @@
 const chai = require('chai');
-const exampleRobotsShort = require('../test-data/example-robots-txt-short.js');
-const parse = require('../../src/parser.js');
+const exampleRobotsShort = require('../test-data/example-robots-txt-short');
+const parse = require('../../src/parser');
 
 const { expect } = chai;
 

diff --git a/test/parser/can-parse-disallow.js b/test/parser/can-parse-disallow.js
@@ -1,6 +1,6 @@
 const chai = require('chai');
-const disallowRobots = require('../test-data/example-robots-txt-disallow.js');
-const parse = require('../../src/parser.js');
+const disallowRobots = require('../test-data/example-robots-txt-disallow');
+const parse = require('../../src/parser');
 
 const { expect } = chai;
 

diff --git a/test/parser/can-parse-sitemaps.js b/test/parser/can-parse-sitemaps.js
@@ -1,7 +1,7 @@
 const chai = require('chai');
 const uniq = require('lodash/uniq');
 const sitemaps = require('../test-data/example-robots-txt-sitemaps');
-const parse = require('../../src/parser.js');
+const parse = require('../../src/parser');
 
 const { expect } = chai;
 

diff --git a/test/parser/can-parse-test-files.js b/test/parser/can-parse-test-files.js
@@ -1,10 +1,10 @@
 const chai = require('chai');
-const exampleRobotsBBC = require('../test-data/example-robots-txt-bbc.js');
-const exampleRobotsBcc = require('../test-data/example-robots-txt-bcc.js');
-const exampleRobotsKarwei = require('../test-data/example-robots-txt-karwei.js');
-const exampleRobotsShort = require('../test-data/example-robots-txt-short.js');
-const exampleRobotsZalando = require('../test-data/example-robots-txt-zalando.js');
-const parse = require('../../src/parser.js');
+const exampleRobotsBBC = require('../test-data/example-robots-txt-bbc');
+const exampleRobotsBcc = require('../test-data/example-robots-txt-bcc');
+const exampleRobotsKarwei = require('../test-data/example-robots-txt-karwei');
+const exampleRobotsShort = require('../test-data/example-robots-txt-short');
+const exampleRobotsZalando = require('../test-data/example-robots-txt-zalando');
+const parse = require('../../src/parser');
 
 const { expect } = chai;
 

diff --git a/test/parser/can-parse-user-agents.js b/test/parser/can-parse-user-agents.js
@@ -1,6 +1,6 @@
 const chai = require('chai');
-const exampleRobotsShort = require('../test-data/example-robots-txt-short.js');
-const parse = require('../../src/parser.js');
+const exampleRobotsShort = require('../test-data/example-robots-txt-short');
+const parse = require('../../src/parser');
 
 const { expect } = chai;
 

diff --git a/test/parser/correct-parse-formatting.js b/test/parser/correct-parse-formatting.js
@@ -1,6 +1,6 @@
 const chai = require('chai');
-const exampleShort = require('../test-data/example-robots-txt-short.js');
-const parse = require('../../src/parser.js');
+const exampleShort = require('../test-data/example-robots-txt-short');
+const parse = require('../../src/parser');
 
 const { expect } = chai;
 

diff --git a/test/parser/ignores-malformed-values.js b/test/parser/ignores-malformed-values.js
@@ -1,6 +1,6 @@
 const chai = require('chai');
-const exampleRobotsMalformed = require('../test-data/example-robots-malformed-short.js');
-const parse = require('../../src/parser.js');
+const exampleRobotsMalformed = require('../test-data/example-robots-malformed-short');
+const parse = require('../../src/parser');
 
 const { expect } = chai;
 

diff --git a/test/robots/can-crawl.js b/test/robots/can-crawl.js
@@ -1,9 +1,9 @@
 const chai = require('chai');
-const robots = require('../../src/index.js');
-const parser = require('../../src/parser.js');
-const util = require('../../src/util.js');
-const testData = require('../test-data/can-crawl-test-data.js');
-const exampleRobotsShort = require('../test-data/example-robots-txt-short.js');
+const robots = require('../../src/index');
+const parser = require('../../src/parser');
+const util = require('../../src/util');
+const testData = require('../test-data/can-crawl-test-data');
+const exampleRobotsShort = require('../test-data/example-robots-txt-short');
 
 const { expect } = chai;
 const robotsParser = robots();

diff --git a/test/robots/fetch.js b/test/robots/fetch.js
@@ -1,5 +1,5 @@
 const chai = require('chai');
-const robots = require('../../src/index.js');
+const robots = require('../../src/index');
 
 const { expect } = chai;
 const robotsParser = robots();

diff --git a/test/robots/get-crawl-delay.js b/test/robots/get-crawl-delay.js
@@ -1,7 +1,7 @@
 const chai = require('chai');
-const robots = require('../../src/index.js');
-const parser = require('../../src/parser.js');
-const exampleRobotsShort = require('../test-data/example-robots-txt-short.js');
+const robots = require('../../src/index');
+const parser = require('../../src/parser');
+const exampleRobotsShort = require('../test-data/example-robots-txt-short');
 
 const { expect } = chai;
 const robotsParser = robots();

diff --git a/test/robots/get-crawlable-links.js b/test/robots/get-crawlable-links.js
@@ -1,6 +1,6 @@
 const chai = require('chai');
-const robots = require('../../src/index.js');
-const exampleRobotsShort = require('../test-data/example-robots-txt-short.js');
+const robots = require('../../src/index');
+const exampleRobotsShort = require('../test-data/example-robots-txt-short');
 
 const { expect } = chai;
 const robotsParser = robots();

diff --git a/test/robots/get-host.js b/test/robots/get-host.js
@@ -1,6 +1,6 @@
 const chai = require('chai');
-const robots = require('../../src/index.js');
-const exampleRobotsShort = require('../test-data/example-robots-txt-short.js');
+const robots = require('../../src/index');
+const exampleRobotsShort = require('../test-data/example-robots-txt-short');
 
 const { expect } = chai;
 const robotsParser = robots();

diff --git a/test/robots/get-sitemaps.js b/test/robots/get-sitemaps.js
@@ -1,7 +1,7 @@
 const chai = require('chai');
-const robots = require('../../src/index.js');
-const parser = require('../../src/parser.js');
-const exampleRobotsShort = require('../test-data/example-robots-txt-short.js');
+const robots = require('../../src/index');
+const parser = require('../../src/parser');
+const exampleRobotsShort = require('../test-data/example-robots-txt-short');
 
 const { expect } = chai;
 const robotsParser = robots();

diff --git a/test/robots/is-cached.js b/test/robots/is-cached.js
@@ -1,6 +1,6 @@
 const chai = require('chai');
-const robots = require('../../src/index.js');
-const exampleRobotsShort = require('../test-data/example-robots-txt-short.js');
+const robots = require('../../src/index');
+const exampleRobotsShort = require('../test-data/example-robots-txt-short');
 
 const { expect } = chai;
 const robotsParser = robots();

diff --git a/test/robots/use-robots-for.js b/test/robots/use-robots-for.js
@@ -1,5 +1,5 @@
 const chai = require('chai');
-const robots = require('../../src/index.js');
+const robots = require('../../src/index');
 
 const { expect } = chai;
 const robotsParser = robots();

diff --git a/test/util/format-link.js b/test/util/format-link.js
@@ -1,5 +1,5 @@
 const chai = require('chai');
-const util = require('../../src/util.js');
+const util = require('../../src/util');
 
 const { expect } = chai;
 

diff --git a/test/util/url-interactions.js b/test/util/url-interactions.js
@@ -1,6 +1,6 @@
 const chai = require('chai');
 const url = require('fast-url-parser');
-const util = require('../../src/util.js');
+const util = require('../../src/util');
 
 const noProtocol = ['bbc.co.uk', 'google.com/robots.txt', 'chaijs.com/api/bdd/', 'www.reddit.com/r/news',
   'example.com/example/example.html'];
@@ -26,7 +26,7 @@ describe('url-interaction', () => {
 
   describe('has-http-protocol', () => {
     it('Expect all links to have a http protocol.', () => {
-      hasHttpsProtocol.forEach((link) => {
+      hasHttpProtocol.forEach((link) => {
         const { protocol } = url.parse(link);
         expect(util.hasHttpProtocol(protocol)).to.be.true;
       });