Added --internal cli switch that avoids checking any crossorigin link…

…s. This allows you to do quick checks on your own sites internal structure
Munter · Jul 29, 2018 · 5fbce69 · 5fbce69
1 parent 177aaae
commit 5fbce69
Show file tree

Hide file tree

Showing 4 changed files with 97 additions and 24 deletions.
diff --git a/lib/cli.js b/lib/cli.js
@@ -34,6 +34,11 @@ const commandLineOptions = optimist
       'Crawl all HTML-pages linked with relative and root relative links. This stays inside your domain.',
     type: 'boolean'
   })
+  .options('internal', {
+    alias: 'i',
+    describe: 'Only check links to assets within your own web root',
+    type: 'boolean'
+  })
   .options('source-maps', {
     describe:
       'Verify the correctness of links to source map files and sources.',
@@ -138,6 +143,7 @@ t.pipe(process.stdout);
         inputUrls: inputUrls,
         followSourceMaps: followSourceMaps,
         recursive: commandLineOptions.recursive,
+        internalOnly: commandLineOptions.internal,
         skipFilter,
         todoFilter,
         verbose: commandLineOptions.verbose,

diff --git a/lib/index.js b/lib/index.js
@@ -53,6 +53,7 @@ async function hyperlink(
     skipFilter = returnFalse,
     todoFilter = returnFalse,
     recursive = false,
+    internalOnly = false,
     followSourceMaps = false,
     verbose = false,
     memdebug = false,
@@ -595,11 +596,13 @@ async function hyperlink(
             ) {
               follow = true;
               relation.to.stopProcessing = true;
-            } else if (relation.fragment && relation.fragment !== '#') {
-              follow = true;
-              relation.to.stopProcessing = true;
-            } else {
-              relation.to.check = true;
+            } else if (!internalOnly) {
+              if (relation.fragment && relation.fragment !== '#') {
+                follow = true;
+                relation.to.stopProcessing = true;
+              } else {
+                relation.to.check = true;
+              }
             }
           }
         } else if (
@@ -668,6 +671,10 @@ async function hyperlink(
   // Check fragments
 
   for (const asset of ag.findAssets()) {
+    if (internalOnly && !asset.isLoaded) {
+      continue;
+    }
+
     if (asset.incomingFragments) {
       for (const {
         fragment,
@@ -702,26 +709,28 @@ async function hyperlink(
   }
 
   // Check urls
-  const assetsToCheck = ag
-    .findAssets({ check: true })
-    .filter(asset => !processedAssets.has(asset));
-  t.push({
-    name: `Crawling ${assetsToCheck.length} outgoing urls`
-  });
+  if (!internalOnly) {
+    const assetsToCheck = ag
+      .findAssets({ check: true })
+      .filter(asset => !processedAssets.has(asset));
+    t.push({
+      name: `Crawling ${assetsToCheck.length} outgoing urls`
+    });
 
-  await new Promise((resolve, reject) =>
-    async.parallelLimit(
-      assetsToCheck.map(asset => httpStatus(asset)),
-      20,
-      err => {
-        if (err) {
-          reject(err);
-        } else {
-          resolve();
+    await new Promise((resolve, reject) =>
+      async.parallelLimit(
+        assetsToCheck.map(asset => httpStatus(asset)),
+        20,
+        err => {
+          if (err) {
+            reject(err);
+          } else {
+            resolve();
+          }
         }
-      }
-    )
-  );
+      )
+    );
+  }
 
   // Check Content-Type vs. incoming relation targetTypes:
 

diff --git a/package.json b/package.json
@@ -35,7 +35,7 @@
   ],
   "homepage": "https://github.com/Munter/hyperlink",
   "dependencies": {
-    "assetgraph": "^4.5.0",
+    "assetgraph": "4.8.0",
     "async": "^2.6.0",
     "optimist": "^0.6.1",
     "pretty-bytes": "^4.0.2",

diff --git a/test/index.js b/test/index.js
@@ -2009,4 +2009,62 @@ describe('hyperlink', function() {
       });
     });
   });
+
+  describe('with internalOnly true', () => {
+    it('should not follow external links', async () => {
+      httpception([
+        {
+          request: 'GET https://example.com/',
+          response: {
+            statusCode: 200,
+            headers: {
+              'Content-Type': 'text/html; charset=UTF-8'
+            },
+            body: `
+              <!DOCTYPE html>
+              <html>
+              <head></head>
+              <body>
+                <a href="otherPage.html">Other Page</a>
+                <a href="https://broken-link.mntr.dk/foo/bar/baz">Broken</a>
+              </body>
+              </html>
+            `
+          }
+        },
+        {
+          request: 'GET https://example.com/otherPage.html',
+          response: {
+            statusCode: 200,
+            headers: {
+              'Content-Type': 'text/html; charset=UTF-8'
+            },
+            body: `
+              <!DOCTYPE html>
+              <html>
+              <head></head>
+              <body>
+                <a href="https://broken-link.mntr.dk/1/2/3#metameta">Broken</a>
+              </body>
+              </html>
+            `
+          }
+        }
+      ]);
+
+      const t = new TapRender();
+      sinon.spy(t, 'push');
+      await hyperlink(
+        {
+          root: 'https://example.com/',
+          inputUrls: ['https://example.com/'],
+          recursive: true,
+          internalOnly: true
+        },
+        t
+      );
+
+      expect(t.close(), 'to satisfy', { fail: 0 });
+    });
+  });
 });