Add removeDuplicates option

Kikobeats · Jul 6, 2018 · 09282e8 · 09282e8
1 parent 9337816
commit 09282e8
Show file tree

Hide file tree

Showing 4 changed files with 129 additions and 34 deletions.
diff --git a/README.md b/README.md
@@ -46,7 +46,7 @@ See [examples](/examples).
 
 ## API
 
-### links([options])
+### htmlUrls([options])
 
 #### options
 
@@ -73,6 +73,13 @@ Default: `[]`
 
 A list of links to be excluded from the final output. It supports regex patterns.
 
+##### removeDuplicates
+
+Type: `boolean`<br>
+Default: true`
+
+Remove duplicated links detected over all the HTML tags.
+
 ## License
 
 **html-urls** © [Kiko Beats](https://kikobeats.com), released under the [MIT](https://github.com/Kikobeats/html-urls/blob/master/LICENSE.md) License.<br>

diff --git a/__snapshots__/index.js.snap-shot b/__snapshots__/index.js.snap-shot
@@ -234,3 +234,44 @@ exports['${tag} (${attributeName}) 27'] = [
   }
 ]
 
+exports['invariant wwww 1'] = [
+  {
+    "url": "https://www.google.com",
+    "normalizedUrl": "https://www.google.com"
+  },
+  {
+    "url": "https://google.com",
+    "normalizedUrl": "https://google.com"
+  },
+  {
+    "url": "https://facebook.com",
+    "normalizedUrl": "https://facebook.com"
+  }
+]
+
+exports['from same tag 2'] = [
+  {
+    "url": "https://google.com",
+    "normalizedUrl": "https://google.com"
+  },
+  {
+    "url": "https://google.com",
+    "normalizedUrl": "https://google.com"
+  },
+  {
+    "url": "https://facebook.com",
+    "normalizedUrl": "https://facebook.com"
+  }
+]
+
+exports['from different tags 2'] = [
+  {
+    "url": "https://google.com",
+    "normalizedUrl": "https://google.com"
+  },
+  {
+    "url": "https://google.com",
+    "normalizedUrl": "https://google.com"
+  }
+]
+
diff --git a/src/index.js b/src/index.js
@@ -33,7 +33,7 @@ const TAGS = {
 }
 
 const reduceSelector = (collection, fn, acc = []) => {
-  collection.each(function (index, element) {
+  collection.each(function () {
     acc = fn(acc, this)
   })
   return acc
@@ -53,30 +53,44 @@ const getLink = ({ url, el, attribute }) => {
   }
 }
 
-const getLinksByAttribute = ({ selector, attribute, url, whitelist }) => {
-  return reduceSelector(
-    selector,
-    (acc, el) => {
-      const link = getLink({ url, el, attribute })
-      const uid = get(link, UID)
-      if (isEmpty(link)) return acc
-      const isAlreadyAdded = includes(acc, item => get(item, UID) === uid)
-      if (isAlreadyAdded) return acc
-      const match = whitelist && matcher([uid], whitelist)
-      return isEmpty(match) ? concat(acc, link) : acc
-    },
-    []
-  )
+const createGetLinksByAttribute = ({ removeDuplicates }) => {
+  const has = removeDuplicates
+    ? (acc, uid) => includes(acc, item => get(item, UID) === uid)
+    : () => false
+
+  return ({ selector, attribute, url, whitelist }) =>
+    reduceSelector(
+      selector,
+      (acc, el) => {
+        const link = getLink({ url, el, attribute })
+        const uid = get(link, UID)
+        if (isEmpty(link)) return acc
+        const isAlreadyAdded = has(acc, uid)
+        if (isAlreadyAdded) return acc
+        const match = whitelist && matcher([uid], whitelist)
+        return isEmpty(match) ? concat(acc, link) : acc
+      },
+      []
+    )
 }
 
+const createAdd = ({ removeDuplicates }) =>
+  removeDuplicates
+    ? (acc, links) => uniqBy(concat(acc, links), UID)
+    : (acc, links) => concat(acc, links)
+
 module.exports = ({
   html = '',
   url = '',
   whitelist = false,
+  removeDuplicates = true,
   cheerioOpts = {}
 } = {}) => {
   const $ = cheerio.load(html, cheerioOpts)
 
+  const add = createAdd({ removeDuplicates })
+  const getLinksByAttribute = createGetLinksByAttribute({ removeDuplicates })
+
   return reduce(
     TAGS,
     (acc, htmlTags, attribute) => {
@@ -86,7 +100,8 @@ module.exports = ({
         url,
         whitelist
       })
-      return uniqBy(concat(acc, links), UID)
+
+      return add(acc, links)
     },
     []
   )

diff --git a/test/index.js b/test/index.js
@@ -24,27 +24,47 @@ describe('html links', () => {
     snapshot(getLinks({ html }))
   })
 
-  describe('normalization', () => {
-    describe('remove duplicate urls', () => {
-      it('from same tag', () => {
-        const html = generateHtml({
-          urls: [
-            'https://google.com',
-            'https://google.com',
-            'https://facebook.com'
-          ]
-        })
-        snapshot(getLinks({ html }))
+  describe('remove duplicate urls', () => {
+    it('from same tag', () => {
+      const html = generateHtml({
+        urls: [
+          'https://google.com',
+          'https://google.com',
+          'https://facebook.com'
+        ]
       })
-      it('from different tags', () => {
-        const html = generateHtml({
-          urls: ['https://google.com'],
-          links: ['https://google.com']
-        })
-        snapshot(getLinks({ html }))
+      snapshot(getLinks({ html }))
+    })
+    it('from different tags', () => {
+      const html = generateHtml({
+        urls: ['https://google.com'],
+        links: ['https://google.com']
+      })
+      snapshot(getLinks({ html }))
+    })
+  })
+
+  describe('non remove duplicate urls', () => {
+    it('from same tag', () => {
+      const html = generateHtml({
+        urls: [
+          'https://google.com',
+          'https://google.com',
+          'https://facebook.com'
+        ]
+      })
+      snapshot(getLinks({ html, removeDuplicates: false }))
+    })
+    it('from different tags', () => {
+      const html = generateHtml({
+        urls: ['https://google.com'],
+        links: ['https://google.com']
       })
+      snapshot(getLinks({ html, removeDuplicates: false }))
     })
+  })
 
+  describe('normalization', () => {
     it('invariant final slash', () => {
       const html = generateHtml({
         urls: [
@@ -57,6 +77,18 @@ describe('html links', () => {
       snapshot(getLinks({ html }))
     })
 
+    it('invariant wwww', () => {
+      const html = generateHtml({
+        urls: [
+          'https://www.google.com',
+          'https://google.com',
+          'https://facebook.com'
+        ]
+      })
+
+      snapshot(getLinks({ html }))
+    })
+
     it('query string parameters position are not relevant', () => {
       const html = generateHtml({
         urls: [