Merge pull request #432 from crisp-dev/master

Add a replacement in cleanURL Closes #439
simplecrawler · Apr 4, 2019 · b184ff3 · b184ff3
2 parents 7083958 + 0ab84d6
commit b184ff3
Showing 1 changed file with 1 addition and 0 deletions.
diff --git a/lib/crawler.js b/lib/crawler.js
@@ -828,6 +828,7 @@ function cleanURL (URL, queueItem) {
         .replace(/&amp;/gi, "&")
         .replace(/&#38;/gi, "&")
         .replace(/&#x00026;/gi, "&")
+        .replace(/&#x2f;/gi, "/")
         .split("#")
         .shift()
         .trim();