Add 13 new useragents

JayBizzle · Jan 2, 2018 · fd88e66 · fd88e66
1 parent 6fa6bd7
commit fd88e66
Show file tree

Hide file tree

Showing 2 changed files with 26 additions and 0 deletions.
diff --git a/src/Fixtures/Crawlers.php b/src/Fixtures/Crawlers.php
@@ -30,6 +30,7 @@ class Crawlers extends AbstractProvider
         '^DangDang',
         '^DavClnt',
         '^FDM ',
+        '^Grabber',
         '^git\/',
         '^Goose\/',
         '^HTTPClient\/',
@@ -158,6 +159,7 @@ class Crawlers extends AbstractProvider
         'Cloudinary\/[0-9]',
         'cmcm\.com',
         'coccoc',
+        'colly -',
         'CommaFeed',
         'Commons-HttpClient',
         'Comodo SSL Checker',
@@ -309,6 +311,7 @@ class Crawlers extends AbstractProvider
         'grouphigh',
         'grokkit',
         'grub-client',
+        'gSOAP\/',
         'GTmetrix',
         'GuzzleHttp',
         'gvfs\/',
@@ -339,6 +342,7 @@ class Crawlers extends AbstractProvider
         'httphr',
         'HTTPMon',
         'PEAR HTTPRequest',
+        'http\.rb\/',
         'httpscheck',
         'httpssites_power',
         'httpunit',
@@ -391,9 +395,11 @@ class Crawlers extends AbstractProvider
         'Jersey\/',
         'Jigsaw',
         'Jobboerse',
+        'JobFeed discovery',
         'Jobg8 URL Monitor',
         'jobo',
         'Jobrapido',
+        'JoinVision Generic',
         'JS-Kit',
         'Kaspersky Lab CFR link resolver',
         'KeepRight OpenStreetMap Checker',
@@ -427,6 +433,7 @@ class Crawlers extends AbstractProvider
         'LinkExaminer',
         'linkfluence',
         'linkpeek',
+        'LinkPreviewGenerator',
         'LinkTiger',
         'LinkWalker',
         'Lipperhey',
@@ -513,6 +520,7 @@ class Crawlers extends AbstractProvider
         'Nymesis',
         'Ocelli\/[0-9]',
         'oegp',
+        'Offline Explorer',
         'okhttp',
         'Omea Reader',
         'omgili',
@@ -542,6 +550,7 @@ class Crawlers extends AbstractProvider
         'Pearltrees',
         'peerindex',
         'Peew',
+        'Perlu -',
         'PhantomJS\/',
         'PhantomJS Screenshoter',
         'Photon\/',
@@ -611,6 +620,7 @@ class Crawlers extends AbstractProvider
         'RSSOwl',
         'safe-agent-scanner',
         'SalesIntelligent',
+        'Saleslift',
         'SauceNAO',
         'SBIder',
         'scalaj-http',
@@ -672,6 +682,7 @@ class Crawlers extends AbstractProvider
         'SniffRSS',
         'sniptracker',
         'Snoopy',
+        'SnowHaze Search',
         'sogou web',
         'SortSite',
         'sovereign\.ai',
@@ -765,6 +776,7 @@ class Crawlers extends AbstractProvider
         'W3C_I18n-Checker',
         'W3C_Unicorn',
         'wangling',
+        'Wappalyzer',
         'WatchMouse',
         'WbSrch\/',
         'web-capture\.net',
@@ -775,6 +787,7 @@ class Crawlers extends AbstractProvider
         'WebClient\/',
         'webcollage',
         'WebCookies',
+        'WebCopier',
         'WebCorp',
         'WebDoc',
         'WebFetch',

diff --git a/tests/crawlers.txt b/tests/crawlers.txt
@@ -3188,3 +3188,16 @@ POE-Component-Client-HTTP/0.943 (perl; N; POE; en; rv:0.943000)
 RecurPost/1.0 https://recurpost.com (KHTML, like Gecko)
 Mozilla/5.0 Firefox Windows WhoRunsCoinHive.com
 Mozilla/5.0 (compatible; Kelny/2.0; +http://www.kelny.com/)
+colly - https://github.com/gocolly/colly
+WebCopier vx.x
+gSOAP/2.8
+http.rb/2.2.2
+Grabber -  https://sites.google.com/3ziele.de/grabber/
+JobFeed discovery script (http://www.jobfeed.com)
+JoinVision Generic Site Loader 1.0
+SSI-LinkPreviewGenerator/1.0
+SnowHaze Search/1.0 support@snowhaze.com
+Saleslift
+Perlu - HTML Analyzer and Relationship Builder 1.1
+Offline Explorer 1.*
+Mozilla/5.0 (compatible; Wappalyzer; https://www.wappalyzer.com)