Add lots of bot user agents (#205)

JayBizzle · Jun 19, 2017 · 55e98a6 · 55e98a6
1 parent fa55dc4
commit 55e98a6
Show file tree

Hide file tree

Showing 2 changed files with 42 additions and 1 deletion.
diff --git a/src/Fixtures/Crawlers.php b/src/Fixtures/Crawlers.php
@@ -135,6 +135,7 @@ class Crawlers extends AbstractProvider
         'CERT\.at-Statistics-Survey',
         'cg-eye',
         'changedetection',
+        'ChangesMeter\/',
         'Charlotte',
         'CheckHost',
         'checkprivacy',
@@ -215,6 +216,7 @@ class Crawlers extends AbstractProvider
         'facebookplatform',
         'fairshare',
         'Faraday v',
+        'fasthttp',
         'Faveeo',
         'Favicon downloader',
         'FavOrg',
@@ -244,6 +246,7 @@ class Crawlers extends AbstractProvider
         'free thumbnails',
         'FreeWebMonitoring SiteChecker',
         'Funnelback',
+        'G-i-g-a-b-o-t',
         'g00g1e\.net',
         'GAChecker',
         'ganarvisitas\/[0-9]',
@@ -256,6 +259,7 @@ class Crawlers extends AbstractProvider
         'GetURLInfo\/[0-9]',
         'Ghost Inspector',
         'GigablastOpenSource',
+        'GIS-LABS',
         'github\.com\/',
         'Go [\d\.]* package http',
         'Go-http-client',
@@ -284,6 +288,7 @@ class Crawlers extends AbstractProvider
         'GoogleDocs',
         'GoogleHC\/',
         'GoogleProducer',
+        'Gookey',
         'GoScraper',
         'GoSpotCheck',
         'GoSquared-Status-Checker',
@@ -294,6 +299,7 @@ class Crawlers extends AbstractProvider
         'grouphigh',
         'grub-client',
         'GTmetrix',
+        'GuzzleHttp',
         'gvfs\/',
         'HAA(A)?RTLAND http client',
         'Hatena',
@@ -309,8 +315,10 @@ class Crawlers extends AbstractProvider
         'ht:\/\/check',
         'htdig',
         'HTMLParser\/',
+        'http-get',
         'HTTP-Header-Abfrage',
         'http-kit',
+        'http-request\/',
         'HTTP-Tiny',
         'HTTP_Compression_Test',
         'http_request2',
@@ -353,6 +361,7 @@ class Crawlers extends AbstractProvider
         'internet_archive',
         'InternetSeer',
         'internetVista monitor',
+        'intraVnews',
         'IODC',
         'IOI',
         'iplabel',
@@ -365,6 +374,7 @@ class Crawlers extends AbstractProvider
         'iskanie',
         'iZSearch',
         'janforman',
+        'Jaunt\/',
         'Jigsaw',
         'Jobboerse',
         'jobo',
@@ -406,13 +416,14 @@ class Crawlers extends AbstractProvider
         'LongURL API',
         'looksystems\.net',
         'ltx71',
+        'lua-resty-http',
         'lwp-trivial',
         'lycos',
         'LYT\.SR',
         'mabontland',
         'MagpieRSS',
         'Mail.Ru',
-        'MailChimp\.com',
+        'MailChimp',
         'Mandrill',
         'MapperCmd',
         'marketinggrader',
@@ -442,6 +453,7 @@ class Crawlers extends AbstractProvider
         'Moreover',
         'Morning Paper',
         'mowser',
+        'MovableType',
         'Mrcgiguy',
         'mShots',
         'MVAClient',
@@ -507,6 +519,7 @@ class Crawlers extends AbstractProvider
         'peerindex',
         'Peew',
         'PhantomJS\/',
+        'PhantomJS Screenshoter',
         'Photon\/',
         'phpcrawl',
         'phpservermon',
@@ -572,6 +585,7 @@ class Crawlers extends AbstractProvider
         'SalesIntelligent',
         'SauceNAO',
         'SBIder',
+        'scalaj-http',
         'Scoop',
         'scooter',
         'ScoutJet',
@@ -635,6 +649,7 @@ class Crawlers extends AbstractProvider
         'Spinn3r',
         'spray-can',
         'Sprinklr ',
+        'sqlmap',
         'spyonweb',
         'Sqworm',
         'SSL Labs',
@@ -652,6 +667,7 @@ class Crawlers extends AbstractProvider
         'Symfony2 BrowserKit',
         'SynHttpClient-Built',
         'Sysomos',
+        'Symfony BrowserKit',
         'T0PHackTeam',
         'Tarantula\/',
         'Taringa UGC',
@@ -671,7 +687,9 @@ class Crawlers extends AbstractProvider
         'topster',
         'touche.com',
         'Traackr.com',
+        'TrapitAgent',
         'truwoGPS',
+        'TulipChain',
         'tweetedtimes\.com',
         'Tweetminster',
         'Tweezler\/',
@@ -680,6 +698,7 @@ class Crawlers extends AbstractProvider
         'ubermetrics-technologies',
         'uclassify',
         'UdmSearch',
+        'UniversalFeedParser',
         'Untiny',
         'UnwindFetchor',
         'updated',

diff --git a/tests/crawlers.txt b/tests/crawlers.txt
@@ -3112,3 +3112,25 @@ Sens.ai http://sovereign.ai/contact
 eContext/1.0 (eContext Classification Engine)
 Owler
 Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Slack/1.2.6 Chrome/45.0.2454.85 AtomShell/0.34.3 Safari/537.36 Slack_SSB/1.2.6
+Mozilla/5.0 (compatible; ChangesMeter/1.9.1; http://intuiware.com/apps/changes-meter)
+Mozilla/5.0 (compatible; Gookey.co/1.0; +http://gookey.co/)
+TrapitAgent/0.1 (feed processor; +http://trapit.com/about)
+sqlmap/1.0-dev-nongit-201612050a8c (http://sqlmap.org)
+UniversalFeedParser/3.3 +http://feedparser.org/
+TulipChain/5.xx (http://ostermiller.org/tulipchain/) Java/1.x.1_0x (http://apple.com/) Mac_OS_X/10.2.8
+scalaj-http/1.0
+HggH PhantomJS Screenshoter
+fasthttp
+fasthttp, Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.101 Safari/537.36
+lua-resty-http/0.10 (Lua) ngx_lua/10000
+GIS-LABS:CertMon
+GuzzleHttp/6.2.1 PHP/7.0.15-0ubuntu0.16.04.4
+http-request/v0.7.0 (http://git.io/tl_S2w) node.js/v0.10.29
+intraVnews/1.x
+MailChimp
+G-i-g-a-b-o-t
+Symfony BrowserKit
+VCPP71 http-get 1.0a
+Jaunt/1.2
+Mozilla/5.0 (compatible; Owler/0.4; +; )
+MovableType/x.x