Add 10 new useragents (#299)

* Add 4 new useragents * Add 6 more useragents
JayBizzle · Oct 23, 2018 · 92805a3 · 92805a3
1 parent bfb7d00
commit 92805a3
Show file tree

Hide file tree

Showing 5 changed files with 33 additions and 6 deletions.
diff --git a/raw/Crawlers.json b/raw/Crawlers.json
diff --git a/raw/Crawlers.txt b/raw/Crawlers.txt
@@ -222,6 +222,7 @@ cybo\.com
 DareBoost
 DatabaseDriverMysqli
 DataCha0s
+Datafeedwatch
 Datanyze
 DataparkSearch
 dataprovider
@@ -236,6 +237,7 @@ Digg
 Digincore
 DigitalPebble
 Dirbuster
+Disqus\/
 Dispatch\/
 DittoSpyder
 dlvr
@@ -273,7 +275,7 @@ ElectricMonk
 elefent
 EMail Exractor
 EMail\ Wolf
-Email%20Extractor%20Lite
+Email%20Extractor
 EmailWolf
 Embed PHP Library
 Embedly
@@ -388,6 +390,7 @@ Google-SearchByImage
 Google-Site-Verification
 Google-Structured-Data-Testing-Tool
 Google-Youtube-Links
+google-xrawler
 GoogleCloudMonitoring
 GoogleDocs
 GoogleHC\/
@@ -439,6 +442,7 @@ ht:\/\/check
 htdig
 HTMLparser
 htmlyse\.com
+HTTP Banner Detection
 HTTP_Compression_Test
 http_request2
 http_requester
@@ -449,6 +453,7 @@ http-request\/
 HTTP-Tiny
 HTTP::Lite
 http\.rb\/
+http_get
 HttpComponents
 httphr
 HTTPMon
@@ -611,6 +616,7 @@ Mata\ Hari
 Mediapartners-Google
 mediawords
 MegaIndex\.ru
+MeltwaterNews
 Melvil Rawi\/
 MergeFlow-PageReader
 Metaspinner
@@ -860,6 +866,7 @@ scooter
 ScoutJet
 ScoutURLMonitor
 ScrapeBox Page Scanner
+SimpleScraper
 Scrapy
 Screaming
 ScreenShotService\/[0-9]
@@ -1010,6 +1017,7 @@ theoldreader\.com
 Thinklab
 Thumbshots
 ThumbSniper
+timewe\.net
 TinEye
 Tiny Tiny RSS
 TLSProbe\/
@@ -1200,6 +1208,7 @@ Your-Website-Sucks\/[0-9]
 yourls\.org
 YoYs\.net
 YP\.PL
+Zabbix
 Zade
 Zao
 Zauba

diff --git a/src/Fixtures/Crawlers.php b/src/Fixtures/Crawlers.php
@@ -243,6 +243,7 @@ class Crawlers extends AbstractProvider
         'DareBoost',
         'DatabaseDriverMysqli',
         'DataCha0s',
+        'Datafeedwatch',
         'Datanyze',
         'DataparkSearch',
         'dataprovider',
@@ -257,6 +258,7 @@ class Crawlers extends AbstractProvider
         'Digincore',
         'DigitalPebble',
         'Dirbuster',
+        'Disqus\/',
         'Dispatch\/',
         'DittoSpyder',
         'dlvr',
@@ -294,7 +296,7 @@ class Crawlers extends AbstractProvider
         'elefent',
         'EMail Exractor',
         'EMail\ Wolf',
-        'Email%20Extractor%20Lite',
+        'Email%20Extractor',
         'EmailWolf',
         'Embed PHP Library',
         'Embedly',
@@ -409,6 +411,7 @@ class Crawlers extends AbstractProvider
         'Google-Site-Verification',
         'Google-Structured-Data-Testing-Tool',
         'Google-Youtube-Links',
+        'google-xrawler',
         'GoogleCloudMonitoring',
         'GoogleDocs',
         'GoogleHC\/',
@@ -460,6 +463,7 @@ class Crawlers extends AbstractProvider
         'htdig',
         'HTMLparser',
         'htmlyse\.com',
+        'HTTP Banner Detection',
         'HTTP_Compression_Test',
         'http_request2',
         'http_requester',
@@ -470,6 +474,7 @@ class Crawlers extends AbstractProvider
         'HTTP-Tiny',
         'HTTP::Lite',
         'http\.rb\/',
+        'http_get',
         'HttpComponents',
         'httphr',
         'HTTPMon',
@@ -632,6 +637,7 @@ class Crawlers extends AbstractProvider
         'Mediapartners-Google',
         'mediawords',
         'MegaIndex\.ru',
+        'MeltwaterNews',
         'Melvil Rawi\/',
         'MergeFlow-PageReader',
         'Metaspinner',
@@ -881,6 +887,7 @@ class Crawlers extends AbstractProvider
         'ScoutJet',
         'ScoutURLMonitor',
         'ScrapeBox Page Scanner',
+        'SimpleScraper',
         'Scrapy',
         'Screaming',
         'ScreenShotService\/[0-9]',
@@ -1031,6 +1038,7 @@ class Crawlers extends AbstractProvider
         'Thinklab',
         'Thumbshots',
         'ThumbSniper',
+        'timewe\.net',
         'TinEye',
         'Tiny Tiny RSS',
         'TLSProbe\/',
@@ -1221,6 +1229,7 @@ class Crawlers extends AbstractProvider
         'yourls\.org',
         'YoYs\.net',
         'YP\.PL',
+        'Zabbix',
         'Zade',
         'Zao',
         'Zauba',

diff --git a/tests/crawlers.txt b/tests/crawlers.txt
@@ -3380,4 +3380,15 @@ Mozilla/5.0 (Windows NT 6.1; StumbleUpon; noc@stumbleupon.com) Chrome/58.0.3029.
 Mozilla/5.0 (Windows NT 6.1; stumbleupon.com) Chrome/58.0.3029.110 Safari/537.36
 Mozilla/5.0 (compatible; tracemyfile/1.0)
 Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/66.0.3336.0 Safari/537.36
-Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36 Pix/1.0 (Criteo; x64-criteo)
+Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36 Pix/1.0 (Criteo; x64-criteo)
+CDR/1.7.1 Simulator/0.7(+http://timewe.net) Profile/MIDP-1.0 Configuration/CLDC-1.0
+Email%20Extractor/1 CFNetwork/760.6.3 Darwin/15.6.0 (x86_64)
+Email%20Extractor/29 CFNetwork/897.15 Darwin/17.5.0 (x86_64)
+HTTP Banner Detection (https://security.ipip.net)
+HTTP Banner Detection(security.ipip.net)
+Mozilla/5.0 (compatible; SimpleScraper)
+http_get
+MeltwaterNews www.meltwater.com
+Disqus/1.0
+Datafeedwatch/2.1.x
+Zabbix
diff --git a/tests/devices.txt b/tests/devices.txt
@@ -151348,7 +151348,6 @@ Mozilla/5.0 (iPhone; CPU iPhone OS 11_2_5 like Mac OS X) AppleWebKit/604.5.6 (KH
 Mozilla/5.0 (iPhone; CPU iPhone OS 11_2_6 like Mac OS X) AppleWebKit/604.5.6 (KHTML, like Gecko) Mobile/15D100 [FBAN/FBIOS;FBAV/164.0.0.56.96;FBBV/98434650;FBDV/iPhone8,4;FBMD/iPhone;FBSN/iOS;FBSV/11.2.6;FBSS/2;FBCR/O2;FBID/phone;FBLC/en_GB;FBOP/5;FBRV/100241679]
 Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:61.0) Gecko/20100101 Firefox/61.0
 Mozilla/5.0 (Linux; Android 7.1.1; XT1650 Build/NPLS26.118-20-5-11; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/65.0.3325.109 Mobile Safari/537.36
-Email%20Extractor/1 CFNetwork/760.6.3 Darwin/15.6.0 (x86_64)
 Mozilla/5.0 (iPhone; CPU iPhone OS 11_2_1 like Mac OS X) AppleWebKit/604.4.7 (KHTML, like Gecko) Mobile/15C153 [FBAN/FBIOS;FBAV/165.0.0.74.96;FBBV/100174821;FBDV/iPhone10,4;FBMD/iPhone;FBSN/iOS;FBSV/11.2.1;FBSS/2;FBCR/O2;FBID/phone;FBLC/en_GB;FBOP/5;FBRV/100483788]
 Mozilla/5.0 (Linux; Android 5.1; XT1039 Build/LPBS23.13-17.3-1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.109 Mobile Safari/537.36
 Mozilla/5.0 (Macintosh; PPC Mac OS X 10.8; rv:48.0) Gecko/20100101 Firefox/48.0
@@ -152368,7 +152367,6 @@ Mozilla/5.0 (Windows; U; Windows NT 6.1; tr; rv:1.9.1.9) Gecko/20100315 Firefox/
 Mozilla/5.0 (iPhone; CPU iPhone OS 11_1_2 like Mac OS X) AppleWebKit/604.3.5 (KHTML, like Gecko) Mobile/15B202 [FBAN/FBIOS;FBAV/170.1.0.80.91;FBBV/106613464;FBDV/iPhone9,3;FBMD/iPhone;FBSN/iOS;FBSV/11.1.2;FBSS/2;FBCR/giffgaff;FBID/phone;FBLC/en_GB;FBOP/5;FBRV/108284014]
 Mozilla/5.0 (Linux; Android 7.0; S6s Build/NRD90M) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.137 Mobile Safari/537.36
 Mozilla/5.0 (iPhone; CPU iPhone OS 11_2 like Mac OS X) AppleWebKit/604.4.7 (KHTML, like Gecko) Mobile/15C114 [FBAN/FBIOS;FBAV/160.0.0.34.96;FBBV/93101108;FBDV/iPhone9,4;FBMD/iPhone;FBSN/iOS;FBSV/11.2;FBSS/3;FBCR/EE;FBID/phone;FBLC/en_GB;FBOP/5;FBRV/93101108]
-Email%20Extractor/29 CFNetwork/897.15 Darwin/17.5.0 (x86_64)
 Mozilla/5.0 (iPad; CPU OS 11_2 like Mac OS X) AppleWebKit/604.1.34 (KHTML, like Gecko) CriOS/62.0.3202.70 Mobile/15C114 Safari/604.1
 Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239 Safari/537.36
 Mozilla/5.0 (Linux; Android 4.4.4; SM-T116 Build/KTU84P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.109 Safari/537.36