Merge f0f32d4 into 624eab2

JayBizzle · Jan 31, 2020 · df3258a · df3258a
2 parents 624eab2 + f0f32d4
commit df3258a
Show file tree

Hide file tree

Showing 4 changed files with 57 additions and 2 deletions.
diff --git a/raw/Crawlers.json b/raw/Crawlers.json
diff --git a/raw/Crawlers.txt b/raw/Crawlers.txt
@@ -103,6 +103,7 @@ Asterias
 Astute
 asynchttp
 Attach
+attohttpc
 autocite
 AutomaticWPTester
 Autonomy
@@ -127,6 +128,7 @@ BBBike
 BCKLINKS
 BDFetch
 BegunAdvertising
+Bewica-security-scan
 Bidtellect
 BigBozz
 Bigfoot
@@ -141,13 +143,15 @@ Black Hole
 Blackboard Safeassign
 BlackWidow
 BlockNote\.Net
+BlogBridge
 Bloglines
 Bloglovin
 BlogPulseLive
 BlogSearch
 Blogtrottr
 BlowFish
 boitho\.com-dc
+Boost\.Beast
 BPImageWalker
 Braintree-Webhooks
 Branch Metrics API
@@ -240,6 +244,7 @@ Daum(oa)?[ \/][0-9]
 dBpoweramp
 ddline
 deeris
+delve\.ai
 Demon
 DeuSu
 developers\.google\.com\/\+\/web\/snippet\/
@@ -258,6 +263,7 @@ DNSPod-reporting
 docoloc
 Dolphin http client
 DomainAppender
+DomainLabz
 Donuts Content Explorer
 dotMailer content retrieval
 dotSemantic
@@ -301,6 +307,7 @@ ExactSearch
 ExaleadCloudview
 Excel\/
 exif
+ExoRank
 Exploratodo
 Express WebPictures
 Extreme Picture Finder
@@ -352,6 +359,7 @@ http:\/\/www.neomo.de\/
 free thumbnails
 Freeuploader
 Funnelback
+Fuzz Faster U Fool
 G-i-g-a-b-o-t
 g00g1e\.net
 ganarvisitas
@@ -443,6 +451,7 @@ HEADMasterSEO
 HeartRails_Capture
 help@dataminr\.com
 heritrix
+Hexometer
 historious
 hkedcity
 hledejLevne\.cz
@@ -589,6 +598,7 @@ Liferea\/
 Lightspeedsystems
 Lighthouse
 Likse
+limber\.io
 Link Valet
 link_thumbnailer
 LinkAlarm\/
@@ -608,6 +618,7 @@ livedoor ScreenShot
 LoadImpactRload
 localsearch-web
 LongURL API
+longurl-r-package
 looid\.com
 looksystems\.net
 ltx71
@@ -756,6 +767,7 @@ Optimizer
 Orbiter
 OrgProbe\/
 orion-semantics
+OSPScraper
 Outlook-Express
 Outlook-iOS
 ow\.ly
@@ -835,6 +847,7 @@ PTST
 PTST\/[0-9]+
 Pulsepoint XT3 web scraper
 Pump
+python-httpx
 Python-httplib2
 python-requests
 Python-urllib
@@ -875,6 +888,7 @@ Robozilla
 ROI Hunter
 RPT-HTTPClient
 RSSOwl
+RyowlEngine
 safe-agent-scanner
 SalesIntelligent
 Saleslift
@@ -883,6 +897,7 @@ SauceNAO
 SBIder
 sc-downloader
 scalaj-http
+Scamadviser-Frontend
 scan\.lol
 ScanAlert
 Scoop
@@ -916,6 +931,7 @@ Seobility
 SEOCentro
 SeoCheck
 SEOkicks
+SEOlizer
 Seomoz
 SEOprofiler
 SEOsearch
@@ -929,6 +945,7 @@ Shelob
 Shodan
 Shoppimon
 ShopWiki
+shortURL lengthener
 ShortLinkTranslate
 shrinktheweb
 Sideqik
@@ -1191,6 +1208,7 @@ WinPodder
 wkhtmlto
 wmtips
 Woko
+Wolfram HTTPClient
 woorankreview
 Word\/
 WordPress\/

diff --git a/src/Fixtures/Crawlers.php b/src/Fixtures/Crawlers.php
@@ -124,6 +124,7 @@ class Crawlers extends AbstractProvider
         'Astute',
         'asynchttp',
         'Attach',
+        'attohttpc',
         'autocite',
         'AutomaticWPTester',
         'Autonomy',
@@ -148,6 +149,7 @@ class Crawlers extends AbstractProvider
         'BCKLINKS',
         'BDFetch',
         'BegunAdvertising',
+        'Bewica-security-scan',
         'Bidtellect',
         'BigBozz',
         'Bigfoot',
@@ -162,13 +164,15 @@ class Crawlers extends AbstractProvider
         'Blackboard Safeassign',
         'BlackWidow',
         'BlockNote\.Net',
+        'BlogBridge',
         'Bloglines',
         'Bloglovin',
         'BlogPulseLive',
         'BlogSearch',
         'Blogtrottr',
         'BlowFish',
         'boitho\.com-dc',
+        'Boost\.Beast',
         'BPImageWalker',
         'Braintree-Webhooks',
         'Branch Metrics API',
@@ -261,6 +265,7 @@ class Crawlers extends AbstractProvider
         'dBpoweramp',
         'ddline',
         'deeris',
+        'delve\.ai',
         'Demon',
         'DeuSu',
         'developers\.google\.com\/\+\/web\/snippet\/',
@@ -279,6 +284,7 @@ class Crawlers extends AbstractProvider
         'docoloc',
         'Dolphin http client',
         'DomainAppender',
+        'DomainLabz',
         'Donuts Content Explorer',
         'dotMailer content retrieval',
         'dotSemantic',
@@ -322,6 +328,7 @@ class Crawlers extends AbstractProvider
         'ExaleadCloudview',
         'Excel\/',
         'exif',
+        'ExoRank',
         'Exploratodo',
         'Express WebPictures',
         'Extreme Picture Finder',
@@ -373,6 +380,7 @@ class Crawlers extends AbstractProvider
         'free thumbnails',
         'Freeuploader',
         'Funnelback',
+        'Fuzz Faster U Fool',
         'G-i-g-a-b-o-t',
         'g00g1e\.net',
         'ganarvisitas',
@@ -464,6 +472,7 @@ class Crawlers extends AbstractProvider
         'HeartRails_Capture',
         'help@dataminr\.com',
         'heritrix',
+        'Hexometer',
         'historious',
         'hkedcity',
         'hledejLevne\.cz',
@@ -610,6 +619,7 @@ class Crawlers extends AbstractProvider
         'Lightspeedsystems',
         'Lighthouse',
         'Likse',
+        'limber\.io',
         'Link Valet',
         'link_thumbnailer',
         'LinkAlarm\/',
@@ -629,6 +639,7 @@ class Crawlers extends AbstractProvider
         'LoadImpactRload',
         'localsearch-web',
         'LongURL API',
+        'longurl-r-package',
         'looid\.com',
         'looksystems\.net',
         'ltx71',
@@ -777,6 +788,7 @@ class Crawlers extends AbstractProvider
         'Orbiter',
         'OrgProbe\/',
         'orion-semantics',
+        'OSPScraper',
         'Outlook-Express',
         'Outlook-iOS',
         'ow\.ly',
@@ -856,6 +868,7 @@ class Crawlers extends AbstractProvider
         'PTST\/[0-9]+',
         'Pulsepoint XT3 web scraper',
         'Pump',
+        'python-httpx',
         'Python-httplib2',
         'python-requests',
         'Python-urllib',
@@ -896,6 +909,7 @@ class Crawlers extends AbstractProvider
         'ROI Hunter',
         'RPT-HTTPClient',
         'RSSOwl',
+        'RyowlEngine',
         'safe-agent-scanner',
         'SalesIntelligent',
         'Saleslift',
@@ -904,6 +918,7 @@ class Crawlers extends AbstractProvider
         'SBIder',
         'sc-downloader',
         'scalaj-http',
+        'Scamadviser-Frontend',
         'scan\.lol',
         'ScanAlert',
         'Scoop',
@@ -937,6 +952,7 @@ class Crawlers extends AbstractProvider
         'SEOCentro',
         'SeoCheck',
         'SEOkicks',
+        'SEOlizer',
         'Seomoz',
         'SEOprofiler',
         'SEOsearch',
@@ -950,6 +966,7 @@ class Crawlers extends AbstractProvider
         'Shodan',
         'Shoppimon',
         'ShopWiki',
+        'shortURL lengthener',
         'ShortLinkTranslate',
         'shrinktheweb',
         'Sideqik',
@@ -1212,6 +1229,7 @@ class Crawlers extends AbstractProvider
         'wkhtmlto',
         'wmtips',
         'Woko',
+        'Wolfram HTTPClient',
         'woorankreview',
         'Word\/',
         'WordPress\/',

diff --git a/tests/crawlers.txt b/tests/crawlers.txt
@@ -3474,4 +3474,23 @@ HaxerMen
 kubectl/v1.12.0 (linux/amd64) kubernetes/0ed3388
 NetSystemsResearch studies the availability of various services across the internet. Our website is netsystemsresearch.com
 Mozilla/5.0(Linux;Android 5.1.1;OPPO A33 Build/LMY47V;wv) AppleWebKit/537.36(KHTML,link Gecko) Version/4.0 Chrome/43.0.2357.121 Mobile Safari/537.36 LieBaoFast/4.51.3
-sc-downloader/2.0 (http://www.searchconcepts.io)
+sc-downloader/2.0 (http://www.searchconcepts.io)
+[object Promise] delve.ai website analyse
+attohttpc/0.9.0
+Bewica-security-scan
+BlogBridge 2.13 (http://www.blogbridge.com/)
+Boost.Beast/248
+DomainLabz
+ExoRank
+Fuzz Faster U Fool v1.0-rc1
+Hexometer
+longurl-r-package
+looid.com Search Engine/0.1
+Mozilla/5.0 (compatible; RyowlEngine/1.0; +https://ryowl.org)
+Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:25.0; https://app.limber.io) Gecko/20100101 Firefox/25.0
+OSPScraper (+https://www.opensyllabusproject.org)
+python-httpx/0.11.1
+shortURL lengthener/0.1 +http://kevindriscoll.info/
+Scamadviser-Frontend
+Wolfram HTTPClient 12.
+SEOlizer/1.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.13) Gecko/20080311 Firefox/2.0.0.13