diff --git a/src/Fixtures/Crawlers.php b/src/Fixtures/Crawlers.php index ceca5da..cbb927f 100644 --- a/src/Fixtures/Crawlers.php +++ b/src/Fixtures/Crawlers.php @@ -81,7 +81,6 @@ class Crawlers extends AbstractProvider 'Arachmo', 'arachnode', 'Arachnophilia', - 'archive-com', 'aria2', 'asafaweb.com', 'AskQuickly', @@ -99,7 +98,6 @@ class Crawlers extends AbstractProvider 'BCKLINKS', 'BDFetch', 'BegunAdvertising\/', - 'bibnum\.bnf', 'BigBozz', 'biglotron', 'BingLocalSearch', @@ -150,7 +148,6 @@ class Crawlers extends AbstractProvider 'contactbigdatafr', 'convera', 'copyright sheriff', - 'cosmos\/[0-9]', 'Covario-IDS', 'CrawlForMe\/[0-9]', 'cron-job\.org', @@ -159,7 +156,6 @@ class Crawlers extends AbstractProvider 'Curious George', 'curl', 'cuwhois\/[0-9]', - 'CyberPatrol', 'cybo\.com', 'DareBoost', 'DataparkSearch', @@ -226,7 +222,6 @@ class Crawlers extends AbstractProvider 'Fever\/[0-9]', 'findlink', 'findthatfile', - 'Flamingo_SearchEngine', 'FlipboardBrowserProxy', 'FlipboardProxy', 'FlipboardRSS', @@ -247,6 +242,7 @@ class Crawlers extends AbstractProvider 'getprismatic\.com', 'GetURLInfo\/[0-9]', 'GigablastOpenSource', + 'github\.com\/', 'Go [\d\.]* package http', 'Go-http-client', 'gofetch', @@ -400,12 +396,10 @@ class Crawlers extends AbstractProvider 'MailChimp\.com', 'Mandrill', 'marketinggrader', - 'masscan\/', 'Mediapartners-Google', 'MegaIndex\.ru', 'Melvil Rawi\/', 'MergeFlow-PageReader', - 'MetaInspector', 'Metaspinner', 'MetaURI', 'Microsearch', @@ -539,7 +533,6 @@ class Crawlers extends AbstractProvider 'Redirect Checker Tool', 'ReederForMac', 'request\.js', - 'resolver\/[0-9]', 'ResponseCodeTest\/[0-9]', 'RestSharp', 'RetrevoPageAnalyzer', @@ -571,7 +564,6 @@ class Crawlers extends AbstractProvider 'SEOCentro', 'SeoCheck', 'SeopultContentAnalyzer', - 'SEOstats', 'Server Density Service Monitoring', 'servernfo\.com', 'Seznam screenshot-generator', @@ -652,11 +644,9 @@ class Crawlers extends AbstractProvider 'Tweetminster', 'Twikle', 'Twingly', - 'Typhoeus', 'ubermetrics-technologies', 'uclassify', 'UdmSearch', - 'unfurlist', 'UnwindFetchor', 'updated', 'Upflow', @@ -672,7 +662,6 @@ class Crawlers extends AbstractProvider 'visionutils', 'vkShare', 'voltron', - 'Vortex\/[0-9]', 'voyager\/', 'VSAgent\/[0-9]', 'VSB-TUO\/[0-9]', @@ -683,7 +672,6 @@ class Crawlers extends AbstractProvider 'W3C_I18n-Checker', 'W3C_Unicorn', 'wangling', - 'Wappalyzer', 'WatchMouse', 'WbSrch\/', 'web-capture\.net', @@ -707,7 +695,6 @@ class Crawlers extends AbstractProvider 'Website Analyzer\/', 'websitepulse[+ ]checker', 'Websnapr\/', - 'Websquash\.com', 'Webthumb\/[0-9]', 'WebThumbnail', 'WeCrawlForThePeace', @@ -727,7 +714,6 @@ class Crawlers extends AbstractProvider 'wkhtmlto', 'wmtips', 'Woko', - 'WomlpeFactory', 'Word\/', 'WordPress\/', 'wotbox', diff --git a/tests/crawlers.txt b/tests/crawlers.txt index b1aded8..e5f19cf 100644 --- a/tests/crawlers.txt +++ b/tests/crawlers.txt @@ -2889,3 +2889,5 @@ Mozilla/5.0 (compatible; Semiocast HTTP client; http://semiocast.com/) Mozilla/5.0 (compatible; The Drop Reaper; +http://tag1consulting.com/dropreaper.html) /n software IPWorks HTTP/S Component - www.nsoftware.com safe-agent-scanner +Mozilla/5.0 (compatible; ACHE/Unknown Version; +https://github.com/ViDA-NYU/ache; ) +ANAM (github.com/dutchcoders/anam)