Merge pull request #410 from sunnydavis/parole-amazon-cloudfront

Remove Amazon CloudFront from the crawlers

raw/Crawlers.txt

@@ -1,13 +1,13 @@
  YLT
 ^b0t$
-^bluefish 
+^bluefish
 ^Calypso v\/
 ^Corax
 ^COMODO DCV
 ^DangDang
 ^DavClnt
 ^DHSH
-^FDM 
+^FDM
 ^git\/
 ^Goose\/
 ^Grabber
@@ -24,7 +24,7 @@
 ^PHP\/[0-9]
 ^RMA\/
 ^Ruby|Ruby\/[0-9]
-^Swurl 
+^Swurl
 ^VSE\/[0-9]
 ^WordPress\.com
 ^XRL\/[0-9]
@@ -75,7 +75,6 @@ allloadin
 AllSubmitter
 alyze\.info
 amagit
-^Amazon CloudFront$
 ^Amazon Simple Notification Service Agent$
 Anarchie
 AndroidDownloadManager
@@ -516,7 +515,7 @@ httpunit
 HttpUrlConnection
 httrack
 huaweisymantec
-HubSpot 
+HubSpot
 Humanlinks
 i2kconnect\/
 Iblog
@@ -685,7 +684,7 @@ MetaURI
 MFC_Tear_Sample
 Microsearch
 Microsoft\.Data\.Mashup
-Microsoft Office 
+Microsoft Office
 Microsoft Outlook
 Microsoft Windows Network Diagnostics
 Microsoft-WebDAV-MiniRedir
@@ -882,7 +881,7 @@ ProWebWalker
 proximic
 PRTG Network Monitor
 pshtt, https scanning
-PTST 
+PTST
 PTST\/[0-9]+
 Pump
 python-httpx
@@ -1045,7 +1044,7 @@ speedy
 SPEng
 Spinn3r
 spray-can
-Sprinklr 
+Sprinklr
 spyonweb
 sqlmap
 Sqlworm
@@ -1067,7 +1066,7 @@ summify
 SuperHTTP
 Surphace Scout
 Suzuran
-swcd 
+swcd
 Symfony BrowserKit
 Symfony2 BrowserKit
 SynHttpClient-Built
@@ -1215,7 +1214,7 @@ WebIndex
 webkit2png
 WebLeacher
 webmastercoffee
-webmon 
+webmon
 WebPix
 WebReaper
 WebSauger
@@ -1311,12 +1310,12 @@ Zemanta Aggregator
 Zend_Http_Client
 Zend\\Http\\Client
 Zermelo
-Zeus 
+Zeus
 zgrab
 ZnajdzFoto
 ZnHTTP
 Zombie\.js
 Zoom\.Mac
 ZoteroTranslationServer
 ZyBorg
-[a-z0-9\-_]*(bot|crawl|archiver|transcoder|spider|uptime|validator|fetcher|cron|checker|reader|extractor|monitoring|analyzer|scraper)
+[a-z0-9\-_]*(bot|crawl|archiver|transcoder|spider|uptime|validator|fetcher|cron|checker|reader|extractor|monitoring|analyzer|scraper)

src/Fixtures/Crawlers.php

@@ -96,7 +96,6 @@ class Crawlers extends AbstractProvider
         'AllSubmitter',
         'alyze\.info',
         'amagit',
-        '^Amazon CloudFront$',
         '^Amazon Simple Notification Service Agent$',
         'Anarchie',
         'AndroidDownloadManager',

tests/crawlers.txt

@@ -3523,7 +3523,6 @@ php-requests/1.7
 nghttp2/1.40.0
 WinHTTP/1.1
 Mozilla/5.0 (compatible; Javelin; +https://about.javelin.io/)
-Amazon CloudFront
 swcd (unknown version) CFNetwork/1125.2 Darwin/19.4.0
 YahooMailProxy; https://help.yahoo.com/kb/yahoo-mail-proxy-SLN28749.html
 Mozilla/5.0 (Java) outbrain
@@ -3560,4 +3559,4 @@ postplanner.com/site-scraping
 Nuclei (@pdiscoveryio)
 myseosnapshot/1.0
 Contextual Code Sites Explorer
-Corax - support@coraxcyber.com
+Corax - support@coraxcyber.com