Merge pull request #257 from JayBizzle/dev

Add bot
JayBizzle · Mar 5, 2018 · 6e9abfe · 6e9abfe
2 parents 878d69c + aa5061e
commit 6e9abfe
Show file tree

Hide file tree

Showing 6 changed files with 8 additions and 3 deletions.
diff --git a/raw/Crawlers.json b/raw/Crawlers.json
diff --git a/raw/Crawlers.txt b/raw/Crawlers.txt
@@ -160,6 +160,7 @@ curl
 cuwhois\/[0-9]
 cybo\.com
 DareBoost
+help@dataminr\.com
 DataparkSearch
 dataprovider
 Daum(oa)?[ \/][0-9]
@@ -855,11 +856,12 @@ Yoleo Consumer
 yoogliFetchAgent
 YottaaMonitor
 yourls\.org
+Your-Website-Sucks\/[0-9]
 Zao
 Zemanta Aggregator
 Zend\\Http\\Client
 Zend_Http_Client
 zgrab
 ZnajdzFoto
 ZyBorg
-[a-z0-9\-_]*((?<!cu)bot|crawler|archiver|transcoder|spider|uptime|validator|fetcher)
+[a-z0-9\-_]*(bot|crawler|archiver|transcoder|spider|uptime|validator|fetcher)
diff --git a/raw/Exclusions.json b/raw/Exclusions.json
@@ -1 +1 @@
-["Safari.[\\d\\.]*","Firefox.[\\d\\.]*","Chrome.[\\d\\.]*","Chromium.[\\d\\.]*","MSIE.[\\d\\.]","Opera\\\/[\\d\\.]*","Mozilla.[\\d\\.]*","AppleWebKit.[\\d\\.]*","Trident.[\\d\\.]*","Windows NT.[\\d\\.]*","Android [\\d\\.]*","Macintosh.","Ubuntu","Linux","[ ]Intel","Mac OS X [\\d_]*","(like )?Gecko(.[\\d\\.]*)?","KHTML,","CriOS.[\\d\\.]*","CPU iPhone OS ([0-9_])* like Mac OS X","CPU OS ([0-9_])* like Mac OS X","iPod","compatible","x86_..","i686","x64","X11","rv:[\\d\\.]*","Version.[\\d\\.]*","WOW64","Win64","Dalvik.[\\d\\.]*"," \\.NET CLR [\\d\\.]*","Presto.[\\d\\.]*","Media Center PC","BlackBerry","Build","Opera Mini\\\/\\d{1,2}\\.\\d{1,2}\\.[\\d\\.]*\\\/\\d{1,2}\\.","Opera"," \\.NET[\\d\\.]*",";"]
+["Safari.[\\d\\.]*","Firefox.[\\d\\.]*","Chrome.[\\d\\.]*","Chromium.[\\d\\.]*","MSIE.[\\d\\.]","Opera\\\/[\\d\\.]*","Mozilla.[\\d\\.]*","AppleWebKit.[\\d\\.]*","Trident.[\\d\\.]*","Windows NT.[\\d\\.]*","Android [\\d\\.]*","Macintosh.","Ubuntu","Linux","[ ]Intel","Mac OS X [\\d_]*","(like )?Gecko(.[\\d\\.]*)?","KHTML,","CriOS.[\\d\\.]*","CPU iPhone OS ([0-9_])* like Mac OS X","CPU OS ([0-9_])* like Mac OS X","iPod","compatible","x86_..","i686","x64","X11","rv:[\\d\\.]*","Version.[\\d\\.]*","WOW64","Win64","Dalvik.[\\d\\.]*"," \\.NET CLR [\\d\\.]*","Presto.[\\d\\.]*","Media Center PC","BlackBerry","Build","Opera Mini\\\/\\d{1,2}\\.\\d{1,2}\\.[\\d\\.]*\\\/\\d{1,2}\\.","Opera"," \\.NET[\\d\\.]*","cubot",";"]
diff --git a/raw/Exclusions.txt b/raw/Exclusions.txt
@@ -38,4 +38,5 @@ Build
 Opera Mini\/\d{1,2}\.\d{1,2}\.[\d\.]*\/\d{1,2}\.
 Opera
  \.NET[\d\.]*
+cubot
 ;
diff --git a/src/Fixtures/Crawlers.php b/src/Fixtures/Crawlers.php
@@ -877,6 +877,7 @@ class Crawlers extends AbstractProvider
         'yoogliFetchAgent',
         'YottaaMonitor',
         'yourls\.org',
+        'Your-Website-Sucks\/[0-9]',
         'Zao',
         'Zemanta Aggregator',
         'Zend\\\\Http\\\\Client',

diff --git a/tests/crawlers.txt b/tests/crawlers.txt
@@ -3234,3 +3234,4 @@ Mozilla/5.0 (compatible; dotMailer content retrieval service)
 Nexgate Ruby Client
 Miniature.io/2.0 (Linux; en-us)
 help@dataminr.com
+Mozilla/5.0 (compatible; Your-Website-Sucks/2.0; +http://www.Your-Website-Sucks.net/WWW.SOMEWEBSITE.COM)
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		["Safari.[\\d\\.]","Firefox.[\\d\\.]","Chrome.[\\d\\.]","Chromium.[\\d\\.]","MSIE.[\\d\\.]","Opera\\\/[\\d\\.]","Mozilla.[\\d\\.]","AppleWebKit.[\\d\\.]","Trident.[\\d\\.]","Windows NT.[\\d\\.]","Android [\\d\\.]","Macintosh.","Ubuntu","Linux","[ ]Intel","Mac OS X [\\d_]","(like )?Gecko(.[\\d\\.])?","KHTML,","CriOS.[\\d\\.]","CPU iPhone OS ([0-9_]) like Mac OS X","CPU OS ([0-9_])* like Mac OS X","iPod","compatible","x86_..","i686","x64","X11","rv:[\\d\\.]","Version.[\\d\\.]","WOW64","Win64","Dalvik.[\\d\\.]"," \\.NET CLR [\\d\\.]","Presto.[\\d\\.]","Media Center PC","BlackBerry","Build","Opera Mini\\\/\\d{1,2}\\.\\d{1,2}\\.[\\d\\.]\\\/\\d{1,2}\\.","Opera"," \\.NET[\\d\\.]*",";"]
		["Safari.[\\d\\.]","Firefox.[\\d\\.]","Chrome.[\\d\\.]","Chromium.[\\d\\.]","MSIE.[\\d\\.]","Opera\\\/[\\d\\.]","Mozilla.[\\d\\.]","AppleWebKit.[\\d\\.]","Trident.[\\d\\.]","Windows NT.[\\d\\.]","Android [\\d\\.]","Macintosh.","Ubuntu","Linux","[ ]Intel","Mac OS X [\\d_]","(like )?Gecko(.[\\d\\.])?","KHTML,","CriOS.[\\d\\.]","CPU iPhone OS ([0-9_]) like Mac OS X","CPU OS ([0-9_])* like Mac OS X","iPod","compatible","x86_..","i686","x64","X11","rv:[\\d\\.]","Version.[\\d\\.]","WOW64","Win64","Dalvik.[\\d\\.]"," \\.NET CLR [\\d\\.]","Presto.[\\d\\.]","Media Center PC","BlackBerry","Build","Opera Mini\\\/\\d{1,2}\\.\\d{1,2}\\.[\\d\\.]\\\/\\d{1,2}\\.","Opera"," \\.NET[\\d\\.]*","cubot",";"]
-Original file line number
+Diff line change
@@ Expand Up / @@ -38,4 +38,5 @@ Build @@
     Opera Mini\/\d{1,2}\.\d{1,2}\.[\d\.]*\/\d{1,2}\.
     Opera
      \.NET[\d\.]*
+    cubot
     ;