BOA: Do not block Yandex and Baidu spiders (Robert)

omega8cc · Apr 29, 2011 · c560d048127a7b631da216c9d22aa329b6dddfea · c560d04
1 parent cb3d7e9
commit c560d048127a7b631da216c9d22aa329b6dddfea
Show file tree

Hide file tree

Showing 3 changed files with 3 additions and 3 deletions.
diff --git a/BARRACUDA.sh.txt b/BARRACUDA.sh.txt
@@ -1463,7 +1463,7 @@ if [ -d "/var/xdrago" ] ; then
   cp -af /var/xdrago /var/backups/dragon/x/xdrago-pre-$_INSTALLER_VERSION-$_NOW &> /dev/null
   rm -f /var/xdrago/log/VISITOR_ABUSE_ONE.log
   rm -f /var/xdrago/log/blackIP.log
-  rm -f /var/xdrago/{enableStatus,graceful,move_sql,run_all,second,Minute,firewall.sh,stop-mysql-innodb.sh,firewall_restarter,FireStart}
+  rm -f /var/xdrago/{enableStatus,graceful,move_sql,run_all,second,Minute,firewall.sh,stop-mysql-innodb.sh,firewall_restarter,FireStart,memcache,redis}
   cp -af /var/spool/cron/crontabs/root /var/backups/dragon/z/cron-root-pre-$_INSTALLER_VERSION-$_NOW &> /dev/null
   cp -af /opt/tmp/$_BOA_REPO_NAME/aegir/tools/auto-healing-monitoring/* /var/xdrago/ &> /dev/null
   if [ -z "$_THISHTIP" ] ; then

diff --git a/aegir/conf/nginx_advanced_include.conf b/aegir/conf/nginx_advanced_include.conf
@@ -5,7 +5,7 @@
     ###
     ### deny crawlers without 403 response
     ###
-    if ( $http_user_agent ~* (?:HTTrack|HTMLParser|libwww|AutomaticSiteMap|Yandex|BaiduSpider|ClickSense|ValueClick|SiteBot|BuzzTracker) ) {
+    if ( $http_user_agent ~* (?:HTTrack|HTMLParser|libwww|PECL|AutomaticSiteMap|ClickSense|ValueClick|SiteBot|BuzzTracker) ) {
          return 444;
     }
 

diff --git a/aegir/conf/nginx_octopus_include.conf b/aegir/conf/nginx_octopus_include.conf
@@ -5,7 +5,7 @@
     ###
     ### deny crawlers without 403 response
     ###
-    if ( $http_user_agent ~* (?:HTTrack|HTMLParser|libwww|AutomaticSiteMap|Yandex|BaiduSpider|ClickSense|ValueClick|SiteBot|BuzzTracker) ) {
+    if ( $http_user_agent ~* (?:HTTrack|HTMLParser|libwww|PECL|AutomaticSiteMap|ClickSense|ValueClick|SiteBot|BuzzTracker) ) {
          return 444;
     }