ignore google bot access to apache

cms-sw · Sep 21, 2016 · db34c70 · db34c70
1 parent b2d67c0
commit db34c70
Show file tree

Hide file tree

Showing 4 changed files with 11 additions and 3 deletions.
diff --git a/es_cmsrep_apache.py b/es_cmsrep_apache.py
@@ -5,9 +5,11 @@
 from es_utils import send_payload
 from hashlib import sha1
 from json import dumps
-from logwatch import logwatch, run_cmd
+from logwatch import logwatch, run_cmd, LOGWATCH_APACHE_IGNORE_AGENTS
 
 def process (line, count):
+  for agent in LOGWATCH_APACHE_IGNORE_AGENTS:
+    if argent in line: return True
   payload = {}
   items = line.split(" ")
   if len(items)<12: return True

diff --git a/es_cmssdt_apache.py b/es_cmssdt_apache.py
@@ -6,9 +6,11 @@
 from es_utils import send_payload
 from hashlib import sha1
 from json import dumps
-from logwatch import logwatch, run_cmd
+from logwatch import logwatch, run_cmd, LOGWATCH_APACHE_IGNORE_AGENTS
 
 def process (line, count):
+  for agent in LOGWATCH_APACHE_IGNORE_AGENTS:
+    if argent in line: return True
   payload = {}
   items = line.split(" ")
   if len(items)<10: return True

diff --git a/es_doxygen_apache.py b/es_doxygen_apache.py
@@ -6,9 +6,11 @@
 from es_utils import send_payload
 from hashlib import sha1
 from json import dumps
-from logwatch import logwatch, run_cmd
+from logwatch import logwatch, run_cmd, LOGWATCH_APACHE_IGNORE_AGENTS
 
 def process (line, count):
+  for agent in LOGWATCH_APACHE_IGNORE_AGENTS:
+    if argent in line: return True
   payload = {}
   items = line.split(" ")
   if len(items)<10: return True

diff --git a/logwatch.py b/logwatch.py
@@ -4,6 +4,8 @@
 from commands import getstatusoutput
 from hashlib import sha256
 
+LOGWATCH_APACHE_IGNORE_AGENTS = ["www.google.com/bot.html", "ahrefs.com/robot", "yandex.com/bots"]
+
 def run_cmd (cmd, exit_on_error=True):
   err, out = getstatusoutput (cmd)
   if err and exit_on_error: