From 990e7a40c9ff455c428b582b04f082de88443a5c Mon Sep 17 00:00:00 2001 From: jsn Date: Fri, 14 May 2021 13:40:19 -0500 Subject: [PATCH 1/4] add python-requests to matomo botlist --- syslog/bin/import_logs.py | 1 + 1 file changed, 1 insertion(+) diff --git a/syslog/bin/import_logs.py b/syslog/bin/import_logs.py index e3e89be4d3..b57999d154 100755 --- a/syslog/bin/import_logs.py +++ b/syslog/bin/import_logs.py @@ -100,6 +100,7 @@ 'netcraftsurvey', 'panopta', 'pingdom.com_bot_', + 'python-requests/', 'robot', 'spider', 'surveybot', From bb64e296dd9998de58f776bf246bef6117b95f4a Mon Sep 17 00:00:00 2001 From: jsn Date: Tue, 25 May 2021 12:35:20 -0500 Subject: [PATCH 2/4] Use 1 Matomo log analysis argument per line --- syslog/bin/matomo.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/syslog/bin/matomo.sh b/syslog/bin/matomo.sh index 6438e30425..401ccf7d09 100755 --- a/syslog/bin/matomo.sh +++ b/syslog/bin/matomo.sh @@ -8,7 +8,9 @@ while IFS= read -r line; do echo ${line} | /srv/syslog/bin/import_logs.py \ --url=https://${MATOMO_FQDN}/ --token-auth=${MATOMO_AUTH_TOKEN} \ --idsite=${MATOMO_SITEID} --recorders=4 \ - --enable-http-errors --enable-http-redirects --enable-bots \ + --enable-http-errors \ + --enable-http-redirects \ + --enable-bots \ --log-format-name=nginx_json - fi done From 7d08808c8359d2df66e961dd963d177689531b2a Mon Sep 17 00:00:00 2001 From: jsn Date: Tue, 25 May 2021 12:36:46 -0500 Subject: [PATCH 3/4] Matomo log analysis ignores requests from the botlist. --- syslog/bin/matomo.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/syslog/bin/matomo.sh b/syslog/bin/matomo.sh index 401ccf7d09..0ce3a9a5bf 100755 --- a/syslog/bin/matomo.sh +++ b/syslog/bin/matomo.sh @@ -10,7 +10,6 @@ while IFS= read -r line; do --idsite=${MATOMO_SITEID} --recorders=4 \ --enable-http-errors \ --enable-http-redirects \ - --enable-bots \ --log-format-name=nginx_json - fi done From b9b71aabc66eb49230c37e44bf18c1c862879971 Mon Sep 17 00:00:00 2001 From: jsn Date: Tue, 25 May 2021 12:40:40 -0500 Subject: [PATCH 4/4] add apache-httpclient to matomo botlist --- syslog/bin/import_logs.py | 1 + 1 file changed, 1 insertion(+) diff --git a/syslog/bin/import_logs.py b/syslog/bin/import_logs.py index b57999d154..40edc11b49 100755 --- a/syslog/bin/import_logs.py +++ b/syslog/bin/import_logs.py @@ -80,6 +80,7 @@ # https://github.com/matomo-org/device-detector/blob/master/regexes/bots.yml # user agents must be lowercase EXCLUDED_USER_AGENTS = ( + 'apache-httpclient', 'adsbot-google', 'ask jeeves', 'baidubot',