Merge pull request #10 from smilebasti/9-container-auto-crashing

9 redirection only + monitoring exclusion list

README.md

@@ -28,10 +28,12 @@ Hope you enjoy
 1) create an influx Organisation called npmgrafstats
 2) Create a Bucket called npmgrafstats and a API-Token for npmgrafstats with write access
 3) Set HOME_IPS to your External/Public IP
-4) get your GeoLite2-City.mmdb from the geoliteupdate container (docker-compose file below) or download it to the /home/docker/geolite directory manually
-5) Start the docker container or docker compose with ajusted settings
-6) Add InfluxDB Bucket npmgrafstats as data source into grafana
-7) Download the dashboard file (NPM Map Dashboard v2.1.1.json) or import it with the ID: 18826 and set the new data source
+4) Set REDIRECTION_LOGS to TRUE for Reverse-Proxy and Redirection logs, to ONLY for only Redirection logs and FALSE for only Reverse-Proxy logs
+5) create the monitoringips.txt file and fill it with the IP's of Uptimerobot, Hetrixtools or similar services. (1 IP per row)(This is optional but is recommended if you monitor your domains via http(s) with external services).
+6) get your GeoLite2-City.mmdb from the geoliteupdate container (docker-compose file below) or download it to the /home/docker/geolite directory manually
+7) Start the docker container or docker-compose with ajusted settings
+8) Add InfluxDB Bucket npmgrafstats as data source into grafana
+9) Download the dashboard file (NPM Map Dashboard v2.1.1.json) or import it with the ID: 18826 and set the new data source in Grafana
 
 ## start docker on the same host where nginx proxy manger runs
 - In the following the working directory is /home/docker !
@@ -42,12 +44,13 @@ Hope you enjoy
 docker run --name npmgraf -it -d
 -v /home/docker/nginx-proxy-manager/data/logs:/logs \
 -v /home/docker/geolite:/geolite \
+-v /home/docker/monitoringips.txt:/monitoringips.txt \ # optional only mount if preexists and a wanted feature
 -e HOME_IPS=<replace with external IP> \
 -e INFLUX_HOST=<replace>:8086 \  # use host IP
 -e INFLUX_BUCKET=npmgrafstats \
 -e INFLUX_ORG=npmgrafstats \
 -e INFLUX_TOKEN=<replace> \
--e REDIRECTION_LOGS=<set> # set to TRUE or FALSE
+-e REDIRECTION_LOGS=<set> # set to TRUE or FALSE or ONLY
 smilebasti/npmgrafstats
 ```
 ### Docker Compose file
@@ -63,10 +66,11 @@ services:
       - INFLUX_BUCKET=npmgrafstats
       - INFLUX_ORG=npmgrafstats
       - INFLUX_TOKEN=<replace>
-      - REDIRECTION_LOGS=<set> # set to TRUE or FALSE
+      - REDIRECTION_LOGS=<set> # set to TRUE or FALSE or ONLY
     volumes:
       - /home/docker/nginx-proxy-manager/data/logs:/logs
       - /home/docker/geolite:/geolite
+      - /home/docker/monitoringips.txt:/monitoringips.txt # optional only mount if preexists and a wanted feature
 ```
 
 ## GeoLite2-City.mmdb Auto update

changelog.md

@@ -1,8 +1,15 @@
 # Changelog
 
 ### Todo list
-- use logtime and not hosttime to save the stats
 - Stop gathering after few days
+### Not planned features
+- maybe analys fallback logs
+
+## v2.2.1
+- add option REDIRECTION_LOGS='ONLY' for only redirection logs analysis
+- add version to startup logs
+- add an external list of ip's to be excluded for exclusion of monitoring ip's
+- move HOME_IPS to internal domain
 
 ## v2.2.0
 Removes duplicate logs of the same Connection in case of a restart of NpmGrafStats

docker-compose.yml

@@ -41,14 +41,15 @@ services:
     restart: unless-stopped
     environment:
       HOME_IPS: 'extrenal IP'
-      REDIRECTION_LOGS: '<set TRUE or FALSE>' # TRUE or FALSE
+      REDIRECTION_LOGS: '<set TRUE or FALSE>' # TRUE or FALSE or ONLY
       INFLUX_HOST: '<replace>:8086' # use host IP
       INFLUX_BUCKET: 'npmgrafstats'
       INFLUX_ORG: 'npmgrafstats'
       INFLUX_TOKEN: '<replace>' # insert after first run and manual token creation
     volumes:
       - ./data/logs:/logs
       - ./geolite:/geolite
+      - ./monitoringips.txt:/monitoringips.txt # optional only mount if preexists and a wanted feature
     depends_on:
       - geoipupdate
       - npm

sendips.sh

@@ -4,10 +4,17 @@
 # To better understand the collection with regular expression
 # {1,3} get one to three characters. [0-9] character from to 0 to 9. \ for special characters. () grouping as an expression. | or
 internalips="(10([\.][0-9]{1,3}){3})|(192.168([\.][0-9]{1,3}){2})|(172.(1[6-9]|2[0-9]|3[0-1])([\.][0-9]{1,3}){2})"
+# check if monitoringips.txt exists
+if [ -f "/monitoringips.txt" ]
+then
+  monitorfile=true
+else 
+  monitorfile=false
+fi
 
 # gets all lines including an IP. 
 # Grep finds the the IP addresses without the HOME_IPS in the access.log
-tail -f /logs/proxy-host-*_access.log | grep --line-buffered -E "([0-9]{1,3}[\.]){3}[0-9]{1,3}" | grep --line-buffered -v "$HOME_IPS" | while read line;
+tail -f /logs/proxy-host-*_access.log | grep --line-buffered -E "([0-9]{1,3}[\.]){3}[0-9]{1,3}" | while read line;
 
 do
   # Domain or subdomain gets found.
@@ -20,6 +27,12 @@ do
   if [[ $outsideip =~ $internalips ]]
   then
     echo "Internal IP-Source: $outsideip called: $targetdomain"
+  elif [[ $outsideip =~ $HOME_IPS ]]
+  then
+    echo "Internal IP-Source called: $targetdomain"
+  elif $monitorfile && grep -qFx $outsideip /monitoringips.txt
+  then
+    echo "A monitoring service checked: $targetdomain"
   else      
     # head -2 and tail -1 because grep finds two (sometimes three) and only the second is needed
     targetip=`echo $line | grep -o -m 1 -E "(([0-9]{1,3}[\.]){3}[0-9]{1,3}|([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]))" | head -2| tail -1` 

sendredirectionips.sh

@@ -4,10 +4,17 @@
 # To better understand the collection with regular expression
 # {1,3} get one to three characters. [0-9] character from to 0 to 9. \ for special characters. () grouping as an expression. | or
 internalips="(10([\.][0-9]{1,3}){3})|(192.168([\.][0-9]{1,3}){2})|(172.(1[6-9]|2[0-9]|3[0-1])([\.][0-9]{1,3}){2})"
+# check if monitoringips.txt exists
+if [ -f "/monitoringips.txt" ]
+then
+  monitorfile=true
+else 
+  monitorfile=false
+fi
 
 # gets all lines including an IP. 
 # Grep finds the the IP addresses without the HOME_IPS in the access.log
-tail -f /logs/redirection-host-*_access.log | grep --line-buffered -E "([0-9]{1,3}[\.]){3}[0-9]{1,3}" | grep --line-buffered -v "$HOME_IPS" | while read line;
+tail -f /logs/redirection-host-*_access.log | grep --line-buffered -E "([0-9]{1,3}[\.]){3}[0-9]{1,3}" | while read line;
 
 do
   # Domain or subdomain gets found.
@@ -16,10 +23,16 @@ do
   # Get the first ip found = outsideip
   # head -1 because grep finds two (sometimes three) and only the first is needed
   outsideip=`echo $line | grep -o -m 1 -E "(([0-9]{1,3}[\.]){3}[0-9]{1,3}|([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]))" | head -1` 
-
-   if [[ $outsideip =~ $internalips ]]
+
+  if [[ $outsideip =~ $internalips ]]
+  then
+    echo "Internal IP-Source: $outsideip called: $targetdomain"
+  elif [[ $outsideip =~ $HOME_IPS ]]
+  then
+    echo "Internal IP-Source called: $targetdomain"
+  elif $monitorfile && grep -qFx $outsideip /monitoringips.txt
   then
-    echo "Internal IP: $outsideip called for redirection: $targetdomain"
+    echo "A monitoring service checked: $targetdomain"
   else  
     #echo "external IP called redirection"    
 

start.sh

@@ -1,15 +1,21 @@
 #!/bin/bash
+echo "npmGrafStats: v2.2.1"
 echo "Startup: lets get logs data and send them to influx"
 
 
 if [ "$REDIRECTION_LOGS" = "TRUE" ]
 then
-    echo "Redirection Logs activated"
+    echo "Redirection and Reverse-Proxy Logs activated"
     bash /root/.config/NPMGRAF/sendips.sh &
     bash /root/.config/NPMGRAF/sendredirectionips.sh &
-
+
+elif [ "$REDIRECTION_LOGS" = "ONLY" ]
+then
+    echo "Only Redirection Logs activated"
+    bash /root/.config/NPMGRAF/sendredirectionips.sh
+
 else
-    echo "Redirection Logs deactivated"
+    echo "Only Reverse-Proxy Logs activated"
     bash /root/.config/NPMGRAF/sendips.sh
 fi