INF-266 reroute nginx paths to new prometheus exporters (new exporter…

… images included) (#3959)

creator-node/nginx_conf/nginx.conf

@@ -29,6 +29,18 @@ http {
 					max_size=4g inactive=12h use_temp_path=off;
     proxy_read_timeout 3600; # 1 hour in seconds
 
+    upstream exporter-postgres {
+        server exporter_postgres:9187;
+    }
+
+    upstream exporter-redis {
+        server exporter_redis:9121;
+    }
+
+    upstream exporter-linux {
+        server host.docker.internal:9100;
+    }
+
     server {
         listen 4000;
 
@@ -96,6 +108,24 @@ http {
             add_header X-Cache-Status $upstream_cache_status always;
         }
 
+        location /prometheus/postgres {
+            proxy_pass http://exporter-postgres/metrics;
+            proxy_set_header Host $http_host;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        }
+
+        location /prometheus/redis {
+            proxy_pass http://exporter-redis/metrics;
+            proxy_set_header Host $http_host;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        }
+
+        location /prometheus/linux {
+            proxy_pass http://exporter-linux/metrics;
+            proxy_set_header Host $http_host;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        }
+
         # Pass all other requests to upstream server
         location / {
             proxy_pass http://127.0.0.1:3000;

discovery-provider/nginx_conf/nginx.conf

@@ -43,6 +43,26 @@ http {
         main.start_update_redirect_weights_timer()
     }
 
+    upstream exporter-postgres {
+        server exporter_postgres:9187;
+    }
+
+    upstream exporter-postgres-read-replica {
+        server exporter_postgres_read_replica:9187;
+    }
+
+    upstream exporter-elasticsearch {
+        server exporter_elasticsearch:9114;
+    }
+
+    upstream exporter-redis {
+        server exporter_redis:9121;
+    }
+
+    upstream exporter-linux {
+        server host.docker.internal:9100;
+    }
+
     server {
         listen 5000;
 
@@ -68,6 +88,45 @@ http {
             proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
         }
 
+        location /prometheus/postgres {
+            proxy_pass http://exporter-postgres/metrics;
+            proxy_set_header Host $http_host;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        }
+
+        location /prometheus/postgres/read-replica {
+            proxy_pass http://exporter-postgres-read-replica/metrics;
+            proxy_set_header Host $http_host;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        }
+
+        location /prometheus/elasticsearch {
+            proxy_pass http://exporter-elasticsearch/metrics;
+            proxy_set_header Host $http_host;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+
+            # don't expose IP addresses, by removing the host label
+            body_filter_by_lua_block {
+                local body = ngx.arg[1]
+                if body then
+                    body = ngx.re.gsub(body, ',(host=[^,/]*)', '')
+                end
+                ngx.arg[1] = body
+            }
+        }
+
+        location /prometheus/redis {
+            proxy_pass http://exporter-redis/metrics;
+            proxy_set_header Host $http_host;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        }
+
+        location /prometheus/linux {
+            proxy_pass http://exporter-linux/metrics;
+            proxy_set_header Host $http_host;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        }
+
         # Do not redirect any /v1/challenges/... requests, which need to resolve
         # to the node that the request was intended for. Selection of
         # nodes to respond to challenge attestations is intentional.

monitoring/exporters/postgres/Dockerfile

@@ -0,0 +1,6 @@
+# Manually built and pushed to https://hub.docker.com/r/audius/exporter-postgres
+FROM quay.io/prometheuscommunity/postgres-exporter:v0.10.1
+
+COPY docker-entrypoint.sh /bin/docker-entrypoint.sh
+ENTRYPOINT [ "/bin/docker-entrypoint.sh" ]
+CMD

monitoring/exporters/postgres/docker-entrypoint.sh

@@ -0,0 +1,19 @@
+#!/bin/sh
+
+# default case for discovery provider
+export DATA_SOURCE_NAME="${audius_db_url}?sslmode=disable"
+
+if [ "$*" = "--read-replica" ]; then
+    export DATA_SOURCE_NAME="${audius_db_url_read_replica}?sslmode=disable"
+fi
+
+if [ "$*" = "--creator-node" ]; then
+    export DATA_SOURCE_NAME=${dbUrl}?sslmode=disable
+fi
+
+if [ "$*" = "--identity-service" ]; then
+    export DATA_SOURCE_NAME=${dbUrl}?sslmode=disable
+fi
+
+# don't expose IP addresses, by overwriting the server label
+/bin/postgres_exporter --constantLabels server=server

monitoring/exporters/redis/Dockerfile

@@ -0,0 +1,6 @@
+# Manually built and pushed to https://hub.docker.com/r/audius/exporter-redis
+FROM quay.io/oliver006/redis_exporter:v1.44.0-alpine
+
+COPY docker-entrypoint.sh /docker-entrypoint.sh
+ENTRYPOINT [ "/docker-entrypoint.sh" ]
+CMD

monitoring/exporters/redis/docker-entrypoint.sh

@@ -0,0 +1,14 @@
+#!/bin/sh
+
+# default case for discovery provider
+export REDIS_ADDR=${audius_redis_url}
+
+if [ "$*" = "--creator-node" ]; then
+    export REDIS_ADDR=redis://${redisHost}:${redisPort}
+fi
+
+if [ "$*" = "--identity-service" ]; then
+    export REDIS_ADDR=redis://${redisHost}:${redisPort}
+fi
+
+/redis_exporter