feat(BOUN-999): some nginx-related tweaks, cleanups

ic-os/boundary-guestos/rootfs/etc/nginx/conf.d/000-nginx-global.conf

@@ -88,22 +88,20 @@ geo $rosetta {
 
 upstream icx_proxy {
     server 127.0.0.1:9500;
-    keepalive 16;
+    keepalive 32;
+    keepalive_time 10m;
 }
 
 upstream ic_boundary {
     server 127.0.0.1:4444;
-    keepalive 16;
+    keepalive 32;
+    keepalive_time 10m;
 }
 
 upstream cert_issuer {
     server 127.0.0.1:3000;
     keepalive 4;
-}
-
-map $remote_addr $traffic_segment {
-    default   "external";
-    127.0.0.1 "internal";
+    keepalive_time 10m;
 }
 
 add_header X-Frame-Options "DENY" always;
@@ -141,8 +139,8 @@ root /var/www/html;
 
 # Any direct HTTPS access without correct domain name will default to returning a 404.
 server {
-    listen       443 ssl;
-    listen       [::]:443 ssl ipv6only=on; # setting ipv6only=on once turns it on for all instances on that port
+    listen       443 ssl reuseport;
+    listen       [::]:443 ssl ipv6only=on reuseport; # setting ipv6only=on once turns it on for all instances on that port
 
     server_name  _;
 
@@ -151,8 +149,8 @@ server {
 
 # Any direct HTTP traffic will be redirected to HTTPS via 301.
 server {
-    listen       80;
-    listen       [::]:80 ipv6only=on; # setting ipv6only=on once turns it on for all instances on that port
+    listen       80 reuseport;
+    listen       [::]:80 ipv6only=on reuseport; # setting ipv6only=on once turns it on for all instances on that port
 
     server_name  _;
 

ic-os/boundary-guestos/rootfs/etc/nginx/conf.d/002-mainnet-nginx.conf

@@ -1,5 +1,3 @@
-include "/var/opt/nginx/ic/ic_upstreams.conf";
-
 # Server blocks for custom domains
 include "/var/opt/nginx/domains.conf";
 

ic-os/boundary-guestos/rootfs/etc/nginx/includes/proxy_headers.conf

@@ -4,3 +4,13 @@ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
 proxy_set_header X-Forwarded-Proto $scheme;
 proxy_set_header X-Request-ID $request_uuid;
 proxy_set_header Connection "";
+
+proxy_hide_header x-ic-error-cause;
+proxy_hide_header x-ic-cache-bypass-reason;
+proxy_hide_header x-ic-node-id;
+proxy_hide_header x-ic-request-type;
+proxy_hide_header x-ic-subnet-type;
+proxy_hide_header x-ic-canister-id;
+proxy_hide_header x-ic-sender;
+proxy_hide_header x-ic-retries;
+proxy_hide_header x-ic-method-name;

ic-os/boundary-guestos/rootfs/etc/nginx/nginx.conf

@@ -1,5 +1,5 @@
 user www-data;
-worker_processes 6;
+worker_processes auto;
 pid /run/nginx.pid;
 include /etc/nginx/modules-enabled/*.conf;
 
@@ -17,7 +17,24 @@ http {
     sendfile on;
     tcp_nopush on;
     tcp_nodelay on;
-    keepalive_timeout 65;
+
+    # Keepalive settings
+    ## Idle connection timeout
+    keepalive_timeout 30; # default 75s
+
+    ## Close keepalive connections more frequently to free per-connection resources
+    keepalive_time 10m; # default 1h
+    keepalive_requests 500; # default 1000
+
+    # Client timeouts
+    ## Full header reading timeout
+    client_header_timeout 15s; # default 60s
+    ## Timeout between two read operations from the client, not full body transmit time
+    client_body_timeout 15s; # default 60s
+    ## Timeout between two write operations to the client, not full body transmit time
+    send_timeout 15s; # default 60s
+    ## More aggressively close timed out connections, don't linger in FIN_WAIT1
+    reset_timedout_connection on;
 
     # Do not emit server version in headers
     server_tokens off;
@@ -116,12 +133,12 @@ http {
     log_format access escape=json '{'
         '"body_bytes_sent":'          '"$body_bytes_sent",'
         '"bytes_sent":'               '"$bytes_sent",'
-        '"cache_bypass_reason":'      '"$sent_http_x_ic_cache_bypass_reason",'
-        '"cache_status":'             '"$sent_http_x_ic_cache_status",'
+        '"cache_bypass_reason":'      '"$upstream_http_x_ic_cache_bypass_reason",'
+        '"cache_status":'             '"$upstream_http_x_ic_cache_status",'
         '"cache_status_nginx":'       '"$upstream_cache_status",'
         '"content_length":'           '"$content_length",'
         '"content_type":'             '"$content_type",'
-        '"error_cause":'              '"$sent_http_x_ic_error_cause",'
+        '"error_cause":'              '"$upstream_http_x_ic_error_cause",'
         '"geo_city_name":'            '"$geo_city_name",'
         '"geo_country_code":'         '"$geo_country_code",'
         '"geo_country_name":'         '"$geo_country_name",'
@@ -132,18 +149,18 @@ http {
         '"http_user_agent":'          '"$http_user_agent",'
         '"https":'                    '"$https",'
         '"ic_canister_id":'           '"$inferred_canister_id",'
-        '"ic_canister_id_cbor":'      '"$sent_http_x_ic_canister_id",'
-        '"ic_method_name":'           '"$sent_http_x_ic_method_name",'
-        '"ic_node_id":'               '"$sent_http_x_ic_node_id",'
-        '"ic_request_type":'          '"$sent_http_x_ic_request_type",'
-        '"ic_sender":'                '"$sent_http_x_ic_sender",'
-        '"ic_subnet_id":'             '"$sent_http_x_ic_subnet_id",'
+        '"ic_canister_id_cbor":'      '"$upstream_http_x_ic_canister_id",'
+        '"ic_method_name":'           '"$upstream_http_x_ic_method_name",'
+        '"ic_node_id":'               '"$upstream_http_x_ic_node_id",'
+        '"ic_request_type":'          '"$upstream_http_x_ic_request_type",'
+        '"ic_sender":'                '"$upstream_http_x_ic_sender",'
+        '"ic_subnet_id":'             '"$upstream_http_x_ic_subnet_id",'
         '"is_bot":'                   '"$is_bot",'
         '"msec":'                     '"$msec",'
         '"pre_isolation_canister":'   '"$is_pre_isolation_canister",'
         '"query_string":'             '"$query_string",'
         '"remote_addr":'              '"$remote_addr",'
-        '"retries":'                  '"$sent_http_x_ic_retries",'
+        '"retries":'                  '"$upstream_http_x_ic_retries",'
         '"request_id":'               '"$request_uuid",'
         '"request_length":'           '"$request_length",'
         '"request_method":'           '"$request_method",'

ic-os/boundary-guestos/rootfs/opt/ic/bin/setup-nginx.sh

@@ -200,35 +200,6 @@ function setup_geolite2_dbs() {
     done
 }
 
-function setup_ic_router() {
-    local -r SNAKEOIL_PEM='/etc/ssl/certs/ssl-cert-snakeoil.pem'
-    local -r IC_ROUTING='/var/opt/nginx/ic'
-    local -r IC_LEGACY_ROUTING='/var/cache/ic_routes'
-    local -r TRUSTED_CERTS="${IC_ROUTING}/trusted_certs.pem"
-    local -r NGINX_TABLE="${IC_ROUTING}/ic_upstreams.conf"
-    local -r IC_ROUTER_TABLE="${IC_ROUTING}/ic_routes.js"
-
-    # Place to store the generated routing tables
-    mkdir -p "${IC_ROUTING}" "${IC_LEGACY_ROUTING}"
-
-    # trusted_cert.pem contains all certificates for the upstream replica. This file
-    # is periodically updated by the proxy+watcher service. To bootstrap the process
-    # we initially place a dummy trusted cert. This dummy is the copy of the
-    # snakeoil cert. This allows the nginx service to start, but upstream routing
-    # will only happen once the control plane pulls the initial set of routes
-    if [[ ! -f "${TRUSTED_CERTS}" ]]; then
-        cp "${SNAKEOIL_PEM}" "${TRUSTED_CERTS}"
-    fi
-
-    if [[ ! -f "${NGINX_TABLE}" ]]; then
-        echo '# PLACEHOLDER' >"${NGINX_TABLE}"
-    fi
-
-    if [[ ! -f "${IC_ROUTER_TABLE}" ]]; then
-        echo "${EMPTY_NJS_EXPORTS}" >"${IC_ROUTER_TABLE}"
-    fi
-}
-
 function setup_custom_domains() {
     local -r SERVER_BLOCKS='/var/opt/nginx/domains.conf'
     mkdir -p "$(dirname ${SERVER_BLOCKS})"
@@ -299,7 +270,6 @@ function main() {
     copy_deny_list
     setup_domains
     setup_geolite2_dbs
-    setup_ic_router
     setup_custom_domains
     setup_pre_isolation_canisters
     setup_canister_id_alises