Helm: prevent query-scheduler from terminating connections

operations/helm/charts/mimir-distributed/CHANGELOG.md

@@ -104,6 +104,7 @@ Entries should include a reference to the Pull Request that introduced the chang
 * [ENHANCEMENT] Add podAntiAffinity to sizing plans (small.yaml, large.yaml, capped-small.yaml, capped-large.yaml). #2906
 * [ENHANCEMENT] Add ability to configure and run mimir-continuous-test. #3117
 * [BUGFIX] Fix wrong label selector in ingester anti affinity rules in the sizing plans. #2906
+* [BUGFIX] Query-scheduler no longer periodically terminates connections from query-frontends and queriers. This caused some queries to time out and EOF errors in the logs. #3262
 
 ## 3.1.0
 

operations/helm/charts/mimir-distributed/templates/query-scheduler/query-scheduler-dep.yaml

@@ -44,6 +44,8 @@ spec:
             - "-target=query-scheduler"
             - "-config.expand-env=true"
             - "-config.file=/etc/mimir/mimir.yaml"
+            - "-server.grpc.keepalive.max-connection-age=2562047h" # 100000 days, effectively infinity
+            - "-server.grpc.keepalive.max-connection-age-grace=2562047h" # 100000 days, effectively infinity
           {{- range $key, $value := .Values.query_scheduler.extraArgs }}
             - "-{{ $key }}={{ $value }}"
           {{- end }}

operations/helm/tests/graphite-enabled-values-generated/mimir-distributed/templates/query-scheduler/query-scheduler-dep.yaml

@@ -45,6 +45,8 @@ spec:
             - "-target=query-scheduler"
             - "-config.expand-env=true"
             - "-config.file=/etc/mimir/mimir.yaml"
+            - "-server.grpc.keepalive.max-connection-age=2562047h" # 100000 days, effectively infinity
+            - "-server.grpc.keepalive.max-connection-age-grace=2562047h" # 100000 days, effectively infinity
           volumeMounts:
             - name: runtime-config
               mountPath: /var/mimir

operations/helm/tests/metamonitoring-values-generated/mimir-distributed/templates/query-scheduler/query-scheduler-dep.yaml

@@ -45,6 +45,8 @@ spec:
             - "-target=query-scheduler"
             - "-config.expand-env=true"
             - "-config.file=/etc/mimir/mimir.yaml"
+            - "-server.grpc.keepalive.max-connection-age=2562047h" # 100000 days, effectively infinity
+            - "-server.grpc.keepalive.max-connection-age-grace=2562047h" # 100000 days, effectively infinity
           volumeMounts:
             - name: runtime-config
               mountPath: /var/mimir

operations/helm/tests/test-enterprise-configmap-values-generated/mimir-distributed/templates/query-scheduler/query-scheduler-dep.yaml

@@ -46,6 +46,8 @@ spec:
             - "-target=query-scheduler"
             - "-config.expand-env=true"
             - "-config.file=/etc/mimir/mimir.yaml"
+            - "-server.grpc.keepalive.max-connection-age=2562047h" # 100000 days, effectively infinity
+            - "-server.grpc.keepalive.max-connection-age-grace=2562047h" # 100000 days, effectively infinity
           volumeMounts:
             - name: runtime-config
               mountPath: /var/mimir

operations/helm/tests/test-enterprise-legacy-label-values-generated/mimir-distributed/templates/query-scheduler/query-scheduler-dep.yaml

@@ -43,6 +43,8 @@ spec:
             - "-target=query-scheduler"
             - "-config.expand-env=true"
             - "-config.file=/etc/mimir/mimir.yaml"
+            - "-server.grpc.keepalive.max-connection-age=2562047h" # 100000 days, effectively infinity
+            - "-server.grpc.keepalive.max-connection-age-grace=2562047h" # 100000 days, effectively infinity
           volumeMounts:
             - name: runtime-config
               mountPath: /var/enterprise-metrics

operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/query-scheduler/query-scheduler-dep.yaml

@@ -45,6 +45,8 @@ spec:
             - "-target=query-scheduler"
             - "-config.expand-env=true"
             - "-config.file=/etc/mimir/mimir.yaml"
+            - "-server.grpc.keepalive.max-connection-age=2562047h" # 100000 days, effectively infinity
+            - "-server.grpc.keepalive.max-connection-age-grace=2562047h" # 100000 days, effectively infinity
           volumeMounts:
             - name: runtime-config
               mountPath: /var/mimir

operations/helm/tests/test-oss-logical-multizone-values-generated/mimir-distributed/templates/query-scheduler/query-scheduler-dep.yaml

@@ -45,6 +45,8 @@ spec:
             - "-target=query-scheduler"
             - "-config.expand-env=true"
             - "-config.file=/etc/mimir/mimir.yaml"
+            - "-server.grpc.keepalive.max-connection-age=2562047h" # 100000 days, effectively infinity
+            - "-server.grpc.keepalive.max-connection-age-grace=2562047h" # 100000 days, effectively infinity
           volumeMounts:
             - name: runtime-config
               mountPath: /var/mimir

operations/helm/tests/test-oss-multizone-values-generated/mimir-distributed/templates/query-scheduler/query-scheduler-dep.yaml

@@ -45,6 +45,8 @@ spec:
             - "-target=query-scheduler"
             - "-config.expand-env=true"
             - "-config.file=/etc/mimir/mimir.yaml"
+            - "-server.grpc.keepalive.max-connection-age=2562047h" # 100000 days, effectively infinity
+            - "-server.grpc.keepalive.max-connection-age-grace=2562047h" # 100000 days, effectively infinity
           volumeMounts:
             - name: runtime-config
               mountPath: /var/mimir

operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/query-scheduler/query-scheduler-dep.yaml

@@ -46,6 +46,8 @@ spec:
             - "-target=query-scheduler"
             - "-config.expand-env=true"
             - "-config.file=/etc/mimir/mimir.yaml"
+            - "-server.grpc.keepalive.max-connection-age=2562047h" # 100000 days, effectively infinity
+            - "-server.grpc.keepalive.max-connection-age-grace=2562047h" # 100000 days, effectively infinity
           volumeMounts:
             - name: runtime-config
               mountPath: /var/mimir