From 3636368de23ec90b2b23ed2d2168949aad56500f Mon Sep 17 00:00:00 2001 From: Lucian Tosa Date: Thu, 4 Sep 2025 15:44:24 +0200 Subject: [PATCH 1/5] Add --remove-destination to cp --- docker/mongodb-agent/setup-agent-files.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/mongodb-agent/setup-agent-files.sh b/docker/mongodb-agent/setup-agent-files.sh index 6a1b8a007..df9371a54 100755 --- a/docker/mongodb-agent/setup-agent-files.sh +++ b/docker/mongodb-agent/setup-agent-files.sh @@ -8,8 +8,8 @@ SCRIPTS_DIR="/opt/scripts" # readiness always returns failure setup_dummy_probes() { echo "Setting up dummy probe scripts..." - cp /usr/local/bin/dummy-probe.sh "$SCRIPTS_DIR/probe.sh" - cp /usr/local/bin/dummy-readinessprobe "$SCRIPTS_DIR/readinessprobe" + cp --remove-destination /usr/local/bin/dummy-probe.sh "$SCRIPTS_DIR/probe.sh" + cp --remove-destination /usr/local/bin/dummy-readinessprobe "$SCRIPTS_DIR/readinessprobe" echo "Dummy probe scripts ready" } From de648e32d40d093f7cc30ed702eede6fe3d2d28c Mon Sep 17 00:00:00 2001 From: Lucian Tosa Date: Thu, 4 Sep 2025 15:45:13 +0200 Subject: [PATCH 2/5] Dump logs of utilities container --- scripts/evergreen/e2e/dump_diagnostic_information.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/evergreen/e2e/dump_diagnostic_information.sh b/scripts/evergreen/e2e/dump_diagnostic_information.sh index f4ea7b986..c44824950 100755 --- a/scripts/evergreen/e2e/dump_diagnostic_information.sh +++ b/scripts/evergreen/e2e/dump_diagnostic_information.sh @@ -129,6 +129,7 @@ dump_pod_logs() { kubectl logs -n "${namespace}" "${pod}" -c "mongodb-agent-monitoring" > "logs/${prefix}${pod}-monitoring-agent-stdout.log" || true kubectl logs -n "${namespace}" "${pod}" -c "mongod" > "logs/${prefix}${pod}-mongod-container.log" || true kubectl logs -n "${namespace}" "${pod}" -c "mongodb-agent" > "logs/${prefix}${pod}-mongodb-agent-container.log" || true + kubectl logs -n "${namespace}" "${pod}" -c "mongodb-agent-operator-utilities" > "logs/${prefix}${pod}-mongodb-utilities-container.log" || true kubectl cp "${namespace}/${pod}:/var/log/mongodb-mms-automation/mongodb.log" "logs/${prefix}${pod}-mongodb.log" &> /dev/null || true # note that this file may get empty if the logs have already grew too much - seems it's better to have it explicitly empty then just omit From 3b88094ae46c76817c102df92d5b423e219372b8 Mon Sep 17 00:00:00 2001 From: Lucian Tosa Date: Thu, 4 Sep 2025 16:24:59 +0200 Subject: [PATCH 3/5] Dump logs of each container always --- .../e2e/dump_diagnostic_information.sh | 30 ++++++++----------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/scripts/evergreen/e2e/dump_diagnostic_information.sh b/scripts/evergreen/e2e/dump_diagnostic_information.sh index c44824950..5b889621a 100755 --- a/scripts/evergreen/e2e/dump_diagnostic_information.sh +++ b/scripts/evergreen/e2e/dump_diagnostic_information.sh @@ -126,30 +126,26 @@ dump_pod_logs() { kubectl cp "${namespace}/${pod}:/var/log/mongodb-mms-automation/automation-agent.log" "logs/${prefix}${pod}-agent.log" &> /dev/null kubectl cp "${namespace}/${pod}:/var/log/mongodb-mms-automation/monitoring-agent-verbose.log" "logs/${prefix}${pod}-monitoring-agent-verbose.log" &> /dev/null kubectl cp "${namespace}/${pod}:/var/log/mongodb-mms-automation/monitoring-agent.log" "logs/${prefix}${pod}-monitoring-agent.log" &> /dev/null - kubectl logs -n "${namespace}" "${pod}" -c "mongodb-agent-monitoring" > "logs/${prefix}${pod}-monitoring-agent-stdout.log" || true - kubectl logs -n "${namespace}" "${pod}" -c "mongod" > "logs/${prefix}${pod}-mongod-container.log" || true - kubectl logs -n "${namespace}" "${pod}" -c "mongodb-agent" > "logs/${prefix}${pod}-mongodb-agent-container.log" || true - kubectl logs -n "${namespace}" "${pod}" -c "mongodb-agent-operator-utilities" > "logs/${prefix}${pod}-mongodb-utilities-container.log" || true kubectl cp "${namespace}/${pod}:/var/log/mongodb-mms-automation/mongodb.log" "logs/${prefix}${pod}-mongodb.log" &> /dev/null || true # note that this file may get empty if the logs have already grew too much - seems it's better to have it explicitly empty then just omit kubectl logs -n "${namespace}" "${pod}" | jq -c -r 'select( .logType == "agent-launcher-script") | .contents' 2> /dev/null > "logs/${prefix}${pod}-launcher.log" - else - # for all other pods we want each log per container from kubectl - for container in $(kubectl get pods -n "${namespace}" "${pod}" -o jsonpath='{.spec.containers[*].name}'); do - echo "Writing log file for pod ${pod} - container ${container} to logs/${pod}-${container}.log" - kubectl logs -n "${namespace}" "${pod}" -c "${container}" > "logs/${pod}-${container}.log" + fi - # Check if the container has restarted by examining its restart count - restartCount=$(kubectl get pod -n "${namespace}" "${pod}" -o jsonpath="{.status.containerStatuses[?(@.name=='${container}')].restartCount}") + for container in $(kubectl get pods -n "${namespace}" "${pod}" -o jsonpath='{.spec.containers[*].name}'); do + echo "Writing log file for pod ${pod} - container ${container} to logs/${pod}-${container}.log" + kubectl logs -n "${namespace}" "${pod}" -c "${container}" > "logs/${pod}-${container}.log" - if [ "${restartCount}" -gt 0 ]; then - echo "Writing log file for restarted ${pod} - container ${container} to logs/${pod}-${container}-previous.log" - kubectl logs --previous -n "${namespace}" "${pod}" -c "${container}" > "logs/${pod}-${container}-previous.log" || true - fi + # Check if the container has restarted by examining its restart count + restartCount=$(kubectl get pod -n "${namespace}" "${pod}" -o jsonpath="{.status.containerStatuses[?(@.name=='${container}')].restartCount}") + + if [ "${restartCount}" -gt 0 ]; then + echo "Writing log file for restarted ${pod} - container ${container} to logs/${pod}-${container}-previous.log" + kubectl logs --previous -n "${namespace}" "${pod}" -c "${container}" > "logs/${pod}-${container}-previous.log" || true + fi + + done - done - fi if kubectl exec "${pod}" -n "${namespace}" -- ls /var/log/mongodb-mms-automation/automation-agent-stderr.log &>/dev/null; then kubectl cp "${namespace}/${pod}:/var/log/mongodb-mms-automation/automation-agent-stderr.log" "logs/${prefix}${pod}-agent-stderr.log" &> /dev/null From 97015fcd6f619b00a7bd9eae5cfe776d4a92d4b3 Mon Sep 17 00:00:00 2001 From: Lucian Tosa Date: Thu, 4 Sep 2025 16:37:39 +0200 Subject: [PATCH 4/5] Changelog --- ...20250904_fix_fix_individual_agent_container_restarts.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 changelog/20250904_fix_fix_individual_agent_container_restarts.md diff --git a/changelog/20250904_fix_fix_individual_agent_container_restarts.md b/changelog/20250904_fix_fix_individual_agent_container_restarts.md new file mode 100644 index 000000000..ea5d4806a --- /dev/null +++ b/changelog/20250904_fix_fix_individual_agent_container_restarts.md @@ -0,0 +1,7 @@ +--- +title: Fix individual agent container restarts +kind: fix +date: 2025-09-04 +--- + +* Fixed a bug where the agent container in the static containers architecture would enter a CrashLoopBackOff state if the container restarted. From 60202228ad64db3523661967bdf0a78ab6e6c5be Mon Sep 17 00:00:00 2001 From: Lucian Tosa Date: Thu, 4 Sep 2025 17:28:51 +0200 Subject: [PATCH 5/5] Remove changelog --- ...20250904_fix_fix_individual_agent_container_restarts.md | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 changelog/20250904_fix_fix_individual_agent_container_restarts.md diff --git a/changelog/20250904_fix_fix_individual_agent_container_restarts.md b/changelog/20250904_fix_fix_individual_agent_container_restarts.md deleted file mode 100644 index ea5d4806a..000000000 --- a/changelog/20250904_fix_fix_individual_agent_container_restarts.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -title: Fix individual agent container restarts -kind: fix -date: 2025-09-04 ---- - -* Fixed a bug where the agent container in the static containers architecture would enter a CrashLoopBackOff state if the container restarted.