Merge pull request #13431 from bharathv/crash_loop_default

config: enable crash_loop_limit by default
redpanda-data · Sep 19, 2023 · 3ad3700 · 3ad3700
2 parents b52b012 + 12b5afa
commit 3ad3700
Show file tree

Hide file tree

Showing 3 changed files with 19 additions and 8 deletions.
diff --git a/src/v/config/node_config.cc b/src/v/config/node_config.cc
@@ -134,9 +134,10 @@ node_config::node_config() noexcept
       *this,
       "crash_loop_limit",
       "Maximum consecutive crashes (unclean shutdowns) allowed after which "
-      "operator intervention is needed to startup the broker.",
+      "operator intervention is needed to startup the broker. Limit is not "
+      "enforced in developer mode.",
       {.visibility = visibility::user},
-      std::nullopt)
+      5)
   , upgrade_override_checks(
       *this,
       "upgrade_override_checks",

diff --git a/src/v/redpanda/application.cc b/src/v/redpanda/application.cc
@@ -747,6 +747,11 @@ void application::check_environment() {
 /// tracker file. This is to prevent on disk state from piling up in
 /// each unclean run and creating more state to recover for the next run.
 void application::check_for_crash_loop() {
+    if (config::node().developer_mode()) {
+        // crash loop tracking has value only in long running clusters
+        // that can potentially accumulate state across restarts.
+        return;
+    }
     auto file_path = config::node().data_directory().path
                      / crash_loop_tracker_file;
     std::optional<crash_tracker_metadata> maybe_crash_md;
@@ -835,11 +840,15 @@ void application::schedule_crash_tracker_file_cleanup() {
     // next run.
     // We emplace it in the front to make it the last task to run.
     _deferred.emplace_front([&] {
-        auto file = config::node().data_directory().path
-                    / crash_loop_tracker_file;
-        ss::remove_file(file.string()).get();
-        ss::sync_directory(config::node().data_directory().as_sstring()).get();
-        vlog(_log.debug, "Deleted crash loop tracker file: {}", file);
+        auto file = (config::node().data_directory().path
+                     / crash_loop_tracker_file)
+                      .string();
+        if (ss::file_exists(file).get()) {
+            ss::remove_file(file).get();
+            ss::sync_directory(config::node().data_directory().as_sstring())
+              .get();
+            vlog(_log.debug, "Deleted crash loop tracker file: {}", file);
+        }
     });
 }
 

diff --git a/tests/rptest/tests/crash_loop_checks_test.py b/tests/rptest/tests/crash_loop_checks_test.py
@@ -37,7 +37,8 @@ def __init__(self, test_context):
                              num_brokers=1,
                              extra_node_conf={
                                  "crash_loop_limit":
-                                 CrashLoopChecksTest.CRASH_LOOP_LIMIT
+                                 CrashLoopChecksTest.CRASH_LOOP_LIMIT,
+                                 "developer_mode": False
                              })
 
     def remove_crash_loop_tracker_file(self, broker):