cockroachdb · craig · Feb 15, 2022 · Feb 1, 2022
@@ -1314,6 +1314,7 @@ RangeProblems describes issues reported by a range. For internal use only.
 | no_lease | [bool](#cockroach.server.serverpb.RaftDebugResponse-bool) |  |  | [reserved](#support-status) |
 | quiescent_equals_ticking | [bool](#cockroach.server.serverpb.RaftDebugResponse-bool) |  | Quiescent ranges do not tick by definition, but we track this in two different ways and suspect that they're getting out of sync. If the replica's quiescent flag doesn't agree with the store's list of replicas that are ticking, warn about it. | [reserved](#support-status) |
 | raft_log_too_large | [bool](#cockroach.server.serverpb.RaftDebugResponse-bool) |  | When the raft log is too large, it can be a symptom of other issues. | [reserved](#support-status) |
+| circuit_breaker_error | [bool](#cockroach.server.serverpb.RaftDebugResponse-bool) |  |  | [reserved](#support-status) |
 
 
 
@@ -1520,6 +1521,7 @@ RangeProblems describes issues reported by a range. For internal use only.
 | no_lease | [bool](#cockroach.server.serverpb.RangesResponse-bool) |  |  | [reserved](#support-status) |
 | quiescent_equals_ticking | [bool](#cockroach.server.serverpb.RangesResponse-bool) |  | Quiescent ranges do not tick by definition, but we track this in two different ways and suspect that they're getting out of sync. If the replica's quiescent flag doesn't agree with the store's list of replicas that are ticking, warn about it. | [reserved](#support-status) |
 | raft_log_too_large | [bool](#cockroach.server.serverpb.RangesResponse-bool) |  | When the raft log is too large, it can be a symptom of other issues. | [reserved](#support-status) |
+| circuit_breaker_error | [bool](#cockroach.server.serverpb.RangesResponse-bool) |  |  | [reserved](#support-status) |
 
 
 
@@ -3049,6 +3051,7 @@ Support status: [reserved](#support-status)
 | overreplicated_range_ids | [int64](#cockroach.server.serverpb.ProblemRangesResponse-int64) | repeated |  | [reserved](#support-status) |
 | quiescent_equals_ticking_range_ids | [int64](#cockroach.server.serverpb.ProblemRangesResponse-int64) | repeated |  | [reserved](#support-status) |
 | raft_log_too_large_range_ids | [int64](#cockroach.server.serverpb.ProblemRangesResponse-int64) | repeated |  | [reserved](#support-status) |
+| circuit_breaker_error_range_ids | [int64](#cockroach.server.serverpb.ProblemRangesResponse-int64) | repeated |  | [reserved](#support-status) |
 
 
 
@@ -3344,6 +3347,7 @@ RangeProblems describes issues reported by a range. For internal use only.
 | no_lease | [bool](#cockroach.server.serverpb.RangeResponse-bool) |  |  | [reserved](#support-status) |
 | quiescent_equals_ticking | [bool](#cockroach.server.serverpb.RangeResponse-bool) |  | Quiescent ranges do not tick by definition, but we track this in two different ways and suspect that they're getting out of sync. If the replica's quiescent flag doesn't agree with the store's list of replicas that are ticking, warn about it. | [reserved](#support-status) |
 | raft_log_too_large | [bool](#cockroach.server.serverpb.RangeResponse-bool) |  | When the raft log is too large, it can be a symptom of other issues. | [reserved](#support-status) |
+| circuit_breaker_error | [bool](#cockroach.server.serverpb.RangeResponse-bool) |  |  | [reserved](#support-status) |
 
 
 

diff --git a/docs/generated/swagger/spec.json b/docs/generated/swagger/spec.json
@@ -1022,6 +1022,10 @@
       "type": "object",
       "title": "RangeProblems describes issues reported by a range. For internal use only.",
       "properties": {
+        "circuit_breaker_error": {
+          "type": "boolean",
+          "x-go-name": "CircuitBreakerError"
+        },
         "leader_not_lease_holder": {
           "type": "boolean",
           "x-go-name": "LeaderNotLeaseHolder"

@@ -129,6 +129,10 @@ func (s *statusServer) ProblemRanges(
 					problems.RaftLogTooLargeRangeIDs =
 						append(problems.RaftLogTooLargeRangeIDs, info.State.Desc.RangeID)
 				}
+				if info.Problems.CircuitBreakerError {
+					problems.CircuitBreakerErrorRangeIDs =
+						append(problems.CircuitBreakerErrorRangeIDs, info.State.Desc.RangeID)
+				}
 			}
 			sort.Sort(roachpb.RangeIDSlice(problems.UnavailableRangeIDs))
 			sort.Sort(roachpb.RangeIDSlice(problems.RaftLeaderNotLeaseHolderRangeIDs))
@@ -138,6 +142,7 @@ func (s *statusServer) ProblemRanges(
 			sort.Sort(roachpb.RangeIDSlice(problems.OverreplicatedRangeIDs))
 			sort.Sort(roachpb.RangeIDSlice(problems.QuiescentEqualsTickingRangeIDs))
 			sort.Sort(roachpb.RangeIDSlice(problems.RaftLogTooLargeRangeIDs))
+			sort.Sort(roachpb.RangeIDSlice(problems.CircuitBreakerErrorRangeIDs))
 			response.ProblemsByNodeID[resp.nodeID] = problems
 		case <-ctx.Done():
 			return nil, status.Errorf(codes.DeadlineExceeded, ctx.Err().Error())

@@ -389,6 +389,7 @@ message RangeProblems {
 
   // When the raft log is too large, it can be a symptom of other issues.
   bool raft_log_too_large = 7;
+  bool circuit_breaker_error = 9;
 }
 
 // RangeStatistics describes statistics reported by a range. For internal use
@@ -1044,6 +1045,11 @@ message ProblemRangesResponse {
       (gogoproto.casttype) =
           "github.com/cockroachdb/cockroach/pkg/roachpb.RangeID"
     ];
+    repeated int64 circuit_breaker_error_range_ids = 10 [
+      (gogoproto.customname) = "CircuitBreakerErrorRangeIDs",
+      (gogoproto.casttype) = 
+          "github.com/cockroachdb/cockroach/pkg/roachpb.RangeID"
+    ];
   }
   reserved 1 to 7;
   // NodeID is the node that submitted all the requests.

@@ -1993,6 +1993,7 @@ func (s *statusServer) rangesHelper(
 				NoLease:                metrics.Leader && !metrics.LeaseValid && !metrics.Quiescent,
 				QuiescentEqualsTicking: raftStatus != nil && metrics.Quiescent == metrics.Ticking,
 				RaftLogTooLarge:        metrics.RaftLogTooLarge,
+				CircuitBreakerError:    len(state.CircuitBreakerError) > 0,
 			},
 			LeaseStatus:                 metrics.LeaseStatus,
 			Quiescent:                   metrics.Quiescent,

@@ -71,6 +71,10 @@ const connectionTableColumns: ConnectionTableColumn[] = [
     title: "Raft log too large",
     extract: problem => problem.raft_log_too_large_range_ids.length,
   },
+  {
+    title: "Circuit breaker error",
+    extract: problem => problem.circuit_breaker_error_range_ids.length,
+  },
   {
     title: "Total",
     extract: problem => {
@@ -82,7 +86,8 @@ const connectionTableColumns: ConnectionTableColumn[] = [
         problem.underreplicated_range_ids.length +
         problem.overreplicated_range_ids.length +
         problem.quiescent_equals_ticking_range_ids.length +
-        problem.raft_log_too_large_range_ids.length
+        problem.raft_log_too_large_range_ids.length +
+        problem.circuit_breaker_error_range_ids.length
       );
     },
   },

@@ -203,6 +203,11 @@ export class ProblemRanges extends React.Component<ProblemRangesProps, {}> {
           problems={problems}
           extract={problem => problem.quiescent_equals_ticking_range_ids}
         />
+        <ProblemRangeList
+          name="Circuit breaker error"
+          problems={problems}
+          extract={problem => problem.circuit_breaker_error_range_ids}
+        />
       </div>
     );
   }

@@ -239,6 +239,11 @@ const rangeTableDisplayList: RangeTableRow[] = [
     display: "Closed timestamp LAI - side transport (centralized state)",
     compareToLeader: false,
   },
+  {
+    variable: "circuitBreakerError",
+    display: "Circuit Breaker Error",
+    compareToLeader: false,
+  },
 ];
 
 const rangeTableEmptyContent: RangeTableCellContent = {
@@ -854,6 +859,9 @@ export default class RangeTable extends React.Component<RangeTableProps, {}> {
             ? "range-table__cell--warning"
             : "",
         ),
+        circuitBreakerError: this.createContent(
+          info.state.circuit_breaker_error,
+        ),
       });
     });
 

@@ -311,6 +311,8 @@ $reports-table
   list-style-type none
   margin 0
   padding 0
+  white-space normal
+  overflow-x auto
 
 .debug-table
   margin 0 0 40px