From 4f35e2185f68bd0e0b80246433e3bd6161064148 Mon Sep 17 00:00:00 2001 From: jirka <6035284+Borda@users.noreply.github.com> Date: Tue, 7 Jul 2026 13:43:59 +0200 Subject: [PATCH 1/2] fix(tracker): harden ByteTrack edge cases - Keep ByteTrack confidence-threshold boundary detections eligible and avoid impossible activation thresholds above score 1.0. - Stop mutating caller-owned detections and assignment cost matrices while preserving matched tracker output. - Filter invalid tensor boxes before Kalman updates and respect minimum consecutive frames on first-frame tensor updates. --- Co-authored-by: Codex --- docs/changelog.md | 1 + src/supervision/tracker/byte_tracker/core.py | 22 ++++- .../tracker/byte_tracker/matching.py | 6 +- .../byte_tracker/single_object_track.py | 11 +-- tests/tracker/test_byte_tracker.py | 99 ++++++++++++++++--- 5 files changed, 115 insertions(+), 24 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index fb3ddb3ba9..5ab523b861 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -18,6 +18,7 @@ date_modified: 2026-07-06 - `sv.mask_non_max_merge` now computes exact mask overlap at the original mask resolution and ignores the deprecated `mask_dimension` parameter. Code that relied on downscaled mask overlap should recalibrate thresholds; passing `mask_dimension` positionally now emits a deprecation warning, and the parameter is scheduled for removal in `0.33.0` ([#2400](https://github.com/roboflow/supervision/pull/2400)). ### Fixed +- `sv.ByteTrack` no longer mutates input `Detections` while assigning tracker IDs. It now keeps detections at the activation-threshold boundary eligible for matching, avoids impossible new-track thresholds above score `1.0`, ignores invalid zero-area/non-finite tensor boxes before Kalman updates, and does not emit unconfirmed `-1` IDs from first-frame tensor updates. - Fixed [#2402](https://github.com/roboflow/supervision/pull/2402): `sv.KeyPoints.as_detections` now accepts NumPy arrays, tuples, and generators in `selected_keypoint_indices` without ambiguous truth-value errors; empty index iterables select all keypoints. Valid zero-area skeletons are preserved, while all-zero and non-finite-only skeletons are filtered out. - Fixed [#2407](https://github.com/roboflow/supervision/pull/2407): `sv.ColorPalette.by_idx()` now raises a clear `ValueError` when called on an empty palette instead of leaking a `ZeroDivisionError`. Non-empty palettes keep the existing index-wrapping behavior. - Fixed [#2393](https://github.com/roboflow/supervision/pull/2393): `sv.CropAnnotator.annotate` no longer raises `cv2.error` when detections extend outside the scene; out-of-bounds boxes are clipped to scene bounds and zero-area results are skipped silently. diff --git a/src/supervision/tracker/byte_tracker/core.py b/src/supervision/tracker/byte_tracker/core.py index 249bc6c369..43548b6d67 100644 --- a/src/supervision/tracker/byte_tracker/core.py +++ b/src/supervision/tracker/byte_tracker/core.py @@ -13,6 +13,16 @@ from supervision.tracker.byte_tracker.utils import IdCounter +def _valid_tracking_tensors( + tensors: npt.NDArray[np.float32], +) -> npt.NDArray[np.bool_]: + """Identify finite tensors with positive-width and positive-height boxes.""" + bboxes = tensors[:, :4] + widths = bboxes[:, 2] - bboxes[:, 0] + heights = bboxes[:, 3] - bboxes[:, 1] + return np.isfinite(tensors).all(axis=1) & (widths > 0) & (heights > 0) + + @deprecated_class( target=TargetMode.NOTIFY, deprecated_in="0.28.0", @@ -65,6 +75,8 @@ def __init__( self.frame_id = 0 self.det_thresh = self.track_activation_threshold + 0.1 + if self.det_thresh > 1.0: + self.det_thresh = self.track_activation_threshold self.max_time_lost = int(frame_rate / 30.0 * lost_track_buffer) self.minimum_consecutive_frames = minimum_consecutive_frames self.kalman_filter = KalmanFilter() @@ -138,13 +150,14 @@ def callback(frame: np.ndarray, index: int) -> np.ndarray: iou_costs: npt.NDArray[np.float32] = 1 - ious matches, _, _ = matching.linear_assignment(iou_costs, 0.5) - detections.tracker_id = np.full(len(detections), -1, dtype=int) + tracked_detections = detections.select(np.arange(len(detections))) + tracked_detections.tracker_id = np.full(len(detections), -1, dtype=int) for i_detection, i_track in matches: - detections.tracker_id[i_detection] = int( + tracked_detections.tracker_id[i_detection] = int( tracks[i_track].external_track_id ) - return detections.select(detections.tracker_id != -1) + return tracked_detections.select(tracked_detections.tracker_id != -1) else: detections = Detections.empty() @@ -184,10 +197,11 @@ def update_with_tensors(self, tensors: npt.NDArray[np.float32]) -> list[STrack]: lost_stracks = [] removed_stracks = [] + tensors = tensors[_valid_tracking_tensors(tensors)] scores = tensors[:, 4] bboxes = tensors[:, :4] - remain_inds = scores > self.track_activation_threshold + remain_inds = scores >= self.track_activation_threshold inds_low = scores > 0.1 inds_high = scores < self.track_activation_threshold diff --git a/src/supervision/tracker/byte_tracker/matching.py b/src/supervision/tracker/byte_tracker/matching.py index b792d9df7a..b32871b772 100644 --- a/src/supervision/tracker/byte_tracker/matching.py +++ b/src/supervision/tracker/byte_tracker/matching.py @@ -23,6 +23,7 @@ def indices_to_matches( def linear_assignment( cost_matrix: npt.NDArray[np.float32], thresh: float ) -> tuple[npt.NDArray[np.int_], tuple[int, ...], tuple[int, ...]]: + """Match rows and columns without mutating the caller-owned cost matrix.""" if cost_matrix.size == 0: return ( np.empty((0, 2), dtype=int), @@ -30,8 +31,9 @@ def linear_assignment( tuple(range(cost_matrix.shape[1])), ) - cost_matrix[cost_matrix > thresh] = thresh + 1e-4 - row_ind, col_ind = linear_sum_assignment(cost_matrix) + assignment_cost_matrix = cost_matrix.copy() + assignment_cost_matrix[assignment_cost_matrix > thresh] = thresh + 1e-4 + row_ind, col_ind = linear_sum_assignment(assignment_cost_matrix) indices = np.column_stack((row_ind, col_ind)) return indices_to_matches(cost_matrix, indices, thresh) diff --git a/src/supervision/tracker/byte_tracker/single_object_track.py b/src/supervision/tracker/byte_tracker/single_object_track.py index 5194b49058..5d540db1b5 100644 --- a/src/supervision/tracker/byte_tracker/single_object_track.py +++ b/src/supervision/tracker/byte_tracker/single_object_track.py @@ -81,19 +81,18 @@ def multi_predict(stracks: list[STrack], shared_kalman: KalmanFilter) -> None: stracks[i].covariance = cov def activate(self, kalman_filter: KalmanFilter, frame_id: int) -> None: - """Start a new tracklet""" + """Start a new tracklet after its first detection.""" self.kalman_filter = kalman_filter self.internal_track_id = self.internal_id_counter.new_id() self.mean, self.covariance = self.kalman_filter.initiate( self.tlwh_to_xyah(self._tlwh) ) - self.tracklet_len = 0 + self.tracklet_len = 1 self.state = TrackState.Tracked - if frame_id == 1: + if frame_id == 1 and self.tracklet_len >= self.minimum_consecutive_frames: self.is_activated = True - if self.minimum_consecutive_frames == 1: - self.external_track_id = self.external_id_counter.new_id() + self.external_track_id = self.external_id_counter.new_id() self.frame_id = frame_id self.start_frame = frame_id @@ -130,7 +129,7 @@ def update(self, new_track: STrack, frame_id: int) -> None: self.mean, self.covariance, self.tlwh_to_xyah(new_tlwh) ) self.state = TrackState.Tracked - if self.tracklet_len == self.minimum_consecutive_frames: + if self.tracklet_len >= self.minimum_consecutive_frames: self.is_activated = True if self.external_track_id == self.external_id_counter.NO_ID: self.external_track_id = self.external_id_counter.new_id() diff --git a/tests/tracker/test_byte_tracker.py b/tests/tracker/test_byte_tracker.py index 83e25b0642..d0732912af 100644 --- a/tests/tracker/test_byte_tracker.py +++ b/tests/tracker/test_byte_tracker.py @@ -2,6 +2,20 @@ import pytest import supervision as sv +from supervision.tracker.byte_tracker import matching + + +def _detections_from_boxes( + boxes: list[list[float]], confidence: list[float] | None = None +) -> sv.Detections: + """Create detections with class ids and confidence for tracker regressions.""" + if confidence is None: + confidence = [1.0] * len(boxes) + return sv.Detections( + xyxy=np.array(boxes, dtype=np.float32), + class_id=np.zeros(len(boxes), dtype=int), + confidence=np.array(confidence, dtype=np.float32), + ) @pytest.mark.parametrize( @@ -33,27 +47,22 @@ def test_byte_tracker( detections: list[sv.Detections], expected_results: sv.Detections, ) -> None: + """ByteTrack should preserve stable tracker ids for repeated detections.""" byte_tracker = sv.ByteTrack() tracked_detections = [byte_tracker.update_with_detections(d) for d in detections] assert tracked_detections[-1] == expected_results def test_byte_tracker_does_not_skip_external_ids_for_short_lived_tracks() -> None: - def detections_from_boxes(boxes: list[list[float]]) -> sv.Detections: - return sv.Detections( - xyxy=np.array(boxes, dtype=np.float32), - class_id=np.zeros(len(boxes), dtype=int), - confidence=np.ones(len(boxes), dtype=np.float32), - ) - + """Unconfirmed short-lived tracks should not consume external ids.""" # A transient false-positive appears and disappears before becoming confirmed. # It should not consume an external tracker id. frames = [ - detections_from_boxes([[0, 0, 10, 10]]), - detections_from_boxes([[0, 0, 10, 10], [100, 100, 110, 110]]), - detections_from_boxes([[0, 0, 10, 10]]), - detections_from_boxes([[0, 0, 10, 10], [200, 200, 210, 210]]), - detections_from_boxes([[0, 0, 10, 10], [200, 200, 210, 210]]), + _detections_from_boxes([[0, 0, 10, 10]]), + _detections_from_boxes([[0, 0, 10, 10], [100, 100, 110, 110]]), + _detections_from_boxes([[0, 0, 10, 10]]), + _detections_from_boxes([[0, 0, 10, 10], [200, 200, 210, 210]]), + _detections_from_boxes([[0, 0, 10, 10], [200, 200, 210, 210]]), ] byte_tracker = sv.ByteTrack(minimum_consecutive_frames=1) @@ -61,3 +70,69 @@ def detections_from_boxes(boxes: list[list[float]]) -> sv.Detections: tracked = [byte_tracker.update_with_detections(frame) for frame in frames] assert tracked[-1].tracker_id is not None assert np.array_equal(np.sort(tracked[-1].tracker_id), np.array([1, 2])) + + +def test_high_activation_threshold_can_start_track_at_score_one() -> None: + """A high activation threshold must still allow maximum-confidence tracks.""" + byte_tracker = sv.ByteTrack(track_activation_threshold=0.95) + detections = _detections_from_boxes([[0, 0, 10, 10]], confidence=[1.0]) + + tracked = byte_tracker.update_with_detections(detections) + + assert tracked.tracker_id is not None + assert np.array_equal(tracked.tracker_id, np.array([1])) + + +def test_update_with_detections_does_not_mutate_input() -> None: + """Tracking should return a copy instead of writing ids into the input.""" + byte_tracker = sv.ByteTrack() + detections = _detections_from_boxes([[0, 0, 10, 10]]) + + _ = byte_tracker.update_with_detections(detections) + + assert detections.tracker_id is None + + +def test_score_equal_to_activation_threshold_keeps_existing_track() -> None: + """A detection exactly at the threshold should remain eligible to match.""" + byte_tracker = sv.ByteTrack(track_activation_threshold=0.5) + _ = byte_tracker.update_with_detections( + _detections_from_boxes([[0, 0, 10, 10]], confidence=[1.0]) + ) + + tracked = byte_tracker.update_with_detections( + _detections_from_boxes([[0, 0, 10, 10]], confidence=[0.5]) + ) + + assert tracked.tracker_id is not None + assert np.array_equal(tracked.tracker_id, np.array([1])) + + +def test_linear_assignment_does_not_mutate_cost_matrix() -> None: + """Assignment should not rewrite the caller-owned cost matrix.""" + cost_matrix = np.array([[0.1, 0.9], [0.8, 0.2]], dtype=np.float32) + original = cost_matrix.copy() + + _ = matching.linear_assignment(cost_matrix, thresh=0.5) + + assert np.array_equal(cost_matrix, original) + + +def test_update_with_tensors_ignores_zero_height_boxes() -> None: + """Zero-height boxes should be dropped before Kalman state creation.""" + byte_tracker = sv.ByteTrack() + tensors = np.array([[0, 0, 10, 0, 0.9]], dtype=np.float32) + + tracks = byte_tracker.update_with_tensors(tensors) + + assert tracks == [] + + +def test_update_with_tensors_respects_min_consecutive_frames_on_first_frame() -> None: + """First-frame tensor updates should not emit unconfirmed track id -1.""" + byte_tracker = sv.ByteTrack(minimum_consecutive_frames=2) + tensors = np.array([[0, 0, 10, 10, 0.9]], dtype=np.float32) + + tracks = byte_tracker.update_with_tensors(tensors) + + assert tracks == [] From 935c3326c432b06fdb2c304c6d22f47944ea35b2 Mon Sep 17 00:00:00 2001 From: jirka <6035284+Borda@users.noreply.github.com> Date: Tue, 7 Jul 2026 19:58:32 +0200 Subject: [PATCH 2/2] fix(tracker): harden ByteTrack edge cases - Avoids per-call np.arange allocation by cloning detections with slice(None) while preserving non-mutation behavior. - Adds regressions for delayed activation on the second consecutive tensor frame and broader invalid-tensor rejection cases. --- Co-authored-by: Codex --- src/supervision/tracker/byte_tracker/core.py | 2 +- tests/tracker/test_byte_tracker.py | 51 ++++++++++++++++++-- 2 files changed, 49 insertions(+), 4 deletions(-) diff --git a/src/supervision/tracker/byte_tracker/core.py b/src/supervision/tracker/byte_tracker/core.py index 43548b6d67..2acb15fbef 100644 --- a/src/supervision/tracker/byte_tracker/core.py +++ b/src/supervision/tracker/byte_tracker/core.py @@ -150,7 +150,7 @@ def callback(frame: np.ndarray, index: int) -> np.ndarray: iou_costs: npt.NDArray[np.float32] = 1 - ious matches, _, _ = matching.linear_assignment(iou_costs, 0.5) - tracked_detections = detections.select(np.arange(len(detections))) + tracked_detections = detections.select(slice(None)) tracked_detections.tracker_id = np.full(len(detections), -1, dtype=int) for i_detection, i_track in matches: tracked_detections.tracker_id[i_detection] = int( diff --git a/tests/tracker/test_byte_tracker.py b/tests/tracker/test_byte_tracker.py index d0732912af..416f18799f 100644 --- a/tests/tracker/test_byte_tracker.py +++ b/tests/tracker/test_byte_tracker.py @@ -93,6 +93,20 @@ def test_update_with_detections_does_not_mutate_input() -> None: assert detections.tracker_id is None +def test_update_with_tensors_activates_on_second_consecutive_frame() -> None: + """A second consecutive tensor frame should activate the delayed track.""" + byte_tracker = sv.ByteTrack(minimum_consecutive_frames=2) + tensors = np.array([[0, 0, 10, 10, 0.9]], dtype=np.float32) + + first_frame = byte_tracker.update_with_tensors(tensors) + second_frame = byte_tracker.update_with_tensors(tensors) + + assert first_frame == [] + assert len(second_frame) == 1 + assert second_frame[0].is_activated + assert second_frame[0].external_track_id == 1 + + def test_score_equal_to_activation_threshold_keeps_existing_track() -> None: """A detection exactly at the threshold should remain eligible to match.""" byte_tracker = sv.ByteTrack(track_activation_threshold=0.5) @@ -118,10 +132,41 @@ def test_linear_assignment_does_not_mutate_cost_matrix() -> None: assert np.array_equal(cost_matrix, original) -def test_update_with_tensors_ignores_zero_height_boxes() -> None: - """Zero-height boxes should be dropped before Kalman state creation.""" +@pytest.mark.parametrize( + "tensors", + [ + pytest.param( + np.array([[np.nan, 0, 10, 10, 0.9]], dtype=np.float32), + id="nan", + ), + pytest.param( + np.array([[0, np.inf, 10, 10, 0.9]], dtype=np.float32), + id="inf", + ), + pytest.param( + np.array([[0, 0, 0, 10, 0.9]], dtype=np.float32), + id="zero-width", + ), + pytest.param( + np.array([[10, 0, 0, 10, 0.9]], dtype=np.float32), + id="negative-width", + ), + pytest.param( + np.array([[0, 0, 10, 0, 0.9]], dtype=np.float32), + id="zero-height", + ), + pytest.param( + np.array([[0, 10, 10, 0, 0.9]], dtype=np.float32), + id="negative-height", + ), + pytest.param(np.empty((0, 5), dtype=np.float32), id="empty"), + ], +) +def test_update_with_tensors_ignores_invalid_boxes( + tensors: np.ndarray, +) -> None: + """Invalid tensors should be dropped before track creation.""" byte_tracker = sv.ByteTrack() - tensors = np.array([[0, 0, 10, 0, 0.9]], dtype=np.float32) tracks = byte_tracker.update_with_tensors(tensors)