Skip to content

Commit

Permalink
Improve 2d spawn heuristics (#5087)
Browse files Browse the repository at this point in the history
### What

* partially solves #4926
* fixes new issues where we get two spaces views in our 2d code examples
for lines

Removes "child of root" heuristics for 2D space views and improves image
grouping behavior a bit.
Had to do slight adjustments to object tracking demo so that it still
looks good:


![image](https://github.com/rerun-io/rerun/assets/1220815/5b0987f7-2c3a-4f7e-b954-087e85a1a548)

But imho the adjusted hierarchy also makes a lot of sense. Soon user can
configure this from blueprint, we can then think about removing
non-explicit bucketing. See
* #3985


### Checklist
* [x] I have read and agree to [Contributor
Guide](https://github.com/rerun-io/rerun/blob/main/CONTRIBUTING.md) and
the [Code of
Conduct](https://github.com/rerun-io/rerun/blob/main/CODE_OF_CONDUCT.md)
* [x] I've included a screenshot or gif (if applicable)
* [x] I have tested the web demo (if applicable):
* Using newly built examples:
[app.rerun.io](https://app.rerun.io/pr/5087/index.html)
* Using examples from latest `main` build:
[app.rerun.io](https://app.rerun.io/pr/5087/index.html?manifest_url=https://app.rerun.io/version/main/examples_manifest.json)
* Using full set of examples from `nightly` build:
[app.rerun.io](https://app.rerun.io/pr/5087/index.html?manifest_url=https://app.rerun.io/version/nightly/examples_manifest.json)
* [x] The PR title and labels are set such as to maximize their
usefulness for the next release's CHANGELOG
* [x] If applicable, add a new check to the [release
checklist](tests/python/release_checklist)!

- [PR Build Summary](https://build.rerun.io/pr/5087)
- [Docs
preview](https://rerun.io/preview/4040710e64651d60a9b1f1afacfa899be19e11cc/docs)
<!--DOCS-PREVIEW-->
- [Examples
preview](https://rerun.io/preview/4040710e64651d60a9b1f1afacfa899be19e11cc/examples)
<!--EXAMPLES-PREVIEW-->
- [Recent benchmark results](https://build.rerun.io/graphs/crates.html)
- [Wasm size tracking](https://build.rerun.io/graphs/sizes.html)
  • Loading branch information
Wumpf committed Feb 7, 2024
1 parent ff1bf9d commit 3e0e1e2
Show file tree
Hide file tree
Showing 3 changed files with 68 additions and 31 deletions.
45 changes: 45 additions & 0 deletions crates/re_log_types/src/path/entity_path.rs
Expand Up @@ -237,6 +237,27 @@ impl EntityPath {
itertools::Either::Right(std::iter::empty())
}
}

/// Returns the first common ancestor of two paths.
///
/// If both paths are the same, the common ancestor is the path itself.
pub fn common_ancestor(&self, other: &EntityPath) -> EntityPath {
let mut common = Vec::new();
for (a, b) in self.iter().zip(other.iter()) {
if a == b {
common.push(a.clone());
} else {
break;
}
}
EntityPath::new(common)
}

/// Returns the first common ancestor of a list of entity paths.
pub fn common_ancestor_of<'a>(mut entities: impl Iterator<Item = &'a Self>) -> Self {
let first = entities.next().cloned().unwrap_or(EntityPath::root());
entities.fold(first, |acc, e| acc.common_ancestor(e))
}
}

impl SizeBytes for EntityPath {
Expand Down Expand Up @@ -492,4 +513,28 @@ mod tests {
vec![EntityPath::from("foo/bar"), EntityPath::from("foo/bar/baz")]
);
}

#[test]
fn test_common_ancestor() {
assert_eq!(
EntityPath::from("foo/bar").common_ancestor(&EntityPath::from("foo/bar")),
EntityPath::from("foo/bar")
);
assert_eq!(
EntityPath::from("foo/bar").common_ancestor(&EntityPath::from("foo/bar/baz")),
EntityPath::from("foo/bar")
);
assert_eq!(
EntityPath::from("foo/bar/baz").common_ancestor(&EntityPath::from("foo/bar")),
EntityPath::from("foo/bar")
);
assert_eq!(
EntityPath::from("foo/bar/mario").common_ancestor(&EntityPath::from("foo/bar/luigi")),
EntityPath::from("foo/bar")
);
assert_eq!(
EntityPath::from("mario/bowser").common_ancestor(&EntityPath::from("luigi/bowser")),
EntityPath::root()
);
}
}
36 changes: 14 additions & 22 deletions crates/re_space_view_spatial/src/space_view_2d.rs
Expand Up @@ -15,8 +15,7 @@ use re_viewer_context::{
use crate::{
contexts::{register_spatial_contexts, PrimitiveCounter},
heuristics::{
default_visualized_entities_for_visualizer_kind, root_space_split_heuristic,
update_object_property_heuristics,
default_visualized_entities_for_visualizer_kind, update_object_property_heuristics,
},
spatial_topology::{SpatialTopology, SubSpace, SubSpaceDimensionality},
ui::SpatialSpaceViewState,
Expand Down Expand Up @@ -178,19 +177,9 @@ impl SpaceViewClass for SpatialSpaceView2D {
// Note that visualizability filtering is all about being in the right subspace,
// so we don't need to call the visualizers' filter functions here.
let mut heuristics = SpatialTopology::access(ctx.entity_db.store_id(), |topo| {
let split_root_spaces =
root_space_split_heuristic(topo, &indicated_entities, SubSpaceDimensionality::TwoD);

SpaceViewSpawnHeuristics {
recommended_space_views: topo
.iter_subspaces()
.flat_map(|subspace| {
if subspace.origin.is_root() && !split_root_spaces.is_empty() {
itertools::Either::Left(split_root_spaces.values())
} else {
itertools::Either::Right(std::iter::once(subspace))
}
})
.flat_map(|subspace| {
if subspace.dimensionality == SubSpaceDimensionality::ThreeD
|| subspace.entities.is_empty()
Expand All @@ -215,17 +204,17 @@ impl SpaceViewClass for SpatialSpaceView2D {
images_by_bucket
.into_iter()
.map(|(_, entity_bucket)| {
// Pick a shared parent as origin, mostly because it looks nicer in the ui.
let root = EntityPath::common_ancestor_of(entity_bucket.iter());

let mut query_filter = EntityPathFilter::default();
for image in entity_bucket {
for image in &entity_bucket {
// This might lead to overlapping subtrees and break the same image size bucketing again.
// We just take that risk, the heuristic doesn't need to be perfect.
query_filter.add_subtree(image);
query_filter.add_subtree(image.clone());
}

RecommendedSpaceView {
root: subspace.origin.clone(),
query_filter,
}
RecommendedSpaceView { root, query_filter }
})
.collect()
}
Expand Down Expand Up @@ -329,10 +318,13 @@ fn bucket_images_in_subspace(
store.query_latest_component::<TensorData>(image_entity, &ctx.current_query())
{
if let Some([height, width, _]) = tensor.image_height_width_channels() {
images_by_bucket
.entry((height, width))
.or_default()
.push(image_entity.clone());
// 1D tensors are typically handled by tensor or bar chart views and make generally for poor image buckets!
if height > 1 && width > 1 {
images_by_bucket
.entry((height, width))
.or_default()
.push(image_entity.clone());
}
}
}
}
Expand Down
18 changes: 9 additions & 9 deletions examples/python/detect_and_track_objects/main.py
Expand Up @@ -50,15 +50,15 @@
### Input Video
The input video is logged as a sequence of
[rr.Image objects](https://www.rerun.io/docs/reference/types/archetypes/image) to the
[image/rgb entity](recording://image/rgb). Since the detection and segmentation model operates on smaller images the
resized images are logged to the separate [image_scaled/rgb entity](recording://image_scaled/rgb). This allows us to
[image entity](recording://image). Since the detection and segmentation model operates on smaller images the
resized images are logged to the separate [segmentation/rgb_scaled entity](recording://segmentation/rgb_scaled). This allows us to
subsequently visualize the segmentation mask on top of the video.
### Segmentations
The [segmentation result](recording://image_scaled/segmentation) is logged through a combination of two archetypes.
The [segmentation result](recording://image_segmentation/segmentation) is logged through a combination of two archetypes.
The segmentation image itself is logged as an
[rr.SegmentationImage archetype](https://www.rerun.io/docs/reference/types/archetypes/segmentation_image) and
contains the id for each pixel. It is logged to the [image_scaled/segmentation entity](recording://image_scaled/segmentation).
contains the id for each pixel. It is logged to the [segmentation entity](recording://segmentation).
The color and label for each class is determined by the
[rr.AnnotationContext archetype](https://www.rerun.io/docs/reference/types/archetypes/annotation_context) which is
Expand Down Expand Up @@ -134,7 +134,7 @@ def detect_objects_to_track(self, rgb: npt.NDArray[np.uint8], frame_idx: int) ->
_, _, scaled_height, scaled_width = inputs["pixel_values"].shape
scaled_size = (scaled_width, scaled_height)
rgb_scaled = cv2.resize(rgb, scaled_size)
rr.log("image_scaled/rgb", rr.Image(rgb_scaled).compress(jpeg_quality=85))
rr.log("segmentation/rgb_scaled", rr.Image(rgb_scaled).compress(jpeg_quality=85))

logging.debug("Pass image to detection network")
outputs = self.model(**inputs)
Expand All @@ -147,7 +147,7 @@ def detect_objects_to_track(self, rgb: npt.NDArray[np.uint8], frame_idx: int) ->
)[0]

mask = segmentation_mask.detach().cpu().numpy().astype(np.uint8)
rr.log("image_scaled/segmentation", rr.SegmentationImage(mask))
rr.log("segmentation", rr.SegmentationImage(mask))

boxes = detections["boxes"].detach().cpu().numpy()
class_ids = detections["labels"].detach().cpu().numpy()
Expand Down Expand Up @@ -178,7 +178,7 @@ def log_detections(self, boxes: npt.NDArray[np.float32], class_ids: list[int], t
thing_boxes = boxes[things_np, :]
thing_class_ids = class_ids_np[things_np]
rr.log(
"image_scaled/detections/things",
"segmentation/detections/things",
rr.Boxes2D(
array=thing_boxes,
array_format=rr.Box2DFormat.XYXY,
Expand All @@ -189,7 +189,7 @@ def log_detections(self, boxes: npt.NDArray[np.float32], class_ids: list[int], t
background_boxes = boxes[~things_np, :]
background_class_ids = class_ids[~things_np]
rr.log(
"image_scaled/detections/background",
"segmentation/detections/background",
rr.Boxes2D(
array=background_boxes,
array_format=rr.Box2DFormat.XYXY,
Expand Down Expand Up @@ -391,7 +391,7 @@ def track_objects(video_path: str, *, max_frame_count: int | None) -> None:
break

rgb = cv2.cvtColor(bgr, cv2.COLOR_BGR2RGB)
rr.log("image/rgb", rr.Image(rgb).compress(jpeg_quality=85))
rr.log("image", rr.Image(rgb).compress(jpeg_quality=85))

if not trackers or frame_idx % 40 == 0:
detections = detector.detect_objects_to_track(rgb=rgb, frame_idx=frame_idx)
Expand Down

0 comments on commit 3e0e1e2

Please sign in to comment.