diff --git a/crates/re_data_store/src/entity_properties.rs b/crates/re_data_store/src/entity_properties.rs index 7be177ea0c33..4699dd692db7 100644 --- a/crates/re_data_store/src/entity_properties.rs +++ b/crates/re_data_store/src/entity_properties.rs @@ -36,6 +36,8 @@ impl EntityPropertyMap { // ---------------------------------------------------------------------------- +// TODO(#1423): We need to properly split entity properties that only apply to specific +// views/primitives. #[cfg(feature = "serde")] #[derive(Clone, Debug, PartialEq)] #[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))] @@ -50,6 +52,20 @@ pub struct EntityProperties { /// Only applies to pinhole cameras when in a spatial view, using 3D navigation. /// pub pinhole_image_plane_distance: EditableAutoValue, + + /// Should the depth texture be backprojected into a point cloud? + /// + /// Only applies to tensors with meaning=depth that are affected by a pinhole transform when + /// in a spatial view, using 3D navigation. + pub backproject_depth: bool, + /// Entity path of the pinhole transform used for the backprojection. + /// + /// `None` means backprojection is disabled. + pub backproject_pinhole_ent_path: Option, + /// Used to scale the resulting point cloud. + pub backproject_scale: EditableAutoValue, + /// Used to scale the radii of the points in the resulting point cloud. + pub backproject_radius_scale: EditableAutoValue, } #[cfg(feature = "serde")] @@ -64,6 +80,16 @@ impl EntityProperties { .pinhole_image_plane_distance .or(&child.pinhole_image_plane_distance) .clone(), + backproject_depth: self.backproject_depth || child.backproject_depth, + backproject_pinhole_ent_path: self + .backproject_pinhole_ent_path + .clone() + .or(child.backproject_pinhole_ent_path.clone()), + backproject_scale: self.backproject_scale.or(&child.backproject_scale).clone(), + backproject_radius_scale: self + .backproject_radius_scale + .or(&child.backproject_radius_scale) + .clone(), } } } @@ -76,6 +102,10 @@ impl Default for EntityProperties { visible_history: ExtraQueryHistory::default(), interactive: true, pinhole_image_plane_distance: EditableAutoValue::default(), + backproject_depth: false, + backproject_pinhole_ent_path: None, + backproject_scale: EditableAutoValue::default(), + backproject_radius_scale: EditableAutoValue::default(), } } } diff --git a/crates/re_renderer/examples/depth_cloud.rs b/crates/re_renderer/examples/depth_cloud.rs index 81da75f34a14..d70db8066537 100644 --- a/crates/re_renderer/examples/depth_cloud.rs +++ b/crates/re_renderer/examples/depth_cloud.rs @@ -78,17 +78,14 @@ impl RenderDepthClouds { } = self; let focal_length = glam::Vec2::new(intrinsics.x_axis.x, intrinsics.y_axis.y); - let offset = glam::Vec2::new(intrinsics.x_axis.z, intrinsics.y_axis.z); + let offset = glam::Vec2::new(intrinsics.z_axis.x, intrinsics.z_axis.y); let point_cloud_draw_data = { let num_points = depth.dimensions.x * depth.dimensions.y; let (points, colors, radii): (Vec<_>, Vec<_>, Vec<_>) = (0..depth.dimensions.y) .flat_map(|y| (0..depth.dimensions.x).map(move |x| glam::UVec2::new(x, y))) .map(|texcoords| { - let linear_depth = depth.get_linear( - depth.dimensions.x - texcoords.x - 1, - depth.dimensions.y - texcoords.y - 1, - ); + let linear_depth = depth.get_linear(texcoords.x, texcoords.y); let pos_in_world = ((texcoords.as_vec2() - offset) * linear_depth / focal_length) .extend(linear_depth); @@ -171,17 +168,12 @@ impl RenderDepthClouds { .. } = self; - let world_from_obj = glam::Mat4::from_cols( - glam::Vec4::NEG_X * *scale, - glam::Vec4::NEG_Y * *scale, - glam::Vec4::Z * *scale, - glam::Vec4::W, - ); + let world_from_obj = glam::Mat4::from_scale(glam::Vec3::splat(*scale)); let depth_cloud_draw_data = DepthCloudDrawData::new( re_ctx, &[DepthCloud { - world_from_obj, + depth_camera_extrinsics: world_from_obj, depth_camera_intrinsics: *intrinsics, radius_scale: *radius_scale, depth_dimensions: depth.dimensions, @@ -261,7 +253,8 @@ impl framework::Example for RenderDepthClouds { Vec3::new(focal_length, 0.0, offset.x), Vec3::new(0.0, focal_length, offset.y), Vec3::new(0.0, 0.0, 1.0), - ); + ) + .transpose(); RenderDepthClouds { depth, diff --git a/crates/re_renderer/shader/depth_cloud.wgsl b/crates/re_renderer/shader/depth_cloud.wgsl index 1b84adc4371b..cb4792ca549b 100644 --- a/crates/re_renderer/shader/depth_cloud.wgsl +++ b/crates/re_renderer/shader/depth_cloud.wgsl @@ -30,7 +30,7 @@ fn compute_point_data(quad_idx: i32) -> PointData { let color = Vec4(linear_from_srgb(Vec3(norm_linear_depth)), 1.0); // TODO(cmc): This assumes a pinhole camera; need to support other kinds at some point. - let intrinsics = transpose(depth_cloud_info.depth_camera_intrinsics); + let intrinsics = depth_cloud_info.depth_camera_intrinsics; let focal_length = Vec2(intrinsics[0][0], intrinsics[1][1]); let offset = Vec2(intrinsics[2][0], intrinsics[2][1]); @@ -39,7 +39,7 @@ fn compute_point_data(quad_idx: i32) -> PointData { norm_linear_depth, ); - let pos_in_world = depth_cloud_info.world_from_obj * Vec4(pos_in_obj, 1.0); + let pos_in_world = depth_cloud_info.extrinsincs * Vec4(pos_in_obj, 1.0); var data: PointData; data.pos_in_world = pos_in_world.xyz; @@ -52,7 +52,8 @@ fn compute_point_data(quad_idx: i32) -> PointData { // --- struct DepthCloudInfo { - world_from_obj: Mat4, + /// The extrinsincs of the camera used for the projection. + extrinsincs: Mat4, /// The intrinsics of the camera used for the projection. /// diff --git a/crates/re_renderer/src/renderer/depth_cloud.rs b/crates/re_renderer/src/renderer/depth_cloud.rs index 7312d64f6dcf..f4db5b0a1a44 100644 --- a/crates/re_renderer/src/renderer/depth_cloud.rs +++ b/crates/re_renderer/src/renderer/depth_cloud.rs @@ -39,7 +39,7 @@ mod gpu_data { #[repr(C, align(256))] #[derive(Clone, Copy, bytemuck::Pod, bytemuck::Zeroable)] pub struct DepthCloudInfoUBO { - pub world_from_obj: crate::wgpu_buffer_types::Mat4, + pub depth_camera_extrinsics: crate::wgpu_buffer_types::Mat4, pub depth_camera_intrinsics: crate::wgpu_buffer_types::Mat3, pub radius_scale: crate::wgpu_buffer_types::F32RowPadded, @@ -65,7 +65,8 @@ pub enum DepthCloudDepthData { } pub struct DepthCloud { - pub world_from_obj: glam::Mat4, + /// The extrinsics of the camera used for the projection. + pub depth_camera_extrinsics: glam::Mat4, /// The intrinsics of the camera used for the projection. /// @@ -87,7 +88,7 @@ pub struct DepthCloud { impl Default for DepthCloud { fn default() -> Self { Self { - world_from_obj: glam::Mat4::IDENTITY, + depth_camera_extrinsics: glam::Mat4::IDENTITY, depth_camera_intrinsics: glam::Mat3::IDENTITY, radius_scale: 1.0, depth_dimensions: glam::UVec2::ZERO, @@ -113,6 +114,17 @@ impl DepthCloudDrawData { ) -> Result { crate::profile_function!(); + let bg_layout = ctx + .renderers + .write() + .get_or_create::<_, DepthCloudRenderer>( + &ctx.shared_renderer_data, + &mut ctx.gpu_resources, + &ctx.device, + &mut ctx.resolver, + ) + .bind_group_layout; + if depth_clouds.is_empty() { return Ok(DepthCloudDrawData { bind_groups: Vec::new(), @@ -123,24 +135,13 @@ impl DepthCloudDrawData { ctx, "depth_cloud_ubos".into(), depth_clouds.iter().map(|info| gpu_data::DepthCloudInfoUBO { - world_from_obj: info.world_from_obj.into(), + depth_camera_extrinsics: info.depth_camera_extrinsics.into(), depth_camera_intrinsics: info.depth_camera_intrinsics.into(), radius_scale: info.radius_scale.into(), end_padding: Default::default(), }), ); - let bg_layout = ctx - .renderers - .write() - .get_or_create::<_, DepthCloudRenderer>( - &ctx.shared_renderer_data, - &mut ctx.gpu_resources, - &ctx.device, - &mut ctx.resolver, - ) - .bind_group_layout; - let mut bind_groups = Vec::with_capacity(depth_clouds.len()); for (depth_cloud, ubo) in depth_clouds.iter().zip(depth_cloud_ubos.into_iter()) { let depth_texture = match &depth_cloud.depth_data { diff --git a/crates/re_viewer/src/ui/selection_panel.rs b/crates/re_viewer/src/ui/selection_panel.rs index 24053087fca8..52262ed9acef 100644 --- a/crates/re_viewer/src/ui/selection_panel.rs +++ b/crates/re_viewer/src/ui/selection_panel.rs @@ -1,5 +1,9 @@ +use egui::NumExt as _; use re_data_store::{query_latest_single, EditableAutoValue, EntityPath, EntityProperties}; -use re_log_types::{TimeType, Transform}; +use re_log_types::{ + component_types::{Tensor, TensorDataMeaning}, + TimeType, Transform, +}; use crate::{ ui::{view_spatial::SpatialNavigationMode, Blueprint}, @@ -371,8 +375,6 @@ fn entity_props_ui( entity_props: &mut EntityProperties, view_state: &ViewState, ) { - use egui::NumExt; - ui.checkbox(&mut entity_props.visible, "Visible"); ui.checkbox(&mut entity_props.interactive, "Interactive") .on_hover_text("If disabled, the entity will not react to any mouse interaction"); @@ -407,31 +409,119 @@ fn entity_props_ui( } ui.end_row(); - // pinhole_image_plane_distance if view_state.state_spatial.nav_mode == SpatialNavigationMode::ThreeD { if let Some(entity_path) = entity_path { - let query = ctx.current_query(); - if let Some(re_log_types::Transform::Pinhole(_)) = - query_latest_single::(&ctx.log_db.entity_db, entity_path, &query) - { - ui.label("Image plane distance"); - let mut distance = *entity_props.pinhole_image_plane_distance.get(); - let speed = (distance * 0.05).at_least(0.01); - if ui - .add( - egui::DragValue::new(&mut distance) - .clamp_range(0.0..=1.0e8) - .speed(speed), - ) - .on_hover_text("Controls how far away the image plane is.") - .changed() - { - entity_props.pinhole_image_plane_distance = - EditableAutoValue::UserEdited(distance); - } - ui.end_row(); - } + pinhole_props_ui(ctx, ui, entity_path, entity_props); + depth_props_ui(ctx, ui, entity_path, entity_props); } } }); } + +fn pinhole_props_ui( + ctx: &mut ViewerContext<'_>, + ui: &mut egui::Ui, + entity_path: &EntityPath, + entity_props: &mut EntityProperties, +) { + let query = ctx.current_query(); + if let Some(re_log_types::Transform::Pinhole(_)) = + query_latest_single::(&ctx.log_db.entity_db, entity_path, &query) + { + ui.label("Image plane distance"); + let mut distance = *entity_props.pinhole_image_plane_distance.get(); + let speed = (distance * 0.05).at_least(0.01); + if ui + .add( + egui::DragValue::new(&mut distance) + .clamp_range(0.0..=1.0e8) + .speed(speed), + ) + .on_hover_text("Controls how far away the image plane is.") + .changed() + { + entity_props.pinhole_image_plane_distance = EditableAutoValue::UserEdited(distance); + } + ui.end_row(); + } +} + +fn depth_props_ui( + ctx: &mut ViewerContext<'_>, + ui: &mut egui::Ui, + entity_path: &EntityPath, + entity_props: &mut EntityProperties, +) { + let query = ctx.current_query(); + + // Find closest pinhole transform, if any. + let mut pinhole_ent_path = None; + let mut cur_path = Some(entity_path.clone()); + while let Some(path) = cur_path { + if let Some(re_log_types::Transform::Pinhole(_)) = + query_latest_single::(&ctx.log_db.entity_db, &path, &query) + { + pinhole_ent_path = Some(path); + break; + } + cur_path = path.parent(); + } + + // Early out if there's no pinhole transform upwards in the tree. + let Some(pinhole_ent_path) = pinhole_ent_path else { return; }; + + entity_props.backproject_pinhole_ent_path = Some(pinhole_ent_path.clone()); + + let tensor = query_latest_single::(&ctx.log_db.entity_db, entity_path, &query); + if tensor.as_ref().map(|t| t.meaning) == Some(TensorDataMeaning::Depth) { + ui.checkbox(&mut entity_props.backproject_depth, "Backproject Depth") + .on_hover_text( + "If enabled, the depth texture will be backprojected into a point cloud rather \ + than simply displayed as an image.", + ); + ui.end_row(); + + if entity_props.backproject_depth { + ui.label("Pinhole"); + ctx.entity_path_button(ui, None, &pinhole_ent_path) + .on_hover_text( + "The entity path of the pinhole transform being used to do the backprojection.", + ); + ui.end_row(); + + ui.label("Backproject scale"); + let mut scale = *entity_props.backproject_scale.get(); + let speed = (scale * 0.05).at_least(0.01); + if ui + .add( + egui::DragValue::new(&mut scale) + .clamp_range(0.0..=1.0e8) + .speed(speed), + ) + .on_hover_text("Scales the backprojected point cloud") + .changed() + { + entity_props.backproject_scale = EditableAutoValue::UserEdited(scale); + } + ui.end_row(); + + ui.label("Backproject radius scale"); + let mut radius_scale = *entity_props.backproject_radius_scale.get(); + let speed = (radius_scale * 0.001).at_least(0.001); + if ui + .add( + egui::DragValue::new(&mut radius_scale) + .clamp_range(0.0..=1.0e8) + .speed(speed), + ) + .on_hover_text("Scales the radii of the points in the backprojected point cloud") + .changed() + { + entity_props.backproject_radius_scale = EditableAutoValue::UserEdited(radius_scale); + } + ui.end_row(); + } else { + entity_props.backproject_pinhole_ent_path = None; + } + } +} diff --git a/crates/re_viewer/src/ui/view_spatial/scene/picking.rs b/crates/re_viewer/src/ui/view_spatial/scene/picking.rs index 52eee1b4ef67..f4a4abc660d1 100644 --- a/crates/re_viewer/src/ui/view_spatial/scene/picking.rs +++ b/crates/re_viewer/src/ui/view_spatial/scene/picking.rs @@ -166,6 +166,7 @@ pub fn picking( line_strips, points, meshes, + depth_clouds: _, // no picking for depth clouds yet } = primitives; picking_points(&context, &mut state, points); diff --git a/crates/re_viewer/src/ui/view_spatial/scene/primitives.rs b/crates/re_viewer/src/ui/view_spatial/scene/primitives.rs index 22a1fd28dd31..d87b49828dd3 100644 --- a/crates/re_viewer/src/ui/view_spatial/scene/primitives.rs +++ b/crates/re_viewer/src/ui/view_spatial/scene/primitives.rs @@ -1,6 +1,9 @@ use egui::Color32; use re_data_store::InstancePathHash; -use re_renderer::{renderer::MeshInstance, LineStripSeriesBuilder, PointCloudBuilder}; +use re_renderer::{ + renderer::{DepthCloud, MeshInstance}, + LineStripSeriesBuilder, PointCloudBuilder, +}; use super::MeshSource; @@ -23,6 +26,7 @@ pub struct SceneSpatialPrimitives { pub points: PointCloudBuilder, pub meshes: Vec, + pub depth_clouds: Vec, } const AXIS_COLOR_X: Color32 = Color32::from_rgb(255, 25, 25); @@ -38,6 +42,7 @@ impl SceneSpatialPrimitives { line_strips: Default::default(), points: PointCloudBuilder::new(re_ctx), meshes: Default::default(), + depth_clouds: Default::default(), } } @@ -55,60 +60,64 @@ impl SceneSpatialPrimitives { line_strips, points, meshes, + depth_clouds, } = &self; textured_rectangles.len() + line_strips.vertices.len() + points.vertices.len() + meshes.len() + + depth_clouds.len() } pub fn recalculate_bounding_box(&mut self) { crate::profile_function!(); - self.bounding_box = macaw::BoundingBox::nothing(); + let Self { + bounding_box, + textured_rectangles_ids: _, + textured_rectangles, + line_strips, + points, + meshes, + depth_clouds: _, // no bbox for depth clouds + } = self; + + *bounding_box = macaw::BoundingBox::nothing(); - for rect in &self.textured_rectangles { - self.bounding_box.extend(rect.top_left_corner_position); - self.bounding_box - .extend(rect.top_left_corner_position + rect.extent_u); - self.bounding_box - .extend(rect.top_left_corner_position + rect.extent_v); - self.bounding_box - .extend(rect.top_left_corner_position + rect.extent_v + rect.extent_u); + for rect in textured_rectangles { + bounding_box.extend(rect.top_left_corner_position); + bounding_box.extend(rect.top_left_corner_position + rect.extent_u); + bounding_box.extend(rect.top_left_corner_position + rect.extent_v); + bounding_box.extend(rect.top_left_corner_position + rect.extent_v + rect.extent_u); } // We don't need a very accurate bounding box, so in order to save some time, // we calculate a per batch bounding box for lines and points. // TODO(andreas): We should keep these around to speed up picking! - for (batch, vertex_iter) in self.points.iter_vertices_by_batch() { + for (batch, vertex_iter) in points.iter_vertices_by_batch() { // Only use points which are an IsoTransform to update the bounding box // This prevents crazy bounds-increases when projecting 3d to 2d // See: https://github.com/rerun-io/rerun/issues/1203 if let Some(transform) = macaw::IsoTransform::from_mat4(&batch.world_from_obj) { let batch_bb = macaw::BoundingBox::from_points(vertex_iter.map(|v| v.position)); - self.bounding_box = self - .bounding_box - .union(batch_bb.transform_affine3(&transform.into())); + *bounding_box = bounding_box.union(batch_bb.transform_affine3(&transform.into())); } } - for (batch, vertex_iter) in self.line_strips.iter_vertices_by_batch() { + for (batch, vertex_iter) in line_strips.iter_vertices_by_batch() { // Only use points which are an IsoTransform to update the bounding box // This prevents crazy bounds-increases when projecting 3d to 2d // See: https://github.com/rerun-io/rerun/issues/1203 if let Some(transform) = macaw::IsoTransform::from_mat4(&batch.world_from_obj) { let batch_bb = macaw::BoundingBox::from_points(vertex_iter.map(|v| v.position)); - self.bounding_box = self - .bounding_box - .union(batch_bb.transform_affine3(&transform.into())); + *bounding_box = bounding_box.union(batch_bb.transform_affine3(&transform.into())); } } - for mesh in &self.meshes { + for mesh in meshes { // TODO(jleibs): is this safe for meshes or should we be doing the equivalent of the above? - self.bounding_box = self - .bounding_box - .union(mesh.mesh.bbox().transform_affine3(&mesh.world_from_mesh)); + *bounding_box = + bounding_box.union(mesh.mesh.bbox().transform_affine3(&mesh.world_from_mesh)); } } diff --git a/crates/re_viewer/src/ui/view_spatial/scene/scene_part/images.rs b/crates/re_viewer/src/ui/view_spatial/scene/scene_part/images.rs index fe9304288752..cbd88433fc8f 100644 --- a/crates/re_viewer/src/ui/view_spatial/scene/scene_part/images.rs +++ b/crates/re_viewer/src/ui/view_spatial/scene/scene_part/images.rs @@ -4,13 +4,17 @@ use egui::NumExt; use glam::Vec3; use itertools::Itertools; -use re_data_store::{EntityPath, EntityProperties, InstancePathHash}; +use re_data_store::{query_latest_single, EntityPath, EntityProperties, InstancePathHash}; use re_log_types::{ - component_types::{ColorRGBA, InstanceKey, Tensor, TensorTrait}, + component_types::{ColorRGBA, InstanceKey, Tensor, TensorData, TensorDataMeaning, TensorTrait}, msg_bundle::Component, + Transform, }; use re_query::{query_primary_with_history, EntityView, QueryError}; -use re_renderer::Size; +use re_renderer::{ + renderer::{DepthCloud, DepthCloudDepthData}, + Size, +}; use crate::{ misc::{caches::AsDynamicImage, SpaceViewHighlights, TransformCache, ViewerContext}, @@ -128,6 +132,7 @@ impl ImagesPart { entity_view: &EntityView, scene: &mut SceneSpatial, ctx: &mut ViewerContext<'_>, + transforms: &TransformCache, properties: &EntityProperties, ent_path: &EntityPath, world_from_obj: glam::Mat4, @@ -146,64 +151,171 @@ impl ImagesPart { return Ok(()); } - let entity_highlight = highlights.entity_highlight(ent_path.hash()); + if tensor.meaning == TensorDataMeaning::Depth { + if let Some(pinhole_ent_path) = properties.backproject_pinhole_ent_path.as_ref() + { + // NOTE: we don't pass in `world_from_obj` because this corresponds to the + // transform of the projection plane, which is of no use to us here. + // What we want are the extrinsics of the depth camera! + Self::process_entity_view_as_depth_cloud( + scene, + ctx, + transforms, + properties, + &tensor, + pinhole_ent_path, + ); + return Ok(()); + }; + } - let instance_path_hash = instance_path_hash_for_picking( - ent_path, - instance_key, + Self::process_entity_view_as_image( entity_view, + scene, + ctx, properties, - entity_highlight, + ent_path, + world_from_obj, + highlights, + instance_key, + tensor, + color, ); + } + } - let annotations = scene.annotation_map.find(ent_path); + Ok(()) + } - let color = annotations.class_description(None).annotation_info().color( - color.map(|c| c.to_array()).as_ref(), - DefaultColor::OpaqueWhite, - ); + #[allow(clippy::too_many_arguments)] + fn process_entity_view_as_image( + entity_view: &EntityView, + scene: &mut SceneSpatial, + ctx: &mut ViewerContext<'_>, + properties: &EntityProperties, + ent_path: &EntityPath, + world_from_obj: glam::Mat4, + highlights: &SpaceViewHighlights, + instance_key: InstanceKey, + tensor: Tensor, + color: Option, + ) { + crate::profile_function!(); - let highlight = entity_highlight.index_highlight(instance_path_hash.instance_key); - if highlight.is_some() { - let color = SceneSpatial::apply_hover_and_selection_effect_color( - re_renderer::Color32::TRANSPARENT, - highlight, - ); - let rect = - glam::vec2(tensor.shape()[1].size as f32, tensor.shape()[0].size as f32); - scene - .primitives - .line_strips - .batch("image outlines") - .world_from_obj(world_from_obj) - .add_axis_aligned_rectangle_outline_2d(glam::Vec2::ZERO, rect) - .color(color) - .radius(Size::new_points(1.0)); - } + let entity_highlight = highlights.entity_highlight(ent_path.hash()); - push_tensor_texture( - scene, - ctx, - &annotations, - world_from_obj, - instance_path_hash, - &tensor, - color.into(), - ); + let instance_path_hash = instance_path_hash_for_picking( + ent_path, + instance_key, + entity_view, + properties, + entity_highlight, + ); - // TODO(jleibs): Meter should really be its own component - let meter = tensor.meter; + let annotations = scene.annotation_map.find(ent_path); - scene.ui.images.push(Image { - instance_path_hash, - tensor, - meter, - annotations, - }); - } + let color = annotations.class_description(None).annotation_info().color( + color.map(|c| c.to_array()).as_ref(), + DefaultColor::OpaqueWhite, + ); + + let highlight = entity_highlight.index_highlight(instance_path_hash.instance_key); + if highlight.is_some() { + let color = SceneSpatial::apply_hover_and_selection_effect_color( + re_renderer::Color32::TRANSPARENT, + highlight, + ); + let rect = glam::vec2(tensor.shape()[1].size as f32, tensor.shape()[0].size as f32); + scene + .primitives + .line_strips + .batch("image outlines") + .world_from_obj(world_from_obj) + .add_axis_aligned_rectangle_outline_2d(glam::Vec2::ZERO, rect) + .color(color) + .radius(Size::new_points(1.0)); } - Ok(()) + push_tensor_texture( + scene, + ctx, + &annotations, + world_from_obj, + instance_path_hash, + &tensor, + color.into(), + ); + + // TODO(jleibs): Meter should really be its own component + let meter = tensor.meter; + + scene.ui.images.push(Image { + instance_path_hash, + tensor, + meter, + annotations, + }); + } + + #[allow(clippy::too_many_arguments)] + fn process_entity_view_as_depth_cloud( + scene: &mut SceneSpatial, + ctx: &mut ViewerContext<'_>, + transforms: &TransformCache, + properties: &EntityProperties, + tensor: &Tensor, + pinhole_ent_path: &EntityPath, + ) { + crate::profile_function!(); + + let Some(re_log_types::Transform::Pinhole(intrinsics)) = query_latest_single::( + &ctx.log_db.entity_db, + pinhole_ent_path, + &ctx.current_query(), + ) else { + re_log::warn_once!("Couldn't fetch pinhole intrinsics at {pinhole_ent_path:?}"); + return; + }; + + // TODO(cmc): getting to those extrinsics is no easy task :| + let Some(extrinsics) = pinhole_ent_path + .parent() + .and_then(|ent_path| transforms.reference_from_entity(&ent_path)) else { + re_log::warn_once!("Couldn't fetch pinhole extrinsics at {pinhole_ent_path:?}"); + return; + }; + + // TODO(cmc): automagically convert as needed for non-natively supported datatypes? + let data = match &tensor.data { + // TODO(cmc): re_renderer needs native interop with Arrow, we shouldn't need to + // allocate this massive buffer and do a memcpy when we could just carry arround the + // original `arrow::Buffer`. + TensorData::U16(data) => DepthCloudDepthData::U16(data.to_vec()), + TensorData::F32(data) => DepthCloudDepthData::F32(data.to_vec()), + _ => { + let discriminant = std::mem::discriminant(&tensor.data); + re_log::warn_once!( + "Tensor datatype is not supported for backprojection ({discriminant:?})" + ); + return; + } + }; + + let scale = *properties.backproject_scale.get(); + let radius_scale = *properties.backproject_radius_scale.get(); + + let (h, w) = (tensor.shape()[0].size, tensor.shape()[1].size); + let dimensions = glam::UVec2::new(w as _, h as _); + + let world_from_obj = extrinsics * glam::Mat4::from_scale(glam::Vec3::splat(scale)); + + scene.primitives.depth_clouds.push(DepthCloud { + depth_camera_extrinsics: world_from_obj, + depth_camera_intrinsics: intrinsics.image_from_cam.into(), + radius_scale, + depth_dimensions: dimensions, + depth_data: data, + }); } } @@ -237,6 +349,7 @@ impl ScenePart for ImagesPart { &entity, scene, ctx, + transforms, &props, ent_path, world_from_obj, diff --git a/crates/re_viewer/src/ui/view_spatial/ui.rs b/crates/re_viewer/src/ui/view_spatial/ui.rs index feaff198f93f..0b811581bcf3 100644 --- a/crates/re_viewer/src/ui/view_spatial/ui.rs +++ b/crates/re_viewer/src/ui/view_spatial/ui.rs @@ -4,6 +4,7 @@ use re_format::format_f32; use egui::{NumExt, WidgetText}; use macaw::BoundingBox; +use re_log_types::component_types::{Tensor, TensorData, TensorDataMeaning}; use crate::{ misc::{ @@ -161,37 +162,100 @@ impl ViewSpatialState { crate::profile_function!(); let scene_size = self.scene_bbox_accum.size().length(); - let default_image_plane_distance = if scene_size.is_finite() && scene_size > 0.0 { - scene_size * 0.05 - } else { - 1.0 - }; let query = ctx.current_query(); let entity_paths = data_blueprint.entity_paths().clone(); // TODO(andreas): Workaround borrow checker for entity_path in entity_paths { - if let Some(re_log_types::Transform::Pinhole(_)) = - query_latest_single::( - &ctx.log_db.entity_db, - &entity_path, - &query, - ) - { - let mut properties = data_blueprint + Self::update_pinhole_property_heuristics( + ctx, + data_blueprint, + &query, + &entity_path, + scene_size, + ); + Self::update_depth_cloud_property_heuristics( + ctx, + data_blueprint, + &query, + &entity_path, + scene_size, + ); + } + } + + fn update_pinhole_property_heuristics( + ctx: &mut ViewerContext<'_>, + data_blueprint: &mut DataBlueprintTree, + query: &re_arrow_store::LatestAtQuery, + entity_path: &EntityPath, + scene_size: f32, + ) { + if let Some(re_log_types::Transform::Pinhole(_)) = + query_latest_single::( + &ctx.log_db.entity_db, + entity_path, + query, + ) + { + let default_image_plane_distance = if scene_size.is_finite() && scene_size > 0.0 { + scene_size * 0.05 + } else { + 1.0 + }; + + let mut properties = data_blueprint.data_blueprints_individual().get(entity_path); + if properties.pinhole_image_plane_distance.is_auto() { + properties.pinhole_image_plane_distance = + EditableAutoValue::Auto(default_image_plane_distance); + data_blueprint .data_blueprints_individual() - .get(&entity_path); - if properties.pinhole_image_plane_distance.is_auto() { - properties.pinhole_image_plane_distance = - EditableAutoValue::Auto(default_image_plane_distance); - data_blueprint - .data_blueprints_individual() - .set(entity_path, properties); - } + .set(entity_path.clone(), properties); } } } + fn update_depth_cloud_property_heuristics( + ctx: &mut ViewerContext<'_>, + data_blueprint: &mut DataBlueprintTree, + query: &re_arrow_store::LatestAtQuery, + entity_path: &EntityPath, + scene_size: f32, + ) { + let tensor = query_latest_single::(&ctx.log_db.entity_db, entity_path, query); + if tensor.as_ref().map(|t| t.meaning) == Some(TensorDataMeaning::Depth) { + let tensor = tensor.as_ref().unwrap(); + + let mut properties = data_blueprint.data_blueprints_individual().get(entity_path); + if properties.backproject_scale.is_auto() { + let auto = tensor.meter.map_or_else( + || match &tensor.data { + TensorData::U16(_) => 1.0 / u16::MAX as f32, + _ => 1.0, + }, + |meter| match &tensor.data { + TensorData::U16(_) => 1.0 / meter * u16::MAX as f32, + _ => meter, + }, + ); + properties.backproject_scale = EditableAutoValue::Auto(auto); + } + + if properties.backproject_radius_scale.is_auto() { + let auto = if scene_size.is_finite() && scene_size > 0.0 { + f32::max(0.02, scene_size * 0.001) + } else { + 0.02 + }; + properties.backproject_radius_scale = EditableAutoValue::Auto(auto); + } + + data_blueprint + .data_blueprints_individual() + .set(entity_path.clone(), properties); + } + } + pub fn selection_ui( &mut self, ctx: &mut ViewerContext<'_>, diff --git a/crates/re_viewer/src/ui/view_spatial/ui_renderer_bridge.rs b/crates/re_viewer/src/ui/view_spatial/ui_renderer_bridge.rs index 30a2758bb653..e04917655fb2 100644 --- a/crates/re_viewer/src/ui/view_spatial/ui_renderer_bridge.rs +++ b/crates/re_viewer/src/ui/view_spatial/ui_renderer_bridge.rs @@ -1,6 +1,6 @@ use egui::mutex::Mutex; use re_renderer::{ - renderer::{GenericSkyboxDrawData, MeshDrawData, RectangleDrawData}, + renderer::{DepthCloudDrawData, GenericSkyboxDrawData, MeshDrawData, RectangleDrawData}, view_builder::{TargetConfiguration, ViewBuilder}, RenderContext, }; @@ -48,6 +48,7 @@ fn create_and_fill_view_builder( view_builder.setup_view(render_ctx, target_config)?; view_builder + .queue_draw(&DepthCloudDrawData::new(render_ctx, &primitives.depth_clouds).unwrap()) .queue_draw(&MeshDrawData::new(render_ctx, &primitives.mesh_instances()).unwrap()) .queue_draw(&primitives.line_strips.to_draw_data(render_ctx)) .queue_draw(&primitives.points.to_draw_data(render_ctx)?)