From 84fd0cbc98d67eb67bbd1e60511a4b6269abb458 Mon Sep 17 00:00:00 2001 From: Andreas Reich Date: Tue, 23 Jan 2024 15:42:06 +0100 Subject: [PATCH 01/28] update fbs files for mesh texture support --- .../definitions/rerun/archetypes/mesh3d.fbs | 9 ++++++--- .../definitions/rerun/components/material.fbs | 2 +- .../definitions/rerun/datatypes/material.fbs | 15 +++++++++++---- .../definitions/rerun/datatypes/tensor_data.fbs | 2 +- 4 files changed, 19 insertions(+), 9 deletions(-) diff --git a/crates/re_types/definitions/rerun/archetypes/mesh3d.fbs b/crates/re_types/definitions/rerun/archetypes/mesh3d.fbs index 6499c920f077..8e7470999a98 100644 --- a/crates/re_types/definitions/rerun/archetypes/mesh3d.fbs +++ b/crates/re_types/definitions/rerun/archetypes/mesh3d.fbs @@ -41,14 +41,17 @@ table Mesh3D ( /// An optional color for each vertex. vertex_colors: [rerun.components.Color] ("attr.rerun.component_optional", nullable, order: 3100); + /// An optional uv texture coordinate for each vertex. + vertex_texcoords: [rerun.components.Position2D] ("attr.rerun.component_optional", nullable, order: 3200); + /// Optional material properties for the mesh as a whole. - mesh_material: rerun.components.Material ("attr.rerun.component_optional", nullable, order: 3200); + mesh_material: rerun.components.Material ("attr.rerun.component_optional", nullable, order: 3300); /// Optional class Ids for the vertices. /// /// The class ID provides colors and labels if not specified explicitly. - class_ids: [rerun.components.ClassId] ("attr.rerun.component_optional", nullable, order: 3300); + class_ids: [rerun.components.ClassId] ("attr.rerun.component_optional", nullable, order: 3400); /// Unique identifiers for each individual vertex in the mesh. - instance_keys: [rerun.components.InstanceKey] ("attr.rerun.component_optional", nullable, order: 3400); + instance_keys: [rerun.components.InstanceKey] ("attr.rerun.component_optional", nullable, order: 3500); } diff --git a/crates/re_types/definitions/rerun/components/material.fbs b/crates/re_types/definitions/rerun/components/material.fbs index a9c6847cf749..a487b16fdd6b 100644 --- a/crates/re_types/definitions/rerun/components/material.fbs +++ b/crates/re_types/definitions/rerun/components/material.fbs @@ -11,7 +11,7 @@ namespace rerun.components; /// Material properties of a mesh. table Material ( - "attr.rust.derive": "PartialEq, Eq" + "attr.rust.derive": "PartialEq" ) { material: rerun.datatypes.Material (order: 100); } diff --git a/crates/re_types/definitions/rerun/datatypes/material.fbs b/crates/re_types/definitions/rerun/datatypes/material.fbs index b42b550f0c8a..941968838479 100644 --- a/crates/re_types/definitions/rerun/datatypes/material.fbs +++ b/crates/re_types/definitions/rerun/datatypes/material.fbs @@ -1,18 +1,25 @@ include "arrow/attributes.fbs"; include "python/attributes.fbs"; include "rust/attributes.fbs"; - include "rerun/attributes.fbs"; -include "rerun/datatypes/rgba32.fbs"; + +include "./rgba32.fbs"; +include "./tensor_buffer.fbs"; namespace rerun.datatypes; // --- /// Material properties of a mesh. -struct Material ( - "attr.rust.derive": "Copy, PartialEq, Eq, Hash" +table Material ( + "attr.rust.derive": "PartialEq" ) { /// Optional color multiplier. albedo_factor: rerun.datatypes.Rgba32 (nullable, order: 100); + + /// Optional albedo texture. + /// + /// Used with `vertex_texcoords` on `Mesh3D`. + /// Currently supports only RGB & RGBA 2D-textures, ignoring alpha. + albedo_texture: rerun.datatypes.TensorData (nullable, order: 200); } diff --git a/crates/re_types/definitions/rerun/datatypes/tensor_data.fbs b/crates/re_types/definitions/rerun/datatypes/tensor_data.fbs index 5ff7be5a81a9..9c60455146e4 100644 --- a/crates/re_types/definitions/rerun/datatypes/tensor_data.fbs +++ b/crates/re_types/definitions/rerun/datatypes/tensor_data.fbs @@ -2,7 +2,7 @@ include "arrow/attributes.fbs"; include "fbs/attributes.fbs"; include "./tensor_dimension.fbs"; -include "./tensor_data.fbs"; +include "./tensor_buffer.fbs"; namespace rerun.datatypes; From 85e39911698bad7637c88c05b94e48f08cfb4786 Mon Sep 17 00:00:00 2001 From: Andreas Reich Date: Tue, 23 Jan 2024 15:51:17 +0100 Subject: [PATCH 02/28] extension fixup --- crates/re_types/src/datatypes/material_ext.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/re_types/src/datatypes/material_ext.rs b/crates/re_types/src/datatypes/material_ext.rs index eed456f3bc51..a4467ecd6c11 100644 --- a/crates/re_types/src/datatypes/material_ext.rs +++ b/crates/re_types/src/datatypes/material_ext.rs @@ -7,6 +7,7 @@ impl Material { pub fn from_albedo_factor(color: impl Into) -> Self { Self { albedo_factor: Some(color.into()), + albedo_texture: None, } } } From c401fe706d04cb69e872be2e7a7db344f9eefbe3 Mon Sep 17 00:00:00 2001 From: Andreas Reich Date: Tue, 23 Jan 2024 15:51:29 +0100 Subject: [PATCH 03/28] re_types tests fixup --- crates/re_types/tests/mesh3d.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/crates/re_types/tests/mesh3d.rs b/crates/re_types/tests/mesh3d.rs index faf1218ab604..cc1837dcd61c 100644 --- a/crates/re_types/tests/mesh3d.rs +++ b/crates/re_types/tests/mesh3d.rs @@ -2,8 +2,8 @@ use std::collections::HashMap; use re_types::{ archetypes::Mesh3D, - components::{ClassId, InstanceKey, Position3D, Vector3D}, - datatypes::{Material, MeshProperties, Rgba32, Vec3D}, + components::{ClassId, InstanceKey, Position2D, Position3D, Vector3D}, + datatypes::{Material, MeshProperties, Rgba32, Vec2D, Vec3D}, Archetype as _, AsComponents as _, }; @@ -28,9 +28,14 @@ fn roundtrip() { Rgba32::from_unmultiplied_rgba(0xAA, 0x00, 0x00, 0xCC).into(), // Rgba32::from_unmultiplied_rgba(0x00, 0xBB, 0x00, 0xDD).into(), ]), + vertex_texcoords: Some(vec![ + Position2D(Vec2D([0.0, 1.0])), // + Position2D(Vec2D([2.0, 3.0])), // + ]), mesh_material: Some( Material { albedo_factor: Some(Rgba32::from_unmultiplied_rgba(0xEE, 0x11, 0x22, 0x33)), + albedo_texture: None, } .into(), ), @@ -51,6 +56,7 @@ fn roundtrip() { ])) .with_vertex_normals([[4.0, 5.0, 6.0], [40.0, 50.0, 60.0]]) .with_vertex_colors([0xAA0000CC, 0x00BB00DD]) + .with_vertex_texcoords([[0.0, 1.0], [2.0, 3.0]]) .with_mesh_material(Material::from_albedo_factor(0xEE112233)) .with_class_ids([126, 127]) .with_instance_keys([u64::MAX - 1, u64::MAX]); From 9742012fe9c2d5ee689163bd13dd276ae4118d37 Mon Sep 17 00:00:00 2001 From: Andreas Reich Date: Tue, 23 Jan 2024 15:42:59 +0100 Subject: [PATCH 04/28] codegen --- crates/re_types/src/archetypes/mesh3d.rs | 40 +++- crates/re_types/src/components/material.rs | 22 ++- crates/re_types/src/datatypes/material.rs | 182 +++++++++++------- .../reference/types/archetypes/mesh3d.md | 2 +- .../reference/types/components/position2d.md | 1 + .../reference/types/datatypes/material.md | 1 + .../reference/types/datatypes/tensor_data.md | 1 + rerun_cpp/src/rerun/archetypes/mesh3d.cpp | 7 +- rerun_cpp/src/rerun/archetypes/mesh3d.hpp | 12 ++ rerun_cpp/src/rerun/components/material.hpp | 15 +- rerun_cpp/src/rerun/datatypes/material.cpp | 24 +++ rerun_cpp/src/rerun/datatypes/material.hpp | 22 +-- rerun_py/rerun_sdk/rerun/archetypes/mesh3d.py | 10 + .../rerun_sdk/rerun/datatypes/material.py | 144 +++++++++++++- 14 files changed, 371 insertions(+), 112 deletions(-) diff --git a/crates/re_types/src/archetypes/mesh3d.rs b/crates/re_types/src/archetypes/mesh3d.rs index ebb4ebc9e6ca..ebe9c97fa162 100644 --- a/crates/re_types/src/archetypes/mesh3d.rs +++ b/crates/re_types/src/archetypes/mesh3d.rs @@ -71,6 +71,9 @@ pub struct Mesh3D { /// An optional color for each vertex. pub vertex_colors: Option>, + /// An optional uv texture coordinate for each vertex. + pub vertex_texcoords: Option>, + /// Optional material properties for the mesh as a whole. pub mesh_material: Option, @@ -90,6 +93,7 @@ impl ::re_types_core::SizeBytes for Mesh3D { + self.mesh_properties.heap_size_bytes() + self.vertex_normals.heap_size_bytes() + self.vertex_colors.heap_size_bytes() + + self.vertex_texcoords.heap_size_bytes() + self.mesh_material.heap_size_bytes() + self.class_ids.heap_size_bytes() + self.instance_keys.heap_size_bytes() @@ -101,6 +105,7 @@ impl ::re_types_core::SizeBytes for Mesh3D { && >::is_pod() && >>::is_pod() && >>::is_pod() + && >>::is_pod() && >::is_pod() && >>::is_pod() && >>::is_pod() @@ -119,17 +124,18 @@ static RECOMMENDED_COMPONENTS: once_cell::sync::Lazy<[ComponentName; 3usize]> = ] }); -static OPTIONAL_COMPONENTS: once_cell::sync::Lazy<[ComponentName; 4usize]> = +static OPTIONAL_COMPONENTS: once_cell::sync::Lazy<[ComponentName; 5usize]> = once_cell::sync::Lazy::new(|| { [ "rerun.components.ClassId".into(), "rerun.components.Color".into(), "rerun.components.InstanceKey".into(), "rerun.components.Material".into(), + "rerun.components.Position2D".into(), ] }); -static ALL_COMPONENTS: once_cell::sync::Lazy<[ComponentName; 8usize]> = +static ALL_COMPONENTS: once_cell::sync::Lazy<[ComponentName; 9usize]> = once_cell::sync::Lazy::new(|| { [ "rerun.components.Position3D".into(), @@ -140,11 +146,12 @@ static ALL_COMPONENTS: once_cell::sync::Lazy<[ComponentName; 8usize]> = "rerun.components.Color".into(), "rerun.components.InstanceKey".into(), "rerun.components.Material".into(), + "rerun.components.Position2D".into(), ] }); impl Mesh3D { - pub const NUM_COMPONENTS: usize = 8usize; + pub const NUM_COMPONENTS: usize = 9usize; } /// Indicator component for the [`Mesh3D`] [`::re_types_core::Archetype`] @@ -240,6 +247,19 @@ impl ::re_types_core::Archetype for Mesh3D { } else { None }; + let vertex_texcoords = + if let Some(array) = arrays_by_name.get("rerun.components.Position2D") { + Some({ + ::from_arrow_opt(&**array) + .with_context("rerun.archetypes.Mesh3D#vertex_texcoords")? + .into_iter() + .map(|v| v.ok_or_else(DeserializationError::missing_data)) + .collect::>>() + .with_context("rerun.archetypes.Mesh3D#vertex_texcoords")? + }) + } else { + None + }; let mesh_material = if let Some(array) = arrays_by_name.get("rerun.components.Material") { ::from_arrow_opt(&**array) .with_context("rerun.archetypes.Mesh3D#mesh_material")? @@ -279,6 +299,7 @@ impl ::re_types_core::Archetype for Mesh3D { mesh_properties, vertex_normals, vertex_colors, + vertex_texcoords, mesh_material, class_ids, instance_keys, @@ -302,6 +323,9 @@ impl ::re_types_core::AsComponents for Mesh3D { self.vertex_colors .as_ref() .map(|comp_batch| (comp_batch as &dyn ComponentBatch).into()), + self.vertex_texcoords + .as_ref() + .map(|comp_batch| (comp_batch as &dyn ComponentBatch).into()), self.mesh_material .as_ref() .map(|comp| (comp as &dyn ComponentBatch).into()), @@ -332,6 +356,7 @@ impl Mesh3D { mesh_properties: None, vertex_normals: None, vertex_colors: None, + vertex_texcoords: None, mesh_material: None, class_ids: None, instance_keys: None, @@ -365,6 +390,15 @@ impl Mesh3D { self } + #[inline] + pub fn with_vertex_texcoords( + mut self, + vertex_texcoords: impl IntoIterator>, + ) -> Self { + self.vertex_texcoords = Some(vertex_texcoords.into_iter().map(Into::into).collect()); + self + } + #[inline] pub fn with_mesh_material( mut self, diff --git a/crates/re_types/src/components/material.rs b/crates/re_types/src/components/material.rs index 856813a7338d..9aafef54ed34 100644 --- a/crates/re_types/src/components/material.rs +++ b/crates/re_types/src/components/material.rs @@ -22,7 +22,7 @@ use ::re_types_core::{ComponentBatch, MaybeOwnedComponentBatch}; use ::re_types_core::{DeserializationError, DeserializationResult}; /// **Component**: Material properties of a mesh. -#[derive(Clone, Debug, PartialEq, Eq)] +#[derive(Clone, Debug, PartialEq)] pub struct Material(pub crate::datatypes::Material); impl ::re_types_core::SizeBytes for Material { @@ -73,12 +73,20 @@ impl ::re_types_core::Loggable for Material { #[inline] fn arrow_datatype() -> arrow2::datatypes::DataType { use arrow2::datatypes::*; - DataType::Struct(vec![Field { - name: "albedo_factor".to_owned(), - data_type: ::arrow_datatype(), - is_nullable: true, - metadata: [].into(), - }]) + DataType::Struct(vec![ + Field { + name: "albedo_factor".to_owned(), + data_type: ::arrow_datatype(), + is_nullable: true, + metadata: [].into(), + }, + Field { + name: "albedo_texture".to_owned(), + data_type: ::arrow_datatype(), + is_nullable: true, + metadata: [].into(), + }, + ]) } #[allow(clippy::wildcard_imports)] diff --git a/crates/re_types/src/datatypes/material.rs b/crates/re_types/src/datatypes/material.rs index 3807b3984387..5a2dc955ebcc 100644 --- a/crates/re_types/src/datatypes/material.rs +++ b/crates/re_types/src/datatypes/material.rs @@ -22,45 +22,28 @@ use ::re_types_core::{ComponentBatch, MaybeOwnedComponentBatch}; use ::re_types_core::{DeserializationError, DeserializationResult}; /// **Datatype**: Material properties of a mesh. -#[derive(Clone, Debug, Copy, PartialEq, Eq, Hash)] +#[derive(Clone, Debug, PartialEq)] pub struct Material { /// Optional color multiplier. pub albedo_factor: Option, + + /// Optional albedo texture. + /// + /// Used with `vertex_texcoords` on `Mesh3D`. + /// Currently supports only RGB & RGBA 2D-textures, ignoring alpha. + pub albedo_texture: Option, } impl ::re_types_core::SizeBytes for Material { #[inline] fn heap_size_bytes(&self) -> u64 { - self.albedo_factor.heap_size_bytes() + self.albedo_factor.heap_size_bytes() + self.albedo_texture.heap_size_bytes() } #[inline] fn is_pod() -> bool { >::is_pod() - } -} - -impl>> From for Material { - fn from(v: T) -> Self { - Self { - albedo_factor: v.into(), - } - } -} - -impl std::borrow::Borrow> for Material { - #[inline] - fn borrow(&self) -> &Option { - &self.albedo_factor - } -} - -impl std::ops::Deref for Material { - type Target = Option; - - #[inline] - fn deref(&self) -> &Option { - &self.albedo_factor + && >::is_pod() } } @@ -78,12 +61,20 @@ impl ::re_types_core::Loggable for Material { #[inline] fn arrow_datatype() -> arrow2::datatypes::DataType { use arrow2::datatypes::*; - DataType::Struct(vec![Field { - name: "albedo_factor".to_owned(), - data_type: ::arrow_datatype(), - is_nullable: true, - metadata: [].into(), - }]) + DataType::Struct(vec![ + Field { + name: "albedo_factor".to_owned(), + data_type: ::arrow_datatype(), + is_nullable: true, + metadata: [].into(), + }, + Field { + name: "albedo_texture".to_owned(), + data_type: ::arrow_datatype(), + is_nullable: true, + metadata: [].into(), + }, + ]) } #[allow(clippy::wildcard_imports)] @@ -109,41 +100,66 @@ impl ::re_types_core::Loggable for Material { }; StructArray::new( ::arrow_datatype(), - vec![{ - let (somes, albedo_factor): (Vec<_>, Vec<_>) = data - .iter() - .map(|datum| { - let datum = datum - .as_ref() + vec![ + { + let (somes, albedo_factor): (Vec<_>, Vec<_>) = data + .iter() + .map(|datum| { + let datum = datum + .as_ref() + .map(|datum| { + let Self { albedo_factor, .. } = &**datum; + albedo_factor.clone() + }) + .flatten(); + (datum.is_some(), datum) + }) + .unzip(); + let albedo_factor_bitmap: Option = { + let any_nones = somes.iter().any(|some| !*some); + any_nones.then(|| somes.into()) + }; + PrimitiveArray::new( + DataType::UInt32, + albedo_factor + .into_iter() .map(|datum| { - let Self { albedo_factor, .. } = &**datum; - albedo_factor.clone() + datum + .map(|datum| { + let crate::datatypes::Rgba32(data0) = datum; + data0 + }) + .unwrap_or_default() }) - .flatten(); - (datum.is_some(), datum) - }) - .unzip(); - let albedo_factor_bitmap: Option = { - let any_nones = somes.iter().any(|some| !*some); - any_nones.then(|| somes.into()) - }; - PrimitiveArray::new( - DataType::UInt32, - albedo_factor - .into_iter() + .collect(), + albedo_factor_bitmap, + ) + .boxed() + }, + { + let (somes, albedo_texture): (Vec<_>, Vec<_>) = data + .iter() .map(|datum| { - datum + let datum = datum + .as_ref() .map(|datum| { - let crate::datatypes::Rgba32(data0) = datum; - data0 + let Self { albedo_texture, .. } = &**datum; + albedo_texture.clone() }) - .unwrap_or_default() + .flatten(); + (datum.is_some(), datum) }) - .collect(), - albedo_factor_bitmap, - ) - .boxed() - }], + .unzip(); + let albedo_texture_bitmap: Option = { + let any_nones = somes.iter().any(|some| !*some); + any_nones.then(|| somes.into()) + }; + { + _ = albedo_texture_bitmap; + crate::datatypes::TensorData::to_arrow_opt(albedo_texture)? + } + }, + ], bitmap, ) .boxed() @@ -165,12 +181,20 @@ impl ::re_types_core::Loggable for Material { .downcast_ref::() .ok_or_else(|| { DeserializationError::datatype_mismatch( - DataType::Struct(vec![Field { - name: "albedo_factor".to_owned(), - data_type: ::arrow_datatype(), - is_nullable: true, - metadata: [].into(), - }]), + DataType::Struct(vec![ + Field { + name: "albedo_factor".to_owned(), + data_type: ::arrow_datatype(), + is_nullable: true, + metadata: [].into(), + }, + Field { + name: "albedo_texture".to_owned(), + data_type: ::arrow_datatype(), + is_nullable: true, + metadata: [].into(), + }, + ]), arrow_data.data_type().clone(), ) }) @@ -208,13 +232,31 @@ impl ::re_types_core::Loggable for Material { .map(|opt| opt.copied()) .map(|res_or_opt| res_or_opt.map(|v| crate::datatypes::Rgba32(v))) }; + let albedo_texture = { + if !arrays_by_name.contains_key("albedo_texture") { + return Err(DeserializationError::missing_struct_field( + Self::arrow_datatype(), + "albedo_texture", + )) + .with_context("rerun.datatypes.Material"); + } + let arrow_data = &**arrays_by_name["albedo_texture"]; + crate::datatypes::TensorData::from_arrow_opt(arrow_data) + .with_context("rerun.datatypes.Material#albedo_texture")? + .into_iter() + }; arrow2::bitmap::utils::ZipValidity::new_with_validity( - ::itertools::izip!(albedo_factor), + ::itertools::izip!(albedo_factor, albedo_texture), arrow_data.validity(), ) .map(|opt| { - opt.map(|(albedo_factor)| Ok(Self { albedo_factor })) - .transpose() + opt.map(|(albedo_factor, albedo_texture)| { + Ok(Self { + albedo_factor, + albedo_texture, + }) + }) + .transpose() }) .collect::>>() .with_context("rerun.datatypes.Material")? diff --git a/docs/content/reference/types/archetypes/mesh3d.md b/docs/content/reference/types/archetypes/mesh3d.md index 627bf593d275..35980211f18d 100644 --- a/docs/content/reference/types/archetypes/mesh3d.md +++ b/docs/content/reference/types/archetypes/mesh3d.md @@ -10,7 +10,7 @@ A 3D triangle mesh as specified by its per-mesh and per-vertex properties. **Recommended**: [`MeshProperties`](../components/mesh_properties.md), [`Vector3D`](../components/vector3d.md) -**Optional**: [`Color`](../components/color.md), [`Material`](../components/material.md), [`ClassId`](../components/class_id.md), [`InstanceKey`](../components/instance_key.md) +**Optional**: [`Color`](../components/color.md), [`Position2D`](../components/position2d.md), [`Material`](../components/material.md), [`ClassId`](../components/class_id.md), [`InstanceKey`](../components/instance_key.md) ## Links * 🌊 [C++ API docs for `Mesh3D`](https://ref.rerun.io/docs/cpp/stable/structrerun_1_1archetypes_1_1Mesh3D.html) diff --git a/docs/content/reference/types/components/position2d.md b/docs/content/reference/types/components/position2d.md index 875baccd1458..f513a60c84d9 100644 --- a/docs/content/reference/types/components/position2d.md +++ b/docs/content/reference/types/components/position2d.md @@ -18,4 +18,5 @@ A position in 2D space. * [`Arrows2D`](../archetypes/arrows2d.md?speculative-link) * [`Boxes2D`](../archetypes/boxes2d.md) +* [`Mesh3D`](../archetypes/mesh3d.md) * [`Points2D`](../archetypes/points2d.md) diff --git a/docs/content/reference/types/datatypes/material.md b/docs/content/reference/types/datatypes/material.md index 3b2a1d1fd1f2..5256976fd2e3 100644 --- a/docs/content/reference/types/datatypes/material.md +++ b/docs/content/reference/types/datatypes/material.md @@ -7,6 +7,7 @@ Material properties of a mesh. ## Fields * albedo_factor: [`Rgba32`](../datatypes/rgba32.md) +* albedo_texture: [`TensorData`](../datatypes/tensor_data.md) ## Links * 🌊 [C++ API docs for `Material`](https://ref.rerun.io/docs/cpp/stable/structrerun_1_1datatypes_1_1Material.html) diff --git a/docs/content/reference/types/datatypes/tensor_data.md b/docs/content/reference/types/datatypes/tensor_data.md index a68ec7f7aeae..95374f70fc8f 100644 --- a/docs/content/reference/types/datatypes/tensor_data.md +++ b/docs/content/reference/types/datatypes/tensor_data.md @@ -25,3 +25,4 @@ which stores a contiguous array of typed values. ## Used by * [`TensorData`](../components/tensor_data.md) +* [`Material`](../datatypes/material.md) diff --git a/rerun_cpp/src/rerun/archetypes/mesh3d.cpp b/rerun_cpp/src/rerun/archetypes/mesh3d.cpp index c3e0a07429d2..194ef40b7566 100644 --- a/rerun_cpp/src/rerun/archetypes/mesh3d.cpp +++ b/rerun_cpp/src/rerun/archetypes/mesh3d.cpp @@ -14,7 +14,7 @@ namespace rerun { ) { using namespace archetypes; std::vector cells; - cells.reserve(8); + cells.reserve(9); { auto result = DataCell::from_loggable(archetype.vertex_positions); @@ -36,6 +36,11 @@ namespace rerun { RR_RETURN_NOT_OK(result.error); cells.push_back(std::move(result.value)); } + if (archetype.vertex_texcoords.has_value()) { + auto result = DataCell::from_loggable(archetype.vertex_texcoords.value()); + RR_RETURN_NOT_OK(result.error); + cells.push_back(std::move(result.value)); + } if (archetype.mesh_material.has_value()) { auto result = DataCell::from_loggable(archetype.mesh_material.value()); RR_RETURN_NOT_OK(result.error); diff --git a/rerun_cpp/src/rerun/archetypes/mesh3d.hpp b/rerun_cpp/src/rerun/archetypes/mesh3d.hpp index b19b9b91809e..42c1cda4b9cb 100644 --- a/rerun_cpp/src/rerun/archetypes/mesh3d.hpp +++ b/rerun_cpp/src/rerun/archetypes/mesh3d.hpp @@ -10,6 +10,7 @@ #include "../components/instance_key.hpp" #include "../components/material.hpp" #include "../components/mesh_properties.hpp" +#include "../components/position2d.hpp" #include "../components/position3d.hpp" #include "../components/vector3d.hpp" #include "../data_cell.hpp" @@ -77,6 +78,9 @@ namespace rerun::archetypes { /// An optional color for each vertex. std::optional> vertex_colors; + /// An optional uv texture coordinate for each vertex. + std::optional> vertex_texcoords; + /// Optional material properties for the mesh as a whole. std::optional mesh_material; @@ -124,6 +128,14 @@ namespace rerun::archetypes { RR_WITH_MAYBE_UNINITIALIZED_DISABLED(return std::move(*this);) } + /// An optional uv texture coordinate for each vertex. + Mesh3D with_vertex_texcoords(Collection _vertex_texcoords + ) && { + vertex_texcoords = std::move(_vertex_texcoords); + // See: https://github.com/rerun-io/rerun/issues/4027 + RR_WITH_MAYBE_UNINITIALIZED_DISABLED(return std::move(*this);) + } + /// Optional material properties for the mesh as a whole. Mesh3D with_mesh_material(rerun::components::Material _mesh_material) && { mesh_material = std::move(_mesh_material); diff --git a/rerun_cpp/src/rerun/components/material.hpp b/rerun_cpp/src/rerun/components/material.hpp index dfe861f11724..eaceb8bddebf 100644 --- a/rerun_cpp/src/rerun/components/material.hpp +++ b/rerun_cpp/src/rerun/components/material.hpp @@ -4,12 +4,11 @@ #pragma once #include "../datatypes/material.hpp" -#include "../datatypes/rgba32.hpp" #include "../result.hpp" #include #include -#include +#include namespace arrow { class Array; @@ -32,18 +31,10 @@ namespace rerun::components { public: Material() = default; - Material(rerun::datatypes::Material material_) : material(material_) {} + Material(rerun::datatypes::Material material_) : material(std::move(material_)) {} Material& operator=(rerun::datatypes::Material material_) { - material = material_; - return *this; - } - - Material(std::optional albedo_factor_) - : material(albedo_factor_) {} - - Material& operator=(std::optional albedo_factor_) { - material = albedo_factor_; + material = std::move(material_); return *this; } diff --git a/rerun_cpp/src/rerun/datatypes/material.cpp b/rerun_cpp/src/rerun/datatypes/material.cpp index 9512650af0c8..3456154676db 100644 --- a/rerun_cpp/src/rerun/datatypes/material.cpp +++ b/rerun_cpp/src/rerun/datatypes/material.cpp @@ -4,6 +4,7 @@ #include "material.hpp" #include "rgba32.hpp" +#include "tensor_data.hpp" #include #include @@ -18,6 +19,11 @@ namespace rerun { Loggable::arrow_datatype(), true ), + arrow::field( + "albedo_texture", + Loggable::arrow_datatype(), + true + ), }); return datatype; } @@ -51,6 +57,24 @@ namespace rerun { } } } + { + auto field_builder = static_cast(builder->field_builder(1)); + ARROW_RETURN_NOT_OK(field_builder->Reserve(static_cast(num_elements))); + for (size_t elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { + const auto& element = elements[elem_idx]; + if (element.albedo_texture.has_value()) { + RR_RETURN_NOT_OK( + Loggable::fill_arrow_array_builder( + field_builder, + &element.albedo_texture.value(), + 1 + ) + ); + } else { + ARROW_RETURN_NOT_OK(field_builder->AppendNull()); + } + } + } ARROW_RETURN_NOT_OK(builder->AppendValues(static_cast(num_elements), nullptr)); return Error::ok(); diff --git a/rerun_cpp/src/rerun/datatypes/material.hpp b/rerun_cpp/src/rerun/datatypes/material.hpp index efd12166804e..eeab33df3cd8 100644 --- a/rerun_cpp/src/rerun/datatypes/material.hpp +++ b/rerun_cpp/src/rerun/datatypes/material.hpp @@ -5,6 +5,7 @@ #include "../result.hpp" #include "rgba32.hpp" +#include "tensor_data.hpp" #include #include @@ -22,23 +23,14 @@ namespace rerun::datatypes { /// Optional color multiplier. std::optional albedo_factor; + /// Optional albedo texture. + /// + /// Used with `vertex_texcoords` on `Mesh3D`. + /// Currently supports only RGB & RGBA 2D-textures, ignoring alpha. + std::optional albedo_texture; + public: Material() = default; - - Material(std::optional albedo_factor_) - : albedo_factor(albedo_factor_) {} - - Material& operator=(std::optional albedo_factor_) { - albedo_factor = albedo_factor_; - return *this; - } - - Material(uint32_t rgba_) : albedo_factor(rgba_) {} - - Material& operator=(uint32_t rgba_) { - albedo_factor = rgba_; - return *this; - } }; } // namespace rerun::datatypes diff --git a/rerun_py/rerun_sdk/rerun/archetypes/mesh3d.py b/rerun_py/rerun_sdk/rerun/archetypes/mesh3d.py index 79eb8c3b9e3d..eea77fe728ae 100644 --- a/rerun_py/rerun_sdk/rerun/archetypes/mesh3d.py +++ b/rerun_py/rerun_sdk/rerun/archetypes/mesh3d.py @@ -60,6 +60,7 @@ def __attrs_clear__(self) -> None: mesh_properties=None, # type: ignore[arg-type] vertex_normals=None, # type: ignore[arg-type] vertex_colors=None, # type: ignore[arg-type] + vertex_texcoords=None, # type: ignore[arg-type] mesh_material=None, # type: ignore[arg-type] class_ids=None, # type: ignore[arg-type] instance_keys=None, # type: ignore[arg-type] @@ -111,6 +112,15 @@ def _clear(cls) -> Mesh3D: # # (Docstring intentionally commented out to hide this field from the docs) + vertex_texcoords: components.Position2DBatch | None = field( + metadata={"component": "optional"}, + default=None, + converter=components.Position2DBatch._optional, # type: ignore[misc] + ) + # An optional uv texture coordinate for each vertex. + # + # (Docstring intentionally commented out to hide this field from the docs) + mesh_material: components.MaterialBatch | None = field( metadata={"component": "optional"}, default=None, diff --git a/rerun_py/rerun_sdk/rerun/datatypes/material.py b/rerun_py/rerun_sdk/rerun/datatypes/material.py index 8bb491b01545..49a840c67db5 100644 --- a/rerun_py/rerun_sdk/rerun/datatypes/material.py +++ b/rerun_py/rerun_sdk/rerun/datatypes/material.py @@ -32,7 +32,11 @@ def _material__albedo_factor__special_field_converter_override( class Material(MaterialExt): """**Datatype**: Material properties of a mesh.""" - def __init__(self: Any, albedo_factor: datatypes.Rgba32Like | None = None): + def __init__( + self: Any, + albedo_factor: datatypes.Rgba32Like | None = None, + albedo_texture: datatypes.TensorDataLike | None = None, + ): """ Create a new instance of the Material datatype. @@ -40,10 +44,15 @@ def __init__(self: Any, albedo_factor: datatypes.Rgba32Like | None = None): ---------- albedo_factor: Optional color multiplier. + albedo_texture: + Optional albedo texture. + + Used with `vertex_texcoords` on `Mesh3D`. + Currently supports only RGB & RGBA 2D-textures, ignoring alpha. """ # You can define your own __init__ function as a member of MaterialExt in material_ext.py - self.__attrs_init__(albedo_factor=albedo_factor) + self.__attrs_init__(albedo_factor=albedo_factor, albedo_texture=albedo_texture) albedo_factor: datatypes.Rgba32 | None = field( default=None, converter=_material__albedo_factor__special_field_converter_override @@ -52,6 +61,14 @@ def __init__(self: Any, albedo_factor: datatypes.Rgba32Like | None = None): # # (Docstring intentionally commented out to hide this field from the docs) + albedo_texture: datatypes.TensorData | None = field(default=None) + # Optional albedo texture. + # + # Used with `vertex_texcoords` on `Mesh3D`. + # Currently supports only RGB & RGBA 2D-textures, ignoring alpha. + # + # (Docstring intentionally commented out to hide this field from the docs) + MaterialLike = Material MaterialArrayLike = Union[ @@ -65,7 +82,128 @@ class MaterialType(BaseExtensionType): def __init__(self) -> None: pa.ExtensionType.__init__( - self, pa.struct([pa.field("albedo_factor", pa.uint32(), nullable=True, metadata={})]), self._TYPE_NAME + self, + pa.struct( + [ + pa.field("albedo_factor", pa.uint32(), nullable=True, metadata={}), + pa.field( + "albedo_texture", + pa.struct( + [ + pa.field( + "shape", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field("size", pa.uint64(), nullable=False, metadata={}), + pa.field("name", pa.utf8(), nullable=True, metadata={}), + ] + ), + nullable=False, + metadata={}, + ) + ), + nullable=False, + metadata={}, + ), + pa.field( + "buffer", + pa.dense_union( + [ + pa.field("_null_markers", pa.null(), nullable=True, metadata={}), + pa.field( + "U8", + pa.list_(pa.field("item", pa.uint8(), nullable=False, metadata={})), + nullable=False, + metadata={}, + ), + pa.field( + "U16", + pa.list_(pa.field("item", pa.uint16(), nullable=False, metadata={})), + nullable=False, + metadata={}, + ), + pa.field( + "U32", + pa.list_(pa.field("item", pa.uint32(), nullable=False, metadata={})), + nullable=False, + metadata={}, + ), + pa.field( + "U64", + pa.list_(pa.field("item", pa.uint64(), nullable=False, metadata={})), + nullable=False, + metadata={}, + ), + pa.field( + "I8", + pa.list_(pa.field("item", pa.int8(), nullable=False, metadata={})), + nullable=False, + metadata={}, + ), + pa.field( + "I16", + pa.list_(pa.field("item", pa.int16(), nullable=False, metadata={})), + nullable=False, + metadata={}, + ), + pa.field( + "I32", + pa.list_(pa.field("item", pa.int32(), nullable=False, metadata={})), + nullable=False, + metadata={}, + ), + pa.field( + "I64", + pa.list_(pa.field("item", pa.int64(), nullable=False, metadata={})), + nullable=False, + metadata={}, + ), + pa.field( + "F16", + pa.list_(pa.field("item", pa.float16(), nullable=False, metadata={})), + nullable=False, + metadata={}, + ), + pa.field( + "F32", + pa.list_(pa.field("item", pa.float32(), nullable=False, metadata={})), + nullable=False, + metadata={}, + ), + pa.field( + "F64", + pa.list_(pa.field("item", pa.float64(), nullable=False, metadata={})), + nullable=False, + metadata={}, + ), + pa.field( + "JPEG", + pa.list_(pa.field("item", pa.uint8(), nullable=False, metadata={})), + nullable=False, + metadata={}, + ), + pa.field( + "NV12", + pa.list_(pa.field("item", pa.uint8(), nullable=False, metadata={})), + nullable=False, + metadata={}, + ), + ] + ), + nullable=False, + metadata={}, + ), + ] + ), + nullable=True, + metadata={}, + ), + ] + ), + self._TYPE_NAME, ) From ce87aeea26ebbf02b04ff0a69e58c8a7ebd9255c Mon Sep 17 00:00:00 2001 From: Andreas Reich Date: Tue, 23 Jan 2024 17:46:44 +0100 Subject: [PATCH 05/28] mesh visualizer picks albedo texture now --- crates/re_renderer/src/lib.rs | 2 + .../re_space_view_spatial/src/mesh_cache.rs | 6 +- .../re_space_view_spatial/src/mesh_loader.rs | 64 +++++++++++++++++-- .../src/visualizers/meshes.rs | 18 +++++- .../re_viewer_context/src/gpu_bridge/mod.rs | 1 + .../src/gpu_bridge/tensor_to_gpu.rs | 4 +- 6 files changed, 83 insertions(+), 12 deletions(-) diff --git a/crates/re_renderer/src/lib.rs b/crates/re_renderer/src/lib.rs index 8d6909a0e7b6..d197d1678f20 100644 --- a/crates/re_renderer/src/lib.rs +++ b/crates/re_renderer/src/lib.rs @@ -81,6 +81,8 @@ pub use self::file_server::FileServer; // Re-export used color types. pub use ecolor::{Color32, Hsva, Rgba}; +// Re-export wgpu +pub use wgpu; // --------------------------------------------------------------------------- diff --git a/crates/re_space_view_spatial/src/mesh_cache.rs b/crates/re_space_view_spatial/src/mesh_cache.rs index f1de1d769556..76e5ca3dbb72 100644 --- a/crates/re_space_view_spatial/src/mesh_cache.rs +++ b/crates/re_space_view_spatial/src/mesh_cache.rs @@ -24,7 +24,11 @@ pub struct MeshCache(ahash::HashMap>>); #[derive(Debug, Clone, Copy)] pub enum AnyMesh<'a> { Asset(&'a re_types::archetypes::Asset3D), - Mesh(&'a re_types::archetypes::Mesh3D), + Mesh { + mesh: &'a re_types::archetypes::Mesh3D, + /// If any textures are in the mesh's material, they use this hash for texture manager lookup. + texture_key: u64, + }, } impl MeshCache { diff --git a/crates/re_space_view_spatial/src/mesh_loader.rs b/crates/re_space_view_spatial/src/mesh_loader.rs index b75e00ce929a..9b4267c15c62 100644 --- a/crates/re_space_view_spatial/src/mesh_loader.rs +++ b/crates/re_space_view_spatial/src/mesh_loader.rs @@ -3,6 +3,7 @@ use re_renderer::{resource_managers::ResourceLifeTime, RenderContext, Rgba32Unmu use re_types::{ archetypes::{Asset3D, Mesh3D}, components::MediaType, + datatypes::TensorBuffer, }; use crate::mesh_cache::AnyMesh; @@ -26,7 +27,9 @@ impl LoadedMesh { // TODO(emilk): load CpuMesh in background thread. match mesh { AnyMesh::Asset(asset3d) => Self::load_asset3d(name, asset3d, render_ctx), - AnyMesh::Mesh(mesh3d) => Ok(Self::load_mesh3d(name, mesh3d, render_ctx)?), + AnyMesh::Mesh { mesh, texture_key } => { + Ok(Self::load_mesh3d(name, mesh, texture_key, render_ctx)?) + } } } @@ -85,6 +88,7 @@ impl LoadedMesh { fn load_mesh3d( name: String, mesh3d: &Mesh3D, + texture_key: u64, render_ctx: &RenderContext, ) -> anyhow::Result { re_tracing::profile_function!(); @@ -94,6 +98,7 @@ impl LoadedMesh { mesh_properties, vertex_normals, vertex_colors, + vertex_texcoords, mesh_material, class_ids: _, instance_keys: _, @@ -135,11 +140,16 @@ impl LoadedMesh { normals.iter().map(|v| v.0.into()).collect::>() } else { // TODO(andreas): Calculate normals - // TODO(cmc): support textured raw meshes vec![glam::Vec3::ZERO; num_positions] }; - let vertex_texcoords = vec![glam::Vec2::ZERO; vertex_normals.len()]; + let vertex_texcoords = if let Some(texcoords) = vertex_texcoords { + re_tracing::profile_scope!("collect_texcoords"); + texcoords.iter().map(|v| v.0.into()).collect::>() + } else { + vec![glam::Vec2::ZERO; num_positions] + }; + let albedo_factor = mesh_material.as_ref().and_then(|mat| mat.albedo_factor); let bbox = { @@ -147,6 +157,18 @@ impl LoadedMesh { macaw::BoundingBox::from_points(vertex_positions.iter().copied()) }; + let albedo = if let Some(albedo_texture) = mesh_material + .as_ref() + .and_then(|mat| mat.albedo_texture.as_ref()) + { + mesh_texture_from_tensor_data(albedo_texture, render_ctx, texture_key)? + } else { + render_ctx + .texture_manager_2d + .white_texture_unorm_handle() + .clone() + }; + let mesh = re_renderer::mesh::Mesh { label: name.clone().into(), triangle_indices, @@ -157,10 +179,7 @@ impl LoadedMesh { materials: smallvec::smallvec![re_renderer::mesh::Material { label: name.clone().into(), index_range: 0..num_indices as _, - albedo: render_ctx - .texture_manager_2d - .white_texture_unorm_handle() - .clone(), + albedo, albedo_multiplier: albedo_factor.map_or(re_renderer::Rgba::WHITE, |c| c.into()), }], }; @@ -190,3 +209,34 @@ impl LoadedMesh { self.bbox } } + +fn mesh_texture_from_tensor_data( + albedo_texture: &re_types::datatypes::TensorData, + render_ctx: &RenderContext, + texture_key: u64, +) -> anyhow::Result { + let [height, width, depth] = + re_viewer_context::gpu_bridge::texture_height_width_channels(albedo_texture)?; + + re_viewer_context::gpu_bridge::try_get_or_create_texture(render_ctx, texture_key, || { + let data = match (depth, &albedo_texture.buffer) { + (3, TensorBuffer::U8(buf)) => re_renderer::pad_rgb_to_rgba(buf, u8::MAX).into(), + (4, TensorBuffer::U8(buf)) => bytemuck::cast_slice(buf.as_slice()).into(), + + _ => { + anyhow::bail!( + "Only 3 and 4 channel u8 tensor data is supported currently for mesh textures." + ); + } + }; + + Ok(re_renderer::resource_managers::Texture2DCreationDesc { + label: "mesh albedo texture from tensor data".into(), + data, + format: re_renderer::wgpu::TextureFormat::Rgba8UnormSrgb, + width, + height, + }) + }) + .map_err(|err| anyhow::format_err!("{err}")) +} diff --git a/crates/re_space_view_spatial/src/visualizers/meshes.rs b/crates/re_space_view_spatial/src/visualizers/meshes.rs index 05f79d3e0c4a..70b24d1a3844 100644 --- a/crates/re_space_view_spatial/src/visualizers/meshes.rs +++ b/crates/re_space_view_spatial/src/visualizers/meshes.rs @@ -3,7 +3,7 @@ use re_query::{ArchetypeView, QueryError}; use re_renderer::renderer::MeshInstance; use re_types::{ archetypes::Mesh3D, - components::{Color, InstanceKey, Material, MeshProperties, Position3D, Vector3D}, + components::{Color, InstanceKey, Material, MeshProperties, Position2D, Position3D, Vector3D}, Archetype, ComponentNameSet, }; use re_viewer_context::{ @@ -81,6 +81,17 @@ impl Mesh3DVisualizer { } else { None }, + vertex_texcoords: if arch_view.has_component::() { + re_tracing::profile_scope!("vertex_texcoords"); + Some( + arch_view + .iter_optional_component::()? + .map(|comp| comp.unwrap_or(Position2D::ZERO)) + .collect(), + ) + } else { + None + }, mesh_properties: arch_view.raw_optional_mono_component::()?, mesh_material: arch_view.raw_optional_mono_component::()?, class_ids: None, @@ -99,7 +110,10 @@ impl Mesh3DVisualizer { versioned_instance_path_hash: picking_instance_hash.versioned(primary_row_id), media_type: None, }, - AnyMesh::Mesh(&mesh), + AnyMesh::Mesh { + mesh: &mesh, + texture_key: re_log_types::hash::Hash64::hash(primary_row_id).hash64(), + }, ctx.render_ctx, ) }); diff --git a/crates/re_viewer_context/src/gpu_bridge/mod.rs b/crates/re_viewer_context/src/gpu_bridge/mod.rs index 2c14f20c19b5..55adf3a1d9be 100644 --- a/crates/re_viewer_context/src/gpu_bridge/mod.rs +++ b/crates/re_viewer_context/src/gpu_bridge/mod.rs @@ -8,6 +8,7 @@ pub use colormap::colormap_dropdown_button_ui; pub use re_renderer_callback::new_renderer_callback; pub use tensor_to_gpu::{ class_id_tensor_to_gpu, color_tensor_to_gpu, depth_tensor_to_gpu, tensor_to_gpu, + texture_height_width_channels, }; use crate::TensorStats; diff --git a/crates/re_viewer_context/src/gpu_bridge/tensor_to_gpu.rs b/crates/re_viewer_context/src/gpu_bridge/tensor_to_gpu.rs index 92b1a0168a95..5c34bcdb9c5b 100644 --- a/crates/re_viewer_context/src/gpu_bridge/tensor_to_gpu.rs +++ b/crates/re_viewer_context/src/gpu_bridge/tensor_to_gpu.rs @@ -533,11 +533,11 @@ fn pad_and_narrow_and_cast( // ----------------------------------------------------------------------------; -fn texture_height_width_channels(tensor: &TensorData) -> anyhow::Result<[u32; 3]> { +pub fn texture_height_width_channels(tensor: &TensorData) -> anyhow::Result<[u32; 3]> { use anyhow::Context as _; let Some([mut height, width, channel]) = tensor.image_height_width_channels() else { - anyhow::bail!("Tensor is not an image"); + anyhow::bail!("Tensor with shape {:?} is not an image", tensor.shape); }; height = match tensor.buffer { // Correct the texture height for NV12, tensor.image_height_width_channels returns the RGB size for NV12 images. The actual texture size has dimensions (h*3/2, w, 1). From 8c51435394229c84e2e97cc687ad3f0311bb7f92 Mon Sep 17 00:00:00 2001 From: Andreas Reich Date: Tue, 23 Jan 2024 17:47:37 +0100 Subject: [PATCH 06/28] mesh & material python extension update --- .../rerun_sdk/rerun/archetypes/mesh3d_ext.py | 5 ++++ .../rerun_sdk/rerun/datatypes/material_ext.py | 26 ++++++++++++++----- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/rerun_py/rerun_sdk/rerun/archetypes/mesh3d_ext.py b/rerun_py/rerun_sdk/rerun/archetypes/mesh3d_ext.py index dfd0480fc159..ce8c64a77e38 100644 --- a/rerun_py/rerun_sdk/rerun/archetypes/mesh3d_ext.py +++ b/rerun_py/rerun_sdk/rerun/archetypes/mesh3d_ext.py @@ -19,6 +19,7 @@ def __init__( mesh_properties: datatypes.MeshPropertiesLike | None = None, vertex_normals: datatypes.Vec3DArrayLike | None = None, vertex_colors: datatypes.Rgba32ArrayLike | None = None, + vertex_texcoords: datatypes.Rgba32ArrayLike | None = None, mesh_material: datatypes.MaterialLike | None = None, class_ids: datatypes.ClassIdArrayLike | None = None, instance_keys: components.InstanceKeyArrayLike | None = None, @@ -41,6 +42,9 @@ def __init__( vertex_normals: An optional normal for each vertex. If specified, this must have as many elements as `vertex_positions`. + vertex_texcoords: + An optional texture coordinate for each vertex. + If specified, this must have as many elements as `vertex_positions`. vertex_colors: An optional color for each vertex. mesh_material: @@ -63,6 +67,7 @@ def __init__( mesh_properties=mesh_properties, vertex_normals=vertex_normals, vertex_colors=vertex_colors, + vertex_texcoords=vertex_texcoords, mesh_material=mesh_material, class_ids=class_ids, instance_keys=instance_keys, diff --git a/rerun_py/rerun_sdk/rerun/datatypes/material_ext.py b/rerun_py/rerun_sdk/rerun/datatypes/material_ext.py index 1771142c3c84..47afad189593 100644 --- a/rerun_py/rerun_sdk/rerun/datatypes/material_ext.py +++ b/rerun_py/rerun_sdk/rerun/datatypes/material_ext.py @@ -1,9 +1,12 @@ from __future__ import annotations -from typing import TYPE_CHECKING +import collections +from typing import TYPE_CHECKING, cast import pyarrow as pa +from rerun.datatypes.tensor_data_ext import TensorDataExt + if TYPE_CHECKING: from . import MaterialArrayLike @@ -15,17 +18,28 @@ class MaterialExt: def native_to_pa_array_override(data: MaterialArrayLike, data_type: pa.DataType) -> pa.Array: from . import Material, Rgba32Type - if isinstance(data, Material): - data = [data] + # If it's a sequence of a single Material, grab the first one + if isinstance(data, collections.abc.Sequence): + if len(data) > 0: + if isinstance(data[0], Material): + if len(data) > 1: + raise ValueError("Materials do not support batches") + data = data[0] + data = cast(Material, data) field_albedo_factors = data_type.field("albedo_factor") + field_albedo_texture = data_type.field("albedo_texture") albedo_factors = pa.array( - [datum.albedo_factor.rgba if datum.albedo_factor is not None else None for datum in data], + [data.albedo_factor.rgba if data.albedo_factor is not None else None], type=Rgba32Type().storage_type, ) + if data.albedo_texture is not None: + albedo_texture = TensorDataExt.native_to_pa_array_override(data.albedo_texture, field_albedo_texture.type) + else: + albedo_texture = pa.array([None], type=field_albedo_texture.type) return pa.StructArray.from_arrays( - arrays=[albedo_factors], - fields=[field_albedo_factors], + arrays=[albedo_factors, albedo_texture], + fields=[field_albedo_factors, field_albedo_texture], ) From 6e31d155c14306aa716407375d36c6065d3a373f Mon Sep 17 00:00:00 2001 From: Andreas Reich Date: Tue, 23 Jan 2024 17:47:57 +0100 Subject: [PATCH 07/28] python raw mesh example now uses new albedo texture support --- examples/python/raw_mesh/main.py | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/examples/python/raw_mesh/main.py b/examples/python/raw_mesh/main.py index cb18d38f8dfa..5c13d59573bb 100755 --- a/examples/python/raw_mesh/main.py +++ b/examples/python/raw_mesh/main.py @@ -53,17 +53,27 @@ def log_scene(scene: trimesh.Scene, node: str, path: str | None = None) -> None: mesh = cast(trimesh.Trimesh, scene.geometry.get(node_data[1])) if mesh: vertex_colors = None + vertex_texcoords = None mesh_material = None + try: - colors = mesh.visual.to_color().vertex_colors - if len(colors) == 4: - # If trimesh gives us a single vertex color for the entire mesh, we can interpret that - # as an albedo factor for the whole primitive. - mesh_material = Material(albedo_factor=np.array(colors)) - else: - vertex_colors = colors + mesh_material = Material(albedo_texture=mesh.visual.material.baseColorTexture) + vertex_texcoords = mesh.visual.uv + # trimesh uses the OpenGL convention for UV coordinates, so we need to flip the V coordinate + # since Rerun uses the Vulkan/Metal/DX12/WebGPU convention. + vertex_texcoords[:, 1] = 1.0 - vertex_texcoords[:, 1] except Exception: - pass + # Didn't have UV coordinates, try with vertex colors instead. + try: + colors = mesh.visual.to_color().vertex_colors + if len(colors) == 4: + # If trimesh gives us a single vertex color for the entire mesh, we can interpret that + # as an albedo factor for the whole primitive. + mesh_material = Material(albedo_factor=np.array(colors)) + else: + vertex_colors = colors + except Exception: + pass rr.log( path, @@ -71,6 +81,7 @@ def log_scene(scene: trimesh.Scene, node: str, path: str | None = None) -> None: vertex_positions=mesh.vertices, vertex_colors=vertex_colors, vertex_normals=mesh.vertex_normals, + vertex_texcoords=vertex_texcoords, indices=mesh.faces, mesh_material=mesh_material, ), From 9aa916b4c48f55505a0e4955abdfab42cdebc6ef Mon Sep 17 00:00:00 2001 From: Andreas Reich Date: Tue, 23 Jan 2024 17:50:07 +0100 Subject: [PATCH 08/28] update rust raw_mesh example --- examples/rust/raw_mesh/src/main.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/examples/rust/raw_mesh/src/main.rs b/examples/rust/raw_mesh/src/main.rs index 8e8fd0494fde..70cb0e6843bc 100644 --- a/examples/rust/raw_mesh/src/main.rs +++ b/examples/rust/raw_mesh/src/main.rs @@ -31,7 +31,7 @@ impl From for Mesh3D { vertex_positions, vertex_colors, vertex_normals, - vertex_texcoords: _, // TODO(cmc): support mesh texturing + vertex_texcoords, } = primitive; let mut mesh = Mesh3D::new(vertex_positions); @@ -48,10 +48,14 @@ impl From for Mesh3D { if let Some(vertex_colors) = vertex_colors { mesh = mesh.with_vertex_colors(vertex_colors); } + if let Some(vertex_texcoords) = vertex_texcoords { + mesh = mesh.with_vertex_texcoords(vertex_texcoords); + } if albedo_factor.is_some() { mesh = mesh.with_mesh_material(rerun::datatypes::Material { albedo_factor: albedo_factor .map(|[r, g, b, a]| ecolor::Rgba::from_rgba_unmultiplied(r, g, b, a).into()), + albedo_texture: None, // TODO(andreas): This would require loading the right texture file and converting it to `TensorData`. }); } From a5fac6a9bc39a3472006af996d657e14289c8193 Mon Sep 17 00:00:00 2001 From: Andreas Reich Date: Tue, 23 Jan 2024 18:10:45 +0100 Subject: [PATCH 09/28] introduce new texcoord2d component --- clippy.toml | 1 + .../src/visualizers/meshes.rs | 8 +- .../definitions/rerun/archetypes/mesh3d.fbs | 2 +- .../re_types/definitions/rerun/components.fbs | 1 + .../rerun/components/texcoord2d.fbs | 37 ++ crates/re_types/src/archetypes/mesh3d.rs | 14 +- crates/re_types/src/components/.gitattributes | 1 + crates/re_types/src/components/mod.rs | 3 + crates/re_types/src/components/texcoord2d.rs | 317 ++++++++++++++++++ .../re_types/src/components/texcoord2d_ext.rs | 52 +++ crates/re_types/tests/mesh3d.rs | 6 +- .../reference/types/archetypes/mesh3d.md | 2 +- docs/content/reference/types/components.md | 1 + .../reference/types/components/.gitattributes | 1 + .../reference/types/components/position2d.md | 1 - .../reference/types/components/texcoord2d.md | 35 ++ .../reference/types/datatypes/vec2d.md | 1 + rerun_cpp/src/rerun/archetypes/mesh3d.hpp | 6 +- rerun_cpp/src/rerun/components.hpp | 1 + rerun_cpp/src/rerun/components/.gitattributes | 2 + rerun_cpp/src/rerun/components/texcoord2d.cpp | 52 +++ rerun_cpp/src/rerun/components/texcoord2d.hpp | 101 ++++++ .../src/rerun/components/texcoord2d_ext.cpp | 27 ++ rerun_py/rerun_sdk/rerun/archetypes/mesh3d.py | 4 +- .../rerun_sdk/rerun/components/.gitattributes | 1 + .../rerun_sdk/rerun/components/__init__.py | 4 + .../rerun_sdk/rerun/components/texcoord2d.py | 46 +++ scripts/clippy_wasm/clippy.toml | 3 +- 28 files changed, 707 insertions(+), 23 deletions(-) create mode 100644 crates/re_types/definitions/rerun/components/texcoord2d.fbs create mode 100644 crates/re_types/src/components/texcoord2d.rs create mode 100644 crates/re_types/src/components/texcoord2d_ext.rs create mode 100644 docs/content/reference/types/components/texcoord2d.md create mode 100644 rerun_cpp/src/rerun/components/texcoord2d.cpp create mode 100644 rerun_cpp/src/rerun/components/texcoord2d.hpp create mode 100644 rerun_cpp/src/rerun/components/texcoord2d_ext.cpp create mode 100644 rerun_py/rerun_sdk/rerun/components/texcoord2d.py diff --git a/clippy.toml b/clippy.toml index 6e08e8c4407c..09221aff8376 100644 --- a/clippy.toml +++ b/clippy.toml @@ -75,6 +75,7 @@ doc-valid-idents = [ "macOS", "NaN", "OBJ", + "OpenGL", "PyPI", "sRGB", "sRGBA", diff --git a/crates/re_space_view_spatial/src/visualizers/meshes.rs b/crates/re_space_view_spatial/src/visualizers/meshes.rs index 70b24d1a3844..32c952b174fc 100644 --- a/crates/re_space_view_spatial/src/visualizers/meshes.rs +++ b/crates/re_space_view_spatial/src/visualizers/meshes.rs @@ -3,7 +3,7 @@ use re_query::{ArchetypeView, QueryError}; use re_renderer::renderer::MeshInstance; use re_types::{ archetypes::Mesh3D, - components::{Color, InstanceKey, Material, MeshProperties, Position2D, Position3D, Vector3D}, + components::{Color, InstanceKey, Material, MeshProperties, Position3D, Texcoord2D, Vector3D}, Archetype, ComponentNameSet, }; use re_viewer_context::{ @@ -81,12 +81,12 @@ impl Mesh3DVisualizer { } else { None }, - vertex_texcoords: if arch_view.has_component::() { + vertex_texcoords: if arch_view.has_component::() { re_tracing::profile_scope!("vertex_texcoords"); Some( arch_view - .iter_optional_component::()? - .map(|comp| comp.unwrap_or(Position2D::ZERO)) + .iter_optional_component::()? + .map(|comp| comp.unwrap_or(Texcoord2D::ZERO)) .collect(), ) } else { diff --git a/crates/re_types/definitions/rerun/archetypes/mesh3d.fbs b/crates/re_types/definitions/rerun/archetypes/mesh3d.fbs index 8e7470999a98..332cf1763872 100644 --- a/crates/re_types/definitions/rerun/archetypes/mesh3d.fbs +++ b/crates/re_types/definitions/rerun/archetypes/mesh3d.fbs @@ -42,7 +42,7 @@ table Mesh3D ( vertex_colors: [rerun.components.Color] ("attr.rerun.component_optional", nullable, order: 3100); /// An optional uv texture coordinate for each vertex. - vertex_texcoords: [rerun.components.Position2D] ("attr.rerun.component_optional", nullable, order: 3200); + vertex_texcoords: [rerun.components.Texcoord2D] ("attr.rerun.component_optional", nullable, order: 3200); /// Optional material properties for the mesh as a whole. mesh_material: rerun.components.Material ("attr.rerun.component_optional", nullable, order: 3300); diff --git a/crates/re_types/definitions/rerun/components.fbs b/crates/re_types/definitions/rerun/components.fbs index 3d6c9bd21bbb..7b113de9725a 100644 --- a/crates/re_types/definitions/rerun/components.fbs +++ b/crates/re_types/definitions/rerun/components.fbs @@ -25,6 +25,7 @@ include "./components/rotation3d.fbs"; include "./components/scalar_scattering.fbs"; include "./components/scalar.fbs"; include "./components/tensor_data.fbs"; +include "./components/texcoord2d.fbs"; include "./components/text_log_level.fbs"; include "./components/text.fbs"; include "./components/transform3d.fbs"; diff --git a/crates/re_types/definitions/rerun/components/texcoord2d.fbs b/crates/re_types/definitions/rerun/components/texcoord2d.fbs new file mode 100644 index 000000000000..1080af59fbbd --- /dev/null +++ b/crates/re_types/definitions/rerun/components/texcoord2d.fbs @@ -0,0 +1,37 @@ +include "arrow/attributes.fbs"; +include "python/attributes.fbs"; +include "rust/attributes.fbs"; + +include "rerun/attributes.fbs"; +include "rerun/datatypes.fbs"; + +namespace rerun.components; + +// --- + +/// A 2D texture UV coordinate. +/// +/// Texture coordinates specify a position on a 2D texture. +/// A range from 0-1 in covers the entire texture in the respective dimension. +/// The behavior for values outside of this range depends on the visualization/renderer, +/// but will most commonly repeat the texture. +/// Rerun uses top-left as the origin for UV coordinates. +/// +/// 0 U 1 +/// 0 + --------- → +/// | . +/// | . +/// V | . +/// | . +/// 1 ↓ . . . . . . +/// +/// This is the same convention as in Vulkan/Metal/DX12/WebGPU, but (!) unlike OpenGL, +/// which places the origin at the bottom-left. +struct Texcoord2D ( + "attr.python.aliases": "npt.NDArray[np.float32], Sequence[float], Tuple[float, float]", + "attr.python.array_aliases": "npt.NDArray[np.float32], Sequence[float]", + "attr.rust.derive": "Copy, PartialEq, bytemuck::Pod, bytemuck::Zeroable", + "attr.rust.repr": "transparent" +) { + uv: rerun.datatypes.Vec2D (order: 100); +} diff --git a/crates/re_types/src/archetypes/mesh3d.rs b/crates/re_types/src/archetypes/mesh3d.rs index ebe9c97fa162..da34bf0a374d 100644 --- a/crates/re_types/src/archetypes/mesh3d.rs +++ b/crates/re_types/src/archetypes/mesh3d.rs @@ -72,7 +72,7 @@ pub struct Mesh3D { pub vertex_colors: Option>, /// An optional uv texture coordinate for each vertex. - pub vertex_texcoords: Option>, + pub vertex_texcoords: Option>, /// Optional material properties for the mesh as a whole. pub mesh_material: Option, @@ -105,7 +105,7 @@ impl ::re_types_core::SizeBytes for Mesh3D { && >::is_pod() && >>::is_pod() && >>::is_pod() - && >>::is_pod() + && >>::is_pod() && >::is_pod() && >>::is_pod() && >>::is_pod() @@ -131,7 +131,7 @@ static OPTIONAL_COMPONENTS: once_cell::sync::Lazy<[ComponentName; 5usize]> = "rerun.components.Color".into(), "rerun.components.InstanceKey".into(), "rerun.components.Material".into(), - "rerun.components.Position2D".into(), + "rerun.components.Texcoord2D".into(), ] }); @@ -146,7 +146,7 @@ static ALL_COMPONENTS: once_cell::sync::Lazy<[ComponentName; 9usize]> = "rerun.components.Color".into(), "rerun.components.InstanceKey".into(), "rerun.components.Material".into(), - "rerun.components.Position2D".into(), + "rerun.components.Texcoord2D".into(), ] }); @@ -248,9 +248,9 @@ impl ::re_types_core::Archetype for Mesh3D { None }; let vertex_texcoords = - if let Some(array) = arrays_by_name.get("rerun.components.Position2D") { + if let Some(array) = arrays_by_name.get("rerun.components.Texcoord2D") { Some({ - ::from_arrow_opt(&**array) + ::from_arrow_opt(&**array) .with_context("rerun.archetypes.Mesh3D#vertex_texcoords")? .into_iter() .map(|v| v.ok_or_else(DeserializationError::missing_data)) @@ -393,7 +393,7 @@ impl Mesh3D { #[inline] pub fn with_vertex_texcoords( mut self, - vertex_texcoords: impl IntoIterator>, + vertex_texcoords: impl IntoIterator>, ) -> Self { self.vertex_texcoords = Some(vertex_texcoords.into_iter().map(Into::into).collect()); self diff --git a/crates/re_types/src/components/.gitattributes b/crates/re_types/src/components/.gitattributes index 501c9e3b69b4..03fe0fe129f6 100644 --- a/crates/re_types/src/components/.gitattributes +++ b/crates/re_types/src/components/.gitattributes @@ -27,6 +27,7 @@ rotation3d.rs linguist-generated=true scalar.rs linguist-generated=true scalar_scattering.rs linguist-generated=true tensor_data.rs linguist-generated=true +texcoord2d.rs linguist-generated=true text.rs linguist-generated=true text_log_level.rs linguist-generated=true transform3d.rs linguist-generated=true diff --git a/crates/re_types/src/components/mod.rs b/crates/re_types/src/components/mod.rs index 6a3917e36be3..3ca7004b8a4a 100644 --- a/crates/re_types/src/components/mod.rs +++ b/crates/re_types/src/components/mod.rs @@ -45,6 +45,8 @@ mod scalar; mod scalar_ext; mod scalar_scattering; mod tensor_data; +mod texcoord2d; +mod texcoord2d_ext; mod text; mod text_ext; mod text_log_level; @@ -83,6 +85,7 @@ pub use self::rotation3d::Rotation3D; pub use self::scalar::Scalar; pub use self::scalar_scattering::ScalarScattering; pub use self::tensor_data::TensorData; +pub use self::texcoord2d::Texcoord2D; pub use self::text::Text; pub use self::text_log_level::TextLogLevel; pub use self::transform3d::Transform3D; diff --git a/crates/re_types/src/components/texcoord2d.rs b/crates/re_types/src/components/texcoord2d.rs new file mode 100644 index 000000000000..9d07c200488d --- /dev/null +++ b/crates/re_types/src/components/texcoord2d.rs @@ -0,0 +1,317 @@ +// DO NOT EDIT! This file was auto-generated by crates/re_types_builder/src/codegen/rust/api.rs +// Based on "crates/re_types/definitions/rerun/components/texcoord2d.fbs". + +#![allow(trivial_numeric_casts)] +#![allow(unused_imports)] +#![allow(unused_parens)] +#![allow(clippy::clone_on_copy)] +#![allow(clippy::iter_on_single_items)] +#![allow(clippy::map_flatten)] +#![allow(clippy::match_wildcard_for_single_variants)] +#![allow(clippy::needless_question_mark)] +#![allow(clippy::new_without_default)] +#![allow(clippy::redundant_closure)] +#![allow(clippy::too_many_arguments)] +#![allow(clippy::too_many_lines)] +#![allow(clippy::unnecessary_cast)] + +use ::re_types_core::external::arrow2; +use ::re_types_core::ComponentName; +use ::re_types_core::SerializationResult; +use ::re_types_core::{ComponentBatch, MaybeOwnedComponentBatch}; +use ::re_types_core::{DeserializationError, DeserializationResult}; + +/// **Component**: A 2D texture UV coordinate. +/// +/// Texture coordinates specify a position on a 2D texture. +/// A range from 0-1 in covers the entire texture in the respective dimension. +/// The behavior for values outside of this range depends on the visualization/renderer, +/// but will most commonly repeat the texture. +/// Rerun uses top-left as the origin for UV coordinates. +/// +/// 0 U 1 +/// 0 + --------- → +/// | . +/// V | . +/// | . +/// 1 ↓ . . . . . . +/// +/// This is the same convention as in Vulkan/Metal/DX12/WebGPU, but (!) unlike OpenGL, +/// which places the origin at the bottom-left. +#[derive(Clone, Debug, Copy, PartialEq, bytemuck::Pod, bytemuck::Zeroable)] +#[repr(transparent)] +pub struct Texcoord2D(pub crate::datatypes::Vec2D); + +impl ::re_types_core::SizeBytes for Texcoord2D { + #[inline] + fn heap_size_bytes(&self) -> u64 { + self.0.heap_size_bytes() + } + + #[inline] + fn is_pod() -> bool { + ::is_pod() + } +} + +impl> From for Texcoord2D { + fn from(v: T) -> Self { + Self(v.into()) + } +} + +impl std::borrow::Borrow for Texcoord2D { + #[inline] + fn borrow(&self) -> &crate::datatypes::Vec2D { + &self.0 + } +} + +impl std::ops::Deref for Texcoord2D { + type Target = crate::datatypes::Vec2D; + + #[inline] + fn deref(&self) -> &crate::datatypes::Vec2D { + &self.0 + } +} + +::re_types_core::macros::impl_into_cow!(Texcoord2D); + +impl ::re_types_core::Loggable for Texcoord2D { + type Name = ::re_types_core::ComponentName; + + #[inline] + fn name() -> Self::Name { + "rerun.components.Texcoord2D".into() + } + + #[allow(clippy::wildcard_imports)] + #[inline] + fn arrow_datatype() -> arrow2::datatypes::DataType { + use arrow2::datatypes::*; + DataType::FixedSizeList( + Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Float32, + is_nullable: false, + metadata: [].into(), + }), + 2usize, + ) + } + + #[allow(clippy::wildcard_imports)] + fn to_arrow_opt<'a>( + data: impl IntoIterator>>>, + ) -> SerializationResult> + where + Self: Clone + 'a, + { + use ::re_types_core::{Loggable as _, ResultExt as _}; + use arrow2::{array::*, datatypes::*}; + Ok({ + let (somes, data0): (Vec<_>, Vec<_>) = data + .into_iter() + .map(|datum| { + let datum: Option<::std::borrow::Cow<'a, Self>> = datum.map(Into::into); + let datum = datum.map(|datum| { + let Self(data0) = datum.into_owned(); + data0 + }); + (datum.is_some(), datum) + }) + .unzip(); + let data0_bitmap: Option = { + let any_nones = somes.iter().any(|some| !*some); + any_nones.then(|| somes.into()) + }; + { + use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; + let data0_inner_data: Vec<_> = data0 + .iter() + .map(|datum| { + datum + .map(|datum| { + let crate::datatypes::Vec2D(data0) = datum; + data0 + }) + .unwrap_or_default() + }) + .flatten() + .map(Some) + .collect(); + let data0_inner_bitmap: Option = + data0_bitmap.as_ref().map(|bitmap| { + bitmap + .iter() + .map(|i| std::iter::repeat(i).take(2usize)) + .flatten() + .collect::>() + .into() + }); + FixedSizeListArray::new( + Self::arrow_datatype(), + PrimitiveArray::new( + DataType::Float32, + data0_inner_data + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect(), + data0_inner_bitmap, + ) + .boxed(), + data0_bitmap, + ) + .boxed() + } + }) + } + + #[allow(clippy::wildcard_imports)] + fn from_arrow_opt( + arrow_data: &dyn arrow2::array::Array, + ) -> DeserializationResult>> + where + Self: Sized, + { + use ::re_types_core::{Loggable as _, ResultExt as _}; + use arrow2::{array::*, buffer::*, datatypes::*}; + Ok({ + let arrow_data = arrow_data + .as_any() + .downcast_ref::() + .ok_or_else(|| { + DeserializationError::datatype_mismatch( + DataType::FixedSizeList( + Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Float32, + is_nullable: false, + metadata: [].into(), + }), + 2usize, + ), + arrow_data.data_type().clone(), + ) + }) + .with_context("rerun.components.Texcoord2D#uv")?; + if arrow_data.is_empty() { + Vec::new() + } else { + let offsets = (0..) + .step_by(2usize) + .zip((2usize..).step_by(2usize).take(arrow_data.len())); + let arrow_data_inner = { + let arrow_data_inner = &**arrow_data.values(); + arrow_data_inner + .as_any() + .downcast_ref::() + .ok_or_else(|| { + DeserializationError::datatype_mismatch( + DataType::Float32, + arrow_data_inner.data_type().clone(), + ) + }) + .with_context("rerun.components.Texcoord2D#uv")? + .into_iter() + .map(|opt| opt.copied()) + .collect::>() + }; + arrow2::bitmap::utils::ZipValidity::new_with_validity( + offsets, + arrow_data.validity(), + ) + .map(|elem| { + elem.map(|(start, end)| { + debug_assert!(end - start == 2usize); + if end as usize > arrow_data_inner.len() { + return Err(DeserializationError::offset_slice_oob( + (start, end), + arrow_data_inner.len(), + )); + } + + #[allow(unsafe_code, clippy::undocumented_unsafe_blocks)] + let data = + unsafe { arrow_data_inner.get_unchecked(start as usize..end as usize) }; + let data = data.iter().cloned().map(Option::unwrap_or_default); + let arr = array_init::from_iter(data).unwrap(); + Ok(arr) + }) + .transpose() + }) + .map(|res_or_opt| { + res_or_opt.map(|res_or_opt| res_or_opt.map(|v| crate::datatypes::Vec2D(v))) + }) + .collect::>>>()? + } + .into_iter() + } + .map(|v| v.ok_or_else(DeserializationError::missing_data)) + .map(|res| res.map(|v| Some(Self(v)))) + .collect::>>>() + .with_context("rerun.components.Texcoord2D#uv") + .with_context("rerun.components.Texcoord2D")?) + } + + #[allow(clippy::wildcard_imports)] + #[inline] + fn from_arrow(arrow_data: &dyn arrow2::array::Array) -> DeserializationResult> + where + Self: Sized, + { + use ::re_types_core::{Loggable as _, ResultExt as _}; + use arrow2::{array::*, buffer::*, datatypes::*}; + if let Some(validity) = arrow_data.validity() { + if validity.unset_bits() != 0 { + return Err(DeserializationError::missing_data()); + } + } + Ok({ + let slice = { + let arrow_data = arrow_data + .as_any() + .downcast_ref::() + .ok_or_else(|| { + DeserializationError::datatype_mismatch( + DataType::FixedSizeList( + Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Float32, + is_nullable: false, + metadata: [].into(), + }), + 2usize, + ), + arrow_data.data_type().clone(), + ) + }) + .with_context("rerun.components.Texcoord2D#uv")?; + let arrow_data_inner = &**arrow_data.values(); + bytemuck::cast_slice::<_, [_; 2usize]>( + arrow_data_inner + .as_any() + .downcast_ref::() + .ok_or_else(|| { + DeserializationError::datatype_mismatch( + DataType::Float32, + arrow_data_inner.data_type().clone(), + ) + }) + .with_context("rerun.components.Texcoord2D#uv")? + .values() + .as_slice(), + ) + }; + { + slice + .iter() + .copied() + .map(|v| crate::datatypes::Vec2D(v)) + .map(|v| Self(v)) + .collect::>() + } + }) + } +} diff --git a/crates/re_types/src/components/texcoord2d_ext.rs b/crates/re_types/src/components/texcoord2d_ext.rs new file mode 100644 index 000000000000..653cb76762d5 --- /dev/null +++ b/crates/re_types/src/components/texcoord2d_ext.rs @@ -0,0 +1,52 @@ +use crate::datatypes::Vec2D; + +use super::Texcoord2D; + +// --- + +impl Texcoord2D { + pub const ZERO: Self = Self::new(0.0, 0.0); + pub const ONE: Self = Self::new(1.0, 1.0); + + #[inline] + pub const fn new(u: f32, v: f32) -> Self { + Self(Vec2D::new(u, v)) + } + + #[inline] + pub fn u(&self) -> f32 { + self.0.x() + } + + #[inline] + pub fn v(&self) -> f32 { + self.0.y() + } +} + +#[cfg(feature = "glam")] +impl From for glam::Vec2 { + #[inline] + fn from(pt: Texcoord2D) -> Self { + Self::new(pt.u(), pt.v()) + } +} + +#[cfg(feature = "mint")] +impl From for mint::Point2 { + #[inline] + fn from(position: Texcoord2D) -> Self { + Self { + x: position.u(), + y: position.v(), + } + } +} + +#[cfg(feature = "mint")] +impl From> for Texcoord2D { + #[inline] + fn from(position: mint::Point2) -> Self { + Self(Vec2D([position.x, position.y])) + } +} diff --git a/crates/re_types/tests/mesh3d.rs b/crates/re_types/tests/mesh3d.rs index cc1837dcd61c..793cc3c4210b 100644 --- a/crates/re_types/tests/mesh3d.rs +++ b/crates/re_types/tests/mesh3d.rs @@ -2,7 +2,7 @@ use std::collections::HashMap; use re_types::{ archetypes::Mesh3D, - components::{ClassId, InstanceKey, Position2D, Position3D, Vector3D}, + components::{ClassId, InstanceKey, Position3D, Texcoord2D, Vector3D}, datatypes::{Material, MeshProperties, Rgba32, Vec2D, Vec3D}, Archetype as _, AsComponents as _, }; @@ -29,8 +29,8 @@ fn roundtrip() { Rgba32::from_unmultiplied_rgba(0x00, 0xBB, 0x00, 0xDD).into(), ]), vertex_texcoords: Some(vec![ - Position2D(Vec2D([0.0, 1.0])), // - Position2D(Vec2D([2.0, 3.0])), // + Texcoord2D(Vec2D([0.0, 1.0])), // + Texcoord2D(Vec2D([2.0, 3.0])), // ]), mesh_material: Some( Material { diff --git a/docs/content/reference/types/archetypes/mesh3d.md b/docs/content/reference/types/archetypes/mesh3d.md index 35980211f18d..1ee5bc5f1473 100644 --- a/docs/content/reference/types/archetypes/mesh3d.md +++ b/docs/content/reference/types/archetypes/mesh3d.md @@ -10,7 +10,7 @@ A 3D triangle mesh as specified by its per-mesh and per-vertex properties. **Recommended**: [`MeshProperties`](../components/mesh_properties.md), [`Vector3D`](../components/vector3d.md) -**Optional**: [`Color`](../components/color.md), [`Position2D`](../components/position2d.md), [`Material`](../components/material.md), [`ClassId`](../components/class_id.md), [`InstanceKey`](../components/instance_key.md) +**Optional**: [`Color`](../components/color.md), [`Texcoord2D`](../components/texcoord2d.md), [`Material`](../components/material.md), [`ClassId`](../components/class_id.md), [`InstanceKey`](../components/instance_key.md) ## Links * 🌊 [C++ API docs for `Mesh3D`](https://ref.rerun.io/docs/cpp/stable/structrerun_1_1archetypes_1_1Mesh3D.html) diff --git a/docs/content/reference/types/components.md b/docs/content/reference/types/components.md index a2a5241d0321..7328f1e8b454 100644 --- a/docs/content/reference/types/components.md +++ b/docs/content/reference/types/components.md @@ -34,6 +34,7 @@ Archetypes are bundles of components * [`Scalar`](components/scalar.md) * [`ScalarScattering`](components/scalar_scattering.md) * [`TensorData`](components/tensor_data.md) +* [`Texcoord2D`](components/texcoord2d.md) * [`Text`](components/text.md) * [`TextLogLevel`](components/text_log_level.md) * [`Transform3D`](components/transform3d.md) diff --git a/docs/content/reference/types/components/.gitattributes b/docs/content/reference/types/components/.gitattributes index 9b6a72842029..cc952f0754e8 100644 --- a/docs/content/reference/types/components/.gitattributes +++ b/docs/content/reference/types/components/.gitattributes @@ -28,6 +28,7 @@ rotation3d.md linguist-generated=true scalar.md linguist-generated=true scalar_scattering.md linguist-generated=true tensor_data.md linguist-generated=true +texcoord2d.md linguist-generated=true text.md linguist-generated=true text_log_level.md linguist-generated=true transform3d.md linguist-generated=true diff --git a/docs/content/reference/types/components/position2d.md b/docs/content/reference/types/components/position2d.md index f513a60c84d9..875baccd1458 100644 --- a/docs/content/reference/types/components/position2d.md +++ b/docs/content/reference/types/components/position2d.md @@ -18,5 +18,4 @@ A position in 2D space. * [`Arrows2D`](../archetypes/arrows2d.md?speculative-link) * [`Boxes2D`](../archetypes/boxes2d.md) -* [`Mesh3D`](../archetypes/mesh3d.md) * [`Points2D`](../archetypes/points2d.md) diff --git a/docs/content/reference/types/components/texcoord2d.md b/docs/content/reference/types/components/texcoord2d.md new file mode 100644 index 000000000000..c3c8dd3c9c99 --- /dev/null +++ b/docs/content/reference/types/components/texcoord2d.md @@ -0,0 +1,35 @@ +--- +title: "Texcoord2D" +--- + +A 2D texture UV coordinate. + +Texture coordinates specify a position on a 2D texture. +A range from 0-1 in covers the entire texture in the respective dimension. +The behavior for values outside of this range depends on the visualization/renderer, +but will most commonly repeat the texture. +Rerun uses top-left as the origin for UV coordinates. + + 0 U 1 +0 + --------- → + | . +V | . + | . +1 ↓ . . . . . . + +This is the same convention as in Vulkan/Metal/DX12/WebGPU, but (!) unlike OpenGL, +which places the origin at the bottom-left. + +## Fields + +* uv: [`Vec2D`](../datatypes/vec2d.md) + +## Links + * 🌊 [C++ API docs for `Texcoord2D`](https://ref.rerun.io/docs/cpp/stable/structrerun_1_1components_1_1Texcoord2D.html) + * 🐍 [Python API docs for `Texcoord2D`](https://ref.rerun.io/docs/python/stable/common/components#rerun.components.Texcoord2D) + * 🦀 [Rust API docs for `Texcoord2D`](https://docs.rs/rerun/latest/rerun/components/struct.Texcoord2D.html) + + +## Used by + +* [`Mesh3D`](../archetypes/mesh3d.md) diff --git a/docs/content/reference/types/datatypes/vec2d.md b/docs/content/reference/types/datatypes/vec2d.md index 54e549e1f44c..149932c06c1d 100644 --- a/docs/content/reference/types/datatypes/vec2d.md +++ b/docs/content/reference/types/datatypes/vec2d.md @@ -17,4 +17,5 @@ A vector in 2D space. * [`LineStrip2D`](../components/line_strip2d.md) * [`Position2D`](../components/position2d.md) * [`Resolution`](../components/resolution.md) +* [`Texcoord2D`](../components/texcoord2d.md) * [`Vector2D`](../components/vector2d.md?speculative-link) diff --git a/rerun_cpp/src/rerun/archetypes/mesh3d.hpp b/rerun_cpp/src/rerun/archetypes/mesh3d.hpp index 42c1cda4b9cb..d7b147a80715 100644 --- a/rerun_cpp/src/rerun/archetypes/mesh3d.hpp +++ b/rerun_cpp/src/rerun/archetypes/mesh3d.hpp @@ -10,8 +10,8 @@ #include "../components/instance_key.hpp" #include "../components/material.hpp" #include "../components/mesh_properties.hpp" -#include "../components/position2d.hpp" #include "../components/position3d.hpp" +#include "../components/texcoord2d.hpp" #include "../components/vector3d.hpp" #include "../data_cell.hpp" #include "../indicator_component.hpp" @@ -79,7 +79,7 @@ namespace rerun::archetypes { std::optional> vertex_colors; /// An optional uv texture coordinate for each vertex. - std::optional> vertex_texcoords; + std::optional> vertex_texcoords; /// Optional material properties for the mesh as a whole. std::optional mesh_material; @@ -129,7 +129,7 @@ namespace rerun::archetypes { } /// An optional uv texture coordinate for each vertex. - Mesh3D with_vertex_texcoords(Collection _vertex_texcoords + Mesh3D with_vertex_texcoords(Collection _vertex_texcoords ) && { vertex_texcoords = std::move(_vertex_texcoords); // See: https://github.com/rerun-io/rerun/issues/4027 diff --git a/rerun_cpp/src/rerun/components.hpp b/rerun_cpp/src/rerun/components.hpp index 64467627d132..46d5dd7d0229 100644 --- a/rerun_cpp/src/rerun/components.hpp +++ b/rerun_cpp/src/rerun/components.hpp @@ -29,6 +29,7 @@ #include "components/scalar.hpp" #include "components/scalar_scattering.hpp" #include "components/tensor_data.hpp" +#include "components/texcoord2d.hpp" #include "components/text.hpp" #include "components/text_log_level.hpp" #include "components/transform3d.hpp" diff --git a/rerun_cpp/src/rerun/components/.gitattributes b/rerun_cpp/src/rerun/components/.gitattributes index e6c7f1cd25f3..05d27bfe2e0d 100644 --- a/rerun_cpp/src/rerun/components/.gitattributes +++ b/rerun_cpp/src/rerun/components/.gitattributes @@ -55,6 +55,8 @@ scalar_scattering.cpp linguist-generated=true scalar_scattering.hpp linguist-generated=true tensor_data.cpp linguist-generated=true tensor_data.hpp linguist-generated=true +texcoord2d.cpp linguist-generated=true +texcoord2d.hpp linguist-generated=true text.cpp linguist-generated=true text.hpp linguist-generated=true text_log_level.cpp linguist-generated=true diff --git a/rerun_cpp/src/rerun/components/texcoord2d.cpp b/rerun_cpp/src/rerun/components/texcoord2d.cpp new file mode 100644 index 000000000000..b462646d8168 --- /dev/null +++ b/rerun_cpp/src/rerun/components/texcoord2d.cpp @@ -0,0 +1,52 @@ +// DO NOT EDIT! This file was auto-generated by crates/re_types_builder/src/codegen/cpp/mod.rs +// Based on "crates/re_types/definitions/rerun/components/texcoord2d.fbs". + +#include "texcoord2d.hpp" + +#include "../datatypes/vec2d.hpp" + +#include +#include + +namespace rerun::components {} + +namespace rerun { + const std::shared_ptr& Loggable::arrow_datatype() { + static const auto datatype = Loggable::arrow_datatype(); + return datatype; + } + + rerun::Error Loggable::fill_arrow_array_builder( + arrow::FixedSizeListBuilder* builder, const components::Texcoord2D* elements, + size_t num_elements + ) { + static_assert(sizeof(rerun::datatypes::Vec2D) == sizeof(components::Texcoord2D)); + RR_RETURN_NOT_OK(Loggable::fill_arrow_array_builder( + builder, + reinterpret_cast(elements), + num_elements + )); + + return Error::ok(); + } + + Result> Loggable::to_arrow( + const components::Texcoord2D* instances, size_t num_instances + ) { + // TODO(andreas): Allow configuring the memory pool. + arrow::MemoryPool* pool = arrow::default_memory_pool(); + auto datatype = arrow_datatype(); + + ARROW_ASSIGN_OR_RAISE(auto builder, arrow::MakeBuilder(datatype, pool)) + if (instances && num_instances > 0) { + RR_RETURN_NOT_OK(Loggable::fill_arrow_array_builder( + static_cast(builder.get()), + instances, + num_instances + )); + } + std::shared_ptr array; + ARROW_RETURN_NOT_OK(builder->Finish(&array)); + return array; + } +} // namespace rerun diff --git a/rerun_cpp/src/rerun/components/texcoord2d.hpp b/rerun_cpp/src/rerun/components/texcoord2d.hpp new file mode 100644 index 000000000000..d95281240d5f --- /dev/null +++ b/rerun_cpp/src/rerun/components/texcoord2d.hpp @@ -0,0 +1,101 @@ +// DO NOT EDIT! This file was auto-generated by crates/re_types_builder/src/codegen/cpp/mod.rs +// Based on "crates/re_types/definitions/rerun/components/texcoord2d.fbs". + +#pragma once + +#include "../datatypes/vec2d.hpp" +#include "../result.hpp" + +#include +#include +#include + +namespace arrow { + class Array; + class DataType; + class FixedSizeListBuilder; +} // namespace arrow + +namespace rerun::components { + /// **Component**: A 2D texture UV coordinate. + /// + /// Texture coordinates specify a position on a 2D texture. + /// A range from 0-1 in covers the entire texture in the respective dimension. + /// The behavior for values outside of this range depends on the visualization/renderer, + /// but will most commonly repeat the texture. + /// Rerun uses top-left as the origin for UV coordinates. + /// + /// 0 U 1 + /// 0 + --------- → + /// | . + /// V | . + /// | . + /// 1 ↓ . . . . . . + /// + /// This is the same convention as in Vulkan/Metal/DX12/WebGPU, but (!) unlike OpenGL, + /// which places the origin at the bottom-left. + struct Texcoord2D { + rerun::datatypes::Vec2D uv; + + public: + // Extensions to generated type defined in 'texcoord2d_ext.cpp' + + /// Construct Texcoord2D from u/v values. + Texcoord2D(float x, float y) : uv{u, v} {} + + float u() const { + return uv.x(); + } + + float v() const { + return uv.y(); + } + + public: + Texcoord2D() = default; + + Texcoord2D(rerun::datatypes::Vec2D uv_) : uv(uv_) {} + + Texcoord2D& operator=(rerun::datatypes::Vec2D uv_) { + uv = uv_; + return *this; + } + + Texcoord2D(std::array xy_) : uv(xy_) {} + + Texcoord2D& operator=(std::array xy_) { + uv = xy_; + return *this; + } + + /// Cast to the underlying Vec2D datatype + operator rerun::datatypes::Vec2D() const { + return uv; + } + }; +} // namespace rerun::components + +namespace rerun { + template + struct Loggable; + + /// \private + template <> + struct Loggable { + static constexpr const char Name[] = "rerun.components.Texcoord2D"; + + /// Returns the arrow data type this type corresponds to. + static const std::shared_ptr& arrow_datatype(); + + /// Fills an arrow array builder with an array of this type. + static rerun::Error fill_arrow_array_builder( + arrow::FixedSizeListBuilder* builder, const components::Texcoord2D* elements, + size_t num_elements + ); + + /// Serializes an array of `rerun::components::Texcoord2D` into an arrow array. + static Result> to_arrow( + const components::Texcoord2D* instances, size_t num_instances + ); + }; +} // namespace rerun diff --git a/rerun_cpp/src/rerun/components/texcoord2d_ext.cpp b/rerun_cpp/src/rerun/components/texcoord2d_ext.cpp new file mode 100644 index 000000000000..ceae81da280e --- /dev/null +++ b/rerun_cpp/src/rerun/components/texcoord2d_ext.cpp @@ -0,0 +1,27 @@ +#include "texcoord2D.hpp" + +// Uncomment for better auto-complete while editing the extension. +// #define EDIT_EXTENSION + +namespace rerun { + namespace components { + +#ifdef EDIT_EXTENSION + // + + /// Construct Texcoord2D from u/v values. + Texcoord2D(float x, float y) : uv{u, v} {} + + float u() const { + return uv.x(); + } + + float v() const { + return uv.y(); + } + + // + }; +#endif +} // namespace components +} // namespace rerun diff --git a/rerun_py/rerun_sdk/rerun/archetypes/mesh3d.py b/rerun_py/rerun_sdk/rerun/archetypes/mesh3d.py index eea77fe728ae..fb67649a1cd2 100644 --- a/rerun_py/rerun_sdk/rerun/archetypes/mesh3d.py +++ b/rerun_py/rerun_sdk/rerun/archetypes/mesh3d.py @@ -112,10 +112,10 @@ def _clear(cls) -> Mesh3D: # # (Docstring intentionally commented out to hide this field from the docs) - vertex_texcoords: components.Position2DBatch | None = field( + vertex_texcoords: components.Texcoord2DBatch | None = field( metadata={"component": "optional"}, default=None, - converter=components.Position2DBatch._optional, # type: ignore[misc] + converter=components.Texcoord2DBatch._optional, # type: ignore[misc] ) # An optional uv texture coordinate for each vertex. # diff --git a/rerun_py/rerun_sdk/rerun/components/.gitattributes b/rerun_py/rerun_sdk/rerun/components/.gitattributes index e695f39d550c..7a1379b36739 100644 --- a/rerun_py/rerun_sdk/rerun/components/.gitattributes +++ b/rerun_py/rerun_sdk/rerun/components/.gitattributes @@ -29,6 +29,7 @@ rotation3d.py linguist-generated=true scalar.py linguist-generated=true scalar_scattering.py linguist-generated=true tensor_data.py linguist-generated=true +texcoord2d.py linguist-generated=true text.py linguist-generated=true text_log_level.py linguist-generated=true transform3d.py linguist-generated=true diff --git a/rerun_py/rerun_sdk/rerun/components/__init__.py b/rerun_py/rerun_sdk/rerun/components/__init__.py index 914c0f82a3dc..50b3453207f0 100644 --- a/rerun_py/rerun_sdk/rerun/components/__init__.py +++ b/rerun_py/rerun_sdk/rerun/components/__init__.py @@ -53,6 +53,7 @@ ScalarScatteringType, ) from .tensor_data import TensorData, TensorDataBatch, TensorDataType +from .texcoord2d import Texcoord2D, Texcoord2DBatch, Texcoord2DType from .text import Text, TextBatch, TextType from .text_log_level import TextLogLevel, TextLogLevelBatch, TextLogLevelType from .transform3d import Transform3D, Transform3DBatch, Transform3DType @@ -172,6 +173,9 @@ "TensorData", "TensorDataBatch", "TensorDataType", + "Texcoord2D", + "Texcoord2DBatch", + "Texcoord2DType", "Text", "TextBatch", "TextLogLevel", diff --git a/rerun_py/rerun_sdk/rerun/components/texcoord2d.py b/rerun_py/rerun_sdk/rerun/components/texcoord2d.py new file mode 100644 index 000000000000..6eadcd9580b8 --- /dev/null +++ b/rerun_py/rerun_sdk/rerun/components/texcoord2d.py @@ -0,0 +1,46 @@ +# DO NOT EDIT! This file was auto-generated by crates/re_types_builder/src/codegen/python.rs +# Based on "crates/re_types/definitions/rerun/components/texcoord2d.fbs". + +# You can extend this class by creating a "Texcoord2DExt" class in "texcoord2d_ext.py". + +from __future__ import annotations + +from .. import datatypes +from .._baseclasses import ComponentBatchMixin + +__all__ = ["Texcoord2D", "Texcoord2DBatch", "Texcoord2DType"] + + +class Texcoord2D(datatypes.Vec2D): + """ + **Component**: A 2D texture UV coordinate. + + Texture coordinates specify a position on a 2D texture. + A range from 0-1 in covers the entire texture in the respective dimension. + The behavior for values outside of this range depends on the visualization/renderer, + but will most commonly repeat the texture. + Rerun uses top-left as the origin for UV coordinates. + + 0 U 1 + 0 + --------- → + | . + V | . + | . + 1 ↓ . . . . . . + + This is the same convention as in Vulkan/Metal/DX12/WebGPU, but (!) unlike OpenGL, + which places the origin at the bottom-left. + """ + + # You can define your own __init__ function as a member of Texcoord2DExt in texcoord2d_ext.py + + # Note: there are no fields here because Texcoord2D delegates to datatypes.Vec2D + pass + + +class Texcoord2DType(datatypes.Vec2DType): + _TYPE_NAME: str = "rerun.components.Texcoord2D" + + +class Texcoord2DBatch(datatypes.Vec2DBatch, ComponentBatchMixin): + _ARROW_TYPE = Texcoord2DType() diff --git a/scripts/clippy_wasm/clippy.toml b/scripts/clippy_wasm/clippy.toml index 4d67b8c20316..47926806dfaa 100644 --- a/scripts/clippy_wasm/clippy.toml +++ b/scripts/clippy_wasm/clippy.toml @@ -55,6 +55,7 @@ disallowed-types = [ # Allow-list of words for markdown in dosctrings https://rust-lang.github.io/rust-clippy/master/index.html#doc_markdown doc-valid-idents = [ # You must also update the same list in the root `clippy.toml`! + "..", "GitHub", "GLB", "GLTF", @@ -62,11 +63,11 @@ doc-valid-idents = [ "macOS", "NaN", "OBJ", + "OpenGL", "PyPI", "sRGB", "sRGBA", "WebGL", "WebSocket", "WebSockets", - "..", ] From a76d34fb23f385a467c2f7a5afbdba6ea72de713 Mon Sep 17 00:00:00 2001 From: Andreas Reich Date: Tue, 23 Jan 2024 18:12:30 +0100 Subject: [PATCH 10/28] clarify texture limitation --- crates/re_types/definitions/rerun/datatypes/material.fbs | 3 ++- crates/re_types/src/datatypes/material.rs | 3 ++- rerun_cpp/src/rerun/datatypes/material.hpp | 3 ++- rerun_py/rerun_sdk/rerun/datatypes/material.py | 6 ++++-- 4 files changed, 10 insertions(+), 5 deletions(-) diff --git a/crates/re_types/definitions/rerun/datatypes/material.fbs b/crates/re_types/definitions/rerun/datatypes/material.fbs index 941968838479..0d9f0a58c330 100644 --- a/crates/re_types/definitions/rerun/datatypes/material.fbs +++ b/crates/re_types/definitions/rerun/datatypes/material.fbs @@ -20,6 +20,7 @@ table Material ( /// Optional albedo texture. /// /// Used with `vertex_texcoords` on `Mesh3D`. - /// Currently supports only RGB & RGBA 2D-textures, ignoring alpha. + /// Currently supports only RGB & RGBA sRGBA textures, ignoring alpha. + /// (meaning that the tensor must have 3 or 4 channels and use the `u8` format) albedo_texture: rerun.datatypes.TensorData (nullable, order: 200); } diff --git a/crates/re_types/src/datatypes/material.rs b/crates/re_types/src/datatypes/material.rs index 5a2dc955ebcc..102031308599 100644 --- a/crates/re_types/src/datatypes/material.rs +++ b/crates/re_types/src/datatypes/material.rs @@ -30,7 +30,8 @@ pub struct Material { /// Optional albedo texture. /// /// Used with `vertex_texcoords` on `Mesh3D`. - /// Currently supports only RGB & RGBA 2D-textures, ignoring alpha. + /// Currently supports only RGB & RGBA sRGBA textures, ignoring alpha. + /// (meaning that the tensor must have 3 or 4 channels and use the `u8` format) pub albedo_texture: Option, } diff --git a/rerun_cpp/src/rerun/datatypes/material.hpp b/rerun_cpp/src/rerun/datatypes/material.hpp index eeab33df3cd8..61f165cf77b4 100644 --- a/rerun_cpp/src/rerun/datatypes/material.hpp +++ b/rerun_cpp/src/rerun/datatypes/material.hpp @@ -26,7 +26,8 @@ namespace rerun::datatypes { /// Optional albedo texture. /// /// Used with `vertex_texcoords` on `Mesh3D`. - /// Currently supports only RGB & RGBA 2D-textures, ignoring alpha. + /// Currently supports only RGB & RGBA sRGBA textures, ignoring alpha. + /// (meaning that the tensor must have 3 or 4 channels and use the `u8` format) std::optional albedo_texture; public: diff --git a/rerun_py/rerun_sdk/rerun/datatypes/material.py b/rerun_py/rerun_sdk/rerun/datatypes/material.py index 49a840c67db5..14191d0017f5 100644 --- a/rerun_py/rerun_sdk/rerun/datatypes/material.py +++ b/rerun_py/rerun_sdk/rerun/datatypes/material.py @@ -48,7 +48,8 @@ def __init__( Optional albedo texture. Used with `vertex_texcoords` on `Mesh3D`. - Currently supports only RGB & RGBA 2D-textures, ignoring alpha. + Currently supports only RGB & RGBA sRGBA textures, ignoring alpha. + (meaning that the tensor must have 3 or 4 channels and use the `u8` format) """ # You can define your own __init__ function as a member of MaterialExt in material_ext.py @@ -65,7 +66,8 @@ def __init__( # Optional albedo texture. # # Used with `vertex_texcoords` on `Mesh3D`. - # Currently supports only RGB & RGBA 2D-textures, ignoring alpha. + # Currently supports only RGB & RGBA sRGBA textures, ignoring alpha. + # (meaning that the tensor must have 3 or 4 channels and use the `u8` format) # # (Docstring intentionally commented out to hide this field from the docs) From e53b16f225ffa4566aee42070bd410526fe73904 Mon Sep 17 00:00:00 2001 From: Andreas Reich Date: Tue, 23 Jan 2024 18:25:53 +0100 Subject: [PATCH 11/28] crude data ui --- crates/re_data_ui/src/data.rs | 36 +-------------------- crates/re_data_ui/src/image.rs | 2 +- crates/re_data_ui/src/lib.rs | 4 ++- crates/re_data_ui/src/material.rs | 53 +++++++++++++++++++++++++++++++ 4 files changed, 58 insertions(+), 37 deletions(-) create mode 100644 crates/re_data_ui/src/material.rs diff --git a/crates/re_data_ui/src/data.rs b/crates/re_data_ui/src/data.rs index 1e97e5bce722..2a74f9fc7d80 100644 --- a/crates/re_data_ui/src/data.rs +++ b/crates/re_data_ui/src/data.rs @@ -1,9 +1,7 @@ use egui::Vec2; use re_format::format_f32; -use re_types::components::{ - Color, LineStrip2D, LineStrip3D, Material, MeshProperties, ViewCoordinates, -}; +use re_types::components::{Color, LineStrip2D, LineStrip3D, MeshProperties, ViewCoordinates}; use re_viewer_context::{UiVerbosity, ViewerContext}; use super::{table_for_verbosity, DataUi}; @@ -227,38 +225,6 @@ impl DataUi for LineStrip3D { } } -impl DataUi for Material { - fn data_ui( - &self, - ctx: &ViewerContext<'_>, - ui: &mut egui::Ui, - verbosity: UiVerbosity, - query: &re_data_store::LatestAtQuery, - store: &re_data_store::DataStore, - ) { - let show_optional_albedo_factor = |ui: &mut egui::Ui| { - if let Some(albedo_factor) = self.albedo_factor { - Color(albedo_factor).data_ui(ctx, ui, verbosity, query, store); - } else { - ui.weak("(empty)"); - } - }; - - match verbosity { - UiVerbosity::Small | UiVerbosity::Reduced => { - show_optional_albedo_factor(ui); - } - UiVerbosity::Full | UiVerbosity::LimitHeight => { - egui::Grid::new("material").num_columns(2).show(ui, |ui| { - ui.label("albedo_factor"); - show_optional_albedo_factor(ui); - ui.end_row(); - }); - } - } - } -} - impl DataUi for MeshProperties { fn data_ui( &self, diff --git a/crates/re_data_ui/src/image.rs b/crates/re_data_ui/src/image.rs index acd11237e004..1fd087801fba 100644 --- a/crates/re_data_ui/src/image.rs +++ b/crates/re_data_ui/src/image.rs @@ -65,7 +65,7 @@ impl EntityDataUi for re_types::components::TensorData { } #[allow(clippy::too_many_arguments)] -fn tensor_ui( +pub fn tensor_ui( ctx: &ViewerContext<'_>, query: &re_data_store::LatestAtQuery, store: &re_data_store::DataStore, diff --git a/crates/re_data_ui/src/lib.rs b/crates/re_data_ui/src/lib.rs index d27ea982e9a5..9b7e58c816a5 100644 --- a/crates/re_data_ui/src/lib.rs +++ b/crates/re_data_ui/src/lib.rs @@ -19,13 +19,15 @@ mod entity_path; mod image; mod image_meaning; mod instance_path; -pub mod item_ui; mod log_msg; +mod material; mod pinhole; mod rotation3d; mod store_id; mod transform3d; +pub mod item_ui; + pub use crate::image::{ show_zoomed_image_region, show_zoomed_image_region_area_outline, tensor_summary_ui_grid_contents, diff --git a/crates/re_data_ui/src/material.rs b/crates/re_data_ui/src/material.rs new file mode 100644 index 000000000000..85ed9c4aa25e --- /dev/null +++ b/crates/re_data_ui/src/material.rs @@ -0,0 +1,53 @@ +use re_types::components::{Color, Material}; +use re_viewer_context::{UiVerbosity, ViewerContext}; + +use crate::DataUi; + +impl DataUi for Material { + fn data_ui( + &self, + ctx: &ViewerContext<'_>, + ui: &mut egui::Ui, + verbosity: UiVerbosity, + query: &re_data_store::LatestAtQuery, + store: &re_data_store::DataStore, + ) { + let show_optional_albedo_factor = |ui: &mut egui::Ui| { + if let Some(albedo_factor) = self.albedo_factor { + Color(albedo_factor).data_ui(ctx, ui, verbosity, query, store); + } else { + ui.weak("(empty)"); + } + }; + let show_optional_albedo_texture = |ui: &mut egui::Ui| { + if let Some(albedo_texture) = &self.albedo_texture { + // TODO(andreas): Either we rethink this so that albedo texture is a regular image, + // or we pluck apart the image preview code to be re-usable here, + // so that we can show an image preview. + if let Some([height, width, _]) = albedo_texture.image_height_width_channels() { + ui.weak(format!("[{height} x {width}]")); + } else { + ui.weak("(not an image)"); + } + } else { + ui.weak("(empty)"); + } + }; + + match verbosity { + UiVerbosity::Small | UiVerbosity::Reduced => { + show_optional_albedo_factor(ui); + } + UiVerbosity::Full | UiVerbosity::LimitHeight => { + egui::Grid::new("material").num_columns(2).show(ui, |ui| { + ui.label("albedo_factor"); + show_optional_albedo_factor(ui); + ui.end_row(); + ui.label("albedo_texture"); + show_optional_albedo_texture(ui); + ui.end_row(); + }); + } + } + } +} From fb42ed55e7515a2f81a001a1be529cea810883b3 Mon Sep 17 00:00:00 2001 From: Andreas Reich Date: Tue, 23 Jan 2024 18:36:43 +0100 Subject: [PATCH 12/28] spelling exception --- docs/cspell.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/cspell.json b/docs/cspell.json index 8288ae0c1460..08788d347a5f 100644 --- a/docs/cspell.json +++ b/docs/cspell.json @@ -274,8 +274,8 @@ "ndarray", "nohash", "noqa", - "nuScenes", "numpy", + "nuScenes", "nyud", "obbs", "obj", @@ -351,6 +351,7 @@ "superquadrics", "tableofcontents", "taplo", + "Texcoord", "thiserror", "timepanel", "timepoint", From d3055783ed4d086d6b044e90a08c6c1cc7d97662 Mon Sep 17 00:00:00 2001 From: Andreas Reich Date: Thu, 25 Jan 2024 11:17:12 +0100 Subject: [PATCH 13/28] reexport wgpu in external namespace --- crates/re_renderer/src/lib.rs | 8 +++++--- crates/re_space_view_spatial/src/mesh_loader.rs | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/crates/re_renderer/src/lib.rs b/crates/re_renderer/src/lib.rs index d197d1678f20..0206c696bc48 100644 --- a/crates/re_renderer/src/lib.rs +++ b/crates/re_renderer/src/lib.rs @@ -79,10 +79,12 @@ pub use self::file_resolver::{ }; pub use self::file_server::FileServer; -// Re-export used color types. +// Re-export used color types directly. pub use ecolor::{Color32, Hsva, Rgba}; -// Re-export wgpu -pub use wgpu; + +pub mod external { + pub use wgpu; +} // --------------------------------------------------------------------------- diff --git a/crates/re_space_view_spatial/src/mesh_loader.rs b/crates/re_space_view_spatial/src/mesh_loader.rs index 9b4267c15c62..aeae83ffe5ec 100644 --- a/crates/re_space_view_spatial/src/mesh_loader.rs +++ b/crates/re_space_view_spatial/src/mesh_loader.rs @@ -233,7 +233,7 @@ fn mesh_texture_from_tensor_data( Ok(re_renderer::resource_managers::Texture2DCreationDesc { label: "mesh albedo texture from tensor data".into(), data, - format: re_renderer::wgpu::TextureFormat::Rgba8UnormSrgb, + format: re_renderer::external::wgpu::TextureFormat::Rgba8UnormSrgb, width, height, }) From acf9f665d660bfd5f14c39c73b0060ceeec77f7d Mon Sep 17 00:00:00 2001 From: Andreas Reich Date: Thu, 25 Jan 2024 11:36:31 +0100 Subject: [PATCH 14/28] better texture_key for meshes --- crates/re_space_view_spatial/src/mesh_cache.rs | 2 +- .../re_space_view_spatial/src/visualizers/meshes.rs | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/crates/re_space_view_spatial/src/mesh_cache.rs b/crates/re_space_view_spatial/src/mesh_cache.rs index 76e5ca3dbb72..409bb3a0b3be 100644 --- a/crates/re_space_view_spatial/src/mesh_cache.rs +++ b/crates/re_space_view_spatial/src/mesh_cache.rs @@ -9,7 +9,7 @@ use crate::mesh_loader::LoadedMesh; // ---------------------------------------------------------------------------- -#[derive(Debug, PartialEq, Eq, Hash)] +#[derive(Debug, PartialEq, Eq, Hash, Clone)] pub struct MeshCacheKey { pub versioned_instance_path_hash: VersionedInstancePathHash, pub media_type: Option, diff --git a/crates/re_space_view_spatial/src/visualizers/meshes.rs b/crates/re_space_view_spatial/src/visualizers/meshes.rs index 32c952b174fc..8efdcdf6b502 100644 --- a/crates/re_space_view_spatial/src/visualizers/meshes.rs +++ b/crates/re_space_view_spatial/src/visualizers/meshes.rs @@ -104,15 +104,16 @@ impl Mesh3DVisualizer { let outline_mask_ids = ent_context.highlight.index_outline_mask(InstanceKey::SPLAT); let mesh = ctx.cache.entry(|c: &mut MeshCache| { + let key = MeshCacheKey { + versioned_instance_path_hash: picking_instance_hash.versioned(primary_row_id), + media_type: None, + }; c.entry( &ent_path.to_string(), - MeshCacheKey { - versioned_instance_path_hash: picking_instance_hash.versioned(primary_row_id), - media_type: None, - }, + key.clone(), AnyMesh::Mesh { mesh: &mesh, - texture_key: re_log_types::hash::Hash64::hash(primary_row_id).hash64(), + texture_key: re_log_types::hash::Hash64::hash(&key).hash64(), }, ctx.render_ctx, ) From bb26200ed33e6ca6a4eeba8bf934d97c8ed829b5 Mon Sep 17 00:00:00 2001 From: Andreas Reich Date: Thu, 25 Jan 2024 11:38:06 +0100 Subject: [PATCH 15/28] fbs doc fixes --- crates/re_types/definitions/rerun/components/texcoord2d.fbs | 5 ++--- crates/re_types/definitions/rerun/datatypes/material.fbs | 2 +- crates/re_types/src/components/texcoord2d.rs | 5 ++--- crates/re_types/src/datatypes/material.rs | 2 +- docs/content/reference/types/components/texcoord2d.md | 5 ++--- rerun_cpp/src/rerun/components/texcoord2d.hpp | 5 ++--- rerun_cpp/src/rerun/datatypes/material.hpp | 2 +- rerun_py/rerun_sdk/rerun/components/texcoord2d.py | 5 ++--- rerun_py/rerun_sdk/rerun/datatypes/material.py | 4 ++-- 9 files changed, 15 insertions(+), 20 deletions(-) diff --git a/crates/re_types/definitions/rerun/components/texcoord2d.fbs b/crates/re_types/definitions/rerun/components/texcoord2d.fbs index 1080af59fbbd..1a678f9efb63 100644 --- a/crates/re_types/definitions/rerun/components/texcoord2d.fbs +++ b/crates/re_types/definitions/rerun/components/texcoord2d.fbs @@ -12,9 +12,8 @@ namespace rerun.components; /// A 2D texture UV coordinate. /// /// Texture coordinates specify a position on a 2D texture. -/// A range from 0-1 in covers the entire texture in the respective dimension. -/// The behavior for values outside of this range depends on the visualization/renderer, -/// but will most commonly repeat the texture. +/// A range from 0-1 covers the entire texture in the respective dimension. +/// Unless configured otherwise, the texture repeats outside of this range. /// Rerun uses top-left as the origin for UV coordinates. /// /// 0 U 1 diff --git a/crates/re_types/definitions/rerun/datatypes/material.fbs b/crates/re_types/definitions/rerun/datatypes/material.fbs index 0d9f0a58c330..faff29883921 100644 --- a/crates/re_types/definitions/rerun/datatypes/material.fbs +++ b/crates/re_types/definitions/rerun/datatypes/material.fbs @@ -20,7 +20,7 @@ table Material ( /// Optional albedo texture. /// /// Used with `vertex_texcoords` on `Mesh3D`. - /// Currently supports only RGB & RGBA sRGBA textures, ignoring alpha. + /// Currently supports only sRGB(A) textures, ignoring alpha. /// (meaning that the tensor must have 3 or 4 channels and use the `u8` format) albedo_texture: rerun.datatypes.TensorData (nullable, order: 200); } diff --git a/crates/re_types/src/components/texcoord2d.rs b/crates/re_types/src/components/texcoord2d.rs index 9d07c200488d..56e865efa8cd 100644 --- a/crates/re_types/src/components/texcoord2d.rs +++ b/crates/re_types/src/components/texcoord2d.rs @@ -24,9 +24,8 @@ use ::re_types_core::{DeserializationError, DeserializationResult}; /// **Component**: A 2D texture UV coordinate. /// /// Texture coordinates specify a position on a 2D texture. -/// A range from 0-1 in covers the entire texture in the respective dimension. -/// The behavior for values outside of this range depends on the visualization/renderer, -/// but will most commonly repeat the texture. +/// A range from 0-1 covers the entire texture in the respective dimension. +/// Unless configured otherwise, the texture repeats outside of this range. /// Rerun uses top-left as the origin for UV coordinates. /// /// 0 U 1 diff --git a/crates/re_types/src/datatypes/material.rs b/crates/re_types/src/datatypes/material.rs index 102031308599..6ae0ba9d6066 100644 --- a/crates/re_types/src/datatypes/material.rs +++ b/crates/re_types/src/datatypes/material.rs @@ -30,7 +30,7 @@ pub struct Material { /// Optional albedo texture. /// /// Used with `vertex_texcoords` on `Mesh3D`. - /// Currently supports only RGB & RGBA sRGBA textures, ignoring alpha. + /// Currently supports only sRGB(A) textures, ignoring alpha. /// (meaning that the tensor must have 3 or 4 channels and use the `u8` format) pub albedo_texture: Option, } diff --git a/docs/content/reference/types/components/texcoord2d.md b/docs/content/reference/types/components/texcoord2d.md index c3c8dd3c9c99..45ccd4ab4973 100644 --- a/docs/content/reference/types/components/texcoord2d.md +++ b/docs/content/reference/types/components/texcoord2d.md @@ -5,9 +5,8 @@ title: "Texcoord2D" A 2D texture UV coordinate. Texture coordinates specify a position on a 2D texture. -A range from 0-1 in covers the entire texture in the respective dimension. -The behavior for values outside of this range depends on the visualization/renderer, -but will most commonly repeat the texture. +A range from 0-1 covers the entire texture in the respective dimension. +Unless configured otherwise, the texture repeats outside of this range. Rerun uses top-left as the origin for UV coordinates. 0 U 1 diff --git a/rerun_cpp/src/rerun/components/texcoord2d.hpp b/rerun_cpp/src/rerun/components/texcoord2d.hpp index d95281240d5f..449f283f5370 100644 --- a/rerun_cpp/src/rerun/components/texcoord2d.hpp +++ b/rerun_cpp/src/rerun/components/texcoord2d.hpp @@ -20,9 +20,8 @@ namespace rerun::components { /// **Component**: A 2D texture UV coordinate. /// /// Texture coordinates specify a position on a 2D texture. - /// A range from 0-1 in covers the entire texture in the respective dimension. - /// The behavior for values outside of this range depends on the visualization/renderer, - /// but will most commonly repeat the texture. + /// A range from 0-1 covers the entire texture in the respective dimension. + /// Unless configured otherwise, the texture repeats outside of this range. /// Rerun uses top-left as the origin for UV coordinates. /// /// 0 U 1 diff --git a/rerun_cpp/src/rerun/datatypes/material.hpp b/rerun_cpp/src/rerun/datatypes/material.hpp index 61f165cf77b4..da30b2677958 100644 --- a/rerun_cpp/src/rerun/datatypes/material.hpp +++ b/rerun_cpp/src/rerun/datatypes/material.hpp @@ -26,7 +26,7 @@ namespace rerun::datatypes { /// Optional albedo texture. /// /// Used with `vertex_texcoords` on `Mesh3D`. - /// Currently supports only RGB & RGBA sRGBA textures, ignoring alpha. + /// Currently supports only sRGB(A) textures, ignoring alpha. /// (meaning that the tensor must have 3 or 4 channels and use the `u8` format) std::optional albedo_texture; diff --git a/rerun_py/rerun_sdk/rerun/components/texcoord2d.py b/rerun_py/rerun_sdk/rerun/components/texcoord2d.py index 6eadcd9580b8..5674f0f59df4 100644 --- a/rerun_py/rerun_sdk/rerun/components/texcoord2d.py +++ b/rerun_py/rerun_sdk/rerun/components/texcoord2d.py @@ -16,9 +16,8 @@ class Texcoord2D(datatypes.Vec2D): **Component**: A 2D texture UV coordinate. Texture coordinates specify a position on a 2D texture. - A range from 0-1 in covers the entire texture in the respective dimension. - The behavior for values outside of this range depends on the visualization/renderer, - but will most commonly repeat the texture. + A range from 0-1 covers the entire texture in the respective dimension. + Unless configured otherwise, the texture repeats outside of this range. Rerun uses top-left as the origin for UV coordinates. 0 U 1 diff --git a/rerun_py/rerun_sdk/rerun/datatypes/material.py b/rerun_py/rerun_sdk/rerun/datatypes/material.py index 14191d0017f5..9e6bf2d2ecc7 100644 --- a/rerun_py/rerun_sdk/rerun/datatypes/material.py +++ b/rerun_py/rerun_sdk/rerun/datatypes/material.py @@ -48,7 +48,7 @@ def __init__( Optional albedo texture. Used with `vertex_texcoords` on `Mesh3D`. - Currently supports only RGB & RGBA sRGBA textures, ignoring alpha. + Currently supports only sRGB(A) textures, ignoring alpha. (meaning that the tensor must have 3 or 4 channels and use the `u8` format) """ @@ -66,7 +66,7 @@ def __init__( # Optional albedo texture. # # Used with `vertex_texcoords` on `Mesh3D`. - # Currently supports only RGB & RGBA sRGBA textures, ignoring alpha. + # Currently supports only sRGB(A) textures, ignoring alpha. # (meaning that the tensor must have 3 or 4 channels and use the `u8` format) # # (Docstring intentionally commented out to hide this field from the docs) From 1570e5a984b2f29bf55d9bf6c1092e13f0d2946a Mon Sep 17 00:00:00 2001 From: Andreas Reich Date: Thu, 25 Jan 2024 17:45:42 +0100 Subject: [PATCH 16/28] fix missing dependency - did this break on main? --- Cargo.lock | 1 + crates/re_space_view_text_log/Cargo.toml | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 65ba187c2f8c..aa80b517a584 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4788,6 +4788,7 @@ dependencies = [ "re_entity_db", "re_log", "re_log_types", + "re_query_cache", "re_renderer", "re_tracing", "re_types", diff --git a/crates/re_space_view_text_log/Cargo.toml b/crates/re_space_view_text_log/Cargo.toml index 6f5b3d0178e4..ca2b1962512e 100644 --- a/crates/re_space_view_text_log/Cargo.toml +++ b/crates/re_space_view_text_log/Cargo.toml @@ -19,8 +19,9 @@ all-features = true re_data_store.workspace = true re_data_ui.workspace = true re_entity_db.workspace = true -re_log.workspace = true re_log_types.workspace = true +re_log.workspace = true +re_query_cache.workspace = true re_renderer.workspace = true re_tracing.workspace = true re_types.workspace = true From 12b9b8d43bb0b0296958a7d96fdeb7de18559081 Mon Sep 17 00:00:00 2001 From: Andreas Reich Date: Thu, 25 Jan 2024 18:01:24 +0100 Subject: [PATCH 17/28] fix python serialization for no texture present but looks like deserialization is actually broken --- rerun_py/rerun_sdk/rerun/datatypes/material_ext.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/rerun_py/rerun_sdk/rerun/datatypes/material_ext.py b/rerun_py/rerun_sdk/rerun/datatypes/material_ext.py index 47afad189593..00126f0e3fc4 100644 --- a/rerun_py/rerun_sdk/rerun/datatypes/material_ext.py +++ b/rerun_py/rerun_sdk/rerun/datatypes/material_ext.py @@ -16,7 +16,7 @@ class MaterialExt: @staticmethod def native_to_pa_array_override(data: MaterialArrayLike, data_type: pa.DataType) -> pa.Array: - from . import Material, Rgba32Type + from . import Material, Rgba32Type, TensorDataType # If it's a sequence of a single Material, grab the first one if isinstance(data, collections.abc.Sequence): @@ -35,9 +35,11 @@ def native_to_pa_array_override(data: MaterialArrayLike, data_type: pa.DataType) type=Rgba32Type().storage_type, ) if data.albedo_texture is not None: - albedo_texture = TensorDataExt.native_to_pa_array_override(data.albedo_texture, field_albedo_texture.type) + albedo_texture = TensorDataExt.native_to_pa_array_override( + data.albedo_texture, TensorDataType().storage_type + ) else: - albedo_texture = pa.array([None], type=field_albedo_texture.type) + albedo_texture = pa.nulls(1, type=TensorDataType().storage_type) return pa.StructArray.from_arrays( arrays=[albedo_factors, albedo_texture], From acd27b7fd2a44ed5a54cc5820105b8152a950661 Mon Sep 17 00:00:00 2001 From: Andreas Reich Date: Thu, 25 Jan 2024 18:28:37 +0100 Subject: [PATCH 18/28] missing unreleased attribute --- crates/re_types/definitions/rerun/components/texcoord2d.fbs | 3 ++- docs/content/reference/types/components/texcoord2d.md | 6 +++--- docs/content/reference/types/datatypes/vec2d.md | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/crates/re_types/definitions/rerun/components/texcoord2d.fbs b/crates/re_types/definitions/rerun/components/texcoord2d.fbs index 1a678f9efb63..8e2b0924f9fb 100644 --- a/crates/re_types/definitions/rerun/components/texcoord2d.fbs +++ b/crates/re_types/definitions/rerun/components/texcoord2d.fbs @@ -30,7 +30,8 @@ struct Texcoord2D ( "attr.python.aliases": "npt.NDArray[np.float32], Sequence[float], Tuple[float, float]", "attr.python.array_aliases": "npt.NDArray[np.float32], Sequence[float]", "attr.rust.derive": "Copy, PartialEq, bytemuck::Pod, bytemuck::Zeroable", - "attr.rust.repr": "transparent" + "attr.rust.repr": "transparent", + "attr.docs.unreleased" ) { uv: rerun.datatypes.Vec2D (order: 100); } diff --git a/docs/content/reference/types/components/texcoord2d.md b/docs/content/reference/types/components/texcoord2d.md index 45ccd4ab4973..caa918cedb17 100644 --- a/docs/content/reference/types/components/texcoord2d.md +++ b/docs/content/reference/types/components/texcoord2d.md @@ -24,9 +24,9 @@ which places the origin at the bottom-left. * uv: [`Vec2D`](../datatypes/vec2d.md) ## Links - * 🌊 [C++ API docs for `Texcoord2D`](https://ref.rerun.io/docs/cpp/stable/structrerun_1_1components_1_1Texcoord2D.html) - * 🐍 [Python API docs for `Texcoord2D`](https://ref.rerun.io/docs/python/stable/common/components#rerun.components.Texcoord2D) - * 🦀 [Rust API docs for `Texcoord2D`](https://docs.rs/rerun/latest/rerun/components/struct.Texcoord2D.html) + * 🌊 [C++ API docs for `Texcoord2D`](https://ref.rerun.io/docs/cpp/stable/structrerun_1_1components_1_1Texcoord2D.html?speculative-link) + * 🐍 [Python API docs for `Texcoord2D`](https://ref.rerun.io/docs/python/stable/common/components?speculative-link#rerun.components.Texcoord2D) + * 🦀 [Rust API docs for `Texcoord2D`](https://docs.rs/rerun/latest/rerun/components/struct.Texcoord2D.html?speculative-link) ## Used by diff --git a/docs/content/reference/types/datatypes/vec2d.md b/docs/content/reference/types/datatypes/vec2d.md index 149932c06c1d..d49ab5101f06 100644 --- a/docs/content/reference/types/datatypes/vec2d.md +++ b/docs/content/reference/types/datatypes/vec2d.md @@ -17,5 +17,5 @@ A vector in 2D space. * [`LineStrip2D`](../components/line_strip2d.md) * [`Position2D`](../components/position2d.md) * [`Resolution`](../components/resolution.md) -* [`Texcoord2D`](../components/texcoord2d.md) +* [`Texcoord2D`](../components/texcoord2d.md?speculative-link) * [`Vector2D`](../components/vector2d.md?speculative-link) From fbc140ddf43340672c67c430fb67cc23460411b8 Mon Sep 17 00:00:00 2001 From: Andreas Reich Date: Thu, 25 Jan 2024 18:29:50 +0100 Subject: [PATCH 19/28] missing newline --- crates/re_space_view_spatial/src/mesh_cache.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/re_space_view_spatial/src/mesh_cache.rs b/crates/re_space_view_spatial/src/mesh_cache.rs index 409bb3a0b3be..7cbdee860c6e 100644 --- a/crates/re_space_view_spatial/src/mesh_cache.rs +++ b/crates/re_space_view_spatial/src/mesh_cache.rs @@ -26,6 +26,7 @@ pub enum AnyMesh<'a> { Asset(&'a re_types::archetypes::Asset3D), Mesh { mesh: &'a re_types::archetypes::Mesh3D, + /// If any textures are in the mesh's material, they use this hash for texture manager lookup. texture_key: u64, }, From 6b3c097982f6001ee1c985759cdb43e6345df55b Mon Sep 17 00:00:00 2001 From: Andreas Reich Date: Thu, 25 Jan 2024 18:31:11 +0100 Subject: [PATCH 20/28] fix typo in texcoord2d cpp --- rerun_cpp/src/rerun/components/texcoord2d.hpp | 2 +- rerun_cpp/src/rerun/components/texcoord2d_ext.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/rerun_cpp/src/rerun/components/texcoord2d.hpp b/rerun_cpp/src/rerun/components/texcoord2d.hpp index 449f283f5370..34101cec6967 100644 --- a/rerun_cpp/src/rerun/components/texcoord2d.hpp +++ b/rerun_cpp/src/rerun/components/texcoord2d.hpp @@ -40,7 +40,7 @@ namespace rerun::components { // Extensions to generated type defined in 'texcoord2d_ext.cpp' /// Construct Texcoord2D from u/v values. - Texcoord2D(float x, float y) : uv{u, v} {} + Texcoord2D(float u, float v) : uv{u, v} {} float u() const { return uv.x(); diff --git a/rerun_cpp/src/rerun/components/texcoord2d_ext.cpp b/rerun_cpp/src/rerun/components/texcoord2d_ext.cpp index ceae81da280e..e3af13fd4e63 100644 --- a/rerun_cpp/src/rerun/components/texcoord2d_ext.cpp +++ b/rerun_cpp/src/rerun/components/texcoord2d_ext.cpp @@ -10,7 +10,7 @@ namespace rerun { // /// Construct Texcoord2D from u/v values. - Texcoord2D(float x, float y) : uv{u, v} {} + Texcoord2D(float u, float v) : uv{u, v} {} float u() const { return uv.x(); From 66937225bcab32e16683b4883d1cf25f98fc1083 Mon Sep 17 00:00:00 2001 From: Andreas Reich Date: Thu, 25 Jan 2024 18:46:43 +0100 Subject: [PATCH 21/28] fix and update mesh3d python tests --- .../rerun_sdk/rerun/archetypes/mesh3d_ext.py | 2 +- rerun_py/tests/unit/test_mesh3d.py | 24 +++++++++++++++++-- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/rerun_py/rerun_sdk/rerun/archetypes/mesh3d_ext.py b/rerun_py/rerun_sdk/rerun/archetypes/mesh3d_ext.py index ce8c64a77e38..f40e9d8895df 100644 --- a/rerun_py/rerun_sdk/rerun/archetypes/mesh3d_ext.py +++ b/rerun_py/rerun_sdk/rerun/archetypes/mesh3d_ext.py @@ -19,7 +19,7 @@ def __init__( mesh_properties: datatypes.MeshPropertiesLike | None = None, vertex_normals: datatypes.Vec3DArrayLike | None = None, vertex_colors: datatypes.Rgba32ArrayLike | None = None, - vertex_texcoords: datatypes.Rgba32ArrayLike | None = None, + vertex_texcoords: datatypes.Vec2DArrayLike | None = None, mesh_material: datatypes.MaterialLike | None = None, class_ids: datatypes.ClassIdArrayLike | None = None, instance_keys: components.InstanceKeyArrayLike | None = None, diff --git a/rerun_py/tests/unit/test_mesh3d.py b/rerun_py/tests/unit/test_mesh3d.py index 0fa3492f737d..c7a9ca071674 100644 --- a/rerun_py/tests/unit/test_mesh3d.py +++ b/rerun_py/tests/unit/test_mesh3d.py @@ -5,6 +5,7 @@ import rerun as rr from rerun.components import InstanceKeyArrayLike, MaterialBatch, MeshPropertiesBatch, Position3DBatch, Vector3DBatch +from rerun.components.texcoord2d import Texcoord2DBatch from rerun.datatypes import ( ClassIdArrayLike, Material, @@ -12,6 +13,7 @@ MeshProperties, MeshPropertiesLike, Rgba32ArrayLike, + Vec2DArrayLike, Vec3DArrayLike, ) @@ -23,6 +25,8 @@ instance_keys_arrays, instance_keys_expected, none_empty_or_value, + vec2ds_arrays, + vec2ds_expected, vec3ds_arrays, vec3ds_expected, ) @@ -55,11 +59,13 @@ def test_mesh3d() -> None: vertex_positions_arrays = vec3ds_arrays vertex_normals_arrays = vec3ds_arrays vertex_colors_arrays = colors_arrays + vertex_texcoord_arrays = vec2ds_arrays all_arrays = itertools.zip_longest( vertex_positions_arrays, vertex_normals_arrays, vertex_colors_arrays, + vertex_texcoord_arrays, mesh_properties_objects, mesh_materials, class_ids_arrays, @@ -70,6 +76,7 @@ def test_mesh3d() -> None: vertex_positions, vertex_normals, vertex_colors, + vertex_texcoords, mesh_properties, mesh_material, class_ids, @@ -81,6 +88,7 @@ def test_mesh3d() -> None: vertex_positions = cast(Vec3DArrayLike, vertex_positions) vertex_normals = cast(Optional[Vec3DArrayLike], vertex_normals) vertex_colors = cast(Optional[Rgba32ArrayLike], vertex_colors) + vertex_texcoords = cast(Optional[Vec2DArrayLike], vertex_texcoords) mesh_properties = cast(Optional[MeshPropertiesLike], mesh_properties) mesh_material = cast(Optional[MaterialLike], mesh_material) class_ids = cast(Optional[ClassIdArrayLike], class_ids) @@ -88,9 +96,10 @@ def test_mesh3d() -> None: print( f"E: rr.Mesh3D(\n" - f" vertex_normals={vertex_positions}\n" + f" vertex_positions={vertex_positions}\n" f" vertex_normals={vertex_normals}\n" f" vertex_colors={vertex_colors}\n" + f" vertex_texcoords={vertex_texcoords}\n" f" mesh_properties={mesh_properties_objects}\n" f" mesh_material={mesh_material}\n" f" class_ids={class_ids}\n" @@ -101,6 +110,7 @@ def test_mesh3d() -> None: vertex_positions=vertex_positions, vertex_normals=vertex_normals, vertex_colors=vertex_colors, + vertex_texcoords=vertex_texcoords, mesh_properties=mesh_properties, mesh_material=mesh_material, class_ids=class_ids, @@ -111,6 +121,7 @@ def test_mesh3d() -> None: assert arch.vertex_positions == vec3ds_expected(vertex_positions, Position3DBatch) assert arch.vertex_normals == vec3ds_expected(vertex_normals, Vector3DBatch) assert arch.vertex_colors == colors_expected(vertex_colors) + assert arch.vertex_texcoords == vec2ds_expected(vertex_texcoords, Texcoord2DBatch) assert arch.mesh_properties == mesh_properties_expected(mesh_properties) assert arch.mesh_material == mesh_material_expected(mesh_material) assert arch.class_ids == class_ids_expected(class_ids) @@ -124,11 +135,20 @@ def test_nullable_albedo_factor() -> None: MaterialBatch( [ Material(albedo_factor=[0xCC, 0x00, 0xCC, 0xFF]), + ] + ) + ) + == 1 + ) + assert ( + len( + MaterialBatch( + [ Material(), ] ) ) - == 2 + == 1 ) From 9b0107b234ea613d3aad63060566d359e8c19290 Mon Sep 17 00:00:00 2001 From: Andreas Reich Date: Thu, 1 Feb 2024 15:55:33 +0100 Subject: [PATCH 22/28] improvements to rgba32 rust extensions --- crates/re_types/src/datatypes/rgba32_ext.rs | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/crates/re_types/src/datatypes/rgba32_ext.rs b/crates/re_types/src/datatypes/rgba32_ext.rs index 6e3f7df89b88..da1594148e85 100644 --- a/crates/re_types/src/datatypes/rgba32_ext.rs +++ b/crates/re_types/src/datatypes/rgba32_ext.rs @@ -1,26 +1,28 @@ use super::Rgba32; impl Rgba32 { + pub const WHITE: Self = Self::from_rgb(255, 255, 255); + #[inline] - pub fn from_rgb(r: u8, g: u8, b: u8) -> Self { - Self::from([r, g, b, 255]) + pub const fn from_rgb(r: u8, g: u8, b: u8) -> Self { + Self::from_unmultiplied_rgba(r, g, b, 255) } #[inline] - pub fn from_unmultiplied_rgba(r: u8, g: u8, b: u8, a: u8) -> Self { + pub const fn from_unmultiplied_rgba(r: u8, g: u8, b: u8, a: u8) -> Self { let [r, g, b, a] = [r as u32, g as u32, b as u32, a as u32]; Self(r << 24 | g << 16 | b << 8 | a) } /// Most significant byte is `r`, least significant byte is `a`. #[inline] - pub fn from_u32(rgba: u32) -> Self { + pub const fn from_u32(rgba: u32) -> Self { Self(rgba) } /// `[r, g, b, a]` #[inline] - pub fn to_array(self) -> [u8; 4] { + pub const fn to_array(self) -> [u8; 4] { [ (self.0 >> 24) as u8, (self.0 >> 16) as u8, @@ -31,7 +33,7 @@ impl Rgba32 { /// Most significant byte is `r`, least significant byte is `a`. #[inline] - pub fn to_u32(self) -> u32 { + pub const fn to_u32(self) -> u32 { self.0 } } From e5f1d37e4e99e5f6202ce03ce851220fce5162d5 Mon Sep 17 00:00:00 2001 From: Andreas Reich Date: Thu, 1 Feb 2024 17:32:07 +0100 Subject: [PATCH 23/28] revert material, put albedo texture on the mesh archetype instead --- crates/re_data_ui/src/material.rs | 17 -- .../re_space_view_spatial/src/mesh_loader.rs | 8 +- .../src/visualizers/meshes.rs | 5 +- .../definitions/rerun/archetypes/mesh3d.fbs | 11 +- .../definitions/rerun/components/material.fbs | 2 +- .../definitions/rerun/datatypes/material.fbs | 16 +- crates/re_types/src/archetypes/mesh3d.rs | 41 +++- crates/re_types/src/components/material.rs | 22 +-- crates/re_types/src/datatypes/material.rs | 183 +++++++----------- crates/re_types/src/datatypes/material_ext.rs | 1 - crates/re_types/tests/mesh3d.rs | 24 ++- .../reference/types/archetypes/mesh3d.md | 2 +- .../reference/types/components/tensor_data.md | 1 + .../reference/types/datatypes/material.md | 1 - .../reference/types/datatypes/tensor_data.md | 1 - examples/python/raw_mesh/main.py | 31 +-- examples/rust/raw_mesh/src/main.rs | 1 - rerun_cpp/src/rerun/archetypes/mesh3d.cpp | 7 +- rerun_cpp/src/rerun/archetypes/mesh3d.hpp | 19 ++ rerun_cpp/src/rerun/components/material.hpp | 15 +- rerun_cpp/src/rerun/datatypes/material.cpp | 24 --- rerun_cpp/src/rerun/datatypes/material.hpp | 23 ++- rerun_py/rerun_sdk/rerun/archetypes/mesh3d.py | 14 ++ .../rerun_sdk/rerun/archetypes/mesh3d_ext.py | 6 + .../rerun_sdk/rerun/datatypes/material.py | 152 +-------------- .../rerun_sdk/rerun/datatypes/material_ext.py | 30 +-- 26 files changed, 260 insertions(+), 397 deletions(-) diff --git a/crates/re_data_ui/src/material.rs b/crates/re_data_ui/src/material.rs index 85ed9c4aa25e..d17aba5c17a9 100644 --- a/crates/re_data_ui/src/material.rs +++ b/crates/re_data_ui/src/material.rs @@ -19,20 +19,6 @@ impl DataUi for Material { ui.weak("(empty)"); } }; - let show_optional_albedo_texture = |ui: &mut egui::Ui| { - if let Some(albedo_texture) = &self.albedo_texture { - // TODO(andreas): Either we rethink this so that albedo texture is a regular image, - // or we pluck apart the image preview code to be re-usable here, - // so that we can show an image preview. - if let Some([height, width, _]) = albedo_texture.image_height_width_channels() { - ui.weak(format!("[{height} x {width}]")); - } else { - ui.weak("(not an image)"); - } - } else { - ui.weak("(empty)"); - } - }; match verbosity { UiVerbosity::Small | UiVerbosity::Reduced => { @@ -43,9 +29,6 @@ impl DataUi for Material { ui.label("albedo_factor"); show_optional_albedo_factor(ui); ui.end_row(); - ui.label("albedo_texture"); - show_optional_albedo_texture(ui); - ui.end_row(); }); } } diff --git a/crates/re_space_view_spatial/src/mesh_loader.rs b/crates/re_space_view_spatial/src/mesh_loader.rs index aeae83ffe5ec..06c56c0bc42e 100644 --- a/crates/re_space_view_spatial/src/mesh_loader.rs +++ b/crates/re_space_view_spatial/src/mesh_loader.rs @@ -102,6 +102,7 @@ impl LoadedMesh { mesh_material, class_ids: _, instance_keys: _, + albedo_texture, } = mesh3d; let vertex_positions: &[glam::Vec3] = bytemuck::cast_slice(vertex_positions.as_slice()); @@ -157,11 +158,8 @@ impl LoadedMesh { macaw::BoundingBox::from_points(vertex_positions.iter().copied()) }; - let albedo = if let Some(albedo_texture) = mesh_material - .as_ref() - .and_then(|mat| mat.albedo_texture.as_ref()) - { - mesh_texture_from_tensor_data(albedo_texture, render_ctx, texture_key)? + let albedo = if let Some(albedo_texture) = &albedo_texture { + mesh_texture_from_tensor_data(&albedo_texture.0, render_ctx, texture_key)? } else { render_ctx .texture_manager_2d diff --git a/crates/re_space_view_spatial/src/visualizers/meshes.rs b/crates/re_space_view_spatial/src/visualizers/meshes.rs index f3677c09f871..d5009ac85dfa 100644 --- a/crates/re_space_view_spatial/src/visualizers/meshes.rs +++ b/crates/re_space_view_spatial/src/visualizers/meshes.rs @@ -3,7 +3,9 @@ use re_query::{ArchetypeView, QueryError}; use re_renderer::renderer::MeshInstance; use re_types::{ archetypes::Mesh3D, - components::{Color, InstanceKey, Material, MeshProperties, Position3D, Texcoord2D, Vector3D}, + components::{ + Color, InstanceKey, Material, MeshProperties, Position3D, TensorData, Texcoord2D, Vector3D, + }, }; use re_viewer_context::{ ApplicableEntities, IdentifiedViewSystem, SpaceViewSystemExecutionError, ViewContextCollection, @@ -94,6 +96,7 @@ impl Mesh3DVisualizer { }, mesh_properties: arch_view.raw_optional_mono_component::()?, mesh_material: arch_view.raw_optional_mono_component::()?, + albedo_texture: arch_view.raw_optional_mono_component::()?, class_ids: None, instance_keys: None, } diff --git a/crates/re_types/definitions/rerun/archetypes/mesh3d.fbs b/crates/re_types/definitions/rerun/archetypes/mesh3d.fbs index 332cf1763872..bc4c7afa5a18 100644 --- a/crates/re_types/definitions/rerun/archetypes/mesh3d.fbs +++ b/crates/re_types/definitions/rerun/archetypes/mesh3d.fbs @@ -47,11 +47,18 @@ table Mesh3D ( /// Optional material properties for the mesh as a whole. mesh_material: rerun.components.Material ("attr.rerun.component_optional", nullable, order: 3300); + /// Optional albedo texture. + /// + /// Used with `vertex_texcoords` on `Mesh3D`. + /// Currently supports only sRGB(A) textures, ignoring alpha. + /// (meaning that the tensor must have 3 or 4 channels and use the `u8` format) + albedo_texture: rerun.components.TensorData ("attr.rerun.component_optional", nullable, order: 3400); + /// Optional class Ids for the vertices. /// /// The class ID provides colors and labels if not specified explicitly. - class_ids: [rerun.components.ClassId] ("attr.rerun.component_optional", nullable, order: 3400); + class_ids: [rerun.components.ClassId] ("attr.rerun.component_optional", nullable, order: 3500); /// Unique identifiers for each individual vertex in the mesh. - instance_keys: [rerun.components.InstanceKey] ("attr.rerun.component_optional", nullable, order: 3500); + instance_keys: [rerun.components.InstanceKey] ("attr.rerun.component_optional", nullable, order: 3600); } diff --git a/crates/re_types/definitions/rerun/components/material.fbs b/crates/re_types/definitions/rerun/components/material.fbs index a487b16fdd6b..a9c6847cf749 100644 --- a/crates/re_types/definitions/rerun/components/material.fbs +++ b/crates/re_types/definitions/rerun/components/material.fbs @@ -11,7 +11,7 @@ namespace rerun.components; /// Material properties of a mesh. table Material ( - "attr.rust.derive": "PartialEq" + "attr.rust.derive": "PartialEq, Eq" ) { material: rerun.datatypes.Material (order: 100); } diff --git a/crates/re_types/definitions/rerun/datatypes/material.fbs b/crates/re_types/definitions/rerun/datatypes/material.fbs index faff29883921..b42b550f0c8a 100644 --- a/crates/re_types/definitions/rerun/datatypes/material.fbs +++ b/crates/re_types/definitions/rerun/datatypes/material.fbs @@ -1,26 +1,18 @@ include "arrow/attributes.fbs"; include "python/attributes.fbs"; include "rust/attributes.fbs"; -include "rerun/attributes.fbs"; -include "./rgba32.fbs"; -include "./tensor_buffer.fbs"; +include "rerun/attributes.fbs"; +include "rerun/datatypes/rgba32.fbs"; namespace rerun.datatypes; // --- /// Material properties of a mesh. -table Material ( - "attr.rust.derive": "PartialEq" +struct Material ( + "attr.rust.derive": "Copy, PartialEq, Eq, Hash" ) { /// Optional color multiplier. albedo_factor: rerun.datatypes.Rgba32 (nullable, order: 100); - - /// Optional albedo texture. - /// - /// Used with `vertex_texcoords` on `Mesh3D`. - /// Currently supports only sRGB(A) textures, ignoring alpha. - /// (meaning that the tensor must have 3 or 4 channels and use the `u8` format) - albedo_texture: rerun.datatypes.TensorData (nullable, order: 200); } diff --git a/crates/re_types/src/archetypes/mesh3d.rs b/crates/re_types/src/archetypes/mesh3d.rs index da34bf0a374d..1c3fefb458af 100644 --- a/crates/re_types/src/archetypes/mesh3d.rs +++ b/crates/re_types/src/archetypes/mesh3d.rs @@ -77,6 +77,13 @@ pub struct Mesh3D { /// Optional material properties for the mesh as a whole. pub mesh_material: Option, + /// Optional albedo texture. + /// + /// Used with `vertex_texcoords` on `Mesh3D`. + /// Currently supports only sRGB(A) textures, ignoring alpha. + /// (meaning that the tensor must have 3 or 4 channels and use the `u8` format) + pub albedo_texture: Option, + /// Optional class Ids for the vertices. /// /// The class ID provides colors and labels if not specified explicitly. @@ -95,6 +102,7 @@ impl ::re_types_core::SizeBytes for Mesh3D { + self.vertex_colors.heap_size_bytes() + self.vertex_texcoords.heap_size_bytes() + self.mesh_material.heap_size_bytes() + + self.albedo_texture.heap_size_bytes() + self.class_ids.heap_size_bytes() + self.instance_keys.heap_size_bytes() } @@ -107,6 +115,7 @@ impl ::re_types_core::SizeBytes for Mesh3D { && >>::is_pod() && >>::is_pod() && >::is_pod() + && >::is_pod() && >>::is_pod() && >>::is_pod() } @@ -124,18 +133,19 @@ static RECOMMENDED_COMPONENTS: once_cell::sync::Lazy<[ComponentName; 3usize]> = ] }); -static OPTIONAL_COMPONENTS: once_cell::sync::Lazy<[ComponentName; 5usize]> = +static OPTIONAL_COMPONENTS: once_cell::sync::Lazy<[ComponentName; 6usize]> = once_cell::sync::Lazy::new(|| { [ "rerun.components.ClassId".into(), "rerun.components.Color".into(), "rerun.components.InstanceKey".into(), "rerun.components.Material".into(), + "rerun.components.TensorData".into(), "rerun.components.Texcoord2D".into(), ] }); -static ALL_COMPONENTS: once_cell::sync::Lazy<[ComponentName; 9usize]> = +static ALL_COMPONENTS: once_cell::sync::Lazy<[ComponentName; 10usize]> = once_cell::sync::Lazy::new(|| { [ "rerun.components.Position3D".into(), @@ -146,12 +156,13 @@ static ALL_COMPONENTS: once_cell::sync::Lazy<[ComponentName; 9usize]> = "rerun.components.Color".into(), "rerun.components.InstanceKey".into(), "rerun.components.Material".into(), + "rerun.components.TensorData".into(), "rerun.components.Texcoord2D".into(), ] }); impl Mesh3D { - pub const NUM_COMPONENTS: usize = 9usize; + pub const NUM_COMPONENTS: usize = 10usize; } /// Indicator component for the [`Mesh3D`] [`::re_types_core::Archetype`] @@ -269,6 +280,16 @@ impl ::re_types_core::Archetype for Mesh3D { } else { None }; + let albedo_texture = if let Some(array) = arrays_by_name.get("rerun.components.TensorData") + { + ::from_arrow_opt(&**array) + .with_context("rerun.archetypes.Mesh3D#albedo_texture")? + .into_iter() + .next() + .flatten() + } else { + None + }; let class_ids = if let Some(array) = arrays_by_name.get("rerun.components.ClassId") { Some({ ::from_arrow_opt(&**array) @@ -301,6 +322,7 @@ impl ::re_types_core::Archetype for Mesh3D { vertex_colors, vertex_texcoords, mesh_material, + albedo_texture, class_ids, instance_keys, }) @@ -329,6 +351,9 @@ impl ::re_types_core::AsComponents for Mesh3D { self.mesh_material .as_ref() .map(|comp| (comp as &dyn ComponentBatch).into()), + self.albedo_texture + .as_ref() + .map(|comp| (comp as &dyn ComponentBatch).into()), self.class_ids .as_ref() .map(|comp_batch| (comp_batch as &dyn ComponentBatch).into()), @@ -358,6 +383,7 @@ impl Mesh3D { vertex_colors: None, vertex_texcoords: None, mesh_material: None, + albedo_texture: None, class_ids: None, instance_keys: None, } @@ -408,6 +434,15 @@ impl Mesh3D { self } + #[inline] + pub fn with_albedo_texture( + mut self, + albedo_texture: impl Into, + ) -> Self { + self.albedo_texture = Some(albedo_texture.into()); + self + } + #[inline] pub fn with_class_ids( mut self, diff --git a/crates/re_types/src/components/material.rs b/crates/re_types/src/components/material.rs index 2b724d35f728..c56fc48b2c57 100644 --- a/crates/re_types/src/components/material.rs +++ b/crates/re_types/src/components/material.rs @@ -22,7 +22,7 @@ use ::re_types_core::{ComponentBatch, MaybeOwnedComponentBatch}; use ::re_types_core::{DeserializationError, DeserializationResult}; /// **Component**: Material properties of a mesh. -#[derive(Clone, Debug, PartialEq)] +#[derive(Clone, Debug, PartialEq, Eq)] pub struct Material(pub crate::datatypes::Material); impl ::re_types_core::SizeBytes for Material { @@ -73,20 +73,12 @@ impl ::re_types_core::Loggable for Material { #[inline] fn arrow_datatype() -> arrow2::datatypes::DataType { use arrow2::datatypes::*; - DataType::Struct(std::sync::Arc::new(vec![ - Field { - name: "albedo_factor".to_owned(), - data_type: ::arrow_datatype(), - is_nullable: true, - metadata: [].into(), - }, - Field { - name: "albedo_texture".to_owned(), - data_type: ::arrow_datatype(), - is_nullable: true, - metadata: [].into(), - }, - ])) + DataType::Struct(std::sync::Arc::new(vec![Field { + name: "albedo_factor".to_owned(), + data_type: ::arrow_datatype(), + is_nullable: true, + metadata: [].into(), + }])) } #[allow(clippy::wildcard_imports)] diff --git a/crates/re_types/src/datatypes/material.rs b/crates/re_types/src/datatypes/material.rs index caaf5815c3b8..a73c76b39f22 100644 --- a/crates/re_types/src/datatypes/material.rs +++ b/crates/re_types/src/datatypes/material.rs @@ -22,29 +22,45 @@ use ::re_types_core::{ComponentBatch, MaybeOwnedComponentBatch}; use ::re_types_core::{DeserializationError, DeserializationResult}; /// **Datatype**: Material properties of a mesh. -#[derive(Clone, Debug, PartialEq)] +#[derive(Clone, Debug, Copy, PartialEq, Eq, Hash)] pub struct Material { /// Optional color multiplier. pub albedo_factor: Option, - - /// Optional albedo texture. - /// - /// Used with `vertex_texcoords` on `Mesh3D`. - /// Currently supports only sRGB(A) textures, ignoring alpha. - /// (meaning that the tensor must have 3 or 4 channels and use the `u8` format) - pub albedo_texture: Option, } impl ::re_types_core::SizeBytes for Material { #[inline] fn heap_size_bytes(&self) -> u64 { - self.albedo_factor.heap_size_bytes() + self.albedo_texture.heap_size_bytes() + self.albedo_factor.heap_size_bytes() } #[inline] fn is_pod() -> bool { >::is_pod() - && >::is_pod() + } +} + +impl>> From for Material { + fn from(v: T) -> Self { + Self { + albedo_factor: v.into(), + } + } +} + +impl std::borrow::Borrow> for Material { + #[inline] + fn borrow(&self) -> &Option { + &self.albedo_factor + } +} + +impl std::ops::Deref for Material { + type Target = Option; + + #[inline] + fn deref(&self) -> &Option { + &self.albedo_factor } } @@ -62,20 +78,12 @@ impl ::re_types_core::Loggable for Material { #[inline] fn arrow_datatype() -> arrow2::datatypes::DataType { use arrow2::datatypes::*; - DataType::Struct(std::sync::Arc::new(vec![ - Field { - name: "albedo_factor".to_owned(), - data_type: ::arrow_datatype(), - is_nullable: true, - metadata: [].into(), - }, - Field { - name: "albedo_texture".to_owned(), - data_type: ::arrow_datatype(), - is_nullable: true, - metadata: [].into(), - }, - ])) + DataType::Struct(std::sync::Arc::new(vec![Field { + name: "albedo_factor".to_owned(), + data_type: ::arrow_datatype(), + is_nullable: true, + metadata: [].into(), + }])) } #[allow(clippy::wildcard_imports)] @@ -101,66 +109,41 @@ impl ::re_types_core::Loggable for Material { }; StructArray::new( ::arrow_datatype(), - vec![ - { - let (somes, albedo_factor): (Vec<_>, Vec<_>) = data - .iter() - .map(|datum| { - let datum = datum - .as_ref() - .map(|datum| { - let Self { albedo_factor, .. } = &**datum; - albedo_factor.clone() - }) - .flatten(); - (datum.is_some(), datum) - }) - .unzip(); - let albedo_factor_bitmap: Option = { - let any_nones = somes.iter().any(|some| !*some); - any_nones.then(|| somes.into()) - }; - PrimitiveArray::new( - DataType::UInt32, - albedo_factor - .into_iter() + vec![{ + let (somes, albedo_factor): (Vec<_>, Vec<_>) = data + .iter() + .map(|datum| { + let datum = datum + .as_ref() .map(|datum| { - datum - .map(|datum| { - let crate::datatypes::Rgba32(data0) = datum; - data0 - }) - .unwrap_or_default() + let Self { albedo_factor, .. } = &**datum; + albedo_factor.clone() }) - .collect(), - albedo_factor_bitmap, - ) - .boxed() - }, - { - let (somes, albedo_texture): (Vec<_>, Vec<_>) = data - .iter() + .flatten(); + (datum.is_some(), datum) + }) + .unzip(); + let albedo_factor_bitmap: Option = { + let any_nones = somes.iter().any(|some| !*some); + any_nones.then(|| somes.into()) + }; + PrimitiveArray::new( + DataType::UInt32, + albedo_factor + .into_iter() .map(|datum| { - let datum = datum - .as_ref() + datum .map(|datum| { - let Self { albedo_texture, .. } = &**datum; - albedo_texture.clone() + let crate::datatypes::Rgba32(data0) = datum; + data0 }) - .flatten(); - (datum.is_some(), datum) + .unwrap_or_default() }) - .unzip(); - let albedo_texture_bitmap: Option = { - let any_nones = somes.iter().any(|some| !*some); - any_nones.then(|| somes.into()) - }; - { - _ = albedo_texture_bitmap; - crate::datatypes::TensorData::to_arrow_opt(albedo_texture)? - } - }, - ], + .collect(), + albedo_factor_bitmap, + ) + .boxed() + }], bitmap, ) .boxed() @@ -182,20 +165,12 @@ impl ::re_types_core::Loggable for Material { .downcast_ref::() .ok_or_else(|| { DeserializationError::datatype_mismatch( - DataType::Struct(std::sync::Arc::new(vec![ - Field { - name: "albedo_factor".to_owned(), - data_type: ::arrow_datatype(), - is_nullable: true, - metadata: [].into(), - }, - Field { - name: "albedo_texture".to_owned(), - data_type: ::arrow_datatype(), - is_nullable: true, - metadata: [].into(), - }, - ])), + DataType::Struct(std::sync::Arc::new(vec![Field { + name: "albedo_factor".to_owned(), + data_type: ::arrow_datatype(), + is_nullable: true, + metadata: [].into(), + }])), arrow_data.data_type().clone(), ) }) @@ -233,31 +208,13 @@ impl ::re_types_core::Loggable for Material { .map(|opt| opt.copied()) .map(|res_or_opt| res_or_opt.map(|v| crate::datatypes::Rgba32(v))) }; - let albedo_texture = { - if !arrays_by_name.contains_key("albedo_texture") { - return Err(DeserializationError::missing_struct_field( - Self::arrow_datatype(), - "albedo_texture", - )) - .with_context("rerun.datatypes.Material"); - } - let arrow_data = &**arrays_by_name["albedo_texture"]; - crate::datatypes::TensorData::from_arrow_opt(arrow_data) - .with_context("rerun.datatypes.Material#albedo_texture")? - .into_iter() - }; arrow2::bitmap::utils::ZipValidity::new_with_validity( - ::itertools::izip!(albedo_factor, albedo_texture), + ::itertools::izip!(albedo_factor), arrow_data.validity(), ) .map(|opt| { - opt.map(|(albedo_factor, albedo_texture)| { - Ok(Self { - albedo_factor, - albedo_texture, - }) - }) - .transpose() + opt.map(|(albedo_factor)| Ok(Self { albedo_factor })) + .transpose() }) .collect::>>() .with_context("rerun.datatypes.Material")? diff --git a/crates/re_types/src/datatypes/material_ext.rs b/crates/re_types/src/datatypes/material_ext.rs index a4467ecd6c11..eed456f3bc51 100644 --- a/crates/re_types/src/datatypes/material_ext.rs +++ b/crates/re_types/src/datatypes/material_ext.rs @@ -7,7 +7,6 @@ impl Material { pub fn from_albedo_factor(color: impl Into) -> Self { Self { albedo_factor: Some(color.into()), - albedo_texture: None, } } } diff --git a/crates/re_types/tests/mesh3d.rs b/crates/re_types/tests/mesh3d.rs index 793cc3c4210b..be2c52c842df 100644 --- a/crates/re_types/tests/mesh3d.rs +++ b/crates/re_types/tests/mesh3d.rs @@ -3,12 +3,29 @@ use std::collections::HashMap; use re_types::{ archetypes::Mesh3D, components::{ClassId, InstanceKey, Position3D, Texcoord2D, Vector3D}, - datatypes::{Material, MeshProperties, Rgba32, Vec2D, Vec3D}, + datatypes::{ + Material, MeshProperties, Rgba32, TensorBuffer, TensorData, TensorDimension, Vec2D, Vec3D, + }, Archetype as _, AsComponents as _, }; #[test] fn roundtrip() { + let tensor_data: re_types::components::TensorData = TensorData { + shape: vec![ + TensorDimension { + size: 2, + name: Some("height".into()), + }, + TensorDimension { + size: 3, + name: Some("width".into()), + }, + ], + buffer: TensorBuffer::U8(vec![1, 2, 3, 4, 5, 6].into()), + } + .into(); + let expected = Mesh3D { vertex_positions: vec![ Position3D(Vec3D([1.0, 2.0, 3.0])), @@ -35,10 +52,10 @@ fn roundtrip() { mesh_material: Some( Material { albedo_factor: Some(Rgba32::from_unmultiplied_rgba(0xEE, 0x11, 0x22, 0x33)), - albedo_texture: None, } .into(), ), + albedo_texture: Some(tensor_data.clone()), class_ids: Some(vec![ ClassId::from(126), // ClassId::from(127), // @@ -59,7 +76,8 @@ fn roundtrip() { .with_vertex_texcoords([[0.0, 1.0], [2.0, 3.0]]) .with_mesh_material(Material::from_albedo_factor(0xEE112233)) .with_class_ids([126, 127]) - .with_instance_keys([u64::MAX - 1, u64::MAX]); + .with_instance_keys([u64::MAX - 1, u64::MAX]) + .with_albedo_texture(tensor_data); similar_asserts::assert_eq!(expected, arch); let expected_extensions: HashMap<_, _> = [ diff --git a/docs/content/reference/types/archetypes/mesh3d.md b/docs/content/reference/types/archetypes/mesh3d.md index 1ee5bc5f1473..3c76f1233bf2 100644 --- a/docs/content/reference/types/archetypes/mesh3d.md +++ b/docs/content/reference/types/archetypes/mesh3d.md @@ -10,7 +10,7 @@ A 3D triangle mesh as specified by its per-mesh and per-vertex properties. **Recommended**: [`MeshProperties`](../components/mesh_properties.md), [`Vector3D`](../components/vector3d.md) -**Optional**: [`Color`](../components/color.md), [`Texcoord2D`](../components/texcoord2d.md), [`Material`](../components/material.md), [`ClassId`](../components/class_id.md), [`InstanceKey`](../components/instance_key.md) +**Optional**: [`Color`](../components/color.md), [`Texcoord2D`](../components/texcoord2d.md), [`Material`](../components/material.md), [`TensorData`](../components/tensor_data.md), [`ClassId`](../components/class_id.md), [`InstanceKey`](../components/instance_key.md) ## Links * 🌊 [C++ API docs for `Mesh3D`](https://ref.rerun.io/docs/cpp/stable/structrerun_1_1archetypes_1_1Mesh3D.html) diff --git a/docs/content/reference/types/components/tensor_data.md b/docs/content/reference/types/components/tensor_data.md index fb2a37846015..2895dec1081e 100644 --- a/docs/content/reference/types/components/tensor_data.md +++ b/docs/content/reference/types/components/tensor_data.md @@ -19,5 +19,6 @@ A multi-dimensional `Tensor` with optionally named arguments. * [`BarChart`](../archetypes/bar_chart.md) * [`DepthImage`](../archetypes/depth_image.md) * [`Image`](../archetypes/image.md) +* [`Mesh3D`](../archetypes/mesh3d.md) * [`SegmentationImage`](../archetypes/segmentation_image.md) * [`Tensor`](../archetypes/tensor.md) diff --git a/docs/content/reference/types/datatypes/material.md b/docs/content/reference/types/datatypes/material.md index 5256976fd2e3..3b2a1d1fd1f2 100644 --- a/docs/content/reference/types/datatypes/material.md +++ b/docs/content/reference/types/datatypes/material.md @@ -7,7 +7,6 @@ Material properties of a mesh. ## Fields * albedo_factor: [`Rgba32`](../datatypes/rgba32.md) -* albedo_texture: [`TensorData`](../datatypes/tensor_data.md) ## Links * 🌊 [C++ API docs for `Material`](https://ref.rerun.io/docs/cpp/stable/structrerun_1_1datatypes_1_1Material.html) diff --git a/docs/content/reference/types/datatypes/tensor_data.md b/docs/content/reference/types/datatypes/tensor_data.md index 95374f70fc8f..a68ec7f7aeae 100644 --- a/docs/content/reference/types/datatypes/tensor_data.md +++ b/docs/content/reference/types/datatypes/tensor_data.md @@ -25,4 +25,3 @@ which stores a contiguous array of typed values. ## Used by * [`TensorData`](../components/tensor_data.md) -* [`Material`](../datatypes/material.md) diff --git a/examples/python/raw_mesh/main.py b/examples/python/raw_mesh/main.py index 5c13d59573bb..38c92c293976 100755 --- a/examples/python/raw_mesh/main.py +++ b/examples/python/raw_mesh/main.py @@ -55,25 +55,31 @@ def log_scene(scene: trimesh.Scene, node: str, path: str | None = None) -> None: vertex_colors = None vertex_texcoords = None mesh_material = None + albedo_texture = None try: - mesh_material = Material(albedo_texture=mesh.visual.material.baseColorTexture) vertex_texcoords = mesh.visual.uv # trimesh uses the OpenGL convention for UV coordinates, so we need to flip the V coordinate # since Rerun uses the Vulkan/Metal/DX12/WebGPU convention. vertex_texcoords[:, 1] = 1.0 - vertex_texcoords[:, 1] except Exception: - # Didn't have UV coordinates, try with vertex colors instead. - try: - colors = mesh.visual.to_color().vertex_colors - if len(colors) == 4: - # If trimesh gives us a single vertex color for the entire mesh, we can interpret that - # as an albedo factor for the whole primitive. - mesh_material = Material(albedo_factor=np.array(colors)) - else: - vertex_colors = colors - except Exception: - pass + pass + + try: + albedo_texture = mesh.visual.material.baseColorTexture + except Exception: + pass + + try: + colors = mesh.visual.to_color().vertex_colors + if len(colors) == 4: + # If trimesh gives us a single vertex color for the entire mesh, we can interpret that + # as an albedo factor for the whole primitive. + mesh_material = Material(albedo_factor=np.array(colors)) + else: + vertex_colors = colors + except Exception: + pass rr.log( path, @@ -82,6 +88,7 @@ def log_scene(scene: trimesh.Scene, node: str, path: str | None = None) -> None: vertex_colors=vertex_colors, vertex_normals=mesh.vertex_normals, vertex_texcoords=vertex_texcoords, + albedo_texture=albedo_texture, indices=mesh.faces, mesh_material=mesh_material, ), diff --git a/examples/rust/raw_mesh/src/main.rs b/examples/rust/raw_mesh/src/main.rs index 0c702d03f127..5b05170e98c3 100644 --- a/examples/rust/raw_mesh/src/main.rs +++ b/examples/rust/raw_mesh/src/main.rs @@ -55,7 +55,6 @@ impl From for Mesh3D { mesh = mesh.with_mesh_material(rerun::datatypes::Material { albedo_factor: albedo_factor .map(|[r, g, b, a]| ecolor::Rgba::from_rgba_unmultiplied(r, g, b, a).into()), - albedo_texture: None, // TODO(andreas): This would require loading the right texture file and converting it to `TensorData`. }); } diff --git a/rerun_cpp/src/rerun/archetypes/mesh3d.cpp b/rerun_cpp/src/rerun/archetypes/mesh3d.cpp index 194ef40b7566..7d774e5acefe 100644 --- a/rerun_cpp/src/rerun/archetypes/mesh3d.cpp +++ b/rerun_cpp/src/rerun/archetypes/mesh3d.cpp @@ -14,7 +14,7 @@ namespace rerun { ) { using namespace archetypes; std::vector cells; - cells.reserve(9); + cells.reserve(10); { auto result = DataCell::from_loggable(archetype.vertex_positions); @@ -46,6 +46,11 @@ namespace rerun { RR_RETURN_NOT_OK(result.error); cells.push_back(std::move(result.value)); } + if (archetype.albedo_texture.has_value()) { + auto result = DataCell::from_loggable(archetype.albedo_texture.value()); + RR_RETURN_NOT_OK(result.error); + cells.push_back(std::move(result.value)); + } if (archetype.class_ids.has_value()) { auto result = DataCell::from_loggable(archetype.class_ids.value()); RR_RETURN_NOT_OK(result.error); diff --git a/rerun_cpp/src/rerun/archetypes/mesh3d.hpp b/rerun_cpp/src/rerun/archetypes/mesh3d.hpp index d7b147a80715..e74daa9f7691 100644 --- a/rerun_cpp/src/rerun/archetypes/mesh3d.hpp +++ b/rerun_cpp/src/rerun/archetypes/mesh3d.hpp @@ -11,6 +11,7 @@ #include "../components/material.hpp" #include "../components/mesh_properties.hpp" #include "../components/position3d.hpp" +#include "../components/tensor_data.hpp" #include "../components/texcoord2d.hpp" #include "../components/vector3d.hpp" #include "../data_cell.hpp" @@ -84,6 +85,13 @@ namespace rerun::archetypes { /// Optional material properties for the mesh as a whole. std::optional mesh_material; + /// Optional albedo texture. + /// + /// Used with `vertex_texcoords` on `Mesh3D`. + /// Currently supports only sRGB(A) textures, ignoring alpha. + /// (meaning that the tensor must have 3 or 4 channels and use the `u8` format) + std::optional albedo_texture; + /// Optional class Ids for the vertices. /// /// The class ID provides colors and labels if not specified explicitly. @@ -143,6 +151,17 @@ namespace rerun::archetypes { RR_WITH_MAYBE_UNINITIALIZED_DISABLED(return std::move(*this);) } + /// Optional albedo texture. + /// + /// Used with `vertex_texcoords` on `Mesh3D`. + /// Currently supports only sRGB(A) textures, ignoring alpha. + /// (meaning that the tensor must have 3 or 4 channels and use the `u8` format) + Mesh3D with_albedo_texture(rerun::components::TensorData _albedo_texture) && { + albedo_texture = std::move(_albedo_texture); + // See: https://github.com/rerun-io/rerun/issues/4027 + RR_WITH_MAYBE_UNINITIALIZED_DISABLED(return std::move(*this);) + } + /// Optional class Ids for the vertices. /// /// The class ID provides colors and labels if not specified explicitly. diff --git a/rerun_cpp/src/rerun/components/material.hpp b/rerun_cpp/src/rerun/components/material.hpp index eaceb8bddebf..dfe861f11724 100644 --- a/rerun_cpp/src/rerun/components/material.hpp +++ b/rerun_cpp/src/rerun/components/material.hpp @@ -4,11 +4,12 @@ #pragma once #include "../datatypes/material.hpp" +#include "../datatypes/rgba32.hpp" #include "../result.hpp" #include #include -#include +#include namespace arrow { class Array; @@ -31,10 +32,18 @@ namespace rerun::components { public: Material() = default; - Material(rerun::datatypes::Material material_) : material(std::move(material_)) {} + Material(rerun::datatypes::Material material_) : material(material_) {} Material& operator=(rerun::datatypes::Material material_) { - material = std::move(material_); + material = material_; + return *this; + } + + Material(std::optional albedo_factor_) + : material(albedo_factor_) {} + + Material& operator=(std::optional albedo_factor_) { + material = albedo_factor_; return *this; } diff --git a/rerun_cpp/src/rerun/datatypes/material.cpp b/rerun_cpp/src/rerun/datatypes/material.cpp index 3456154676db..9512650af0c8 100644 --- a/rerun_cpp/src/rerun/datatypes/material.cpp +++ b/rerun_cpp/src/rerun/datatypes/material.cpp @@ -4,7 +4,6 @@ #include "material.hpp" #include "rgba32.hpp" -#include "tensor_data.hpp" #include #include @@ -19,11 +18,6 @@ namespace rerun { Loggable::arrow_datatype(), true ), - arrow::field( - "albedo_texture", - Loggable::arrow_datatype(), - true - ), }); return datatype; } @@ -57,24 +51,6 @@ namespace rerun { } } } - { - auto field_builder = static_cast(builder->field_builder(1)); - ARROW_RETURN_NOT_OK(field_builder->Reserve(static_cast(num_elements))); - for (size_t elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { - const auto& element = elements[elem_idx]; - if (element.albedo_texture.has_value()) { - RR_RETURN_NOT_OK( - Loggable::fill_arrow_array_builder( - field_builder, - &element.albedo_texture.value(), - 1 - ) - ); - } else { - ARROW_RETURN_NOT_OK(field_builder->AppendNull()); - } - } - } ARROW_RETURN_NOT_OK(builder->AppendValues(static_cast(num_elements), nullptr)); return Error::ok(); diff --git a/rerun_cpp/src/rerun/datatypes/material.hpp b/rerun_cpp/src/rerun/datatypes/material.hpp index da30b2677958..efd12166804e 100644 --- a/rerun_cpp/src/rerun/datatypes/material.hpp +++ b/rerun_cpp/src/rerun/datatypes/material.hpp @@ -5,7 +5,6 @@ #include "../result.hpp" #include "rgba32.hpp" -#include "tensor_data.hpp" #include #include @@ -23,15 +22,23 @@ namespace rerun::datatypes { /// Optional color multiplier. std::optional albedo_factor; - /// Optional albedo texture. - /// - /// Used with `vertex_texcoords` on `Mesh3D`. - /// Currently supports only sRGB(A) textures, ignoring alpha. - /// (meaning that the tensor must have 3 or 4 channels and use the `u8` format) - std::optional albedo_texture; - public: Material() = default; + + Material(std::optional albedo_factor_) + : albedo_factor(albedo_factor_) {} + + Material& operator=(std::optional albedo_factor_) { + albedo_factor = albedo_factor_; + return *this; + } + + Material(uint32_t rgba_) : albedo_factor(rgba_) {} + + Material& operator=(uint32_t rgba_) { + albedo_factor = rgba_; + return *this; + } }; } // namespace rerun::datatypes diff --git a/rerun_py/rerun_sdk/rerun/archetypes/mesh3d.py b/rerun_py/rerun_sdk/rerun/archetypes/mesh3d.py index fb67649a1cd2..b68b2acd52db 100644 --- a/rerun_py/rerun_sdk/rerun/archetypes/mesh3d.py +++ b/rerun_py/rerun_sdk/rerun/archetypes/mesh3d.py @@ -62,6 +62,7 @@ def __attrs_clear__(self) -> None: vertex_colors=None, # type: ignore[arg-type] vertex_texcoords=None, # type: ignore[arg-type] mesh_material=None, # type: ignore[arg-type] + albedo_texture=None, # type: ignore[arg-type] class_ids=None, # type: ignore[arg-type] instance_keys=None, # type: ignore[arg-type] ) @@ -130,6 +131,19 @@ def _clear(cls) -> Mesh3D: # # (Docstring intentionally commented out to hide this field from the docs) + albedo_texture: components.TensorDataBatch | None = field( + metadata={"component": "optional"}, + default=None, + converter=components.TensorDataBatch._optional, # type: ignore[misc] + ) + # Optional albedo texture. + # + # Used with `vertex_texcoords` on `Mesh3D`. + # Currently supports only sRGB(A) textures, ignoring alpha. + # (meaning that the tensor must have 3 or 4 channels and use the `u8` format) + # + # (Docstring intentionally commented out to hide this field from the docs) + class_ids: components.ClassIdBatch | None = field( metadata={"component": "optional"}, default=None, diff --git a/rerun_py/rerun_sdk/rerun/archetypes/mesh3d_ext.py b/rerun_py/rerun_sdk/rerun/archetypes/mesh3d_ext.py index f40e9d8895df..985d76971e0c 100644 --- a/rerun_py/rerun_sdk/rerun/archetypes/mesh3d_ext.py +++ b/rerun_py/rerun_sdk/rerun/archetypes/mesh3d_ext.py @@ -20,6 +20,7 @@ def __init__( vertex_normals: datatypes.Vec3DArrayLike | None = None, vertex_colors: datatypes.Rgba32ArrayLike | None = None, vertex_texcoords: datatypes.Vec2DArrayLike | None = None, + albedo_texture: datatypes.TensorDataLike | None = None, mesh_material: datatypes.MaterialLike | None = None, class_ids: datatypes.ClassIdArrayLike | None = None, instance_keys: components.InstanceKeyArrayLike | None = None, @@ -49,6 +50,10 @@ def __init__( An optional color for each vertex. mesh_material: Optional material properties for the mesh as a whole. + albedo_texture: + Optional albedo texture. Used with `vertex_texcoords` on `Mesh3D`. + Currently supports only sRGB(A) textures, ignoring alpha. + (meaning that the tensor must have 3 or 4 channels and use the `u8` format) class_ids: Optional class Ids for the vertices. The class ID provides colors and labels if not specified explicitly. @@ -68,6 +73,7 @@ def __init__( vertex_normals=vertex_normals, vertex_colors=vertex_colors, vertex_texcoords=vertex_texcoords, + albedo_texture=albedo_texture, mesh_material=mesh_material, class_ids=class_ids, instance_keys=instance_keys, diff --git a/rerun_py/rerun_sdk/rerun/datatypes/material.py b/rerun_py/rerun_sdk/rerun/datatypes/material.py index 0f6f64e2287d..8bb491b01545 100644 --- a/rerun_py/rerun_sdk/rerun/datatypes/material.py +++ b/rerun_py/rerun_sdk/rerun/datatypes/material.py @@ -32,11 +32,7 @@ def _material__albedo_factor__special_field_converter_override( class Material(MaterialExt): """**Datatype**: Material properties of a mesh.""" - def __init__( - self: Any, - albedo_factor: datatypes.Rgba32Like | None = None, - albedo_texture: datatypes.TensorDataLike | None = None, - ): + def __init__(self: Any, albedo_factor: datatypes.Rgba32Like | None = None): """ Create a new instance of the Material datatype. @@ -44,16 +40,10 @@ def __init__( ---------- albedo_factor: Optional color multiplier. - albedo_texture: - Optional albedo texture. - - Used with `vertex_texcoords` on `Mesh3D`. - Currently supports only sRGB(A) textures, ignoring alpha. - (meaning that the tensor must have 3 or 4 channels and use the `u8` format) """ # You can define your own __init__ function as a member of MaterialExt in material_ext.py - self.__attrs_init__(albedo_factor=albedo_factor, albedo_texture=albedo_texture) + self.__attrs_init__(albedo_factor=albedo_factor) albedo_factor: datatypes.Rgba32 | None = field( default=None, converter=_material__albedo_factor__special_field_converter_override @@ -62,15 +52,6 @@ def __init__( # # (Docstring intentionally commented out to hide this field from the docs) - albedo_texture: datatypes.TensorData | None = field(default=None) - # Optional albedo texture. - # - # Used with `vertex_texcoords` on `Mesh3D`. - # Currently supports only sRGB(A) textures, ignoring alpha. - # (meaning that the tensor must have 3 or 4 channels and use the `u8` format) - # - # (Docstring intentionally commented out to hide this field from the docs) - MaterialLike = Material MaterialArrayLike = Union[ @@ -84,134 +65,7 @@ class MaterialType(BaseExtensionType): def __init__(self) -> None: pa.ExtensionType.__init__( - self, - pa.struct( - [ - pa.field("albedo_factor", pa.uint32(), nullable=True, metadata={}), - pa.field( - "albedo_texture", - pa.struct( - [ - pa.field( - "shape", - pa.list_( - pa.field( - "item", - pa.struct( - [ - pa.field("size", pa.uint64(), nullable=False, metadata={}), - pa.field("name", pa.utf8(), nullable=True, metadata={}), - ] - ), - nullable=False, - metadata={}, - ) - ), - nullable=False, - metadata={}, - ), - pa.field( - "buffer", - pa.dense_union( - [ - pa.field("_null_markers", pa.null(), nullable=True, metadata={}), - pa.field( - "U8", - pa.list_(pa.field("item", pa.uint8(), nullable=False, metadata={})), - nullable=False, - metadata={}, - ), - pa.field( - "U16", - pa.list_(pa.field("item", pa.uint16(), nullable=False, metadata={})), - nullable=False, - metadata={}, - ), - pa.field( - "U32", - pa.list_(pa.field("item", pa.uint32(), nullable=False, metadata={})), - nullable=False, - metadata={}, - ), - pa.field( - "U64", - pa.list_(pa.field("item", pa.uint64(), nullable=False, metadata={})), - nullable=False, - metadata={}, - ), - pa.field( - "I8", - pa.list_(pa.field("item", pa.int8(), nullable=False, metadata={})), - nullable=False, - metadata={}, - ), - pa.field( - "I16", - pa.list_(pa.field("item", pa.int16(), nullable=False, metadata={})), - nullable=False, - metadata={}, - ), - pa.field( - "I32", - pa.list_(pa.field("item", pa.int32(), nullable=False, metadata={})), - nullable=False, - metadata={}, - ), - pa.field( - "I64", - pa.list_(pa.field("item", pa.int64(), nullable=False, metadata={})), - nullable=False, - metadata={}, - ), - pa.field( - "F16", - pa.list_(pa.field("item", pa.float16(), nullable=False, metadata={})), - nullable=False, - metadata={}, - ), - pa.field( - "F32", - pa.list_(pa.field("item", pa.float32(), nullable=False, metadata={})), - nullable=False, - metadata={}, - ), - pa.field( - "F64", - pa.list_(pa.field("item", pa.float64(), nullable=False, metadata={})), - nullable=False, - metadata={}, - ), - pa.field( - "JPEG", - pa.list_(pa.field("item", pa.uint8(), nullable=False, metadata={})), - nullable=False, - metadata={}, - ), - pa.field( - "NV12", - pa.list_(pa.field("item", pa.uint8(), nullable=False, metadata={})), - nullable=False, - metadata={}, - ), - pa.field( - "YUY2", - pa.list_(pa.field("item", pa.uint8(), nullable=False, metadata={})), - nullable=False, - metadata={}, - ), - ] - ), - nullable=False, - metadata={}, - ), - ] - ), - nullable=True, - metadata={}, - ), - ] - ), - self._TYPE_NAME, + self, pa.struct([pa.field("albedo_factor", pa.uint32(), nullable=True, metadata={})]), self._TYPE_NAME ) diff --git a/rerun_py/rerun_sdk/rerun/datatypes/material_ext.py b/rerun_py/rerun_sdk/rerun/datatypes/material_ext.py index 00126f0e3fc4..1771142c3c84 100644 --- a/rerun_py/rerun_sdk/rerun/datatypes/material_ext.py +++ b/rerun_py/rerun_sdk/rerun/datatypes/material_ext.py @@ -1,12 +1,9 @@ from __future__ import annotations -import collections -from typing import TYPE_CHECKING, cast +from typing import TYPE_CHECKING import pyarrow as pa -from rerun.datatypes.tensor_data_ext import TensorDataExt - if TYPE_CHECKING: from . import MaterialArrayLike @@ -16,32 +13,19 @@ class MaterialExt: @staticmethod def native_to_pa_array_override(data: MaterialArrayLike, data_type: pa.DataType) -> pa.Array: - from . import Material, Rgba32Type, TensorDataType + from . import Material, Rgba32Type - # If it's a sequence of a single Material, grab the first one - if isinstance(data, collections.abc.Sequence): - if len(data) > 0: - if isinstance(data[0], Material): - if len(data) > 1: - raise ValueError("Materials do not support batches") - data = data[0] - data = cast(Material, data) + if isinstance(data, Material): + data = [data] field_albedo_factors = data_type.field("albedo_factor") - field_albedo_texture = data_type.field("albedo_texture") albedo_factors = pa.array( - [data.albedo_factor.rgba if data.albedo_factor is not None else None], + [datum.albedo_factor.rgba if datum.albedo_factor is not None else None for datum in data], type=Rgba32Type().storage_type, ) - if data.albedo_texture is not None: - albedo_texture = TensorDataExt.native_to_pa_array_override( - data.albedo_texture, TensorDataType().storage_type - ) - else: - albedo_texture = pa.nulls(1, type=TensorDataType().storage_type) return pa.StructArray.from_arrays( - arrays=[albedo_factors, albedo_texture], - fields=[field_albedo_factors, field_albedo_texture], + arrays=[albedo_factors], + fields=[field_albedo_factors], ) From b630dfd3833458805455ffaae12c1e19fb217b76 Mon Sep 17 00:00:00 2001 From: Andreas Reich Date: Thu, 1 Feb 2024 18:04:09 +0100 Subject: [PATCH 24/28] fix c++ build --- rerun_cpp/src/rerun/components/texcoord2d_ext.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rerun_cpp/src/rerun/components/texcoord2d_ext.cpp b/rerun_cpp/src/rerun/components/texcoord2d_ext.cpp index e3af13fd4e63..0fd79fd78e93 100644 --- a/rerun_cpp/src/rerun/components/texcoord2d_ext.cpp +++ b/rerun_cpp/src/rerun/components/texcoord2d_ext.cpp @@ -1,4 +1,4 @@ -#include "texcoord2D.hpp" +#include "texcoord2d.hpp" // Uncomment for better auto-complete while editing the extension. // #define EDIT_EXTENSION From 3ae6938cf2ab1b7cba464b9ff353b7fa86ee67b7 Mon Sep 17 00:00:00 2001 From: Andreas Reich Date: Thu, 1 Feb 2024 18:05:01 +0100 Subject: [PATCH 25/28] fix typo that I didn't cause --- crates/re_space_view/src/data_query_blueprint.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/re_space_view/src/data_query_blueprint.rs b/crates/re_space_view/src/data_query_blueprint.rs index ed3f00f96945..70b81a1caec0 100644 --- a/crates/re_space_view/src/data_query_blueprint.rs +++ b/crates/re_space_view/src/data_query_blueprint.rs @@ -419,7 +419,7 @@ impl DataQueryPropertyResolver<'_> { /// Recursively walk the [`DataResultTree`] and update the [`PropertyOverrides`] for each node. /// /// This will accumulate the group properties at each step down the tree, and then finally merge - /// with individual overrides at the leafs. + /// with individual overrides at the leaves. fn update_overrides_recursive( &self, ctx: &StoreContext<'_>, From 4229db958dc4feea24f8f4720b954e2766d4c296 Mon Sep 17 00:00:00 2001 From: Andreas Reich Date: Thu, 1 Feb 2024 18:26:29 +0100 Subject: [PATCH 26/28] fix too dark mesh - don't convert the trimesh material if you don't have to --- examples/python/raw_mesh/main.py | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/examples/python/raw_mesh/main.py b/examples/python/raw_mesh/main.py index 38c92c293976..91edb1497c75 100755 --- a/examples/python/raw_mesh/main.py +++ b/examples/python/raw_mesh/main.py @@ -68,18 +68,17 @@ def log_scene(scene: trimesh.Scene, node: str, path: str | None = None) -> None: try: albedo_texture = mesh.visual.material.baseColorTexture except Exception: - pass - - try: - colors = mesh.visual.to_color().vertex_colors - if len(colors) == 4: - # If trimesh gives us a single vertex color for the entire mesh, we can interpret that - # as an albedo factor for the whole primitive. - mesh_material = Material(albedo_factor=np.array(colors)) - else: - vertex_colors = colors - except Exception: - pass + # Try vertex colors instead. + try: + colors = mesh.visual.to_color().vertex_colors + if len(colors) == 4: + # If trimesh gives us a single vertex color for the entire mesh, we can interpret that + # as an albedo factor for the whole primitive. + mesh_material = Material(albedo_factor=np.array(colors)) + else: + vertex_colors = colors + except Exception: + pass rr.log( path, From d89fed970c3f73ed0338758b407db69e20266835 Mon Sep 17 00:00:00 2001 From: Clement Rey Date: Fri, 2 Feb 2024 10:14:17 +0100 Subject: [PATCH 27/28] fix python raw_mesh example not picking up albedo factor --- examples/python/raw_mesh/main.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/examples/python/raw_mesh/main.py b/examples/python/raw_mesh/main.py index 91edb1497c75..5bfd677919fa 100755 --- a/examples/python/raw_mesh/main.py +++ b/examples/python/raw_mesh/main.py @@ -67,6 +67,8 @@ def log_scene(scene: trimesh.Scene, node: str, path: str | None = None) -> None: try: albedo_texture = mesh.visual.material.baseColorTexture + if mesh.visual.material.baseColorTexture is None: + raise ValueError() except Exception: # Try vertex colors instead. try: From cf2f654b4ebec5348300032f7ec7c0494d3663e4 Mon Sep 17 00:00:00 2001 From: Clement Rey Date: Fri, 2 Feb 2024 10:26:03 +0100 Subject: [PATCH 28/28] review --- crates/re_space_view_spatial/src/mesh_cache.rs | 3 ++- examples/rust/raw_mesh/src/main.rs | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/re_space_view_spatial/src/mesh_cache.rs b/crates/re_space_view_spatial/src/mesh_cache.rs index 7cbdee860c6e..bfc28fb4443b 100644 --- a/crates/re_space_view_spatial/src/mesh_cache.rs +++ b/crates/re_space_view_spatial/src/mesh_cache.rs @@ -27,7 +27,8 @@ pub enum AnyMesh<'a> { Mesh { mesh: &'a re_types::archetypes::Mesh3D, - /// If any textures are in the mesh's material, they use this hash for texture manager lookup. + /// If there are any textures associated with that mesh (albedo etc), they use this + /// hash for texture manager lookup. texture_key: u64, }, } diff --git a/examples/rust/raw_mesh/src/main.rs b/examples/rust/raw_mesh/src/main.rs index 5b05170e98c3..dc40f77cd18f 100644 --- a/examples/rust/raw_mesh/src/main.rs +++ b/examples/rust/raw_mesh/src/main.rs @@ -272,6 +272,8 @@ fn node_primitives<'data>( let vertex_texcoords = reader.read_tex_coords(0); // TODO(cmc): pick correct set let vertex_texcoords = vertex_texcoords.map(|texcoords| texcoords.into_f32().collect()); + // TODO(cmc): support for albedo textures + GltfPrimitive { albedo_factor, vertex_positions,