enhancement(StreamByImport): allow importId in deep links

3D: support PoseInFrame and PosesInFrame (#4419)

**User-Facing Changes**
None (feature is already documented and was a regression in the new 3d
panel)

**Description**
Fixes #4417

3D: support foxglove Image types (#4425)

**User-Facing Changes**
Added support for Foxglove RawImage, CompressedImage, and
CameraCalibration schemas to the new 3D panel.

**Description**
Closes #4323

packages/studio-base/src/panels/ThreeDeeRender/foxglove.ts

@@ -8,9 +8,24 @@ addFoxgloveSchema(FRAME_TRANSFORM_DATATYPES, "foxglove.FrameTransform");
 export const POINTCLOUD_DATATYPES = new Set<string>();
 addFoxgloveSchema(POINTCLOUD_DATATYPES, "foxglove.PointCloud");
 
+export const RAW_IMAGE_DATATYPES = new Set<string>();
+addFoxgloveSchema(RAW_IMAGE_DATATYPES, "foxglove.RawImage");
+
+export const COMPRESSED_IMAGE_DATATYPES = new Set<string>();
+addFoxgloveSchema(COMPRESSED_IMAGE_DATATYPES, "foxglove.CompressedImage");
+
+export const CAMERA_CALIBRATION_DATATYPES = new Set<string>();
+addFoxgloveSchema(CAMERA_CALIBRATION_DATATYPES, "foxglove.CameraCalibration");
+
 export const SCENE_UPDATE_DATATYPES = new Set<string>();
 addFoxgloveSchema(SCENE_UPDATE_DATATYPES, "foxglove.SceneUpdate");
 
+export const POSE_IN_FRAME_DATATYPES = new Set<string>();
+addFoxgloveSchema(POSE_IN_FRAME_DATATYPES, "foxglove.PoseInFrame");
+
+export const POSES_IN_FRAME_DATATYPES = new Set<string>();
+addFoxgloveSchema(POSES_IN_FRAME_DATATYPES, "foxglove.PosesInFrame");
+
 // Expand a single Foxglove dataType into variations for ROS1 and ROS2 then add
 // them to the given output set
 function addFoxgloveSchema(output: Set<string>, dataType: string): Set<string> {

packages/studio-base/src/panels/ThreeDeeRender/renderables/Cameras.ts

@@ -5,6 +5,7 @@
 import { PinholeCameraModel } from "@foxglove/den/image";
 import Logger from "@foxglove/log";
 import { toNanoSec } from "@foxglove/rostime";
+import { CameraCalibration } from "@foxglove/schemas/schemas/typescript";
 import { SettingsTreeAction, SettingsTreeFields } from "@foxglove/studio";
 import type { RosValue } from "@foxglove/studio-base/players/types";
 import { MutablePoint } from "@foxglove/studio-base/types/Messages";
@@ -14,10 +15,11 @@ import { Renderer } from "../Renderer";
 import { PartialMessage, PartialMessageEvent, SceneExtension } from "../SceneExtension";
 import { SettingsTreeEntry } from "../SettingsManager";
 import { makeRgba, rgbaToCssString, stringToRgba } from "../color";
-import { normalizeHeader } from "../normalizeMessages";
+import { CAMERA_CALIBRATION_DATATYPES } from "../foxglove";
+import { normalizeHeader, normalizeTime } from "../normalizeMessages";
 import {
   CameraInfo,
-  CAMERA_INFO_DATATYPES,
+  CAMERA_INFO_DATATYPES as ROS_CAMERA_INFO_DATATYPES,
   IncomingCameraInfo,
   Marker,
   MarkerAction,
@@ -61,6 +63,7 @@ export type CameraInfoUserData = BaseUserData & {
   settings: LayerSettingsCameraInfo;
   topic: string;
   cameraInfo: CameraInfo | undefined;
+  originalMessage: Record<string, RosValue> | undefined;
   cameraModel: PinholeCameraModel | undefined;
   lines: RenderableLineList | undefined;
 };
@@ -72,23 +75,27 @@ export class CameraInfoRenderable extends Renderable<CameraInfoUserData> {
   }
 
   public override details(): Record<string, RosValue> {
-    return this.userData.cameraInfo ?? {};
+    return this.userData.originalMessage ?? {};
   }
 }
 
 export class Cameras extends SceneExtension<CameraInfoRenderable> {
   public constructor(renderer: Renderer) {
     super("foxglove.Cameras", renderer);
 
-    renderer.addDatatypeSubscriptions(CAMERA_INFO_DATATYPES, this.handleCameraInfo);
+    renderer.addDatatypeSubscriptions(ROS_CAMERA_INFO_DATATYPES, this.handleCameraInfo);
+    renderer.addDatatypeSubscriptions(CAMERA_CALIBRATION_DATATYPES, this.handleCameraInfo);
   }
 
   public override settingsNodes(): SettingsTreeEntry[] {
     const configTopics = this.renderer.config.topics;
     const handler = this.handleSettingsAction;
     const entries: SettingsTreeEntry[] = [];
     for (const topic of this.renderer.topics ?? []) {
-      if (CAMERA_INFO_DATATYPES.has(topic.datatype)) {
+      if (
+        ROS_CAMERA_INFO_DATATYPES.has(topic.datatype) ||
+        CAMERA_CALIBRATION_DATATYPES.has(topic.datatype)
+      ) {
         const config = (configTopics[topic.name] ?? {}) as Partial<LayerSettingsCameraInfo>;
 
         // prettier-ignore
@@ -125,17 +132,25 @@ export class Cameras extends SceneExtension<CameraInfoRenderable> {
     const topicName = path[1]!;
     const renderable = this.renderables.get(topicName);
     if (renderable) {
-      const { cameraInfo, receiveTime } = renderable.userData;
+      const { cameraInfo, receiveTime, originalMessage } = renderable.userData;
       if (cameraInfo) {
         const settings = this.renderer.config.topics[topicName] as
           | Partial<LayerSettingsCameraInfo>
           | undefined;
-        this._updateCameraInfoRenderable(renderable, cameraInfo, receiveTime, settings);
+        this._updateCameraInfoRenderable(
+          renderable,
+          cameraInfo,
+          originalMessage,
+          receiveTime,
+          settings,
+        );
       }
     }
   };
 
-  private handleCameraInfo = (messageEvent: PartialMessageEvent<IncomingCameraInfo>): void => {
+  private handleCameraInfo = (
+    messageEvent: PartialMessageEvent<IncomingCameraInfo | CameraCalibration>,
+  ): void => {
     const topic = messageEvent.topic;
     const cameraInfo = normalizeCameraInfo(messageEvent.message);
     const receiveTime = toNanoSec(messageEvent.receiveTime);
@@ -160,6 +175,7 @@ export class Cameras extends SceneExtension<CameraInfoRenderable> {
         settings,
         topic,
         cameraInfo: undefined,
+        originalMessage: undefined,
         cameraModel: undefined,
         lines: undefined,
       });
@@ -171,6 +187,7 @@ export class Cameras extends SceneExtension<CameraInfoRenderable> {
     this._updateCameraInfoRenderable(
       renderable,
       cameraInfo,
+      messageEvent.message,
       receiveTime,
       renderable.userData.settings,
     );
@@ -179,6 +196,7 @@ export class Cameras extends SceneExtension<CameraInfoRenderable> {
   private _updateCameraInfoRenderable(
     renderable: CameraInfoRenderable,
     cameraInfo: CameraInfo,
+    originalMessage: Record<string, RosValue> | undefined,
     receiveTime: bigint,
     settings: Partial<LayerSettingsCameraInfo> | undefined,
   ): void {
@@ -200,6 +218,7 @@ export class Cameras extends SceneExtension<CameraInfoRenderable> {
     if (!dataEqual) {
       // log.warn(`CameraInfo changed on topic "${topic}", updating rectification model`);
       renderable.userData.cameraInfo = cameraInfo;
+      renderable.userData.originalMessage = originalMessage;
 
       if (cameraInfo.P.length === 12) {
         try {
@@ -431,7 +450,9 @@ function normalizeRegionOfInterest(
   };
 }
 
-function normalizeCameraInfo(message: PartialMessage<IncomingCameraInfo>): CameraInfo {
+function normalizeCameraInfo(
+  message: PartialMessage<IncomingCameraInfo> & PartialMessage<CameraCalibration>,
+): CameraInfo {
   // Handle lowercase field names as well (ROS2 compatibility)
   const D = message.D ?? message.d;
   const K = message.K ?? message.k;
@@ -444,7 +465,10 @@ function normalizeCameraInfo(message: PartialMessage<IncomingCameraInfo>): Camer
   const Plen = P?.length ?? 0;
 
   return {
-    header: normalizeHeader(message.header),
+    header:
+      "timestamp" in message
+        ? { stamp: normalizeTime(message.timestamp), frame_id: message.frame_id ?? "" }
+        : normalizeHeader(message.header),
     height: message.height ?? 0,
     width: message.width ?? 0,
     distortion_model: message.distortion_model ?? "",