feat: messages format renderer

packages/table/src/lib/views/cells/cell-contents/ImageContent.svelte

@@ -62,7 +62,7 @@
       return null;
     }
     if (typeof img == "string") {
-      if (img.startsWith("data:")) {
+      if (img.startsWith("data:") || img.startsWith("http://") || img.startsWith("https://")) {
         return img;
       } else {
         let type = detectImageType(base64Decode(img));

packages/viewer/package.json

@@ -48,7 +48,7 @@
     "@sveltejs/vite-plugin-svelte": "^6.1.0",
     "@tailwindcss/forms": "^0.5.10",
     "@tailwindcss/postcss": "^4.1.11",
-    "@tailwindcss/typography": "0.5.19",
+    "@tailwindcss/typography": "^0.5.19",
     "@tsconfig/svelte": "^5.0.4",
     "@types/d3-array": "^3.2.1",
     "@types/d3-color": "^3.1.3",

packages/viewer/src/assets/chart_icons.ts

@@ -1,6 +1,7 @@
 // Copyright (c) 2025 Apple Inc. Licensed under MIT License.
 
 import chart_boxplot from "./chart-boxplot.svg?raw";
+import chart_content_viewer from "./chart-content-viewer.svg?raw";
 import chart_embedding from "./chart-embedding.svg?raw";
 import chart_h_bar from "./chart-h-bar.svg?raw";
 import chart_heatmap from "./chart-heatmap.svg?raw";
@@ -20,4 +21,5 @@ export const chartIcons: Record<string, string> = {
   "chart-spec": chart_spec,
   "chart-predicates": chart_predicates,
   "chart-markdown": chart_markdown,
+  "chart-content-viewer": chart_content_viewer,
 };

packages/viewer/src/charts/basic/ContentViewer.svelte

@@ -0,0 +1,40 @@
+<!-- Copyright (c) 2025 Apple Inc. Licensed under MIT License. -->
+<script lang="ts">
+  import * as SQL from "@uwdata/mosaic-sql";
+
+  import ContentRenderer from "../../renderers/ContentRenderer.svelte";
+  import Container from "../common/Container.svelte";
+
+  import type { ChartViewProps } from "../chart.js";
+  import type { ContentViewerSpec } from "./types.js";
+
+  let { context, width, height, spec }: ChartViewProps<ContentViewerSpec, {}> = $props();
+
+  let { columnStyles } = context;
+
+  let value = $state<any>(undefined);
+
+  $effect.pre(() =>
+    context.highlight.subscribe(async (id) => {
+      if (id == null) {
+        return;
+      }
+      let r = await context.coordinator.query(
+        SQL.Query.from(context.table)
+          .select({ value: spec.field })
+          .where(SQL.eq(SQL.column(context.id, context.table), SQL.literal(id))),
+      );
+      value = r.get(0)?.value ?? undefined;
+    }),
+  );
+
+  let renderer = $derived($columnStyles[spec.field]?.renderer);
+</script>
+
+<Container width={width} height={height} scrollY={true}>
+  {#if value != null}
+    <ContentRenderer value={value} renderer={renderer} />
+  {:else}
+    (null)
+  {/if}
+</Container>

packages/viewer/src/charts/basic/types.ts

@@ -72,3 +72,9 @@ export interface MarkdownSpec {
   title?: string;
   content: string;
 }
+
+export interface ContentViewerSpec {
+  type: "content-viewer";
+  title?: string;
+  field: string;
+}

packages/viewer/src/charts/builder/Builder.svelte

@@ -85,9 +85,9 @@
     return input;
   }
 
-  function getField(name: string): { name: string; type: "continuous" | "discrete" | "discrete[]" } | null {
+  function getField(name: string): { name: string; type: "continuous" | "discrete" | "discrete[]" | "unknown" } | null {
     let c = columns.find((x) => x.name == name);
-    if (c == null || c.jsType == null) {
+    if (c == null) {
       return null;
     }
     switch (c.jsType) {
@@ -107,13 +107,16 @@
           type: "discrete[]",
         };
       default:
-        return null;
+        return {
+          name: c.name,
+          type: "unknown",
+        };
     }
   }
 
   function filteredColumns(columns: ColumnDesc[], types: JSType[] | null | undefined): ColumnDesc[] {
     if (types == null) {
-      return columns.filter((c) => c.jsType != null);
+      return columns;
     }
     return columns.filter((c) => c.jsType != null && types.indexOf(c.jsType) >= 0);
   }

packages/viewer/src/charts/chart_types.ts

@@ -3,6 +3,7 @@
 import type { Component } from "svelte";
 
 import BoxPlot from "./basic/BoxPlot.svelte";
+import ContentViewer from "./basic/ContentViewer.svelte";
 import CountPlot from "./basic/CountPlot.svelte";
 import CountPlotList from "./basic/CountPlotList.svelte";
 import Histogram from "./basic/Histogram.svelte";
@@ -17,6 +18,7 @@ import Table from "./table/Table.svelte";
 
 import type {
   BoxPlotSpec,
+  ContentViewerSpec,
   CountPlotSpec,
   Histogram2DSpec,
   HistogramSpec,
@@ -99,6 +101,7 @@ registerChartType("embedding", Embedding);
 registerChartType("predicates", Predicates);
 registerChartType("table", Table);
 registerChartType("markdown", Markdown, { supportsEditMode: true });
+registerChartType("content-viewer", ContentViewer);
 
 // Spec type for all builtin chart types
 export type BuiltinChartSpec =
@@ -110,7 +113,8 @@ export type BuiltinChartSpec =
   | PredicatesSpec
   | EmbeddingSpec
   | TableSpec
-  | MarkdownSpec;
+  | MarkdownSpec
+  | ContentViewerSpec;
 
 // Chart builders
 
@@ -258,6 +262,18 @@ registerChartBuilder({
   }),
 });
 
+registerChartBuilder({
+  icon: "chart-content-viewer",
+  description: "Create a view that displays a given field's content for the last selected point",
+  preview: false,
+  ui: [{ field: { key: "field", label: "Field", required: true } }] as const,
+  create: ({ field }): ContentViewerSpec | undefined => ({
+    type: "content-viewer",
+    title: field.name,
+    field: field.name,
+  }),
+});
+
 registerChartBuilder({
   icon: "chart-spec",
   description: "Create a chart with custom spec",

packages/viewer/src/layouts/dashboard/DashboardChartPanel.svelte

@@ -129,7 +129,7 @@
   }}
 >
   <div
-    class="absolute left-0 right-0 top-0 bottom-0 bg-white dark:bg-black rounded-md overflow-hidden flex flex-col shadow-md border border-slate-300 dark:border-slate-700"
+    class="absolute left-0 right-0 top-0 bottom-0 bg-white dark:bg-black rounded-md overflow-hidden flex flex-col shadow-md border border-slate-300 dark:border-slate-700 group"
   >
     <div class="p-2 bg-slate-100 dark:bg-slate-900 flex cursor-move select-none">
       <div
@@ -143,7 +143,7 @@
       >
         {spec.title}
       </div>
-      <div class="flex gap-1">
+      <div class="flex gap-1 sm:opacity-0 group-hover:opacity-100">
         <CornerButton
           icon={chartMode == "edit" ? IconCheck : IconEdit}
           onClick={() => {

packages/viewer/src/renderers/index.ts

@@ -5,6 +5,7 @@ import type { CustomCell } from "@embedding-atlas/table";
 import { ImageRenderer } from "./image.js";
 import { JSONRenderer, safeJSONStringify } from "./json.js";
 import { MarkdownRenderer } from "./markdown.js";
+import { MessagesRenderer } from "./messages.js";
 import { URLRenderer } from "./url.js";
 
 /** A type describing how to display a column in the table, tooltip, and search results */
@@ -22,13 +23,15 @@ export let textRendererClasses: Record<string, any> = {
   image: ImageRenderer,
   url: URLRenderer,
   json: JSONRenderer,
+  messages: MessagesRenderer,
 };
 
 export let renderersList = [
   { renderer: "markdown", label: "Markdown" },
   { renderer: "image", label: "Image" },
   { renderer: "url", label: "Link" },
   { renderer: "json", label: "JSON" },
+  { renderer: "messages", label: "Messages" },
 ];
 
 export function getRenderer(value: string | CustomCell | null | undefined) {

packages/viewer/src/renderers/messages.ts

@@ -0,0 +1,159 @@
+// Copyright (c) 2025 Apple Inc. Licensed under MIT License.
+
+import { marked } from "marked";
+
+import { safeJSONStringify } from "./json.js";
+
+type ResolvedContent = { type: "text"; text: string } | { type: "image"; imageUrl: string };
+
+interface ResolvedMessage {
+  role: string;
+  content: ResolvedContent[];
+  remaining: any;
+}
+
+function resolveContent(value: any, output: ResolvedContent[]) {
+  if (value == null) {
+    return [];
+  }
+  if (typeof value == "string" && value.length > 0) {
+    output.push({ type: "text", text: value });
+  } else if (value instanceof Array) {
+    for (let item of value) {
+      resolveContent(item, output);
+    }
+  } else if (typeof value == "object") {
+    if (value.text != undefined && typeof value.text == "string" && value.text.length > 0) {
+      output.push({ type: "text", text: value.text });
+    }
+    if (value.image != undefined && typeof value.image == "string" && value.image.length > 0) {
+      output.push({ type: "image", imageUrl: value.image });
+    }
+    if (value.image_url != undefined && typeof value.image_url == "string" && value.image_url.length > 0) {
+      output.push({ type: "image", imageUrl: value.image_url });
+    }
+  }
+}
+
+function resolveMessage(item: any): ResolvedMessage | undefined {
+  if (item == null || typeof item != "object") {
+    return;
+  }
+
+  let role = item.role?.toString() ?? "(null)";
+  let content: ResolvedContent[] = [];
+  let remaining = { ...item };
+
+  delete remaining["role"];
+
+  for (let key of ["content", "contents"]) {
+    let value = item[key];
+    if (value != null) {
+      resolveContent(value, content);
+      delete remaining[key];
+    }
+  }
+
+  for (let key in remaining) {
+    if (remaining[key] == null) {
+      delete remaining[key];
+    }
+  }
+
+  return { role, content, remaining };
+}
+
+export class MessagesRenderer {
+  element: HTMLDivElement;
+
+  constructor(element: HTMLDivElement, props: { value: any }) {
+    this.element = element;
+    this.update(props);
+  }
+
+  update(props: { value: any }) {
+    let div = document.createElement("div");
+    if (props.value == null) {
+      div.innerText = "(null)";
+    } else if (typeof props.value == "string") {
+      div.innerText = props.value;
+    } else if (props.value instanceof Array) {
+      for (let item of props.value) {
+        let resolved = resolveMessage(item);
+        if (resolved == undefined) {
+          continue;
+        }
+        div.appendChild(
+          E("div", {
+            class: "mb-1 flex flex-col gap-1",
+            children: [
+              // Role
+              E("div", {
+                class:
+                  "text-xs font-bold border-b text-gray-400 dark:text-gray-500 border-gray-400 dark:border-gray-500",
+                innerText: resolved.role,
+              }),
+              // Content
+              ...resolved.content.map((c) => {
+                if (c.type == "text") {
+                  return E("div", {
+                    class: "prose dark:prose-invert max-w-none",
+                    innerHTML: marked(c.text, { async: false }),
+                  });
+                } else if (c.type == "image") {
+                  return E("img", {
+                    class: "max-w-120 max-h-120 object-contain",
+                    attrs: {
+                      src: c.imageUrl,
+                    },
+                  });
+                }
+              }),
+
+              // Remaining Properties
+              Object.keys(resolved.remaining).length > 0
+                ? E("pre", {
+                    class:
+                      "border rounded-md p-1 bg-gray-100 border-gray-200 dark:bg-gray-800 dark:border-gray-700 text-xs",
+                    innerText: safeJSONStringify(resolved.remaining, 2),
+                  })
+                : null,
+            ],
+          }),
+        );
+      }
+    }
+    this.element.replaceChildren(div);
+  }
+}
+
+function E(
+  tag: string,
+  options: {
+    innerText?: string;
+    innerHTML?: string;
+    class?: string;
+    attrs?: Record<string, any>;
+    children?: (HTMLElement | null | undefined)[];
+  },
+) {
+  let e = document.createElement(tag);
+  if (options.innerText != null) {
+    e.innerText = options.innerText;
+  }
+  if (options.innerHTML != null) {
+    e.innerHTML = options.innerHTML;
+  }
+  if (options.class != null) {
+    e.className = options.class;
+  }
+  if (options.attrs != null) {
+    for (let [key, value] of Object.entries(options.attrs)) {
+      e.setAttribute(key, value);
+    }
+  }
+  if (options.children != null) {
+    e.replaceChildren(...options.children.filter((x) => x != null));
+  }
+  return e;
+}