feat(dialog-protocol): widget channels + Image codec for the Mosaico toolbox

CLAUDE.md

@@ -89,6 +89,24 @@ Before committing, always run:
 
 Code formatting and linting are enforced via pre-commit hooks (clang-format v17).
 
+## Release Versioning
+
+In **every PR**, proactively raise whether it warrants a new Conan release, and
+propose the version bump rather than waiting to be asked. Pre-1.0 versioning
+convention (`0.MINOR.PATCH`):
+
+- **MINOR** bump (`0.X.0`) — any API or ABI **break**: removing/reordering ABI
+  vtable slots, changing existing struct layouts or function signatures, or any
+  source-incompatible SDK change.
+- **PATCH** bump (`0.x.Y`) — **backward-compatible** changes: tail-appended ABI
+  slots (gated by `struct_size`), additive SDK helpers, bug fixes, docs.
+
+The version is declared in two places that **must stay in sync**: `version` in
+`conanfile.py` and `PJ_PACKAGE_VERSION` in the root `CMakeLists.txt` (also update
+the example tag in the `conanfile.py` docstring). Tagging and pushing the release
+is a separate, explicitly-authorized step — never tag or push a release without
+the user's go-ahead.
+
 ## Instructions Glossary
 
 - **"Read all documentation"** means: find and read every `.md` file in the entire project tree (all subdirectories). Use `find . -name "*.md"` or equivalent. This includes docs in `docs/`, `pj_datastore/docs/`, `pj_plugins/docs/`, and any other location.

CMakeLists.txt

@@ -183,7 +183,7 @@ endif()
 if(PJ_INSTALL_SDK)
   include(CMakePackageConfigHelpers)
 
-  set(PJ_PACKAGE_VERSION "0.3.0")
+  set(PJ_PACKAGE_VERSION "0.3.1")
   set(PJ_PACKAGE_CMAKE_DIR ${CMAKE_INSTALL_LIBDIR}/cmake/plotjuggler_core)
 
   install(EXPORT plotjuggler_coreTargets

conanfile.py

@@ -7,7 +7,7 @@
   plugin_sdk   — umbrella for plugin authors (base + dialog SDK + parser SDK)
   plugin_host  — umbrella for host loaders (data_source/parser/toolbox/dialog)
 
-A consuming Conan recipe declares e.g. `plotjuggler_core/0.3.0` and then:
+A consuming Conan recipe declares e.g. `plotjuggler_core/0.3.1` and then:
 
     find_package(plotjuggler_core REQUIRED COMPONENTS plugin_sdk)
     target_link_libraries(my_plugin PRIVATE plotjuggler_core::plugin_sdk)
@@ -27,7 +27,7 @@
 
 class PlotjugglerCoreConan(ConanFile):
     name = "plotjuggler_core"
-    version = "0.3.0"
+    version = "0.3.1"
     license = "MIT"
     url = "https://github.com/PlotJuggler/plotjuggler_core"
     description = "C++20 foundation libraries for PlotJuggler: storage engine, plugin SDK, plugin host loaders."

pj_base/include/pj_base/plugin_data_api.h

@@ -183,6 +183,15 @@ typedef struct {
   uint32_t id;
 } PJ_field_handle_t;
 
+/* ABI-FROZEN: layout permanent; changes = v5 break.
+ *
+ * Declared up here (rather than next to the object-store host vtables below)
+ * because PJ_toolbox_host_vtable_t now references it: toolbox plugins can
+ * register/push canonical media topics directly through the toolbox host. */
+typedef struct {
+  uint32_t id; /* 0 == invalid handle */
+} PJ_object_topic_handle_t;
+
 /* ==========================================================================
  * Protocol v4 core types
  *
@@ -489,6 +498,26 @@ typedef struct PJ_toolbox_host_vtable_t {
   bool (*read_series_arrow)(
       void* ctx, PJ_field_handle_t field, struct ArrowSchema* out_schema, struct ArrowArray* out_array,
       PJ_error_t* out_error) PJ_NOEXCEPT;
+
+  /* [main-thread] Register an object topic for media payloads (images, point
+   * clouds, annotations). The topic is namespaced under the data source
+   * previously created via create_data_source. `metadata_json` is opaque to
+   * the host — viewers and parsers read it to pick a renderer (e.g.
+   * {"object_type":"image"} for the MediaViewerWidget). Returns false (with
+   * out_error populated) if the source handle is unknown or a topic with
+   * this name already exists for the data source. */
+  bool (*register_object_topic)(
+      void* ctx, PJ_data_source_handle_t source, PJ_string_view_t topic_name, PJ_string_view_t metadata_json,
+      PJ_object_topic_handle_t* out_handle, PJ_error_t* out_error) PJ_NOEXCEPT;
+
+  /* [main-thread] Eager push of an object payload — host copies the bytes
+   * into ObjectStore. Appropriate for both small messages and the
+   * one-shot media writes that toolbox plugins typically perform; lazy
+   * push is not offered on the toolbox surface (plugin holds the bytes
+   * already by the time it calls). */
+  bool (*push_owned_object)(
+      void* ctx, PJ_object_topic_handle_t topic, int64_t timestamp_ns, const uint8_t* data, size_t size,
+      PJ_error_t* out_error) PJ_NOEXCEPT;
 } PJ_toolbox_host_vtable_t;
 
 typedef struct {
@@ -511,10 +540,8 @@ typedef struct {
  * `pj.parser_object_write.v1` in addition to `pj.parser_write.v1`.
  * ========================================================================== */
 
-/* ABI-FROZEN: layout permanent; changes = ABI break. */
-typedef struct {
-  uint32_t id; /* 0 == invalid handle */
-} PJ_object_topic_handle_t;
+/* PJ_object_topic_handle_t is declared earlier (next to PJ_field_handle_t)
+ * because the toolbox host vtable references it as well. */
 
 /* Lazy-fetch callback type. Invoked by the host on-demand when a consumer
  * reads an entry stored via push_lazy. On success the plugin populates

pj_base/include/pj_base/sdk/plugin_data_api.hpp

@@ -1178,12 +1178,63 @@ class ToolboxHostView {
     return MaterializedSeriesView(std::move(schema), std::move(array));
   }
 
+  /// Register an object topic for media payloads (images, point clouds,
+  /// annotations) under a previously created data source. `metadata_json`
+  /// is opaque to the store and retained verbatim; viewers and parsers
+  /// read it to pick a renderer.
+  ///
+  /// Returns `unexpected` if the host predates this ABI slot — older hosts
+  /// can be detected via the ABI's `PJ_HAS_TAIL_SLOT` macro; in this
+  /// view's terms, the function pointer will be null.
+  [[nodiscard]] Expected<ObjectTopicHandle> registerObjectTopic(
+      DataSourceHandle source, std::string_view name, std::string_view metadata_json) const {
+    if (!valid()) {
+      return unexpected("toolbox host is not bound");
+    }
+    if (!hasTailSlot(offsetof(PJ_toolbox_host_vtable_t, register_object_topic), host_.vtable->register_object_topic)) {
+      return unexpected("toolbox host does not support object topics (older host)");
+    }
+    ObjectTopicHandle handle{};
+    PJ_error_t err{};
+    if (!host_.vtable->register_object_topic(
+            host_.ctx, source, toAbiString(name), toAbiString(metadata_json), &handle, &err)) {
+      return unexpected(errorToString(err));
+    }
+    return handle;
+  }
+
+  /// Eager push of an object payload — host copies the bytes into its own
+  /// storage. Returns `unexpected` on older hosts that don't expose the
+  /// slot (see registerObjectTopic).
+  [[nodiscard]] Status pushOwnedObject(ObjectTopicHandle topic, Timestamp ts, Span<const uint8_t> payload) const {
+    if (!valid()) {
+      return unexpected("toolbox host is not bound");
+    }
+    if (!hasTailSlot(offsetof(PJ_toolbox_host_vtable_t, push_owned_object), host_.vtable->push_owned_object)) {
+      return unexpected("toolbox host does not support object payloads (older host)");
+    }
+    PJ_error_t err{};
+    if (!host_.vtable->push_owned_object(host_.ctx, topic, ts, payload.data(), payload.size(), &err)) {
+      return unexpected(errorToString(err));
+    }
+    return okStatus();
+  }
+
   [[nodiscard]] const PJ_toolbox_host_t& raw() const noexcept {
     return host_;
   }
 
  private:
   PJ_toolbox_host_t host_{};
+
+  /// Tail-slot guard mirroring PJ_HAS_TAIL_SLOT from the C ABI: the host's
+  /// struct_size must be large enough to cover the field, AND the slot
+  /// must be non-null. Templated on the function-pointer type so the
+  /// sizeof check stays accurate without naming the typedef.
+  template <class Fn>
+  [[nodiscard]] bool hasTailSlot(std::size_t field_offset, Fn fn) const noexcept {
+    return host_.vtable != nullptr && host_.vtable->struct_size >= field_offset + sizeof(Fn) && fn != nullptr;
+  }
 };
 
 // ---------------------------------------------------------------------------

pj_base/tests/abi_layout_sentinels_test.cpp

@@ -172,7 +172,9 @@ static_assert(offsetof(PJ_toolbox_host_vtable_t, struct_size) == 4, "toolbox hos
 static_assert(offsetof(PJ_toolbox_host_vtable_t, append_bound_record) == 40, "toolbox host baseline pinned");
 static_assert(offsetof(PJ_toolbox_host_vtable_t, append_arrow_stream) == 48, "toolbox host bulk slot pinned");
 static_assert(offsetof(PJ_toolbox_host_vtable_t, read_series_arrow) == 64, "toolbox host read slot pinned");
-static_assert(sizeof(PJ_toolbox_host_vtable_t) == 72, "Toolbox host size");
+static_assert(offsetof(PJ_toolbox_host_vtable_t, register_object_topic) == 72, "toolbox host object-topic slot pinned");
+static_assert(offsetof(PJ_toolbox_host_vtable_t, push_owned_object) == 80, "toolbox host object-push tail slot pinned");
+static_assert(sizeof(PJ_toolbox_host_vtable_t) == 88, "Toolbox host size (update deliberately on append)");
 
 // --- ABI version symbol ------------------------------------------------------
 static_assert(PJ_ABI_VERSION == 5, "v5 ABI version");

pj_datastore/include/pj_datastore/plugin_data_host.hpp

@@ -117,7 +117,11 @@ class DatastoreParserObjectWriteHost {
 
 class DatastoreToolboxHost {
  public:
-  explicit DatastoreToolboxHost(DataEngine& engine);
+  /// Construct with both an engine (scalar/Arrow column writes) and an
+  /// object store (canonical media payloads — images, point clouds,
+  /// annotations). The two are independent storage backends; toolbox
+  /// plugins write into one or both via the same host fat pointer.
+  DatastoreToolboxHost(DataEngine& engine, ObjectStore& object_store);
   ~DatastoreToolboxHost();
 
   DatastoreToolboxHost(const DatastoreToolboxHost&) = delete;

pj_datastore/src/plugin_data_host.cpp

@@ -921,8 +921,17 @@ struct DatastoreParserWriteHostState {
 };
 
 struct DatastoreToolboxHostState {
-  explicit DatastoreToolboxHostState(DataEngine& engine) : core(engine) {}
+  DatastoreToolboxHostState(DataEngine& engine, ObjectStore& store) : core(engine), object_store(store) {}
   ToolboxCore core;
+  // Toolbox plugins share the session's object store; the host holds a
+  // reference so register_object_topic + push_owned_object can forward
+  // without going back through the engine.
+  ObjectStore& object_store;
+  std::string object_last_error;
+
+  void setObjectError(std::string msg) {
+    object_last_error = std::move(msg);
+  }
 };
 
 struct DatastoreSourceObjectWriteHostState {
@@ -1199,6 +1208,75 @@ bool toolboxReadSeriesArrow(
   });
 }
 
+bool toolboxRegisterObjectTopic(
+    void* ctx, DataSourceHandle source, PJ_string_view_t topic_name, PJ_string_view_t metadata_json,
+    PJ_object_topic_handle_t* out_handle, PJ_error_t* out_error) noexcept {
+  auto* impl = static_cast<DatastoreToolboxHostState*>(ctx);
+  if (out_handle == nullptr) {
+    propagateError(out_error, "out_handle must not be null");
+    return false;
+  }
+  // Validate the source handle against the engine — same check used by
+  // scalar ensureTopic so the toolbox can't register a topic against a
+  // dataset that doesn't exist.
+  if (impl->core.engine_.getDataset(source.id) == nullptr) {
+    impl->setObjectError(fmt::format("data source {} not found", source.id));
+    propagateError(out_error, impl->object_last_error.c_str());
+    return false;
+  }
+  try {
+    ObjectTopicDescriptor desc{};
+    desc.dataset_id = source.id;
+    desc.topic_name = std::string(toStringView(topic_name));
+    desc.metadata_json = std::string(toStringView(metadata_json));
+    auto result = impl->object_store.registerTopic(desc);
+    if (!result) {
+      impl->setObjectError(result.error());
+      propagateError(out_error, impl->object_last_error.c_str());
+      return false;
+    }
+    out_handle->id = result->id;
+    impl->object_last_error.clear();
+    return true;
+  } catch (const std::exception& e) {
+    impl->setObjectError(e.what());
+    propagateError(out_error, impl->object_last_error.c_str());
+    return false;
+  } catch (...) {
+    impl->setObjectError("registerObjectTopic: unknown exception");
+    propagateError(out_error, impl->object_last_error.c_str());
+    return false;
+  }
+}
+
+bool toolboxPushOwnedObject(
+    void* ctx, PJ_object_topic_handle_t topic, int64_t timestamp_ns, const uint8_t* data, std::size_t size,
+    PJ_error_t* out_error) noexcept {
+  auto* impl = static_cast<DatastoreToolboxHostState*>(ctx);
+  try {
+    std::vector<uint8_t> bytes;
+    if (data != nullptr && size > 0) {
+      bytes.assign(data, data + size);
+    }
+    auto result = impl->object_store.pushOwned(ObjectTopicId{topic.id}, timestamp_ns, std::move(bytes));
+    if (!result) {
+      impl->setObjectError(result.error());
+      propagateError(out_error, impl->object_last_error.c_str());
+      return false;
+    }
+    impl->object_last_error.clear();
+    return true;
+  } catch (const std::exception& e) {
+    impl->setObjectError(e.what());
+    propagateError(out_error, impl->object_last_error.c_str());
+    return false;
+  } catch (...) {
+    impl->setObjectError("pushOwnedObject: unknown exception");
+    propagateError(out_error, impl->object_last_error.c_str());
+    return false;
+  }
+}
+
 /// RAII holder for the plugin-owned `fetch_ctx` passed to push_lazy. Stores
 /// the destroy callback pointer and the ctx value; destroys both on drop.
 /// Wrapped in a shared_ptr so the lambda that ObjectStore stores remains
@@ -1597,6 +1675,8 @@ const PJ_toolbox_host_vtable_t kToolboxVTable = {
     toolboxAppendArrowStream,
     toolboxAcquireCatalogSnapshot,
     toolboxReadSeriesArrow,
+    toolboxRegisterObjectTopic,
+    toolboxPushOwnedObject,
 };
 
 const PJ_object_write_host_vtable_t kSourceObjectWriteVTable = {
@@ -1647,8 +1727,8 @@ void DatastoreParserWriteHost::flushPending() {
   state_->core.flushPending();
 }
 
-DatastoreToolboxHost::DatastoreToolboxHost(DataEngine& engine)
-    : state_(std::make_unique<DatastoreToolboxHostState>(engine)) {}
+DatastoreToolboxHost::DatastoreToolboxHost(DataEngine& engine, ObjectStore& object_store)
+    : state_(std::make_unique<DatastoreToolboxHostState>(engine, object_store)) {}
 DatastoreToolboxHost::~DatastoreToolboxHost() = default;
 DatastoreToolboxHost::DatastoreToolboxHost(DatastoreToolboxHost&&) noexcept = default;
 DatastoreToolboxHost& DatastoreToolboxHost::operator=(DatastoreToolboxHost&&) noexcept = default;

pj_datastore/tests/arrow_stream_round_trip_test.cpp

@@ -26,6 +26,7 @@
 #include "pj_base/type_tree.hpp"
 #include "pj_base/types.hpp"
 #include "pj_datastore/engine.hpp"
+#include "pj_datastore/object_store.hpp"
 #include "pj_datastore/plugin_data_host.hpp"
 
 namespace PJ {
@@ -152,7 +153,8 @@ TEST(ArrowStreamRoundTripTest, WriteViaAppendArrowStreamReadViaReadSeriesArrow)
   write_host.flushPending();
 
   // Catalog snapshot — look up the field handle for "value".
-  DatastoreToolboxHost tb_host(engine);
+  ObjectStore object_store;
+  DatastoreToolboxHost tb_host(engine, object_store);
   auto tb_vtable = tb_host.raw();
 
   PJ_catalog_snapshot_t snapshot{};
@@ -239,7 +241,8 @@ TEST(ArrowStreamRoundTripTest, ParserWriteHostAppendArrowStreamWritesBoundTopic)
 
   parser_write_host.flushPending();
 
-  DatastoreToolboxHost tb_host(engine);
+  ObjectStore object_store;
+  DatastoreToolboxHost tb_host(engine, object_store);
   auto tb_vtable = tb_host.raw();
 
   PJ_catalog_snapshot_t snapshot{};

pj_datastore/tests/plugin_host_read_test.cpp

@@ -13,6 +13,7 @@
 #include "pj_base/sdk/plugin_data_api.hpp"
 #include "pj_base/type_tree.hpp"
 #include "pj_datastore/engine.hpp"
+#include "pj_datastore/object_store.hpp"
 #include "pj_datastore/plugin_data_host.hpp"
 #include "pj_datastore/writer.hpp"
 
@@ -23,7 +24,8 @@ using namespace PJ::sdk;
 
 struct Fixture {
   DataEngine engine;
-  DatastoreToolboxHost toolbox_impl{engine};
+  ObjectStore object_store;
+  DatastoreToolboxHost toolbox_impl{engine, object_store};
   ToolboxHostView toolbox{toolbox_impl.raw()};
 };
 

pj_datastore/tests/plugin_host_write_test.cpp

@@ -14,6 +14,7 @@
 #include "nanoarrow/nanoarrow_ipc.h"
 #include "pj_base/sdk/plugin_data_api.hpp"
 #include "pj_datastore/engine.hpp"
+#include "pj_datastore/object_store.hpp"
 #include "pj_datastore/plugin_data_host.hpp"
 
 namespace PJ {
@@ -50,7 +51,8 @@ std::vector<uint8_t> serializeToIpc(ArrowSchema* schema, ArrowArray* array) {
 
 struct Fixture {
   DataEngine engine;
-  DatastoreToolboxHost toolbox_impl{engine};
+  ObjectStore object_store;
+  DatastoreToolboxHost toolbox_impl{engine, object_store};
   ToolboxHostView toolbox{toolbox_impl.raw()};
 };