From ad9ccddd51f2f7f2ea9fa04ec1b71d4b2ffa16c3 Mon Sep 17 00:00:00 2001 From: mdheller Date: Mon, 3 Aug 2026 06:22:38 -0400 Subject: [PATCH] surface: crossmodal-embeddings (D6 text<->image shared vector space) Text+image co-embedded into one space, matched crossmodally; image encoder handed off from imagelab#multimodal-handoff (provenance kept, no mutable image-model state -> lab boundary preserved). Additive: repo validate.py still green. --- docs/CROSSMODAL_SURFACE.md | 2 ++ examples/crossmodal-query.example.json | 7 +++++ lab.manifest.json | 7 ++++- schemas/crossmodal-embedding.schema.json | 35 ++++++++++++++++++++++++ tools/validate_crossmodal.py | 25 +++++++++++++++++ 5 files changed, 75 insertions(+), 1 deletion(-) create mode 100644 docs/CROSSMODAL_SURFACE.md create mode 100644 examples/crossmodal-query.example.json create mode 100644 schemas/crossmodal-embedding.schema.json create mode 100644 tools/validate_crossmodal.py diff --git a/docs/CROSSMODAL_SURFACE.md b/docs/CROSSMODAL_SURFACE.md new file mode 100644 index 0000000..d0c3124 --- /dev/null +++ b/docs/CROSSMODAL_SURFACE.md @@ -0,0 +1,2 @@ +# Crossmodal embeddings surface (embeddinglab) +D6: text and image inputs embedded into **one shared vector space** and matched crossmodally (a text query retrieves images; an image query retrieves text). embeddinglab owns the shared space (`embeddings`/`semantic-indexing` surfaces); the **image encoder is a handoff from `imagelab#multimodal-handoff`** — the `image_encoding_handoff` receipt keeps encoder provenance, so embeddinglab never carries mutable image-model state (lab-boundary preserved). Feeds the marketplace/search deliverable-matching (crossmodal product/skill search). Contract: `schemas/crossmodal-embedding.schema.json`; validate `python3 tools/validate_crossmodal.py`. diff --git a/examples/crossmodal-query.example.json b/examples/crossmodal-query.example.json new file mode 100644 index 0000000..e6c3295 --- /dev/null +++ b/examples/crossmodal-query.example.json @@ -0,0 +1,7 @@ +{ "query": {"mode": "combined", "text": "red high-heeled shoe", + "image_ref": "cid:bafyimageexample", "image_encoding_handoff": "imagelab:handoff:rcpt-001"}, + "unified_space_ref": "embeddinglab:space:text-image-v0", "top_k": 3, + "results": [ + {"id": "product:shoe-42", "modality": "image", "score": 0.95}, + {"id": "listing:red-heels", "modality": "text", "score": 0.92}, + {"id": "product:shoe-17", "modality": "image", "score": 0.89}] } diff --git a/lab.manifest.json b/lab.manifest.json index 4658e20..8a91aba 100644 --- a/lab.manifest.json +++ b/lab.manifest.json @@ -9,7 +9,8 @@ "rerankers", "hybrid-retrieval-preparation", "vector-evaluation", - "semantic-indexing" + "semantic-indexing", + "crossmodal-embeddings" ], "serviceManifests": [ "service-manifest/functional-service.v1.json" @@ -25,5 +26,9 @@ "allowed": true, "carriesMutableModelState": false, "clientRefRequired": true + }, + "imageHandoff": { + "seam": "SociOS-Linux/imagelab#multimodal-handoff", + "note": "image encoder lives in imagelab; embeddinglab owns the shared text+image vector space" } } diff --git a/schemas/crossmodal-embedding.schema.json b/schemas/crossmodal-embedding.schema.json new file mode 100644 index 0000000..854ce56 --- /dev/null +++ b/schemas/crossmodal-embedding.schema.json @@ -0,0 +1,35 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://schemas.socioprophet.org/embedding/crossmodal-embedding.v0.json", + "title": "CrossmodalQuery", + "description": "D6 (crossmodal search): text and/or image inputs embedded into ONE shared vector space, matched crossmodally (text<->image). The shared space is embeddinglab's; the image encoding is handed off from imagelab#multimodal-handoff (provenance kept). Lab-only contract: emits a governed match, never self-promotes a model.", + "type": "object", + "additionalProperties": false, + "required": ["query", "unified_space_ref", "results"], + "properties": { + "query": { + "type": "object", "additionalProperties": false, "required": ["mode"], + "description": "at least one of text/image_ref must be present, consistent with mode", + "properties": { + "mode": {"type": "string", "enum": ["text", "image", "combined"]}, + "text": {"type": ["string", "null"]}, + "image_ref": {"type": ["string", "null"], "description": "content-addressed image ref or data: URI"}, + "image_encoding_handoff": {"type": ["string", "null"], "description": "imagelab multimodal-handoff receipt id (image-encoder provenance)"} + }, + "allOf": [ + {"if": {"properties": {"mode": {"const": "text"}}}, "then": {"required": ["text"]}}, + {"if": {"properties": {"mode": {"const": "image"}}}, "then": {"required": ["image_ref"]}}, + {"if": {"properties": {"mode": {"const": "combined"}}}, "then": {"required": ["text", "image_ref"]}} + ] + }, + "unified_space_ref": {"type": "string", "description": "embeddinglab shared vector-space id (text+image co-embedded)"}, + "top_k": {"type": "integer", "minimum": 1, "maximum": 200, "default": 20}, + "results": {"type": "array", "items": { + "type": "object", "additionalProperties": false, "required": ["id", "modality", "score"], + "properties": { + "id": {"type": "string"}, + "modality": {"type": "string", "enum": ["text", "image"]}, + "score": {"type": "number", "minimum": 0, "maximum": 1} + }}} + } +} diff --git a/tools/validate_crossmodal.py b/tools/validate_crossmodal.py new file mode 100644 index 0000000..acfccf0 --- /dev/null +++ b/tools/validate_crossmodal.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 +import json, sys +from pathlib import Path +R=Path(__file__).resolve().parents[1] +s=json.load(open(R/"schemas/crossmodal-embedding.schema.json")) +ex=json.load(open(R/"examples/crossmodal-query.example.json")) +try: + from jsonschema import Draft202012Validator as V + errs=list(V(s).iter_errors(ex)) + if errs: print("FAIL:", errs[0].message); sys.exit(1) +except ImportError: + pass +# teeth: shared space required; combined mode must carry both modalities; results labelled by modality +assert ex["unified_space_ref"], "no shared vector space" +if ex["query"]["mode"]=="combined": + assert ex["query"].get("text") and ex["query"].get("image_ref"), "combined must carry text+image" +assert all(r["modality"] in ("text","image") for r in ex["results"]), "results must be modality-labelled" +# adversarial: image-mode query without image_ref must be schema-invalid +try: + from jsonschema import Draft202012Validator as V + bad={"query":{"mode":"image"},"unified_space_ref":"x","results":[]} + assert list(V(s).iter_errors(bad)), "teeth: image mode without image_ref must be rejected" +except ImportError: + pass +print("OK: crossmodal-embedding contract validates (shared space + modality teeth)")