surface: crossmodal-embeddings (text<->image shared vector space)

docs/CROSSMODAL_SURFACE.md

@@ -0,0 +1,2 @@
+# Crossmodal embeddings surface (embeddinglab)
+D6: text and image inputs embedded into **one shared vector space** and matched crossmodally (a text query retrieves images; an image query retrieves text). embeddinglab owns the shared space (`embeddings`/`semantic-indexing` surfaces); the **image encoder is a handoff from `imagelab#multimodal-handoff`** — the `image_encoding_handoff` receipt keeps encoder provenance, so embeddinglab never carries mutable image-model state (lab-boundary preserved). Feeds the marketplace/search deliverable-matching (crossmodal product/skill search). Contract: `schemas/crossmodal-embedding.schema.json`; validate `python3 tools/validate_crossmodal.py`.

examples/crossmodal-query.example.json

@@ -0,0 +1,7 @@
+{ "query": {"mode": "combined", "text": "red high-heeled shoe",
+    "image_ref": "cid:bafyimageexample", "image_encoding_handoff": "imagelab:handoff:rcpt-001"},
+  "unified_space_ref": "embeddinglab:space:text-image-v0", "top_k": 3,
+  "results": [
+    {"id": "product:shoe-42", "modality": "image", "score": 0.95},
+    {"id": "listing:red-heels", "modality": "text", "score": 0.92},
+    {"id": "product:shoe-17", "modality": "image", "score": 0.89}] }

lab.manifest.json

@@ -9,7 +9,8 @@
     "rerankers",
     "hybrid-retrieval-preparation",
     "vector-evaluation",
-    "semantic-indexing"
+    "semantic-indexing",
+    "crossmodal-embeddings"
   ],
   "serviceManifests": [
     "service-manifest/functional-service.v1.json"
@@ -25,5 +26,9 @@
     "allowed": true,
     "carriesMutableModelState": false,
     "clientRefRequired": true
+  },
+  "imageHandoff": {
+    "seam": "SociOS-Linux/imagelab#multimodal-handoff",
+    "note": "image encoder lives in imagelab; embeddinglab owns the shared text+image vector space"
   }
 }

schemas/crossmodal-embedding.schema.json

@@ -0,0 +1,35 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://schemas.socioprophet.org/embedding/crossmodal-embedding.v0.json",
+  "title": "CrossmodalQuery",
+  "description": "D6 (crossmodal search): text and/or image inputs embedded into ONE shared vector space, matched crossmodally (text<->image). The shared space is embeddinglab's; the image encoding is handed off from imagelab#multimodal-handoff (provenance kept). Lab-only contract: emits a governed match, never self-promotes a model.",
+  "type": "object",
+  "additionalProperties": false,
+  "required": ["query", "unified_space_ref", "results"],
+  "properties": {
+    "query": {
+      "type": "object", "additionalProperties": false, "required": ["mode"],
+      "description": "at least one of text/image_ref must be present, consistent with mode",
+      "properties": {
+        "mode": {"type": "string", "enum": ["text", "image", "combined"]},
+        "text": {"type": ["string", "null"]},
+        "image_ref": {"type": ["string", "null"], "description": "content-addressed image ref or data: URI"},
+        "image_encoding_handoff": {"type": ["string", "null"], "description": "imagelab multimodal-handoff receipt id (image-encoder provenance)"}
+      },
+      "allOf": [
+        {"if": {"properties": {"mode": {"const": "text"}}}, "then": {"required": ["text"]}},
+        {"if": {"properties": {"mode": {"const": "image"}}}, "then": {"required": ["image_ref"]}},
+        {"if": {"properties": {"mode": {"const": "combined"}}}, "then": {"required": ["text", "image_ref"]}}
+      ]
+    },
+    "unified_space_ref": {"type": "string", "description": "embeddinglab shared vector-space id (text+image co-embedded)"},
+    "top_k": {"type": "integer", "minimum": 1, "maximum": 200, "default": 20},
+    "results": {"type": "array", "items": {
+      "type": "object", "additionalProperties": false, "required": ["id", "modality", "score"],
+      "properties": {
+        "id": {"type": "string"},
+        "modality": {"type": "string", "enum": ["text", "image"]},
+        "score": {"type": "number", "minimum": 0, "maximum": 1}
+      }}}
+  }
+}

tools/validate_crossmodal.py

@@ -0,0 +1,25 @@
+#!/usr/bin/env python3
+import json, sys
+from pathlib import Path
+R=Path(__file__).resolve().parents[1]
+s=json.load(open(R/"schemas/crossmodal-embedding.schema.json"))
+ex=json.load(open(R/"examples/crossmodal-query.example.json"))
+try:
+    from jsonschema import Draft202012Validator as V
+    errs=list(V(s).iter_errors(ex))
+    if errs: print("FAIL:", errs[0].message); sys.exit(1)
+except ImportError:
+    pass
+# teeth: shared space required; combined mode must carry both modalities; results labelled by modality
+assert ex["unified_space_ref"], "no shared vector space"
+if ex["query"]["mode"]=="combined":
+    assert ex["query"].get("text") and ex["query"].get("image_ref"), "combined must carry text+image"
+assert all(r["modality"] in ("text","image") for r in ex["results"]), "results must be modality-labelled"
+# adversarial: image-mode query without image_ref must be schema-invalid
+try:
+    from jsonschema import Draft202012Validator as V
+    bad={"query":{"mode":"image"},"unified_space_ref":"x","results":[]}
+    assert list(V(s).iter_errors(bad)), "teeth: image mode without image_ref must be rejected"
+except ImportError:
+    pass
+print("OK: crossmodal-embedding contract validates (shared space + modality teeth)")