Multicam web feature phase 1

WEB_MULTICAM_PLAN.MD

@@ -4,9 +4,9 @@ Bring stereo/multicam parity to the Girder/web platform by modeling multicam as
 
 ## Implementation Checklist
 
-- [ ] **Server: constants & models** — Add `MultiType` + multiCam/subType/calibration markers and pydantic models (`server/dive_utils/constants.py`, `server/dive_utils/models.py`)
-- [ ] **Server: verify_dataset** — Relax `verify_dataset` to accept `type=multi`; keep fps mirrored from default-display camera
-- [ ] **Server: get_dataset / multiCamMedia** — Extend `crud_dataset.get_dataset` to embed `multiCamMedia` by fanning out to child `get_media` calls
+- [x] **Server: constants & models** — Add `MultiType` + multiCam/subType/calibration markers and pydantic models (`server/dive_utils/constants.py`, `server/dive_utils/models.py`)
+- [x] **Server: verify_dataset** — Relax `verify_dataset` to accept `type=multi`; keep fps mirrored from default-display camera
+- [x] **Server: get_dataset / multiCamMedia** — Extend `crud_dataset.get_dataset` to embed `multiCamMedia` by fanning out to child `get_media` calls
 - [ ] **Server: create multicam** — Implement `POST /dive_dataset/multicam` and `crud_dataset.create_multicam` to move child folders + write parent meta + attach calibration
 - [ ] **Server: clone & export** — Extend `createSoftClone` and `export_datasets_zipstream` to recurse into child cameras; include calibration
 - [ ] **Server: pipelines** — Add multicam/stereo pipeline dispatch in `crud_rpc` that fans inputs/outputs per camera and passes calibration for stereo

server/dive_server/crud.py

@@ -101,12 +101,28 @@ def verify_dataset(folder: GirderModel):
     if not asbool(fromMeta(folder, constants.DatasetMarker, False)):
         raise RestException('Source folder is not a valid DIVE dataset', code=404)
     dstype = fromMeta(folder, 'type')
-    if dstype not in [constants.ImageSequenceType, constants.VideoType, constants.LargeImageType]:
+    valid_types = [
+        constants.ImageSequenceType,
+        constants.VideoType,
+        constants.LargeImageType,
+        constants.MultiType,
+    ]
+    if dstype not in valid_types:
         raise ValueError(f'Source folder is marked as dataset but has invalid type {dstype}')
-    if dstype == constants.VideoType:
+    if dstype in (constants.VideoType, constants.MultiType):
         fps = fromMeta(folder, 'fps')
         if type(fps) not in [int, float]:
-            raise ValueError(f'Video missing numerical fps, found {fps}')
+            raise ValueError(f'Dataset missing numerical fps, found {fps}')
+    if dstype == constants.MultiType:
+        multi_cam = fromMeta(folder, constants.MultiCamMarker)
+        if not multi_cam or not multi_cam.get('defaultDisplay'):
+            raise ValueError('Multi camera dataset missing multiCam.defaultDisplay')
+        cameras = multi_cam.get('cameras') or {}
+        if not cameras:
+            raise ValueError('Multi camera dataset missing multiCam.cameras')
+        for name, cam in cameras.items():
+            if not cam.get('folderId'):
+                raise ValueError(f'Multi camera entry "{name}" missing folderId')
     return True
 
 

server/dive_server/crud_dataset.py

@@ -93,17 +93,62 @@ def list_datasets(
     return [Folder().filter(doc, additionalKeys=['ownerLogin']) for doc in response['results']]
 
 
+def get_multi_cam_media(
+    dsFolder: types.GirderModel, user: types.GirderUserModel
+) -> models.MultiCamMedia:
+    """Build MultiCamMedia by loading media for each child camera folder."""
+    multi_cam = fromMeta(dsFolder, constants.MultiCamMarker)
+    if not multi_cam:
+        raise ValueError('Multi camera dataset missing multiCam metadata')
+    default_display = multi_cam.get('defaultDisplay')
+    if not default_display:
+        raise ValueError('Multi camera dataset missing defaultDisplay')
+    cameras_meta = multi_cam.get('cameras') or {}
+    cameras: Dict[str, models.MultiCamMediaCamera] = {}
+    for name, cam_info in cameras_meta.items():
+        folder_id = cam_info.get('folderId')
+        if not folder_id:
+            raise ValueError(f'Camera "{name}" missing folderId')
+        child = Folder().load(folder_id, level=AccessType.READ, user=user)
+        if child is None:
+            raise RestException(
+                f'Camera folder {folder_id} for "{name}" was not found',
+                code=404,
+            )
+        child_media = get_media(child, user)
+        cam_type = cam_info.get('type') or fromMeta(child, constants.TypeMarker)
+        video_url = child_media.video.url if child_media.video else ''
+        cameras[name] = models.MultiCamMediaCamera(
+            type=cam_type,
+            imageData=child_media.imageData,
+            videoUrl=video_url,
+        )
+    return models.MultiCamMedia(
+        defaultDisplay=default_display,
+        cameras=cameras,
+    )
+
+
 def get_dataset(
     dsFolder: types.GirderModel, user: types.GirderUserModel
 ) -> models.GirderMetadataStatic:
     """Transform a girder folder into a dataset metadata object"""
     crud.verify_dataset(dsFolder)
+    meta = dict(dsFolder.get('meta', {}))
+    source_type = fromMeta(dsFolder, constants.TypeMarker)
+    multi_cam_media = None
+    if source_type == constants.MultiType:
+        multi_cam_media = get_multi_cam_media(dsFolder, user)
+    sub_type = meta.pop(constants.SubTypeMarker, None)
+    meta.pop(constants.MultiCamMarker, None)
     return models.GirderMetadataStatic(
         id=str(dsFolder['_id']),
         createdAt=str(dsFolder['created']),
         name=dsFolder['name'],
         foreign_media_id=dsFolder.get(constants.ForeignMediaIdMarker, None),
-        **dsFolder['meta'],
+        subType=sub_type,
+        multiCamMedia=multi_cam_media,
+        **meta,
     )
 
 
@@ -115,7 +160,10 @@ def get_media(
     imageData: List[models.MediaResource] = []
     crud.verify_dataset(dsFolder)
     source_type = fromMeta(dsFolder, constants.TypeMarker)
-    print(f'Source Type: {source_type}')
+    if source_type == constants.MultiType:
+        return models.DatasetSourceMedia(
+            imageData=imageData, video=videoResource, sourceVideo=sourceVideoResource
+        )
     if source_type == constants.VideoType:
         # Find a video tagged with an h264 codec left by the transcoder
         videoItem = Item().findOne(

server/dive_utils/constants.py

@@ -7,6 +7,7 @@
 ImageSequenceType = "image-sequence"
 VideoType = "video"
 LargeImageType = "large-image"
+MultiType = "multi"
 DefaultVideoFPS = -1
 JsonMetaCurrentVersion = 1
 SettingsCurrentVersion = 1
@@ -54,6 +55,7 @@
 jsonRegex = re.compile(r"\.json$", re.IGNORECASE)
 ymlRegex = re.compile(r"\.ya?ml$", re.IGNORECASE)
 zipRegex = re.compile(r"\.zip$", re.IGNORECASE)
+npzRegex = re.compile(r"\.npz$", re.IGNORECASE)
 metaRegex = re.compile(r"^.*\.?(meta|config)\.json$", re.IGNORECASE)
 # .json or .csv file
 possibleAnnotationRegex = re.compile(r"\.(json|csv)$", re.IGNORECASE)
@@ -106,6 +108,9 @@
 ForeignMediaIdMarker = "foreign_media_id"
 TrainedPipelineMarker = "trained_pipeline"
 TypeMarker = "type"
+SubTypeMarker = "subType"
+MultiCamMarker = "multiCam"
+CalibrationItemIdMarker = "calibrationItemId"
 AssetstoreSourceMarker = "import_source"
 AssetstoreSourcePathMarker = "import_path"
 MarkForPostProcess = "MarkForPostProcess"

server/dive_utils/models.py

@@ -253,6 +253,40 @@ def is_dive_configuration(value: dict):
         return any([value.get(key, False) for key in keys])
 
 
+class MediaResource(BaseModel):
+    url: str
+    id: str
+    filename: str
+
+
+class MultiCamCameraMeta(BaseModel):
+    """Per-camera entry stored on the parent folder meta.multiCam.cameras."""
+
+    folderId: str
+    type: str
+
+
+class MultiCamMetaStorage(BaseModel):
+    """Parent-folder multiCam metadata (storage shape)."""
+
+    defaultDisplay: str
+    cameras: Dict[str, MultiCamCameraMeta]
+    calibrationItemId: Optional[str] = None
+
+
+class MultiCamMediaCamera(BaseModel):
+    """Per-camera media returned to the client (matches dive-common MultiCamMedia)."""
+
+    type: str
+    imageData: List[MediaResource] = Field(default_factory=list)
+    videoUrl: str = ''
+
+
+class MultiCamMedia(BaseModel):
+    cameras: Dict[str, MultiCamMediaCamera]
+    defaultDisplay: str
+
+
 class GirderMetadataStatic(MetadataMutable):
     # Required
     id: str
@@ -268,12 +302,8 @@ class GirderMetadataStatic(MetadataMutable):
     originalFps: Optional[Union[float, int]]
     ffprobe_info: Optional[Dict[str, Any]]
     foreign_media_id: Optional[str]
-
-
-class MediaResource(BaseModel):
-    url: str
-    id: str
-    filename: str
+    subType: Optional[Literal['stereo', 'multicam']] = None
+    multiCamMedia: Optional[MultiCamMedia] = None
 
 
 class DatasetSourceMedia(BaseModel):

server/tests/test_multicam_dataset.py

@@ -0,0 +1,131 @@
+from unittest.mock import MagicMock, patch
+
+import pytest
+from girder.exceptions import RestException
+
+from dive_server import crud, crud_dataset
+from dive_utils import constants
+from dive_utils.models import (
+    DatasetSourceMedia,
+    GirderMetadataStatic,
+    MediaResource,
+    MultiCamMedia,
+    MultiCamMediaCamera,
+)
+
+
+def _multi_parent_folder():
+    return {
+        '_id': 'parent-id',
+        'name': 'stereo-dataset',
+        'created': '2020-01-01T00:00:00',
+        'meta': {
+            'annotate': True,
+            'type': constants.MultiType,
+            'fps': 5,
+            'subType': 'stereo',
+            'multiCam': {
+                'defaultDisplay': 'left',
+                'cameras': {
+                    'left': {'folderId': 'left-id', 'type': 'image-sequence'},
+                    'right': {'folderId': 'right-id', 'type': 'image-sequence'},
+                },
+            },
+        },
+    }
+
+
+def _child_folder(folder_id: str, name: str):
+    return {
+        '_id': folder_id,
+        'name': name,
+        'meta': {
+            'annotate': True,
+            'type': 'image-sequence',
+            'fps': 5,
+        },
+    }
+
+
+class TestVerifyDatasetMulti:
+    def test_accepts_valid_multi_dataset(self):
+        crud.verify_dataset(_multi_parent_folder())
+
+    def test_rejects_multi_without_fps(self):
+        folder = _multi_parent_folder()
+        folder['meta'].pop('fps')
+        with pytest.raises(ValueError, match='missing numerical fps'):
+            crud.verify_dataset(folder)
+
+    def test_rejects_multi_without_multicam_config(self):
+        folder = _multi_parent_folder()
+        folder['meta'].pop('multiCam')
+        with pytest.raises(ValueError, match='multiCam.defaultDisplay'):
+            crud.verify_dataset(folder)
+
+
+@patch('dive_server.crud_dataset.get_media')
+@patch('dive_server.crud_dataset.Folder')
+@patch('dive_server.crud_dataset.crud.verify_dataset')
+def test_get_dataset_includes_multicam_media(_verify, folder_cls, get_media_mock):
+    parent = _multi_parent_folder()
+    user = {'login': 'tester'}
+
+    def load_folder(folder_id, level=None, user=None):
+        if folder_id == 'left-id':
+            return _child_folder('left-id', 'left')
+        if folder_id == 'right-id':
+            return _child_folder('right-id', 'right')
+        return None
+
+    folder_cls.return_value.load.side_effect = load_folder
+
+    left_image = MediaResource(
+        id='img-left', url='/api/v1/.../left.png', filename='left.png'
+    )
+    right_image = MediaResource(
+        id='img-right', url='/api/v1/.../right.png', filename='right.png'
+    )
+
+    def media_for_child(child_folder, child_user):
+        if child_folder['_id'] == 'left-id':
+            return DatasetSourceMedia(imageData=[left_image], video=None, sourceVideo=None)
+        return DatasetSourceMedia(imageData=[right_image], video=None, sourceVideo=None)
+
+    get_media_mock.side_effect = media_for_child
+
+    result = crud_dataset.get_dataset(parent, user)
+
+    assert isinstance(result, GirderMetadataStatic)
+    assert result.type == constants.MultiType
+    assert result.subType == 'stereo'
+    assert result.multiCamMedia is not None
+    assert result.multiCamMedia.defaultDisplay == 'left'
+    assert set(result.multiCamMedia.cameras.keys()) == {'left', 'right'}
+    assert result.multiCamMedia.cameras['left'].imageData[0].filename == 'left.png'
+    assert 'multiCam' not in result.dict()
+
+
+@patch('dive_server.crud_dataset.crud.verify_dataset')
+def test_get_media_multi_parent_returns_empty(_verify):
+    parent = _multi_parent_folder()
+    user = {'login': 'tester'}
+
+    result = crud_dataset.get_media(parent, user)
+
+    assert result.imageData == []
+    assert result.video is None
+    assert result.sourceVideo is None
+
+
+@patch('dive_server.crud_dataset.get_media')
+@patch('dive_server.crud_dataset.Folder')
+def test_get_multi_cam_media_missing_child_raises(folder_cls, get_media_mock):
+    folder_cls.return_value.load.return_value = None
+    parent = _multi_parent_folder()
+    user = {'login': 'tester'}
+
+    with pytest.raises(RestException, match='not found'):
+        crud_dataset.get_multi_cam_media(parent, user)
+
+    get_media_mock.assert_not_called()