diff --git a/WEB_MULTICAM_PLAN.MD b/WEB_MULTICAM_PLAN.MD index 72aa1b573..cb8458458 100644 --- a/WEB_MULTICAM_PLAN.MD +++ b/WEB_MULTICAM_PLAN.MD @@ -4,9 +4,9 @@ Bring stereo/multicam parity to the Girder/web platform by modeling multicam as ## Implementation Checklist -- [ ] **Server: constants & models** — Add `MultiType` + multiCam/subType/calibration markers and pydantic models (`server/dive_utils/constants.py`, `server/dive_utils/models.py`) -- [ ] **Server: verify_dataset** — Relax `verify_dataset` to accept `type=multi`; keep fps mirrored from default-display camera -- [ ] **Server: get_dataset / multiCamMedia** — Extend `crud_dataset.get_dataset` to embed `multiCamMedia` by fanning out to child `get_media` calls +- [x] **Server: constants & models** — Add `MultiType` + multiCam/subType/calibration markers and pydantic models (`server/dive_utils/constants.py`, `server/dive_utils/models.py`) +- [x] **Server: verify_dataset** — Relax `verify_dataset` to accept `type=multi`; keep fps mirrored from default-display camera +- [x] **Server: get_dataset / multiCamMedia** — Extend `crud_dataset.get_dataset` to embed `multiCamMedia` by fanning out to child `get_media` calls - [ ] **Server: create multicam** — Implement `POST /dive_dataset/multicam` and `crud_dataset.create_multicam` to move child folders + write parent meta + attach calibration - [ ] **Server: clone & export** — Extend `createSoftClone` and `export_datasets_zipstream` to recurse into child cameras; include calibration - [ ] **Server: pipelines** — Add multicam/stereo pipeline dispatch in `crud_rpc` that fans inputs/outputs per camera and passes calibration for stereo diff --git a/server/dive_server/crud.py b/server/dive_server/crud.py index d348601c2..1508f7968 100644 --- a/server/dive_server/crud.py +++ b/server/dive_server/crud.py @@ -101,12 +101,28 @@ def verify_dataset(folder: GirderModel): if not asbool(fromMeta(folder, constants.DatasetMarker, False)): raise RestException('Source folder is not a valid DIVE dataset', code=404) dstype = fromMeta(folder, 'type') - if dstype not in [constants.ImageSequenceType, constants.VideoType, constants.LargeImageType]: + valid_types = [ + constants.ImageSequenceType, + constants.VideoType, + constants.LargeImageType, + constants.MultiType, + ] + if dstype not in valid_types: raise ValueError(f'Source folder is marked as dataset but has invalid type {dstype}') - if dstype == constants.VideoType: + if dstype in (constants.VideoType, constants.MultiType): fps = fromMeta(folder, 'fps') if type(fps) not in [int, float]: - raise ValueError(f'Video missing numerical fps, found {fps}') + raise ValueError(f'Dataset missing numerical fps, found {fps}') + if dstype == constants.MultiType: + multi_cam = fromMeta(folder, constants.MultiCamMarker) + if not multi_cam or not multi_cam.get('defaultDisplay'): + raise ValueError('Multi camera dataset missing multiCam.defaultDisplay') + cameras = multi_cam.get('cameras') or {} + if not cameras: + raise ValueError('Multi camera dataset missing multiCam.cameras') + for name, cam in cameras.items(): + if not cam.get('folderId'): + raise ValueError(f'Multi camera entry "{name}" missing folderId') return True diff --git a/server/dive_server/crud_dataset.py b/server/dive_server/crud_dataset.py index b799581a1..a97f3886b 100644 --- a/server/dive_server/crud_dataset.py +++ b/server/dive_server/crud_dataset.py @@ -93,17 +93,62 @@ def list_datasets( return [Folder().filter(doc, additionalKeys=['ownerLogin']) for doc in response['results']] +def get_multi_cam_media( + dsFolder: types.GirderModel, user: types.GirderUserModel +) -> models.MultiCamMedia: + """Build MultiCamMedia by loading media for each child camera folder.""" + multi_cam = fromMeta(dsFolder, constants.MultiCamMarker) + if not multi_cam: + raise ValueError('Multi camera dataset missing multiCam metadata') + default_display = multi_cam.get('defaultDisplay') + if not default_display: + raise ValueError('Multi camera dataset missing defaultDisplay') + cameras_meta = multi_cam.get('cameras') or {} + cameras: Dict[str, models.MultiCamMediaCamera] = {} + for name, cam_info in cameras_meta.items(): + folder_id = cam_info.get('folderId') + if not folder_id: + raise ValueError(f'Camera "{name}" missing folderId') + child = Folder().load(folder_id, level=AccessType.READ, user=user) + if child is None: + raise RestException( + f'Camera folder {folder_id} for "{name}" was not found', + code=404, + ) + child_media = get_media(child, user) + cam_type = cam_info.get('type') or fromMeta(child, constants.TypeMarker) + video_url = child_media.video.url if child_media.video else '' + cameras[name] = models.MultiCamMediaCamera( + type=cam_type, + imageData=child_media.imageData, + videoUrl=video_url, + ) + return models.MultiCamMedia( + defaultDisplay=default_display, + cameras=cameras, + ) + + def get_dataset( dsFolder: types.GirderModel, user: types.GirderUserModel ) -> models.GirderMetadataStatic: """Transform a girder folder into a dataset metadata object""" crud.verify_dataset(dsFolder) + meta = dict(dsFolder.get('meta', {})) + source_type = fromMeta(dsFolder, constants.TypeMarker) + multi_cam_media = None + if source_type == constants.MultiType: + multi_cam_media = get_multi_cam_media(dsFolder, user) + sub_type = meta.pop(constants.SubTypeMarker, None) + meta.pop(constants.MultiCamMarker, None) return models.GirderMetadataStatic( id=str(dsFolder['_id']), createdAt=str(dsFolder['created']), name=dsFolder['name'], foreign_media_id=dsFolder.get(constants.ForeignMediaIdMarker, None), - **dsFolder['meta'], + subType=sub_type, + multiCamMedia=multi_cam_media, + **meta, ) @@ -115,7 +160,10 @@ def get_media( imageData: List[models.MediaResource] = [] crud.verify_dataset(dsFolder) source_type = fromMeta(dsFolder, constants.TypeMarker) - print(f'Source Type: {source_type}') + if source_type == constants.MultiType: + return models.DatasetSourceMedia( + imageData=imageData, video=videoResource, sourceVideo=sourceVideoResource + ) if source_type == constants.VideoType: # Find a video tagged with an h264 codec left by the transcoder videoItem = Item().findOne( diff --git a/server/dive_utils/constants.py b/server/dive_utils/constants.py index 06bd4787a..2a37b3650 100644 --- a/server/dive_utils/constants.py +++ b/server/dive_utils/constants.py @@ -7,6 +7,7 @@ ImageSequenceType = "image-sequence" VideoType = "video" LargeImageType = "large-image" +MultiType = "multi" DefaultVideoFPS = -1 JsonMetaCurrentVersion = 1 SettingsCurrentVersion = 1 @@ -54,6 +55,7 @@ jsonRegex = re.compile(r"\.json$", re.IGNORECASE) ymlRegex = re.compile(r"\.ya?ml$", re.IGNORECASE) zipRegex = re.compile(r"\.zip$", re.IGNORECASE) +npzRegex = re.compile(r"\.npz$", re.IGNORECASE) metaRegex = re.compile(r"^.*\.?(meta|config)\.json$", re.IGNORECASE) # .json or .csv file possibleAnnotationRegex = re.compile(r"\.(json|csv)$", re.IGNORECASE) @@ -106,6 +108,9 @@ ForeignMediaIdMarker = "foreign_media_id" TrainedPipelineMarker = "trained_pipeline" TypeMarker = "type" +SubTypeMarker = "subType" +MultiCamMarker = "multiCam" +CalibrationItemIdMarker = "calibrationItemId" AssetstoreSourceMarker = "import_source" AssetstoreSourcePathMarker = "import_path" MarkForPostProcess = "MarkForPostProcess" diff --git a/server/dive_utils/models.py b/server/dive_utils/models.py index a2725105c..b6d3bbc13 100644 --- a/server/dive_utils/models.py +++ b/server/dive_utils/models.py @@ -253,6 +253,40 @@ def is_dive_configuration(value: dict): return any([value.get(key, False) for key in keys]) +class MediaResource(BaseModel): + url: str + id: str + filename: str + + +class MultiCamCameraMeta(BaseModel): + """Per-camera entry stored on the parent folder meta.multiCam.cameras.""" + + folderId: str + type: str + + +class MultiCamMetaStorage(BaseModel): + """Parent-folder multiCam metadata (storage shape).""" + + defaultDisplay: str + cameras: Dict[str, MultiCamCameraMeta] + calibrationItemId: Optional[str] = None + + +class MultiCamMediaCamera(BaseModel): + """Per-camera media returned to the client (matches dive-common MultiCamMedia).""" + + type: str + imageData: List[MediaResource] = Field(default_factory=list) + videoUrl: str = '' + + +class MultiCamMedia(BaseModel): + cameras: Dict[str, MultiCamMediaCamera] + defaultDisplay: str + + class GirderMetadataStatic(MetadataMutable): # Required id: str @@ -268,12 +302,8 @@ class GirderMetadataStatic(MetadataMutable): originalFps: Optional[Union[float, int]] ffprobe_info: Optional[Dict[str, Any]] foreign_media_id: Optional[str] - - -class MediaResource(BaseModel): - url: str - id: str - filename: str + subType: Optional[Literal['stereo', 'multicam']] = None + multiCamMedia: Optional[MultiCamMedia] = None class DatasetSourceMedia(BaseModel): diff --git a/server/tests/test_multicam_dataset.py b/server/tests/test_multicam_dataset.py new file mode 100644 index 000000000..acba666f5 --- /dev/null +++ b/server/tests/test_multicam_dataset.py @@ -0,0 +1,131 @@ +from unittest.mock import MagicMock, patch + +import pytest +from girder.exceptions import RestException + +from dive_server import crud, crud_dataset +from dive_utils import constants +from dive_utils.models import ( + DatasetSourceMedia, + GirderMetadataStatic, + MediaResource, + MultiCamMedia, + MultiCamMediaCamera, +) + + +def _multi_parent_folder(): + return { + '_id': 'parent-id', + 'name': 'stereo-dataset', + 'created': '2020-01-01T00:00:00', + 'meta': { + 'annotate': True, + 'type': constants.MultiType, + 'fps': 5, + 'subType': 'stereo', + 'multiCam': { + 'defaultDisplay': 'left', + 'cameras': { + 'left': {'folderId': 'left-id', 'type': 'image-sequence'}, + 'right': {'folderId': 'right-id', 'type': 'image-sequence'}, + }, + }, + }, + } + + +def _child_folder(folder_id: str, name: str): + return { + '_id': folder_id, + 'name': name, + 'meta': { + 'annotate': True, + 'type': 'image-sequence', + 'fps': 5, + }, + } + + +class TestVerifyDatasetMulti: + def test_accepts_valid_multi_dataset(self): + crud.verify_dataset(_multi_parent_folder()) + + def test_rejects_multi_without_fps(self): + folder = _multi_parent_folder() + folder['meta'].pop('fps') + with pytest.raises(ValueError, match='missing numerical fps'): + crud.verify_dataset(folder) + + def test_rejects_multi_without_multicam_config(self): + folder = _multi_parent_folder() + folder['meta'].pop('multiCam') + with pytest.raises(ValueError, match='multiCam.defaultDisplay'): + crud.verify_dataset(folder) + + +@patch('dive_server.crud_dataset.get_media') +@patch('dive_server.crud_dataset.Folder') +@patch('dive_server.crud_dataset.crud.verify_dataset') +def test_get_dataset_includes_multicam_media(_verify, folder_cls, get_media_mock): + parent = _multi_parent_folder() + user = {'login': 'tester'} + + def load_folder(folder_id, level=None, user=None): + if folder_id == 'left-id': + return _child_folder('left-id', 'left') + if folder_id == 'right-id': + return _child_folder('right-id', 'right') + return None + + folder_cls.return_value.load.side_effect = load_folder + + left_image = MediaResource( + id='img-left', url='/api/v1/.../left.png', filename='left.png' + ) + right_image = MediaResource( + id='img-right', url='/api/v1/.../right.png', filename='right.png' + ) + + def media_for_child(child_folder, child_user): + if child_folder['_id'] == 'left-id': + return DatasetSourceMedia(imageData=[left_image], video=None, sourceVideo=None) + return DatasetSourceMedia(imageData=[right_image], video=None, sourceVideo=None) + + get_media_mock.side_effect = media_for_child + + result = crud_dataset.get_dataset(parent, user) + + assert isinstance(result, GirderMetadataStatic) + assert result.type == constants.MultiType + assert result.subType == 'stereo' + assert result.multiCamMedia is not None + assert result.multiCamMedia.defaultDisplay == 'left' + assert set(result.multiCamMedia.cameras.keys()) == {'left', 'right'} + assert result.multiCamMedia.cameras['left'].imageData[0].filename == 'left.png' + assert 'multiCam' not in result.dict() + + +@patch('dive_server.crud_dataset.crud.verify_dataset') +def test_get_media_multi_parent_returns_empty(_verify): + parent = _multi_parent_folder() + user = {'login': 'tester'} + + result = crud_dataset.get_media(parent, user) + + assert result.imageData == [] + assert result.video is None + assert result.sourceVideo is None + + +@patch('dive_server.crud_dataset.get_media') +@patch('dive_server.crud_dataset.Folder') +def test_get_multi_cam_media_missing_child_raises(folder_cls, get_media_mock): + folder_cls.return_value.load.return_value = None + parent = _multi_parent_folder() + user = {'login': 'tester'} + + with pytest.raises(RestException, match='not found'): + crud_dataset.get_multi_cam_media(parent, user) + + get_media_mock.assert_not_called()