file metadata

python/dendro/api_helpers/clients/db.py

@@ -363,6 +363,18 @@ async def insert_file(file: DendroFile):
     files_collection = client['dendro']['files']
     await files_collection.insert_one(_model_dump(file, exclude_none=True))
 
+async def update_file_metadata(*, project_id, file_id: str, metadata: dict):
+    client = _get_mongo_client()
+    files_collection = client['dendro']['files']
+    await files_collection.update_one({
+        'projectId': project_id,
+        'fileId': file_id
+    }, {
+        '$set': {
+            'metadata': metadata
+        }
+    })
+
 class UserNotFoundError(Exception):
     pass
 
@@ -415,6 +427,21 @@ async def fetch_files_with_content_string(content_string: str) -> List[DendroFil
     files = [DendroFile(**file) for file in files] # validate files
     return files
 
+async def fetch_files_with_metadata(metadata_query: dict) -> List[DendroFile]:
+    # if it's an empty query, raise an error
+    if not metadata_query:
+        raise ValueError("metadata_query cannot be empty")
+    client = _get_mongo_client()
+    files_collection = client['dendro']['files']
+    # find files where file['metadata'] matches metadata_query
+    files = await files_collection.find({
+        'metadata': metadata_query
+    }).to_list(length=100)
+    for file in files:
+        _remove_id_field(file)
+    files = [DendroFile(**file) for file in files] # validate files
+    return files
+
 class ScriptNotFoundException(Exception):
     pass
 

python/dendro/api_helpers/routers/client/router.py

@@ -9,7 +9,7 @@
 from ...core.settings import get_settings
 from ..gui._authenticate_gui_request import _authenticate_gui_request
 from ...core._get_project_role import _check_user_can_edit_project
-from ...services.gui.set_file import set_file as service_set_file
+from ...services.gui.set_file import set_file as service_set_file, set_file_metadata as service_set_file_metadata
 
 router = APIRouter()
 
@@ -99,6 +99,40 @@ async def set_project_file(project_id, file_name, data: SetProjectFileRequest, d
     return SetProjectFileResponse(success=True)
 
 
+# set project file metadata
+class SetProjectFileMetadataRequest(BaseModel):
+    metadata: dict
+
+class SetProjectFileMetadataResponse(BaseModel):
+    success: bool
+
+@router.put("/projects/{project_id}/files-metadata/{file_name:path}")
+@api_route_wrapper
+async def set_project_file_metadata(project_id, file_name, data: SetProjectFileMetadataRequest, dendro_api_key: Union[str, None] = Header(None)) -> SetProjectFileMetadataResponse:
+    metadata = data.metadata
+
+    user_id = await _authenticate_gui_request(
+        github_access_token=None,
+        dendro_api_key=dendro_api_key,
+        raise_on_not_authenticated=True
+    )
+    if user_id is None:
+        raise Exception("User not authenticated")
+
+    project = await fetch_project(project_id)
+    assert project is not None, f"No project with ID {project_id}"
+
+    _check_user_can_edit_project(project, user_id)
+
+    await service_set_file_metadata(
+        project_id=project_id,
+        file_name=file_name,
+        metadata=metadata
+    )
+
+    return SetProjectFileMetadataResponse(success=True)
+
+
 # get project jobs
 class GetProjectJobsResponse(BaseModel):
     jobs: List[DendroJob]

python/dendro/api_helpers/routers/gui/find_routes.py

@@ -2,8 +2,8 @@
 from fastapi import APIRouter
 from ..common import api_route_wrapper
 from .... import BaseModel
-from ....common.dendro_types import DendroProject
-from ...clients.db import fetch_files_with_content_string, fetch_project
+from ....common.dendro_types import DendroProject, DendroFile
+from ...clients.db import fetch_files_with_content_string, fetch_project, fetch_files_with_metadata
 
 
 router = APIRouter()
@@ -30,3 +30,16 @@ async def find_projects(data: FindProjectsRequest) -> CreateProjectResponse:
         if p is not None:
             projects.append(p)
     return CreateProjectResponse(projects=projects, success=True)
+
+# find files with metadata
+class FindFilesWithMetadataRequest(BaseModel):
+    query: dict
+
+class FindFilesWithMetadataResponse(BaseModel):
+    files: List[DendroFile]
+
+@router.post("/find_files_with_metadata")
+@api_route_wrapper
+async def find_files_with_metadata(data: FindFilesWithMetadataRequest) -> FindFilesWithMetadataResponse:
+    files = await fetch_files_with_metadata(data.query)
+    return FindFilesWithMetadataResponse(files=files)

python/dendro/api_helpers/services/gui/set_file.py

@@ -1,6 +1,6 @@
 import time
 from typing import Union
-from ...clients.db import fetch_file, delete_file, insert_file, update_project
+from ...clients.db import fetch_file, delete_file, insert_file, update_project, update_file_metadata
 from ....common.dendro_types import DendroFile
 from ...core._create_random_id import _create_random_id
 from .._remove_detached_files_and_jobs import _remove_detached_files_and_jobs
@@ -48,3 +48,17 @@ async def set_file(
     )
 
     return new_file.fileId
+
+async def set_file_metadata(
+    project_id: str,
+    file_name: str,
+    metadata: dict
+):
+    existing_file = await fetch_file(project_id, file_name)
+    if existing_file is None:
+        raise Exception(f"Cannot set metadata. File {file_name} not found in project {project_id}")
+    await update_file_metadata(
+        project_id=project_id,
+        file_id=existing_file.fileId,
+        metadata=metadata
+    )

python/dendro/client/__init__.py

@@ -1,4 +1,4 @@
 from .Project import Project, load_project # noqa: F401
 from .submit_job import submit_job, SubmitJobInputFile, SubmitJobOutputFile, SubmitJobParameter # noqa: F401
-from .set_file import set_file # noqa: F401
+from .set_file import set_file, set_file_metadata # noqa: F401
 from ..common.dendro_types import DendroJobRequiredResources # noqa: F401

python/dendro/client/set_file.py

@@ -1,23 +1,36 @@
 import os
+import json
 from .Project import Project
 from ..common._api_request import _client_put_api_request
-from ..api_helpers.routers.client.router import SetProjectFileRequest
+from ..api_helpers.routers.client.router import SetProjectFileRequest, SetProjectFileMetadataRequest
 
 
 def set_file(*,
     project: Project,
     file_name: str,
     url: str,
+    metadata: dict = {}
 ):
     # check if a file already exists
     for file in project._files:
         if file.file_name == file_name:
             if file._file_data.content == f'url:{url}':
-                return
+                if _metadata_is_same(file._file_data.metadata, metadata):
+                    return
+                else:
+                    # Let's just update the metadata
+                    # It's important not to replace the entire file because it would
+                    # trigger deleting of jobs and other files
+                    set_file_metadata(
+                        project=project,
+                        file_name=file_name,
+                        metadata=metadata
+                    )
+                    return
 
     req = SetProjectFileRequest(
         content=f'url:{url}',
-        metadata={}
+        metadata=metadata
     )
     dendro_api_key = os.environ.get('DENDRO_API_KEY', None)
     if not dendro_api_key:
@@ -30,9 +43,37 @@ def set_file(*,
     )
     print(f'File {file_name} set to {url}')
 
+def set_file_metadata(*,
+    project: Project,
+    file_name: str,
+    metadata: dict
+):
+    # Check if file already exists
+    for file in project._files:
+        if file.file_name == file_name:
+            if _metadata_is_same(file._file_data.metadata, metadata):
+                return
+
+    req = SetProjectFileMetadataRequest(
+        metadata=metadata
+    )
+    dendro_api_key = os.environ.get('DENDRO_API_KEY', None)
+    if not dendro_api_key:
+        raise ValueError('DENDRO_API_KEY environment variable is not set')
+    url_path = f'/api/client/projects/{project._project_id}/files-metadata/{file_name}'
+    _client_put_api_request(
+        url_path=url_path,
+        data=_model_dump(req),
+        dendro_api_key=dendro_api_key
+    )
+    print(f'File {file_name} metadata updated')
+
 def _model_dump(model, exclude_none=False):
     # handle both pydantic v1 and v2
     if hasattr(model, 'model_dump'):
         return model.model_dump(exclude_none=exclude_none)
     else:
         return model.dict(exclude_none=exclude_none)
+
+def _metadata_is_same(metadata1, metadata2):
+    return json.dumps(metadata1, sort_keys=True) == json.dumps(metadata2, sort_keys=True)

python/dendro/client/submit_job.py

@@ -98,15 +98,16 @@ def submit_job(*,
     ]
     matching_job = None
     for job in project._jobs:
-        if _job_matches(
-            job=job,
-            processor_name=processor_name,
-            input_files=input_files,
-            output_files=output_files,
-            parameters=parameters
-        ):
-            matching_job = job
-            break
+        if not job.deleted:
+            if _job_matches(
+                job=job,
+                processor_name=processor_name,
+                input_files=input_files,
+                output_files=output_files,
+                parameters=parameters
+            ):
+                matching_job = job
+                break
     if matching_job:
         rerun = True
         if rerun_policy == 'never':

src/pages/ProjectPage/FileView/NwbFileView.tsx

@@ -128,6 +128,10 @@ const FileViewTable: FunctionComponent<FileViewTableProps> = ({fileName, additio
                     <td>URI:</td>
                     <td>{theUri}</td>
                 </tr>
+                <tr>
+                    <td>Metadata:</td>
+                    <td>{theFile ? JSON.stringify(theFile.metadata || {}) : ''}</td>
+                </tr>
                 {
                     jobProducingThisFile && (
                         <>
@@ -194,6 +198,9 @@ const NwbFileView: FunctionComponent<Props> = ({fileName, width, height}) => {
         if (metadata.dandisetVersion) {
             additionalQueryParams += `&dandisetVersion=${metadata.dandisetVersion}`
         }
+        if (metadata.dandiAssetId) {
+            additionalQueryParams += `&dandiAssetId=${metadata.dandiAssetId}`
+        }
         if (metadata.dandiAssetPath) {
             const dandiAssetPathEncoded = encodeURIComponent(metadata.dandiAssetPath)
             additionalQueryParams += `&dandiAssetPath=${dandiAssetPathEncoded}`

src/pages/ProjectPage/openFilesInNeurosift.ts

@@ -8,8 +8,10 @@ const openFilesInNeurosift = async (files: DendroFile[], dendroProjectId: string
     // const neurosiftUrl = `https://flatironinstitute.github.io/neurosift/?p=/nwb&${urlQuery}&dendroProjectId=${dendroProjectId}&${fileNameQuery}`;
     const dandisetId = files[0]?.metadata.dandisetId;
     const dandisetVersion = files[0]?.metadata.dandisetVersion;
+    const dandiAssetId = files[0]?.metadata.dandiAssetId;
     if (dandisetId) urlQuery += `&dandisetId=${dandisetId}`;
     if (dandisetVersion) urlQuery += `&dandisetVersion=${dandisetVersion}`;
+    if (dandiAssetId) urlQuery += `&dandiAssetId=${dandiAssetId}`;
     const neurosiftUrl = `https://flatironinstitute.github.io/neurosift/?p=/nwb&${urlQuery}`;
     window.open(neurosiftUrl, '_blank');
 }