diff --git a/python/dendro/api_helpers/clients/db.py b/python/dendro/api_helpers/clients/db.py index 69999ac..d5f7e86 100644 --- a/python/dendro/api_helpers/clients/db.py +++ b/python/dendro/api_helpers/clients/db.py @@ -363,6 +363,18 @@ async def insert_file(file: DendroFile): files_collection = client['dendro']['files'] await files_collection.insert_one(_model_dump(file, exclude_none=True)) +async def update_file_metadata(*, project_id, file_id: str, metadata: dict): + client = _get_mongo_client() + files_collection = client['dendro']['files'] + await files_collection.update_one({ + 'projectId': project_id, + 'fileId': file_id + }, { + '$set': { + 'metadata': metadata + } + }) + class UserNotFoundError(Exception): pass @@ -415,6 +427,21 @@ async def fetch_files_with_content_string(content_string: str) -> List[DendroFil files = [DendroFile(**file) for file in files] # validate files return files +async def fetch_files_with_metadata(metadata_query: dict) -> List[DendroFile]: + # if it's an empty query, raise an error + if not metadata_query: + raise ValueError("metadata_query cannot be empty") + client = _get_mongo_client() + files_collection = client['dendro']['files'] + # find files where file['metadata'] matches metadata_query + files = await files_collection.find({ + 'metadata': metadata_query + }).to_list(length=100) + for file in files: + _remove_id_field(file) + files = [DendroFile(**file) for file in files] # validate files + return files + class ScriptNotFoundException(Exception): pass diff --git a/python/dendro/api_helpers/routers/client/router.py b/python/dendro/api_helpers/routers/client/router.py index 96de6c0..86c520a 100644 --- a/python/dendro/api_helpers/routers/client/router.py +++ b/python/dendro/api_helpers/routers/client/router.py @@ -9,7 +9,7 @@ from ...core.settings import get_settings from ..gui._authenticate_gui_request import _authenticate_gui_request from ...core._get_project_role import _check_user_can_edit_project -from ...services.gui.set_file import set_file as service_set_file +from ...services.gui.set_file import set_file as service_set_file, set_file_metadata as service_set_file_metadata router = APIRouter() @@ -99,6 +99,40 @@ async def set_project_file(project_id, file_name, data: SetProjectFileRequest, d return SetProjectFileResponse(success=True) +# set project file metadata +class SetProjectFileMetadataRequest(BaseModel): + metadata: dict + +class SetProjectFileMetadataResponse(BaseModel): + success: bool + +@router.put("/projects/{project_id}/files-metadata/{file_name:path}") +@api_route_wrapper +async def set_project_file_metadata(project_id, file_name, data: SetProjectFileMetadataRequest, dendro_api_key: Union[str, None] = Header(None)) -> SetProjectFileMetadataResponse: + metadata = data.metadata + + user_id = await _authenticate_gui_request( + github_access_token=None, + dendro_api_key=dendro_api_key, + raise_on_not_authenticated=True + ) + if user_id is None: + raise Exception("User not authenticated") + + project = await fetch_project(project_id) + assert project is not None, f"No project with ID {project_id}" + + _check_user_can_edit_project(project, user_id) + + await service_set_file_metadata( + project_id=project_id, + file_name=file_name, + metadata=metadata + ) + + return SetProjectFileMetadataResponse(success=True) + + # get project jobs class GetProjectJobsResponse(BaseModel): jobs: List[DendroJob] diff --git a/python/dendro/api_helpers/routers/gui/find_routes.py b/python/dendro/api_helpers/routers/gui/find_routes.py index 309a881..906fe6a 100644 --- a/python/dendro/api_helpers/routers/gui/find_routes.py +++ b/python/dendro/api_helpers/routers/gui/find_routes.py @@ -2,8 +2,8 @@ from fastapi import APIRouter from ..common import api_route_wrapper from .... import BaseModel -from ....common.dendro_types import DendroProject -from ...clients.db import fetch_files_with_content_string, fetch_project +from ....common.dendro_types import DendroProject, DendroFile +from ...clients.db import fetch_files_with_content_string, fetch_project, fetch_files_with_metadata router = APIRouter() @@ -30,3 +30,16 @@ async def find_projects(data: FindProjectsRequest) -> CreateProjectResponse: if p is not None: projects.append(p) return CreateProjectResponse(projects=projects, success=True) + +# find files with metadata +class FindFilesWithMetadataRequest(BaseModel): + query: dict + +class FindFilesWithMetadataResponse(BaseModel): + files: List[DendroFile] + +@router.post("/find_files_with_metadata") +@api_route_wrapper +async def find_files_with_metadata(data: FindFilesWithMetadataRequest) -> FindFilesWithMetadataResponse: + files = await fetch_files_with_metadata(data.query) + return FindFilesWithMetadataResponse(files=files) diff --git a/python/dendro/api_helpers/services/gui/set_file.py b/python/dendro/api_helpers/services/gui/set_file.py index e24d811..29ffe18 100644 --- a/python/dendro/api_helpers/services/gui/set_file.py +++ b/python/dendro/api_helpers/services/gui/set_file.py @@ -1,6 +1,6 @@ import time from typing import Union -from ...clients.db import fetch_file, delete_file, insert_file, update_project +from ...clients.db import fetch_file, delete_file, insert_file, update_project, update_file_metadata from ....common.dendro_types import DendroFile from ...core._create_random_id import _create_random_id from .._remove_detached_files_and_jobs import _remove_detached_files_and_jobs @@ -48,3 +48,17 @@ async def set_file( ) return new_file.fileId + +async def set_file_metadata( + project_id: str, + file_name: str, + metadata: dict +): + existing_file = await fetch_file(project_id, file_name) + if existing_file is None: + raise Exception(f"Cannot set metadata. File {file_name} not found in project {project_id}") + await update_file_metadata( + project_id=project_id, + file_id=existing_file.fileId, + metadata=metadata + ) diff --git a/python/dendro/client/__init__.py b/python/dendro/client/__init__.py index 7e80952..72d061e 100644 --- a/python/dendro/client/__init__.py +++ b/python/dendro/client/__init__.py @@ -1,4 +1,4 @@ from .Project import Project, load_project # noqa: F401 from .submit_job import submit_job, SubmitJobInputFile, SubmitJobOutputFile, SubmitJobParameter # noqa: F401 -from .set_file import set_file # noqa: F401 +from .set_file import set_file, set_file_metadata # noqa: F401 from ..common.dendro_types import DendroJobRequiredResources # noqa: F401 diff --git a/python/dendro/client/set_file.py b/python/dendro/client/set_file.py index 859f7a9..bc3cb22 100644 --- a/python/dendro/client/set_file.py +++ b/python/dendro/client/set_file.py @@ -1,23 +1,36 @@ import os +import json from .Project import Project from ..common._api_request import _client_put_api_request -from ..api_helpers.routers.client.router import SetProjectFileRequest +from ..api_helpers.routers.client.router import SetProjectFileRequest, SetProjectFileMetadataRequest def set_file(*, project: Project, file_name: str, url: str, + metadata: dict = {} ): # check if a file already exists for file in project._files: if file.file_name == file_name: if file._file_data.content == f'url:{url}': - return + if _metadata_is_same(file._file_data.metadata, metadata): + return + else: + # Let's just update the metadata + # It's important not to replace the entire file because it would + # trigger deleting of jobs and other files + set_file_metadata( + project=project, + file_name=file_name, + metadata=metadata + ) + return req = SetProjectFileRequest( content=f'url:{url}', - metadata={} + metadata=metadata ) dendro_api_key = os.environ.get('DENDRO_API_KEY', None) if not dendro_api_key: @@ -30,9 +43,37 @@ def set_file(*, ) print(f'File {file_name} set to {url}') +def set_file_metadata(*, + project: Project, + file_name: str, + metadata: dict +): + # Check if file already exists + for file in project._files: + if file.file_name == file_name: + if _metadata_is_same(file._file_data.metadata, metadata): + return + + req = SetProjectFileMetadataRequest( + metadata=metadata + ) + dendro_api_key = os.environ.get('DENDRO_API_KEY', None) + if not dendro_api_key: + raise ValueError('DENDRO_API_KEY environment variable is not set') + url_path = f'/api/client/projects/{project._project_id}/files-metadata/{file_name}' + _client_put_api_request( + url_path=url_path, + data=_model_dump(req), + dendro_api_key=dendro_api_key + ) + print(f'File {file_name} metadata updated') + def _model_dump(model, exclude_none=False): # handle both pydantic v1 and v2 if hasattr(model, 'model_dump'): return model.model_dump(exclude_none=exclude_none) else: return model.dict(exclude_none=exclude_none) + +def _metadata_is_same(metadata1, metadata2): + return json.dumps(metadata1, sort_keys=True) == json.dumps(metadata2, sort_keys=True) diff --git a/python/dendro/client/submit_job.py b/python/dendro/client/submit_job.py index 87a2224..80a72ce 100644 --- a/python/dendro/client/submit_job.py +++ b/python/dendro/client/submit_job.py @@ -98,15 +98,16 @@ def submit_job(*, ] matching_job = None for job in project._jobs: - if _job_matches( - job=job, - processor_name=processor_name, - input_files=input_files, - output_files=output_files, - parameters=parameters - ): - matching_job = job - break + if not job.deleted: + if _job_matches( + job=job, + processor_name=processor_name, + input_files=input_files, + output_files=output_files, + parameters=parameters + ): + matching_job = job + break if matching_job: rerun = True if rerun_policy == 'never': diff --git a/src/pages/ProjectPage/FileView/NwbFileView.tsx b/src/pages/ProjectPage/FileView/NwbFileView.tsx index b8ef65d..c7c2284 100644 --- a/src/pages/ProjectPage/FileView/NwbFileView.tsx +++ b/src/pages/ProjectPage/FileView/NwbFileView.tsx @@ -128,6 +128,10 @@ const FileViewTable: FunctionComponent = ({fileName, additio URI: {theUri} + + Metadata: + {theFile ? JSON.stringify(theFile.metadata || {}) : ''} + { jobProducingThisFile && ( <> @@ -194,6 +198,9 @@ const NwbFileView: FunctionComponent = ({fileName, width, height}) => { if (metadata.dandisetVersion) { additionalQueryParams += `&dandisetVersion=${metadata.dandisetVersion}` } + if (metadata.dandiAssetId) { + additionalQueryParams += `&dandiAssetId=${metadata.dandiAssetId}` + } if (metadata.dandiAssetPath) { const dandiAssetPathEncoded = encodeURIComponent(metadata.dandiAssetPath) additionalQueryParams += `&dandiAssetPath=${dandiAssetPathEncoded}` diff --git a/src/pages/ProjectPage/openFilesInNeurosift.ts b/src/pages/ProjectPage/openFilesInNeurosift.ts index 049323a..38945a1 100644 --- a/src/pages/ProjectPage/openFilesInNeurosift.ts +++ b/src/pages/ProjectPage/openFilesInNeurosift.ts @@ -8,8 +8,10 @@ const openFilesInNeurosift = async (files: DendroFile[], dendroProjectId: string // const neurosiftUrl = `https://flatironinstitute.github.io/neurosift/?p=/nwb&${urlQuery}&dendroProjectId=${dendroProjectId}&${fileNameQuery}`; const dandisetId = files[0]?.metadata.dandisetId; const dandisetVersion = files[0]?.metadata.dandisetVersion; + const dandiAssetId = files[0]?.metadata.dandiAssetId; if (dandisetId) urlQuery += `&dandisetId=${dandisetId}`; if (dandisetVersion) urlQuery += `&dandisetVersion=${dandisetVersion}`; + if (dandiAssetId) urlQuery += `&dandiAssetId=${dandiAssetId}`; const neurosiftUrl = `https://flatironinstitute.github.io/neurosift/?p=/nwb&${urlQuery}`; window.open(neurosiftUrl, '_blank'); }