Merge branch 'main' into ebs-pipelines

package.json

@@ -1,7 +1,7 @@
 {
   "name": "dendro",
   "private": true,
-  "version": "0.1.43",
+  "version": "0.2.12",
   "scripts": {
     "dev": "vite",
     "build": "tsc && vite build",
@@ -21,11 +21,13 @@
     "@fortawesome/free-solid-svg-icons": "^6.4.0",
     "@fortawesome/react-fontawesome": "^0.2.0",
     "@isomorphic-git/lightning-fs": "^4.6.0",
+    "@monaco-editor/react": "^4.6.0",
     "@mui/icons-material": "^5.11.16",
     "@mui/material": "^5.13.0",
     "dompurify": "^3.0.8",
     "github-markdown-css": "^5.2.0",
     "isomorphic-git": "^1.25.2",
+    "monaco-editor": "^0.46.0",
     "nunjucks": "^3.2.4",
     "plotly.js": "^2.27.1",
     "pubnub": "^7.4.1",

python/.gitignore

@@ -1,3 +1,5 @@
+_dendro
+file_cache
 script_jobs
 example-data
 tmp

python/dendro/api_helpers/clients/db.py

@@ -2,7 +2,7 @@
 from typing import List, Union
 from ._get_mongo_client import _get_mongo_client
 from ._remove_id_field import _remove_id_field
-from ...common.dendro_types import DendroProject, DendroFile, DendroJob, DendroComputeResource, ComputeResourceSpec, DendroUser
+from ...common.dendro_types import DendroProject, DendroFile, DendroJob, DendroComputeResource, ComputeResourceSpec, DendroScript, DendroUser
 from ..core._get_project_role import _project_has_user
 from ..core._model_dump import _model_dump
 from ..core._hide_secret_params_in_job import _hide_secret_params_in_job
@@ -101,6 +101,15 @@ async def fetch_project_jobs(project_id: str, include_private_keys=False) -> Lis
         _hide_secret_params_in_job(job)
     return jobs
 
+async def fetch_project_scripts(project_id: str) -> List[DendroScript]:
+    client = _get_mongo_client()
+    scripts_collection = client['dendro']['scripts']
+    scripts = await scripts_collection.find({
+        'projectId': project_id
+    }).to_list(length=None) # type: ignore
+    scripts = [DendroScript(**script) for script in scripts] # validate scripts
+    return scripts
+
 async def update_project(project_id: str, update: dict):
     client = _get_mongo_client()
     projects_collection = client['dendro']['projects']
@@ -246,7 +255,7 @@ async def set_compute_resource_spec(compute_resource_id: str, spec: ComputeResou
 class JobNotFoundError(Exception):
     pass
 
-async def fetch_job(job_id: str, *, include_dandi_api_key: bool = False, include_secret_params: bool = False, include_private_key: bool = False, raise_on_not_found=False):
+async def fetch_job(job_id: str, *, include_dandi_api_key: bool = False, include_secret_params: bool = False, include_private_key: bool = False, raise_on_not_found=False, include_deleted=False):
     client = _get_mongo_client()
     jobs_collection = client['dendro']['jobs']
     job = await jobs_collection.find_one({'jobId': job_id})
@@ -257,6 +266,11 @@ async def fetch_job(job_id: str, *, include_dandi_api_key: bool = False, include
         else:
             return None # pragma: no cover (not ever run with raise_on_not_found=False)
     job = DendroJob(**job) # validate job
+    if job.deleted and not include_deleted:
+        if raise_on_not_found:
+            raise JobNotFoundError(f"Job with ID {job_id} has been deleted")
+        else:
+            return None
     if not include_dandi_api_key:
         job.dandiApiKey = None
     if not include_secret_params:
@@ -400,3 +414,52 @@ async def fetch_files_with_content_string(content_string: str) -> List[DendroFil
         _remove_id_field(file)
     files = [DendroFile(**file) for file in files] # validate files
     return files
+
+class ScriptNotFoundException(Exception):
+    pass
+
+async def fetch_script(script_id: str) -> DendroScript:
+    client = _get_mongo_client()
+    scripts_collection = client['dendro']['scripts']
+    script = await scripts_collection.find_one({
+        'scriptId': script_id
+    })
+    if script is None:
+        raise ScriptNotFoundException(f"No script with ID {script_id}")
+    _remove_id_field(script)
+    script = DendroScript(**script) # validate script
+    return script
+
+async def delete_script(script_id: str):
+    client = _get_mongo_client()
+    scripts_collection = client['dendro']['scripts']
+    await scripts_collection.delete_one({
+        'scriptId': script_id
+    })
+
+async def set_script_content(script_id: str, content: str):
+    client = _get_mongo_client()
+    scripts_collection = client['dendro']['scripts']
+    await scripts_collection.update_one({
+        'scriptId': script_id
+    }, {
+        '$set': {
+            'content': content
+        }
+    })
+
+async def rename_script(script_id: str, new_name: str):
+    client = _get_mongo_client()
+    scripts_collection = client['dendro']['scripts']
+    await scripts_collection.update_one({
+        'scriptId': script_id
+    }, {
+        '$set': {
+            'scriptName': new_name
+        }
+    })
+
+async def insert_script(script: DendroScript):
+    client = _get_mongo_client()
+    scripts_collection = client['dendro']['scripts']
+    await scripts_collection.insert_one(_model_dump(script, exclude_none=True))

python/dendro/api_helpers/routers/compute_resource/router.py

@@ -166,6 +166,6 @@ def _authenticate_compute_resource_request(
     expected_payload: str
 ):
     if compute_resource_payload != expected_payload:
-        raise UnexpectedException('Unexpected payload')
+        raise UnexpectedException('Unexpected payload: ' + compute_resource_payload)
     if not _verify_signature_str(compute_resource_payload, compute_resource_id, compute_resource_signature):
-        raise InvalidSignatureException('Invalid signature')
+        raise InvalidSignatureException(f'Invalid signature: {compute_resource_signature}')

python/dendro/api_helpers/routers/gui/project_routes.py

@@ -5,10 +5,10 @@
 from fastapi import APIRouter, Header
 from .... import BaseModel
 from ...core._create_random_id import _create_random_id
-from ....common.dendro_types import DendroJob, DendroProject, DendroProjectUser
+from ....common.dendro_types import DendroJob, DendroProject, DendroProjectUser, DendroScript
 from ._authenticate_gui_request import _authenticate_gui_request
 from ...core._get_project_role import _check_user_can_edit_project, _check_user_is_project_admin
-from ...clients.db import fetch_project, insert_project, update_project, fetch_project_jobs, fetch_projects_for_user, fetch_all_projects, fetch_projects_with_tag
+from ...clients.db import fetch_project, fetch_project_scripts, insert_project, update_project, fetch_project_jobs, fetch_projects_for_user, fetch_all_projects, fetch_projects_with_tag, insert_script
 from ...services.gui.delete_project import delete_project as service_delete_project
 from ..common import api_route_wrapper
 
@@ -198,6 +198,55 @@ async def get_jobs(project_id):
     jobs = await fetch_project_jobs(project_id, include_private_keys=False)
     return GetJobsResponse(jobs=jobs, success=True)
 
+# get scripts
+class GetScriptsResponse(BaseModel):
+    scripts: List[DendroScript]
+    success: bool
+
+@router.get("/{project_id}/scripts")
+@api_route_wrapper
+async def get_scripts(project_id):
+    scripts = await fetch_project_scripts(project_id)
+    return GetScriptsResponse(scripts=scripts, success=True)
+
+# add script
+class AddScriptRequest(BaseModel):
+    name: str
+
+class AddScriptResponse(BaseModel):
+    scriptId: str
+    success: bool
+
+@router.post("/{project_id}/scripts")
+@api_route_wrapper
+async def add_script(project_id, data: AddScriptRequest, github_access_token: str = Header(...)) -> AddScriptResponse:
+    # authenticate the request
+    user_id = await _authenticate_gui_request(github_access_token=github_access_token, raise_on_not_authenticated=True)
+    assert user_id
+
+    project = await fetch_project(project_id, raise_on_not_found=True)
+    assert project
+
+    _check_user_can_edit_project(project, user_id)
+
+    # parse the request
+    name = data.name
+
+    script_id = _create_random_id(8)
+    script = DendroScript(
+        scriptId=script_id,
+        projectId=project_id,
+        userId=user_id,
+        scriptName=name,
+        content='',
+        timestampCreated=time.time(),
+        timestampModified=time.time()
+    )
+
+    await insert_script(script)
+
+    return AddScriptResponse(scriptId=script_id, success=True)
+
 # set project publicly readable
 class SetProjectPubliclyReadableRequest(BaseModel):
     publiclyReadable: bool

python/dendro/api_helpers/routers/gui/router.py

@@ -4,6 +4,7 @@
 from .compute_resource_routes import router as compute_resource_router
 from .file_routes import router as file_router
 from .job_routes import router as job_router
+from .script_routes import router as script_router
 from .github_auth_routes import router as github_auth_router
 from .user_routes import router as user_router
 from .usage_routes import router as usage_router
@@ -16,6 +17,7 @@
 router.include_router(compute_resource_router, prefix="/compute_resources")
 router.include_router(file_router)
 router.include_router(job_router, prefix="/jobs")
+router.include_router(script_router, prefix="/scripts")
 router.include_router(github_auth_router)
 router.include_router(user_router, prefix="/users")
 router.include_router(usage_router, prefix="/usage")

python/dendro/api_helpers/routers/gui/script_routes.py

@@ -0,0 +1,100 @@
+from fastapi import APIRouter, Header
+from .... import BaseModel
+from ....common.dendro_types import DendroScript
+from ._authenticate_gui_request import _authenticate_gui_request
+from ...core._get_project_role import _check_user_can_edit_project
+from ..common import api_route_wrapper
+from ...clients.db import fetch_script as db_fetch_script, delete_script as db_delete_script, set_script_content as db_set_script_content, rename_script as db_rename_script
+from ...clients.db import fetch_project as db_fetch_project
+
+router = APIRouter()
+
+# get script
+class GetScriptResponse(BaseModel):
+    script: DendroScript
+    success: bool
+
+class ScriptNotFoundException(Exception):
+    pass
+
+@router.get("/{script_id}")
+@api_route_wrapper
+async def get_script(script_id) -> GetScriptResponse:
+    script = await db_fetch_script(script_id)
+    assert script
+    return GetScriptResponse(script=script, success=True)
+
+# delete script
+class DeleteScriptResponse(BaseModel):
+    success: bool
+
+@router.delete("/{script_id}")
+@api_route_wrapper
+async def delete_script(script_id, github_access_token: str = Header(...)) -> DeleteScriptResponse:
+    # authenticate the request
+    user_id = await _authenticate_gui_request(github_access_token=github_access_token, raise_on_not_authenticated=True)
+    assert user_id
+
+    script = await db_fetch_script(script_id)
+    assert script
+
+    project = await db_fetch_project(script.projectId, raise_on_not_found=True)
+    assert project
+
+    _check_user_can_edit_project(project, user_id)
+
+    await db_delete_script(script_id)
+
+    return DeleteScriptResponse(success=True)
+
+# Set script content
+class SetScriptContentRequest(BaseModel):
+    content: str
+
+class SetScriptContentResponse(BaseModel):
+    success: bool
+
+@router.put("/{script_id}/content")
+@api_route_wrapper
+async def set_script_content(script_id, data: SetScriptContentRequest, github_access_token: str = Header(...)) -> SetScriptContentResponse:
+    # authenticate the request
+    user_id = await _authenticate_gui_request(github_access_token=github_access_token, raise_on_not_authenticated=True)
+    assert user_id
+
+    script = await db_fetch_script(script_id)
+    assert script
+
+    project = await db_fetch_project(script.projectId, raise_on_not_found=True)
+    assert project
+
+    _check_user_can_edit_project(project, user_id)
+
+    await db_set_script_content(script_id, data.content)
+
+    return SetScriptContentResponse(success=True)
+
+# Rename script
+class RenameScriptRequest(BaseModel):
+    name: str
+
+class RenameScriptResponse(BaseModel):
+    success: bool
+
+@router.put("/{script_id}/name")
+@api_route_wrapper
+async def rename_script(script_id, data: RenameScriptRequest, github_access_token: str = Header(...)) -> RenameScriptResponse:
+    # authenticate the request
+    user_id = await _authenticate_gui_request(github_access_token=github_access_token, raise_on_not_authenticated=True)
+    assert user_id
+
+    script = await db_fetch_script(script_id)
+    assert script
+
+    project = await db_fetch_project(script.projectId, raise_on_not_found=True)
+    assert project
+
+    _check_user_can_edit_project(project, user_id)
+
+    await db_rename_script(script_id, data.name)
+
+    return RenameScriptResponse(success=True)