From 4020e606c05f22dfb2acd670f9bb57d6d7071f1a Mon Sep 17 00:00:00 2001 From: Jeremy Magland Date: Mon, 23 Oct 2023 21:18:17 -0400 Subject: [PATCH] pyright for api_helpers --- api/index.py | 2 +- api_helpers/clients/_get_mongo_client.py | 5 +- api_helpers/clients/db.py | 25 +++--- api_helpers/clients/pubsub.py | 7 +- api_helpers/core/settings.py | 19 ++--- api_helpers/routers/client/router.py | 11 ++- .../routers/compute_resource/router.py | 28 ++++--- .../routers/gui/_authenticate_gui_request.py | 5 +- .../routers/gui/compute_resource_routes.py | 50 ++++++------ api_helpers/routers/gui/create_job_route.py | 20 ++--- api_helpers/routers/gui/file_routes.py | 30 +++---- api_helpers/routers/gui/job_routes.py | 19 +++-- api_helpers/routers/gui/project_routes.py | 79 +++++++++---------- api_helpers/routers/processor/router.py | 5 +- api_helpers/services/_create_output_file.py | 11 ++- api_helpers/services/_crypto_keys.py | 6 +- .../_remove_detached_files_and_jobs.py | 4 +- api_helpers/services/gui/create_job.py | 15 +++- api_helpers/services/gui/delete_project.py | 1 - api_helpers/services/gui/set_file.py | 2 +- 20 files changed, 189 insertions(+), 155 deletions(-) diff --git a/api/index.py b/api/index.py index 6437eb5..d9464b2 100755 --- a/api/index.py +++ b/api/index.py @@ -26,4 +26,4 @@ app.include_router(client_router, prefix="/api/client", tags=["Client"]) # requests from the GUI -app.include_router(gui_router, prefix="/api/gui", tags=["GUI"]) \ No newline at end of file +app.include_router(gui_router, prefix="/api/gui", tags=["GUI"]) diff --git a/api_helpers/clients/_get_mongo_client.py b/api_helpers/clients/_get_mongo_client.py index 624e515..6ef6974 100644 --- a/api_helpers/clients/_get_mongo_client.py +++ b/api_helpers/clients/_get_mongo_client.py @@ -1,5 +1,4 @@ import asyncio -import os from motor.motor_asyncio import AsyncIOMotorClient from ..core.settings import get_settings @@ -8,13 +7,13 @@ def _get_mongo_client() -> AsyncIOMotorClient: # We want one async mongo client per event loop loop = asyncio.get_event_loop() if hasattr(loop, '_mongo_client'): - return loop._mongo_client + return loop._mongo_client # type: ignore # Otherwise, create a new client and store it in the global variable mongo_uri = get_settings().MONGO_URI if mongo_uri is None: print('MONGO_URI environment variable not set') - raise Exception("MONGO_URI environment variable not set") + raise KeyError("MONGO_URI environment variable not set") client = AsyncIOMotorClient(mongo_uri) setattr(loop, '_mongo_client', client) diff --git a/api_helpers/clients/db.py b/api_helpers/clients/db.py index 130d655..d445e2a 100644 --- a/api_helpers/clients/db.py +++ b/api_helpers/clients/db.py @@ -10,7 +10,7 @@ async def fetch_projects_for_user(user_id: Union[str, None]) -> List[ProtocaasProject]: client = _get_mongo_client() projects_collection = client['protocaas']['projects'] - projects = await projects_collection.find({}).to_list(length=None) + projects = await projects_collection.find({}).to_list(length=None) # type: ignore for project in projects: _remove_id_field(project) projects = [ProtocaasProject(**project) for project in projects] # validate projects @@ -26,13 +26,13 @@ async def fetch_projects_with_tag(tag: str) -> List[ProtocaasProject]: projects = await projects_collection.find({ # When you use a query like { "tags": tag } against an array field in MongoDB, it checks if any element of the array matches the value. 'tags': tag - }).to_list(length=None) + }).to_list(length=None) # type: ignore for project in projects: _remove_id_field(project) projects = [ProtocaasProject(**project) for project in projects] # validate projects return projects -async def fetch_project(project_id: str) -> ProtocaasProject: +async def fetch_project(project_id: str) -> Union[ProtocaasProject, None]: client = _get_mongo_client() projects_collection = client['protocaas']['projects'] project = await projects_collection.find_one({'projectId': project_id}) @@ -45,7 +45,7 @@ async def fetch_project(project_id: str) -> ProtocaasProject: async def fetch_project_files(project_id: str) -> List[ProtocaasFile]: client = _get_mongo_client() files_collection = client['protocaas']['files'] - files = await files_collection.find({'projectId': project_id}).to_list(length=None) + files = await files_collection.find({'projectId': project_id}).to_list(length=None) # type: ignore for file in files: _remove_id_field(file) files = [ProtocaasFile(**file) for file in files] # validate files @@ -54,7 +54,7 @@ async def fetch_project_files(project_id: str) -> List[ProtocaasFile]: async def fetch_project_jobs(project_id: str, include_private_keys=False) -> List[ProtocaasJob]: client = _get_mongo_client() jobs_collection = client['protocaas']['jobs'] - jobs = await jobs_collection.find({'projectId': project_id}).to_list(length=None) + jobs = await jobs_collection.find({'projectId': project_id}).to_list(length=None) # type: ignore for job in jobs: _remove_id_field(job) jobs = [ProtocaasJob(**job) for job in jobs] # validate jobs @@ -114,7 +114,7 @@ async def fetch_compute_resource(compute_resource_id: str): async def fetch_compute_resources_for_user(user_id: str): client = _get_mongo_client() compute_resources_collection = client['protocaas']['computeResources'] - compute_resources = await compute_resources_collection.find({'ownerId': user_id}).to_list(length=None) + compute_resources = await compute_resources_collection.find({'ownerId': user_id}).to_list(length=None) # type: ignore for compute_resource in compute_resources: _remove_id_field(compute_resource) compute_resources = [ProtocaasComputeResource(**compute_resource) for compute_resource in compute_resources] # validate compute resources @@ -142,7 +142,7 @@ async def register_compute_resource(compute_resource_id: str, name: str, user_id compute_resource = await compute_resources_collection.find_one({'computeResourceId': compute_resource_id}) if compute_resource is not None: - compute_resources_collection.update_one({'computeResourceId': compute_resource_id}, { + await compute_resources_collection.update_one({'computeResourceId': compute_resource_id}, { '$set': { 'ownerId': user_id, 'name': name, @@ -157,7 +157,7 @@ async def register_compute_resource(compute_resource_id: str, name: str, user_id timestampCreated=time.time(), apps=[] ) - compute_resources_collection.insert_one(new_compute_resource.dict(exclude_none=True)) + await compute_resources_collection.insert_one(new_compute_resource.dict(exclude_none=True)) async def fetch_compute_resource_jobs(compute_resource_id: str, statuses: Union[List[str], None], include_private_keys: bool) -> List[ProtocaasJob]: client = _get_mongo_client() @@ -166,11 +166,11 @@ async def fetch_compute_resource_jobs(compute_resource_id: str, statuses: Union[ jobs = await jobs_collection.find({ 'computeResourceId': compute_resource_id, 'status': {'$in': statuses} - }).to_list(length=None) + }).to_list(length=None) # type: ignore else: jobs = await jobs_collection.find({ 'computeResourceId': compute_resource_id - }).to_list(length=None) + }).to_list(length=None) # type: ignore for job in jobs: _remove_id_field(job) jobs = [ProtocaasJob(**job) for job in jobs] # validate jobs @@ -197,12 +197,15 @@ async def update_compute_resource_node(compute_resource_id: str, compute_resourc } }, upsert=True) +class ComputeResourceNotFoundError(Exception): + pass + async def set_compute_resource_spec(compute_resource_id: str, spec: ComputeResourceSpec): client = _get_mongo_client() compute_resources_collection = client['protocaas']['computeResources'] compute_resource = await compute_resources_collection.find_one({'computeResourceId': compute_resource_id}) if compute_resource is None: - raise Exception(f"No compute resource with ID {compute_resource_id}") + raise ComputeResourceNotFoundError(f"No compute resource with ID {compute_resource_id}") await compute_resources_collection.update_one({'computeResourceId': compute_resource_id}, { '$set': { 'spec': spec.dict(exclude_none=True) diff --git a/api_helpers/clients/pubsub.py b/api_helpers/clients/pubsub.py index 63f772a..fb3a141 100644 --- a/api_helpers/clients/pubsub.py +++ b/api_helpers/clients/pubsub.py @@ -1,9 +1,12 @@ -import os import json import aiohttp import urllib.parse from ..core.settings import get_settings + +class PubsubError(Exception): + pass + async def publish_pubsub_message(*, channel: str, message: dict): settings = get_settings() # see https://www.pubnub.com/docs/sdks/rest-api/publish-message-to-channel @@ -23,5 +26,5 @@ async def publish_pubsub_message(*, channel: str, message: dict): async with aiohttp.ClientSession() as session: async with session.get(url, headers=headers) as resp: if resp.status != 200: - raise Exception(f"Error publishing to pubsub: {resp.status} {resp.text}") + raise PubsubError(f"Error publishing to pubsub: {resp.status} {resp.text}") return True diff --git a/api_helpers/core/settings.py b/api_helpers/core/settings.py index 20c1c30..3c148d8 100644 --- a/api_helpers/core/settings.py +++ b/api_helpers/core/settings.py @@ -1,3 +1,4 @@ +from typing import Optional from pydantic import BaseModel import os @@ -7,19 +8,19 @@ class Settings(BaseModel): # General app config - MONGO_URI: str = os.environ.get("MONGO_URI") + MONGO_URI: Optional[str] = os.environ.get("MONGO_URI") - PUBNUB_SUBSCRIBE_KEY: str = os.environ.get("VITE_PUBNUB_SUBSCRIBE_KEY") - PUBNUB_PUBLISH_KEY: str = os.environ.get("PUBNUB_PUBLISH_KEY") + PUBNUB_SUBSCRIBE_KEY: Optional[str] = os.environ.get("VITE_PUBNUB_SUBSCRIBE_KEY") + PUBNUB_PUBLISH_KEY: Optional[str] = os.environ.get("PUBNUB_PUBLISH_KEY") - GITHUB_CLIENT_ID: str = os.environ.get("VITE_GITHUB_CLIENT_ID") - GITHUB_CLIENT_SECRET: str = os.environ.get("GITHUB_CLIENT_SECRET") + GITHUB_CLIENT_ID: Optional[str] = os.environ.get("VITE_GITHUB_CLIENT_ID") + GITHUB_CLIENT_SECRET: Optional[str] = os.environ.get("GITHUB_CLIENT_SECRET") - DEFAULT_COMPUTE_RESOURCE_ID: str = os.environ.get("VITE_DEFAULT_COMPUTE_RESOURCE_ID") + DEFAULT_COMPUTE_RESOURCE_ID: Optional[str] = os.environ.get("VITE_DEFAULT_COMPUTE_RESOURCE_ID") - OUTPUT_BUCKET_URI: str = os.environ.get("OUTPUT_BUCKET_URI") - OUTPUT_BUCKET_CREDENTIALS: str = os.environ.get("OUTPUT_BUCKET_CREDENTIALS") - OUTPUT_BUCKET_BASE_URL: str = os.environ.get("OUTPUT_BUCKET_BASE_URL") + OUTPUT_BUCKET_URI: Optional[str] = os.environ.get("OUTPUT_BUCKET_URI") + OUTPUT_BUCKET_CREDENTIALS: Optional[str] = os.environ.get("OUTPUT_BUCKET_CREDENTIALS") + OUTPUT_BUCKET_BASE_URL: Optional[str] = os.environ.get("OUTPUT_BUCKET_BASE_URL") def get_settings(): return Settings() diff --git a/api_helpers/routers/client/router.py b/api_helpers/routers/client/router.py index 46357da..11c7da5 100644 --- a/api_helpers/routers/client/router.py +++ b/api_helpers/routers/client/router.py @@ -12,16 +12,19 @@ class GetProjectResponse(BaseModel): project: ProtocaasProject success: bool +class ProjectError(Exception): + pass + @router.get("/projects/{project_id}") async def get_project(project_id) -> GetProjectResponse: try: project = await fetch_project(project_id) if project is None: - raise Exception(f"No project with ID {project_id}") + raise ProjectError(f"No project with ID {project_id}") return GetProjectResponse(project=project, success=True) except Exception as e: traceback.print_exc() - raise HTTPException(status_code=500, detail=str(e)) + raise HTTPException(status_code=500, detail=str(e)) from e # get project files class GetProjectFilesResponse(BaseModel): @@ -35,7 +38,7 @@ async def get_project_files(project_id) -> GetProjectFilesResponse: return GetProjectFilesResponse(files=files, success=True) except Exception as e: traceback.print_exc() - raise HTTPException(status_code=500, detail=str(e)) + raise HTTPException(status_code=500, detail=str(e)) from e # get project jobs class GetProjectJobsResponse(BaseModel): @@ -49,4 +52,4 @@ async def get_project_jobs(project_id) -> GetProjectJobsResponse: return GetProjectJobsResponse(jobs=jobs, success=True) except Exception as e: traceback.print_exc() - raise HTTPException(status_code=500, detail=str(e)) + raise HTTPException(status_code=500, detail=str(e)) from e diff --git a/api_helpers/routers/compute_resource/router.py b/api_helpers/routers/compute_resource/router.py index 832b8e4..62225e6 100644 --- a/api_helpers/routers/compute_resource/router.py +++ b/api_helpers/routers/compute_resource/router.py @@ -14,6 +14,9 @@ class GetAppsResponse(BaseModel): apps: List[ProtocaasComputeResourceApp] success: bool +class ComputeResourceNotFoundException(Exception): + pass + @router.get("/compute_resources/{compute_resource_id}/apps") async def compute_resource_get_apps( compute_resource_id: str, @@ -32,12 +35,12 @@ async def compute_resource_get_apps( compute_resource = await fetch_compute_resource(compute_resource_id) if compute_resource is None: - raise Exception(f"No compute resource with ID {compute_resource_id}") + raise ComputeResourceNotFoundException(f"No compute resource with ID {compute_resource_id}") apps = compute_resource.apps return GetAppsResponse(apps=apps, success=True) except Exception as e: traceback.print_exc() - raise HTTPException(status_code=500, detail=str(e)) + raise HTTPException(status_code=500, detail=str(e)) from e # get pubsub subscription class GetPubsubSubscriptionResponse(BaseModel): @@ -62,10 +65,10 @@ async def compute_resource_get_pubsub_subscription( compute_resource = await fetch_compute_resource(compute_resource_id) if compute_resource is None: - raise Exception(f"No compute resource with ID {compute_resource_id}") + raise ComputeResourceNotFoundException(f"No compute resource with ID {compute_resource_id}") VITE_PUBNUB_SUBSCRIBE_KEY = get_settings().PUBNUB_SUBSCRIBE_KEY if VITE_PUBNUB_SUBSCRIBE_KEY is None: - raise Exception('Environment variable not set: VITE_PUBNUB_SUBSCRIBE_KEY') + raise KeyError('Environment variable not set: VITE_PUBNUB_SUBSCRIBE_KEY') subscription = PubsubSubscription( pubnubSubscribeKey=VITE_PUBNUB_SUBSCRIBE_KEY, pubnubChannel=compute_resource_id, @@ -74,7 +77,7 @@ async def compute_resource_get_pubsub_subscription( return GetPubsubSubscriptionResponse(subscription=subscription, success=True) except Exception as e: traceback.print_exc() - raise HTTPException(status_code=500, detail=str(e)) + raise HTTPException(status_code=500, detail=str(e)) from e # get unfinished jobs class GetUnfinishedJobsResponse(BaseModel): @@ -110,7 +113,7 @@ async def compute_resource_get_unfinished_jobs( return GetUnfinishedJobsResponse(jobs=jobs, success=True) except Exception as e: traceback.print_exc() - raise HTTPException(status_code=500, detail=str(e)) + raise HTTPException(status_code=500, detail=str(e)) from e # set spec class SetSpecRequest(BaseModel): @@ -143,7 +146,14 @@ async def compute_resource_set_spec( return SetSpecResponse(success=True) except Exception as e: traceback.print_exc() - raise HTTPException(status_code=500, detail=str(e)) + raise HTTPException(status_code=500, detail=str(e)) from e + +class UnexpectedException(Exception): + pass + +class InvalidSignatureException(Exception): + pass + def _authenticate_compute_resource_request( compute_resource_id: str, @@ -152,6 +162,6 @@ def _authenticate_compute_resource_request( expected_payload: str ): if compute_resource_payload != expected_payload: - raise Exception('Unexpected payload') + raise UnexpectedException('Unexpected payload') if not _verify_signature_str(compute_resource_payload, compute_resource_id, compute_resource_signature): - raise Exception('Invalid signature') + raise InvalidSignatureException('Invalid signature') diff --git a/api_helpers/routers/gui/_authenticate_gui_request.py b/api_helpers/routers/gui/_authenticate_gui_request.py index b78540d..6607902 100644 --- a/api_helpers/routers/gui/_authenticate_gui_request.py +++ b/api_helpers/routers/gui/_authenticate_gui_request.py @@ -22,6 +22,9 @@ async def _authenticate_gui_request(github_access_token: str): } return user_id +class AuthException(Exception): + pass + async def _get_user_id_for_access_token(github_access_token: str): url = 'https://api.github.com/user' headers = { @@ -31,6 +34,6 @@ async def _get_user_id_for_access_token(github_access_token: str): async with aiohttp.ClientSession() as session: async with session.get(url, headers=headers) as response: if response.status != 200: - raise Exception(f'Error getting user ID from github access token: {response.status}') + raise AuthException(f'Error getting user ID from github access token: {response.status}') data = await response.json() return data['login'] diff --git a/api_helpers/routers/gui/compute_resource_routes.py b/api_helpers/routers/gui/compute_resource_routes.py index 1c8c077..d3f47b2 100644 --- a/api_helpers/routers/gui/compute_resource_routes.py +++ b/api_helpers/routers/gui/compute_resource_routes.py @@ -18,36 +18,42 @@ class GetComputeResourceResponse(BaseModel): computeResource: ProtocaasComputeResource success: bool +class ComputeResourceNotFoundException(Exception): + pass + @router.get("/{compute_resource_id}") async def get_compute_resource(compute_resource_id) -> GetComputeResourceResponse: try: compute_resource = await fetch_compute_resource(compute_resource_id) if compute_resource is None: - raise Exception(f"No compute resource with ID {compute_resource_id}") + raise ComputeResourceNotFoundException(f"No compute resource with ID {compute_resource_id}") return GetComputeResourceResponse(computeResource=compute_resource, success=True) except Exception as e: traceback.print_exc() - raise HTTPException(status_code=500, detail=str(e)) + raise HTTPException(status_code=500, detail=str(e)) from e # get compute resources class GetComputeResourcesResponse(BaseModel): computeResources: List[ProtocaasComputeResource] success: bool +class AuthException(Exception): + pass + @router.get("") async def get_compute_resources(github_access_token: str=Header(...)): try: # authenticate the request user_id = await _authenticate_gui_request(github_access_token) if not user_id: - raise Exception('User is not authenticated') + raise AuthException('User is not authenticated') compute_resources = await fetch_compute_resources_for_user(user_id) return GetComputeResourcesResponse(computeResources=compute_resources, success=True) except Exception as e: traceback.print_exc() - raise HTTPException(status_code=500, detail=str(e)) + raise HTTPException(status_code=500, detail=str(e)) from e # set compute resource apps class SetComputeResourceAppsRequest(BaseModel): @@ -62,17 +68,17 @@ async def set_compute_resource_apps(compute_resource_id, data: SetComputeResourc # authenticate the request user_id = await _authenticate_gui_request(github_access_token) if not user_id: - raise Exception('User is not authenticated') + raise AuthException('User is not authenticated') # parse the request apps = data.apps compute_resource = await fetch_compute_resource(compute_resource_id) if compute_resource is None: - raise Exception(f"No compute resource with ID {compute_resource_id}") + raise ComputeResourceNotFoundException(f"No compute resource with ID {compute_resource_id}") if compute_resource.ownerId != user_id: - raise Exception('User does not have permission to admin this compute resource') + raise AuthException('User does not have permission to admin this compute resource') await update_compute_resource(compute_resource_id, update={ 'apps': [app.dict(exclude_none=True) for app in apps], @@ -82,7 +88,7 @@ async def set_compute_resource_apps(compute_resource_id, data: SetComputeResourc return SetComputeResourceAppsResponse(success=True) except Exception as e: traceback.print_exc() - raise HTTPException(status_code=500, detail=str(e)) + raise HTTPException(status_code=500, detail=str(e)) from e # delete compute resource class DeleteComputeResourceResponse(BaseModel): @@ -94,20 +100,20 @@ async def delete_compute_resource(compute_resource_id, github_access_token: str= # authenticate the request user_id = await _authenticate_gui_request(github_access_token) if not user_id: - raise Exception('User is not authenticated') + raise AuthException('User is not authenticated') compute_resource = await fetch_compute_resource(compute_resource_id) if compute_resource is None: - raise Exception(f"No compute resource with ID {compute_resource_id}") + raise ComputeResourceNotFoundException(f"No compute resource with ID {compute_resource_id}") if compute_resource.ownerId != user_id: - raise Exception('User does not have permission to delete this compute resource') + raise AuthException('User does not have permission to delete this compute resource') await delete_compute_resource(compute_resource_id) return DeleteComputeResourceResponse(success=True) except Exception as e: traceback.print_exc() - raise HTTPException(status_code=500, detail=str(e)) + raise HTTPException(status_code=500, detail=str(e)) from e # get pubsub subscription class GetPubsubSubscriptionResponse(BaseModel): @@ -119,11 +125,11 @@ async def get_pubsub_subscription(compute_resource_id): try: compute_resource = await fetch_compute_resource(compute_resource_id) if compute_resource is None: - raise Exception(f"No compute resource with ID {compute_resource_id}") + raise ComputeResourceNotFoundException(f"No compute resource with ID {compute_resource_id}") VITE_PUBNUB_SUBSCRIBE_KEY = get_settings().PUBNUB_SUBSCRIBE_KEY if VITE_PUBNUB_SUBSCRIBE_KEY is None: - raise Exception('Environment variable not set: VITE_PUBNUB_SUBSCRIBE_KEY') + raise KeyError('Environment variable not set: VITE_PUBNUB_SUBSCRIBE_KEY') subscription = PubsubSubscription( pubnubSubscribeKey=VITE_PUBNUB_SUBSCRIBE_KEY, pubnubChannel=compute_resource_id, @@ -132,7 +138,7 @@ async def get_pubsub_subscription(compute_resource_id): return GetPubsubSubscriptionResponse(subscription=subscription, success=True) except Exception as e: traceback.print_exc() - raise HTTPException(status_code=500, detail=str(e)) + raise HTTPException(status_code=500, detail=str(e)) from e # register compute resource class RegisterComputeResourceRequest(BaseModel): @@ -149,7 +155,7 @@ async def register_compute_resource(data: RegisterComputeResourceRequest, github # authenticate the request user_id = await _authenticate_gui_request(github_access_token) if user_id is None: - raise Exception('User is not authenticated') + raise AuthException('User is not authenticated') # parse the request compute_resource_id = data.computeResourceId @@ -158,14 +164,14 @@ async def register_compute_resource(data: RegisterComputeResourceRequest, github ok = _verify_resource_code(compute_resource_id, resource_code) if not ok: - raise Exception('Invalid resource code') + raise AuthException('Invalid resource code') await db_register_compute_resource(compute_resource_id=compute_resource_id, name=name, user_id=user_id) return RegisterComputeResourceResponse(success=True) except Exception as e: traceback.print_exc() - raise HTTPException(status_code=500, detail=str(e)) + raise HTTPException(status_code=500, detail=str(e)) from e # get jobs for compute resource class GetJobsForComputeResourceResponse(BaseModel): @@ -178,20 +184,20 @@ async def get_jobs_for_compute_resource(compute_resource_id, github_access_token # authenticate the request user_id = await _authenticate_gui_request(github_access_token) if not user_id: - raise Exception('User is not authenticated') + raise AuthException('User is not authenticated') compute_resource = await fetch_compute_resource(compute_resource_id) if compute_resource is None: - raise Exception(f"No compute resource with ID {compute_resource_id}") + raise ComputeResourceNotFoundException(f"No compute resource with ID {compute_resource_id}") if compute_resource.ownerId != user_id: - raise Exception('User does not have permission to view jobs for this compute resource') + raise AuthException('User does not have permission to view jobs for this compute resource') jobs = await fetch_compute_resource_jobs(compute_resource_id, statuses=None, include_private_keys=False) return GetJobsForComputeResourceResponse(jobs=jobs, success=True) except Exception as e: traceback.print_exc() - raise HTTPException(status_code=500, detail=str(e)) + raise HTTPException(status_code=500, detail=str(e)) from e def _verify_resource_code(compute_resource_id: str, resource_code: str) -> bool: # check that timestamp is within 5 minutes of current time diff --git a/api_helpers/routers/gui/create_job_route.py b/api_helpers/routers/gui/create_job_route.py index fd35688..cab1eff 100644 --- a/api_helpers/routers/gui/create_job_route.py +++ b/api_helpers/routers/gui/create_job_route.py @@ -2,26 +2,15 @@ import traceback from pydantic import BaseModel from fastapi import APIRouter, HTTPException, Header + from ._authenticate_gui_request import _authenticate_gui_request -from ...services.gui.create_job import create_job +from ...services.gui.create_job import create_job, CreateJobRequestInputFile, CreateJobRequestOutputFile, CreateJobRequestInputParameter from ...core.protocaas_types import ComputeResourceSpecProcessor router = APIRouter() # create job -class CreateJobRequestInputFile(BaseModel): - name: str - fileName: str - -class CreateJobRequestOutputFile(BaseModel): - name: str - fileName: str - -class CreateJobRequestInputParameter(BaseModel): - name: str - value: Union[Any, None] - class CreateJobRequest(BaseModel): projectId: str processorName: str @@ -36,13 +25,16 @@ class CreateJobResponse(BaseModel): jobId: str success: bool +class AuthException(Exception): + pass + @router.post("/jobs") async def create_job_handler(data: CreateJobRequest, github_access_token: str=Header(...)) -> CreateJobResponse: try: # authenticate the request user_id = await _authenticate_gui_request(github_access_token) if not user_id: - raise Exception('User is not authenticated') + raise AuthException('User is not authenticated') # parse the request project_id = data.projectId diff --git a/api_helpers/routers/gui/file_routes.py b/api_helpers/routers/gui/file_routes.py index 11f7133..ed81c6c 100644 --- a/api_helpers/routers/gui/file_routes.py +++ b/api_helpers/routers/gui/file_routes.py @@ -20,12 +20,11 @@ class GetFileResponse(BaseModel): async def get_file(project_id, file_name): try: file = await fetch_file(project_id, file_name) - if file is None: - raise Exception(f"No file with name {file_name} in project with ID {project_id}") + assert file is not None, f"No file with name {file_name} in project with ID {project_id}" return GetFileResponse(file=file, success=True) except Exception as e: traceback.print_exc() - raise HTTPException(status_code=500, detail=str(e)) + raise HTTPException(status_code=500, detail=str(e)) from e # get files class GetFilesResponse(BaseModel): @@ -39,7 +38,7 @@ async def get_files(project_id): return GetFilesResponse(files=files, success=True) except Exception as e: traceback.print_exc() - raise HTTPException(status_code=500, detail=str(e)) + raise HTTPException(status_code=500, detail=str(e)) from e # set file class SetFileRequest(BaseModel): @@ -52,13 +51,16 @@ class SetFileResponse(BaseModel): fileId: str success: bool +class AuthException(Exception): + pass + @router.put("/projects/{project_id}/files/{file_name:path}") async def set_file(project_id, file_name, data: SetFileRequest, github_access_token: str=Header(...)): try: # authenticate the request user_id = await _authenticate_gui_request(github_access_token) if not user_id: - raise Exception('User not authenticated') + raise AuthException('User not authenticated') # parse the request content = data.content @@ -66,19 +68,20 @@ async def set_file(project_id, file_name, data: SetFileRequest, github_access_to size = data.size metadata = data.metadata + assert size is not None, "size must be specified" + project = await fetch_project(project_id) - if project is None: - raise Exception(f"No project with ID {project_id}") + assert project is not None, f"No project with ID {project_id}" if not _project_is_editable(project, user_id): - raise Exception('User does not have permission to set file content in this project') + raise AuthException('User does not have permission to set file content in this project') file_id = await service_set_file(project_id=project_id, user_id=user_id, file_name=file_name, content=content, job_id=job_id, size=size, metadata=metadata) return SetFileResponse(fileId=file_id, success=True) except Exception as e: traceback.print_exc() - raise HTTPException(status_code=500, detail=str(e)) + raise HTTPException(status_code=500, detail=str(e)) from e # delete file class DeleteFileResponse(BaseModel): @@ -90,14 +93,13 @@ async def delete_file(project_id, file_name, github_access_token: str=Header(... # authenticate the request user_id = await _authenticate_gui_request(github_access_token) if not user_id: - raise Exception('User not authenticated') + raise AuthException('User not authenticated') project = await fetch_project(project_id) - if project is None: - raise Exception(f"No project with ID {project_id}") + assert project is not None, f"No project with ID {project_id}" if not _project_is_editable(project, user_id): - raise Exception('User does not have permission to delete files in this project') + raise AuthException('User does not have permission to delete files in this project') await db_delete_file(project_id, file_name) @@ -107,4 +109,4 @@ async def delete_file(project_id, file_name, github_access_token: str=Header(... return DeleteFileResponse(success=True) except Exception as e: traceback.print_exc() - raise HTTPException(status_code=500, detail=str(e)) + raise HTTPException(status_code=500, detail=str(e)) from e diff --git a/api_helpers/routers/gui/job_routes.py b/api_helpers/routers/gui/job_routes.py index 0e22d48..0a24c15 100644 --- a/api_helpers/routers/gui/job_routes.py +++ b/api_helpers/routers/gui/job_routes.py @@ -15,16 +15,22 @@ class GetJobResponse(BaseModel): job: ProtocaasJob success: bool +class AuthException(Exception): + pass + +class JobNotFoundException(Exception): + pass + @router.get("/{job_id}") async def get_job(job_id) -> GetJobResponse: try: job = await fetch_job(job_id) if job is None: - raise Exception(f"No job with ID {job_id}") + raise JobNotFoundException(f"No job with ID {job_id}") return GetJobResponse(job=job, success=True) except Exception as e: traceback.print_exc() - raise HTTPException(status_code=500, detail=str(e)) + raise HTTPException(status_code=500, detail=str(e)) from e # delete job class DeleteJobResponse(BaseModel): @@ -36,16 +42,17 @@ async def delete_job(job_id, github_access_token: str=Header(...)) -> DeleteJobR # authenticate the request user_id = await _authenticate_gui_request(github_access_token) if not user_id: - raise Exception('User is not authenticated') + raise AuthException('User is not authenticated') job = await fetch_job(job_id) if job is None: - raise Exception(f"No job with ID {job_id}") + raise JobNotFoundException(f"No job with ID {job_id}") project = await fetch_project(job.projectId) + assert project is not None, f"No project with ID {job.projectId}" if not _project_is_editable(project, user_id): - raise Exception('User does not have permission to delete jobs in this project') + raise AuthException('User does not have permission to delete jobs in this project') await db_delete_job(job_id) @@ -55,4 +62,4 @@ async def delete_job(job_id, github_access_token: str=Header(...)) -> DeleteJobR return DeleteJobResponse(success=True) except Exception as e: traceback.print_exc() - raise HTTPException(status_code=500, detail=str(e)) + raise HTTPException(status_code=500, detail=str(e)) from e diff --git a/api_helpers/routers/gui/project_routes.py b/api_helpers/routers/gui/project_routes.py index 62c57b1..11a72b4 100644 --- a/api_helpers/routers/gui/project_routes.py +++ b/api_helpers/routers/gui/project_routes.py @@ -22,12 +22,11 @@ class GetProjectReponse(BaseModel): async def get_project(project_id): try: project = await fetch_project(project_id) - if project is None: - raise Exception(f"No project with ID {project_id}") + assert project is not None, f"No project with ID {project_id}" return GetProjectReponse(project=project, success=True) except Exception as e: traceback.print_exc() - raise HTTPException(status_code=500, detail=str(e)) + raise HTTPException(status_code=500, detail=str(e)) from e # get projects class GetProjectsResponse(BaseModel): @@ -46,7 +45,7 @@ async def get_projects(github_access_token: str=Header(...), tag: Optional[str]= return GetProjectsResponse(projects=projects, success=True) except Exception as e: traceback.print_exc() - raise HTTPException(status_code=500, detail=str(e)) + raise HTTPException(status_code=500, detail=str(e)) from e # create project class CreateProjectRequest(BaseModel): @@ -56,13 +55,16 @@ class CreateProjectResponse(BaseModel): projectId: str success: bool +class AuthException(Exception): + pass + @router.post("") async def create_project(data: CreateProjectRequest, github_access_token: str=Header(...)) -> CreateProjectResponse: try: # authenticate the request user_id = await _authenticate_gui_request(github_access_token) if not user_id: - raise Exception('User is not authenticated') + raise AuthException('User is not authenticated') # parse the request name = data.name @@ -86,7 +88,7 @@ async def create_project(data: CreateProjectRequest, github_access_token: str=He return CreateProjectResponse(projectId=project_id, success=True) except Exception as e: traceback.print_exc() - raise HTTPException(status_code=500, detail=str(e)) + raise HTTPException(status_code=500, detail=str(e)) from e # set project name class SetProjectNameRequest(BaseModel): @@ -101,14 +103,13 @@ async def set_project_name(project_id, data: SetProjectNameRequest, github_acces # authenticate the request user_id = await _authenticate_gui_request(github_access_token) if not user_id: - raise Exception('User is not authenticated') + raise AuthException('User is not authenticated') project = await fetch_project(project_id) - if project is None: - raise Exception(f"No project with ID {project_id}") + assert project is not None, f"No project with ID {project_id}" if not _project_is_editable(project, user_id): - raise Exception('User does not have permission to set project name') + raise AuthException('User does not have permission to set project name') # parse the request name = data.name @@ -121,7 +122,7 @@ async def set_project_name(project_id, data: SetProjectNameRequest, github_acces return SetProjectNameResponse(success=True) except Exception as e: traceback.print_exc() - raise HTTPException(status_code=500, detail=str(e)) + raise HTTPException(status_code=500, detail=str(e)) from e # set project description class SetProjectDescriptionRequest(BaseModel): @@ -136,14 +137,13 @@ async def set_project_description(project_id, data: SetProjectDescriptionRequest # authenticate the request user_id = await _authenticate_gui_request(github_access_token) if not user_id: - raise Exception('User is not authenticated') + raise AuthException('User is not authenticated') project = await fetch_project(project_id) - if project is None: - raise Exception(f"No project with ID {project_id}") + assert project is not None, f"No project with ID {project_id}" if not _project_is_editable(project, user_id): - raise Exception('User does not have permission to set project description') + raise AuthException('User does not have permission to set project description') # parse the request description = data.description @@ -156,7 +156,7 @@ async def set_project_description(project_id, data: SetProjectDescriptionRequest return SetProjectDescriptionResponse(success=True) except Exception as e: traceback.print_exc() - raise HTTPException(status_code=500, detail=str(e)) + raise HTTPException(status_code=500, detail=str(e)) from e # set project tags class SetProjectTagsRequest(BaseModel): @@ -171,14 +171,13 @@ async def set_project_tags(project_id, data: SetProjectTagsRequest, github_acces # authenticate the request user_id = await _authenticate_gui_request(github_access_token) if not user_id: - raise Exception('User is not authenticated') + raise AuthException('User is not authenticated') project = await fetch_project(project_id) - if project is None: - raise Exception(f"No project with ID {project_id}") + assert project is not None, f"No project with ID {project_id}" if not _project_is_editable(project, user_id): - raise Exception('User does not have permission to set project tags') + raise AuthException('User does not have permission to set project tags') # parse the request tags = data.tags @@ -191,7 +190,7 @@ async def set_project_tags(project_id, data: SetProjectTagsRequest, github_acces return SetProjectTagsResponse(success=True) except Exception as e: traceback.print_exc() - raise HTTPException(status_code=500, detail=str(e)) + raise HTTPException(status_code=500, detail=str(e)) from e # delete project class DeleteProjectResponse(BaseModel): @@ -203,21 +202,20 @@ async def delete_project(project_id, github_access_token: str=Header(...)) -> De # authenticate the request user_id = await _authenticate_gui_request(github_access_token) if not user_id: - raise Exception('User is not authenticated') + raise AuthException('User is not authenticated') project = await fetch_project(project_id) - if project is None: - raise Exception(f"No project with ID {project_id}") + assert project is not None, f"No project with ID {project_id}" if not _user_is_project_admin(project, user_id): - raise Exception('User does not have permission to delete this project') + raise AuthException('User does not have permission to delete this project') await service_delete_project(project) return DeleteProjectResponse(success=True) except Exception as e: traceback.print_exc() - raise HTTPException(status_code=500, detail=str(e)) + raise HTTPException(status_code=500, detail=str(e)) from e # get jobs class GetJobsResponse(BaseModel): @@ -231,7 +229,7 @@ async def get_jobs(project_id): return GetJobsResponse(jobs=jobs, success=True) except Exception as e: traceback.print_exc() - raise HTTPException(status_code=500, detail=str(e)) + raise HTTPException(status_code=500, detail=str(e)) from e # set project publicly readable class SetProjectPubliclyReadableRequest(BaseModel): @@ -246,13 +244,12 @@ async def set_project_public(project_id, data: SetProjectPubliclyReadableRequest # authenticate the request user_id = await _authenticate_gui_request(github_access_token) if not user_id: - raise Exception('User is not authenticated') + raise AuthException('User is not authenticated') project = await fetch_project(project_id) - if project is None: - raise Exception(f"No project with ID {project_id}") + assert project is not None, f"No project with ID {project_id}" if not _user_is_project_admin(project, user_id): - raise Exception('User does not have permission to admin this project') + raise AuthException('User does not have permission to admin this project') # parse the request publicly_readable = data.publiclyReadable @@ -265,7 +262,7 @@ async def set_project_public(project_id, data: SetProjectPubliclyReadableRequest return SetProjectPubliclyReadableResponse(success=True) except Exception as e: traceback.print_exc() - raise HTTPException(status_code=500, detail=str(e)) + raise HTTPException(status_code=500, detail=str(e)) from e # set project compute resource id @@ -281,13 +278,12 @@ async def set_project_compute_resource_id(project_id, data: SetProjectComputeRes # authenticate the request user_id = await _authenticate_gui_request(github_access_token) if not user_id: - raise Exception('User is not authenticated') + raise AuthException('User is not authenticated') project = await fetch_project(project_id) - if project is None: - raise Exception(f"No project with ID {project_id}") + assert project is not None, f"No project with ID {project_id}" if not _user_is_project_admin(project, user_id): - raise Exception('User does not have permission to admin this project') + raise AuthException('User does not have permission to admin this project') # parse the request compute_resource_id = data.computeResourceId @@ -300,7 +296,7 @@ async def set_project_compute_resource_id(project_id, data: SetProjectComputeRes return SetProjectComputeResourceIdResponse(success=True) except Exception as e: traceback.print_exc() - raise HTTPException(status_code=500, detail=str(e)) + raise HTTPException(status_code=500, detail=str(e)) from e # set project users class SetProjectUsersRequest(BaseModel): @@ -315,13 +311,12 @@ async def set_project_users(project_id, data: SetProjectUsersRequest, github_acc # authenticate the request user_id = await _authenticate_gui_request(github_access_token) if not user_id: - raise Exception('User is not authenticated') + raise AuthException('User is not authenticated') project = await fetch_project(project_id) - if project is None: - raise Exception(f"No project with ID {project_id}") + assert project is not None, f"No project with ID {project_id}" if not _user_is_project_admin(project, user_id): - raise Exception('User does not have permission to admin this project') + raise AuthException('User does not have permission to admin this project') # parse the request users = data.users @@ -334,4 +329,4 @@ async def set_project_users(project_id, data: SetProjectUsersRequest, github_acc return SetProjectUsersResponse(success=True) except Exception as e: traceback.print_exc() - raise HTTPException(status_code=500, detail=str(e)) + raise HTTPException(status_code=500, detail=str(e)) from e diff --git a/api_helpers/routers/processor/router.py b/api_helpers/routers/processor/router.py index 33ba75f..21bc916 100644 --- a/api_helpers/routers/processor/router.py +++ b/api_helpers/routers/processor/router.py @@ -156,11 +156,12 @@ class ProcessorGetJobOutputUploadUrlResponse(BaseModel): async def processor_get_upload_url(job_id: str, output_name: str, job_private_key: str = Header(...)) -> ProcessorGetJobOutputUploadUrlResponse: try: job = await fetch_job(job_id) + assert job, f"No job with ID {job_id}" if job.jobPrivateKey != job_private_key: - raise Exception(f"Invalid job private key for job {job_id}") + raise ValueError(f"Invalid job private key for job {job_id}") signed_upload_url = await get_upload_url(job=job, output_name=output_name) return ProcessorGetJobOutputUploadUrlResponse(uploadUrl=signed_upload_url, success=True) except Exception as e: traceback.print_exc() - raise HTTPException(status_code=500, detail=str(e)) + raise HTTPException(status_code=500, detail=str(e)) from e diff --git a/api_helpers/services/_create_output_file.py b/api_helpers/services/_create_output_file.py index 9df34b7..cb426d8 100644 --- a/api_helpers/services/_create_output_file.py +++ b/api_helpers/services/_create_output_file.py @@ -58,14 +58,17 @@ async def _create_output_file(*, return new_file.fileId +class GetSizeForRemoteFileException(Exception): + pass + async def _get_size_for_remote_file(url: str) -> int: response = await _head_request(url) if response is None: - raise Exception(f"Unable to HEAD {url}") - size = int(response.headers.get('content-length')) + raise GetSizeForRemoteFileException(f"Unable to HEAD {url}") + size = response.headers.get('content-length') if size is None: - raise Exception(f"Unable to get content-length for {url}") - return size + raise GetSizeForRemoteFileException(f"Unable to get content-length for {url}") + return int(size) async def _head_request(url: str): async with aiohttp.ClientSession() as session: diff --git a/api_helpers/services/_crypto_keys.py b/api_helpers/services/_crypto_keys.py index 06a6f14..36f3c46 100644 --- a/api_helpers/services/_crypto_keys.py +++ b/api_helpers/services/_crypto_keys.py @@ -40,7 +40,7 @@ def _verify_signature_str(msg: str, public_key_hex: str, signature: str): pubk = Ed25519PublicKey.from_public_bytes(bytes.fromhex(public_key_hex)) try: pubk.verify(bytes.fromhex(signature), msg_bytes) - except: + except: # pylint: disable=bare-except return False return True @@ -53,11 +53,11 @@ def generate_keypair(): encoding=serialization.Encoding.Raw, format=serialization.PrivateFormat.Raw, encryption_algorithm=serialization.NoEncryption() - ).hex() + ).hex() # type: ignore public_key_hex = pubk.public_bytes( encoding=serialization.Encoding.Raw, format=serialization.PublicFormat.Raw - ).hex() + ).hex() # type: ignore test_msg = {'a': 1} test_signature = _sign_message(test_msg, public_key_hex, private_key_hex) assert _verify_signature(test_msg, public_key_hex, test_signature) diff --git a/api_helpers/services/_remove_detached_files_and_jobs.py b/api_helpers/services/_remove_detached_files_and_jobs.py index 1adcda6..6eda86b 100644 --- a/api_helpers/services/_remove_detached_files_and_jobs.py +++ b/api_helpers/services/_remove_detached_files_and_jobs.py @@ -10,10 +10,10 @@ async def _remove_detached_files_and_jobs(project_id: str): files = await files_collection.find({ 'projectId': project_id - }).to_list(length=None) + }).to_list(length=None) # type: ignore jobs = await jobs_collection.find({ 'projectId': project_id - }).to_list(length=None) + }).to_list(length=None) # type: ignore for file in files: _remove_id_field(file) diff --git a/api_helpers/services/gui/create_job.py b/api_helpers/services/gui/create_job.py index 41548d9..fe35aec 100644 --- a/api_helpers/services/gui/create_job.py +++ b/api_helpers/services/gui/create_job.py @@ -21,6 +21,12 @@ class CreateJobRequestInputParameter(BaseModel): name: str value: Union[Any, None] +class AuthException(Exception): + pass + +class CreateJobException(Exception): + pass + async def create_job( project_id: str, processor_name: str, @@ -33,21 +39,22 @@ async def create_job( dandi_api_key: Union[str, None] = None ): project = await fetch_project(project_id) + assert project is not None, f"No project with ID {project_id}" if not _project_is_editable(project, user_id): - raise Exception('User does not have permission to create jobs') + raise AuthException('User does not have permission to create jobs') compute_resource_id = project.computeResourceId if not compute_resource_id: compute_resource_id = get_settings().DEFAULT_COMPUTE_RESOURCE_ID if compute_resource_id is None: - raise Exception('Project does not have a compute resource ID, and no default VITE_DEFAULT_COMPUTE_RESOURCE_ID is set in the environment.') + raise KeyError('Project does not have a compute resource ID, and no default VITE_DEFAULT_COMPUTE_RESOURCE_ID is set in the environment.') input_files: List[ProtocaasJobInputFile] = [] # {name, fileId, fileName} for input_file in input_files_from_request: file = await fetch_file(project_id, input_file.fileName) if file is None: - raise Exception(f"Project input file does not exist: {input_file.fileName}") + raise CreateJobException(f"Project input file does not exist: {input_file.fileName}") input_files.append( ProtocaasJobInputFile( name=input_file.name, @@ -102,7 +109,7 @@ def filter_output_file_name(file_name): for input_parameter in input_parameters: pp = next((x for x in processor_spec.parameters if x.name == input_parameter.name), None) if not pp: - raise Exception(f"Processor parameter not found: {input_parameter.name}") + raise CreateJobException(f"Processor parameter not found: {input_parameter.name}") input_parameters2.append( ProtocaasJobInputParameter( name=input_parameter.name, diff --git a/api_helpers/services/gui/delete_project.py b/api_helpers/services/gui/delete_project.py index e0d7049..b9e5f0c 100644 --- a/api_helpers/services/gui/delete_project.py +++ b/api_helpers/services/gui/delete_project.py @@ -1,4 +1,3 @@ -import time from ...core.protocaas_types import ProtocaasProject from ...clients.db import delete_all_files_in_project, delete_all_jobs_in_project, delete_project as db_delete_project diff --git a/api_helpers/services/gui/set_file.py b/api_helpers/services/gui/set_file.py index 57b471e..a7ce6ac 100644 --- a/api_helpers/services/gui/set_file.py +++ b/api_helpers/services/gui/set_file.py @@ -1,5 +1,5 @@ import time -from typing import Union, List, Any +from typing import Union from ...clients.db import fetch_file, delete_file, insert_file, update_project from ...core.protocaas_types import ProtocaasFile from ...core._create_random_id import _create_random_id