From cb04c75839f47d59b0af5f55da3bf2fb997058f1 Mon Sep 17 00:00:00 2001 From: Mike Gevaert Date: Mon, 2 Jun 2025 13:10:45 +0200 Subject: [PATCH] Wire in assets to the BrainAtlas and BrainAtlasRead --- app/schemas/brain_atlas.py | 5 +- app/service/brain_atlas.py | 12 +++-- app/service/emodel.py | 1 - tests/routers/test_asset.py | 71 +++++++++++--------------- tests/test_brain_atlas.py | 53 +++++++++++++++++-- tests/test_emodel.py | 12 +++-- tests/test_ion_channel_model.py | 12 +++-- tests/test_single_neuron_simulation.py | 19 ++++--- tests/utils.py | 27 ++++++++++ 9 files changed, 147 insertions(+), 65 deletions(-) diff --git a/app/schemas/brain_atlas.py b/app/schemas/brain_atlas.py index 236bac8d..5e196481 100644 --- a/app/schemas/brain_atlas.py +++ b/app/schemas/brain_atlas.py @@ -2,6 +2,7 @@ from pydantic import BaseModel, ConfigDict +from app.schemas.asset import AssetsMixin from app.schemas.base import CreationMixin, IdentifiableMixin, SpeciesRead @@ -13,7 +14,7 @@ class BrainAtlasBase(BaseModel): species: SpeciesRead -class BrainAtlasRead(BrainAtlasBase, CreationMixin, IdentifiableMixin): +class BrainAtlasRead(BrainAtlasBase, CreationMixin, IdentifiableMixin, AssetsMixin): pass @@ -27,5 +28,5 @@ class BrainAtlasRegionBase(BaseModel): brain_region_id: uuid.UUID -class BrainAtlasRegionRead(BrainAtlasRegionBase, CreationMixin, IdentifiableMixin): +class BrainAtlasRegionRead(BrainAtlasRegionBase, CreationMixin, IdentifiableMixin, AssetsMixin): pass diff --git a/app/service/brain_atlas.py b/app/service/brain_atlas.py index d6b6b409..9a091315 100644 --- a/app/service/brain_atlas.py +++ b/app/service/brain_atlas.py @@ -1,5 +1,7 @@ import uuid +from sqlalchemy.orm import selectinload + import app.queries.common from app.db.model import BrainAtlas, BrainAtlasRegion from app.dependencies.auth import UserContextDep @@ -25,7 +27,7 @@ def read_many( facets=None, aliases=None, apply_filter_query_operations=None, - apply_data_query_operations=None, + apply_data_query_operations=lambda select: select.options(selectinload(BrainAtlas.assets)), pagination_request=pagination_request, response_schema_class=BrainAtlasRead, name_to_facet_query_params=None, @@ -40,7 +42,7 @@ def read_one(user_context: UserContextDep, atlas_id: uuid.UUID, db: SessionDep) db_model_class=BrainAtlas, authorized_project_id=user_context.project_id, response_schema_class=BrainAtlasRead, - apply_operations=None, + apply_operations=lambda select: select.options(selectinload(BrainAtlas.assets)), ) @@ -62,7 +64,7 @@ def read_many_region( apply_filter_query_operations=lambda q: q.filter( BrainAtlasRegion.brain_atlas_id == atlas_id ), - apply_data_query_operations=None, + apply_data_query_operations=lambda s: s.options(selectinload(BrainAtlasRegion.assets)), pagination_request=pagination_request, response_schema_class=BrainAtlasRegionRead, name_to_facet_query_params=None, @@ -79,5 +81,7 @@ def read_one_region( db_model_class=BrainAtlasRegion, authorized_project_id=user_context.project_id, response_schema_class=BrainAtlasRegionRead, - apply_operations=lambda q: q.filter(BrainAtlasRegion.brain_atlas_id == atlas_id), + apply_operations=lambda select: select.filter( + BrainAtlasRegion.brain_atlas_id == atlas_id + ).options(selectinload(BrainAtlasRegion.assets)), ) diff --git a/app/service/emodel.py b/app/service/emodel.py index 131071f3..1767f2c0 100644 --- a/app/service/emodel.py +++ b/app/service/emodel.py @@ -40,7 +40,6 @@ def _load(select: sa.Select[tuple[EModel]]): selectinload(EModel.contributions).joinedload(Contribution.role), joinedload(EModel.mtypes), joinedload(EModel.etypes), - selectinload(EModel.assets), selectinload(EModel.ion_channel_models).joinedload(IonChannelModel.species), selectinload(EModel.ion_channel_models).joinedload(IonChannelModel.strain), selectinload(EModel.ion_channel_models).joinedload(IonChannelModel.brain_region), diff --git a/tests/routers/test_asset.py b/tests/routers/test_asset.py index 2381e6ca..de9418c6 100644 --- a/tests/routers/test_asset.py +++ b/tests/routers/test_asset.py @@ -1,12 +1,10 @@ from unittest.mock import ANY -from uuid import UUID import pytest from app.db.model import Asset, Entity from app.db.types import AssetLabel, AssetStatus, EntityType from app.errors import ApiErrorCode -from app.routers.asset import EntityRoute from app.schemas.api import ErrorResponse from app.schemas.asset import AssetRead from app.utils.s3 import build_s3_path @@ -18,6 +16,8 @@ VIRTUAL_LAB_ID, add_db, create_reconstruction_morphology_id, + route, + upload_entity_asset, ) DIFFERENT_ENTITY_TYPE = "experimental_bouton_density" @@ -27,28 +27,6 @@ FILE_EXAMPLE_SIZE = 31 -def _entity_type_to_route(entity_type: EntityType) -> EntityRoute: - return EntityRoute[entity_type.name] - - -def _route(entity_type: EntityType) -> str: - return f"/{_entity_type_to_route(entity_type)}" - - -def _upload_entity_asset( - client, entity_type: EntityType, entity_id: UUID, label: str | None = None -): - with FILE_EXAMPLE_PATH.open("rb") as f: - files = { - # (filename, file (or bytes), content_type, headers) - "file": ("a/b/c.txt", f, "text/plain") - } - data = None - if label: - data = {"label": label} - return client.post(f"{_route(entity_type)}/{entity_id}/assets", files=files, data=data) - - def _get_expected_full_path(entity, path): return build_s3_path( vlab_id=VIRTUAL_LAB_ID, @@ -73,6 +51,17 @@ def entity(client, species_id, strain_id, brain_region_id) -> Entity: return Entity(id=entity_id, type=entity_type) +def _upload_entity_asset(client, entity_type, entity_id, label=None): + with FILE_EXAMPLE_PATH.open("rb") as f: + files = { + # (filename, file (or bytes), content_type, headers) + "file": ("a/b/c.txt", f, "text/plain") + } + return upload_entity_asset( + client=client, entity_type=entity_type, entity_id=entity_id, files=files, label=label + ) + + @pytest.fixture def asset(client, entity) -> AssetRead: response = _upload_entity_asset(client, entity_type=entity.type, entity_id=entity.id) @@ -178,7 +167,7 @@ def test_upload_entity_asset__label(monkeypatch, client, entity): def test_get_entity_asset(client, entity, asset): - response = client.get(f"{_route(entity.type)}/{entity.id}/assets/{asset.id}") + response = client.get(f"{route(entity.type)}/{entity.id}/assets/{asset.id}") assert response.status_code == 200, f"Failed to get asset: {response.text}" data = response.json() @@ -197,20 +186,20 @@ def test_get_entity_asset(client, entity, asset): } # try to get an asset with non-existent entity id - response = client.get(f"{_route(entity.type)}/{MISSING_ID}/assets/{asset.id}") + response = client.get(f"{route(entity.type)}/{MISSING_ID}/assets/{asset.id}") assert response.status_code == 404, f"Unexpected result: {response.text}" error = ErrorResponse.model_validate(response.json()) assert error.error_code == ApiErrorCode.ENTITY_NOT_FOUND # try to get an asset with non-existent asset id - response = client.get(f"{_route(entity.type)}/{entity.id}/assets/{MISSING_ID}") + response = client.get(f"{route(entity.type)}/{entity.id}/assets/{MISSING_ID}") assert response.status_code == 404, f"Unexpected result: {response.text}" error = ErrorResponse.model_validate(response.json()) assert error.error_code == ApiErrorCode.ASSET_NOT_FOUND def test_get_entity_assets(client, entity, asset): - response = client.get(f"{_route(entity.type)}/{entity.id}/assets") + response = client.get(f"{route(entity.type)}/{entity.id}/assets") assert response.status_code == 200, f"Failed to get asset: {response.text}" data = response.json()["data"] @@ -231,7 +220,7 @@ def test_get_entity_assets(client, entity, asset): ] # try to get assets with non-existent entity id - response = client.get(f"{_route(entity.type)}/{MISSING_ID}/assets") + response = client.get(f"{route(entity.type)}/{MISSING_ID}/assets") assert response.status_code == 404, f"Unexpected result: {response.text}" error = ErrorResponse.model_validate(response.json()) assert error.error_code == ApiErrorCode.ENTITY_NOT_FOUND @@ -239,7 +228,7 @@ def test_get_entity_assets(client, entity, asset): def test_download_entity_asset(client, entity, asset): response = client.get( - f"{_route(entity.type)}/{entity.id}/assets/{asset.id}/download", + f"{route(entity.type)}/{entity.id}/assets/{asset.id}/download", follow_redirects=False, ) @@ -250,20 +239,20 @@ def test_download_entity_asset(client, entity, asset): assert expected_params.issubset(response.next_request.url.params) # try to download an asset with non-existent entity id - response = client.get(f"{_route(entity.type)}/{MISSING_ID}/assets/{asset.id}/download") + response = client.get(f"{route(entity.type)}/{MISSING_ID}/assets/{asset.id}/download") assert response.status_code == 404, f"Unexpected result: {response.text}" error = ErrorResponse.model_validate(response.json()) assert error.error_code == ApiErrorCode.ENTITY_NOT_FOUND # try to download an asset with non-existent asset id - response = client.get(f"{_route(entity.type)}/{entity.id}/assets/{MISSING_ID}/download") + response = client.get(f"{route(entity.type)}/{entity.id}/assets/{MISSING_ID}/download") assert response.status_code == 404, f"Unexpected result: {response.text}" error = ErrorResponse.model_validate(response.json()) assert error.error_code == ApiErrorCode.ASSET_NOT_FOUND # when downloading a single file asset_path should not be passed as a parameter response = client.get( - f"{_route(entity.type)}/{entity.id}/assets/{asset.id}/download", + f"{route(entity.type)}/{entity.id}/assets/{asset.id}/download", params={"asset_path": "foo"}, follow_redirects=False, ) @@ -273,21 +262,21 @@ def test_download_entity_asset(client, entity, asset): def test_delete_entity_asset(client, entity, asset): - response = client.delete(f"{_route(entity.type)}/{entity.id}/assets/{asset.id}") + response = client.delete(f"{route(entity.type)}/{entity.id}/assets/{asset.id}") assert response.status_code == 200, f"Failed to delete asset: {response.text}" data = response.json() assert data == asset.model_copy(update={"status": AssetStatus.DELETED}).model_dump(mode="json") # try to delete again the same asset - response = client.delete(f"{_route(entity.type)}/{entity.id}/assets/{asset.id}") + response = client.delete(f"{route(entity.type)}/{entity.id}/assets/{asset.id}") assert response.status_code == 404, f"Unexpected result: {response.text}" # try to delete an asset with non-existent entity id - response = client.delete(f"{_route(entity.type)}/{MISSING_ID}/assets/{asset.id}") + response = client.delete(f"{route(entity.type)}/{MISSING_ID}/assets/{asset.id}") assert response.status_code == 404, f"Unexpected result: {response.text}" # try to delete an asset with non-existent asset id - response = client.delete(f"{_route(entity.type)}/{entity.id}/assets/{MISSING_ID}") + response = client.delete(f"{route(entity.type)}/{entity.id}/assets/{MISSING_ID}") assert response.status_code == 404, f"Unexpected result: {response.text}" @@ -297,7 +286,7 @@ def test_upload_delete_upload_entity_asset(client, entity): data = response.json() asset0 = AssetRead.model_validate(data) - response = client.delete(f"{_route(entity.type)}/{entity.id}/assets/{asset0.id}") + response = client.delete(f"{route(entity.type)}/{entity.id}/assets/{asset0.id}") assert response.status_code == 200, f"Failed to delete asset: {response.text}" # upload the asset with the same path @@ -307,7 +296,7 @@ def test_upload_delete_upload_entity_asset(client, entity): asset1 = AssetRead.model_validate(data) # test that the deleted assets are filtered out - response = client.get(f"{_route(entity.type)}/{entity.id}/assets") + response = client.get(f"{route(entity.type)}/{entity.id}/assets") assert response.status_code == 200, f"Failed to get assest: {response.text}" data = response.json()["data"] @@ -320,7 +309,7 @@ def test_upload_delete_upload_entity_asset(client, entity): def test_download_directory_file(client, entity, asset_directory): response = client.get( - url=f"{_route(entity.type)}/{entity.id}/assets/{asset_directory.id}/download", + url=f"{route(entity.type)}/{entity.id}/assets/{asset_directory.id}/download", params={"asset_path": "file1.txt"}, follow_redirects=False, ) @@ -328,7 +317,7 @@ def test_download_directory_file(client, entity, asset_directory): # asset_path is mandatory if the asset is a direcotory response = client.get( - url=f"{_route(entity.type)}/{entity.id}/assets/{asset_directory.id}/download", + url=f"{route(entity.type)}/{entity.id}/assets/{asset_directory.id}/download", follow_redirects=False, ) assert response.status_code == 409, ( diff --git a/tests/test_brain_atlas.py b/tests/test_brain_atlas.py index e1b4fb67..61770a6a 100644 --- a/tests/test_brain_atlas.py +++ b/tests/test_brain_atlas.py @@ -2,10 +2,12 @@ from unittest.mock import ANY from app.db.model import BrainAtlas, BrainAtlasRegion +from app.db.types import EntityType from . import utils ROUTE = "/brain-atlas" +FILE_EXAMPLE_PATH = utils.TEST_DATA_DIR / "example.json" HIERARCHY = { @@ -59,6 +61,13 @@ def test_brain_atlas(db, client, species_id): authorized_public=True, ), ) + with FILE_EXAMPLE_PATH.open("rb") as f: + utils.upload_entity_asset( + client, + EntityType.brain_atlas, + brain_atlas0.id, + files={"file": ("a/b/c.txt", f, "text/plain")}, + ) brain_atlas1 = utils.add_db( db, BrainAtlas( @@ -71,6 +80,20 @@ def test_brain_atlas(db, client, species_id): ), ) expected = { + "assets": [ + { + "content_type": "text/plain", + "full_path": ANY, + "id": ANY, + "is_directory": False, + "label": None, + "meta": {}, + "path": "a/b/c.txt", + "sha256_digest": "a8124f083a58b9a8ff80cb327dd6895a10d0bc92bb918506da0c9c75906d3f91", + "size": 31, + "status": "created", + } + ], "creation_date": ANY, "hierarchy_id": str(hierarchy_name.id), "id": str(brain_atlas0.id), @@ -99,10 +122,7 @@ def test_brain_atlas(db, client, species_id): data = (("root", False, None), ("blue", False, None), ("red", True, 15), ("grey", True, 10)) ids = {} for brain_atlas, (name, leaf, volume) in it.product( - ( - brain_atlas0, - brain_atlas1, - ), + (brain_atlas0, brain_atlas1), data, ): row = BrainAtlasRegion( @@ -115,12 +135,33 @@ def test_brain_atlas(db, client, species_id): ) ids[brain_atlas.name, name] = utils.add_db(db, row) + with FILE_EXAMPLE_PATH.open("rb") as f: + utils.upload_entity_asset( + client, + EntityType.brain_atlas_region, + entity_id=ids[brain_atlas.name, name].id, + files={"file": ("a/b/c.txt", f, "text/plain")}, + ).raise_for_status() + response = client.get( f"{ROUTE}/{brain_atlas0.id}/regions", params={"order_by": "+creation_date"} ) assert response.status_code == 200 + expected_asset = { + "content_type": "text/plain", + "full_path": ANY, + "id": ANY, + "is_directory": False, + "label": None, + "meta": {}, + "path": "a/b/c.txt", + "sha256_digest": "a8124f083a58b9a8ff80cb327dd6895a10d0bc92bb918506da0c9c75906d3f91", + "size": 31, + "status": "created", + } assert response.json()["data"] == [ { + "assets": [expected_asset], "brain_atlas_id": str(brain_atlas0.id), "brain_region_id": str(regions["root"].id), "creation_date": ANY, @@ -130,6 +171,7 @@ def test_brain_atlas(db, client, species_id): "volume": None, }, { + "assets": [expected_asset], "brain_atlas_id": str(brain_atlas0.id), "brain_region_id": str(regions["blue"].id), "creation_date": ANY, @@ -139,6 +181,7 @@ def test_brain_atlas(db, client, species_id): "volume": None, }, { + "assets": [expected_asset], "brain_atlas_id": str(brain_atlas0.id), "brain_region_id": str(regions["red"].id), "creation_date": ANY, @@ -148,6 +191,7 @@ def test_brain_atlas(db, client, species_id): "volume": 15.0, }, { + "assets": [expected_asset], "brain_atlas_id": str(brain_atlas0.id), "brain_region_id": str(regions["grey"].id), "creation_date": ANY, @@ -161,6 +205,7 @@ def test_brain_atlas(db, client, species_id): response = client.get(f"{ROUTE}/{brain_atlas0.id}/regions/{ids[brain_atlas0.name, 'root'].id}") assert response.status_code == 200 assert response.json() == { + "assets": [expected_asset], "brain_atlas_id": str(brain_atlas0.id), "brain_region_id": str(regions["root"].id), "creation_date": ANY, diff --git a/tests/test_emodel.py b/tests/test_emodel.py index 75de7c9f..472c2ef1 100644 --- a/tests/test_emodel.py +++ b/tests/test_emodel.py @@ -6,9 +6,9 @@ from app.db.types import EntityType from .conftest import CreateIds, EModelIds -from .utils import create_reconstruction_morphology_id -from tests.routers.test_asset import _upload_entity_asset +from .utils import TEST_DATA_DIR, create_reconstruction_morphology_id, upload_entity_asset +FILE_EXAMPLE_PATH = TEST_DATA_DIR / "example.json" ROUTE = "/emodel" @@ -43,7 +43,13 @@ def test_create_emodel(client: TestClient, species_id, strain_id, brain_region_i def test_get_emodel(client: TestClient, emodel_id: str): - _upload_entity_asset(client, EntityType.emodel, uuid.UUID(emodel_id)) + with FILE_EXAMPLE_PATH.open("rb") as f: + upload_entity_asset( + client, + EntityType.emodel, + uuid.UUID(emodel_id), + files={"file": ("a/b/c.txt", f, "text/plain")}, + ) response = client.get(f"{ROUTE}/{emodel_id}") diff --git a/tests/test_ion_channel_model.py b/tests/test_ion_channel_model.py index 0edd9caf..2255005c 100644 --- a/tests/test_ion_channel_model.py +++ b/tests/test_ion_channel_model.py @@ -7,9 +7,9 @@ from app.db.types import EntityType from app.schemas.ion_channel_model import IonChannelModelRead -from .utils import PROJECT_ID, check_brain_region_filter -from tests.routers.test_asset import _upload_entity_asset +from .utils import PROJECT_ID, TEST_DATA_DIR, check_brain_region_filter, upload_entity_asset +FILE_EXAMPLE_PATH = TEST_DATA_DIR / "example.json" ROUTE = "/ion-channel-model" @@ -56,7 +56,13 @@ def test_read_one(client: TestClient, species_id: str, strain_id: str, brain_reg icm_res = create(client, species_id, strain_id, brain_region_id) icm: dict = icm_res.json() icm_id = icm.get("id") - _upload_entity_asset(client, EntityType.ion_channel_model, uuid.UUID(icm_id)) + with FILE_EXAMPLE_PATH.open("rb") as f: + upload_entity_asset( + client, + EntityType.ion_channel_model, + uuid.UUID(icm_id), + files={"file": ("a/b/c.txt", f, "text/plain")}, + ) response = client.get(f"{ROUTE}/{icm_id}") diff --git a/tests/test_single_neuron_simulation.py b/tests/test_single_neuron_simulation.py index 769059dc..0cb1161f 100644 --- a/tests/test_single_neuron_simulation.py +++ b/tests/test_single_neuron_simulation.py @@ -9,13 +9,15 @@ MISSING_ID, MISSING_ID_COMPACT, PROJECT_ID, + TEST_DATA_DIR, add_db, assert_request, check_brain_region_filter, create_brain_region, + upload_entity_asset, ) -from tests.routers.test_asset import _upload_entity_asset +FILE_EXAMPLE_PATH = TEST_DATA_DIR / "example.json" ROUTE = "/single-neuron-simulation" @@ -45,12 +47,15 @@ def test_single_neuron_simulation(client, brain_region_id, memodel_id): ) data = response.json() - _upload_entity_asset( - client, - EntityType.single_neuron_simulation, - data["id"], - label=AssetLabel.single_cell_simulation_data, - ) + + with FILE_EXAMPLE_PATH.open("rb") as f: + upload_entity_asset( + client, + EntityType.single_neuron_simulation, + data["id"], + label=AssetLabel.single_cell_simulation_data, + files={"file": ("a/b/c.txt", f, "text/plain")}, + ) assert data["brain_region"]["id"] == str(brain_region_id), ( f"Failed to get id for reconstruction morphology: {data}" ) diff --git a/tests/utils.py b/tests/utils.py index f3dfec50..c4986fe2 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -2,6 +2,7 @@ import uuid from pathlib import Path from unittest.mock import ANY +from uuid import UUID import sqlalchemy as sa from httpx import Headers @@ -14,6 +15,7 @@ MTypeClassification, ) from app.db.types import EntityType +from app.routers.asset import EntityRoute TEST_DATA_DIR = Path(__file__).parent / "data" @@ -349,3 +351,28 @@ def recurse(i): ret = {row.acronym: row for row in db.execute(sa.select(BrainRegion)).scalars()} return ret + + +def _entity_type_to_route(entity_type: EntityType) -> EntityRoute: + return EntityRoute[entity_type.name] + + +def route(entity_type: EntityType) -> str: + return f"/{_entity_type_to_route(entity_type)}" + + +def upload_entity_asset( + client, + entity_type: EntityType, + entity_id: UUID, + files: dict[str, tuple], + label: str | None = None, +): + """Attach a file to an entity + + files maps to: (filename, file (or bytes), content_type, headers) + """ + data = None + if label: + data = {"label": label} + return client.post(f"{route(entity_type)}/{entity_id}/assets", files=files, data=data)