From 118b0eda76e01ce42c4b9bf0aab1c6b56a38c9ad Mon Sep 17 00:00:00 2001 From: Gianluca Ficarelli Date: Wed, 12 Nov 2025 15:16:00 +0100 Subject: [PATCH] Add notebook_required_files to analysis notebook template and result --- ...ed96_add_asset_label_analysis_notebook_.py | 196 ++++++++++++++++++ app/db/model.py | 4 +- app/db/types.py | 11 +- docs/asset-labels.md | 92 ++++---- scripts/export/build_database_archive.sh | 4 +- 5 files changed, 258 insertions(+), 49 deletions(-) create mode 100644 alembic/versions/20251112_151700_4b526a1ded96_add_asset_label_analysis_notebook_.py diff --git a/alembic/versions/20251112_151700_4b526a1ded96_add_asset_label_analysis_notebook_.py b/alembic/versions/20251112_151700_4b526a1ded96_add_asset_label_analysis_notebook_.py new file mode 100644 index 00000000..7ee21dcb --- /dev/null +++ b/alembic/versions/20251112_151700_4b526a1ded96_add_asset_label_analysis_notebook_.py @@ -0,0 +1,196 @@ +"""Add asset label analysis_notebook_template + +Revision ID: 4b526a1ded96 +Revises: a33cdde1ea6a +Create Date: 2025-11-12 15:17:00.381264 + +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from alembic_postgresql_enum import TableReference + +from sqlalchemy import Text +import app.db.types + +# revision identifiers, used by Alembic. +revision: str = "4b526a1ded96" +down_revision: Union[str, None] = "a33cdde1ea6a" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.sync_enum_values( + enum_schema="public", + enum_name="contenttype", + new_values=[ + "application/json", + "application/swc", + "application/nrrd", + "application/obj", + "application/hoc", + "application/asc", + "application/abf", + "application/nwb", + "application/x-hdf5", + "text/plain", + "application/vnd.directory", + "application/mod", + "application/pdf", + "image/png", + "image/jpeg", + "model/gltf-binary", + "application/gzip", + "image/webp", + "application/x-ipynb+json", + "application/zip", + ], + affected_columns=[ + TableReference(table_schema="public", table_name="asset", column_name="content_type") + ], + enum_values_to_rename=[], + ) + op.sync_enum_values( + enum_schema="public", + enum_name="assetlabel", + new_values=[ + "morphology", + "morphology_with_spines", + "cell_composition_summary", + "cell_composition_volumes", + "single_neuron_synaptome_config", + "single_neuron_synaptome_simulation_data", + "single_neuron_simulation_data", + "sonata_circuit", + "compressed_sonata_circuit", + "circuit_figures", + "circuit_analysis_data", + "circuit_connectivity_matrices", + "nwb", + "neuron_hoc", + "emodel_optimization_output", + "sonata_simulation_config", + "simulation_generation_config", + "ion_channel_modeling_generation_config", + "custom_node_sets", + "campaign_generation_config", + "campaign_summary", + "replay_spikes", + "voltage_report", + "spike_report", + "neuron_mechanisms", + "brain_atlas_annotation", + "brain_atlas_region_mesh", + "voxel_densities", + "validation_result_figure", + "validation_result_details", + "simulation_designer_image", + "circuit_visualization", + "node_stats", + "network_stats_a", + "network_stats_b", + "cell_surface_mesh", + "jupyter_notebook", + "requirements", + "notebook_required_files", + "ion_channel_model_figure", + "ion_channel_model_figure_summary_json", + "ion_channel_model_thumbnail", + "circuit_extraction_config", + ], + affected_columns=[ + TableReference(table_schema="public", table_name="asset", column_name="label") + ], + enum_values_to_rename=[], + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.sync_enum_values( + enum_schema="public", + enum_name="assetlabel", + new_values=[ + "morphology", + "morphology_with_spines", + "cell_composition_summary", + "cell_composition_volumes", + "single_neuron_synaptome_config", + "single_neuron_synaptome_simulation_data", + "single_neuron_simulation_data", + "sonata_circuit", + "compressed_sonata_circuit", + "circuit_figures", + "circuit_analysis_data", + "circuit_connectivity_matrices", + "nwb", + "neuron_hoc", + "emodel_optimization_output", + "sonata_simulation_config", + "simulation_generation_config", + "ion_channel_modeling_generation_config", + "custom_node_sets", + "campaign_generation_config", + "campaign_summary", + "replay_spikes", + "voltage_report", + "spike_report", + "neuron_mechanisms", + "brain_atlas_annotation", + "brain_atlas_region_mesh", + "voxel_densities", + "validation_result_figure", + "validation_result_details", + "simulation_designer_image", + "circuit_visualization", + "node_stats", + "network_stats_a", + "network_stats_b", + "cell_surface_mesh", + "jupyter_notebook", + "requirements", + "ion_channel_model_figure", + "ion_channel_model_figure_summary_json", + "ion_channel_model_thumbnail", + "circuit_extraction_config", + ], + affected_columns=[ + TableReference(table_schema="public", table_name="asset", column_name="label") + ], + enum_values_to_rename=[], + ) + op.sync_enum_values( + enum_schema="public", + enum_name="contenttype", + new_values=[ + "application/json", + "application/swc", + "application/nrrd", + "application/obj", + "application/hoc", + "application/asc", + "application/abf", + "application/nwb", + "application/x-hdf5", + "text/plain", + "application/vnd.directory", + "application/mod", + "application/pdf", + "image/png", + "image/jpeg", + "model/gltf-binary", + "application/gzip", + "image/webp", + "application/x-ipynb+json", + ], + affected_columns=[ + TableReference(table_schema="public", table_name="asset", column_name="content_type") + ], + enum_values_to_rename=[], + ) + # ### end Alembic commands ### diff --git a/app/db/model.py b/app/db/model.py index 51eec901..df2dac26 100644 --- a/app/db/model.py +++ b/app/db/model.py @@ -1963,6 +1963,7 @@ class AnalysisNotebookTemplate(Entity, NameDescriptionVectorMixin): Assets: - a .ipynb file. - requirements.txt produced with `pip freeze` if possible. + - an optional zip archive with label `notebook_required_files`. """ __tablename__ = EntityType.analysis_notebook_template.value @@ -2040,7 +2041,8 @@ class AnalysisNotebookResult(Entity, NameDescriptionVectorMixin): id (uuid.UUID): Primary key, referencing the entity ID. Assets: - - a .ipynb file + - a .ipynb file. + - an optional zip archive with label `notebook_required_files`. """ __tablename__ = EntityType.analysis_notebook_result.value diff --git a/app/db/types.py b/app/db/types.py index deebadc8..6a43a4db 100644 --- a/app/db/types.py +++ b/app/db/types.py @@ -379,6 +379,7 @@ class ContentType(StrEnum): gzip = "application/gzip" webp = "image/webp" ipynb = "application/x-ipynb+json" + zip = "application/zip" class AssetLabel(StrEnum): @@ -422,6 +423,7 @@ class AssetLabel(StrEnum): cell_surface_mesh = auto() jupyter_notebook = auto() requirements = auto() + notebook_required_files = auto() ion_channel_model_figure = auto() ion_channel_model_figure_summary_json = auto() ion_channel_model_thumbnail = auto() @@ -460,6 +462,7 @@ class LabelRequirements(BaseModel): ), ContentType.webp: (".webp",), ContentType.ipynb: (".ipynb",), + ContentType.zip: (".zip",), } ALLOWED_ASSET_LABELS_PER_ENTITY: dict[ @@ -469,6 +472,9 @@ class LabelRequirements(BaseModel): AssetLabel.jupyter_notebook: [ LabelRequirements(content_type=ContentType.ipynb, is_directory=False) ], + AssetLabel.notebook_required_files: [ + LabelRequirements(content_type=ContentType.zip, is_directory=False) + ], AssetLabel.requirements: [ LabelRequirements(content_type=ContentType.text, is_directory=False) ], @@ -481,7 +487,10 @@ class LabelRequirements(BaseModel): EntityType.analysis_notebook_result: { AssetLabel.jupyter_notebook: [ LabelRequirements(content_type=ContentType.ipynb, is_directory=False) - ] + ], + AssetLabel.notebook_required_files: [ + LabelRequirements(content_type=ContentType.zip, is_directory=False) + ], }, EntityType.brain_atlas: { AssetLabel.brain_atlas_annotation: [ diff --git a/docs/asset-labels.md b/docs/asset-labels.md index 06514eef..edc6f12f 100644 --- a/docs/asset-labels.md +++ b/docs/asset-labels.md @@ -1,45 +1,47 @@ -| Type | Value | Content-Type | Suffix | Description | -| ------------------------------------- | ------------------------------------------------- | ------------------ | --------------- | ----------------------------------------------------------------------------------------------------------- | -| analysis_notebook_template | jupyter_notebook | application/x-ipynb+json | .ipynb | Jupyter notebook file. | -| analysis_notebook_template | requirements | text/plain | .txt | requirements.txt file with the required packages, frozen if possible. | -| analysis_notebook_environment | requirements | text/plain | .txt | requirements.txt file with the frozen packages. | -| analysis_notebook_result | jupyter_notebook | application/x-ipynb+json | .ipynb | Jupyter notebook file. | -| brain_atlas | brain_atlas_annotation | application/nrrd | .nrrd | Brain atlas annotation nrrd volume. | -| brain_atlas_region | brain_atlas_region_mesh | application/obj | .obj | Brain atlas region mesh geometry object. | -| cell_composition | cell_composition_summary | application/json | .json | Region/mtype/etype densities summary | -| cell_composition | cell_composition_volumes | application/json | .json | mtype/etype voxel densities composition | -| em_cell_mesh | cell_surface_mesh | application/x-hdf5 | .h5 | a triangle mesh describing the surface of a cell in h5 format | -| | | application/obj | .obj | a triangle mesh describing the surface of a cell in obj format | -| circuit | sonata_circuit | N/A | N/A (directory) | SONATA circuit, but have a circuit_config.json in the root of the directory | -| circuit | compressed_sonata_circuit | application/gzip | .gz | Compressed SONATA circuit in GZIP format | -| circuit | circuit_connectivity_matrices | N/A | N/A (directory) | Connectivity matrices in ConnectomeUtilities format, with a matrix_config.json in the root of the directory | -| circuit | simulation_designer_image | image/png | .png | Circuit image used by simulation designer GUI | -| circuit | circuit_visualization | image/webp | .webp | Circuit visualization image | -| circuit | node_stats | image/webp | .webp | Circuit node statistics image | -| circuit | network_stats_a | image/webp | .webp | Circuit network statistics image a | -| circuit | network_stats_b | image/webp | .webp | Circuit network statistics image b | -| circuit | circuit_analysis_data | N/A | N/A (directory) | Circuit analysis data with an analysis_config.json in the root of the directory | -| circuit | circuit_figures | N/A | N/A (directory) | Circuit figures with a figure_config.json in the root of the directory | -| electrical_cell_recording | nwb | application/nwb | .nwb | Electrophysiological timeseries data | -| emodel | emodel_optimization_output | application/json | .json | Electrical model optimized parameters, and electrical feature: values and scores | -| emodel | neuron_hoc | application/hoc | .hoc | Electrical model NEURON template | -| ion_channel_model | neuron_mechanisms | application/mod | .mod | Ionic mechanisms file | -| me_type_density | voxel_densities | application/nrrd | .nrrd | Morpho-electric cell voxel densities | -| cell_morphology | morphology | application/asc | .asc | Morphology in Neurolucida ASCII format | -| | | application/swc | .swc | Morphology in SWC format | -| | | application/x-hdf5 | .h5 | Morphology in HDF5 format | -| simulation | custom_node_sets | application/json | .json | Node set groups for regions, mtypes, etc. | -| simulation | replay_spikes | application/x-hdf5 | .h5 | | -| simulation | simulation_generation_config | application/json | .json | | -| simulation | sonata_simulation_config | application/json | .json | Simulation SONATA configuration | -| simulation_campaign | campaign_generation_config | application/json | .json | Campaign configuration | -| simulation_campaign | campaign_summary | application/json | .json | Summary of generated campaign listing all created simulation configs | -| simulation_result | spike_report | application/x-hdf5 | .h5 | Simulation spikes report | -| simulation_result | voltage_report | application/x-hdf5 | .h5 | Simulation voltage report | -| | | application/nwb | .nwb | Simulation voltage report in NWB format | -| single_neuron_simulation | single_neuron_simulation_data | application/json | .json | single neuron simulation configuration and timeseries output | -| single_neuron_synaptome | single_neuron_synaptome_config | application/json | .json | single neuron synaptome configuration | -| single_neuron_synaptome_simulation | single_neuron_synaptome_simulation_data | application/json | .json | single neuron synaptome simulation configuration and timeseries output | -| validation_result | validation_result_figure | application/pdf | .pdf | Validation result figure, pdf | -| | | application/png | .png | Validation result figure, png (legacy) | -| validation_result | validation_result_details | text/plain | .txt | Log and details about the validation execution | +| Type | Value | Content-Type | Suffix | Description | +| ---------------------------------- | --------------------------------------- | ------------------------ | --------------- | ----------------------------------------------------------------------------------------------------------- | +| analysis_notebook_template | jupyter_notebook | application/x-ipynb+json | .ipynb | Jupyter notebook file. | +| analysis_notebook_template | notebook_required_files | application/zip | .zip | Optional archive for additional required files not available as entities. | +| analysis_notebook_template | requirements | text/plain | .txt | requirements.txt file with the required packages, frozen if possible. | +| analysis_notebook_environment | requirements | text/plain | .txt | requirements.txt file with the frozen packages. | +| analysis_notebook_result | jupyter_notebook | application/x-ipynb+json | .ipynb | Jupyter notebook file. | +| analysis_notebook_result | notebook_required_files | application/zip | .zip | Optional archive for additional required files not available as entities. | +| brain_atlas | brain_atlas_annotation | application/nrrd | .nrrd | Brain atlas annotation nrrd volume. | +| brain_atlas_region | brain_atlas_region_mesh | application/obj | .obj | Brain atlas region mesh geometry object. | +| cell_composition | cell_composition_summary | application/json | .json | Region/mtype/etype densities summary | +| cell_composition | cell_composition_volumes | application/json | .json | mtype/etype voxel densities composition | +| em_cell_mesh | cell_surface_mesh | application/x-hdf5 | .h5 | a triangle mesh describing the surface of a cell in h5 format | +| | | application/obj | .obj | a triangle mesh describing the surface of a cell in obj format | +| circuit | sonata_circuit | N/A | N/A (directory) | SONATA circuit, but have a circuit_config.json in the root of the directory | +| circuit | compressed_sonata_circuit | application/gzip | .gz | Compressed SONATA circuit in GZIP format | +| circuit | circuit_connectivity_matrices | N/A | N/A (directory) | Connectivity matrices in ConnectomeUtilities format, with a matrix_config.json in the root of the directory | +| circuit | simulation_designer_image | image/png | .png | Circuit image used by simulation designer GUI | +| circuit | circuit_visualization | image/webp | .webp | Circuit visualization image | +| circuit | node_stats | image/webp | .webp | Circuit node statistics image | +| circuit | network_stats_a | image/webp | .webp | Circuit network statistics image a | +| circuit | network_stats_b | image/webp | .webp | Circuit network statistics image b | +| circuit | circuit_analysis_data | N/A | N/A (directory) | Circuit analysis data with an analysis_config.json in the root of the directory | +| circuit | circuit_figures | N/A | N/A (directory) | Circuit figures with a figure_config.json in the root of the directory | +| electrical_cell_recording | nwb | application/nwb | .nwb | Electrophysiological timeseries data | +| emodel | emodel_optimization_output | application/json | .json | Electrical model optimized parameters, and electrical feature: values and scores | +| emodel | neuron_hoc | application/hoc | .hoc | Electrical model NEURON template | +| ion_channel_model | neuron_mechanisms | application/mod | .mod | Ionic mechanisms file | +| me_type_density | voxel_densities | application/nrrd | .nrrd | Morpho-electric cell voxel densities | +| cell_morphology | morphology | application/asc | .asc | Morphology in Neurolucida ASCII format | +| | | application/swc | .swc | Morphology in SWC format | +| | | application/x-hdf5 | .h5 | Morphology in HDF5 format | +| simulation | custom_node_sets | application/json | .json | Node set groups for regions, mtypes, etc. | +| simulation | replay_spikes | application/x-hdf5 | .h5 | | +| simulation | simulation_generation_config | application/json | .json | | +| simulation | sonata_simulation_config | application/json | .json | Simulation SONATA configuration | +| simulation_campaign | campaign_generation_config | application/json | .json | Campaign configuration | +| simulation_campaign | campaign_summary | application/json | .json | Summary of generated campaign listing all created simulation configs | +| simulation_result | spike_report | application/x-hdf5 | .h5 | Simulation spikes report | +| simulation_result | voltage_report | application/x-hdf5 | .h5 | Simulation voltage report | +| | | application/nwb | .nwb | Simulation voltage report in NWB format | +| single_neuron_simulation | single_neuron_simulation_data | application/json | .json | single neuron simulation configuration and timeseries output | +| single_neuron_synaptome | single_neuron_synaptome_config | application/json | .json | single neuron synaptome configuration | +| single_neuron_synaptome_simulation | single_neuron_synaptome_simulation_data | application/json | .json | single neuron synaptome simulation configuration and timeseries output | +| validation_result | validation_result_figure | application/pdf | .pdf | Validation result figure, pdf | +| | | application/png | .png | Validation result figure, png (legacy) | +| validation_result | validation_result_details | text/plain | .txt | Log and details about the validation execution | diff --git a/scripts/export/build_database_archive.sh b/scripts/export/build_database_archive.sh index 626d50c1..f8fdbe56 100755 --- a/scripts/export/build_database_archive.sh +++ b/scripts/export/build_database_archive.sh @@ -2,7 +2,7 @@ # Automatically generated, do not edit! set -euo pipefail SCRIPT_VERSION="1" -SCRIPT_DB_VERSION="a33cdde1ea6a" +SCRIPT_DB_VERSION="4b526a1ded96" echo "DB dump (version $SCRIPT_VERSION for db version $SCRIPT_DB_VERSION)" @@ -261,7 +261,7 @@ install -m 755 /dev/stdin "$WORK_DIR/load.sh" <<'EOF_LOAD_SCRIPT' # Automatically generated, do not edit! set -euo pipefail SCRIPT_VERSION="1" -SCRIPT_DB_VERSION="a33cdde1ea6a" +SCRIPT_DB_VERSION="4b526a1ded96" echo "DB load (version $SCRIPT_VERSION for db version $SCRIPT_DB_VERSION)"