feat: compound feat

.github/workflows/deploy-helm.yml

@@ -18,7 +18,7 @@ jobs:
     - name: 'Deploy'
       run: |
         kubecfg_path=${{ runner.temp }}/.kube_config
-        version=$(echo api/VERSION)
+        version=$(cat api/VERSION)
         
         echo "${{ secrets.KUBECONFIG }}" > $kubecfg_path
         helm --kubeconfig=$kubecfg_path status ${{ inputs.DEPLOYMENT_NAME }}

.helm/README.md

@@ -1,6 +1,6 @@
 # Deployment
 
-This document is intended for a documentation on how siibra-explorer can be deployed to a [kubernetes (k8s)](https://kubernetes.io/) cluster via [helm](https://helm.sh/).
+This document is intended for a documentation on how siibra-api can be deployed to a [kubernetes (k8s)](https://kubernetes.io/) cluster via [helm](https://helm.sh/).
 
 ## Active deployments
 

.helm/siibra-api/templates/deployment-worker.yaml

@@ -16,7 +16,6 @@ spec:
     matchLabels:
       role: worker
       queuename: {{ . }}
-      sapiVersion: {{ $.Values.sapiVersion }}
       sapiFlavor: {{ $.Values.sapiFlavor }}
       {{- include "siibra-api.selectorLabels" $ | nindent 6 }}
   template:
@@ -28,7 +27,6 @@ spec:
       labels:
         role: worker
         queuename: {{ . }}
-        sapiVersion: {{ $.Values.sapiVersion }}
         sapiFlavor: {{ $.Values.sapiFlavor }}
         {{- include "siibra-api.labels" $ | nindent 8 }}
 	{{- with $.Values.podLabels }}

.helm/siibra-api/templates/hpa.yaml

@@ -1,32 +1,37 @@
 {{- if .Values.autoscaling.enabled }}
+{{- range .Values.sapiWorkerQueues }}
+---
 apiVersion: autoscaling/v2
 kind: HorizontalPodAutoscaler
 metadata:
-  name: {{ include "siibra-api.fullname" . }}
+  name: {{ include "siibra-api.fullname" $ }}-worker-hpa-{{ . }}
   labels:
-    {{- include "siibra-api.labels" . | nindent 4 }}
+    queuename: {{ . }}
+    {{- include "siibra-api.labels" $ | nindent 4 }}
 spec:
   scaleTargetRef:
     apiVersion: apps/v1
     kind: Deployment
-    name: {{ include "siibra-api.fullname" . }}
-  minReplicas: {{ .Values.autoscaling.minReplicas }}
-  maxReplicas: {{ .Values.autoscaling.maxReplicas }}
+    name: {{ include "siibra-api.fullname" $ }}-worker-{{ . }}
+  minReplicas: {{ $.Values.autoscaling.minReplicas }}
+  maxReplicas: {{ $.Values.autoscaling.maxReplicas }}
   metrics:
-    {{- if .Values.autoscaling.targetCPUUtilizationPercentage }}
+    {{- if $.Values.autoscaling.targetCPUUtilizationPercentage }}
     - type: Resource
       resource:
         name: cpu
         target:
           type: Utilization
-          averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }}
+          averageUtilization: {{ $.Values.autoscaling.targetCPUUtilizationPercentage }}
     {{- end }}
-    {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }}
+    {{- if $.Values.autoscaling.targetMemoryUtilizationPercentage }}
     - type: Resource
       resource:
         name: memory
         target:
           type: Utilization
-          averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }}
+          averageUtilization: {{ $.Values.autoscaling.targetCPUUtilizationPercentage }}
     {{- end }}
+
+{{- end }}
 {{- end }}

.helm/siibra-api/values.yaml

@@ -92,7 +92,7 @@ resourcesWorkerPod:
 autoscaling:
   enabled: true
   minReplicas: 1
-  maxReplicas: 100
+  maxReplicas: 10
   targetCPUUtilizationPercentage: 80
   # targetMemoryUtilizationPercentage: 80
 

api/VERSION

@@ -1 +1 @@
-0.3.16
+0.3.17

api/common/data_handlers/features/types.py

@@ -4,6 +4,7 @@
 from hashlib import md5
 from pathlib import Path
 from zipfile import ZipFile
+import json
 
 @data_decorator(ROLE)
 def all_feature_types() -> List[Dict[str, str]]:
@@ -32,15 +33,15 @@ def get_hierarchy_type(Cls: Type[Any]) -> str:
         {
             'name': get_hierarchy_type(Cls),
             'category': Cls.category,
-        } for Cls in Feature.SUBCLASSES
+        } for Cls in Feature._SUBCLASSES
     ]
 
 @data_decorator(ROLE)
 def get_single_feature_from_id(feature_id: str, **kwargs):
     import siibra
     from api.serialization.util import instance_to_model
     try:
-        feature = siibra.features.Feature.get_instance_by_id(feature_id)
+        feature = siibra.features.Feature._get_instance_by_id(feature_id)
     except Exception as e:
         raise NotFound(str(e))
     else:
@@ -53,18 +54,44 @@ def get_single_feature_plot_from_id(feature_id: str, template="plotly", **kwargs
     import json
 
     try:
-        feature = siibra.features.Feature.get_instance_by_id(feature_id)
+        feature = siibra.features.Feature._get_instance_by_id(feature_id)
     except Exception as e:
         raise NotFound from e
 
     try:
-        plotly_fig = feature.plot(backend="plotly", template=template)
+        plotly_fig = feature.plot(backend="plotly", template=template, **kwargs)
         json_str = to_json(plotly_fig)
         return json.loads(json_str)
 
     except NotImplementedError:
         raise NotFound(f"feature with id {feature_id} is found, but the plot function has not been implemented")
 
+@data_decorator(ROLE)
+def get_single_feature_intents_from_id(feature_id: str, **kwargs):
+    import siibra
+    from api.serialization.util.siibra import RegionalConnectivity
+    from api.serialization.util import instance_to_model
+    from api.models.intents.colorization import RegionMapping, ColorizationIntent
+
+    try:
+        feature = siibra.features.Feature._get_instance_by_id(feature_id)
+    except Exception as e:
+        raise NotFound from e
+
+    if not isinstance(feature, RegionalConnectivity):
+        return []
+
+    region_mappings = [RegionMapping(
+        region=instance_to_model(reg),
+        rgb=rgb,
+    ) for reg, rgb in feature.get_profile_colorscale(**kwargs)]
+
+    return [
+        ColorizationIntent(
+            region_mappings=region_mappings
+        ).dict()
+    ]
+
 @data_decorator(ROLE)
 def get_single_feature_download_zip_path(feature_id: str, **kwargs):
     assert feature_id, f"feature_id must be defined"
@@ -79,7 +106,7 @@ def get_single_feature_download_zip_path(feature_id: str, **kwargs):
         return str(full_filename)
     import siibra
     try:
-        feat = siibra.features.Feature.get_instance_by_id(feature_id)
+        feat = siibra.features.Feature._get_instance_by_id(feature_id)
     except Exception as e:
         logger.error(f"Error finding single feature {feature_id=}, {str(e)}")
         raise NotFound from e
@@ -94,21 +121,21 @@ def get_single_feature_download_zip_path(feature_id: str, **kwargs):
         return str(error_filename)
 
 
-def extract_concept(*, space_id: str=None, parcellation_id: str=None, region_id: str=None, **kwargs):
-    import siibra
+def extract_concept(*, space_id: str=None, parcellation_id: str=None, region_id: str=None, bbox: str=None, **kwargs):
+    from api.serialization.util.siibra import Region, Parcellation, BoundingBox, siibra
     if region_id is not None:
         if parcellation_id is None:
             raise InsufficientParameters(f"if region_id is defined, parcellation_id must also be defined!")
-        region: siibra.core.parcellation.region.Region = siibra.get_region(parcellation_id, region_id)
+        region: Region = siibra.get_region(parcellation_id, region_id)
         return region
 
     if parcellation_id:
-        parcellation: siibra._parcellation.Parcellation = siibra.parcellations[parcellation_id]
+        parcellation: Parcellation = siibra.parcellations[parcellation_id]
         return parcellation
 
     if space_id:
-        space: siibra._space.Space = siibra.spaces[space_id]
-        return space
+        assert bbox, f"If space_id is defined, bbox must be defined!"
+        return BoundingBox(*json.loads(bbox), space_id)
     raise InsufficientParameters(f"concept cannot be extracted")
 
 @data_decorator(ROLE)
@@ -132,29 +159,16 @@ def get_all_all_features(*, space_id: str=None, parcellation_id: str=None, regio
 
 def _get_all_features(*, space_id: str, parcellation_id: str, region_id: str, type: str, bbox: str=None, **kwargs):
     import siibra
-    from siibra.features.image.image import Image
-    import json
     if type is None:
         raise InsufficientParameters(f"type is a required kwarg")
 
     *_, type = type.split(".")
 
-    concept = extract_concept(space_id=space_id, parcellation_id=parcellation_id, region_id=region_id)
+    concept = extract_concept(space_id=space_id, parcellation_id=parcellation_id, region_id=region_id, bbox=bbox)
 
     if concept is None:
         raise InsufficientParameters(f"at least one of space_id, parcellation_id and/or region_id must be defined.")
-
-    features = siibra.features.get(concept, type, **kwargs)
-
-    if bbox:
-        assert isinstance(concept, siibra.core.space.Space)
-        bounding_box = concept.get_bounding_box(*json.loads(bbox))
-        features = [f
-                for f in features
-                if isinstance(f, Image) and
-                f.anchor.location.intersects(bounding_box)]
-
-    return features
+    return siibra.features.get(concept, type, **kwargs)
 
 @data_decorator(ROLE)
 def all_features(*, space_id: str, parcellation_id: str, region_id: str, type: str, **kwargs):

api/common/decorators.py

@@ -1,5 +1,5 @@
 from .siibra_api_typing import ROLE_TYPE
-from api.common import logger
+from api.common.logger import logger
 from functools import wraps, partial
 import inspect
 import asyncio
@@ -32,6 +32,8 @@ def outer_wrapper(fn):
                 from api.worker import app
 
                 def celery_task_wrapper(self, *args, **kwargs):
+                    if role == "worker":
+                        logger.info(f"Task Received: {fn.__name__=}, {args=}, {kwargs=}")
                     return fn(*args, **kwargs)
 
                 return app.task(bind=True)(

api/common/logger.py

@@ -58,6 +58,9 @@
     import socket
     filename = log_dir + f"{socket.gethostname()}.general.log"
     warn_fh = TimedRotatingFileHandler(filename, when="d", encoding="utf-8")
-    warn_fh.setLevel(logging.INFO)
-    warn_fh.setFormatter(formatter)
-    logger.addHandler(warn_fh)
+else:
+    warn_fh = logging.StreamHandler()
+
+warn_fh.setLevel(logging.INFO)
+warn_fh.setFormatter(formatter)
+logger.addHandler(warn_fh)

api/models/_commons.py

@@ -108,6 +108,11 @@ class DataFrameModel(ConfigBaseModel):
         ]]
     ]
 
+class TimedeltaModel(ConfigBaseModel):
+    """TimedeltaModel"""
+    total_seconds: float
+
+
 class TaskIdResp(BaseModel):
     """TaskIdResp"""
     task_id: str

api/models/features/_basetypes/feature.py

@@ -1,7 +1,8 @@
 from api.models._commons import ConfigBaseModel
 from api.models._retrieval.datasets import EbrainsDatasetModel
 from api.models.features.anchor import SiibraAnchorModel
-from typing import List, Optional
+from api.models.locations.point import CoordinatePointModel
+from typing import List, Optional, Union
 from abc import ABC
 
 class _FeatureModel(ConfigBaseModel, ABC, type="feature"):
@@ -20,3 +21,11 @@ class _FeatureModel(ConfigBaseModel, ABC, type="feature"):
 class FeatureModel(_FeatureModel):
     """FeatureModel"""
     pass
+
+class SubfeatureModel(ConfigBaseModel):
+    id: str
+    index: Union[str, CoordinatePointModel]
+    name: str
+
+class CompoundFeatureModel(_FeatureModel, type="compoundfeature"):
+    indices: List[SubfeatureModel]

api/models/features/_basetypes/regional_connectivity.py

@@ -5,5 +5,6 @@
 class SiibraRegionalConnectivityModel(_FeatureModel, type="regional_connectivity"):
     """SiibraRegionalConnectivityModel"""
     cohort: str
-    subjects: List[str]
-    matrices: Optional[Dict[str, DataFrameModel]]
+    subject: str
+    feature: Optional[str]
+    matrix: Optional[DataFrameModel]

api/models/intents/base.py

@@ -0,0 +1,5 @@
+from abc import ABC
+from .._commons import ConfigBaseModel
+
+class _BaseIntent(ConfigBaseModel, ABC, type="intent"):
+    pass

api/models/intents/colorization.py

@@ -0,0 +1,14 @@
+from typing import List
+from pydantic import BaseModel
+
+from .base import _BaseIntent
+
+from ..core.region import ParcellationEntityVersionModel
+
+class RegionMapping(BaseModel):
+    region: ParcellationEntityVersionModel
+    rgb: List[int]
+
+class ColorizationIntent(_BaseIntent, type="colorization"):
+    region_mappings: List[RegionMapping]
+

api/serialization/_common.py

@@ -1,7 +1,7 @@
 from api.serialization.util import serialize, instance_to_model
 from api.serialization.util.siibra import MapIndex
-from api.models._commons import MapIndexModel, SeriesModel, DataFrameModel
-from pandas import Series, DataFrame
+from api.models._commons import MapIndexModel, SeriesModel, DataFrameModel, TimedeltaModel
+from pandas import Series, DataFrame, Timedelta
 import numpy as np
 
 @serialize(MapIndex)
@@ -55,7 +55,7 @@ def pdseries_to_model(series: Series, **kwargs) -> SeriesModel:
     )
 
 @serialize(DataFrame)
-def pddf_to_model(df: DataFrame, detail: str=False, **kwargs) -> DataFrameModel:
+def pddf_to_model(df: DataFrame, detail: bool=False, **kwargs) -> DataFrameModel:
     """Serialize pandas dataframe
     
     Args:
@@ -71,3 +71,9 @@ def pddf_to_model(df: DataFrame, detail: str=False, **kwargs) -> DataFrameModel:
         ndim=df.ndim,
         data=instance_to_model(df.values.tolist()) if detail else None,
     )
+
+@serialize(Timedelta)
+def pdtd_to_model(dti: Timedelta, detail: bool=False, **kwargs) -> TimedeltaModel:
+    return TimedeltaModel(
+        total_seconds=dti.total_seconds()
+    )

api/serialization/features/_basetypes/feature.py

@@ -1,6 +1,14 @@
-from api.serialization.util.siibra import Feature
+from api.serialization.util.siibra import Feature, CompoundFeature
 from api.serialization.util import serialize, instance_to_model
-from api.models.features._basetypes.feature import FeatureModel
+from api.models.features._basetypes.feature import FeatureModel, CompoundFeatureModel, SubfeatureModel
+
+@serialize(CompoundFeature, pass_super_model=True)
+def cmpdfeature_to_model(cf: CompoundFeature, detail: bool=False, super_model_dict={}, **kwargs):
+    return CompoundFeatureModel(
+        **super_model_dict,
+        indices=[SubfeatureModel(id=ft.id, index=instance_to_model(idx), name=ft.name) for (idx, ft) in zip(cf.indices, cf.elements)]
+    )
+
 
 @serialize(Feature)
 def feature_to_model(feat: Feature, detail: bool=False, **kwargs) -> FeatureModel:

api/serialization/features/_basetypes/regional_connectivity.py

@@ -19,9 +19,7 @@ def regional_conn_to_model(conn: RegionalConnectivity, subject:str=None, detail:
     """
     return SiibraRegionalConnectivityModel(
         **super_model_dict,
-        subjects=conn.subjects,
+        subject=conn.subject,
         cohort=conn.cohort,
-        matrices={
-            subject if subject else '_average': instance_to_model(conn.get_matrix(subject), detail=detail, **kwargs)
-        } if detail else None
+        matrix=instance_to_model(conn.data, detail=detail, **kwargs) if detail else None
     )

api/serialization/features/_basetypes/volume_of_interest.py

@@ -27,5 +27,5 @@ def voi_to_model(feat: Image, **kwargs) -> SiibraVoiModel:
     return SiibraVoiModel(
         **feature_dict,
         volume=volume_super_model,
-        boundingbox=instance_to_model(feat.boundingbox),
+        boundingbox=instance_to_model(feat.get_boundingbox()),
     )