fix(bakend.storage): put temporary osekit replacement

backend/api/models/data/dataset.py

@@ -1,12 +1,10 @@
 """Datasets models"""
 import csv
 from os.path import join
-from pathlib import Path
 
 from django.conf import settings
 from django.db import models
 from metadatax.acquisition.models import ChannelConfiguration
-from osekit.public_api.dataset import Dataset as OSEkitDataset
 from typing_extensions import deprecated
 
 from backend.aplose.models import User
@@ -73,13 +71,3 @@ def get_config_folder(self) -> str:
                 return ""
             dataset = datasets[0]
         return f"{dataset['spectro_duration']}_{dataset['dataset_sr']}"
-
-    def get_osekit_dataset(self) -> OSEkitDataset:
-        """Get OSEkit dataset object"""
-        json_path = join(
-            settings.VOLUMES_ROOT,
-            settings.DATASET_EXPORT_PATH,
-            self.path,
-            "dataset.json",
-        )
-        return OSEkitDataset.from_json(Path(json_path))

backend/api/models/data/spectrogram.py

@@ -9,8 +9,10 @@
 from django.db.models import Q, F, QuerySet
 from metadatax.data.models import FileFormat
 from osekit.config import TIMESTAMP_FORMAT_EXPORTED_FILES_LOCALIZED
-from osekit.core_api.spectro_data import SpectroData
-from osekit.core_api.spectro_dataset import SpectroDataset
+
+# from osekit.core_api.spectro_data import SpectroData
+# from osekit.core_api.spectro_dataset import SpectroDataset
+from backend.utils.osekit_replace import SpectroDataset, SpectroData
 
 from .__abstract_file import AbstractFile
 from .__abstract_time_segment import TimeSegment

backend/api/models/data/spectrogram_analysis.py

@@ -10,10 +10,12 @@
 from django.conf import settings
 from django.db import models
 from django.db.models import CheckConstraint, Q
-from osekit.core_api.spectro_dataset import SpectroDataset
 from typing_extensions import deprecated
 
 from backend.aplose.models import User
+
+# from osekit.core_api.spectro_dataset import SpectroDataset
+from backend.utils.osekit_replace import SpectroDataset
 from .__abstract_analysis import AbstractAnalysis
 from .colormap import Colormap
 from .dataset import Dataset
@@ -128,6 +130,7 @@ def import_for_dataset(
                 )
             )
         )
+        sd.load_data()  # TODO: remove - debug osekit only
         colormap, _ = Colormap.objects.get_or_create(name=sd.colormap)
         dynamic_min = [d.v_lim[0] for d in sd.data]
         dynamic_max = [d.v_lim[1] for d in sd.data]
@@ -323,9 +326,9 @@ def legacy_spectrogram_configuration_csv(self) -> str:
 
     def get_osekit_spectro_dataset(self) -> SpectroDataset:
         """Get OSEkit dataset object"""
-        return SpectroDataset.from_json(
-            self.get_osekit_spectro_dataset_serialized_path()
-        )
+        sd = SpectroDataset.from_json(self.get_osekit_spectro_dataset_serialized_path())
+        sd.load_data()  # TODO: remove - debug osekit only
+        return sd
 
     def get_osekit_spectro_dataset_serialized_path(self) -> Path:
         """Get OSEkit dataset object"""

backend/api/schema/nodes/annotation_spectrogram.py

@@ -10,8 +10,10 @@
 from django_extension.schema.fields import AuthenticatedPaginationConnectionField
 from django_extension.schema.types import ExtendedNode
 from graphql import GraphQLResolveInfo
-from osekit.core_api.spectro_data import SpectroData
-from osekit.core_api.spectro_dataset import SpectroDataset
+
+# from osekit.core_api.spectro_data import SpectroData
+# from osekit.core_api.spectro_dataset import SpectroDataset
+from backend.utils.osekit_replace import SpectroDataset, SpectroData
 
 from backend.api.models import (
     Spectrogram,

backend/storage/schema/resolver/analysis.py

@@ -1,9 +1,11 @@
 from typing import Optional
 
 from django.conf import settings
-from osekit.core_api.spectro_dataset import SpectroDataset
 
 from backend.api.models import SpectrogramAnalysis as AnalysisModel
+
+# from osekit.core_api.spectro_dataset import SpectroDataset
+from backend.utils.osekit_replace import SpectroDataset
 from .base_resolver import BaseResolver
 from .exceptions import AnalysisBrowseException
 from .types import ImportStatus

backend/storage/schema/resolver/dataset.py

@@ -2,12 +2,13 @@
 from typing import Optional
 
 from django.conf import settings
-from osekit.public_api.dataset import (
-    Dataset as OSEkitDataset,
-    SpectroDataset as OSEkitSpectroDataset,
-)
 
-from backend.api.models import Dataset as DatasetModel
+from backend.api.models.data.dataset import Dataset as DatasetModel
+
+# from osekit.public_api.dataset import (
+#     SpectroDataset,
+# )
+from backend.utils.osekit_replace import SpectroDataset, OSEkitDataset
 from .analysis import Analysis
 from .base_resolver import BaseResolver
 from .types import LegacyCSVDataset, ImportStatus
@@ -108,7 +109,7 @@ def browse(self) -> list[Analysis]:
         # Browse datasets
         if self.osekit is not None:
             for [_name, d] in self.osekit.datasets.items():
-                if d["class"] != OSEkitSpectroDataset.__name__:
+                if d["class"] != SpectroDataset.__name__:
                     continue
                 path = (
                     PureWindowsPath(d["dataset"].folder)

backend/storage/tests/schema/import_analysis.py

@@ -20,6 +20,11 @@
             id
         }
     }
+  _debug {
+    exceptions {
+      stack
+    }
+  }
 }
 """
 VARIABLES = {

backend/utils/osekit_replace.py

@@ -0,0 +1,142 @@
+import json
+from os.path import join
+from pathlib import PureWindowsPath, Path
+
+import numpy as np
+from pandas import Timestamp, Timedelta
+from scipy.signal import ShortTimeFFT
+
+
+class TFile:
+    begin: Timestamp
+    end: Timestamp
+    path: str
+
+    def __init__(self, d: dict, dataset_path):
+        self.begin = d["begin"]
+        self.end = d["end"]
+        self.path = join(
+            dataset_path,
+            PureWindowsPath(d["path"])
+            .as_posix()
+            .split(PureWindowsPath(dataset_path).stem)
+            .pop()
+            .strip("/"),
+        )
+
+
+class AudioData:
+    dataset_path: Path
+
+    files: list[TFile]
+
+    def __init__(self, d: dict, dataset_path):
+        self.dataset_path = dataset_path
+        self.audio_data = [TFile(d, dataset_path) for name, d in d["files"].items()]
+
+
+class SpectroData:
+    dataset_path: Path
+    name: str
+    v_lim: list[int]  # TODO
+    begin: Timestamp
+    end: Timestamp
+    duration: Timedelta
+    audio_data: AudioData
+
+    def __init__(self, d: dict, name, dataset_path):
+        self.dataset_path = dataset_path
+        self.name = name
+        self.begin = Timestamp(d["begin"])
+        self.end = Timestamp(d["end"])
+        self.v_lim = d["v_lim"]
+        self.duration = self.end - self.begin
+        self.audio_data = AudioData(d["audio_data"], dataset_path)
+
+
+class SpectroDataset:
+    folder: Path
+    name: str
+    data: list[SpectroData] = []
+
+    def __init__(self, d: dict, path):
+        self.d = d
+        self.folder = path
+        self.name = d["name"]
+
+    @property
+    def fft(self) -> ShortTimeFFT:
+        sft = list(self.d["sft"].values())[0]
+        return ShortTimeFFT(
+            win=np.array(sft["win"]),
+            hop=sft["hop"],
+            fs=sft["fs"],
+            mfft=sft["mfft"],
+        )
+
+    @property
+    def colormap(self) -> str | None:
+        return list(self.d["data"].values())[0]["colormap"]
+
+    @property
+    def begin(self) -> Timestamp:
+        """Begin of the first data object."""
+        return min(data.begin for data in self.data)
+
+    @property
+    def end(self) -> Timestamp:
+        """End of the last data object."""
+        return max(data.end for data in self.data)
+
+    @property
+    def data_duration(self) -> Timedelta:
+        data_durations = [
+            Timedelta(data.duration).round(freq="1s") for data in self.data
+        ]
+        return max(set(data_durations), key=data_durations.count)
+
+    def load_data(self):
+        self.data = [
+            SpectroData(d, name, self.folder) for name, d in self.d["data"].items()
+        ]
+
+    @staticmethod
+    def from_json(json_path: Path) -> "SpectroDataset":
+        with json_path.open("r") as f:
+            return SpectroDataset(
+                json.loads(f.read()),
+                PureWindowsPath(json_path).as_posix()[
+                    : -len(f"/{json_path.stem}{json_path.suffix}")
+                ],
+            )
+
+
+class OSEkitDataset:
+    datasets: dict
+
+    def __init__(self, d: dict, path):
+        self.datasets = {}
+        for name, dataset in d["datasets"].items():
+            analysis_json_path = join(
+                path,
+                PureWindowsPath(dataset["json"])
+                .as_posix()
+                .split(PureWindowsPath(path).stem)
+                .pop()
+                .strip("/"),
+            )
+            self.datasets[name] = {
+                "class": dataset["class"],
+                "analysis": dataset["analysis"],
+                "dataset": SpectroDataset.from_json(
+                    Path(PureWindowsPath(analysis_json_path).as_posix())
+                ),
+            }
+
+    @staticmethod
+    def from_json(json_path: Path) -> "OSEkitDataset":
+        with json_path.open("r") as f:
+            return OSEkitDataset(
+                json.loads(f.read()),
+                PureWindowsPath(json_path).as_posix()[: -len("/dataset.json")],
+            )