Sasdata database redux

.github/workflows/test-fair-database.yml

@@ -0,0 +1,54 @@
+name: Tests
+
+on:
+  [push, pull_request]
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  unit-test:
+
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [macos-latest, ubuntu-latest, windows-latest]
+        python-version: ['3.12']
+      fail-fast: false
+
+    steps:
+
+    - name: Obtain SasData source from git
+      uses: actions/checkout@v4
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python-version }}
+        cache: 'pip'
+        cache-dependency-path: |
+          **/test.yml
+          **/requirements*.txt
+
+    ### Installation of build-dependencies
+
+    - name: Install Python dependencies
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install wheel setuptools
+        python -m pip install -r requirements.txt
+        python -m pip install -r sasdata/fair_database/requirements.txt
+
+    ### Build and test sasdata
+
+    - name: Build sasdata
+      run: |
+        # BUILD SASDATA
+        python -m pip install -e .
+
+    ### Build documentation (if enabled)
+
+    - name: Test with Django tests
+      run: |
+        python sasdata/fair_database/manage.py test sasdata.fair_database

.gitignore

@@ -24,6 +24,7 @@
 **/build
 /dist
 .mplconfig
+**/db.sqlite3
 
 # INSTALL.md recommends a venv that should not be committed
 venv

sasdata/data.py

@@ -1,4 +1,6 @@
+import json
 import typing
+from typing import Any
 
 import h5py
 import numpy as np
@@ -125,6 +127,40 @@ def save_h5(data: dict[str, typing.Self], path: str | typing.BinaryIO):
                     sasentry.attrs["sasview_key"] = key
                 data._save_h5(sasentry)
 
+    @staticmethod
+    def deserialise(data: str) -> "SasData":
+        json_data = json.loads(data)
+        return SasData.deserialise_json(json_data)
+
+    @staticmethod
+    def deserialise_json(json_data: dict) -> "SasData":
+        name = json_data["name"]
+        data_contents = {}
+        dataset_type = json_data["dataset_type"] # TODO: update when DatasetType is more finalized
+        metadata = json_data["metadata"].deserialise_json()
+        for quantity in json_data["data_contents"]:
+            data_contents[quantity["label"]] = Quantity.deserialise_json(quantity)
+        return SasData(name, data_contents, dataset_type, metadata)
+
+    def serialise(self) -> str:
+        return json.dumps(self._serialise_json())
+
+    # TODO: fix serializers eventually
+    def _serialise_json(self) -> dict[str, Any]:
+        data = []
+        for d in self._data_contents:
+            quantity = self._data_contents[d]
+            quantity["label"] = d
+            data.append(quantity)
+        return {
+            "name": self.name,
+            "data_contents": data,
+            "dataset_type": None, # TODO: update when DatasetType is more finalized
+            "verbose": self._verbose,
+            "metadata": self.metadata.serialise_json(),
+            "mask": {},
+            "model_requirements": {}
+        }
 
 
 class SasDataEncoder(MetadataEncoder):

sasdata/data_backing.py

@@ -36,6 +36,34 @@ def summary(self, indent_amount: int = 0, indent: str = "  ") -> str:
 
         return s
 
+    @staticmethod
+    def deserialise_json(json_data: dict) -> "Dataset":
+        name = json_data["name"]
+        data = "" # TODO: figure out QuantityType serialisation
+        attributes = {}
+        for key in json_data["attributes"]:
+            value = json_data["attributes"][key]
+            if isinstance(value, dict):
+                attributes[key] = Dataset.deserialise_json(value)
+            else:
+                attributes[key] = value
+        return Dataset(name, data, attributes)
+
+    def serialise_json(self):
+        content = {
+            "name": self.name,
+            "data": "", # TODO: figure out QuantityType serialisation
+            "attributes": {},
+            "type": "dataset"
+        }
+        for key in self.attributes:
+            value = self.attributes[key]
+            if isinstance(value, (Group, Dataset)):
+                content["attributes"]["key"] = value.serialise_json()
+            else:
+                content["attributes"]["key"] = value
+        return content
+
 @dataclass
 class Group:
     name: str
@@ -48,6 +76,27 @@ def summary(self, indent_amount: int=0, indent="  "):
 
         return s
 
+    @staticmethod
+    def deserialise_json(json_data: dict) -> "Group":
+        name = json_data["name"]
+        children = {}
+        for key in json_data["children"]:
+            value = json_data["children"][key]
+            if value["type"] == "group":
+                children[key] = Group.deserialise_json(value)
+            else:
+                children[key] = Dataset.deserialise_json(value)
+        return Group(name, children)
+
+    def serialise_json(self):
+        return {
+            "name": self.name,
+            "children": {
+                key: self.children[key].serialise_json() for key in self.children
+            },
+            "type": "group"
+        }
+
 class Function:
     """ Representation of a (data driven) function, such as I vs Q """
 

sasdata/fair_database/data/admin.py

@@ -0,0 +1,11 @@
+from data import models
+from django.contrib import admin
+
+admin.site.register(models.DataFile)
+admin.site.register(models.Session)
+admin.site.register(models.PublishedState)
+admin.site.register(models.DataSet)
+admin.site.register(models.MetaData)
+admin.site.register(models.Quantity)
+admin.site.register(models.OperationTree)
+admin.site.register(models.ReferenceQuantity)

sasdata/fair_database/data/apps.py

@@ -0,0 +1,6 @@
+from django.apps import AppConfig
+
+
+class DataConfig(AppConfig):
+    default_auto_field = "django.db.models.BigAutoField"
+    name = "data"

sasdata/fair_database/data/forms.py

@@ -0,0 +1,9 @@
+from data.models import DataFile
+from django import forms
+
+
+# Create the form class.
+class DataFileForm(forms.ModelForm):
+    class Meta:
+        model = DataFile
+        fields = ["file", "is_public"]