Merge pull request #29 from NII-DG/develop

Revised code and docs based on previous example and demo

Dockerfile

@@ -10,7 +10,7 @@ FROM python:3.8.15-slim-buster
 LABEL org.opencontainers.image.authors="National Institute of Informatics, Japan"
 LABEL org.opencontainers.image.url="https://github.com/NII-DG/nii-dg"
 LABEL org.opencontainers.image.source="https://raw.githubusercontent.com/NII-DG/nii-dg/main/Dockerfile"
-LABEL org.opencontainers.image.version="0.1.1"
+LABEL org.opencontainers.image.version="0.1.2"
 LABEL org.opencontainers.image.licenses="Apache2.0"
 
 RUN apt update && \

compose.api.yml

@@ -1,7 +1,7 @@
 version: "3.5"
 services:
   app:
-    image: ghcr.io/NII-DG/nii-dg:0.1.1
+    image: ghcr.io/NII-DG/nii-dg:0.1.2
     container_name: nii-dg
     volumes:
       - ${PWD}:/app

compose.yml

@@ -1,7 +1,7 @@
 version: "3.5"
 services:
   app:
-    image: ghcr.io/NII-DG/nii-dg:0.1.1
+    image: ghcr.io/NII-DG/nii-dg:0.1.2
     container_name: nii-dg
     volumes:
       - ${PWD}:/app

docs/conf.py

@@ -27,9 +27,9 @@
 author = 'NII'
 
 # The short X.Y version
-version = '0.1.1'
+version = '0.1.2'
 # The full version, including alpha/beta/rc tags
-release = '0.1.1'
+release = '0.1.2'
 
 
 # -- General configuration ---------------------------------------------------

nii_dg/api.py

@@ -131,7 +131,7 @@ def get_results(request_id: str) -> Response:
             results = job.result()
         elif isinstance(job.exception(), CrateValidationError):
             status = "FAILED"
-            results = result_wrapper(job.exception().entity_errors)  # type:ignore
+            results = result_wrapper(job.exception().errors)  # type:ignore
         else:
             status = "EXECUTOR_ERROR"
             results = [{"err_msg": str(job.exception())}]

nii_dg/check_functions.py

@@ -233,7 +233,7 @@ def is_url_accessible(url: str) -> bool:
         bool: True if the URL is accessible, False otherwise.
     """
     try:
-        req = Request(url, method='HEAD')
+        req = Request(url, method="HEAD")
         res = urlopen(req)
         return res.status < 400  # type: ignore
     except Exception:

nii_dg/module_info.py

@@ -8,7 +8,7 @@
 GH_REPO: str = "NII-DG/nii-dg"
 """str: The GitHub repository name for the nii_dg package."""
 
-GH_REF: str = "0.1.1"
+GH_REF: str = "0.1.2"
 """str: The GitHub reference (tag or branch) for the nii_dg package."""
 
 

nii_dg/ro_crate.py

@@ -7,7 +7,7 @@
 
 import json
 from pathlib import Path
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List, Optional, Type, Union
 
 from nii_dg.const import DOWNLOADED_SCHEMA_DIR_NAME, RO_CRATE_CONTEXT
 from nii_dg.entity import (ContextualEntity, DataEntity, DefaultEntity, Entity,
@@ -140,7 +140,7 @@ def get_by_id(self, id_: str) -> List[Entity]:
         """
         return [entity for entity in self.all_entities if entity.id == id_]
 
-    def get_by_type(self, type_: str) -> List[Entity]:
+    def get_by_type(self, type_: Type[Entity]) -> List[Entity]:
         """\
         Get entities by type.
 
@@ -150,9 +150,9 @@ def get_by_type(self, type_: str) -> List[Entity]:
         Returns:
             A list of entities with the specified type.
         """
-        return [entity for entity in self.all_entities if entity.type == type_]
+        return [entity for entity in self.all_entities if type(entity) == type_]
 
-    def get_by_id_and_type(self, id_: str, type_: str) -> List[Entity]:
+    def get_by_id_and_type(self, id_: str, type_: Type[Entity]) -> List[Entity]:
         """\
         Get entities by ID and type.
 
@@ -163,7 +163,7 @@ def get_by_id_and_type(self, id_: str, type_: str) -> List[Entity]:
         Returns:
             A list of entities with the specified ID and type.
         """
-        return [entity for entity in self.all_entities if entity.id == id_ and entity.type == type_]
+        return [entity for entity in self.all_entities if entity.id == id_ and type(entity) == type_]
 
     def from_jsonld(self, jsonld: Dict[str, Any]) -> None:
         """\
@@ -250,7 +250,7 @@ def as_jsonld(self) -> Dict[str, Any]:
             "@graph": [entity.as_jsonld() for entity in self.all_entities]
         }
 
-    def dump(self, path: str) -> None:
+    def dump(self, path: Union[str, Path]) -> None:
         """\
         Dump the RO-Crate to a file.
 

nii_dg/schema/amed.py

@@ -58,7 +58,7 @@ def validate(self, crate: "ROCrate") -> None:
                 error.add("hostingInstitution", "The hostingInstitution property is required when the hasPart property is not empty.")
             if "dataManager" not in self:
                 error.add("dataManager", "The dataManager property is required when the hasPart property is not empty.")
-        if len(self["hasPart"]) > len(crate.get_by_type("DMP")):
+        if len(self["hasPart"]) > len(crate.get_by_type(DMP)):
             error.add("hasPart", "The number of the hasPart property MUST be equal to the number of DMP entities.")
 
         if error.has_error():
@@ -89,7 +89,7 @@ def validate(self, crate: "ROCrate") -> None:
 
         error = EntityError(self)
 
-        dmp_metadata_ents = crate.get_by_type("DMPMetadata")
+        dmp_metadata_ents = crate.get_by_type(DMPMetadata)
         if len(dmp_metadata_ents) == 0:
             error.add("AnotherEntity", "Entity `DMPMetadata` MUST be required with DMP entity.")
         else:
@@ -113,7 +113,7 @@ def validate(self, crate: "ROCrate") -> None:
 
         if "contentSize" in self:
             target_files = []
-            for ent in crate.get_by_type("File"):
+            for ent in crate.get_by_type(File):
                 if ent["dmpDataNumber"] == self:
                     target_files.append(ent)
 

nii_dg/schema/base.py

@@ -122,7 +122,7 @@ def validate(self, crate: "ROCrate") -> None:
             if self["name"] not in name_list:
                 error.add("name", f"The name MUST be one of {name_list} registered in ROR.")
         else:
-            if is_url_accessible(self.id):
+            if not is_url_accessible(self.id):
                 error.add("@id", "Failed to access the URL.")
 
         if error.has_error():
@@ -155,7 +155,7 @@ def validate(self, crate: "ROCrate") -> None:
 
         error = EntityError(self)
 
-        if is_url_accessible(self.id):
+        if not is_url_accessible(self.id):
             error.add("@id", "Failed to access the URL.")
 
         if error.has_error():
@@ -183,7 +183,7 @@ def validate(self, crate: "ROCrate") -> None:
 
         error = EntityError(self)
 
-        if is_url_accessible(self.id):
+        if not is_url_accessible(self.id):
             error.add("@id", "Failed to access the URL.")
 
         if error.has_error():
@@ -230,7 +230,7 @@ def validate(self, crate: "ROCrate") -> None:
 
         error = EntityError(self)
 
-        if is_url_accessible(self.id):
+        if not is_url_accessible(self.id):
             error.add("@id", "Failed to access the URL.")
 
         if error.has_error():

nii_dg/schema/cao.py

@@ -52,7 +52,7 @@ def validate(self, crate: "ROCrate") -> None:
 
         if self["about"] != crate.root and self["about"] != {"@id": "./"}:
             error.add("about", "The value of the about property MUST be the RootDataEntity of this crate.")
-        if len(self["hasPart"]) != len(crate.get_by_type("DMP")):
+        if len(self["hasPart"]) != len(crate.get_by_type(DMP)):
             error.add("hasPart", "The number of the hasPart property MUST be equal to the number of DMP entities.")
 
         if error.has_error():
@@ -83,7 +83,7 @@ def validate(self, crate: "ROCrate") -> None:
 
         error = EntityError(self)
 
-        dmp_metadata_ents = crate.get_by_type("DMPMetadata")
+        dmp_metadata_ents = crate.get_by_type(DMPMetadata)
         if len(dmp_metadata_ents) == 0:
             error.add("AnotherEntity", "Entity `DMPMetadata` MUST be required with DMP entity.")
         else:
@@ -108,7 +108,7 @@ def validate(self, crate: "ROCrate") -> None:
 
         if "contentSize" in self:
             target_files = []
-            for ent in crate.get_by_type("File"):
+            for ent in crate.get_by_type(File):
                 if ent["dmpDataNumber"] == self:
                     target_files.append(ent)
 
@@ -148,7 +148,7 @@ def validate(self, crate: "ROCrate") -> None:
 
         error = EntityError(self)
 
-        if is_url_accessible(self.id) is False:
+        if not is_url_accessible(self.id):
             error.add("@id", "The value MUST be a valid URL.")
 
         if error.has_error():

nii_dg/schema/ginfork.py

@@ -7,6 +7,7 @@
 from nii_dg.check_functions import is_absolute_path, is_url
 from nii_dg.entity import ContextualEntity, EntityDef
 from nii_dg.error import EntityError
+from nii_dg.schema.base import Dataset
 from nii_dg.schema.base import File as BaseFile
 from nii_dg.utils import load_schema_file, sum_file_size
 
@@ -47,14 +48,13 @@ def validate(self, crate: "ROCrate") -> None:
             error.add(
                 "about", "The value of this property MUST be the RootDataEntity of this crate.")
 
-        targets = [ent for ent in crate.get_by_type(
-            "File") if ent["experimentPackageFlag"] is True]
+        targets = [ent for ent in crate.get_by_type(File) if ent["experimentPackageFlag"] is True]
         sum_size = sum_file_size(self["contentSize"][-2:], targets)
         if sum_size > int(self["contentSize"][:-2]):
             error.add(
                 "contentSize", "The total file size of ginfork.File labeled as an experimental package is larger than the defined size.")
 
-        dir_paths = [Path(dir_.id) for dir_ in crate.get_by_type("Dataset")]
+        dir_paths = [Path(dir_.id) for dir_ in crate.get_by_type(Dataset)]
         required_dirs = [Path(experiment_dir).joinpath(required_dir_name)
                          for experiment_dir in self["experimentPackageList"]
                          for required_dir_name in self.REQUIRED_DIRECTORIES[self["datasetStructure"]]]

nii_dg/schema/meti.py

@@ -50,9 +50,9 @@ def validate(self, crate: "ROCrate") -> None:
 
         if self["about"] != crate.root and self["about"] != {"@id": "./"}:
             error.add("about", "The value of this property MUST be the RootDataEntity of this crate.")
-        if len(self["hasPart"]) != len(crate.get_by_type("DMP")):
+        if len(self["hasPart"]) != len(crate.get_by_type(DMP)):
             diff = []
-            for dmp in crate.get_by_type("DMP"):
+            for dmp in crate.get_by_type(DMP):
                 if dmp not in self["hasPart"]:
                     diff.append(dmp)
             error.add("hasPart", f"There is an omission of DMP entity in the list: {diff}.")
@@ -85,7 +85,7 @@ def validate(self, crate: "ROCrate") -> None:
 
         error = EntityError(self)
 
-        dmp_metadata_ents = crate.get_by_type("DMPMetadata")
+        dmp_metadata_ents = crate.get_by_type(DMPMetadata)
         if len(dmp_metadata_ents) == 0:
             error.add("AnotherEntity", "Entity 'DMPMetadata' MUST be required with DMP entity.")
         else:
@@ -121,7 +121,7 @@ def validate(self, crate: "ROCrate") -> None:
 
         if "contentSize" in self:
             target_files = []
-            for ent in crate.get_by_type("File"):
+            for ent in crate.get_by_type(File):
                 if ent["dmpDataNumber"] == self:
                     target_files.append(ent)
 

nii_dg/schema/sapporo.py

@@ -12,7 +12,7 @@
 import tempfile
 import time
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Dict, List, Optional
+from typing import TYPE_CHECKING, Any, Dict, List
 from urllib.parse import urlencode
 from urllib.request import Request, urlopen
 
@@ -57,7 +57,7 @@ def validate(self, crate: "ROCrate") -> None:
 
         error = EntityError(self)
 
-        sapporo_run_ents = crate.get_by_type("SapporoRun")
+        sapporo_run_ents = crate.get_by_type(SapporoRun)
         if len(sapporo_run_ents) == 0:
             error.add("AnotherEntity", "Entity `SapporoRun` MUST be required with sapporo.File entity.")
 
@@ -78,7 +78,7 @@ def check_props(self) -> None:
 
         if not self.id.endswith("/"):
             error.add("@id", "The value MUST end with '/'.")
-        if is_relative_path(self.id):
+        if not is_relative_path(self.id):
             error.add("@id", "The value MUST be relative path to the directory, neither absolute path nor URL.")
 
         if error.has_error():
@@ -89,7 +89,7 @@ def validate(self, crate: "ROCrate") -> None:
 
         error = EntityError(self)
 
-        sapporo_run_ents = crate.get_by_type("SapporoRun")
+        sapporo_run_ents = crate.get_by_type(SapporoRun)
         if len(sapporo_run_ents) == 0:
             error.add("AnotherEntity", "Entity `SapporoRun` MUST be required with sapporo.File entity.")
 
@@ -119,22 +119,21 @@ def check_props(self) -> None:
             raise error
 
     @classmethod
-    def generate_run_request_json(cls, sapporo_run: "SapporoRun") -> Dict[str, Optional[str]]:
+    def generate_run_request_json(cls, sapporo_run: "SapporoRun") -> Dict[str, str]:
         request_keys = ["workflow_params", "workflow_type", "workflow_type_version", "tags", "workflow_engine_name",
                         "workflow_engine_parameters", "workflow_url", "workflow_name", "workflow_attachment"]
-        run_request: Dict[str, Optional[str]] = {key: None for key in request_keys}
+        run_request: Dict[str, str] = {}
         for key in request_keys:
             if key in sapporo_run:
                 run_request[key] = sapporo_run[key]
 
         return run_request
 
     @classmethod
-    def execute_wf(cls, run_request: Dict[str, Optional[str]], endpoint: str) -> str:
+    def execute_wf(cls, run_request: Dict[str, str], endpoint: str) -> str:
         data = urlencode(run_request).encode("utf-8")  # type: ignore
-        headers = {"Content-Type": "application/x-www-form-urlencoded"}
         # remove trailing slash from endpoint
-        request = Request(f"{endpoint.rstrip('/')}/runs", data=data, headers=headers)  # type: ignore
+        request = Request(f"{endpoint.rstrip('/')}/runs", data=data)  # type: ignore
 
         with urlopen(request) as response:
             result = json.load(response)
@@ -210,7 +209,7 @@ def validate(self, crate: "ROCrate") -> None:
             raise error
 
         if isinstance(self["outputs"], dict):
-            outputs = crate.get_by_id_and_type(self["outputs"]["@id"], "Dataset")
+            outputs = crate.get_by_id_and_type(self["outputs"]["@id"], Dataset)
             if len(outputs) > 0:
                 self["outputs"] = outputs[0]
             else:
@@ -219,7 +218,7 @@ def validate(self, crate: "ROCrate") -> None:
         outputs_entities: List[Entity] = []
         for ent in self["outputs"]["hasPart"]:
             if isinstance(ent, dict):
-                file = crate.get_by_id_and_type(ent["@id"], "File")
+                file = crate.get_by_id_and_type(ent["@id"], File)
                 if len(file) > 0:
                     outputs_entities.append(file[0])
                 else:

sapporo_example/compose.yml

@@ -3,12 +3,13 @@ services:
   nii-dg:
     build:
       context: ..
-      dockerfile: Dockerfile-api
+      dockerfile: Dockerfile
     image: nii-dg
     container_name: nii-dg
     ports:
       - 0.0.0.0:5000:5000
     restart: on-failure
+    command: [ "python", "/app/nii_dg/api.py" ]
     networks:
       - nii-dg-network
   sapporo-service:

sapporo_example/sapporo_use_case.md

@@ -258,8 +258,8 @@ $ curl localhost:5000/32012249-d8f7-4f64-b20d-853ea5be67b5
   "results": [
     {
       "entityId": "#sapporo-run",
-      "props": "sapporo.SapporoRun:outputs_ERR034597_1.small_fastqc.html",
-      "reason": "The file ERR034597_1.small_fastqc.html is included in the result of re-execution, so File entity with @id ERR034597_1.small_fastqc.html is required in this crate."
+      "props": "sapporo.SapporoRun:outputs",
+      "reason": "The file ERR034597_1.small_fastqc.html is included in the result of re-execution, but this crate does not have File entity with @id ERR034597_1.small_fastqc.html."
     }
   ],
   "status": "FAILED"