Updated bids_dataset to validate datasets with library schema

docs/api.rst

@@ -4,21 +4,21 @@ API Reference
 .. _base_ref:
 
 
-:mod:`hedtools.hed.models`: Class models
+:mod:`hed.models`: Class models
 ------------------------------------------
 
-.. autosummary:: hedtools.hed.models
+.. autosummary:: hed.models
    :toctree:
 
-   hedtools.hed.models.base_input.BaseInput
-   hedtools.hed.models.column_mapper.ColumnMapper
-   hedtools.hed.models.column_metadata.ColumnMetadata
-   hedtools.hed.models.column_metadata.ColumnType
-   hedtools.hed.models.def_dict.DefEntry
-   hedtools.hed.models.def_dict.DefDict
-   hedtools.hed.models.def_mapper.DefinitionMapper
+   hed.models.base_input.BaseInput
+   hed.models.column_mapper.ColumnMapper
+   hed.models.column_metadata.ColumnMetadata
+   hed.models.column_metadata.ColumnType
+   hed.models.def_dict.DefEntry
+   hed.models.def_dict.DefDict
+   hed.models.def_mapper.DefinitionMapper
 
-.. currentmodule:: hedtools
+.. currentmodule:: hed
 
 .. _calibration_ref:
 

docs/conf.py

@@ -27,7 +27,10 @@
 # The full version, including alpha/beta/rc tags
 release = '0.0.1'
 
-currentdir = os.path.abspath(os.path.dirname(__file__))
+source_dirs = [
+    os.path.abspath("../hedtools/"),
+]
+
 
 # -- General configuration ---------------------------------------------------
 

docs/index.rst

@@ -17,6 +17,8 @@ Note:  this is a work in progress.
    :maxdepth: 2
    :caption: Resources:
 
+api
+
 Trying stuff
 ==================
 .. automodule:: hedtools.hed.tools.models
@@ -39,6 +41,3 @@ Indices and tables
 * :ref:`modindex`
 * :ref:`search`
 
-
-   api.rst
-   changes.rst

hedtools/hed/models/def_dict.py

@@ -63,14 +63,14 @@ def get_definition(self, replace_tag, placeholder_value=None):
 class DefDict:
     """Class responsible for gathering and storing a group of definitions to be considered a single source.
 
-        A bids style file might have many of these(one for each json dict, and another for the actual file)
+        A bids_old style file might have many of these(one for each json dict, and another for the actual file)
     """
 
     def __init__(self):
         """
         Class responsible for gathering and storing a group of definitions to be considered a single source.
 
-        A bids style file might have many of these(one for each json dict, and another for the actual file)
+        A bids_old style file might have many of these(one for each json dict, and another for the actual file)
         Parameters
         ----------
         """

hedtools/hed/models/events_input.py

@@ -5,7 +5,7 @@
 
 
 class EventsInput(BaseInput):
-    """A class to parse bids style spreadsheets into a more general format."""
+    """A class to parse bids_old style spreadsheets into a more general format."""
     HED_COLUMN_NAME = "HED"
 
     def __init__(self, file=None, sidecars=None, attribute_columns=None, extra_def_dicts=None,
@@ -47,7 +47,7 @@ def __init__(self, file=None, sidecars=None, attribute_columns=None, extra_def_d
                          def_mapper=def_mapper, name=name)
 
         if not self._has_column_names:
-            raise ValueError("You are attempting to open a bids style file with no column headers provided.\n"
+            raise ValueError("You are attempting to open a bids_old style file with no column headers provided.\n"
                              "This is probably not intended.")
 
     def create_def_mapper(self, column_mapper, extra_def_dicts=None):

hedtools/hed/tools/bids/bids_dataset.py

@@ -1,15 +1,17 @@
 import os
 import json
-from hed.tools.data_util import get_new_dataframe
-from hed.schema.hed_schema_file import load_schema, load_schema_version
+from hed.errors.error_reporter import get_printable_issue_string
+from hed.schema.hed_schema_io import load_schema, load_schema_version
 from hed.schema.hed_schema_group import HedSchemaGroup
+from hed.tools.data_util import get_new_dataframe
 from hed.tools.bids.bids_event_files import BidsEventFiles
+from hed.validator import HedValidator
 
 LIBRARY_URL_BASE = "https://raw.githubusercontent.com/hed-standard/hed-schema-library/main/hedxml/HED_"
 
 
 class BidsDataset:
-    """Represents a bids dataset."""
+    """Represents a bids_old dataset."""
 
     def __init__(self, root_path):
         self.root_path = os.path.abspath(root_path)
@@ -19,6 +21,11 @@ def __init__(self, root_path):
         self.hed_schema = HedSchemaGroup(self._schema_from_description())
         self.event_files = BidsEventFiles(root_path)
 
+    def validate(self, check_for_warnings=True):
+        validator = HedValidator(hed_schema=self.hed_schema)
+        issues = self.event_files.validate(validators=[validator], check_for_warnings=check_for_warnings)
+        return issues
+
     def _schema_from_description(self):
         hed = self.dataset_description.get("HEDVersion", None)
         if isinstance(hed, str):
@@ -32,12 +39,19 @@ def _schema_from_description(self):
         if 'libraries' in hed:
             for key, library in hed['libraries'].items():
                 url = LIBRARY_URL_BASE + library + '.xml'
-                x = load_schema(hed_url_path=url, library_prefix=key)
+                x = load_schema(url, library_prefix=key)
                 x.set_library_prefix(key)  # TODO: temporary work around
                 hed_list.append(x)
         return hed_list
 
 
 if __name__ == '__main__':
-    path = 'D:\\Research\\HED\\hed-examples\\datasets\\eeg_ds003654s_hed_library'
+    path = os.path.join(os.path.dirname(os.path.abspath(__file__)),
+                        '../../../tests/data/bids/eeg_ds003654s_hed_library')
     bids = BidsDataset(path)
+    issue_list = bids.validate()
+    if issue_list:
+        issue_str = get_printable_issue_string(issue_list, "HED_library")
+    else:
+        issue_str = "No issues"
+    print(issue_str)

hedtools/hed/tools/bids/bids_event_file.py

@@ -4,7 +4,7 @@
 
 
 class BidsEventFile(BidsFile):
-    """Represents a bids file."""
+    """Represents a bids_old file."""
 
     def __init__(self, file_path):
         super().__init__(os.path.abspath(file_path))

hedtools/hed/tools/bids/bids_event_files.py

@@ -1,6 +1,6 @@
 import os
 from hed.errors.error_reporter import get_printable_issue_string
-from hed.schema.hed_schema_file import load_schema
+from hed.schema.hed_schema_io import load_schema
 from hed.tools.bids.bids_event_file import BidsEventFile
 from hed.tools.bids.bids_sidecar_file import BidsSidecarFile
 from hed.models.events_input import EventsInput
@@ -78,9 +78,8 @@ def validate(self, validators, check_for_warnings=True, keep_events=False):
 
 
 if __name__ == '__main__':
-    # path = 'D:/Research/HED/hed-examples/datasets/eeg_ds003654s_inheritance'
-    path = 'D:/Research/HED/hed-examples/datasets/eeg_ds003654s'
-    # path = 'G:\\WH_working3'
+    path = os.path.join(os.path.dirname(os.path.abspath(__file__)),
+                        '../../../tests/data/bids/eeg_ds003654s_hed')
     bids = BidsEventFiles(path)
 
     for file_obj in bids.sidecar_dict.values():
@@ -90,8 +89,8 @@ def validate(self, validators, check_for_warnings=True, keep_events=False):
         print(file_obj)
 
     print("Now validating.....")
-    hed_schema = load_schema(
-            hed_url_path='https://raw.githubusercontent.com/hed-standard/hed-specification/master/hedxml/HED8.0.0.xml')
+    hed_schema = \
+        load_schema('https://raw.githubusercontent.com/hed-standard/hed-specification/master/hedxml/HED8.0.0.xml')
     validator = HedValidator(hed_schema=hed_schema)
     validation_issues = bids.validate(validators=[validator], check_for_warnings=False)
     issue_str = get_printable_issue_string(validation_issues, skip_filename=False)

hedtools/hed/tools/bids/bids_file.py

@@ -4,7 +4,7 @@
 
 
 class BidsFile:
-    """Represents a bids file."""
+    """Represents a bids_old file."""
 
     def __init__(self, file_path):
         self.file_path = os.path.abspath(file_path)

hedtools/hed/tools/bids/bids_json_file.py

@@ -4,7 +4,7 @@
 
 
 class BidsJsonFile(BidsFile):
-    """Represents a bids file."""
+    """Represents a bids_old file."""
 
     def __init__(self, file_path, set_contents=False):
         super().__init__(os.path.abspath(file_path))

hedtools/hed/tools/bids/bids_sidecar_file.py

@@ -4,7 +4,7 @@
 
 
 class BidsSidecarFile(BidsFile):
-    """Represents a bids file."""
+    """Represents a bids_old file."""
 
     def __init__(self, file_path, set_contents=False):
         super().__init__(os.path.abspath(file_path))

hedtools/hed/tools/hed_def_summary.py

@@ -34,7 +34,7 @@ def get_condition_variables(self):
 
 
 if __name__ == '__main__':
-    the_path = '../../tests/data/bids/task-FacePerception_events.json'
+    the_path = '../../tests/data/bids_old/task-FacePerception_events.json'
     sidecar = Sidecar(the_path)
     hed_schema = load_schema(
             hed_url_path='https://raw.githubusercontent.com/hed-standard/hed-specification/master/hedxml/HED8.0.0.xml')

hedtools/hed/tools/hed_group_summary.py

@@ -66,7 +66,7 @@ def __str__(self):
 
 
 if __name__ == '__main__':
-    the_path = '../../tests/data/bids/task-FacePerception_events.json'
+    the_path = '../../tests/data/bids_old/task-FacePerception_events.json'
     sidecar = Sidecar(the_path)
     hed_schema = load_schema('https://raw.githubusercontent.com/hed-standard/hed-specification/master/hedxml/HED8.0.0.xml')
     def_dicts = [column_entry.def_dict for column_entry in sidecar]

hedtools/hed/tools/sidecar_map.py

@@ -4,7 +4,7 @@
 
 
 class SidecarMap:
-    """A class to parse bids style spreadsheets into a more general format."""
+    """A class to parse bids_old style spreadsheets into a more general format."""
 
     def __init__(self, header_char="*"):
         """ Takes representing an event file and creates a template for remapping.

hedtools/tests/data/bids/eeg_ds003654s_hed/CHANGES

@@ -0,0 +1,6 @@
+1.0.0 2021-05-11
+  - First release
+Revision history for Face Recognition experiment by Wakeman-Henson
+
+version 1.0 - April 2021
+ - Initial release of EEG data in this experiment for HED education purposes

hedtools/tests/data/bids/eeg_ds003654s_hed/dataset_description.json

@@ -0,0 +1,24 @@
+{
+    "Name": "Face processing MEEG dataset with HED annotation",
+    "BIDSVersion": "1.8.4",
+    "HEDVersion": "8.0.0",
+    "License": "CC0",
+    "Authors": [
+        "Daniel G. Wakeman",
+        "Richard N Henson",
+        "Dung Truong (curation)",
+        "Kay Robbins (curation)",
+        "Scott Makeig (curation)",
+        "Arno Delorme (curation)"
+    ],
+    "ReferencesAndLinks": [
+        "Wakeman, D., Henson, R. (2015). A multi-subject, multi-modal human neuroimaging dataset. Sci Data 2, 150001. https://doi.org/10.1038/sdata.2015.1",
+        "Robbins, K., Truong, D., Appelhoff, S., Delorme, A., & Makeig, S. (2021). Capturing the nature of events and event context using Hierarchical Event Descriptors (HED). In press for NeuroImage Special Issue Practice in MEEG. NeuroImage 245 (2021) 118766. Online: https://www.sciencedirect.com/science/article/pii/S1053811921010387.",
+        "Robbins, K., Truong, D., Jones, A., Callanan, I., & Makeig, S. (2021). Building FAIR functionality: Annotating events in time series data using Hierarchical Event Descriptors (HED). Neuroinformatics Special Issue Building the NeuroCommons. Neuroinformatics https://doi.org/10.1007/s12021-021-09537-4. Online: https://link.springer.com/article/10.1007/s12021-021-09537-4."
+    ],
+    "Funding": [
+        "Experiment was supported by the UK Medical Research Council (MC_A060_5PR10) and Elekta Ltd.",
+        "Curation was supported by: Army Research Laboratory W911NF-10-2-0022, NIH R01 EB023297-03, NIH R01 NS047293-l4, and NIH R24 MH120037-01."
+    ],
+    "DatasetDOI": "10.18112/openneuro.ds003645.v1.0.0"
+}

hedtools/tests/data/bids/eeg_ds003654s_hed/stimuli/s010.bmp

@@ -0,0 +1 @@
+{"error": "an unknown error occurred accessing this file"}