Fix termset reqs, rename deps tag (#943)

.github/workflows/check_external_links.yml

@@ -28,7 +28,7 @@ jobs:
       - name: Install Sphinx dependencies and package
         run: |
           python -m pip install --upgrade pip
-          python -m pip install -r requirements-doc.txt
+          python -m pip install -r requirements-doc.txt -r requirements-opt.txt
           python -m pip install .
 
       - name: Check Sphinx external links

.github/workflows/run_all_tests.yml

@@ -94,12 +94,15 @@ jobs:
       matrix:
         include:
           - { name: linux-gallery-python3.8-minimum      , test-tox-env: gallery-py38-minimum    , python-ver: "3.8" , os: ubuntu-latest }
+          - { name: linux-gallery-python3.11-optional    , test-tox-env: gallery-py311-optional  , python-ver: "3.11", os: ubuntu-latest }
           - { name: linux-gallery-python3.11-upgraded    , test-tox-env: gallery-py311-upgraded  , python-ver: "3.11", os: ubuntu-latest }
           - { name: linux-gallery-python3.11-prerelease  , test-tox-env: gallery-py311-prerelease, python-ver: "3.11", os: ubuntu-latest }
           - { name: windows-gallery-python3.8-minimum    , test-tox-env: gallery-py38-minimum    , python-ver: "3.8" , os: windows-latest }
+          - { name: windows-gallery-python3.11-optional  , test-tox-env: gallery-py311-optional  , python-ver: "3.11", os: windows-latest }
           - { name: windows-gallery-python3.11-upgraded  , test-tox-env: gallery-py311-upgraded  , python-ver: "3.11", os: windows-latest }
           - { name: windows-gallery-python3.11-prerelease, test-tox-env: gallery-py311-prerelease, python-ver: "3.11", os: windows-latest }
           - { name: macos-gallery-python3.8-minimum      , test-tox-env: gallery-py38-minimum    , python-ver: "3.8" , os: macos-latest }
+          - { name: macos-gallery-python3.11-optional    , test-tox-env: gallery-py311-optional  , python-ver: "3.11", os: macos-latest }
           - { name: macos-gallery-python3.11-upgraded    , test-tox-env: gallery-py311-upgraded  , python-ver: "3.11", os: macos-latest }
           - { name: macos-gallery-python3.11-prerelease  , test-tox-env: gallery-py311-prerelease, python-ver: "3.11", os: macos-latest }
     steps:

.github/workflows/run_hdmf_zarr_tests.yml

@@ -6,7 +6,7 @@ on:
   workflow_dispatch:
 
 jobs:
-  run-tests:
+  run-hdmf-zarr-tests:
     runs-on: ubuntu-latest
     steps:
       - name: Cancel non-latest runs

.github/workflows/run_pynwb_tests.yml

@@ -6,7 +6,7 @@ on:
   workflow_dispatch:
 
 jobs:
-  run-tests:
+  run-pynwb-tests:
     runs-on: ubuntu-latest
     steps:
       - name: Cancel non-latest runs

.readthedocs.yaml

@@ -25,6 +25,7 @@ formats: all
 python:
   install:
     - requirements: requirements-doc.txt
+    - requirements: requirements-opt.txt
     - requirements: requirements.txt
     - path: .
 

docs/gallery/plot_term_set.py

@@ -63,12 +63,11 @@
 """
 from hdmf.common import DynamicTable, VectorData
 import os
-import sys
 
 try:
     import linkml_runtime  # noqa: F401
-except ImportError:
-    sys.exit(0)
+except ImportError as e:
+    raise ImportError("Please install linkml-runtime to run this example: pip install linkml-runtime") from e
 from hdmf.term_set import TermSet
 
 try:

pyproject.toml

@@ -42,10 +42,10 @@ dynamic = ["version"]
 [project.optional-dependencies]
 zarr = ["zarr>=2.12.0"]
 tqdm = ["tqdm>=4.41.0"]
-termset_reqs = ["linkml-runtime<=1.5.5; python_version >= '3.9'",
-                "schemasheets>=0.1.23; python_version >= '3.9'",
-                "oaklib>=0.5.12; python_version >= '3.9'",
-                "pyyaml>=6.0.1; python_version >= '3.9'"]
+termset = ["linkml-runtime>=1.5.5; python_version >= '3.9'",
+           "schemasheets>=0.1.23; python_version >= '3.9'",
+           "oaklib>=0.5.12; python_version >= '3.9'",
+           "pyyaml>=6.0.1; python_version >= '3.9'"]
 
 [project.urls]
 "Homepage" = "https://github.com/hdmf-dev/hdmf"

requirements-doc.txt

@@ -4,7 +4,3 @@ sphinx>=4  # improved support for docutils>=0.17
 sphinx_rtd_theme>=1  # <1 does not work with docutils>=0.17
 sphinx-gallery
 sphinx-copybutton
-linkml-runtime==1.5.5; python_version >= "3.9"
-schemasheets==0.1.23; python_version >= "3.9"
-oaklib==0.5.12; python_version >= "3.9"
-pyyaml==6.0.1; python_version >= "3.9"

requirements-min.txt

@@ -6,9 +6,10 @@ numpy==1.18
 pandas==1.0.5  # when this is changed to >=1.5.0, see TODO items referenced in #762
 ruamel.yaml==0.16
 scipy==1.4
-linkml-runtime==1.5.5; python_version >= "3.9"
-schemasheets==0.1.23; python_version >= "3.9"
-oaklib==0.5.12; python_version >= "3.9"
-pyyaml==6.0.1; python_version >= "3.9"
+# this file is currently used to test only python~=3.8 so these dependencies are not needed
+# linkml-runtime==1.5.5; python_version >= "3.9"
+# schemasheets==0.1.23; python_version >= "3.9"
+# oaklib==0.5.12; python_version >= "3.9"
+# pyyaml==6.0.1; python_version >= "3.9"
 tqdm==4.41.0
 zarr==2.12.0

src/hdmf/term_set.py

@@ -5,12 +5,12 @@
 import warnings
 
 
-class TermSet():
+class TermSet:
     """
     Class for implementing term sets from ontologies and other resources used to define the
     meaning and/or identify of terms.
 
-    :ivar term_schema_path: The path to LinkML YAML enumeration schema
+    :ivar term_schema_path: The path to the LinkML YAML enumeration schema
     :ivar sources: The prefixes for the ontologies used in the TermSet
     :ivar view: SchemaView of the term set schema
     :ivar schemasheets_folder: The path to the folder containing the LinkML TSV files
@@ -22,7 +22,7 @@ def __init__(self,
                  dynamic: bool=False
                  ):
         """
-        :param term_schema_path: The path to LinkML YAML enumeration schema
+        :param term_schema_path: The path to the LinkML YAML enumeration schema
         :param schemasheets_folder: The path to the folder containing the LinkML TSV files
         :param dynamic: Boolean parameter denoting whether the schema uses Dynamic Enumerations
 
@@ -132,8 +132,8 @@ def __schemasheets_convert(self):
             from linkml_runtime.utils.schema_as_dict import schema_as_dict
             from schemasheets.schemamaker import SchemaMaker
         except ImportError:   # pragma: no cover
-            msg="Install schemasheets."   # pragma: no cover
-            raise ValueError(msg)   # pragma: no cover
+            msg = "Install schemasheets."
+            raise ValueError(msg)
         schema_maker = SchemaMaker()
         tsv_file_paths = glob.glob(self.schemasheets_folder + "/*.tsv")
         schema = schema_maker.create_schema(tsv_file_paths)
@@ -154,11 +154,12 @@ def __enum_expander(self):
         This method returns a path to the new schema to be viewed via SchemaView.
         """
         try:
-            warnings.filterwarnings("ignore", category=DeprecationWarning)
-            from oaklib.utilities.subsets.value_set_expander import ValueSetExpander
+            with warnings.catch_warnings():
+                warnings.filterwarnings("ignore", category=DeprecationWarning)
+                from oaklib.utilities.subsets.value_set_expander import ValueSetExpander
         except ImportError:   # pragma: no cover
-            msg = 'Install oaklib.'  # pragma: no cover
-            raise ValueError(msg)  # pragma: no cover
+            msg = 'Install oaklib.'
+            raise ValueError(msg)
         expander = ValueSetExpander()
         # TODO: linkml should raise a warning if the schema does not have dynamic enums
         enum = list(self.view.all_enums())

test_gallery.py

@@ -71,6 +71,11 @@ def run_gallery_tests():
                     category=RuntimeWarning,
                 )
                 _import_from_file(script)
+        except (ImportError, ValueError) as e:
+            if "linkml" in str(e) and sys.version_info < (3, 9):
+                pass  # this is OK because plot_term_set.py and plot_external_resources.py cannot be run on Python 3.8
+            else:
+                raise e
         except Exception:
             print(traceback.format_exc())
             FAILURES += 1

tests/unit/test_term_set.py

@@ -1,5 +1,4 @@
 import os
-import unittest
 
 from hdmf.term_set import TermSet
 from hdmf.testing import TestCase, remove_test_file
@@ -19,48 +18,44 @@
 
 class TestTermSet(TestCase):
 
-    @unittest.skipIf(not REQUIREMENTS_INSTALLED, "optional LinkML module is not installed")
+    def setUp(self):
+        if not REQUIREMENTS_INSTALLED:
+            self.skipTest("optional LinkML module is not installed")
+
     def test_termset_setup(self):
         termset = TermSet(term_schema_path='tests/unit/example_test_term_set.yaml')
         self.assertEqual(list(termset.sources), ['NCBI_TAXON'])
 
-    @unittest.skipIf(not REQUIREMENTS_INSTALLED, "optional LinkML module is not installed")
     def test_view_set(self):
         termset = TermSet(term_schema_path='tests/unit/example_test_term_set.yaml')
         expected = ['Homo sapiens', 'Mus musculus', 'Ursus arctos horribilis', 'Myrmecophaga tridactyla']
         self.assertEqual(list(termset.view_set), expected)
         self.assertIsInstance(termset.view, SchemaView)
 
-    @unittest.skipIf(not REQUIREMENTS_INSTALLED, "optional LinkML module is not installed")
     def test_termset_validate(self):
         termset = TermSet(term_schema_path='tests/unit/example_test_term_set.yaml')
         self.assertEqual(termset.validate('Homo sapiens'), True)
 
-    @unittest.skipIf(not REQUIREMENTS_INSTALLED, "optional LinkML module is not installed")
     def test_termset_validate_false(self):
         termset = TermSet(term_schema_path='tests/unit/example_test_term_set.yaml')
         self.assertEqual(termset.validate('missing_term'), False)
 
-    @unittest.skipIf(not REQUIREMENTS_INSTALLED, "optional LinkML module is not installed")
     def test_get_item(self):
         termset = TermSet(term_schema_path='tests/unit/example_test_term_set.yaml')
         self.assertEqual(termset['Homo sapiens'].id, 'NCBI_TAXON:9606')
         self.assertEqual(termset['Homo sapiens'].description, 'the species is human')
         self.assertEqual(termset['Homo sapiens'].meaning, 'https://www.ncbi.nlm.nih.gov/Taxonomy/Browser/wwwtax.cgi?mode=Info&id=9606')
 
-    @unittest.skipIf(not REQUIREMENTS_INSTALLED, "optional LinkML module is not installed")
     def test_get_item_key_error(self):
         termset = TermSet(term_schema_path='tests/unit/example_test_term_set.yaml')
         with self.assertRaises(ValueError):
             termset['Homo Ssapiens']
 
-    @unittest.skipIf(not REQUIREMENTS_INSTALLED, "optional LinkML module is not installed")
     def test_schema_sheets_and_path_provided_error(self):
         folder = os.path.join(CUR_DIR, "test_term_set_input", "schemasheets")
         with self.assertRaises(ValueError):
             TermSet(term_schema_path='tests/unit/example_test_term_set.yaml', schemasheets_folder=folder)
 
-    @unittest.skipIf(not REQUIREMENTS_INSTALLED, "optional LinkML module is not installed")
     def test_view_set_sheets(self):
         folder = os.path.join(CUR_DIR, "test_term_set_input", "schemasheets")
         termset = TermSet(schemasheets_folder=folder)
@@ -69,7 +64,6 @@ def test_view_set_sheets(self):
         self.assertEqual(list(termset.view_set), expected)
         self.assertIsInstance(termset.view, SchemaView)
 
-    @unittest.skipIf(not REQUIREMENTS_INSTALLED, "optional LinkML module is not installed")
     def test_enum_expander(self):
         schema_path = 'tests/unit/example_dynamic_term_set.yaml'
         termset = TermSet(term_schema_path=schema_path, dynamic=True)
@@ -90,7 +84,6 @@ def test_enum_expander(self):
         filename = os.path.splitext(os.path.basename(schema_path))[0]
         remove_test_file(f"tests/unit/expanded_{filename}.yaml")
 
-    @unittest.skipIf(not REQUIREMENTS_INSTALLED, "optional LinkML module is not installed")
     def test_enum_expander_output(self):
         schema_path = 'tests/unit/example_dynamic_term_set.yaml'
         termset = TermSet(term_schema_path=schema_path, dynamic=True)
@@ -102,7 +95,6 @@ def test_enum_expander_output(self):
 
         self.assertEqual(convert_path, expected_path)
 
-    @unittest.skipIf(not REQUIREMENTS_INSTALLED, "optional LinkML module is not installed")
     def test_folder_output(self):
         folder = os.path.join(CUR_DIR, "test_term_set_input", "schemasheets")
         termset = TermSet(schemasheets_folder=folder)

tox.ini

@@ -152,6 +152,15 @@ basepython = python3.11
 deps = {[testenv:gallery]deps}
 commands = {[testenv:gallery]commands}
 
+[testenv:gallery-py311-optional]
+basepython = python3.11
+deps =
+    -rrequirements-dev.txt
+    -rrequirements.txt
+    -rrequirements-doc.txt
+    -rrequirements-opt.txt
+commands = {[testenv:gallery]commands}
+
 # Test with python 3.11; pinned dev, doc, and optional reqs; upgraded run reqs
 [testenv:gallery-py311-upgraded]
 basepython = python3.11