Next major release

.github/workflows/ci.yml

@@ -5,9 +5,9 @@ name: Continuous integration Unit tests
 
 on:
   push:
-    branches: [ main ]
+    branches: [ main, dev ]
   pull_request:
-    branches: [ main ]
+    branches: [ main, dev ]
 
 jobs:
   PythonBlack:
@@ -16,16 +16,16 @@ jobs:
       - uses: actions/checkout@v4
 
       # Set up the specific Python version for Black
-      - name: Set up Python 3.12
+      - name: Set up Python 3.10
         uses: actions/setup-python@v5
         with:
-          python-version: "3.12"  # Set your desired Python version here
+          python-version: "3.10"  # Set your desired Python version here
 
       # Install Black and dependencies if necessary
       - name: Install Black
         run: |
           python -m pip install --upgrade pip
-          pip install black
+          python -m pip install black
 
       # Run Black linting
       - name: Check code lints with Black
@@ -64,20 +64,40 @@ jobs:
           isortVersion: "latest"
           requirementsFiles: "requirements.txt requirements-dev.txt"
 
+  mypy:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out source-code repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install -r requirements.txt -r requirements-dev.txt
+      - name: Install package
+        run: pip install .
+      - name: Test type annotations with mypy
+        run: |
+          mypy sdrf_pipelines
+
   build:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v4
-    - name: Set up Python 3.10
-      uses: actions/setup-python@v5
-      with:
-        python-version: "3.10"
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install -r requirements.txt -r requirements-dev.txt
-    - name: Install package
-      run: python setup.py install
-    - name: Test OpenMS converter
-      run: |
-        python -m pytest tests
+      - uses: actions/checkout@v4
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install -r requirements.txt -r requirements-dev.txt
+      - name: Install package
+        run: pip install .
+      - name: Test OpenMS converter
+        run: |
+          python -m pytest

.github/workflows/conda-build.yml

@@ -27,6 +27,10 @@ jobs:
           python -V
           conda install conda-build conda-verify anaconda-client
 
+      - name: Verify conda recipe
+        run: |
+          conda-verify .
+
       - name: Build the sdrf-pipelines package
         run: |
           conda build --package-format .tar.bz2 recipe

.github/workflows/pythonapp.yml

@@ -16,14 +16,15 @@ jobs:
 
     steps:
     - uses: actions/checkout@v4
-    - name: Set up Python 3.11
+    - name: Set up Python 3.10
       uses: actions/setup-python@v5
       with:
-        python-version: 3.11
+        python-version: '3.10'
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install -r requirements.txt
+        python -m pip install poetry
+        python -m poetry install --with=test --no-interaction
     - name: Lint with flake8
       run: |
         pip install flake8
@@ -33,6 +34,6 @@ jobs:
         flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
     - name: Test with pytest
       run: |
-        python setup.py install
-        pip install pytest pytest-datadir
-        python -m pytest tests
+        python -m pip install .
+        python -m pip install -r requirements-dev.txt
+        python -m pytest -s -vv -k reduced

.github/workflows/pythonpackage.yml

@@ -15,7 +15,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.9", "3.10", "3.11", "3.12"]
+        python-version: ["3.10", "3.11", "3.12", "3.13"]
     steps:
     - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
@@ -25,16 +25,16 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install -r requirements.txt
+        python -m pip install -r requirements.txt
     - name: Lint with flake8
       run: |
         pip install flake8
         # stop the build if there are Python syntax errors or undefined names
         flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
-        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
-        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+        # exit-zero treats all errors as warnings.
+        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=120 --statistics
     - name: Test with pytest
       run: |
-        pip install pytest pytest-datadir
-        python setup.py install
-        python -m pytest tests
+        python -m pip install pytest pytest-datadir
+        python -m pip install .
+        python -m pytest

.github/workflows/pythonpublish.yml

@@ -29,7 +29,7 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install build
+        python -m pytest pip install build
     - name: Build package
       run: python -m build
     - name: Publish package

.gitignore

@@ -121,3 +121,5 @@ ENV/
 pip-wheel-metadata
 .vscode
 .*.sw?
+
+.qodo

.pre-commit-config.yaml

@@ -1,15 +1,20 @@
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.4.0
+    rev: v5.0.0
     hooks:
-    - id: end-of-file-fixer
-    - id: trailing-whitespace
-    - id: detect-private-key
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+      - id: detect-private-key
   - repo: https://github.com/psf/black
-    rev: 23.9.1
+    rev: 25.1.0
     hooks:
       - id: black
   - repo: https://github.com/pycqa/isort
-    rev: 5.12.0
+    rev: 6.0.1
     hooks:
       - id: isort
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v1.15.0
+    hooks:
+      - id: mypy
+        additional_dependencies: ['pandas-stubs', 'pydantic', 'types-requests', 'types-PyYAML', 'types-defusedxml']

CHANGELOG.md

@@ -1,9 +1,15 @@
 # History of changes for sdrf-pipelines
 
-## Version 0.0.33
-
-### Important fixes
-
-* The TMT label set inference now works for files with more lines than labels in the used label set.
-
-----
+## Version 1.0.0 From 0.0.32
+
+### Development
+- GitHub action workflow set to use python 3.10 and add mypy and verification of conda recipe into the test suite
+- GitHub action workflow remove python 3.9 from testing
+- Add isort and mypy into pre-commit hooks
+- Usage of poetry for dependency management and packaging
+
+### Major Changes
+- Add yaml-defined schemas for validation and schema inheritance with improve support for multiple ontologies
+- SchemaRegistry can be used to programmatically use to access all the built-in schemas as well as adding custom schemas
+- Remove usage of deprecated pkg_resources
+- Add validate-sdrf-simple command to quickly validate a sdrf file

CONTRIBUTING.md

@@ -1,33 +1,46 @@
 # CONTRIBUTING
 
-## Testing
+## Setting up the development environment
 
 ```
-python -m venv venv
+python3.10 -m venv venv
 source venv/bin/activate
-pip install -r requirements-dev.txt -r requirements.txt
-pytest tests
+pip install -e .
+pip install -r requirements-dev.txt
+```
+
+## Running the tests
+
+After setting up the development environment, it is as easy as running
+
+```
+pytest
+```
+
+## Static type checking
+
+We use Python type hints and when added, they need to be checked with `mypy`
+
+```
+mypy sdrf_pipelines
 ```
 
 ## Code formatting
 
 We delegate code formatting to isort and black.
 To pass the CI tests, the code must adhere to the code formatting standards set in `pyproject.toml` file.
 
-You can install the required code formatters either with
+## `pre-commit` hooks
 
-```
-pip install -r requirements-dev.txt
-```
+As a convenience, we provide pre-commit hooks.
 
-or
+They can be run manually with
 
 ```
-pip install isort black
+pre-commit run --all-files
 ```
 
-As a convenience, we provide pre-commit hooks.
-They can be installed with
+or they can be installed to run before each commit
 
 ```
 pre-commit install --install-hooks