Merge branch 'main' into reconcile-dates

.github/dependabot.yml

@@ -0,0 +1,11 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
+
+version: 2
+updates:
+  - package-ecosystem: "github-actions" # See documentation for possible values
+    directory: "/" # Location of package manifests
+    schedule:
+      interval: "weekly"

.github/workflows/code_checks.yml

@@ -26,16 +26,23 @@ jobs:
   run-code-check:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - name: Install poetry
-        run: python3 -m pip install --upgrade pip && python3 -m pip install poetry
-      - uses: actions/setup-python@v4.7.1
+      - uses: actions/checkout@v4.1.6
+      - name: Install and configure Poetry
+        uses: snok/install-poetry@v1
+        with:
+          virtualenvs-create: true
+          virtualenvs-in-project: true
+      - uses: actions/setup-python@v5.1.0
         with:
           python-version: '3.10'
           cache: 'poetry'
       - name: Install dependencies and check code
         run: |
           poetry env use '3.10'
-          source $(poetry env info --path)/bin/activate
+          source .venv/bin/activate
           poetry install --with test --all-extras
           pre-commit run --all-files
+      - name: pip-audit (gh-action-pip-audit)
+        uses: pypa/gh-action-pip-audit@v1.0.8
+        with:
+          virtual-environment: .venv/

.github/workflows/docker.yml

@@ -22,22 +22,22 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4.1.6
 
       - name: Log in to Docker Hub
-        uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a
+        uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
 
       - name: Extract metadata (tags, labels) for Docker
         id: meta
-        uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
+        uses: docker/metadata-action@f7b4ed12385588c3f9bc252f0a2b520d83b52d48
         with:
           images: vectorinstitute/cyclops
 
       - name: Build and push Docker image
-        uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671
+        uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0
         with:
           context: .
           file: ./Dockerfile

.github/workflows/docs_build.yml

@@ -9,7 +9,9 @@ on:
       - .github/workflows/docs_build.yml
       - '**.py'
       - '**.ipynb'
+      - '**.html'
       - '**.jinja'
+      - '**.js'
       - poetry.lock
       - pyproject.toml
       - '**.rst'
@@ -20,12 +22,12 @@ jobs:
   build:
     runs-on: [self-hosted, db, gpu]
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4.1.6
         with:
           submodules: 'true'
       - name: Install dependencies, build docs and coverage report
         run: python3 -m pip install --upgrade pip && python3 -m pip install poetry
-      - uses: actions/setup-python@v4.7.1
+      - uses: actions/setup-python@v5.1.0
         with:
           python-version: '3.10'
           cache: 'poetry'
@@ -37,21 +39,20 @@ jobs:
           poetry env use '3.10'
           source $(poetry env info --path)/bin/activate
           env MPICC=/opt/openmpi-4.1.5/bin/mpicc poetry install --with docs,dev,test --all-extras
-          # pandoc README.md -f markdown -t rst -s -o docs/source/intro.rst
           cd docs && rm -rf source/reference/api/_autosummary && make html
-          cd .. && coverage run -m pytest -m "not integration_test" && coverage xml && coverage report -m
+          cd .. && pytest -m "not integration_test" --cov-report term --cov-report xml:./coverage.xml --cov=cyclops -n auto --dist loadscope
       - name: Upload coverage to Codecov
-        uses: Wandalen/wretry.action@v1.0.36
+        uses: Wandalen/wretry.action@v3.5.0
         with:
-          action: codecov/codecov-action@v3.1.3
+          action: codecov/codecov-action@v4.0.0
           with: |
             token: ${{ secrets.CODECOV_TOKEN }}
             file: ./coverage.xml
             name: codecov-umbrella
             fail_ci_if_error: true
           attempt_limit: 5
           attempt_delay: 30000
-      - uses: actions/setup-node@v3
+      - uses: actions/setup-node@v4.0.2
         with:
           node-version: 18
           cache: yarn

.github/workflows/docs_deploy.yml

@@ -12,7 +12,9 @@ on:
       - .github/workflows/integration_tests.yml
       - '**.py'
       - '**.ipynb'
+      - '**.html'
       - '**.jinja'
+      - '**.js'
       - poetry.lock
       - pyproject.toml
       - '**.rst'
@@ -23,12 +25,12 @@ jobs:
   deploy:
     runs-on: [self-hosted, db, gpu]
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4.1.6
         with:
           submodules: 'true'
       - name: Install dependencies, build docs and coverage report
         run: python3 -m pip install --upgrade pip && python3 -m pip install poetry
-      - uses: actions/setup-python@v4.7.1
+      - uses: actions/setup-python@v5.1.0
         with:
           python-version: '3.10'
           cache: 'poetry'
@@ -38,22 +40,21 @@ jobs:
           ompi_info
           poetry env use '3.10'
           source $(poetry env info --path)/bin/activate
-          env MPICC=/opt/openmpi-4.1.5/bin/mpicc poetry install --with test,dev --all-extras
-          # pandoc README.md -f markdown -t rst -s -o docs/source/intro.rst
+          env MPICC=/opt/openmpi-4.1.5/bin/mpicc poetry install --with docs,test,dev --all-extras
           cd docs && rm -rf source/reference/api/_autosummary && make html
-          cd .. && coverage run -m pytest -m "not integration_test" && coverage xml && coverage report -m
+          cd .. && pytest -m "not integration_test" --cov-report term --cov-report xml:./coverage.xml --cov=cyclops -n auto --dist loadscope
       - name: Upload coverage to Codecov
-        uses: Wandalen/wretry.action@v1.0.36
+        uses: Wandalen/wretry.action@v3.5.0
         with:
-          action: codecov/codecov-action@v3.1.3
+          action: codecov/codecov-action@v4.0.0
           with: |
             token: ${{ secrets.CODECOV_TOKEN }}
             file: ./coverage.xml
             name: codecov-umbrella
             fail_ci_if_error: true
           attempt_limit: 5
           attempt_delay: 30000
-      - uses: actions/setup-node@v3
+      - uses: actions/setup-node@v4.0.2
         with:
           node-version: 18
           cache: yarn
@@ -65,7 +66,7 @@ jobs:
           yarn build
           cp -r ../build/html build/api
       - name: Deploy to GitHub Pages
-        uses: peaceiris/actions-gh-pages@v3
+        uses: peaceiris/actions-gh-pages@v4.0.0
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           publish_branch: github_pages

.github/workflows/integration_tests.yml

@@ -12,6 +12,9 @@ on:
       - .github/workflows/integration_tests.yml
       - '**.py'
       - '**.ipynb'
+      - '**.html'
+      - '**.jinja'
+      - '**.js'
       - poetry.lock
       - pyproject.toml
       - '**.rst'
@@ -28,6 +31,9 @@ on:
       - .github/workflows/integration_tests.yml
       - '**.py'
       - '**.ipynb'
+      - '**.html'
+      - '**.jinja'
+      - '**.js'
       - poetry.lock
       - pyproject.toml
       - '**.rst'
@@ -36,12 +42,12 @@ on:
 
 jobs:
   integration-tests:
-    runs-on: [self-hosted, gpu, db]
+    runs-on: [self-hosted, gpu, db, cyclops]
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4.1.6
       - name: Install poetry
-        run: pip install poetry
-      - uses: actions/setup-python@v4.7.1
+        run: python3 -m pip install --upgrade pip && python3 -m pip install poetry
+      - uses: actions/setup-python@v5.1.0
         with:
           python-version: '3.10'
       - name: Install dependencies and check code
@@ -52,11 +58,12 @@ jobs:
           poetry env use '3.10'
           source $(poetry env info --path)/bin/activate
           env MPICC=/opt/openmpi-4.1.5/bin/mpicc poetry install --with docs,dev,test --all-extras
-          coverage run -m pytest -m integration_test && coverage xml && coverage report -m
+          mpirun -n 2 python -m pytest --only-mpi
+          pytest -m integration_test --cov-report term --cov-report xml:./coverage.xml --cov=cyclops -n auto --dist loadscope
       - name: Upload coverage to Codecov
-        uses: Wandalen/wretry.action@v1.0.36
+        uses: Wandalen/wretry.action@v3.5.0
         with:
-          action: codecov/codecov-action@v3.1.3
+          action: codecov/codecov-action@v4.0.0
           with: |
             token: ${{ secrets.CODECOV_TOKEN }}
             file: ./coverage.xml

.github/workflows/package.yml

@@ -26,10 +26,10 @@ jobs:
   base-package-install-check:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4.1.6
       - name: Install pip
         run: python3 -m pip install --upgrade pip
-      - uses: actions/setup-python@v4.7.1
+      - uses: actions/setup-python@v5.1.0
         with:
           python-version: '3.10'
       - name: Install package and test import

.github/workflows/publish.yml

@@ -12,16 +12,16 @@ jobs:
         run: |
           sudo apt-get update
           sudo apt-get install libcurl4-openssl-dev libssl-dev
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4.1.6
       - name: Install poetry
         run: python3 -m pip install --upgrade pip && python3 -m pip install poetry
-      - uses: actions/setup-python@v4.7.1
+      - uses: actions/setup-python@v5.1.0
         with:
           python-version: '3.10'
       - name: Build package
         run: poetry build
       - name: Publish package
-        uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29
+        uses: pypa/gh-action-pypi-publish@81e9d935c883d0b210363ab89cf05f3894778450
         with:
           user: __token__
           password: ${{ secrets.PYPI_API_TOKEN }}

.gitignore

@@ -8,6 +8,7 @@ htmlcov
 venv
 .ipynb_checkpoints
 *.pt
+*.onnx
 *.csv
 mlruns
 .profile
@@ -26,7 +27,7 @@ dist/
 outputs/
 multirun/
 _autosummary
-*cyclops_reports*
+*cyclops_report*
 *dummy_reports*
 .mypy_cache
 *code-workspace*

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.4.0  # Use the ref you want to point at
+    rev: v4.6.0  # Use the ref you want to point at
     hooks:
     - id: trailing-whitespace
     - id: check-ast
@@ -15,37 +15,33 @@ repos:
     - id: check-yaml
     - id: check-toml
 
-  - repo: https://github.com/psf/black
-    rev: 23.7.0
-    hooks:
-    - id: black
-
-  - repo: https://github.com/charliermarsh/ruff-pre-commit
-    rev: 'v0.1.7'
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: 'v0.4.4'
     hooks:
     - id: ruff
       args: [--fix, --exit-non-zero-on-fix]
       types_or: [python, jupyter]
+    - id: ruff-format
+      types_or: [python, jupyter]
 
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.7.1
+    rev: v1.10.0
     hooks:
     - id: mypy
       entry: python3 -m mypy --config-file pyproject.toml
       language: system
       types: [python]
-      exclude: "use_cases|tests|cyclops/(models|monitor|report/plot)"
+      exclude: "tests|cyclops/(models|monitor|report/plot)"
 
   - repo: local
     hooks:
     - id: nbstripout
       name: nbstripout
       language: system
       entry: python3 -m nbstripout
-      exclude: ^docs/source/tutorials/gemini/.*\.ipynb$
 
   - repo: https://github.com/nbQA-dev/nbQA
-    rev: 1.7.0
+    rev: 1.8.5
     hooks:
     - id: nbqa-black
     - id: nbqa-ruff
@@ -63,7 +59,19 @@ repos:
     hooks:
     - id: pytest
       name: pytest
-      entry: python3 -m pytest -m "not integration_test"
+      entry: python3 -m pytest -m "not integration_test" -n auto --dist loadscope
       language: system
       pass_filenames: false
       always_run: true
+
+ci:
+    autofix_commit_msg: |
+        [pre-commit.ci] Add auto fixes from pre-commit.com hooks
+
+        for more information, see https://pre-commit.ci
+    autofix_prs: true
+    autoupdate_branch: ''
+    autoupdate_commit_msg: '[pre-commit.ci] pre-commit autoupdate'
+    autoupdate_schedule: weekly
+    skip: [pytest,doctest,nbstripout,mypy]
+    submodules: false

CONTRIBUTING.md

@@ -15,17 +15,12 @@ pre-commit run --all-files
 
 ## Coding guidelines
 
-For code style, we recommend the [google style guide](https://google.github.io/styleguide/pyguide.html).
-
-Pre-commit hooks apply the [black](https://black.readthedocs.io/en/stable/the_black_code_style/current_style.html)
-code formatting.
+For code style, we recommend the [PEP 8 style guide](https://peps.python.org/pep-0008/).
 
 For docstrings we use [numpy format](https://numpydoc.readthedocs.io/en/latest/format.html).
 
-We also use [flake8](https://flake8.pycqa.org/en/latest/) and [pylint](https://pylint.pycqa.org/en/stable/)
-for further static code analysis. The pre-commit hooks show errors which you need
-to fix before submitting a PR.
+We use [ruff](https://docs.astral.sh/ruff/) for code formatting and static code
+analysis. Ruff checks various rules including [flake8](https://docs.astral.sh/ruff/faq/#how-does-ruff-compare-to-flake8). The pre-commit hooks show errors which you need to fix before submitting a PR.
 
 Last but not the least, we use type hints in our code which is then checked using
-[mypy](https://mypy.readthedocs.io/en/stable/). Currently, mypy checks are not
-strict, but will be enforced more as the API code becomes more stable.
+[mypy](https://mypy.readthedocs.io/en/stable/).