Dev into main

.github/dependabot.yml

@@ -2,14 +2,14 @@ version: 2
 updates:
   - package-ecosystem: "pip"
     directory: "/"
-    target-branch: "dev"
+    target-branch: "main"
     labels:
       - "dependencies"
     schedule:
       interval: "weekly"
 
   - package-ecosystem: "github-actions"
     directory: "/"
-    target-branch: "dev"
+    target-branch: "main"
     schedule:
       interval: "weekly"

.pre-commit-config.yaml

@@ -30,40 +30,25 @@ repos:
   # Formatters: hooks that re-write Python and RST files
   ########################################################################################
 
+  # Linter and formatter that replaces flake8, isort, etc.
+  - repo: https://github.com/charliermarsh/ruff-pre-commit
+    rev: "v0.0.258"
+    hooks:
+      - id: ruff
+        args: [--fix, --exit-non-zero-on-fix]
+
   # Convert relative imports to absolute imports
   - repo: https://github.com/MarcoGorelli/absolufy-imports
     rev: v0.3.1
     hooks:
       - id: absolufy-imports
 
-  # Make sure import statements are sorted uniformly.
-  - repo: https://github.com/PyCQA/isort
-    rev: 5.12.0
-    hooks:
-      - id: isort
-
   # Remove f-string prefix when there's nothing in the string to format.
   - repo: https://github.com/dannysepler/rm_unneeded_f_str
     rev: v0.2.0
     hooks:
       - id: rm-unneeded-f-str
 
-  # Use built-in types for annotations as per PEP585, with __futures__ for Python 3.8/3.9
-  # Note that this sometimes conflicts with Pydantic on Python 3.8/3.9
-  - repo: https://github.com/sondrelg/pep585-upgrade
-    rev: "v1.0"
-    hooks:
-      - id: upgrade-type-hints
-        args: ["--futures=true"]
-
-  # Update a bunch of Python syntax, using __futures__ for Python 3.8/3.9
-  # Note that this sometimes conflicts with Pydantic on Python 3.8/3.9
-  - repo: https://github.com/asottile/pyupgrade
-    rev: v3.3.1
-    hooks:
-      - id: pyupgrade
-        args: ["--py39-plus"]
-
   # Deterministic python formatting:
   - repo: https://github.com/psf/black
     rev: 23.1.0
@@ -88,32 +73,6 @@ repos:
   # Linters: hooks that check but don't alter Python & RST files
   ########################################################################################
 
-  # Check for PEP8 non-compliance, code complexity, style, errors, etc:
-  - repo: https://github.com/PyCQA/flake8
-    rev: 6.0.0
-    hooks:
-      - id: flake8
-        additional_dependencies:
-          - flake8-docstrings
-          - flake8-colors
-          - pydocstyle
-          - flake8-builtins
-          - mccabe
-          - pep8-naming
-          - pycodestyle
-          - pyflakes
-          - flake8-rst-docstrings
-          - flake8-pyproject
-          - flake8-use-fstring
-
-  # Check for known security vulnerabilities:
-  - repo: https://github.com/PyCQA/bandit
-    rev: 1.7.4
-    hooks:
-      - id: bandit
-        args: ["-c", "pyproject.toml"]
-        additional_dependencies: ["bandit[toml]"]
-
   # Check for errors in restructuredtext (.rst) files under the doc hierarchy
   - repo: https://github.com/PyCQA/doc8
     rev: v1.1.1

docs/release_notes.rst

@@ -89,6 +89,18 @@ What's New?
     optional.
 *   Updates to :class:`.DispatchModel` to work with the new simpler, cleaner
     :class:`.DataZip`.
+*   Added checks to make sure that the datetime indexes of ``load_profile``,
+    ``dispatchable_profiles``, and ``re_profiles`` match.
+*   Prep for deprecating :meth:`.DispatchModel.from_patio` and
+    :meth:`.DispatchModel.from_fresh`.
+*   Extracted :func:`.calculate_generator_output` from :func:`.dispatch_engine` to make
+    the latter easier to read and to more easily test the former's logic.
+*   Many updates to internal variable names in :mod:`.engine` to make the code easier to
+    read.
+*   Renamed :func:`.apply_op_ret_date` to
+    :func:`.zero_profiles_outside_operating_dates` for clarity, use of the former name
+    will be removed in the future.
+*  Code cleanup along with adoption of ruff and removal of bandit, flake8, isort, etc.
 
 
 Bug Fixes
@@ -102,7 +114,7 @@ Bug Fixes
     cases of division be zero in these calculations, the result is now 0 rather than
     :class:`numpy.nan`. Tests now make sure that no new :class:`numpy.nan` show up.
 *   Fixed a bug in :meth:`.DispatchModel.dispatchable_summary` where ``pct_replaced``
-    would be :class:`numpy.nan` because of division be zero in these calculations, the
+    would be :class:`numpy.nan` because of division by zero in these calculations, the
     result is now 0 rather than :class:`numpy.nan`. Tests now make sure that no new
     :class:`numpy.nan` show up.
 *   Fixed an issue where :meth:`.DispatchModel.full_output` and methods that use it,

pyproject.toml

@@ -18,57 +18,47 @@ classifiers = [
 ]
 requires-python = ">=3.10,<3.11"
 dependencies = [
-    "bottleneck >= 1.3,< 1.3.7",
+    "bottleneck >= 1.3,< 1.3.8",
     "numba >= 0.55.1,<0.57",
     "numexpr >= 2.8, < 2.8.5",
     "numpy >= 1.18.5,<2",
     "pandas >= 1.4,< 1.6",
-    "pandera >= 0.12, <0.14",
-    "pyarrow>=7, <11",
+    "pandera >= 0.12, < 0.15",
+    "pyarrow>=7, <12",
     "rmi.etoolbox @ git+https://github.com/rmi/etoolbox.git",
 ]
 
 [project.optional-dependencies]
 dev = [
-    "black[jupyter] >= 22,<23",
+    "black[jupyter] >= 22,< 24",
     "docformatter>=1.5,<1.6",
-    "isort>=5.0,<5.12",
-    "tox>=3.20,<4.4",
+    "ruff>0.0.215,<0.0.261",
+    "tox>=3.20,<4.5",
     "twine>=3.3,<4.1",  # Used to make releases to PyPI
 ]
 doc = [
     "doc8>=0.9,<1.2",
     "furo>=2022.4.7",
     "sphinx>=4,!=5.1.0,<6.1.4",
-    "sphinx-autoapi>=1.8,<2.1",
+    "sphinx-autoapi>=1.8,<2.2",
     "sphinx-issues>=1.2,<3.1",
-    "sphinx-autodoc-typehints>1.19,<1.21.8",
-    "sphinxcontrib-mermaid>0.7,<=0.7.1",
+    "sphinx-autodoc-typehints>1.19,<1.23.0",
+    "sphinxcontrib-mermaid>0.7,<0.9.0",
 ]
 tests = [
-    "bandit[toml]>=1.6,<1.8",
     "blackdoc>=0.3.8,<0.3.9",  # runs black on doctests
-    "coverage>=5.3,<7.1",
+    "coverage>=5.3,<7.3",
     "doc8>=0.9,<1.2",
-    "flake8>=4.0,<6.1",
-    "flake8-builtins>=1.5,<2.2",
-    "flake8-docstrings>=1.5,<1.7",
-    "flake8-pyproject>1.0,<1.3",
-    "flake8-rst-docstrings>=0.2,<0.4",
-    "flake8-use-fstring>=1.0,<1.5",
-    "mccabe>=0.6,<0.8",
-    "mypy>=0.942,<0.992",
-    "pep8-naming>=0.12,<0.14",
-    "pre-commit>=2.9,<2.22",
-    "pydocstyle>=5.1,<6.4",
+    "pre-commit>=2.9,<3.3",
     "pytest>=6.2,<7.3",
     "pytest-console-scripts>=1.1,<1.4",
     "pytest-cov>=2.10,<4.1",
     "rstcheck[sphinx,toml]>=5.0,<6.2",
-    "tox>=3.20,<4.4",
+    "ruff>0.0.215,<0.0.261",
+    "tox>=3.20,<4.5",
 ]
 viz = [
-    "plotly>5.10,<5.13",
+    "plotly>5.10,<5.15",
     "kaleido>0.2,<0.2.2",
 ]
 
@@ -81,11 +71,6 @@ write_to = "src/dispatch/_version.py"
 #######################################################################################
 # Configurations for linters and other tools
 #######################################################################################
-
-[tool.bandit]
-exclude_dirs = ["tests"]
-skips = ["B101"]
-
 [tool.black]
 line-length = 88
 target-version = ["py310", "py311"]
@@ -99,56 +84,6 @@ wrap-descriptions = 88
 max-line-length = 88
 ignore-path = "docs/_build"
 
-[tool.flake8]
-extend-ignore = [
-    # * W503, W504: Line break before / after binary operator.
-    "W503", "W504",
-    "D401", # * D401: Imperative mood.
-    "E501", # * E501: Overlong line
-    "E203", # * E203: Space before ':' (black recommends to ignore)
-    # * RST201,RST203,RST301: Google docstrings aren't RST until after being processed by
-    #   Napoleon. See https://github.com/peterjc/flake8-rst-docstrings/issues/17
-    "RST201",  "RST203",  "RST301",
-    "D105",
-#    "C901" # We have a backlog of complex functions being skipped with noqa: C901
-]
-max-complexity = 10
-inline-quotes = "double"
-max-line-length = 88
-docstring-convention = "google"
-# Files and directories that should not be subject to linting
-extend-exclude = [
-    ".env_tox",
-    ".eggs",
-    "build",
-]
-rst-roles =[
-    "attr",
-    "class",
-    "doc",
-    "func",
-    "meth",
-    "mod",
-    "obj",
-    "py:const",
-    "ref",
-    "user",
-]
-rst-directives =[
-    "envvar",
-    "exception",
-]
-percent-greedy = 2
-format-greedy = 2
-per-file-ignores = [
-    "__init__.py:F401",
-    "engine.py:C901"  # the dispatch engine is quite complicated
-]
-
-[tool.isort]
-profile = "black"
-known_first_party = ["dispatch"]
-
 [tool.pytest.ini_options]
 testpaths = ["."]
 addopts = ["--verbose", "--doctest-modules"]
@@ -167,3 +102,53 @@ report_level = "WARNING"
 ignore_roles = ["pr", "issue", "user", "include"]
 ignore_messages = '(Hyperlink target .* is not referenced\.$|Duplicate implicit target name:|An `AttributeError` error occured.*)'
 ignore_directives = ["bibliography", "todo", "include"]
+
+[tool.ruff]
+# Enable pycodestyle (`E`) and Pyflakes (`F`) codes by default.
+select = ["A", "B", "C", "D", "E", "F", "G", "I", "N", "PGH", "S", "SIM", "TCH", "UP", "W", "RUF"]
+ignore = ["D401", "D105", "D407", "D416", "D417", "D203", "D213"]
+# Allow autofix for all enabled rules (when `--fix`) is provided.
+fixable = ["A", "B", "C", "D", "E", "F", "I", "UP", ]
+unfixable = []
+# By default, always enumerate fixed violations.
+show-fixes = true
+# Exclude a variety of commonly ignored directories.
+exclude = [
+    ".bzr",
+    ".direnv",
+    ".eggs",
+    ".git",
+    ".hg",
+    ".mypy_cache",
+    ".nox",
+    ".pants.d",
+    ".pytype",
+    ".ruff_cache",
+    ".svn",
+    ".tox",
+    ".venv",
+    "__pypackages__",
+    "_build",
+    "buck-out",
+    "build",
+    "dist",
+    "node_modules",
+    "venv",
+]
+# Same as Black.
+line-length = 88
+# Allow unused variables when underscore-prefixed.
+dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$"
+# Assume Python 3.10.
+target-version = "py310"
+
+[tool.ruff.per-file-ignores]
+"__init__.py" = ["F401"]
+"docs/conf.py" = ["E501"]
+"engine.py" = ["E501", "B905", "S101"]
+"model.py" = ["E501"]
+"tests/*.py" = ["S101", "SIM117", "F841", "D101", "PD901"]
+
+[tool.ruff.mccabe]
+# Unlike Flake8, default to a complexity level of 10.
+max-complexity = 15

src/dispatch/__init__.py

@@ -14,12 +14,17 @@
 except ImportError:
     __version__ = "unknown"
 
-from dispatch.helpers import apply_op_ret_date, copy_profile
+from dispatch.helpers import (
+    apply_op_ret_date,
+    copy_profile,
+    zero_profiles_outside_operating_dates,
+)
 from dispatch.model import DispatchModel
 
 __all__ = [
     "DispatchModel",
     "copy_profile",
+    "zero_profiles_outside_operating_dates",
     "apply_op_ret_date",
     "__version__",
 ]