RMI · arengel · Oct 22, 2022 · Oct 8, 2022 · Oct 11, 2022 · Oct 11, 2022
diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml
@@ -7,6 +7,12 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
+      - name: Get history and tags for SCM versioning to work
+        run: |
+          git fetch --prune --unshallow
+          git fetch --depth=1 origin +refs/tags/*:refs/tags/*
+          git describe --tags
+          git describe --tags $(git rev-list --tags --max-count=1)
       - uses: actions/setup-python@v4
         with:
           python-version: "3.10"

diff --git a/README.rst b/README.rst
@@ -10,8 +10,8 @@ Dispatch: A simple and efficient electricity dispatch model
    :target: https://rmi-electricity.github.io/dispatch/
    :alt: GitHub Pages Status
 
-.. image:: https://coveralls.io/repos/github/rmi-electricity/dispatch/badge.svg?branch=dev
-   :target: https://coveralls.io/github/rmi-electricity/dispatch?branch=dev
+.. image:: https://coveralls.io/repos/github/rmi-electricity/dispatch/badge.svg
+   :target: https://coveralls.io/github/rmi-electricity/dispatch
 
 .. image:: https://img.shields.io/badge/code%20style-black-000000.svg
    :target: https://github.com/psf/black>

diff --git a/docs/conf.py b/docs/conf.py
@@ -24,16 +24,19 @@
 # number via pkg_resources.get_distribution() so we need more than just an
 # importable path.
 
-# The full version, including alpha/beta/rc tags
-release = version_func("rmi.dispatch")
-version = ".".join(release.split(".")[:2])
 
 # -- Project information -----------------------------------------------------
 
 project = "Dispatch"
 copyright = f"{datetime.today().year}, RMI, CC-BY-4.0"  # noqa: A001
 author = "RMI"
 
+# The full version, including alpha/beta/rc tags
+release = version_func("rmi.dispatch")
+version = ".".join(release.split(".")[:2])
+html_title = f"{project} {version} documentation"
+
+
 # -- General configuration ---------------------------------------------------
 
 # Add any Sphinx extension module names here, as strings. They can be
@@ -58,8 +61,19 @@
 autoapi_ignore = [
     "*_test.py",
     "*/package_data/*",
+    "_*.py",
+    "*constants.py",
 ]
 autoapi_python_class_content = "both"
+autoapi_options = [
+    "members",
+    # "undoc-members",
+    # "private-members",
+    "show-inheritance",
+    "show-module-summary",
+    "special-members",
+    "imported-members",
+]
 autodoc_typehints = "description"
 # autoapi_keep_files = True
 
@@ -73,13 +87,16 @@
     "numba": ("https://numba.readthedocs.io/en/stable", None),
     "pandas": ("https://pandas.pydata.org/pandas-docs/stable", None),
     "pandera": ("https://pandera.readthedocs.io/en/stable", None),
+    "plotly": ("https://plotly.com/python-api-reference/", None),
     "pytest": ("https://docs.pytest.org/en/latest/", None),
     "python": ("https://docs.python.org/3", None),
     "scipy": ("https://docs.scipy.org/doc/scipy/", None),
     "setuptools": ("https://setuptools.pypa.io/en/latest/", None),
     "tox": ("https://tox.wiki/en/latest/", None),
 }
 
+"https://plotly.com/python-api-reference/generated/plotly.graph_objects.Figure.html"
+"https://plotly.com/python-api-reference/generated/plotly.graph_objs.Figure"
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ["_templates"]
 

diff --git a/docs/release_notes.rst b/docs/release_notes.rst
@@ -11,9 +11,63 @@ Release Notes
 
 What's New?
 ^^^^^^^^^^^
-*   Test for :func:`.dispatch_engine`.
+*   Tests for :func:`.dispatch_engine_py`, :func:`.copy_profile`.
+*   :meth:`.DispatchModel.hourly_data_check` to help in checking for dispatch errors,
+    and running down why deficits are occuring.
+*   :class:`.DispatchModel` now takes ``load_profile`` that resources will be
+    dispatched against. If ``re_profiles`` and ``re_plant_specs`` are not provided,
+    this should be a net load profile. If they are provided, this *must* be a gross
+    load profile, or at least, gross of those RE resources. These calculations are done
+    by :meth:`.DispatchModel.re_and_net_load`.
+*   :class:`.DispatchModel` now accepts (and requires) raw DC ``re_profiles``, it
+    determines actual renewable output using capacity data and ilr provided in
+    ``re_plant_specs``. This will allow :class:`.DispatchModel` to model DC-coupled
+    RE+Storage facilities that can charge from otherwise clipped generation. The
+    calculations for the amount of charging from DC-coupled RE is in
+    :meth:`.DispatchModel.dc_charge`.
+*   Updates to :func:`.dispatch_engine_py` and :func:`.engine.validate_inputs_py` to
+    accommodate DC-coupled RE charging data. Storage can now be charged from
+    DC-coupled RE in addition to the grid. This includes tracking ``gridcharge``
+    in addition to ``charge``, where the latter includes charging from the grid
+    and DC-coupled RE.
+*   All output charging metrics use the ``gridcharge`` data because from the grid's
+    perspective, this is what matters. ``discharge`` data does not distinguish,
+    so in some cases net charge data may be positive, this reflects RE generation
+    run through the battery that otherwise would have been curtailed.
+*   :class:`.DataZip`, a subclass of :class:`zipfile.ZipFile` that has methods for
+    easily reading and writing :class:`pandas.DataFrame` as ``parquet`` and
+    :class:`dict` as ``json``. This includes storing column names separately that
+    cannot be included in a ``parquet``.
+*   Extracted :func:`dispatch.engine.charge_storage_py` and
+    :func:`dispatch.engine.make_rank_arrays_py` from :func:`.dispatch_engine_py`. This
+    allows easier unit testing and, in the former case, makes sure all charging is
+    implemented consistently.
+*   Added plotting functions :meth:`.DispatchModel.plot_output` to visualize columns
+    from :meth:`.DispatchModel.full_output` and updated
+    :meth:`.DispatchModel.plot_period` to display data by generator if ``by_gen=True``.
+    :meth:`.DispatchModel.plot_year` can now display the results with daily or hourly
+    frequency.
 
 
+Known Issues
+^^^^^^^^^^^^
+*   The storage in DC-coupled RE+Storage system can be charged by either the grid or
+    excess RE that would have been curtailed because of the size of the inverter. It is
+    not possible to restrict grid charging in these systems. It is also not possible to
+    charge storage rather than export to the grid when RE output can fit through the
+    inverter.
+*   It is possible that output from DC-coupled RE+Storage facilities during some hours
+    will exceed the system's inverter capacity because when we discharge these storage
+    facilities, we do not know how much 'room' there is in the inverter because we do
+    not know the RE-side's output.
+*   :class:`.DataZip` are effectively immutable once they are created so the ``a`` mode
+    is not allowed and the ``w`` mode is not allowed on existing files. This is because
+    it is not possible to overwrite or remove a file already in a
+    :class:`zipfile.ZipFile`. That fact prevents us from updating metadata about
+    :class:`pandas.DataFrame` that cannot be stored in the ``parquet`` itself. Ways of
+    addressing this get messy and still wouldn't allow updating existing data without
+    copying everything which a user can do if that is needed.
+
 .. _release-v0-3-0:
 
 ---------------------------------------------------------------------------------------
@@ -28,24 +82,24 @@ What's New?
     :class:`.Validator` to organize and specialize data input
     checking.
 *   Adding cost component details and capacity data to
-    :meth:`.DispatchModel.operations_summary`.
+    :meth:`.DispatchModel.dispatchable_summary`.
 *   We now automatically apply ``operating_date`` and ``retirement_date`` from
     :attr:`.DispatchModel.dispatchable_plant_specs` to
     :attr:`.DispatchModel.dispatchable_profiles` using
     :func:`.apply_op_ret_date`.
 *   Added validation and processing for :attr:`.DispatchModel.re_plant_specs` and
     :attr:`.DispatchModel.re_profiles`, as well as :meth:`.DispatchModel.re_summary`
     to, when the data is provided create a summary of renewable operations analogous
-    to :meth:`.DispatchModel.operations_summary`.
+    to :meth:`.DispatchModel.dispatchable_summary`.
 *   Added :meth:`.DispatchModel.storage_summary` to create a summary of storage
-    operations analogous to :meth:`.DispatchModel.operations_summary`.
+    operations analogous to :meth:`.DispatchModel.dispatchable_summary`.
 *   Added :meth:`.DispatchModel.full_output` to create the kind of outputs needed by
     Optimus and other post-dispatch analysis tools.
 *   Added validation steps for each type of specs that raise an error when an
     operating_date is after the dispatch period which would otherwise result in
     dispatch errors.
-*   New helpers (:func:`.dfs_to_zip` and :func:`.dfs_from_zip`) that simplify saving
-    and reading in groups of :class:`pandas.DataFrame`.
+*   New helpers (:meth:`.DataZip.dfs_to_zip` and :meth:`.DataZip.dfs_from_zip`) that
+    simplify saving and reading in groups of :class:`pandas.DataFrame`.
 *   Added plotting functions :meth:`.DispatchModel.plot_period` and
     :meth:`.DispatchModel.plot_year`.
 

diff --git a/setup.cfg b/setup.cfg
@@ -32,12 +32,12 @@ where=src
 [options.extras_require]
 dev =
     black[jupyter] >= 22,<23
-    black>=22.0,<22.9
+    black>=22.0,<22.11
     isort>=5.0,<5.11
     tox>=3.20,<3.27
 doc =
     doc8>=0.9,<1.1
-    sphinx>=4,!=5.1.0,<5.2.4
+    sphinx>=4,!=5.1.0,<5.3.1
     sphinx-autoapi>=1.8,<2.1
     sphinx-autodoc-typehints
     sphinxcontrib-mermaid
@@ -53,7 +53,7 @@ tests =
     ; A framework for linting & static analysis
     flake8>=4.0,<5.1
     ; Avoid shadowing Python built-in names
-    flake8-builtins>=1.5,<1.6
+    flake8-builtins>=1.5,<2.1
     ; Ensure docstrings are formatted well
     flake8-docstrings>=1.5,<1.7
     ; Allow use of ReST in docstrings

diff --git a/src/dispatch/__init__.py b/src/dispatch/__init__.py
@@ -14,24 +14,17 @@
 except ImportError:
     __version__ = "unknown"
 
-# try:
-#     __version__ = version("rmi.dispatch")
-# except PackageNotFoundError:
-#     # package is not installed
-#     pass
-
-from dispatch.engine import dispatch_engine, dispatch_engine_compiled
-from dispatch.helpers import apply_op_ret_date, copy_profile, dfs_from_zip, dfs_to_zip
+from dispatch.engine import dispatch_engine, dispatch_engine_py
+from dispatch.helpers import DataZip, apply_op_ret_date, copy_profile
 from dispatch.model import DispatchModel
 
 __all__ = [
     "DispatchModel",
+    "dispatch_engine_py",
     "dispatch_engine",
-    "dispatch_engine_compiled",
     "copy_profile",
     "apply_op_ret_date",
-    "dfs_to_zip",
-    "dfs_from_zip",
+    "DataZip",
     "__version__",
 ]
 

diff --git a/src/dispatch/constants.py b/src/dispatch/constants.py
@@ -0,0 +1,77 @@
+"""Constants."""
+from __future__ import annotations
+
+import pandas as pd
+
+MTDF = pd.DataFrame()
+COLOR_MAP = {
+    "Gas CC": "#c85c19",
+    "Gas CT": "#f58228",
+    "Gas RICE": "#fbbb7d",
+    "Gas ST": "#ffdaab",
+    "Coal": "#5f2803",
+    "Other Fossil": "#7d492c",
+    "Biomass": "#556940",
+    "Solar": "#ffcb05",
+    "Onshore Wind": "#005d7f",
+    "Offshore Wind": "#529cba",
+    "Storage": "#7b76ad",
+    "Charge": "#7b76ad",
+    "Discharge": "#7b76ad",
+    "Curtailment": "#dae4c1",
+    "Deficit": "#df897b",
+    "Net Load": "#58585b",
+    "Grossed Load": "#58585b",
+    "Gross Load": "#58585b",
+}
+PLOT_MAP = {
+    "Petroleum Liquids": "Other Fossil",
+    "Natural Gas Steam Turbine": "Gas ST",
+    "Conventional Steam Coal": "Coal",
+    "Natural Gas Fired Combined Cycle": "Gas CC",
+    "Natural Gas Fired Combustion Turbine": "Gas CT",
+    "Natural Gas Internal Combustion Engine": "Gas RICE",
+    "Coal Integrated Gasification Combined Cycle": "Coal",
+    "Other Gases": "Other Fossil",
+    "Petroleum Coke": "Other Fossil",
+    "Wood/Wood Waste Biomass": "Biomass",
+    "Other Waste Biomass": "Biomass",
+    "Landfill Gas": "Biomass",
+    "Municipal Solid Waste": "Biomass",
+    "All Other": "Other Fossil",
+    "solar": "Solar",
+    "Solar Photovoltaic with Energy Storage": "Solar",
+    "onshore_wind": "Onshore Wind",
+    "offshore_wind": "Offshore Wind",
+    "curtailment": "Curtailment",
+    "deficit": "Deficit",
+    # "charge": "Storage",
+    # "discharge": "Storage",
+    "Batteries": "Storage",
+}
+ORDERING = {
+    "gas cc": "001",
+    "gas ct": "003",
+    "gas rice": "004",
+    "gas st": "002",
+    "coal": "000",
+    "other fossil": "005",
+    "biomass": "006",
+    "solar": "009",
+    "onshore wind": "007",
+    "offshore wind": "008",
+    "storage": "010",
+    "curtailment": "011",
+    "january": "101",
+    "february": "102",
+    "march": "103",
+    "april": "104",
+    "may": "105",
+    "june": "106",
+    "july": "107",
+    "august": "108",
+    "september": "109",
+    "october": "110",
+    "november": "111",
+    "december": "112",
+}