diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index ccc2cd6d..f074081d 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -7,6 +7,12 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 + - name: Get history and tags for SCM versioning to work + run: | + git fetch --prune --unshallow + git fetch --depth=1 origin +refs/tags/*:refs/tags/* + git describe --tags + git describe --tags $(git rev-list --tags --max-count=1) - uses: actions/setup-python@v4 with: python-version: "3.10" diff --git a/README.rst b/README.rst index 4236f7ca..9eaabe21 100644 --- a/README.rst +++ b/README.rst @@ -10,8 +10,8 @@ Dispatch: A simple and efficient electricity dispatch model :target: https://rmi-electricity.github.io/dispatch/ :alt: GitHub Pages Status -.. image:: https://coveralls.io/repos/github/rmi-electricity/dispatch/badge.svg?branch=dev - :target: https://coveralls.io/github/rmi-electricity/dispatch?branch=dev +.. image:: https://coveralls.io/repos/github/rmi-electricity/dispatch/badge.svg + :target: https://coveralls.io/github/rmi-electricity/dispatch .. image:: https://img.shields.io/badge/code%20style-black-000000.svg :target: https://github.com/psf/black> diff --git a/docs/conf.py b/docs/conf.py index be93a27f..11f5fde5 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -24,9 +24,6 @@ # number via pkg_resources.get_distribution() so we need more than just an # importable path. -# The full version, including alpha/beta/rc tags -release = version_func("rmi.dispatch") -version = ".".join(release.split(".")[:2]) # -- Project information ----------------------------------------------------- @@ -34,6 +31,12 @@ copyright = f"{datetime.today().year}, RMI, CC-BY-4.0" # noqa: A001 author = "RMI" +# The full version, including alpha/beta/rc tags +release = version_func("rmi.dispatch") +version = ".".join(release.split(".")[:2]) +html_title = f"{project} {version} documentation" + + # -- General configuration --------------------------------------------------- # Add any Sphinx extension module names here, as strings. They can be @@ -58,8 +61,19 @@ autoapi_ignore = [ "*_test.py", "*/package_data/*", + "_*.py", + "*constants.py", ] autoapi_python_class_content = "both" +autoapi_options = [ + "members", + # "undoc-members", + # "private-members", + "show-inheritance", + "show-module-summary", + "special-members", + "imported-members", +] autodoc_typehints = "description" # autoapi_keep_files = True @@ -73,6 +87,7 @@ "numba": ("https://numba.readthedocs.io/en/stable", None), "pandas": ("https://pandas.pydata.org/pandas-docs/stable", None), "pandera": ("https://pandera.readthedocs.io/en/stable", None), + "plotly": ("https://plotly.com/python-api-reference/", None), "pytest": ("https://docs.pytest.org/en/latest/", None), "python": ("https://docs.python.org/3", None), "scipy": ("https://docs.scipy.org/doc/scipy/", None), @@ -80,6 +95,8 @@ "tox": ("https://tox.wiki/en/latest/", None), } +"https://plotly.com/python-api-reference/generated/plotly.graph_objects.Figure.html" +"https://plotly.com/python-api-reference/generated/plotly.graph_objs.Figure" # Add any paths that contain templates here, relative to this directory. templates_path = ["_templates"] diff --git a/docs/release_notes.rst b/docs/release_notes.rst index f7b596a1..7ed8b9b1 100644 --- a/docs/release_notes.rst +++ b/docs/release_notes.rst @@ -11,9 +11,63 @@ Release Notes What's New? ^^^^^^^^^^^ -* Test for :func:`.dispatch_engine`. +* Tests for :func:`.dispatch_engine_py`, :func:`.copy_profile`. +* :meth:`.DispatchModel.hourly_data_check` to help in checking for dispatch errors, + and running down why deficits are occuring. +* :class:`.DispatchModel` now takes ``load_profile`` that resources will be + dispatched against. If ``re_profiles`` and ``re_plant_specs`` are not provided, + this should be a net load profile. If they are provided, this *must* be a gross + load profile, or at least, gross of those RE resources. These calculations are done + by :meth:`.DispatchModel.re_and_net_load`. +* :class:`.DispatchModel` now accepts (and requires) raw DC ``re_profiles``, it + determines actual renewable output using capacity data and ilr provided in + ``re_plant_specs``. This will allow :class:`.DispatchModel` to model DC-coupled + RE+Storage facilities that can charge from otherwise clipped generation. The + calculations for the amount of charging from DC-coupled RE is in + :meth:`.DispatchModel.dc_charge`. +* Updates to :func:`.dispatch_engine_py` and :func:`.engine.validate_inputs_py` to + accommodate DC-coupled RE charging data. Storage can now be charged from + DC-coupled RE in addition to the grid. This includes tracking ``gridcharge`` + in addition to ``charge``, where the latter includes charging from the grid + and DC-coupled RE. +* All output charging metrics use the ``gridcharge`` data because from the grid's + perspective, this is what matters. ``discharge`` data does not distinguish, + so in some cases net charge data may be positive, this reflects RE generation + run through the battery that otherwise would have been curtailed. +* :class:`.DataZip`, a subclass of :class:`zipfile.ZipFile` that has methods for + easily reading and writing :class:`pandas.DataFrame` as ``parquet`` and + :class:`dict` as ``json``. This includes storing column names separately that + cannot be included in a ``parquet``. +* Extracted :func:`dispatch.engine.charge_storage_py` and + :func:`dispatch.engine.make_rank_arrays_py` from :func:`.dispatch_engine_py`. This + allows easier unit testing and, in the former case, makes sure all charging is + implemented consistently. +* Added plotting functions :meth:`.DispatchModel.plot_output` to visualize columns + from :meth:`.DispatchModel.full_output` and updated + :meth:`.DispatchModel.plot_period` to display data by generator if ``by_gen=True``. + :meth:`.DispatchModel.plot_year` can now display the results with daily or hourly + frequency. +Known Issues +^^^^^^^^^^^^ +* The storage in DC-coupled RE+Storage system can be charged by either the grid or + excess RE that would have been curtailed because of the size of the inverter. It is + not possible to restrict grid charging in these systems. It is also not possible to + charge storage rather than export to the grid when RE output can fit through the + inverter. +* It is possible that output from DC-coupled RE+Storage facilities during some hours + will exceed the system's inverter capacity because when we discharge these storage + facilities, we do not know how much 'room' there is in the inverter because we do + not know the RE-side's output. +* :class:`.DataZip` are effectively immutable once they are created so the ``a`` mode + is not allowed and the ``w`` mode is not allowed on existing files. This is because + it is not possible to overwrite or remove a file already in a + :class:`zipfile.ZipFile`. That fact prevents us from updating metadata about + :class:`pandas.DataFrame` that cannot be stored in the ``parquet`` itself. Ways of + addressing this get messy and still wouldn't allow updating existing data without + copying everything which a user can do if that is needed. + .. _release-v0-3-0: --------------------------------------------------------------------------------------- @@ -28,7 +82,7 @@ What's New? :class:`.Validator` to organize and specialize data input checking. * Adding cost component details and capacity data to - :meth:`.DispatchModel.operations_summary`. + :meth:`.DispatchModel.dispatchable_summary`. * We now automatically apply ``operating_date`` and ``retirement_date`` from :attr:`.DispatchModel.dispatchable_plant_specs` to :attr:`.DispatchModel.dispatchable_profiles` using @@ -36,16 +90,16 @@ What's New? * Added validation and processing for :attr:`.DispatchModel.re_plant_specs` and :attr:`.DispatchModel.re_profiles`, as well as :meth:`.DispatchModel.re_summary` to, when the data is provided create a summary of renewable operations analogous - to :meth:`.DispatchModel.operations_summary`. + to :meth:`.DispatchModel.dispatchable_summary`. * Added :meth:`.DispatchModel.storage_summary` to create a summary of storage - operations analogous to :meth:`.DispatchModel.operations_summary`. + operations analogous to :meth:`.DispatchModel.dispatchable_summary`. * Added :meth:`.DispatchModel.full_output` to create the kind of outputs needed by Optimus and other post-dispatch analysis tools. * Added validation steps for each type of specs that raise an error when an operating_date is after the dispatch period which would otherwise result in dispatch errors. -* New helpers (:func:`.dfs_to_zip` and :func:`.dfs_from_zip`) that simplify saving - and reading in groups of :class:`pandas.DataFrame`. +* New helpers (:meth:`.DataZip.dfs_to_zip` and :meth:`.DataZip.dfs_from_zip`) that + simplify saving and reading in groups of :class:`pandas.DataFrame`. * Added plotting functions :meth:`.DispatchModel.plot_period` and :meth:`.DispatchModel.plot_year`. diff --git a/setup.cfg b/setup.cfg index 192f671f..b62bc755 100644 --- a/setup.cfg +++ b/setup.cfg @@ -32,12 +32,12 @@ where=src [options.extras_require] dev = black[jupyter] >= 22,<23 - black>=22.0,<22.9 + black>=22.0,<22.11 isort>=5.0,<5.11 tox>=3.20,<3.27 doc = doc8>=0.9,<1.1 - sphinx>=4,!=5.1.0,<5.2.4 + sphinx>=4,!=5.1.0,<5.3.1 sphinx-autoapi>=1.8,<2.1 sphinx-autodoc-typehints sphinxcontrib-mermaid @@ -53,7 +53,7 @@ tests = ; A framework for linting & static analysis flake8>=4.0,<5.1 ; Avoid shadowing Python built-in names - flake8-builtins>=1.5,<1.6 + flake8-builtins>=1.5,<2.1 ; Ensure docstrings are formatted well flake8-docstrings>=1.5,<1.7 ; Allow use of ReST in docstrings diff --git a/src/dispatch/__init__.py b/src/dispatch/__init__.py index eb1dfded..c4929a58 100644 --- a/src/dispatch/__init__.py +++ b/src/dispatch/__init__.py @@ -14,24 +14,17 @@ except ImportError: __version__ = "unknown" -# try: -# __version__ = version("rmi.dispatch") -# except PackageNotFoundError: -# # package is not installed -# pass - -from dispatch.engine import dispatch_engine, dispatch_engine_compiled -from dispatch.helpers import apply_op_ret_date, copy_profile, dfs_from_zip, dfs_to_zip +from dispatch.engine import dispatch_engine, dispatch_engine_py +from dispatch.helpers import DataZip, apply_op_ret_date, copy_profile from dispatch.model import DispatchModel __all__ = [ "DispatchModel", + "dispatch_engine_py", "dispatch_engine", - "dispatch_engine_compiled", "copy_profile", "apply_op_ret_date", - "dfs_to_zip", - "dfs_from_zip", + "DataZip", "__version__", ] diff --git a/src/dispatch/constants.py b/src/dispatch/constants.py new file mode 100644 index 00000000..c73474cd --- /dev/null +++ b/src/dispatch/constants.py @@ -0,0 +1,77 @@ +"""Constants.""" +from __future__ import annotations + +import pandas as pd + +MTDF = pd.DataFrame() +COLOR_MAP = { + "Gas CC": "#c85c19", + "Gas CT": "#f58228", + "Gas RICE": "#fbbb7d", + "Gas ST": "#ffdaab", + "Coal": "#5f2803", + "Other Fossil": "#7d492c", + "Biomass": "#556940", + "Solar": "#ffcb05", + "Onshore Wind": "#005d7f", + "Offshore Wind": "#529cba", + "Storage": "#7b76ad", + "Charge": "#7b76ad", + "Discharge": "#7b76ad", + "Curtailment": "#dae4c1", + "Deficit": "#df897b", + "Net Load": "#58585b", + "Grossed Load": "#58585b", + "Gross Load": "#58585b", +} +PLOT_MAP = { + "Petroleum Liquids": "Other Fossil", + "Natural Gas Steam Turbine": "Gas ST", + "Conventional Steam Coal": "Coal", + "Natural Gas Fired Combined Cycle": "Gas CC", + "Natural Gas Fired Combustion Turbine": "Gas CT", + "Natural Gas Internal Combustion Engine": "Gas RICE", + "Coal Integrated Gasification Combined Cycle": "Coal", + "Other Gases": "Other Fossil", + "Petroleum Coke": "Other Fossil", + "Wood/Wood Waste Biomass": "Biomass", + "Other Waste Biomass": "Biomass", + "Landfill Gas": "Biomass", + "Municipal Solid Waste": "Biomass", + "All Other": "Other Fossil", + "solar": "Solar", + "Solar Photovoltaic with Energy Storage": "Solar", + "onshore_wind": "Onshore Wind", + "offshore_wind": "Offshore Wind", + "curtailment": "Curtailment", + "deficit": "Deficit", + # "charge": "Storage", + # "discharge": "Storage", + "Batteries": "Storage", +} +ORDERING = { + "gas cc": "001", + "gas ct": "003", + "gas rice": "004", + "gas st": "002", + "coal": "000", + "other fossil": "005", + "biomass": "006", + "solar": "009", + "onshore wind": "007", + "offshore wind": "008", + "storage": "010", + "curtailment": "011", + "january": "101", + "february": "102", + "march": "103", + "april": "104", + "may": "105", + "june": "106", + "july": "107", + "august": "108", + "september": "109", + "october": "110", + "november": "111", + "december": "112", +} diff --git a/src/dispatch/engine.py b/src/dispatch/engine.py index 76d64115..39986ac2 100644 --- a/src/dispatch/engine.py +++ b/src/dispatch/engine.py @@ -4,10 +4,17 @@ import numpy as np from numba import njit -__all__ = ["dispatch_engine_compiled", "dispatch_engine"] +__all__ = [ + "dispatch_engine", + "make_rank_arrays", + "charge_storage", + "dispatch_engine_py", + "make_rank_arrays_py", + "charge_storage_py", +] -def dispatch_engine( +def dispatch_engine_py( net_load: np.ndarray, hr_to_cost_idx: np.ndarray, historical_dispatch: np.ndarray, @@ -16,8 +23,9 @@ def dispatch_engine( dispatchable_marginal_cost: np.ndarray, storage_mw: np.ndarray, storage_hrs: np.ndarray, - storage_eff: np.ndarray = np.array((0.9, 0.5)), - storage_op_hour: np.ndarray = np.array((0, 0)), + storage_eff: np.ndarray, + storage_op_hour: np.ndarray, + storage_dc_charge: np.ndarray, ) -> tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]: """Dispatch engine that can be compiled with :func:`numba.jit`. @@ -32,27 +40,37 @@ def dispatch_engine( net_load: net load, as in net of RE generation, negative net load means excess renewables hr_to_cost_idx: an array that contains for each hour, the index of the correct - column in ``dispatchable_marginal_cost`` that contains cost data for that hour - historical_dispatch: historic plant dispatch, acts as an hourly upper constraint - on this dispatch + column in ``dispatchable_marginal_cost`` that contains cost data for that + hour + historical_dispatch: historic plant dispatch, acts as an hourly upper + constraint on this dispatch dispatchable_ramp_mw: max one hour ramp in MW dispatchable_startup_cost: startup cost in $ for each dispatchable generator - dispatchable_marginal_cost: annual marginal cost for each dispatchable generator in $/MWh - rows are generators and columns are years + dispatchable_marginal_cost: annual marginal cost for each dispatchable + generator in $/MWh rows are generators and columns are years storage_mw: max charge/discharge rate for storage in MW storage_hrs: duration of storage storage_eff: storage round-trip efficiency storage_op_hour: first hour in which storage is available, i.e. the index of the operating date + storage_dc_charge: an array whose columns match each storage facility, and a + row for each hour representing how much energy would be curtailed at an + attached renewable facility because it exceeds the system's inverter. This + then represents how much the storage in an RE+Storage facility could be + charged by otherwise curtailed energy when ilr>1. Returns: - redispatch: new hourly dispatch - storage: hourly charge, discharge, and state of charge data - system_level: hourly deficit, dirty charge, and total curtailment data - starts: count of starts for each plant in each year + - **redispatch** (:class:`numpy.ndarray`) - new hourly dispatch + - **storage** (:class:`numpy.ndarray`) - hourly charge, discharge, and state + of charge data + - **system_level** (:class:`numpy.ndarray`) - hourly deficit, dirty charge, + and total curtailment dat + - **starts** (:class:`numpy.ndarray`) - count of starts for each plant in + each year + """ - _validate_inputs( + validate_inputs( net_load, hr_to_cost_idx, historical_dispatch, @@ -62,58 +80,36 @@ def dispatch_engine( storage_mw, storage_hrs, storage_eff, + storage_dc_charge, ) storage_soc_max: np.ndarray = storage_mw * storage_hrs - # internal dispatch data we need to track; (0) current run op_hours - # (1) whether we touched the plant in the first round of dispatch - op_data: np.ndarray = np.zeros((dispatchable_ramp_mw.shape[0], 2), dtype=np.int64) - # need to set op_hours to 1 for plants that we are starting off as operating - op_data[:, 0] = np.where(historical_dispatch[0, :] > 0.0, 1, 0) + marginal_ranks, start_ranks = make_rank_arrays( + dispatchable_marginal_cost, dispatchable_startup_cost + ) # create an array to keep track of re-dispatch redispatch: np.ndarray = np.zeros_like(historical_dispatch) - # to avoid having to do the first hour differently, we just assume original - # dispatch in that hour and then skip it - redispatch[0, :] = historical_dispatch[0, :] - - # create an array to determine the marginal cost dispatch order for each year - # the values in each column represent the canonical indexes for each resource - # and they are in the order of increasing marginal cost for that year (column) - marginal_ranks: np.ndarray = np.hstack( - ( - np.arange(dispatchable_marginal_cost.shape[0]).reshape(-1, 1), - dispatchable_marginal_cost, - ) - ) - for i in range(1, 1 + dispatchable_marginal_cost[0, :].shape[0]): - marginal_ranks[:, i] = marginal_ranks[marginal_ranks[:, i].argsort()][:, 0] - marginal_ranks = marginal_ranks[:, 1:].astype(np.int64) - - # create an array to determine the startup cost order for each year - # the values in each column represent the canonical indexes for each resource - # and they are in the order of increasing startup cost for that year (column) - start_ranks: np.ndarray = np.hstack( - ( - np.arange(dispatchable_startup_cost.shape[0]).reshape(-1, 1), - dispatchable_startup_cost, - ) - ) - for i in range(1, 1 + dispatchable_startup_cost[0, :].shape[0]): - start_ranks[:, i] = start_ranks[start_ranks[:, i].argsort()][:, 0] - start_ranks = start_ranks[:, 1:].astype(np.int64) - # array to keep track of starts by year starts: np.ndarray = np.zeros_like(marginal_ranks) - # array to keep track of hourly storage data. rows are hours, columns - # (0) charge, (1) discharge, (2) state of charge - storage: np.ndarray = np.zeros((len(net_load), 3, len(storage_mw))) - + # (0) charge, (1) discharge, (2) state of charge, (3) grid charge + storage: np.ndarray = np.zeros((len(net_load), 4, len(storage_mw))) # array to keep track of system level data (0) deficits, (1) dirty charge, # (2) curtailment - system_level: np.ndarray = np.zeros_like(storage[:, :, 0]) + system_level: np.ndarray = np.zeros((len(net_load), 3)) + # internal dispatch data we need to track; (0) current run op_hours + # (1) whether we touched the plant in the first round of dispatch + op_data: np.ndarray = np.zeros( + (dispatchable_marginal_cost.shape[0], 2), dtype=np.int64 + ) + + # to avoid having to do the first hour differently, we just assume original + # dispatch in that hour and then skip it + redispatch[0, :] = historical_dispatch[0, :] + # need to set op_hours to 1 for plants that we are starting off as operating + op_data[:, 0] = np.where(historical_dispatch[0, :] > 0.0, 1, 0) # the big loop where we iterate through all the hours for hr, (deficit, yr) in enumerate(zip(net_load, hr_to_cost_idx)): @@ -122,21 +118,24 @@ def dispatch_engine( # historical fossil dispatch if hr == 0: # if there is excess RE we charge the battery in the first hour - if deficit < 0.0: + if deficit < 0.0 or np.any(storage_dc_charge[hr, :] > 0.0): for es_i in range(storage.shape[2]): # skip the `es_i` storage resource if it is not yet in operation if storage_op_hour[es_i] > hr: continue - # charge storage - storage[hr, 0, es_i] = min( - -deficit, - storage_mw[es_i], - storage_soc_max[es_i] / storage_eff[es_i], + + storage[hr, :, es_i] = charge_storage( + deficit=deficit, + soc=0.0, # previous soc + dc_charge=storage_dc_charge[hr, es_i], + mw=storage_mw[es_i], + soc_max=storage_soc_max[es_i], + eff=storage_eff[es_i], ) - storage[hr, 2, es_i] = storage[hr, 0, es_i] * storage_eff[es_i] - deficit += storage[hr, 0, es_i] - if deficit == 0.0: - break + + # update the deficit if we are grid charging + if deficit < 0.0: + deficit += storage[hr, 3, es_i] continue # want to figure out how much we'd like to dispatch fossil given @@ -174,41 +173,47 @@ def dispatch_engine( redispatch[hr, r] = r_out # keep a running total of remaining deficit, having this value be negative # just makes the loop code more complicated, if it actually should be - # negative we capture that below + # negative we capture that when we calculate the actual deficit based + # on redispatch below prov_deficit = max(0, prov_deficit - r_out) # calculate the true deficit as the hour's net load less actual dispatch # of fossil plants in hr that were also operating in hr - 1 deficit -= np.sum(redispatch[hr, :]) - # # negative deficit means excess generation, so we charge the battery - # # and move on to the next hour - if deficit < 0.0: + # negative deficit means excess generation, so we charge the battery + # if there is DC-coupled storage, we also charge that storage if there is + # RE generation that would otherwise be curtailed + if deficit < 0.0 or np.any(storage_dc_charge[hr, :] > 0.0): for es_i in range(storage.shape[2]): # skip the `es_i` storage resource if it is not yet in operation if storage_op_hour[es_i] > hr: continue - # calculate the amount of charging, to account for battery capacity, we - # make sure that `charge` would not put `soc` over `storage_soc_max` - soc = storage[hr - 1, 2, es_i] - charge = min( - -deficit, - storage_mw[es_i], - (storage_soc_max[es_i] - soc) / storage_eff[es_i], + storage[hr, :, es_i] = charge_storage( + deficit=deficit, + soc=storage[hr - 1, 2, es_i], # previous soc + dc_charge=storage_dc_charge[hr, es_i], + mw=storage_mw[es_i], + soc_max=storage_soc_max[es_i], + eff=storage_eff[es_i], ) - # calculate new `soc` and check that it makes sense - soc = soc + charge * storage_eff[es_i] - assert soc <= storage_soc_max[es_i] - # store charge and new soc - storage[hr, 0, es_i], storage[hr, 2, es_i] = charge, soc + # alias grid_charge + grid_charge = storage[hr, 3, es_i] # calculate the amount of charging that was dirty # TODO check that this calculation is actually correct - system_level[hr, 1] += min(max(0, charge - net_load[hr] * -1), charge) - # calculate the amount of total curtailment - # system_level[hr, 2] += -deficit - charge - deficit += charge + system_level[hr, 1] += min( + max(0, grid_charge - net_load[hr] * -1), grid_charge + ) + # if we are charging from the grid, need to update the deficit + if deficit < 0.0: + deficit += grid_charge # store the amount of total curtailment # TODO check that this calculation is actually correct + + # this 'continue' and storing of the deficit needs an extra check because + # sometimes we charge storage direct from DC-coupled RE even when there + # is a positive deficit + if deficit <= 0.0: system_level[hr, 2] = -deficit continue @@ -217,8 +222,9 @@ def dispatch_engine( # or discharge every hour whether there is a deficit or not to propagate # state of charge forward for es_i in range(storage.shape[2]): - # skip the `es_i` storage resource if it is not yet in operation - if storage_op_hour[es_i] > hr: + # skip the `es_i` storage resource if it is not yet in operation or + # there was excess generation from a DC-coupled RE facility + if storage_op_hour[es_i] > hr or storage_dc_charge[hr, es_i] > 0.0: continue discharge = min(storage[hr - 1, 2, es_i], deficit, storage_mw[es_i]) storage[hr, 1, es_i] = discharge @@ -272,11 +278,115 @@ def dispatch_engine( redispatch <= historical_dispatch * (1 + 1e-4) ), "redispatch exceeded historical dispatch in at least 1 hour" + if np.all(storage_dc_charge == 0.0): + assert np.all( + storage[:, 0, :] == storage[:, 3, :] + ), "charge != gridcharge when storage_dc_charge is all 0.0" + else: + assert np.all( + storage[:, 0, :] >= storage[:, 3, :] + ), "gridcharge exceeded charge for at least one storage facility/hour" + + # for es_i in range(storage.shape[2]): + # if np.any( + # storage[storage[:, 1, es_i] > np.roll(storage[:, 2, es_i], 1)] + # ): + # raise AssertionError(f"discharge exceeded previous state of charge in at least 1 hour for {es_i}") + return redispatch, storage, system_level, starts -@njit -def _validate_inputs( +def make_rank_arrays_py( + marginal_cost: np.ndarray, startup_cost: np.ndarray +) -> tuple[np.ndarray, np.ndarray]: + """Turn cost arrays into rank arrays. + + Args: + marginal_cost: array of marginal costs + startup_cost: array of startup cost + + Returns: + - **marginal_ranks** (:class:`numpy.ndarray`) - marginal rank array + - **start_ranks** (:class:`numpy.ndarray`) - start rank array + """ + # create an array to determine the marginal cost dispatch order for each year + # the values in each column represent the canonical indexes for each resource + # and they are in the order of increasing marginal cost for that year (column) + marginal_ranks: np.ndarray = np.hstack( + ( + np.arange(marginal_cost.shape[0]).reshape(-1, 1), + marginal_cost, + ) + ) + for i in range(1, 1 + marginal_cost[0, :].shape[0]): + marginal_ranks[:, i] = marginal_ranks[marginal_ranks[:, i].argsort()][:, 0] + marginal_ranks = marginal_ranks[:, 1:].astype(np.int64) + # create an array to determine the startup cost order for each year + # the values in each column represent the canonical indexes for each resource + # and they are in the order of increasing startup cost for that year (column) + start_ranks: np.ndarray = np.hstack( + ( + np.arange(startup_cost.shape[0]).reshape(-1, 1), + startup_cost, + ) + ) + for i in range(1, 1 + startup_cost[0, :].shape[0]): + start_ranks[:, i] = start_ranks[start_ranks[:, i].argsort()][:, 0] + start_ranks = start_ranks[:, 1:].astype(np.int64) + return marginal_ranks, start_ranks + + +def charge_storage_py( + deficit: float, + soc: float, + dc_charge: float, + mw: float, + soc_max: float, + eff: float, +) -> tuple[float, float, float, float]: + """Calculations for charging storage. + + Args: + deficit: amount of charging possible from the grid + soc: state of charge before charging + dc_charge: power available from DC-coupled RE + mw: storage power capacity + soc_max: storage energy capacity + eff: round-trip efficiency of storage + + Returns: + A tuple with the same organization of columns of internal ``storage`` in + :func:`.dispatch_engine_py`. + + - **charge** (:class:`float`) - total charge in the hour + - **discharge** (:class:`float`) - always 0.0, a placeholder + - **soc** (:class:`float`) - tate of charge after charging + - **grid_charge** (:class:`float`) - portion of ``charge`` that came from + the grid + + """ + # because we can now end up in this loop when deficit is positive, + # we need to prevent that positive deficit from mucking up our + # calculations + _grid_charge = -deficit if deficit < 0.0 else 0.0 + charge = min( + # _grid_charge represents grid charging, + # dc_charge represents charging from a DC-coupled RE facility + _grid_charge + dc_charge, + mw, + # calculate the amount of charging, to account for battery capacity, we + # make sure that `charge` would not put `soc` over `soc_max` + (soc_max - soc) / eff, + ) + # we charge from DC-coupled RE before charging from the grid + grid_charge = max(0.0, charge - dc_charge) + # calculate new `soc` and check that it makes sense + soc = soc + charge * eff + assert soc <= soc_max + return charge, 0.0, soc, grid_charge + + +def validate_inputs_py( net_load, hr_to_cost_idx, historical_dispatch, @@ -286,8 +396,15 @@ def _validate_inputs( storage_mw, storage_hrs, storage_eff, + storage_dc_charge, ): - if not (len(storage_mw) == len(storage_hrs) == len(storage_eff)): + """Validate shape of inputs.""" + if not ( + len(storage_mw) + == len(storage_hrs) + == len(storage_eff) + == storage_dc_charge.shape[1] + ): raise AssertionError("storage data does not match") if not ( ramp_mw.shape[0] @@ -296,7 +413,12 @@ def _validate_inputs( == historical_dispatch.shape[1] ): raise AssertionError("shapes of dispatchable plant data do not match") - if not (len(net_load) == len(hr_to_cost_idx) == len(historical_dispatch)): + if not ( + len(net_load) + == len(hr_to_cost_idx) + == len(historical_dispatch) + == len(storage_dc_charge) + ): raise AssertionError("profile lengths do not match") if not ( len(np.unique(hr_to_cost_idx)) @@ -309,4 +431,14 @@ def _validate_inputs( ) -dispatch_engine_compiled = njit(dispatch_engine, error_model="numpy") +validate_inputs = njit(validate_inputs_py) +""":mod:`numba` compiled version of :func:`.validate_inputs_py`.""" + +make_rank_arrays = njit(make_rank_arrays_py) +""":mod:`numba` compiled version of :func:`.make_rank_arrays_py`.""" + +charge_storage = njit(charge_storage_py) +""":mod:`numba` compiled version of :func:`.charge_storage_py`.""" + +dispatch_engine = njit(dispatch_engine_py, error_model="numpy") +""":mod:`numba` compiled version of :func:`.dispatch_engine_py`.""" diff --git a/src/dispatch/helpers.py b/src/dispatch/helpers.py index 83d8e2f2..205b4826 100644 --- a/src/dispatch/helpers.py +++ b/src/dispatch/helpers.py @@ -2,14 +2,19 @@ from __future__ import annotations import json +import logging from collections.abc import Generator from io import BytesIO from pathlib import Path -from zipfile import ZipFile +from zipfile import ZipFile, ZipInfo import numpy as np import pandas as pd +from dispatch.constants import ORDERING + +LOGGER = logging.getLogger(__name__) + def copy_profile( profiles: pd.DataFrame | pd.Series, years: range | tuple @@ -99,102 +104,181 @@ def _str_cols(df, *args): return df.set_axis(list(map(str, range(df.shape[1]))), axis="columns") -def _to_frame(df, n): - return df.to_frame(name=n) - - -def _null(df, *args): - return df - - -def dfs_to_zip(df_dict: dict[str, pd.DataFrame], path: Path, clobber=False) -> None: - """Create a zip of parquets. - - Args: - df_dict: dict of dfs to put into a zip - path: path for the zip - clobber: if True, overwrite exiting file with same path - - Returns: None - - """ - bad_cols = {} - other_stuff = {} - path = path.with_suffix(".zip") - if path.exists(): - if not clobber: - raise FileExistsError(f"{path} exists, to overwrite set `clobber=True`") - path.unlink() - with ZipFile(path, "w") as z: - for key, val in df_dict.items(): - if isinstance(val, pd.DataFrame) and not val.empty: - try: - z.writestr(f"{key}.parquet", val.to_parquet()) - except ValueError: - bad_cols.update({key: (list(val.columns), list(val.columns.names))}) - z.writestr(f"{key}.parquet", _str_cols(val).to_parquet()) - elif isinstance(val, pd.Series) and not val.empty: - z.writestr(f"{key}.parquet", val.to_frame(name=key).to_parquet()) - elif isinstance(val, (float, int, str, tuple, dict, list)): - other_stuff.update({key: val}) - z.writestr( - "other_stuff.json", json.dumps(other_stuff, ensure_ascii=False, indent=4) - ) - z.writestr("bad_cols.json", json.dumps(bad_cols, ensure_ascii=False, indent=4)) +def dispatch_key(item): + """Key function for use sorting, including with :mod:`pandas` objects.""" + if isinstance(item, pd.Series): + return item.str.casefold().replace(ORDERING) + if isinstance(item, pd.Index): + return pd.Index([ORDERING.get(x.casefold(), str(x)) for x in item]) + return ORDERING.get(item.casefold(), str(item)) -def dfs_from_zip(path: Path, lazy=False) -> dict | Generator: - """Dict of dfs from a zip of parquets. - - Args: - path: path of the zip to load - lazy: if True, return a generator rather than a dict +class DataZip(ZipFile): + """SubClass of :class:`ZipFile` with methods for easier use with :mod:`pandas`. - Returns: dict of dfs or Generator of name, df pairs + z = DataZip(file, mode="r", compression=ZIP_STORED, allowZip64=True, + compresslevel=None) """ - if lazy: - return _lazy_load(path) - out_dict = {} - with ZipFile(path.with_suffix(".zip"), "r") as z: - bad_cols = json.loads(z.read("bad_cols.json")) - for name in z.namelist(): - if "parquet" in name: - out_dict.update( - { - name.removesuffix(".parquet"): pd.read_parquet( - BytesIO(z.read(name)) - ).squeeze() - } - ) - out_dict = out_dict | json.loads(z.read("other_stuff.json")) - for df_name, (cols, names) in bad_cols.items(): - if isinstance(names, (tuple, list)) and len(names) > 1: - cols = pd.MultiIndex.from_tuples(cols, names=names) + def __init__(self, file: str | Path, mode="r", *args, **kwargs): + """Open the ZIP file. + + Args: + file: Either the path to the file, or a file-like object. + If it is a path, the file will be opened and closed by ZipFile. + mode: The mode can be either read 'r', write 'w', exclusive create 'x', + or append 'a'. + compression: ZIP_STORED (no compression), ZIP_DEFLATED (requires zlib), + ZIP_BZIP2 (requires bz2) or ZIP_LZMA (requires lzma). + allowZip64: if True ZipFile will create files with ZIP64 extensions when + needed, otherwise it will raise an exception when this would + be necessary. + compresslevel: None (default for the given compression type) or an integer + specifying the level to pass to the compressor. + When using ZIP_STORED or ZIP_LZMA this keyword has no effect. + When using ZIP_DEFLATED integers 0 through 9 are accepted. + When using ZIP_BZIP2 integers 1 through 9 are accepted. + """ + if not isinstance(file, Path): + file = Path(file) + file = file.with_suffix(".zip") + if mode in ("a", "x"): + raise ValueError("DataZip does not support modes 'a' or 'x'") + if file.exists() and mode == "w": + raise FileExistsError( + f"{file} exists, you cannot write or append to an existing DataZip." + ) + super().__init__(file, mode, *args, **kwargs) + try: + self.bad_cols = self._read_dict("bad_cols") + except KeyError: + self.bad_cols = {} + + def read( + self, name: str | ZipInfo, pwd: bytes | None = ... + ) -> bytes | pd.DataFrame | pd.Series | dict: + """Return obj or bytes for name.""" + if "parquet" in name or f"{name}.parquet" in self.namelist(): + return self._read_df(name) + if "json" in name or f"{name}.json" in self.namelist(): + return self._read_dict(name) + return super().read(name) + + def read_dfs(self) -> Generator[tuple[str, pd.DataFrame | pd.Series]]: + """Read all dfs lazily.""" + for name, *suffix in map(lambda x: x.split("."), self.namelist()): + if "parquet" in suffix: + yield name, self.read(name) + + def _read_df(self, name) -> pd.DataFrame | pd.Series: + name = name.removesuffix(".parquet") + out = pd.read_parquet(BytesIO(super().read(name + ".parquet"))) + + if name in self.bad_cols: + cols, names = self.bad_cols[name] + if isinstance(names, (tuple, list)) and len(names) > 1: + cols = pd.MultiIndex.from_tuples(cols, names=names) + else: + cols = pd.Index(cols, name=names[0]) + out.columns = cols + return out.squeeze() + + def _read_dict(self, name) -> dict: + return json.loads(super().read(name.removesuffix(".json") + ".json")) + + def writed( + self, + name: str, + data: str | dict | pd.DataFrame | pd.Series, + ) -> None: + """Write dict, df, str, to name.""" + if data is None: + LOGGER.info("Unable to write data %s because it is None.", name) + return None + name = name.removesuffix(".json").removesuffix(".parquet") + if isinstance(data, dict): + self._write_dict(name, data) + elif isinstance(data, (pd.DataFrame, pd.Series)): + self._write_df(name, data) + else: + raise TypeError("`data` must be a dict, pd.DataFrame, or pd.Series") + + def _write_df(self, name: str, df: pd.DataFrame | pd.Series) -> None: + """Write a df in the ZIP as parquet.""" + if df.empty: + LOGGER.info("Unable to write df %s because it is empty.", name) + return None + if f"{name}.parquet" not in self.namelist(): + if isinstance(df, pd.Series): + df = df.to_frame(name=name) + try: + self.writestr(f"{name}.parquet", df.to_parquet()) + except ValueError: + self.bad_cols.update({name: (list(df.columns), list(df.columns.names))}) + self.writestr(f"{name}.parquet", _str_cols(df).to_parquet()) + else: + raise FileExistsError(f"{name}.parquet already in {self.filename}") + + def _write_dict( + self, name, dct: dict[int | str, list | tuple | dict | str | float | int] + ) -> None: + """Write a dict in the ZIP as json.""" + if f"{name}.json" not in self.namelist(): + self.writestr(f"{name}.json", json.dumps(dct, ensure_ascii=False, indent=4)) else: - cols = pd.Index(cols, name=names[0]) - out_dict[df_name].columns = cols - - return out_dict - - -def _lazy_load(path: Path) -> Generator[tuple[str, pd.DataFrame]]: - with ZipFile(path.with_suffix(".zip"), "r") as z: - bad_cols = json.loads(z.read("bad_cols.json")) - for name in z.namelist(): - if "parquet" in name: - key = name.removesuffix(".parquet") - df = pd.read_parquet(BytesIO(z.read(name))).squeeze() - if key in bad_cols: - cols, names = bad_cols[key] - if isinstance(names, (tuple, list)) and len(names) > 1: - cols = pd.MultiIndex.from_tuples(cols, names=names) - else: - cols = pd.Index(cols, name=names[0]) - df.columns = cols - - yield name, df + raise FileExistsError(f"{name}.json already in {self.filename}") + + def __exit__(self, exc_type, exc_val, exc_tb): + if self.mode == "w": + self._write_dict("bad_cols", self.bad_cols) + super().__exit__(exc_type, exc_val, exc_tb) + + @classmethod + def dfs_to_zip(cls, path: Path, df_dict: dict[str, pd.DataFrame], clobber=False): + """Create a zip of parquets. + + Args: + df_dict: dict of dfs to put into a zip + path: path for the zip + clobber: if True, overwrite exiting file with same path + + Returns: None + + """ + path = path.with_suffix(".zip") + if path.exists(): + if not clobber: + raise FileExistsError(f"{path} exists, to overwrite set `clobber=True`") + path.unlink() + with cls(path, "w") as z: + other_stuff = {} + for key, val in df_dict.items(): + if isinstance(val, (pd.Series, pd.DataFrame, dict)): + z.writed(key, val) + elif isinstance(val, (float, int, str, tuple, dict, list)): + other_stuff.update({key: val}) + z.writed("other_stuff", other_stuff) + + @classmethod + def dfs_from_zip(cls, path: Path) -> dict: + """Dict of dfs from a zip of parquets. + + Args: + path: path of the zip to load + + Returns: dict of dfs + + """ + with cls(path, "r") as z: + out_dict = dict(z.read_dfs()) + try: + other = z.read("other_stuff") + except KeyError: + other = {} + out_dict = out_dict | other + + return out_dict def idfn(val): diff --git a/src/dispatch/metadata.py b/src/dispatch/metadata.py index c2be478e..33ecf0d1 100644 --- a/src/dispatch/metadata.py +++ b/src/dispatch/metadata.py @@ -12,7 +12,7 @@ DT_SCHEMA = pa.Index(pa.Timestamp, name="datetime") PID_SCHEMA = pa.Index(int, name="plant_id_eia") GID_SCHEMA = pa.Index(str, name="generator_id") -NET_LOAD_SCHEMA = pa.SeriesSchema(pa.Float, index=DT_SCHEMA, coerce=True) +LOAD_PROFILE_SCHEMA = pa.SeriesSchema(pa.Float, index=DT_SCHEMA, coerce=True) class Validator: @@ -39,21 +39,24 @@ def __init__(self, obj: Any, gen_set: pd.Index): """Init Validator.""" self.obj = obj self.gen_set = gen_set - self.net_load_profile = obj.net_load_profile + self.load_profile = obj.load_profile self.storage_specs_schema = pa.DataFrameSchema( + index=pa.MultiIndex( + [PID_SCHEMA, GID_SCHEMA], + unique=["plant_id_eia"], + strict=True, + coerce=True, + ), columns={ - "plant_id_eia": pa.Column(int, required=False), - "generator_id": pa.Column(str, required=False), "capacity_mw": pa.Column(float, pa.Check.greater_than(0)), "duration_hrs": pa.Column(int, pa.Check.greater_than(0)), "roundtrip_eff": pa.Column(float, pa.Check.in_range(0, 1)), "operating_date": pa.Column( pa.Timestamp, - pa.Check.less_than(self.net_load_profile.index.max()), + pa.Check.less_than(self.load_profile.index.max()), description="operating_date in storage_specs", ), }, - index=pa.Index(int, unique=True), # strict=True, coerce=True, title="storage_specs", @@ -61,15 +64,16 @@ def __init__(self, obj: Any, gen_set: pd.Index): self.renewable_specs_schema = pa.DataFrameSchema( index=pa.MultiIndex( [PID_SCHEMA, GID_SCHEMA], - unique=["plant_id_eia", "generator_id"], + unique=["plant_id_eia"], strict=True, coerce=True, ), columns={ "capacity_mw": pa.Column(float, pa.Check.greater_than(0)), + "ilr": pa.Column(float, pa.Check.in_range(1.0, 10.0)), "operating_date": pa.Column( pa.Timestamp, - pa.Check.less_than(self.net_load_profile.index.max()), + pa.Check.less_than(self.load_profile.index.max()), description="operating_date in renewable_specs", ), "retirement_date": pa.Column(pa.Timestamp, nullable=True), @@ -88,7 +92,7 @@ def __init__(self, obj: Any, gen_set: pd.Index): "ramp_rate": pa.Column(float, pa.Check.greater_than(0)), "operating_date": pa.Column( pa.Timestamp, - pa.Check.less_than(self.net_load_profile.index.max()), + pa.Check.less_than(self.load_profile.index.max()), description="operating_date in dispatchable_specs", ), "retirement_date": pa.Column(pa.Timestamp, nullable=True), @@ -124,9 +128,9 @@ def dispatchable_profiles( coerce=True, strict=True, ).validate(dispatchable_profiles) - if not np.all(dispatchable_profiles.index == self.net_load_profile.index): + if not np.all(dispatchable_profiles.index == self.load_profile.index): raise AssertionError( - "`dispatchable_profiles` and `net_load_profile` indexes must match" + "`dispatchable_profiles` and `load_profile` indexes must match" ) return dispatchable_profiles @@ -151,7 +155,7 @@ def dispatchable_cost(self, dispatchable_cost: pd.DataFrame) -> pd.DataFrame: marg_dts = dispatchable_cost.index.get_level_values("datetime") missing_prds = [ d - for d in self.net_load_profile.resample(marg_freq).first().index + for d in self.load_profile.resample(marg_freq).first().index if d not in marg_dts ] if missing_prds: @@ -164,17 +168,20 @@ def dispatchable_cost(self, dispatchable_cost: pd.DataFrame) -> pd.DataFrame: def storage_specs(self, storage_specs: pd.DataFrame) -> pd.DataFrame: """Validate storage_specs.""" - if storage_specs is None: - LOGGER.warning("Careful, dispatch without storage is untested") - storage_specs = pd.DataFrame( - [0.0, 0, 1.0, self.net_load_profile.index.max()], - columns=[ - "capacity_mw", - "duration_hrs", - "roundtrip_eff", - "operating_date", - ], - ) + # if storage_specs is None: + # LOGGER.warning("Careful, dispatch without storage is untested") + # storage_specs = pd.DataFrame( + # [0.0, 0, 1.0, self.load_profile.index.max()], + # columns=[ + # "capacity_mw", + # "duration_hrs", + # "roundtrip_eff", + # "operating_date", + # ], + # index=pd.MultiIndex.from_tuples( + # [(-99, "es")], names=["plant_id_eia", "generator_id"] + # ), + # ) return self.storage_specs_schema.validate(storage_specs) def renewables( @@ -186,7 +193,10 @@ def renewables( re_plant_specs = self.renewable_specs_schema.validate(re_plant_specs) re_profiles = pa.DataFrameSchema( index=DT_SCHEMA, - columns={x: pa.Column(float) for x in re_plant_specs.index}, + columns={ + x: pa.Column(float, pa.Check.in_range(0.0, 1.0)) + for x in re_plant_specs.index + }, ordered=True, coerce=True, strict=True, diff --git a/src/dispatch/model.py b/src/dispatch/model.py index ed57b4a4..ca707b32 100644 --- a/src/dispatch/model.py +++ b/src/dispatch/model.py @@ -2,119 +2,83 @@ from __future__ import annotations import inspect -import json import logging from collections.abc import Callable from datetime import datetime -from io import BytesIO from pathlib import Path -from zipfile import ZIP_DEFLATED, ZipFile +from zipfile import ZIP_DEFLATED import numpy as np import pandas as pd try: import plotly.express as px + from plotly.graph_objects import Figure PLOTLY_INSTALLED = True except ModuleNotFoundError: + from typing import Any + + Figure = Any PLOTLY_INSTALLED = False __all__ = ["DispatchModel"] from dispatch import __version__ -from dispatch.engine import dispatch_engine, dispatch_engine_compiled -from dispatch.helpers import _null, _str_cols, _to_frame, apply_op_ret_date -from dispatch.metadata import NET_LOAD_SCHEMA, Validator +from dispatch.constants import COLOR_MAP, MTDF, PLOT_MAP +from dispatch.engine import dispatch_engine, dispatch_engine_py +from dispatch.helpers import DataZip, apply_op_ret_date, dispatch_key +from dispatch.metadata import LOAD_PROFILE_SCHEMA, Validator LOGGER = logging.getLogger(__name__) -MTDF = pd.DataFrame() -"""An empty :class:`pandas.DataFrame`.""" -COLOR_MAP = { - "Gas CC": "#c85c19", - "Gas CT": "#f58228", - "Gas RICE": "#fbbb7d", - "Gas ST": "#ffdaab", - "Coal": "#5f2803", - "Other Fossil": "#7d492c", - "Biomass": "#556940", - "Solar": "#ffcb05", - "Onshore Wind": "#005d7f", - "Offshore Wind": "#529cba", - "Storage": "#7b76ad", - "Charge": "#7b76ad", - "Discharge": "#7b76ad", - "Curtailment": "#eec7b7", - "Deficit": "#df897b", - "Net Load": "#58585b", - "Grossed Load": "#58585b", -} -PLOT_MAP = { - "Petroleum Liquids": "Other Fossil", - "Natural Gas Steam Turbine": "Gas ST", - "Conventional Steam Coal": "Coal", - "Natural Gas Fired Combined Cycle": "Gas CC", - "Natural Gas Fired Combustion Turbine": "Gas CT", - "Natural Gas Internal Combustion Engine": "Gas RICE", - "Coal Integrated Gasification Combined Cycle": "Coal", - "Other Gases": "Other Fossil", - "Petroleum Coke": "Other Fossil", - "Wood/Wood Waste Biomass": "Biomass", - "Other Waste Biomass": "Biomass", - "Landfill Gas": "Biomass", - "Municipal Solid Waste": "Biomass", - "All Other": "Other Fossil", - "solar": "Solar", - "onshore_wind": "Onshore Wind", - "offshore_wind": "Offshore Wind", - "curtailment": "Curtailment", - "deficit": "Deficit", - "charge": "Storage", - "discharge": "Storage", -} - class DispatchModel: """Class to contain the core dispatch model functionality. - allow the core dispatch model to accept data set up for different uses - - provide a nicer API that accepts pandas objects on top of :func:`.dispatch_engine` + - provide a nicer API on top of :func:`.dispatch_engine_py` that accepts + :mod:`pandas` objects - methods for common analysis of dispatch results """ - __slots__ = ( - "net_load_profile", + # __slots__ = ( + # "load_profile", + # "net_load_profile", + # "dispatchable_specs", + # "dispatchable_cost", + # "dispatchable_profiles", + # "storage_specs", + # "re_profiles_ac", + # "re_excess", + # "re_plant_specs", + # "dt_idx", + # "yrs_idx", + # "redispatch", + # "storage_dispatch", + # "system_data", + # "starts", + # "_metadata", + # "_cached", + # ) + _parquet_out = ( + "re_profiles_ac", + "re_excess", + "re_plant_specs", + "storage_dispatch", + "system_data", + "storage_specs", "dispatchable_specs", "dispatchable_cost", "dispatchable_profiles", - "storage_specs", - "re_profiles", - "re_plant_specs", - "dt_idx", - "yrs_idx", "redispatch", - "storage_dispatch", - "system_data", - "starts", - "_metadata", - "_cached", + "load_profile", + "net_load_profile", ) - _parquet_out = { - "re_profiles": _null, - "storage_dispatch": _null, - "system_data": _null, - "storage_specs": _null, - "dispatchable_specs": _null, - "dispatchable_cost": _null, - "dispatchable_profiles": _str_cols, - "redispatch": _str_cols, - "net_load_profile": _to_frame, - } def __init__( self, - net_load_profile: pd.Series[float], + load_profile: pd.Series[float], dispatchable_specs: pd.DataFrame, dispatchable_profiles: pd.DataFrame, dispatchable_cost: pd.DataFrame, @@ -127,8 +91,10 @@ def __init__( """Initialize DispatchModel. Args: - net_load_profile: net load, as in net of RE generation, negative net - load means excess renewable generation + load_profile: load profile that resources will be dispatched against. If + ``re_profiles`` and ``re_plant_specs`` are not provided, this should + be a net load profile. If they are provided, this *must* be a gross + profile, or at least, gross of those RE resources. dispatchable_specs: rows are dispatchable generators, columns must include: - capacity_mw: generator nameplate/operating capacity @@ -136,9 +102,13 @@ def __init__( - operating_date: the date the plant entered or will enter service - retirement_date: the date the plant will retire - dispatchable_profiles: set the maximum output of each generator in each hour - dispatchable_cost: cost metrics for each dispatchable generator in each year - must be tidy with :class:`pandas.MultiIndex` of + The index must be a :class:`pandas.MultiIndex` of + ``['plant_id_eia', 'generator_id']``. + + dispatchable_profiles: set the maximum output of each generator in + each hour. + dispatchable_cost: cost metrics for each dispatchable generator in + each year must be tidy with :class:`pandas.MultiIndex` of ``['plant_id_eia', 'generator_id', 'datetime']``, columns must include: @@ -147,15 +117,35 @@ def __init__( - fom_per_kw: fixed O&M (USD/kW) - start_per_kw: generator startup cost (USD/kW) - storage_specs: rows are types of storage, columns must include: + storage_specs: rows are storage facilities, for RE+Storage facilities, + the ``plant_id_eia`` for the storage component must match the + ``plant_id_eia`` for the RE component in ``re_profiles`` and + ``re_plant_specs``. Columns must include: - capacity_mw: max charge/discharge capacity in MW - duration_hrs: storage duration in hours - roundtrip_eff: roundtrip efficiency - operating_date: datetime unit starts operating - re_profiles: ?? - re_plant_specs: ?? + The index must be a :class:`pandas.MultiIndex` of + ``['plant_id_eia', 'generator_id']``. + + re_profiles: normalized renewable profiles, these should be DC profiles, + especially when they are part of RE+Storage resources, if they are + AC profiles, make sure the ilr in ``re_plant_specs`` is 1.0. + re_plant_specs: rows are renewable facilities, for RE+Storage facilities, + the ``plant_id_eia`` for the RE component must match the + ``plant_id_eia`` for the storage component in ``storage_specs``. + Columns must include: + + - capacity_mw: AC capacity of the generator + - ilr: inverter loading ratio, if ilr != 1, the corresponding + profile must be a DC profile. + - operating_date: datetime unit starts operating + + The index must be a :class:`pandas.MultiIndex` of + ``['plant_id_eia', 'generator_id']``. + jit: if ``True``, use numba to compile the dispatch engine, ``False`` is mostly for debugging name: a name, only used in the ``repr`` @@ -169,9 +159,9 @@ def __init__( "jit": jit, } - self.net_load_profile: pd.Series = NET_LOAD_SCHEMA.validate(net_load_profile) + self.load_profile: pd.Series = LOAD_PROFILE_SCHEMA.validate(load_profile) - self.dt_idx = self.net_load_profile.index + self.dt_idx = self.load_profile.index self.yrs_idx = self.dt_idx.to_series().groupby([pd.Grouper(freq="YS")]).first() validator = Validator(self, gen_set=dispatchable_specs.index) @@ -187,17 +177,27 @@ def __init__( self.dispatchable_specs.operating_date, self.dispatchable_specs.retirement_date, ) - self.re_plant_specs, self.re_profiles = validator.renewables( + self.re_plant_specs, re_profiles = validator.renewables( re_plant_specs, re_profiles ) + ( + self.net_load_profile, + self.re_profiles_ac, + self.re_excess, + ) = self.re_and_net_load(re_profiles) # create vars with correct column names that will be replaced after dispatch self.redispatch = MTDF.reindex(columns=self.dispatchable_specs.index) self.storage_dispatch = MTDF.reindex( columns=[ col - for i in range(self.storage_specs.shape[0]) - for col in (f"charge_{i}", f"discharge_{i}", f"soc_{i}") + for i in self.storage_specs.index.get_level_values("plant_id_eia") + for col in ( + f"charge_{i}", + f"discharge_{i}", + f"soc_{i}", + f"gridcharge_{i}", + ) ] ) self.system_data = MTDF.reindex( @@ -206,6 +206,36 @@ def __init__( self.starts = MTDF.reindex(columns=self.dispatchable_specs.index) self._cached = {} + def re_and_net_load(self, re_profiles): + """Create net_load_profile based on what RE data was provided.""" + if self.re_plant_specs is None or re_profiles is None: + return ( + self.load_profile, + None, + None, + ) + # ILR adjusted normalized profiles + temp = re_profiles * self.re_plant_specs.ilr.to_numpy() + ac_out = np.minimum(temp, 1) * self.re_plant_specs.capacity_mw.to_numpy() + excess = temp * self.re_plant_specs.capacity_mw.to_numpy() - ac_out + return self.load_profile - ac_out.sum(axis=1), ac_out, excess + + def dc_charge(self): + """Align excess_re to match the storage facilities it could charge.""" + dc_charge = pd.DataFrame( + np.nan, index=self.load_profile.index, columns=self.storage_specs.index + ) + if self.re_excess is None: + return dc_charge.fillna(0.0) + dc_charge = dc_charge.droplevel("generator_id", axis=1) + return ( + dc_charge.combine_first(self.re_excess.droplevel("generator_id", axis=1))[ + dc_charge.columns + ] + .set_axis(self.storage_specs.index, axis=1) + .fillna(0.0) + ) + def add_total_costs(self, df: pd.DataFrame) -> pd.DataFrame: """Add columns for total FOM and total startup from respective unit costs.""" df = ( @@ -236,19 +266,10 @@ def _type_check(meta): ) del meta["__qualname__"] - data_dict = {} - with ZipFile(path.with_suffix(".zip"), "r") as z: - metadata = json.loads(z.read("metadata.json")) + with DataZip(path, "r") as z: + metadata = z.read("metadata") _type_check(metadata) - plant_index = pd.MultiIndex.from_tuples( - metadata.pop("plant_index"), names=["plant_id_eia", "generator_id"] - ) - for df_name in cls._parquet_out: - if (x := df_name + ".parquet") in z.namelist(): - df_in = pd.read_parquet(BytesIO(z.read(x))).squeeze() - if df_name in ("dispatchable_profiles", "redispatch"): - df_in.columns = plant_index - data_dict[df_name] = df_in + data_dict = dict(z.read_dfs()) sig = inspect.signature(cls).parameters self = cls( @@ -272,9 +293,15 @@ def from_patio( ) -> DispatchModel: """Create :class:`.DispatchModel` with data from patio.BAScenario.""" if "operating_date" not in storage_specs: - storage_specs = storage_specs.assign(operating_date=net_load.index.min()) + storage_specs = storage_specs.assign( + operating_date=net_load.index.min(), + plant_id_eia=lambda x: x.index.to_series() * -1, + generator_id=lambda x: ( + x.groupby(["plant_id_eia"]).transform("cumcount") + 1 + ).astype(str), + ).set_index(["plant_id_eia", "generator_id"]) return cls( - net_load_profile=net_load, + load_profile=net_load, dispatchable_specs=plant_data, dispatchable_cost=cost_data, dispatchable_profiles=dispatchable_profiles, @@ -321,17 +348,16 @@ def from_fresh( # make a boolean array for whether a particular hour comes between # a generator's `operating_date` and `retirement_date` or not - dispatchable_profiles = apply_op_ret_date( - pd.DataFrame( - 1, index=net_load_profile.index, columns=dispatchable_specs.index + dispatchable_profiles = pd.DataFrame( + np.expand_dims(dispatchable_specs.capacity_mw.to_numpy(), axis=0).repeat( + len(net_load_profile.index), axis=0 ), - dispatchable_specs.operating_date, - dispatchable_specs.retirement_date, - dispatchable_specs.capacity_mw, + index=net_load_profile.index, + columns=dispatchable_specs.index, ) return cls( - net_load_profile=net_load_profile, + load_profile=net_load_profile, dispatchable_specs=dispatchable_specs, dispatchable_cost=dispatchable_cost, dispatchable_profiles=dispatchable_profiles, @@ -342,7 +368,7 @@ def from_fresh( @property def dispatch_func(self) -> Callable: """Appropriate dispatch engine depending on ``jit`` setting.""" - return dispatch_engine_compiled if self._metadata["jit"] else dispatch_engine + return dispatch_engine if self._metadata["jit"] else dispatch_engine_py @property def is_redispatch(self) -> bool: @@ -377,7 +403,7 @@ def redispatch_cost(self) -> dict[str, pd.DataFrame]: return self._cost(self.redispatch) # TODO probably a bad idea to use __call__, but nice to not have to think of a name - def __call__(self) -> None: + def __call__(self) -> DispatchModel: """Run dispatch model.""" fos_prof, storage, deficits, starts = self.dispatch_func( net_load=self.net_load_profile.to_numpy(dtype=np.float_), # type: ignore @@ -407,27 +433,22 @@ def __call__(self) -> None: >= self.storage_specs.operating_date.to_numpy(), axis=0, ), + storage_dc_charge=self.dc_charge().to_numpy(dtype=np.float_), ) self.redispatch = pd.DataFrame( - fos_prof.astype(np.float32), + fos_prof, index=self.dt_idx, columns=self.dispatchable_profiles.columns, ) self.storage_dispatch = pd.DataFrame( - np.hstack([storage[:, :, x] for x in range(storage.shape[2])]).astype( - np.float32 - ), + np.hstack([storage[:, :, x] for x in range(storage.shape[2])]), index=self.dt_idx, - columns=[ - col - for i in range(storage.shape[2]) - for col in (f"charge_{i}", f"discharge_{i}", f"soc_{i}") - ], + columns=self.storage_dispatch.columns, ) self.system_data = pd.DataFrame( - deficits.astype(np.float32), + deficits, index=self.dt_idx, - columns=["deficit", "dirty_charge", "curtailment"], + columns=self.system_data.columns, ) self.starts = ( pd.DataFrame( @@ -439,22 +460,23 @@ def __call__(self) -> None: .reorder_levels([1, 2, 0]) .sort_index() ) + return self def _cost(self, profiles: pd.DataFrame) -> dict[str, pd.DataFrame]: """Determine total cost based on hourly production and starts.""" profs = profiles.to_numpy() fuel_cost = profs * self.dispatchable_cost.fuel_per_mwh.unstack( level=("plant_id_eia", "generator_id") - ).reindex(index=self.net_load_profile.index, method="ffill") + ).reindex(index=self.load_profile.index, method="ffill") vom_cost = profs * self.dispatchable_cost.vom_per_mwh.unstack( level=("plant_id_eia", "generator_id") - ).reindex(index=self.net_load_profile.index, method="ffill") + ).reindex(index=self.load_profile.index, method="ffill") start_cost = np.where( (profs == 0) & (np.roll(profs, -1, axis=0) > 0), 1, 0 ) * self.dispatchable_cost.startup_cost.unstack( level=("plant_id_eia", "generator_id") ).reindex( - index=self.net_load_profile.index, method="ffill" + index=self.load_profile.index, method="ffill" ) fom = ( apply_op_ret_date( @@ -468,9 +490,9 @@ def _cost(self, profiles: pd.DataFrame) -> dict[str, pd.DataFrame]: lambda x: x.replace(day=1, month=1) ), ) - .reindex(index=self.net_load_profile.index, method="ffill") + .reindex(index=self.load_profile.index, method="ffill") .divide( - self.net_load_profile.groupby(pd.Grouper(freq="YS")).transform("count"), + self.load_profile.groupby(pd.Grouper(freq="YS")).transform("count"), axis=0, ) ) @@ -567,7 +589,7 @@ def lost_load( ) -> pd.Series[int]: """Number of hours during which deficit was in various duration bins.""" if comparison is None: - durs = self.system_data.deficit / self.net_load_profile + durs = self.system_data.deficit / self.load_profile else: durs = self.system_data.deficit / comparison bins = map( @@ -590,9 +612,7 @@ def hrs_to_check( positive deficit hours. """ if comparison is None: - comparison = self.net_load_profile.groupby( - [pd.Grouper(freq="YS")] - ).transform( + comparison = self.load_profile.groupby([pd.Grouper(freq="YS")]).transform( "max" ) # type: ignore td_1h = np.timedelta64(1, "h") @@ -603,10 +623,44 @@ def hrs_to_check( self.system_data.deficit / comparison > cutoff ].index for hr in (dhr - 2 * td_1h, dhr - td_1h, dhr) - if hr in self.net_load_profile.index + if hr in self.load_profile.index } ) + def hourly_data_check(self, cutoff: float = 0.01): + """Aggregate data for :meth:`.DispatchModel.hrs_to_checl`.""" + max_disp = apply_op_ret_date( + pd.DataFrame( + 1.0, + index=self.load_profile.index, + columns=self.dispatchable_profiles.columns, + ), + self.dispatchable_specs.operating_date, + self.dispatchable_specs.retirement_date, + self.dispatchable_specs.capacity_mw, + ) + out = pd.concat( + { + "gross_load": self.load_profile, + "net_load": self.net_load_profile, + "deficit": self.system_data.deficit, + "max_dispatch": max_disp.sum(axis=1), + "redispatch": self.redispatch.sum(axis=1), + "historical_dispatch": self.dispatchable_profiles.sum(axis=1), + "net_storage": ( + self.storage_dispatch.filter(regex="^discharge").sum(axis=1) + - self.storage_dispatch.filter(regex="^gridcharge").sum(axis=1) + ), + "state_of_charge": self.storage_dispatch.filter(regex="^soc").sum( + axis=1 + ), + "re": self.re_profiles_ac.sum(axis=1), + "re_excess": self.re_excess.sum(axis=1), + }, + axis=1, + ).loc[self.hrs_to_check(cutoff=cutoff), :] + return out + def storage_capacity(self) -> pd.DataFrame: """Number of hours when storage charge or discharge was in various bins.""" rates = self.storage_dispatch.filter(like="charge") @@ -648,17 +702,15 @@ def system_level_summary(self, freq: str = "YS", **kwargs) -> pd.DataFrame: self.system_data.groupby(pd.Grouper(freq=freq)) .sum() .rename(columns={c: f"{c}_mwh" for c in self.system_data}), - # max deficit pct of net load + # max deficit pct of load self.system_data[["deficit"]] .groupby(pd.Grouper(freq=freq)) .max() .rename(columns={"deficit": "deficit_max_pct_net_load"}) - / self.net_load_profile.max(), + / self.load_profile.max(), # count of deficit greater than 2% pd.Series( - self.system_data[ - self.system_data / self.net_load_profile.max() > 0.02 - ] + self.system_data[self.system_data / self.load_profile.max() > 0.02] .groupby(pd.Grouper(freq=freq)) .deficit.count(), name="deficit_gt_2pct_count", @@ -669,12 +721,16 @@ def system_level_summary(self, freq: str = "YS", **kwargs) -> pd.DataFrame: f"storage_{i}_max_mw": self.storage_dispatch.filter( like=f"e_{i}" ).max(axis=1) - for i in self.storage_specs.index + for i in self.storage_specs.index.get_level_values( + "plant_id_eia" + ) }, **{ f"storage_{i}_max_hrs": self.storage_dispatch[f"soc_{i}"] / self.storage_specs.loc[i, "capacity_mw"] - for i in self.storage_specs.index + for i in self.storage_specs.index.get_level_values( + "plant_id_eia" + ) }, ) .groupby(pd.Grouper(freq=freq)) @@ -687,12 +743,12 @@ def system_level_summary(self, freq: str = "YS", **kwargs) -> pd.DataFrame: **{ f"storage_{i}_mw_utilization": out[f"storage_{i}_max_mw"] / self.storage_specs.loc[i, "capacity_mw"] - for i in self.storage_specs.index + for i in self.storage_specs.index.get_level_values("plant_id_eia") }, **{ f"storage_{i}_hrs_utilization": out[f"storage_{i}_max_hrs"] / self.storage_specs.loc[i, "duration_hrs"] - for i in self.storage_specs.index + for i in self.storage_specs.index.get_level_values("plant_id_eia") }, ) @@ -703,12 +759,12 @@ def re_summary( **kwargs, ) -> pd.DataFrame: """Create granular summary of renewable plant metrics.""" - if self.re_profiles is None or self.re_plant_specs is None: + if self.re_profiles_ac is None or self.re_plant_specs is None: raise AssertionError( "at least one of `re_profiles` and `re_plant_specs` is `None`" ) out = ( - self.re_profiles.groupby([pd.Grouper(freq=freq)]) + self.re_profiles_ac.groupby([pd.Grouper(freq=freq)]) .sum() .stack(["plant_id_eia", "generator_id"]) .to_frame(name="redispatch_mwh") @@ -737,26 +793,35 @@ def storage_summary( ) -> pd.DataFrame: """Create granular summary of storage plant metrics.""" out = ( - self.storage_dispatch.groupby([pd.Grouper(freq=freq)]) + self.storage_dispatch.filter(regex="^dis|^grid") + .groupby([pd.Grouper(freq=freq)]) .sum() .stack() .reset_index() .rename(columns={0: "redispatch_mwh"}) ) - out[["kind", "index"]] = out.level_1.str.split("_", expand=True) + out[["kind", "plant_id_eia"]] = out.level_1.str.split("_", expand=True) out = ( - out.query("kind != 'soc'") + out.astype({"plant_id_eia": int}) .assign( redispatch_mwh=lambda x: x.redispatch_mwh.mask( - x.kind == "charge", x.redispatch_mwh * -1 - ) + x.kind == "gridcharge", x.redispatch_mwh * -1 + ), + generator_id=lambda x: x.plant_id_eia.replace( + self.storage_specs.reset_index( + "generator_id" + ).generator_id.to_dict() + ), ) - .groupby(["index", "datetime"]) + .groupby(["plant_id_eia", "generator_id", "datetime"]) .redispatch_mwh.sum() .reset_index() - .astype({"index": int}) - .merge(self.storage_specs.reset_index(), on="index", validate="m:1") + .merge( + self.storage_specs.reset_index(), + on=["plant_id_eia", "generator_id"], + validate="m:1", + ) .assign( capacity_mw=lambda x: x.capacity_mw.where( x.operating_date <= x.datetime, 0 @@ -764,14 +829,7 @@ def storage_summary( ) ) if by is None: - if ( - None in self.storage_specs.index.names - and len(self.storage_specs.index.names) == 1 - ): - col = ["index"] - else: - col = self.storage_specs.index.names - return out.set_index(col + ["datetime"]) + return out.set_index(["plant_id_eia", "generator_id", "datetime"]) return out.groupby([by, "datetime"]).sum() def full_output(self, freq: str = "YS") -> pd.DataFrame: @@ -805,14 +863,64 @@ def full_output(self, freq: str = "YS") -> pd.DataFrame: + [col for col in cols if col in self.dispatchable_specs] ] ) + + # setup deficit/curtailment as if they were resources for full output, the idea + # here is that you could rename them purchase/sales. + def_cur = self.grouper(self.system_data, by=None)[["deficit", "curtailment"]] + def_cur.columns = pd.MultiIndex.from_tuples( + [(0, "deficit"), (0, "curtailment")], names=["plant_id_eia", "generator_id"] + ) + def_cur = ( + def_cur.stack([0, 1]) + .reorder_levels([1, 2, 0]) + .sort_index() + .to_frame(name="redispatch_mwh") + .assign( + technology_description=lambda x: x.index.get_level_values( + "generator_id" + ) + ) + ) + return pd.concat( [ dispatchable, self.re_summary(by=None, freq=freq), - self.storage_summary(by=None, freq=freq), + self.storage_summary(by=None, freq=freq) + .reset_index() + .set_index(["plant_id_eia", "generator_id", "datetime"]), + def_cur, ] ).sort_index() + def load_summary(self, **kwargs): + """Create summary of load data.""" + return pd.concat( + [ + self.strict_grouper( + self.net_load_profile.to_frame("net_load"), by=None, freq="YS" + ), + self.strict_grouper( + self.net_load_profile.to_frame("net_load_peak"), + by=None, + freq="YS", + freq_agg="max", + ), + self.strict_grouper( + self.load_profile.to_frame("gross_load"), + by=None, + freq="YS", + ), + self.strict_grouper( + self.load_profile.to_frame("gross_load_peak"), + by=None, + freq="YS", + freq_agg="max", + ), + ], + axis=1, + ) + def dispatchable_summary( self, by: str | None = "technology_description", @@ -833,7 +941,7 @@ def dispatchable_summary( apply_op_ret_date( pd.DataFrame( 1, - index=self.net_load_profile.index, + index=self.load_profile.index, columns=self.dispatchable_specs.index, ), self.dispatchable_specs.operating_date, @@ -900,54 +1008,39 @@ def to_file( clobber=False, **kwargs, ) -> None: - """Save :class:`.DispatchModel` to disk. - - A very ugly process at the moment because of our goal not to use pickle - and to try to keep the file small-ish. Also need to manage the fact that - the parquet requirement for string column names causes some problems. - """ - if not isinstance(path, Path): - path = Path(path) - path = path.with_suffix(".zip") - if path.exists() and not clobber: + """Save :class:`.DispatchModel` to disk.""" + if Path(path).with_suffix(".zip").exists() and not clobber: raise FileExistsError(f"{path} exists, to overwrite set `clobber=True`") + if clobber: + Path(path).with_suffix(".zip").unlink(missing_ok=True) - metadata = { - **self._metadata, - "__qualname__": self.__class__.__qualname__, - "plant_index": list(self.dispatchable_specs.index), - } - - with ZipFile(path, "w", compression=compression) as z: - for df_name, func in self._parquet_out.items(): - try: - df_out = getattr(self, df_name) - if df_out is not None and not df_out.empty: - z.writestr( - f"{df_name}.parquet", func(df_out, df_name).to_parquet() - ) - except Exception as exc: - raise RuntimeError(f"{df_name} {exc!r}") from exc + with DataZip(path, "w", compression=compression) as z: + for df_name in self._parquet_out: + z.writed(df_name, getattr(self, df_name)) if include_output and not self.redispatch.empty: - for df_name in ("system_level_summary", "dispatchable_summary"): - z.writestr( - f"{df_name}.parquet", - getattr(self, df_name)(**kwargs).to_parquet(), + for df_name in ("full_output", "load_summary"): + z.writed( + df_name, + getattr(self, df_name)(**kwargs), ) - z.writestr( - "metadata.json", json.dumps(metadata, ensure_ascii=False, indent=4) + z.writed( + "metadata", + { + **self._metadata, + "__qualname__": self.__class__.__qualname__, + }, ) def _plot_prep(self): if "plot_prep" not in self._cached: storage = self.storage_dispatch.assign( - charge=lambda x: -1 * x.filter(regex="^charge").sum(axis=1), + charge=lambda x: -1 * x.filter(regex="^gridcharge").sum(axis=1), discharge=lambda x: x.filter(regex="^discharge").sum(axis=1), ) try: re = self.re_summary(freq="H").redispatch_mwh.unstack(level=0) except AssertionError: - re = MTDF.reindex(index=self.net_load_profile.index) + re = MTDF.reindex(index=self.load_profile.index) def _grp(df): return df.rename(columns=PLOT_MAP).groupby(level=0, axis=1).sum() @@ -957,7 +1050,7 @@ def _grp(df): [ self.grouper(self.redispatch, freq="H").pipe(_grp), re.pipe(_grp), - storage[["charge", "discharge"]].pipe(_grp), + storage[["charge", "discharge"]], ], axis=1, ) @@ -971,58 +1064,173 @@ def _grp(df): ) return self._cached["plot_prep"] - def plot_period(self, begin, end): - """Plot hourly dispatch.""" + def _plot_prep_detail(self, begin, end): + to_cat = [ + self.redispatch.set_axis( + pd.MultiIndex.from_frame( + self.dispatchable_specs.technology_description.reset_index() + ), + axis=1, + ), + self.re_profiles_ac.set_axis( + pd.MultiIndex.from_frame( + self.re_plant_specs.technology_description.reset_index() + ), + axis=1, + ), + self.storage_dispatch.filter(like="discharge").set_axis( + pd.MultiIndex.from_frame( + self.storage_specs.assign( + technology_description="discharge" + ).technology_description.reset_index() + ), + axis=1, + ), + -1 + * self.storage_dispatch.filter(like="gridcharge").set_axis( + pd.MultiIndex.from_frame( + self.storage_specs.assign( + technology_description="charge" + ).technology_description.reset_index() + ), + axis=1, + ), + -1 + * self.system_data.curtailment.to_frame( + name=(999, "curtailment", "Curtailment") + ), + self.system_data.deficit.to_frame(name=(999, "deficit", "Deficit")), + ] + return ( + pd.concat( + to_cat, + axis=1, + ) + .loc[begin:end, :] + .stack([0, 1, 2]) + .to_frame(name="net_generation_mwh") + .reset_index() + .assign(resource=lambda x: x.technology_description.replace(PLOT_MAP)) + .query("net_generation_mwh != 0.0") + ) + + def plot_period(self, begin, end=None, by_gen=True) -> Figure: + """Plot hourly dispatch by generator.""" + begin = pd.Timestamp(begin) + if end is None: + end = begin + pd.Timedelta(days=7) + else: + end = pd.Timestamp(end) net_load = self.net_load_profile.loc[begin:end] - out = px.bar( - self._plot_prep().loc[net_load.index, :].reset_index(), - x="datetime", - y="net_generation_mwh", - color="resource", - color_discrete_map=COLOR_MAP, - ).add_scatter( - x=net_load.index, - y=net_load, - name="Net Load", - mode="lines", - line_color=COLOR_MAP["Net Load"], - line_dash="dot", + data = self._plot_prep_detail(begin, end) + hover_name = "plant_id_eia" + if not by_gen: + data = data.groupby(["datetime", "resource"]).sum().reset_index() + hover_name = "resource" + out = ( + px.bar( + data.replace( + {"resource": {"charge": "Storage", "discharge": "Storage"}} + ).sort_values(["resource"], key=dispatch_key), + x="datetime", + y="net_generation_mwh", + color="resource", + hover_name=hover_name, + color_discrete_map=COLOR_MAP, + ) + .add_scatter( + x=net_load.index, + y=net_load, + name="Net Load", + mode="lines", + line_color=COLOR_MAP["Net Load"], + line_dash="dot", + ) + .update_layout(xaxis_title=None, yaxis_title="MW", yaxis_tickformat=",.0r") ) - if self.re_profiles is None or self.re_plant_specs is None: + if self.re_profiles_ac is None or self.re_plant_specs is None: return out - gross = self.re_profiles.loc[net_load.index, :].sum(axis=1) + net_load return out.add_scatter( - x=gross.index, - y=gross, - name="Grossed Load", + x=self.load_profile.loc[begin:end].index, + y=self.load_profile.loc[begin:end], + name="Gross Load", mode="lines", - line_color=COLOR_MAP["Grossed Load"], + line_color=COLOR_MAP["Gross Load"], ) - def plot_year(self, year): + def plot_year(self, year: int, freq="D") -> Figure: """Monthly facet plot of daily dispatch for a year.""" + assert freq in ("H", "D"), "`freq` must be 'D' for day or 'H' for hour" out = ( self._plot_prep() + .loc[str(year), :] .reset_index() - .groupby([pd.Grouper(freq="D", key="datetime"), "resource"]) + .groupby([pd.Grouper(freq=freq, key="datetime"), "resource"]) .sum() + .reset_index() .assign( - year=lambda x: x.index.get_level_values("datetime").year, - month=lambda x: x.index.get_level_values("datetime").month, + day=lambda z: z.datetime.dt.day, + hour=lambda z: z.datetime.dt.day * 24 + z.datetime.dt.hour, + month=lambda z: z.datetime.dt.strftime("%B"), + resource=lambda z: z.resource.replace( + {"charge": "Storage", "discharge": "Storage"} + ), ) + .sort_values(["resource", "month"], key=dispatch_key) ) - return px.bar( - out.loc[str(year), :].reset_index(), - x="datetime", - y="net_generation_mwh", - color="resource", - facet_col="month", - facet_col_wrap=4, + x, yt, ht = {"D": ("day", "MWh", "resource"), "H": ("hour", "MW", "datetime")}[ + freq + ] + return ( + px.bar( + out, + x=x, + y="net_generation_mwh", + color="resource", + facet_col="month", + facet_col_wrap=4, + height=800, + width=1000, + hover_name=ht, + color_discrete_map=COLOR_MAP, + ) + .for_each_annotation(lambda a: a.update(text=a.text.split("=")[-1])) + .update_xaxes(title=None) + .for_each_yaxis( + lambda yaxis: (yaxis.update(title=yt) if yaxis.title.text else None) + # lambda yaxis: print(yaxis) + ) + .update_layout(bargap=0) ) + def plot_output(self, y: str, color="resource") -> Figure: + """Plot a columns from :meth:`.DispatchModel.full_output`.""" + return px.bar( + self.full_output() + .reset_index() + .assign( + year=lambda x: x.datetime.dt.year, + resource=lambda x: x.technology_description.replace(PLOT_MAP), + redispatch_cost=lambda x: x.filter(like="redispatch_cost").sum(axis=1), + historical_cost=lambda x: x.filter(like="historical_cost").sum(axis=1), + ) + .sort_values(["resource"], key=dispatch_key), + x="year", + y=y, + color=color, + hover_name="plant_id_eia", + color_discrete_map=COLOR_MAP, + width=1000, + ).update_layout(xaxis_title=None) + def __repr__(self) -> str: + if self.re_plant_specs is None: + re_len = 0 + else: + re_len = len(self.re_plant_specs) return ( self.__class__.__qualname__ + f"({', '.join(f'{k}={v}' for k, v in self._metadata.items())}, " - f"n_plants={len(self.dispatchable_specs)})".replace("self.", "") + f"n_dispatchable={len(self.dispatchable_specs)}, n_re={re_len}, " + f"n_storage={len(self.storage_specs)})" ) diff --git a/tests/conftest.py b/tests/conftest.py index 7c5ea3b6..b797f375 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -5,6 +5,8 @@ import pandas as pd import pytest +from dispatch import DataZip + logger = logging.getLogger(__name__) @@ -63,3 +65,23 @@ def fossil_cost(test_dir) -> pd.DataFrame: # df.columns.name = "datetime" # return df.stack() return pd.read_parquet(test_dir / "data/fossil_cost.parquet") + + +@pytest.fixture +def ent_fresh(test_dir) -> dict: + """Fossil Profiles.""" + # df = pd.read_parquet(test_dir / "data/plant_specs.parquet").filter(like="20") + # df.columns = df.columns.map(lambda x: pd.Timestamp(x)) + # df.columns.name = "datetime" + # return df.stack() + return DataZip.dfs_from_zip(test_dir / "data/8fresh.zip") + + +@pytest.fixture +def ent_redispatch(test_dir) -> dict: + """Fossil Profiles.""" + # df = pd.read_parquet(test_dir / "data/plant_specs.parquet").filter(like="20") + # df.columns = df.columns.map(lambda x: pd.Timestamp(x)) + # df.columns.name = "datetime" + # return df.stack() + return DataZip.dfs_from_zip(test_dir / "data/8redispatch.zip") diff --git a/tests/data/8fresh.zip b/tests/data/8fresh.zip index 2fde60fa..e2c00b34 100644 Binary files a/tests/data/8fresh.zip and b/tests/data/8fresh.zip differ diff --git a/tests/data/8redispatch.zip b/tests/data/8redispatch.zip index 47788e90..571638ab 100644 Binary files a/tests/data/8redispatch.zip and b/tests/data/8redispatch.zip differ diff --git a/tests/engine_test.py b/tests/engine_test.py index 4de36039..28ce9af1 100644 --- a/tests/engine_test.py +++ b/tests/engine_test.py @@ -2,8 +2,16 @@ import numpy as np +import pytest -from dispatch.engine import dispatch_engine +# noinspection PyProtectedMember +from dispatch.engine import ( + charge_storage_py, + dispatch_engine_py, + make_rank_arrays_py, + validate_inputs_py, +) +from dispatch.helpers import idfn NL = [ -500.0, @@ -30,7 +38,7 @@ def test_engine(): """Trivial test for the dispatch engine.""" - re, es, sl, st = dispatch_engine( + redispatch, es, sl, st = dispatch_engine_py( net_load=np.array(NL), hr_to_cost_idx=np.zeros(len(NL), dtype=int), historical_dispatch=np.array([CAP] * len(NL)), @@ -41,5 +49,78 @@ def test_engine(): storage_hrs=np.array([4, 12]), storage_eff=np.array((0.9, 0.9)), storage_op_hour=np.array((0, 0)), + storage_dc_charge=np.zeros((len(NL), 2)), ) - assert True + assert np.all(redispatch.sum(axis=1) + es[:, 1, :].sum(axis=1) >= NL) + + +@pytest.mark.parametrize( + "override, expected", + [ + ({"storage_mw": np.array([400, 200, 100])}, AssertionError), + ({"storage_dc_charge": np.zeros((len(NL), 5))}, AssertionError), + ({"startup_cost": np.array([[1000.0], [1000.0]])}, AssertionError), + ({"net_load": np.array(NL)[1:]}, AssertionError), + ( + {"hr_to_cost_idx": np.append(np.zeros(len(NL) - 1, dtype=int), 1)}, + AssertionError, + ), + ], + ids=idfn, +) +def test_validate_inputs(override, expected): + """Test input validator.""" + base = dict( + net_load=np.array(NL), + hr_to_cost_idx=np.zeros(len(NL), dtype=int), + historical_dispatch=np.array([CAP] * len(NL)), + ramp_mw=np.array(CAP), + startup_cost=np.array([[1000.0], [1000.0], [100.0]]), + marginal_cost=np.array([[10.0], [20.0], [50.0]]), + storage_mw=np.array([400, 200]), + storage_hrs=np.array([4, 12]), + storage_eff=np.array((0.9, 0.9)), + storage_dc_charge=np.zeros((len(NL), 2)), + ) + validate_inputs_py(**base) + over = base | override + with pytest.raises(expected): + validate_inputs_py(**over) + + +@pytest.mark.parametrize( + "deficit, soc, dc_charge, mw, soc_max, eff, expected", + [ + (0, 0, 0, 10, 20, 1, (0, 0, 0, 0)), + (-5, 0, 0, 10, 20, 1, (5, 0, 5, 5)), + (0, 0, 5, 10, 20, 1, (5, 0, 5, 0)), + (500, 0, 5, 10, 20, 1, (5, 0, 5, 0)), + (-5, 0, 5, 5, 20, 1, (5, 0, 5, 0)), + (-5, 0, 3, 5, 20, 1, (5, 0, 5, 2)), + (-5, 0, 5, 10, 20, 1, (10, 0, 10, 5)), + (-5, 5, 5, 10, 20, 1, (10, 0, 15, 5)), + (-5, 5, 5, 10, 20, 0.5, (10, 0, 10, 5)), + (-5, 15, 5, 10, 20, 1, (5, 0, 20, 0)), + (-2, 15, 5, 10, 20, 1, (5, 0, 20, 0)), + (-5, 0, 0, 1, 20, 1, (1, 0, 1, 1)), + (0, 0, 5, 1, 20, 1, (1, 0, 1, 0)), + (500, 0, 5, 1, 20, 1, (1, 0, 1, 0)), + (-5, 0, 5, 1, 20, 1, (1, 0, 1, 0)), + (-5, 5, 5, 1, 20, 1, (1, 0, 6, 0)), + (-5, 5, 5, 1, 20, 0.9, (1, 0, 5.9, 0)), + (-2, 15, 5, 1, 20, 1, (1, 0, 16, 0)), + ], + ids=idfn, +) +def test_charge_storage(deficit, soc, dc_charge, mw, soc_max, eff, expected): + """Test storage charging calculations.""" + assert charge_storage_py(deficit, soc, dc_charge, mw, soc_max, eff) == expected + + +def test_make_rank_arrays(): + """Test cost rank setup.""" + m_cost = np.array([[50.0, 50.0], [25.0, 75.0]]) + s_cost = np.array([[250.0, 250.0], [500.0, 500.0]]) + m, s = make_rank_arrays_py(m_cost, s_cost) + assert np.all(m == np.array([[1, 0], [0, 1]])) + assert np.all(s == np.array([[0, 0], [1, 1]])) diff --git a/tests/helper_test.py b/tests/helper_test.py new file mode 100644 index 00000000..d84c42e0 --- /dev/null +++ b/tests/helper_test.py @@ -0,0 +1,125 @@ +"""Tests for :mod:`dispatch.helpers`.""" + +import pandas as pd +import pytest + +from dispatch.helpers import DataZip, copy_profile, dispatch_key + + +def test_copy_profile(ent_fresh): + """Test copy_profile.""" + load = ent_fresh["load_profile"] + yr_range = range(2050, 2055) + out = copy_profile(load.loc["2025"], yr_range) + assert tuple(out.index.year.unique()) == tuple(yr_range) + + +def test_dispatch_key_series(): + """Test dispatch key func on :class:`pandas.Series`.""" + idx = ( + pd.Series(["May", "April", "Solar", "Gas CC", "March"]) + .sort_values(key=dispatch_key) + .reset_index(drop=True) + ) + pd.testing.assert_series_equal( + idx, pd.Series(["Gas CC", "Solar", "March", "April", "May"]) + ) + + +def test_dispatch_key_idx(): + """Test dispatch key func on :class:`pandas.Index`.""" + idx = pd.Index(["May", "April", "Solar", "Gas CC", "March"]).sort_values( + key=dispatch_key + ) + pd.testing.assert_index_equal( + idx, pd.Index(["Gas CC", "Solar", "March", "April", "May"]) + ) + + +def test_dispatch_key_list(): + """Test dispatch key func on :class:`list`.""" + idx = sorted(["May", "April", "Solar", "Gas CC", "March"], key=dispatch_key) + assert idx == ["Gas CC", "Solar", "March", "April", "May"] + + +def test_dfs_to_from_zip(test_dir): + """Dfs are same after being written and read back.""" + df_dict = { + "a": pd.DataFrame( + [[0, 1], [2, 3]], + columns=pd.MultiIndex.from_tuples([(0, "a"), (1, "b")]), + ), + "b": pd.Series([1, 2, 3, 4]), + } + try: + DataZip.dfs_to_zip( + test_dir / "df_test", + df_dict, + ) + df_load = DataZip.dfs_from_zip(test_dir / "df_test") + for a, b in zip(df_dict.values(), df_load.values()): + assert a.compare(b).empty + except Exception as exc: + raise AssertionError("Something broke") from exc + finally: + (test_dir / "df_test.zip").unlink(missing_ok=True) + + +def test_datazip(test_dir): + """Test :class:`.DataZip`.""" + df_dict = { + "a": pd.DataFrame( + [[0, 1], [2, 3]], + columns=pd.MultiIndex.from_tuples([(0, "a"), (1, "b")]), + ), + "b": pd.Series([1, 2, 3, 4]), + } + try: + with DataZip(test_dir / "obj.zip", "w") as z: + z.writed("a", df_dict["a"]) + z.writed("b", df_dict["b"]) + z.writed("c", {1: 3, "3": "fifteen", 5: (0, 1)}) + with pytest.raises(TypeError): + z.writed("d", "hello world") + with pytest.raises(FileExistsError): + z.writed("c", {1: 3, "3": "fifteen", 5: (0, 1)}) + with pytest.raises(FileExistsError): + z.writed("b", df_dict["b"]) + except Exception as exc: + raise AssertionError("Something broke") from exc + else: + with DataZip(test_dir / "obj.zip", "r") as z1: + for n in ("a.parquet", "b.parquet", "c.json"): + assert n in z1.namelist() + assert "a" in z1.bad_cols + finally: + (test_dir / "obj.zip").unlink(missing_ok=True) + + +def test_datazip_w(test_dir): + """Test writing to existing :class:`.DataZip`.""" + df_dict = { + "a": pd.DataFrame( + [[0, 1], [2, 3]], + columns=pd.MultiIndex.from_tuples([(0, "a"), (1, "b")]), + ), + } + try: + with DataZip(test_dir / "obj.zip", "w") as z0: + z0.writed("a", df_dict["a"]) + except Exception as exc: + raise AssertionError("Something broke") from exc + else: + with DataZip(test_dir / "obj.zip", "r") as z1: + assert "a" in z1.bad_cols + with pytest.raises(ValueError): + with DataZip(test_dir / "obj.zip", "a") as z2a: + z2a.namelist() + with pytest.raises(ValueError): + with DataZip(test_dir / "obj.zip", "x") as z2x: + z2x.namelist() + with pytest.raises(FileExistsError): + with DataZip(test_dir / "obj.zip", "w") as z3: + z3.namelist() + finally: + (test_dir / "obj.zip").unlink(missing_ok=True) diff --git a/tests/dispatch_test.py b/tests/model_test.py similarity index 62% rename from tests/dispatch_test.py rename to tests/model_test.py index 15f7bcc5..783e501b 100644 --- a/tests/dispatch_test.py +++ b/tests/model_test.py @@ -3,8 +3,8 @@ import pandas as pd import pytest -from dispatch import DispatchModel, apply_op_ret_date, dfs_from_zip -from dispatch.helpers import dfs_to_zip, idfn +from dispatch import DispatchModel +from dispatch.helpers import apply_op_ret_date, idfn def setup_dm(fossil_profiles, fossil_specs, fossil_cost, re_profiles, re, storage): @@ -51,6 +51,9 @@ def test_new(fossil_profiles, re_profiles, fossil_specs, fossil_cost): storage_specs=pd.DataFrame( [(5000, 4, 0.9), (2000, 8760, 0.5)], columns=["capacity_mw", "duration_hrs", "roundtrip_eff"], + index=pd.MultiIndex.from_tuples( + [(-99, "es"), (-98, "es")], names=["plant_id_eia", "generator_id"] + ), ), jit=True, ) @@ -58,6 +61,33 @@ def test_new(fossil_profiles, re_profiles, fossil_specs, fossil_cost): assert self +def test_new_no_dates(fossil_profiles, re_profiles, fossil_specs, fossil_cost): + """Dummy test to quiet pytest.""" + fossil_specs.iloc[ + 0, fossil_specs.columns.get_loc("retirement_date") + ] = fossil_profiles.index.max() - pd.Timedelta(weeks=15) + self = DispatchModel.from_fresh( + net_load_profile=fossil_profiles.sum(axis=1), + dispatchable_specs=fossil_specs.drop( + columns=["retirement_date", "operating_date"] + ), + dispatchable_cost=fossil_cost, + storage_specs=pd.DataFrame( + [(5000, 4, 0.9), (2000, 8760, 0.5)], + columns=["capacity_mw", "duration_hrs", "roundtrip_eff"], + index=pd.MultiIndex.from_tuples( + [(-99, "es"), (-98, "es")], names=["plant_id_eia", "generator_id"] + ), + ), + jit=True, + ) + dates = self.dispatchable_specs[ + ["operating_date", "retirement_date"] + ].drop_duplicates() + assert fossil_profiles.index.min() == dates.operating_date.item() + assert fossil_profiles.index.max() == dates.retirement_date.item() + + def test_new_with_dates(fossil_profiles, re_profiles, fossil_specs, fossil_cost): """Test operating and retirement dates for fossil and storage.""" fossil_specs.iloc[ @@ -76,6 +106,9 @@ def test_new_with_dates(fossil_profiles, re_profiles, fossil_specs, fossil_cost) (2000, 8760, 0.5, pd.Timestamp(year=2019, month=1, day=1)), ], columns=["capacity_mw", "duration_hrs", "roundtrip_eff", "operating_date"], + index=pd.MultiIndex.from_tuples( + [(-99, "es"), (-98, "es")], names=["plant_id_eia", "generator_id"] + ), ), jit=True, ) @@ -118,13 +151,30 @@ def test_write_and_read( ), ) file = test_dir / "test_obj.zip" + try: + dm.to_file(file) + x = DispatchModel.from_file(file) + x() + x.to_file(file, clobber=True, include_output=False) + except Exception as exc: + raise AssertionError(f"{exc!r}") from exc + else: + assert True + finally: + file.unlink(missing_ok=True) + + +def test_write_and_read_full(test_dir, ent_fresh): + """Test that DispatchModel can be written and read.""" + dm = DispatchModel(**ent_fresh) + file = test_dir / "test_obj.zip" try: dm.to_file(file) x = DispatchModel.from_file(file) x() x.to_file(file, clobber=True, include_output=True) except Exception as exc: - raise exc + raise AssertionError(f"{exc!r}") from exc else: assert True finally: @@ -187,15 +237,50 @@ def test_storage_summary(fossil_profiles, re_profiles, fossil_specs, fossil_cost assert not x.empty -def test_full_output(test_dir): +def test_storage_capacity(ent_fresh): + """Test full_output.""" + self = DispatchModel(**ent_fresh)() + df = self.storage_capacity() + assert not df.empty + + +def test_storage_durations(ent_fresh): + """Test full_output.""" + self = DispatchModel(**ent_fresh)() + df = self.storage_durations() + assert not df.empty + + +def test_system_summary(ent_fresh): + """Test full_output.""" + self = DispatchModel(**ent_fresh)() + df = self.system_level_summary() + assert not df.empty + + +def test_dc_charge(ent_fresh): + """Test full_output.""" + self = DispatchModel(**ent_fresh) + df = self.dc_charge() + assert not df.empty + + +def test_full_output(ent_fresh): """Test full_output.""" - patio = dfs_from_zip(test_dir / "data/8fresh.zip") - self = DispatchModel(**patio) + self = DispatchModel(**ent_fresh) self() df = self.full_output() assert not df.empty +def test_load_summary(ent_fresh): + """Test full_output.""" + self = DispatchModel(**ent_fresh) + self() + df = self.load_summary() + assert not df.empty + + def test_plotting(fossil_profiles, re_profiles, fossil_specs, fossil_cost, test_dir): """Testing plotting function.""" fossil_profiles.columns = fossil_specs.index @@ -211,42 +296,72 @@ def test_plotting(fossil_profiles, re_profiles, fossil_specs, fossil_cost, test_ ), ) self() - self.plot_year(2015) - x = self.plot_period("2015-01-01", "2015-01-05") + y = self.plot_year(2015) + # x = self.plot_period("2015-01-01", "2015-01-05") img_path = test_dir / "plot.pdf" try: - x.write_image(str(img_path)) + y.write_image(str(img_path)) except Exception as exc: raise AssertionError("unable to write image") from exc finally: img_path.unlink(missing_ok=True) -def test_dfs_to_from_zip(test_dir): - """Dfs are same after being written and read back.""" - df_dict = { - "a": pd.DataFrame( - [[0, 1], [2, 3]], - columns=pd.MultiIndex.from_tuples([(0, "a"), (1, "b")]), - ), - "b": pd.Series([1, 2, 3, 4]), - } +def test_plot_detail_ent(ent_fresh, test_dir): + """Testing plotting function.""" + img_path = test_dir / "plot.pdf" + self = DispatchModel(**ent_fresh)() + x = self.plot_period("2034-01-01", "2034-01-05") try: - dfs_to_zip(df_dict, test_dir / "df_test") - df_load = dfs_from_zip(test_dir / "df_test") - for a, b in zip(df_dict.values(), df_load.values()): - assert a.compare(b).empty + x.write_image(img_path) except Exception as exc: - raise AssertionError("Something broke") from exc + raise AssertionError("unable to write image") from exc + else: + assert True finally: - (test_dir / "df_test.zip").unlink(missing_ok=True) + img_path.unlink(missing_ok=True) + + +def test_plot_year_ent(ent_fresh, test_dir): + """Testing plotting function.""" + img_path = test_dir / "plot.pdf" + self = DispatchModel(**ent_fresh)() + x = self.plot_year(2034) + try: + x.write_image(img_path) + except Exception as exc: + raise AssertionError("unable to write image") from exc + else: + assert True + finally: + img_path.unlink(missing_ok=True) + + +def test_plot_output(ent_fresh, test_dir): + """Testing plotting function.""" + img_path = test_dir / "plot.pdf" + self = DispatchModel(**ent_fresh)() + x = self.plot_output("redispatch_mwh") + try: + x.write_image(img_path) + except Exception as exc: + raise AssertionError("unable to write image") from exc + else: + assert True + finally: + img_path.unlink(missing_ok=True) + + +def test_repr(ent_fresh): + """Test repr.""" + self = DispatchModel(**ent_fresh) + assert "n_dispatchable=24" in repr(self) @pytest.mark.parametrize("existing", ["existing", "additions"], ids=idfn) -def test_redispatch_different(test_dir, existing): +def test_redispatch_different(ent_redispatch, existing): """Test that redispatch and historical are not the same.""" - patio = dfs_from_zip(test_dir / "data/8redispatch.zip") - self = DispatchModel(**patio) + self = DispatchModel(**ent_redispatch) self() if existing == "existing": cols = [tup for tup in self.dispatchable_profiles.columns if tup[0] > 0] @@ -261,10 +376,9 @@ def test_redispatch_different(test_dir, existing): @pytest.mark.parametrize("existing", ["existing", "additions"], ids=idfn) -def test_fresh_different(test_dir, existing): +def test_fresh_different(ent_fresh, existing): """Test that dispatch and full capacity profiles (fresh) are not the same.""" - patio = dfs_from_zip(test_dir / "data/8fresh.zip") - self = DispatchModel(**patio) + self = DispatchModel(**ent_fresh) self() if existing == "existing": cols = [tup for tup in self.dispatchable_profiles.columns if tup[0] > 0] @@ -278,10 +392,17 @@ def test_fresh_different(test_dir, existing): assert not comp.empty, f"dispatch of {existing} failed" +def test_hourly_data_check(ent_redispatch): + """Harness for testing dispatch.""" + self = DispatchModel(**ent_redispatch) + self() + df = self.hourly_data_check() + assert not df.empty + + @pytest.mark.skip(reason="for debugging only") -def test_ent(test_dir): +def test_ent(ent_fresh): """Harness for testing dispatch.""" - patio = dfs_from_zip(test_dir / "data/8fresh.zip") - self = DispatchModel(**patio, jit=False) + self = DispatchModel(**ent_fresh, jit=False) self() assert False diff --git a/tox.ini b/tox.ini index 31ddf5e7..0ba611c3 100644 --- a/tox.ini +++ b/tox.ini @@ -109,7 +109,7 @@ commands = {[testenv:doc8]commands} {[testenv:rstcheck]commands} ; sphinx-build -W -b html docs docs/_build/html - sphinx-build -b html docs docs/_build/html + sphinx-build -b html docs docs/_build