diff --git a/.github/workflows/ci-cd-workflow.yml b/.github/workflows/ci-cd-workflow.yml index ceb0de6c..bd98d6b1 100644 --- a/.github/workflows/ci-cd-workflow.yml +++ b/.github/workflows/ci-cd-workflow.yml @@ -18,8 +18,8 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install dev dependencies run: | - pip install --upgrade pip - pip install -e .[dev] --use-feature=2020-resolver + pip install --upgrade --user pip wheel + pip install -e .[dev] # TODO: add `pylint pymagicc` # TODO: add pydocstyle pymagicc - name: Formatting and linters @@ -86,11 +86,18 @@ jobs: restore-keys: | ${{ runner.os }}-pip- + # no windows wheel for Python 3.6 cftime 1.3.1 + # https://github.com/Unidata/cftime/issues/224 + - name: Install cftime 1.3.0 (${{ runner.os }}) + if: startsWith(runner.os, 'Windows') && endsWith(matrix.python-version, '3.6') + run: | + pip install --upgrade --user pip wheel + pip install cftime==1.3.0 - name: Install test dependencies run: | - pip install --upgrade pip - pip install -e .[tests] --use-feature=2020-resolver + pip install --upgrade --user pip wheel + pip install -e .[tests] - name: Test with pytest (${{ runner.os }}) @@ -111,6 +118,10 @@ jobs: run: | pytest tests -r a + - name: Test scripts (${{ runner.os }}) + run: | + python scripts/plot_example.py + - name: Upload coverage to Codecov if: startsWith(runner.os, 'Linux') && ${{ matrix.python-version }} == 3.7 @@ -158,10 +169,18 @@ jobs: restore-keys: | ${{ runner.os }}-notebooks-pip- + # no windows wheel for Python 3.6 cftime 1.3.1 + # https://github.com/Unidata/cftime/issues/224 + - name: Install cftime 1.3.0 (${{ runner.os }}) + if: startsWith(runner.os, 'Windows') && endsWith(matrix.python-version, '3.6') + run: | + pip install --upgrade --user pip wheel + pip install cftime==1.3.0 + - name: Install notebook dependencies run: | - pip install --upgrade pip - pip install -e .[tests,notebooks] --use-feature=2020-resolver + pip install --upgrade --user pip wheel + pip install -e .[tests,notebooks] - name: Test notebooks with nbval if: startsWith(runner.os, 'Linux') run: | @@ -171,8 +190,40 @@ jobs: run: | pytest notebooks -r a --nbval --sanitize-with tests/notebook-tests.cfg --no-cov + test-install: + needs: build + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: ["ubuntu-latest", "windows-latest"] + python-version: [3.6, 3.7, 3.8] + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + - name: Setup python + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + + # no windows wheel for Python 3.6 cftime 1.3.1 + # https://github.com/Unidata/cftime/issues/224 + - name: Install cftime 1.3.0 (${{ runner.os }}) + if: startsWith(runner.os, 'Windows') && endsWith(matrix.python-version, '3.6') + run: | + pip install --upgrade --user pip wheel + pip install cftime==1.3.0 + + - name: Install (${{ runner.os }}) + run: | + pip install --upgrade pip wheel + pip install . + - name: Test installation + run: | + python scripts/test_install.py + deploy-pypi: - needs: test-notebooks + needs: [test-notebooks,test-install] if: startsWith(github.ref, 'refs/tags/v') runs-on: ubuntu-latest @@ -189,8 +240,8 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | - pip install --upgrade pip - pip install -e .[dev] --use-feature=2020-resolver + pip install --upgrade --user pip wheel + pip install -e .[dev] - name: Create package run: python setup.py sdist bdist_wheel --universal - name: Publish package to PyPI diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 1440f4ea..075feea4 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,69 +1,111 @@ Changelog ========= +All notable changes to this project will be documented in this file. + +The format is based on `Keep a Changelog`_, and this project adheres to `Semantic Versioning `_. + +The changes listed in this file are categorised as follows: + + - Added: new features + - Changed: changes in existing functionality + - Deprecated: soon-to-be removed features + - Removed: now removed features + - Fixed: any bug fixes + - Security: in case of vulnerabilities. + master ------ -- (`#322 `_) Change to 3-Clause BSD License -- (`#321 `_) Raise :obj:`ValueError` if we attempt to run MAGICC with conflicting config keys (which can occur because FORTRAN is not case sensitive) +v2.0.0 - 2021-01-19 +------------------- + +Added +~~~~~ + - (`#317 `_) Add support for more MAGICC7 output variables - (`#315 `_) Include links to AR6 region abbreviations if they are written in ``.MAG`` files - (`#314 `_) Add AR6 regions to the list of known regions - (`#313 `_) Add tests for global-only binary files written with ``out_binary_format=2`` +- (`#305 `_) Added functionality to read new MAGICC binary format which includes units +- (`#238 `_) Add documentation for handling of World region in ``.SCEN7`` files +- (`#301 `_) Add MAGICC7 variables ``AEROSOL_RF``, ``HEAT_EARTH`` and ``HEAT_NONOCEAN`` +- (`#277 `_) Add MAGICC7 compact output file readers +- (`#288 `_) Add ``pymagicc.io.read_mag_file_metadata``, which allows fast reading of metadata from a ``.MAG`` file +- (`#282 `_) Expose ``MAGICCBase.get_tcr_ecs_from_diagnosis_results`` method +- (`#274 `_) Add better readers and writers for ``.DAT`` files +- (`#272 `_) Add support for new ``THISFILE_TIMESERIESTYPE`` in ``.MAG`` files +- (`#259 `_) Added ``strict`` option for downgrading configuration exceptions to warnings +- (`#256 `_) Capture stderr output from MAGICC7 and above (not available in MAGICC6) +- (`#253 `_) Add support for ``out_dynamic_vars`` parameter +- (`#250 `_) Add support for ``.MAG`` files +- (`#229 `_) Add more robust tests of io, in particular that column order and spacing in files is preserved +- (`#226 `_) Add ``SURFACE_TEMP.IN`` writer, closing `#211 `_ +- (`#224 `_) Add ``INVERSEEMIS.OUT`` reader +- (`#208 `_) Add :meth:`set_zero_config`. Also adds scenarios module, tidies up the notebooks and adds a notebook showing how to run in different modes. +- (`#208 `_) Add ``pymagicc.scenarios`` module +- (`#208 `_) Tidy up the notebooks and add a notebook showing how to run in different modes. +- (`#187 `_) Added ``pymagicc.io.join_timeseries`` which simplifies joining/merging scenarios to create custom scenarios +- (`#185 `_) Added ability to read RCP files from http://www.pik-potsdam.de/~mmalte/rcps/ as requested in `#176 `_ +- (`#183 `_) Added ability to read MHALO files (see `#182 `_) +- (`#180 `_) Added reference which explains MAGICC's variables to docs +- (`#170 `_) Added pyam as a dependency and gave an example of how to integrate with it +- (`#168 `_) Added MAGICC7 compatibility +- (`#162 `_) Added basic tests of integration with MAGICC binaries +- (`#139 `_) Added the ability to read all MAGICC output files/throw an explanatory error with ``pymagicc.io.MAGICCData`` +- (`#79 `_) Confirmed that keeping track of config state works and added example to TCR/ECS diagnosis notebook +- (`#102 `_) Added ability to read and write SCEN7 files +- (`#108 `_) Added ability to read all files in MAGICC6 run folder (``pymagicc/MAGICC6/run``) to a common format + + - Note that this change means that only files which follow the MAGICC6 or MAGICC7 naming convention are supported. These are very similar to MAGICC5 except that emissions files must be named in the form ``*.SCEN``, ``*.SCEN7`` or ``*EMISX.IN`` where ``X`` is ``I`` if the file contains fossil and industrial emissions and ``B`` if the file contains agriculture, land-use and land-use change emissions. The suffixes ``FOSSIL&IND`` and ``LANDUSE`` are no longer supported. + - The renamed files are + + - ``pymagicc/MAGICC6/run/EDGAR_NOX_EMIS_LANDUSE.IN`` => ``pymagicc/MAGICC6/run/EDGAR_NOXB_EMIS.IN`` + - ``pymagicc/MAGICC6/run/EDGAR_NOX_EMIS_FOSSIL&IND.IN`` => ``pymagicc/MAGICC6/run/EDGAR_NOXI_EMIS.IN`` + - ``pymagicc/MAGICC6/run/HOUGHTON_CO2_EMIS_LANDUSE.IN`` => ``pymagicc/MAGICC6/run/HOUGHTON_CO2B_EMIS.IN`` + - ``pymagicc/MAGICC6/run/MARLAND_CO2_EMIS_FOSSIL&IND.IN`` => ``pymagicc/MAGICC6/run/MARLAND_CO2I_EMIS.IN`` + + - Deleted ``pymagicc/MAGICC6/run/HIST_SEALEVEL_CHURCHWHITE2006_RF.IN`` as it's empty + - Added ``scripts/check_run_dir_file_read.py`` so we can quickly check which files in a MAGICC ``run`` directory can be read by ``pymagicc`` + - Added new section to docs, ``docs/file_conventions.rst`` which will document all of the relevant information related to MAGICC's file conventions + +Changed +~~~~~~~ + +- (`#323 `_) Writers raise an :obj:`AssertionError` if the user tries to write a MAGICC input file which has timesteps where some values are nan whilst others are not. Such input files would have nans in them hence would cause MAGICC's run to fail. +- (`#322 `_) Re-write CHANGELOG to follow `Keep a Changelog`_ style +- (`#322 `_) Change to 3-Clause BSD License +- (`#321 `_) Raise :obj:`ValueError` if we attempt to run MAGICC with conflicting config keys (which can occur because FORTRAN is not case sensitive) - (`#311 `_) Fix naming of ocean heat content and ocean heat uptake output variables to match RCMIP conventions -- (`#310 `_) Rename ``pymagicc.io.prn`` to ``pymagicc.io.prn_files`` as PRN is a reserved filename on Windows - (`#307 `_) Use ``scmdata.ScmRun`` as a base class for ``MAGICCData`` instead of the deprecated ``scmdata.ScmDataFrame`` (closes `#295 `_) -- (`#305 `_) Added functionality to read new MAGICC binary format which includes units - (`#306 `_) Copy ``run`` folder recursively when creating temporary copy -- (`#238 `_) Add documentation for handling of World region in ``.SCEN7`` files - (`#303 `_) Refactor ``pymagicc.io`` into multiple files -- (`#301 `_) Add MAGICC7 variables ``AEROSOL_RF``, ``HEAT_EARTH`` and ``HEAT_NONOCEAN`` - (`#299 `_) Make conversion of FORTRAN safe units apply to ``.MAG`` files too and be more consistent -- (`#300 `_) Fix name in docs (closes `#205 `_) -- (`#298 `_) Make SCEN7 writing work with single variables -- (`#297 `_) Make Binary reader able to handle global-only binary output - (`#293 `_) Update CI to use GitHub actions -- (`#294 `_) Convert the direct aerosols variable names from MAGICC in a consistent way. Renamed ``definitions/magicc_emisssions_units.csv`` to ``definitions/magicc_emissions_units.csv`` +- (`#294 `_) Convert the direct aerosols variable names from MAGICC in a consistent way. +- (`#294 `_) Renamed ``definitions/magicc_emisssions_units.csv`` to ``definitions/magicc_emissions_units.csv`` - (`#291 `_) Switch to using the ``_ERF`` suffix for IPCC definition of Effective Radiative Forcing variables. This replaces ``_EFFRF`` which is a MAGICC internal variable and was incorrectly labelled as Effective Radiative Forcing. -- (`#277 `_) Add MAGICC7 compact output file readers -- (`#281 `_) Hotfix readers and writers for ``.DAT`` files (``thisfile_datacolumns`` was wrong) -- (`#288 `_) Add ``pymagicc.io.read_mag_file_metadata``, which allows fast reading of metadata from a ``.MAG`` file - (`#290 `_) Update minimum ``scmdata`` version to v0.4.3 - (`#285 `_) Return ``pint.quantity.Quantity`` from all ECS, TCR and TCRE diagnostic methods - (`#284 `_) Update ECS, TCR and TCRE diagnosis to use 1pctCO2 and abrupt-2xCO2 experiments - (`#283 `_) Diagnose TCRE alongisde ECS and TCR, changes ``diagnose_tcr_ecs`` to ``diagnose_tcr_ecs_tcre`` and ``get_tcr_ecs_from_diagnosis_results`` method to ``get_tcr_ecs__tcre_from_diagnosis_results`` -- (`#282 `_) Expose ``MAGICCBase.get_tcr_ecs_from_diagnosis_results`` method - (`#280 `_) Also include source distribution in pypi release -- (`#271 `_) Update requirements of pyam, make error messages include ``stderr`` and remove overwrite of ``file_emisscen`` when creating MAGICC7 copies if ``not self.strict`` -- (`#274 `_) Add better readers and writers for ``.DAT`` files -- (`#272 `_) Add support for new ``THISFILE_TIMESERIESTYPE`` in ``.MAG`` files -- (`#269 `_) Break circular dependency on OpenSCM by switching to using scmdata +- (`#271 `_) Update requirements of pyam. +- (`#271 `_) Make error messages include ``stderr`` +- (`#271 `_) Remove overwrite of ``file_emisscen`` when creating MAGICC7 copies if ``not self.strict`` - (`#268 `_) Update region mapping to match SSP database - (`#266 `_) Use a whitelist of `OUT_` parameters which are converted to 1/0's - (`#264 `_) Allowed an empty dataframe to be returned from ``MAGICCBase.run`` if no output is produced -- (`#267 `_) Hotfix appveyor failures - (`#261 `_) Improve mapping of MAGICC7 to OpenSCM variables -- (`#259 `_) Added ``strict`` option for downgrading configuration exceptions to warnings -- (`#256 `_) Capture stderr output from MAGICC7 and above (not available in MAGICC6) -- (`#252 `_) Improve header writing, upgrade MAGICC time conversions and fix wine not installed error handling -- (`#253 `_) Add support for ``out_dynamic_vars`` parameter -- (`#250 `_) Add support for ``.MAG`` files -- (`#249 `_) Update to keep pace with MAGICC7 development +- (`#252 `_) Improve header writing +- (`#252 `_) Upgrade MAGICC time conversions +- (`#249 `_) Update MAGICC7 support - (`#247 `_) Upgrade pyam dependency to use nominated release -- (`#244 `_) Use openscm from pip, hence drop Python3.6 support, and drop pyam dependency (moved into notebooks dependencies) - (`#236 `_) Made all subannual files raise an InvalidTemporalResError exception as ScmDataFrame can't handle merging annual and subannual timeseries together yet - (`#239 `_) Explicitly overwrite tuning model and emission scenario parameters for MAGICC7 when a temporary copy is created -- (`#229 `_) Add more robust tests of io, in particular that column order and spacing in files is preserved -- (`#233 `_) Fix inplace append hard coding as identified in `#232 `_ - (`#234 `_) Raise ``ValueError`` if ``only`` doesn't match an output variable in ``MAGICC.run`` (solves `#231 `_) - (`#227 `_) Fixed up permafrost naming to avoid confusing inclusion when summing up "Emissions|CO2" -- (`#226 `_) Add ``SURFACE_TEMP.IN`` writer, closing `#211 `_ -- (`#225 `_) Fix reading of ``DAT_CO2PF_EMIS.OUT`` -- (`#224 `_) Add ``INVERSEEMIS.OUT`` reader -- (`#223 `_) Ensure `pymagicc.io._BinaryOutReader` closes the input file -- (`#222 `_) Remove trailing ``/`` in ``MANIFEST.IN`` recursive includes as this is invalid syntax on windows. - (`#220 `_) If binary and ascii output files exist for a given variable only read the binary file -- (`#208 `_) Add set zero config method. Also adds scenarios module, tidies up the notebooks and adds a notebook showing how to run in different modes. - (`#214 `_) Refactor to use the timeseries capabilities of ScmDataFrameBase - (`#210 `_) Updated to match new openscm naming - (`#199 `_) Switched to OpenSCMDataFrameBase for the backend, also includes: @@ -77,17 +119,9 @@ master - (`#204 `_) Addressed potential bug identified in (`#203 `_) and updated robustness of output file read in - (`#198 `_) Move all install requirements into ``setup.py`` - (`#190 `_) Speed up diagnosis of TCR and ECS by removing writing of scenario file -- (`#191 `_) Fixed bugs which meant config passed to MAGICC wasn't handled correctly and renamed `tests/test_api.py` to `tests/test_core.py`. -- (`#187 `_) Added `pymagicc.io.join_timeseries` which simplifies joining/merging scenarios to create custom scenarios -- (`#185 `_) Added ability to read RCP files from http://www.pik-potsdam.de/~mmalte/rcps/ as requested in `#176 `_ -- (`#184 `_) Remove redundant mapping of region names for SCEN to SCEN7 conversions -- (`#183 `_) Added ability to read MHALO files (see `#182 `_) -- (`#180 `_) Added reference which explains MAGICC's variables to docs -- (`#177 `_) Fixed SCEN reading bug, can now read SCEN files with "YEAR" in first column rather than "YEARS" -- (`#170 `_) Added pyam as a dependency and gave an example of how to integrate with it -- (`#173 `_) Renamed - ``pymagicc.api`` to ``pymagicc.core`` -- (`#168 `_) Added MAGICC7 compatibility +- (`#191 `_) Fixed bugs which meant config passed to MAGICC wasn't handled correctly +- (`#191 `_) Renamed ``tests/test_api.py`` to ``tests/test_core.py`` +- (`#173 `_) Renamed ``pymagicc.api`` to ``pymagicc.core`` - (`#165 `_) Moved to one unified backend for all run functionality. This one got a bit out of hand so also includes: - Breaking the API, hence requiring significantly re-writing the tests to match the new API, bumping the major version number and updating the examples. @@ -97,36 +131,52 @@ master - Only passing ``filepath`` (i.e. the combination of path and name) to reading/writing functions to remove ambiguity in previous language which used ``file``, ``filepath``, ``path``, ``name`` and ``filename``, sometimes in a self-contradictory way. - (`#167 `_) Updated release instructions -- (`#162 `_) Added basic tests of integration with MAGICC binaries -- (`#163 `_) Confirmed HFC-245fa misnaming in MAGICC6. Accordingly, we: - - - fixed this naming in the SRES scenarios - - removed ``pymagicc/MAGICC6/run/HISTRCP_HFC245ca_CONC.IN`` to avoid repeating this confusion - - ensured that anyone who finds a file with "HFC-245ca" in it in future will get a warning, see ``tests/test_definitions.py`` - - (`#164 `_) Improved missing MAGICC binary message in tests as discussed in `#124 `_ - (`#154 `_) Change to using OpenSCM variables for all user facing data as well as preparing to move to using OpenSCM dataframes - Note that this change breaks direct access but that we will gain a lot of features once we start using the capabilities of pyam as part of an OpenSCM dataframe - (`#160 `_) Made notebooks CI more opinionated (`#158 `_) -- (`#139 `_) Added the ability to read all MAGICC output files/throw an explanatory error with ``pymagicc.io.MAGICCData`` - (`#135 `_) Moved emissions definitions to a single csv and packaged all of the definitions files using the `data package standard `_ -- (`#79 `_) Confirmed that keeping track of config state works and added example to TCR/ECS diagnosis notebook - (`#146 `_) Removed path alteration from docs buiding - (`#143 `_) Only read ``PARAMETERS.OUT`` file if it exists. ``MAGICCBase.config`` now defaults to ``None`` until a valid ``PARAMETERS.OUT`` file is read. - (`#133 `_) Put definitions of MAGICC6's expected emissions into a standalone module -- (`#102 `_) Added ability to read and write SCEN7 files -- (`#108 `_) Added ability to read all files in MAGICC6 run folder (``pymagicc/MAGICC6/run``) to a common format - - Note that this change means that only files which follow the MAGICC6 or MAGICC7 naming convention are supported. These are very similar to MAGICC5 except that emissions files must be named in the form ``*.SCEN``, ``*.SCEN7`` or ``*EMISX.IN`` where ``X`` is ``I`` if the file contains fossil and industrial emissions and ``B`` if the file contains agriculture, land-use and land-use change emissions. The suffixes ``FOSSIL&IND`` and ``LANDUSE`` are no longer supported. - - The renamed files are - - ``pymagicc/MAGICC6/run/EDGAR_NOX_EMIS_LANDUSE.IN`` => ``pymagicc/MAGICC6/run/EDGAR_NOXB_EMIS.IN`` - - ``pymagicc/MAGICC6/run/EDGAR_NOX_EMIS_FOSSIL&IND.IN`` => ``pymagicc/MAGICC6/run/EDGAR_NOXI_EMIS.IN`` - - ``pymagicc/MAGICC6/run/HOUGHTON_CO2_EMIS_LANDUSE.IN`` => ``pymagicc/MAGICC6/run/HOUGHTON_CO2B_EMIS.IN`` - - ``pymagicc/MAGICC6/run/MARLAND_CO2_EMIS_FOSSIL&IND.IN`` => ``pymagicc/MAGICC6/run/MARLAND_CO2I_EMIS.IN`` - - Deleted ``pymagicc/MAGICC6/run/HIST_SEALEVEL_CHURCHWHITE2006_RF.IN`` as it's empty - - Added ``scripts/check_run_dir_file_read.py`` so we can quickly check which files in a MAGICC ``run`` directory can be read by ``pymagicc`` - - Added new section to docs, ``docs/file_conventions.rst`` which will document all of the relevant information related to MAGICC's file conventions + +Deprecated +~~~~~~~~~~ + +Removed +~~~~~~~ + +- (`#244 `_) Use openscm from pip, hence drop Python3.6 support, and drop pyam dependency (moved into notebooks dependencies) +- (`#184 `_) Remove redundant mapping of region names for SCEN to SCEN7 conversions + +Fixed +~~~~~ + +- (`#323 `_) Writers now automatically drop all nan timesteps before writing MAGICC input files +- (`#323 `_) ``pymagicc.scenarios.rcps`` now contains all the rcps rather than just rcp26 +- (`#310 `_) Rename ``pymagicc.io.prn`` to ``pymagicc.io.prn_files`` as PRN is a reserved filename on Windows +- (`#300 `_) Fix name in docs (closes `#205 `_) +- (`#298 `_) Make SCEN7 writing work with single variables +- (`#297 `_) Make Binary reader able to handle global-only binary output +- (`#281 `_) Hotfix readers and writers for ``.DAT`` files (``thisfile_datacolumns`` was wrong) +- (`#269 `_) Break circular dependency on OpenSCM by switching to using scmdata +- (`#267 `_) Hotfix appveyor failures +- (`#252 `_) Fix wine not installed error handling +- (`#233 `_) Fix inplace append hard coding as identified in `#232 `_ +- (`#225 `_) Fix reading of ``DAT_CO2PF_EMIS.OUT`` +- (`#223 `_) Ensure `pymagicc.io._BinaryOutReader` closes the input file +- (`#222 `_) Remove trailing ``/`` in ``MANIFEST.IN`` recursive includes as this is invalid syntax on windows. +- (`#177 `_) Fixed SCEN reading bug, can now read SCEN files with "YEAR" in first column rather than "YEARS" +- (`#163 `_) Confirmed HFC-245fa misnaming in MAGICC6 (i.e. HFC-245fa was mistakenly labelled as HFC-245ca). Accordingly, we: + + - fixed this naming in the SRES scenarios (changing HFC-245ca to HFC-245fa) + - removed ``pymagicc/MAGICC6/run/HISTRCP_HFC245ca_CONC.IN`` to avoid repeating this confusion + - ensured that anyone who finds a file with "HFC-245ca" in it in future will get a warning, see ``tests/test_definitions.py`` + +For versions before 2.0 we did not follow the `Keep a Changelog`_ style. +The notes made whilst developing versions <2.0 are included below for posterity. 1.3.2 ----- @@ -220,3 +270,6 @@ master --- Initial release. + + +.. _Keep a Changelog: https://keepachangelog.com/en/1.0.0/ diff --git a/README.rst b/README.rst index b422c9b7..f1c649b8 100644 --- a/README.rst +++ b/README.rst @@ -91,28 +91,25 @@ Basic Usage import matplotlib.pyplot as plt import pymagicc - from pymagicc import scenarios + import scmdata + from pymagicc import rcps - for name, scen in scenarios.items(): - results = pymagicc.run(scen) - results_df = results.df - results_df.set_index("time", inplace=True) + results = [] + for scen in rcps.groupby("scenario"): + results_scen = pymagicc.run(scen) + results.append(results_scen) - global_temp_time_rows = ( - (results_df.variable == "Surface Temperature") - & (results_df.region == "World") - ) + results = scmdata.run_append(results) - temp = ( - results_df.value[global_temp_time_rows].loc[1850:] - - results_df.value[global_temp_time_rows].loc[1850:1900].mean() - ) - temp.plot(label=name) + temperature_rel_to_1850_1900 = ( + results + .filter(variable="Surface Temperature", region="World") + .relative_to_ref_period_mean(year=range(1850, 1900 + 1)) + ) - plt.legend() + temperature_rel_to_1850_1900.lineplot() plt.title("Global Mean Temperature Projection") plt.ylabel("°C over pre-industrial (1850-1900 mean)"); - plt.legend(loc="best") # Run `plt.show()` to display the plot when running this example # interactively or add `%matplotlib inline` on top when in a Jupyter Notebook. @@ -266,16 +263,16 @@ Use an included scenario .. code:: python - from pymagicc import rcp26 + from pymagicc.scenarios import rcp26 - rcp26.df.head() + rcp26.head() Read a MAGICC scenario file *************************** .. code:: python - from pymagicc import read_scen_file + from pymagicc.scenarios import read_scen_file scenario = read_scen_file("PATHWAY.SCEN") @@ -284,18 +281,17 @@ Run MAGICC for a scenario .. code:: python - results = pymagicc.run(scenario) - results_df = results.df - results_df.set_index("time", inplace=True) + import pymagicc + from pymagicc.scenarios import read_scen_file - global_temp_time_rows = ( - (results_df.variable == "Surface Temperature") - & (results_df.region == "World") - ) + scenario = read_scen_file("PATHWAY.SCEN") + + results = pymagicc.run(scenario) - temp = ( - results_df.value[global_temp_time_rows].loc[1850:] - - results_df.value[global_temp_time_rows].loc[1850:1900].mean() + temperature_rel_to_1850_1900 = ( + results + .filter(variable="Surface Temperature") + .relative_to_ref_period_mean(year=range(1850, 1900 + 1)) ) Using a different MAGICC version diff --git a/pymagicc/io/base.py b/pymagicc/io/base.py index 036bdf05..ca379ea6 100644 --- a/pymagicc/io/base.py +++ b/pymagicc/io/base.py @@ -743,10 +743,20 @@ def _ensure_file_region_type_consistency(self, regions): # no checks required except for certain cases return regions - def _get_data_block(self): - data_block = self.minput.timeseries( + def _get_timeseries_no_nans(self): + out = self.minput.timeseries( meta=["variable", "todo", "unit", "region"] - ).T + ).T.dropna(how="all") + if out.isnull().any().any(): + raise AssertionError( + "Your data contains timesteps where some values are nan whilst others " + "are not. This will not work in MAGICC." + ) + + return out + + def _get_data_block(self): + data_block = self._get_timeseries_no_nans() self._check_data_block_column_names(data_block) self._check_data_filename_variable_consistency(data_block) diff --git a/pymagicc/io/prn_files.py b/pymagicc/io/prn_files.py index 819197c4..e3b97b4b 100644 --- a/pymagicc/io/prn_files.py +++ b/pymagicc/io/prn_files.py @@ -255,9 +255,7 @@ def _get_unit(self): return unit def _get_data_block(self): - data_block = self.minput.timeseries( - meta=["variable", "todo", "unit", "region"] - ).T + data_block = self._get_timeseries_no_nans() self._check_data_block_column_names(data_block) regions = data_block.columns.get_level_values("region").unique() diff --git a/pymagicc/scenarios/__init__.py b/pymagicc/scenarios/__init__.py index e526d4c6..7f78c821 100644 --- a/pymagicc/scenarios/__init__.py +++ b/pymagicc/scenarios/__init__.py @@ -62,7 +62,7 @@ def read_scen_file( rcps = deepcopy(rcp26) for rcp in [rcp45, rcp60, rcp85]: - rcps.append(rcp) + rcps = rcps.append(rcp) zero_emissions = MAGICCData( join(dirname(abspath(__file__)), "RCP3PD_EMISSIONS.DAT"), diff --git a/scripts/example-plot.png b/scripts/example-plot.png index 5b94e3b7..24125644 100644 Binary files a/scripts/example-plot.png and b/scripts/example-plot.png differ diff --git a/scripts/plot_example.py b/scripts/plot_example.py index cd7e2e63..c03c6f48 100644 --- a/scripts/plot_example.py +++ b/scripts/plot_example.py @@ -1,9 +1,9 @@ import os -import matplotlib.pyplot as plt - +import matplotlib.pyplot as plt import pymagicc -from pymagicc import scenarios +import scmdata +from pymagicc import rcps plt.style.use("ggplot") @@ -17,25 +17,21 @@ './example-plot.png' ) -for name, scen in scenarios.items(): - results = pymagicc.run(scen) - results_df = results.df - results_df.set_index("time", inplace=True) +results = [] +for scen in rcps.groupby("scenario"): + results_scen = pymagicc.run(scen) + results.append(results_scen) - global_temp_time_rows = ( - (results_df.variable == "Surface Temperature") - & (results_df.region == "World") - ) +results = scmdata.run_append(results) - temp = ( - results_df.value[global_temp_time_rows].loc[1850:] - - results_df.value[global_temp_time_rows].loc[1850:1900].mean() - ) - temp.plot(label=name) +temperature_rel_to_1850_1900 = ( + results + .filter(variable="Surface Temperature", region="World") + .relative_to_ref_period_mean(year=range(1850, 1900 + 1)) +) -plt.legend() +temperature_rel_to_1850_1900.lineplot() plt.title("Global Mean Temperature Projection") -plt.ylabel("°C over pre-industrial (1850-1900 mean)"); -plt.legend(loc="best") +plt.ylabel("°C over pre-industrial (1850-1900 mean)") plt.savefig(output_path, dpi=96) diff --git a/setup.py b/setup.py index fb6d76ac..79141e78 100644 --- a/setup.py +++ b/setup.py @@ -46,6 +46,7 @@ "Programming Language :: Python :: 3.8", ] REQUIREMENTS_INSTALL = [ + "cftime", "pandas-datapackage-reader", "f90nml", "PyYAML", @@ -67,6 +68,7 @@ "codecov", "goodtables", "scipy", + "seaborn", ] REQUIREMENTS_DOCS = [ "sphinx>2.1", diff --git a/tests/test_io.py b/tests/test_io.py index 9fb935d4..5f0eb77f 100644 --- a/tests/test_io.py +++ b/tests/test_io.py @@ -2907,7 +2907,7 @@ def test_conc_in_reader_get_variable_from_filepath(test_filepath, expected_varia assert conc_reader._get_variable_from_filepath() == expected_variable -@pytest.mark.parametrize( +ALL_FILE_TYPES = pytest.mark.parametrize( "magicc_version, starting_fpath, starting_fname, confusing_metadata, old_namelist", [ (6, MAGICC6_DIR, "HISTRCP_CO2I_EMIS.IN", False, False), @@ -2953,6 +2953,9 @@ def test_conc_in_reader_get_variable_from_filepath(test_filepath, expected_varia (7, TEST_DATA_DIR, "MAG_FORMAT_SAMPLE_TWO.MAG", False, False), ], ) + + +@ALL_FILE_TYPES def test_in_file_read_write_functionally_identical( magicc_version, starting_fpath, @@ -2993,6 +2996,56 @@ def test_in_file_read_write_functionally_identical( ) +@ALL_FILE_TYPES +def test_nans_stripped_before_writing( + magicc_version, + starting_fpath, + starting_fname, + confusing_metadata, + old_namelist, + temp_dir, +): + mi_writer = MAGICCData(join(starting_fpath, starting_fname)) + + nan_idx = mi_writer.shape[1] // 2 + nan_timestep = mi_writer["time"].iloc[nan_idx] + + assert nan_timestep in mi_writer["time"].values + + mi_writer.values[:, nan_idx] = np.nan + + mi_writer.write(join(temp_dir, starting_fname), magicc_version=magicc_version) + + mi_written = MAGICCData(join(temp_dir, starting_fname)) + assert nan_timestep not in mi_written["time"].values + + +@ALL_FILE_TYPES +def test_raises_if_nans_not_uniform( + magicc_version, + starting_fpath, + starting_fname, + confusing_metadata, + old_namelist, + temp_dir, +): + mi_writer = MAGICCData(join(starting_fpath, starting_fname)) + + if mi_writer.shape[0] == 1: + pytest.skip("Only one timeseries so can't create mismatch") + + nan_row = mi_writer.shape[0] // 2 + nan_col = mi_writer.shape[1] // 2 + mi_writer.values[nan_row, nan_col] = np.nan + + error_msg = re.escape( + "Your data contains timesteps where some values are nan whilst others " + "are not. This will not work in MAGICC." + ) + with pytest.raises(AssertionError, match=error_msg): + mi_writer.write(join(temp_dir, starting_fname), magicc_version=magicc_version) + + emissions_valid = [ "CO2I", "CO2B", diff --git a/tests/test_readme.py b/tests/test_readme.py new file mode 100644 index 00000000..0161b558 --- /dev/null +++ b/tests/test_readme.py @@ -0,0 +1,48 @@ +import os +import shutil + +import pytest + + +def _get_readme_codeblocks(): + codeblocks = [] + with open("README.rst", "r") as fh: + in_codeblock = False + codeblock = [] + for line in fh.readlines(): + line = line.strip(os.linesep) + if line == ".. code:: python": + in_codeblock = True + + elif in_codeblock: + if line and not line.startswith(" "): + in_codeblock = False + codeblocks.append(os.linesep.join(codeblock)) + codeblock = [] + else: + if not line: + codeblock.append(os.linesep) + else: + codeblock.append(line[4:]) + + return codeblocks + + +@pytest.mark.parametrize("codeblock", _get_readme_codeblocks()) +def test_readme(codeblock): + pathway_scen_file = "PATHWAY.SCEN" + scen_required = pathway_scen_file in codeblock + if scen_required: + shutil.copyfile( + os.path.join("tests", "test_data", "WORLD_ONLY.SCEN"), pathway_scen_file + ) + + try: + # https://stackoverflow.com/a/62851176/353337 + exec(codeblock, {"__MODULE__": "__main__"}) + except Exception: + print("Codeblock failed:\n{}".format(codeblock)) + raise + finally: + if scen_required: + os.remove(pathway_scen_file) diff --git a/tests/test_scenarios.py b/tests/test_scenarios.py new file mode 100644 index 00000000..81303c1e --- /dev/null +++ b/tests/test_scenarios.py @@ -0,0 +1,5 @@ +from pymagicc.scenarios import rcps + + +def test_all_rcps_included(): + assert set(rcps["scenario"]) == {"RCP26", "RCP45", "RCP60", "RCP85"}