Ouranosinc · RondeauG · Jul 10, 2023 · Jul 7, 2023 · Jul 7, 2023
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -46,7 +46,7 @@ jobs:
 #            tox-build: "py39"
           - python-version: "3.10"
 #            tox-build: "py310"
-#          - python-version: "3.11"
+          - python-version: "3.11"
 #            tox-build: "py311"
     defaults:
       run:

diff --git a/HISTORY.rst b/HISTORY.rst
@@ -8,7 +8,7 @@ Contributors to this version: Gabriel Rondeau-Genesse (:user:`RondeauG`), Pascal
 
 Announcements
 ^^^^^^^^^^^^^
-* Dropped support for Python 3.8. (:pull:`199`).
+* Dropped support for Python 3.8, added support for 3.11. (:pull:`199`,:pull:`222`).
 
 New features and enhancements
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -19,6 +19,9 @@ New features and enhancements
 
 Breaking changes
 ^^^^^^^^^^^^^^^^
+* Columns ``date_start`` and ``date_end`` now use a ``datetime64[ms]`` dtype. (:pull:`222`).
+* The default output of ``date_parser`` is now ``pd.Timestamp`` (``output_dtype='datetime'``). (:pull:`222`).
+* ``driving_institution`` was removed from the "default" xscen columns. (:pull:`222`).
 * Folder parsing utilities (``parse_directory``) moved to ``xscen.catutils``. Signature changed : ``globpattern`` removed, ``dirglob`` added, new ``patterns`` specifications. See doc for all changes. (:pull:`205`).
 
 Bug fixes
@@ -35,7 +38,6 @@ Internal changes
 * Updated GitHub Actions to remove deprecation warnings. (:pull:`187`).
 * Updated the cookiecutter used to generate boilerplate documentation and code via `cruft`. (:pull:`212`).
 * A few changes to `subset_warming_level` so it doesn't need `driving_institution`. (:pull:`215`).
-* Injection of a better ``__format__`` method in ``pd.Period``, allowing use of these in f-strings with strftime-like format strings. (:pull:`218`).
 
 v0.6.0 (2023-05-04)
 -------------------

diff --git a/docs/columns.rst b/docs/columns.rst
@@ -28,9 +28,6 @@ the four columns will be combined in a single Dataset when any of the first thre
 - ``activity``: Model Intercomparison Project (MIP) associated with the data. This is the same as `activity_id` in CMIP6 data. CMIP is the activity for the historical experiment and the DECK experiments.
     - E.g. "CMIP", "CORDEX", "HighResMIP"
 
-- ``driving_institution``: Institution of the driver model.
-    - E.g. "CCCma"
-
 - ``driving_model``: Name of the driver. Following the `driving_model` convention from ES-DOC, this is in the format "institution-model".
     - E.g. "CCCma-CanESM2"
 
@@ -58,11 +55,11 @@ the four columns will be combined in a single Dataset when any of the first thre
 - ``domain``: Name of the region covered by the dataset. It can also contain information on the grid.
     - E.g. "global", "NAM", "ARC-44",  "ARC-22"
 
-- ``date_start``: First date of the dataset. This a pd.Period object with an hourly frequency.
-    - E.g. "2022-06-03 00:00"
+- ``date_start``: First date of the dataset. This usually is a Datetime object with a ms resolution.
+    - E.g. "2022-06-03 00:00:00"
 
-- ``date_end``: Last date of the dataset. This a pd.Period object with an hourly frequency.
-    - E.g. "2022-06-03 00:00"
+- ``date_end``: Last date of the dataset. This usually is a Datetime object with a ms resolution.
+    - E.g. "2022-06-03 00:00:00"
 
 - ``version``: Version of the dataset.
     - E.g. "1.0"

diff --git a/docs/notebooks/1_catalog.ipynb b/docs/notebooks/1_catalog.ipynb
@@ -36,7 +36,6 @@
     "| bias_adjust_project | Name of the project that computed the bias adjustment. |\n",
     "| mip_era | CMIP Generation associated with the data. |\n",
     "| activity | Model Intercomparison Project (MIP) associated with the data. |\n",
-    "| driving_institution | Institution of the driver model. |\n",
     "| driving_model | Name of the driver. |\n",
     "| institution | Institution associated with the source. |\n",
     "| source | Name of the model or the dataset. |\n",
@@ -1023,14 +1022,6 @@
     "PC.df[\"path\"] = newdf[\"new_path\"]\n",
     "PC.update()"
    ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "f34b3727",
-   "metadata": {},
-   "outputs": [],
-   "source": []
   }
  ],
  "metadata": {
@@ -1053,7 +1044,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.11"
+   "version": "3.11.4"
   }
  },
  "nbformat": 4,

diff --git a/docs/notebooks/2_getting_started.ipynb b/docs/notebooks/2_getting_started.ipynb
@@ -1270,7 +1270,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.10"
+   "version": "3.11.4"
   }
  },
  "nbformat": 4,

diff --git a/docs/notebooks/3_diagnostics.ipynb b/docs/notebooks/3_diagnostics.ipynb
@@ -399,7 +399,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.10"
+   "version": "3.11.4"
   }
  },
  "nbformat": 4,

diff --git a/docs/notebooks/4_ensemble_reduction.ipynb b/docs/notebooks/4_ensemble_reduction.ipynb
@@ -171,7 +171,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.10"
+   "version": "3.11.4"
   }
  },
  "nbformat": 4,

diff --git a/docs/notebooks/5_warminglevels.ipynb b/docs/notebooks/5_warminglevels.ipynb
@@ -460,7 +460,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.10"
+   "version": "3.11.4"
   }
  },
  "nbformat": 4,

diff --git a/docs/notebooks/6_config.ipynb b/docs/notebooks/6_config.ipynb
@@ -383,7 +383,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.10"
+   "version": "3.11.4"
   }
  },
  "nbformat": 4,

diff --git a/environment-dev.yml b/environment-dev.yml
@@ -2,14 +2,15 @@ name: xscen-dev
 channels:
   - conda-forge
 dependencies:
-  - python >=3.9,<3.11 # FIXME: CLISOPS is blocking Python3.11
+  - python >=3.9
   # Don't forget to sync the changes here with environment.yml!
-  # Some pins here are meant to accelerate conda and might not be related to real needs
+  # FIXME pins
+  - pydantic < 2  # breaks intake-esm, See: https://github.com/intake/intake-esm/issues/617
   # Main packages
   - cartopy
   - cftime
   - cf_xarray >=0.7.6
-  - clisops >=0.9.5
+  - clisops >=0.10
   - dask
   - flox
   - fsspec
@@ -20,9 +21,8 @@ dependencies:
   - matplotlib
   - netCDF4
   - numpy
-  - pandas !=1.5.3
+  - pandas >= 2
   - parse
-  - pydantic <2.0  # See: https://github.com/intake/intake-esm/issues/617
   - pygeos
   - pyyaml
   - rechunker

diff --git a/environment.yml b/environment.yml
@@ -2,13 +2,15 @@ name: xscen
 channels:
   - conda-forge
 dependencies:
-  - python >=3.9,<3.11 # FIXME: CLISOPS is blocking Python3.11
+  - python >=3.9
   # Don't forget to sync the changes here with environment-dev.yml!
+  # FIXME
+  - pydantic <2.0  # See: https://github.com/intake/intake-esm/issues/617
   # Main packages
   - cartopy
   - cftime
   - cf_xarray >=0.7.6
-  - clisops >=0.9.5
+  - clisops >=0.10
   - dask
   - flox
   - fsspec
@@ -19,9 +21,8 @@ dependencies:
   - matplotlib
   - netCDF4
   - numpy
-  - pandas !=1.5.3
+  - pandas >= 2
   - parse
-  - pydantic <2.0  # See: https://github.com/intake/intake-esm/issues/617
   - pygeos
   - pyyaml
   - rechunker

diff --git a/setup.py b/setup.py
@@ -1,9 +1,5 @@
 #!/usr/bin/env python
-
 """The setup script."""
-
-import re
-
 from setuptools import find_packages, setup
 
 with open("README.rst") as readme_file:
@@ -25,7 +21,7 @@
     "cartopy",
     "cftime",
     "cf_xarray>=0.7.6",
-    "clisops>=0.9.5",
+    "clisops>=0.10",
     "dask",
     "flox",
     "fsspec",
@@ -36,7 +32,7 @@
     "matplotlib",
     "netCDF4",
     "numpy",
-    "pandas!=1.5.3",
+    "pandas>=2",
     "parse",
     "pyarrow",  # Used when opening catalogs.
     "pydantic<2.0",  # See: https://github.com/intake/intake-esm/issues/617
@@ -45,7 +41,7 @@
     "rechunker",
     "shapely",
     "xarray",
-    "xclim>=0.37",
+    "xclim>=0.43",
     "xesmf>=0.7",
     "zarr",
 ]
@@ -63,7 +59,7 @@
         "Programming Language :: Python :: 3",
         "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
-        # "Programming Language :: Python :: 3.11",
+        "Programming Language :: Python :: 3.11",
         "Topic :: Scientific/Engineering :: Atmospheric Science",
     ],
     description="A climate change scenario-building analysis framework, built with xclim/xarray.",

diff --git a/tests/test_catutils.py b/tests/test_catutils.py
@@ -122,10 +122,10 @@ def test_parse_directory():
     assert (
         df[df["experiment"] != "ssp126"]["driving_model"].isnull().all()
     )  # CVS complex
-    assert df.date_start.dtype == "period[H]"
-    assert df.date_end.dtype == "period[H]"
+    assert df.date_start.dtype == "<M8[ms]"
+    assert df.date_end.dtype == "<M8[ms]"
     assert (
-        df[df["frequency"] == "day"]["date_end"] == pd.Period("2050-12-31", "H")
+        df[df["frequency"] == "day"]["date_end"] == pd.Timestamp("2050-12-31")
     ).all()  # Read from file
     # Read from file + attrs cvs
     assert set(
@@ -150,7 +150,7 @@ def test_parse_directory_readgroups():
     )
     assert len(df) == 10
     t2m = df.variable.apply(lambda v: "t2m" in v)
-    assert (df[t2m]["date_end"] == pd.Period("2050-12-31", "H")).all()
+    assert (df[t2m]["date_end"] == pd.Timestamp("2050-12-31")).all()
     assert (df[~t2m].variable.apply(len) == 0).all()
 
 

diff --git a/tests/test_utils.py b/tests/test_utils.py
@@ -0,0 +1,42 @@
+from datetime import datetime
+
+import cftime
+import numpy as np
+import pandas as pd
+import pytest
+
+import xscen as xs
+
+
+class TestDateParser:
+    @pytest.mark.parametrize(
+        "date,end_of_period,dtype,exp",
+        [
+            ("2001", True, "datetime", pd.Timestamp("2001-12-31")),
+            ("150004", True, "datetime", pd.Timestamp("1500-04-30")),
+            ("31231212", None, "datetime", pd.Timestamp("3123-12-12")),
+            ("2001-07-08", None, "period", pd.Period("2001-07-08", "H")),
+            (pd.Timestamp("1993-05-20T12:07"), None, "str", "1993-05-20"),
+            (
+                cftime.Datetime360Day(1981, 2, 30),
+                None,
+                "datetime",
+                pd.Timestamp("1981-02-28"),
+            ),
+            (np.datetime64("1200-11-12"), "Y", "datetime", pd.Timestamp("1200-12-31")),
+            (
+                datetime(2045, 5, 2, 13, 45),
+                None,
+                "datetime",
+                pd.Timestamp("2045-05-02T13:45"),
+            ),
+            ("abc", None, "datetime", pd.Timestamp("NaT")),
+            ("", True, "datetime", pd.Timestamp("NaT")),
+        ],
+    )
+    def test_normal(self, date, end_of_period, dtype, exp):
+        out = xs.utils.date_parser(date, end_of_period=end_of_period, out_dtype=dtype)
+        if pd.isna(exp):
+            assert pd.isna(out)
+        else:
+            assert out == exp