Merge 4777dca into 79c85e4

flexmeasures/utils/tests/test_unit_utils.py

@@ -4,6 +4,7 @@
 import pytest
 
 import pandas as pd
+import timely_beliefs as tb
 
 from flexmeasures.utils.unit_utils import (
     convert_units,
@@ -64,6 +65,115 @@ def test_convert_unit(
     pd.testing.assert_series_equal(converted_data, expected_data)
 
 
+@pytest.mark.parametrize(
+    "from_unit, to_unit, timezone, input_values, expected_values",
+    [
+        # datetimes are converted to seconds since UNIX epoch
+        (
+            "datetime",
+            "s",
+            None,
+            ["1970-01-01", "1970-01-02", "1970-01-03"],
+            [0, 60 * 60 * 24, 60 * 60 * 48],
+        ),
+        # nothing overflows for the next 100 years
+        (
+            "datetime",
+            "s",
+            None,
+            ["2123-05-02", "2123-05-03", "2123-05-04"],
+            [4838659200, 4838659200 + 60 * 60 * 24, 4838659200 + 60 * 60 * 48],
+        ),
+        # Same as above, but day precedes month in input
+        (
+            "dayfirst datetime",
+            "s",
+            None,
+            ["02-05-2123", "03-05-2123", "04-05-2123"],
+            [4838659200, 4838659200 + 60 * 60 * 24, 4838659200 + 60 * 60 * 48],
+        ),
+        # Localize timezone-naive datetimes to UTC in case there is no sensor information available
+        (
+            "datetime",
+            "s",
+            None,
+            ["2023-05-02 00:00:01", "2023-05-02 00:00:02", "2023-05-02 00:00:03"],
+            [1682985601, 1682985602, 1682985603],
+        ),
+        # Localize timezone-naive datetimes to sensor's timezone in case that is available
+        (
+            "datetime",
+            "s",
+            "Europe/Amsterdam",
+            ["2023-05-02 00:00:01", "2023-05-02 00:00:02", "2023-05-02 00:00:03"],
+            [
+                1682985601 - 60 * 60 * 2,
+                1682985602 - 60 * 60 * 2,
+                1682985603 - 60 * 60 * 2,
+            ],
+        ),
+        # Timezone-aware datetimes work don't require localization
+        (
+            "datetime",
+            "s",
+            None,
+            [
+                "2023-05-02 00:00:01 +02:00",
+                "2023-05-02 00:00:02 +02:00",
+                "2023-05-02 00:00:03 +02:00",
+            ],
+            [
+                1682985601 - 60 * 60 * 2,
+                1682985602 - 60 * 60 * 2,
+                1682985603 - 60 * 60 * 2,
+            ],
+        ),
+        # Timezone-aware datetimes also means that the sensor timezone is irrelevant
+        (
+            "datetime",
+            "s",
+            "Asia/Seoul",
+            [
+                "2023-05-02 00:00:01 +02:00",
+                "2023-05-02 00:00:02 +02:00",
+                "2023-05-02 00:00:03 +02:00",
+            ],
+            [
+                1682985601 - 60 * 60 * 2,
+                1682985602 - 60 * 60 * 2,
+                1682985603 - 60 * 60 * 2,
+            ],
+        ),
+        # Timedeltas can be converted to units of time
+        ("timedelta", "s", None, ["1 minute", "1 minute 2 seconds"], [60.0, 62.0]),
+        # Convertible timedeltas include absolute days of 24 hours
+        ("timedelta", "d", None, ["1 day", "1 day 12 hours"], [1.0, 1.5]),
+        # Convertible timedeltas exclude nominal durations like month or year, which cannot be represented as a datetime.timedelta object
+        # ("timedelta", "d", None, ["1 month", "1 year"], [30., 365.]),  # fails
+    ],
+)
+def test_convert_special_unit(
+    from_unit,
+    to_unit,
+    timezone,
+    input_values,
+    expected_values,
+):
+    """Check some special unit conversions."""
+    data = pd.Series(input_values)
+    if timezone:
+        data.sensor = tb.Sensor("test", timezone=timezone)
+    converted_data: pd.Series = convert_units(
+        data=data,
+        from_unit=from_unit,
+        to_unit=to_unit,
+    )
+    print(converted_data)
+    expected_data = pd.Series(expected_values)
+    print(expected_data)
+    pd.testing.assert_series_equal(converted_data, expected_data)
+
+
 @pytest.mark.parametrize(
     "unit, time_unit, expected_unit",
     [

flexmeasures/utils/unit_utils.py

@@ -239,10 +239,15 @@ def _convert_time_units(
     if not to_unit[0].isdigit():
         # unit abbreviations passed to pd.Timedelta need a number (so, for example, h becomes 1h)
         to_unit = f"1{to_unit}"
-    if from_unit == "datetime":
-        return (
-            pd.to_datetime(data, utc=True) - pd.Timestamp("1970-01-01", tz="utc")
-        ) // pd.Timedelta(to_unit)
+    if "datetime" in from_unit:
+        dt_data = pd.to_datetime(
+            data, dayfirst=True if "dayfirst" in from_unit else False
+        )
+        # localize timezone naive data to the sensor's timezone, if available
+        if dt_data.dt.tz is None:
+            timezone = data.sensor.timezone if hasattr(data, "sensor") else "utc"
+            dt_data = dt_data.dt.tz_localize(timezone)
+        return (dt_data - pd.Timestamp("1970-01-01", tz="utc")) // pd.Timedelta(to_unit)
     else:
         return data / pd.Timedelta(to_unit)
 
@@ -254,8 +259,14 @@ def convert_units(
     event_resolution: timedelta | None = None,
     capacity: str | None = None,
 ) -> pd.Series | list[int | float] | int | float:
-    """Updates data values to reflect the given unit conversion."""
-    if from_unit in ("datetime", "timedelta"):
+    """Updates data values to reflect the given unit conversion.
+
+    Handles units in short scientific notation (e.g. m³/h, kW, and ºC), as well as three special units to convert from:
+    - from_unit="datetime"          (with data point such as "2023-05-02", "2023-05-02 05:14:49" or "2023-05-02 05:14:49 +02:00")
+    - from_unit="dayfirst datetime" (with data point such as "02-05-2023")
+    - from_unit="timedelta"         (with data point such as "0 days 01:18:25")
+    """
+    if from_unit in ("datetime", "dayfirst datetime", "timedelta"):
         return _convert_time_units(data, from_unit, to_unit)
 
     if from_unit != to_unit: