Merge branch 'main' into documentation/shiftable_load_tutorial

documentation/changelog.rst

@@ -14,6 +14,7 @@ New features
 -------------
 
 * Users can select a new chart type (daily heatmap) on the sensor page of the UI, showing how sensor values are distributed over the time of day [see `PR #715 <https://www.github.com/FlexMeasures/flexmeasures/pull/715>`_]
+* Users are warned in the UI on when the data they are seeing includes one or more Daylight Saving Time (DST) transitions, and heatmaps (see previous feature) visualize these transitions intuitively [see `PR #723 <https://www.github.com/FlexMeasures/flexmeasures/pull/723>`_]
 * Allow deleting multiple sensors with a single call to ``flexmeasures delete sensor`` by passing the ``--id`` option multiple times [see `PR #734 <https://www.github.com/FlexMeasures/flexmeasures/pull/734>`_]
 * Make it a lot easier to read off the color legend on the asset page, especially when showing many sensors, as they will now be ordered from top to bottom in the same order as they appear in the chart (as defined in the ``sensors_to_show`` attribute), rather than alphabetically [see `PR #742 <https://www.github.com/FlexMeasures/flexmeasures/pull/742>`_]
 * Users on FlexMeasures servers in play mode (``FLEXMEASURES_MODE = "play"``) can use the ``sensors_to_show`` attribute to show any sensor on their asset pages, rather than only sensors registered to assets in their own account or to public assets [see `PR #740 <https://www.github.com/FlexMeasures/flexmeasures/pull/740>`_]

flexmeasures/data/models/charts/belief_charts.py

@@ -100,7 +100,7 @@ def daily_heatmap(
         formatType="quantityWithUnitFormat",
         stack=None,
         **FIELD_DEFINITIONS["event_value"],
-        scale={"scheme": "blueorange", "domainMid": 0},
+        scale={"scheme": "blueorange", "domainMid": 0, "domain": {"unionWith": [0]}},
     )
     event_start_field_definition = dict(
         field="event_start",
@@ -147,33 +147,35 @@ def daily_heatmap(
                 event_ends_before.timestamp() * 10**3,
             ],
         }
+    mark = {"type": "rect", "clip": True, "opacity": 0.7}
+    tooltip = [
+        FIELD_DEFINITIONS["full_date"],
+        {
+            **event_value_field_definition,
+            **dict(title=f"{capitalize(sensor.sensor_type)}"),
+        },
+        FIELD_DEFINITIONS["source_name_and_id"],
+        FIELD_DEFINITIONS["source_model"],
+    ]
     chart_specs = {
         "description": "A daily heatmap showing sensor data.",
         # the sensor type is already shown as the y-axis title (avoid redundant info)
         "title": capitalize(sensor.name) if sensor.name != sensor.sensor_type else None,
         "layer": [
             {
-                "mark": {
-                    "type": "rect",
-                    "clip": True,
-                },
+                "mark": mark,
                 "encoding": {
                     "x": event_start_field_definition,
                     "y": event_start_date_field_definition,
                     "color": event_value_field_definition,
                     "detail": FIELD_DEFINITIONS["source"],
-                    "opacity": {"value": 0.7},
-                    "tooltip": [
-                        FIELD_DEFINITIONS["full_date"],
-                        {
-                            **event_value_field_definition,
-                            **dict(title=f"{capitalize(sensor.sensor_type)}"),
-                        },
-                        FIELD_DEFINITIONS["source_name_and_id"],
-                        FIELD_DEFINITIONS["source_model"],
-                    ],
+                    "tooltip": tooltip,
                 },
                 "transform": [
+                    {
+                        # Mask overlapping data during the fall DST transition, which we show later with a special layer
+                        "filter": "timezoneoffset(datum.event_start) >= timezoneoffset(datum.event_start + 60 * 60 * 1000) && timezoneoffset(datum.event_start) <= timezoneoffset(datum.event_start - 60 * 60 * 1000)"
+                    },
                     {
                         "calculate": "datum.source.name + ' (ID: ' + datum.source.id + ')'",
                         "as": "source_name_and_id",
@@ -227,6 +229,13 @@ def daily_heatmap(
                     },
                 },
             },
+            create_fall_dst_transition_layer(
+                sensor.timezone,
+                mark,
+                event_value_field_definition,
+                event_start_field_definition,
+                tooltip,
+            ),
         ],
     }
     for k, v in override_chart_specs.items():
@@ -238,6 +247,60 @@ def daily_heatmap(
     return chart_specs
 
 
+def create_fall_dst_transition_layer(
+    timezone, mark, event_value_field_definition, event_start_field_definition, tooltip
+) -> dict:
+    """Special layer for showing data during the daylight savings time transition in fall."""
+    return {
+        "mark": mark,
+        "encoding": {
+            "x": event_start_field_definition,
+            "y": {
+                "field": "dst_transition_event_start",
+                "type": "temporal",
+                "title": None,
+                "timeUnit": {"unit": "yearmonthdatehours", "step": 12},
+            },
+            "y2": {
+                "field": "dst_transition_event_start_next",
+                "timeUnit": {"unit": "yearmonthdatehours", "step": 12},
+            },
+            "color": event_value_field_definition,
+            "detail": FIELD_DEFINITIONS["source"],
+            "tooltip": [
+                {
+                    "field": "event_start",
+                    "type": "temporal",
+                    "title": "Timezone",
+                    "timeUnit": "utc",
+                    "format": [timezone],
+                    "formatType": "timezoneFormat",
+                },
+                *tooltip,
+            ],
+        },
+        "transform": [
+            {
+                "filter": "timezoneoffset(datum.event_start) < timezoneoffset(datum.event_start + 60 * 60 * 1000) || timezoneoffset(datum.event_start) > timezoneoffset(datum.event_start - 60 * 60 * 1000)"
+            },
+            {
+                # Push the more recent hour into the second 12-hour bin
+                "calculate": "timezoneoffset(datum.event_start + 60 * 60 * 1000) > timezoneoffset(datum.event_start) ? datum.event_start : datum.event_start + 12 * 60 * 60 * 1000",
+                "as": "dst_transition_event_start",
+            },
+            {
+                # Calculate a time point in the next 12-hour bin
+                "calculate": "datum.dst_transition_event_start + 12 * 60 * 60 * 1000 - 60 * 60 * 1000",
+                "as": "dst_transition_event_start_next",
+            },
+            {
+                "calculate": "datum.source.name + ' (ID: ' + datum.source.id + ')'",
+                "as": "source_name_and_id",
+            },
+        ],
+    }
+
+
 def chart_for_multiple_sensors(
     sensors_to_show: list["Sensor", list["Sensor"]],  # noqa F821
     event_starts_after: datetime | None = None,

flexmeasures/ui/static/css/flexmeasures.css

@@ -189,7 +189,7 @@ p.error {
     font-weight: 700;
 }
 
-#tzwarn,#sourcewarn {
+#tzwarn,#dstwarn,#sourcewarn {
     margin-top: 20px;
     margin-bottom: 20px;
     color: var(--secondary-color);

flexmeasures/ui/static/js/daterange-utils.js

@@ -38,4 +38,33 @@ function getTimeZoneOffset(date, timeZone) {
     // Positive values are West of GMT, opposite of ISO 8601
     // this matches the output of `Date.getTimeZoneOffset`
     return -(lie - date) / 60 / 1000;
-}
+}
+
+/**
+ * Count the number of Daylight Saving Time (DST) transitions within a given datetime range.
+ * @param {Date} startDate - The start date of the datetime range.
+ * @param {Date} endDate - The end date of the datetime range.
+ * @param {number} increment - The number of days to increment between iterations.
+ * @returns {number} The count of DST transitions within the specified range.
+ */
+export function countDSTTransitions(startDate, endDate, increment) {
+    let transitions = 0;
+    let currentDate = new Date(startDate);
+    let nextDate = new Date(startDate);
+
+    while (currentDate <= endDate) {
+        const currentOffset = currentDate.getTimezoneOffset();
+        nextDate.setDate(currentDate.getDate() + increment);
+        if (nextDate > endDate) {
+            nextDate = endDate;
+        }
+        const nextOffset = nextDate.getTimezoneOffset();
+
+        if (currentOffset !== nextOffset) {
+            transitions++;
+        }
+        currentDate.setDate(currentDate.getDate() + increment);
+    }
+
+    return transitions;
+}

flexmeasures/ui/static/js/flexmeasures.js

@@ -311,28 +311,31 @@ function submit_sensor_type() {
     $("#sensor_type-form").attr("action", empty_location).submit();
 }
 
-/** Tooltips: Register custom formatters
- *
- *          - Quantities incl. units
- *            Usage:
- *                 {
- *                     'format': [<d3-format>, <sensor unit>],
- *                     'formatType': 'quantityWithUnitFormat'
- *                 }
- *          - Timedeltas measured in human-readable quantities (usually not milliseconds)
- *            Usage:
- *                 {
- *                     'format': [<d3-format>, <breakpoint>],
- *                     'formatType': 'timedeltaFormat'
- *                 }
- *            <d3-format>  is a d3 format identifier, e.g. 'd' for decimal notation, rounded to integer.
- *                         See https://github.com/d3/d3-format for more details.
- *            <breakpoint> is a scalar that decides the breakpoint from one duration unit to the next larger unit.
-  *                        For example, a breakpoint of 4 means we format 4 days as '4 days', but 3.96 days as '95 hours'.
+/** Tooltips: Register custom formatters */
+
+/* Quantities incl. units
+ * Usage:
+ *     {
+ *         'format': [<d3-format>, <sensor unit>],
+ *         'formatType': 'quantityWithUnitFormat'
+ *     }
  */
 vega.expressionFunction('quantityWithUnitFormat', function(datum, params) {
     return d3.format(params[0])(datum) + " " + params[1];
 });
+
+/*
+ * Timedeltas measured in human-readable quantities (usually not milliseconds)
+ * Usage:
+ *     {
+ *         'format': [<d3-format>, <breakpoint>],
+ *         'formatType': 'timedeltaFormat'
+ *     }
+ * <d3-format>  is a d3 format identifier, e.g. 'd' for decimal notation, rounded to integer.
+ *              See https://github.com/d3/d3-format for more details.
+ * <breakpoint> is a scalar that decides the breakpoint from one duration unit to the next larger unit.
+ *              For example, a breakpoint of 4 means we format 4 days as '4 days', but 3.96 days as '95 hours'.
+ */
 vega.expressionFunction('timedeltaFormat', function(timedelta, params) {
     return (Math.abs(timedelta) > 1000 * 60 * 60 * 24 * 365.2425 * params[1] ? d3.format(params[0])(timedelta / (1000 * 60 * 60 * 24 * 365.2425)) + " years"
         : Math.abs(timedelta) > 1000 * 60 * 60 * 24 * params[1] ? d3.format(params[0])(timedelta / (1000 * 60 * 60 * 24)) + " days"
@@ -341,3 +344,18 @@ vega.expressionFunction('timedeltaFormat', function(timedelta, params) {
         : Math.abs(timedelta) > 1000 * params[1] ? d3.format(params[0])(timedelta / 1000) + " seconds"
         : d3.format(params[0])(timedelta) + " milliseconds");
 });
+
+/*
+ * Timezone offset including IANA timezone name
+ * Usage:
+ *     {
+ *         'format': [<IANA timezone name, e.g. 'Europe/Amsterdam'>],
+ *         'formatType': 'timezoneFormat'
+ *     }
+ */
+vega.expressionFunction('timezoneFormat', function(date, params) {
+    const timezoneString = params[0];
+    const tzOffsetNumber = date.getTimezoneOffset();
+    const tzDate = new Date(0,0,0,0,Math.abs(tzOffsetNumber));
+    return `${ tzOffsetNumber > 0 ? '-' : '+'}${("" + tzDate.getHours()).padStart(2, '0')}:${("" + tzDate.getMinutes()).padStart(2, '0')}` + ' (' + timezoneString + ')';
+});

flexmeasures/ui/templates/base.html

@@ -234,7 +234,7 @@
     <script type="module" type="text/javascript">
 
     import { getUniqueValues } from "{{ url_for('flexmeasures_ui.static', filename='js/data-utils.js') }}";
-    import { subtract, thisMonth, lastNMonths, getOffsetBetweenTimezonesForDate } from "{{ url_for('flexmeasures_ui.static', filename='js/daterange-utils.js') }}";
+    import { subtract, thisMonth, lastNMonths, countDSTTransitions, getOffsetBetweenTimezonesForDate } from "{{ url_for('flexmeasures_ui.static', filename='js/daterange-utils.js') }}";
     import { partition, updateBeliefs, beliefTimedelta, setAbortableTimeout} from "{{ url_for('flexmeasures_ui.static', filename='js/replay-utils.js') }}";
 
     let vegaView;
@@ -246,6 +246,7 @@
     {% if event_starts_after and event_ends_before %}
         storeStartDate = new Date('{{ event_starts_after }}');
         storeEndDate = new Date('{{ event_ends_before }}');
+        checkDSTTransitions(storeStartDate, storeEndDate);
     {% endif %}
     let replaySpeed = 100
     let chartType = '{{ chart_type }}';  // initial chart type from session variable
@@ -281,6 +282,21 @@
         }
     });
 
+    function checkDSTTransitions(startDate, endDate) {
+        var numDSTTransitions = countDSTTransitions(startDate, endDate, 90)
+        if (numDSTTransitions != 0) {
+            document.getElementById('dstwarn').style.display = 'block';
+            if (numDSTTransitions == 1) {
+                document.getElementById('dstwarn').innerHTML = 'Please note that the sensor data you are viewing includes a daylight saving time (DST) transition.';
+            } else {
+                document.getElementById('dstwarn').innerHTML = 'Please note that the sensor data you are viewing includes ' + numDSTTransitions + ' daylight saving time (DST) transitions.';
+            }
+        }
+        else {
+            document.getElementById('dstwarn').style.display = 'none';
+        }
+    }
+
     function checkSourceMasking(data) {
         var uniqueSourceIds = getUniqueValues(data, 'source.id');
         if (chartType == 'daily_heatmap' && uniqueSourceIds.length > 1) {
@@ -385,6 +401,7 @@
         stopReplay()
 
         $("#spinner").show();
+        checkDSTTransitions(startDate, endDate)
         Promise.all([
             // Fetch time series data
             fetch(dataPath + '/chart_data/?event_starts_after=' + queryStartDate + '&event_ends_before=' + queryEndDate, {
@@ -446,8 +463,8 @@
                 var offsetDifference = getOffsetBetweenTimezonesForDate(new Date(), data.timezone, jstz.determine().name());
                 if (offsetDifference != 0) {
                     document.getElementById('tzwarn').style.display = 'block';
-                    var offsetNotice = (offsetDifference > 0) ? 'which is currently ahead by ' + offsetDifference + 'minutes' : 'which is currently behind by ' + offsetDifference + ' minutes'
-                    document.getElementById('tzwarn').innerHTML = 'Please note that the sensor data you are viewing is located in a different timezone (' + offsetNotice + ').<br/>To view the data from a local perspective, set your locale timezone to ' + data.timezone + '.'
+                    var offsetNotice = (offsetDifference > 0) ? 'which is currently ahead by ' + offsetDifference + 'minutes' : 'which is currently behind by ' + offsetDifference + ' minutes';
+                    document.getElementById('tzwarn').innerHTML = 'Please note that the sensor data you are viewing is located in a different timezone (' + offsetNotice + ').<br/>To view the data from a local perspective, set your locale timezone to ' + data.timezone + '.';
                 }
                 {% if active_page == "assets" %}
                     var timerangeVar = 'timerange_of_sensors_to_show';

flexmeasures/ui/templates/crud/asset.html

@@ -7,7 +7,10 @@
 {% block divs %}
 
 <div class="container-fluid">
-    <div class="row"><div class="alert alert-info" id="tzwarn" style="display:none;"></div></div>
+    <div class="row">
+        <div class="alert alert-info" id="tzwarn" style="display:none;"></div>
+        <div class="alert alert-info" id="dstwarn" style="display:none;"></div>
+    </div>
     <div class="row">
         <div class="col-sm-2 on-top-md">
             <div class="header-action-button">

flexmeasures/ui/templates/views/sensors.html

@@ -9,6 +9,7 @@
       <div class="sensor-data charts text-center">
           <div class="row">
               <div class="alert alert-info" id="tzwarn" style="display:none;"></div>
+              <div class="alert alert-info" id="dstwarn" style="display:none;"></div>
               <div class="alert alert-info" id="sourcewarn" style="display:none;"></div>
           </div>
           <div class="row on-top-md">