Merge bf29897 into 16a1a79

flexmeasures/data/models/charts/belief_charts.py

@@ -143,33 +143,35 @@ def matrix_chart(
                 event_ends_before.timestamp() * 10**3,
             ],
         }
+    mark = {"type": "rect", "clip": True, "opacity": 0.7}
+    tooltip = [
+        FIELD_DEFINITIONS["full_date"],
+        {
+            **event_value_field_definition,
+            **dict(title=f"{capitalize(sensor.sensor_type)}"),
+        },
+        FIELD_DEFINITIONS["source_name_and_id"],
+        FIELD_DEFINITIONS["source_model"],
+    ]
     chart_specs = {
         "description": "A simple heatmap chart showing sensor data.",
         # the sensor type is already shown as the y-axis title (avoid redundant info)
         "title": capitalize(sensor.name) if sensor.name != sensor.sensor_type else None,
         "layer": [
             {
-                "mark": {
-                    "type": "rect",
-                    "clip": True,
-                },
+                "mark": mark,
                 "encoding": {
                     "x": event_start_field_definition,
                     "y": event_start_date_field_definition,
                     "color": event_value_field_definition,
                     "detail": FIELD_DEFINITIONS["source"],
-                    "opacity": {"value": 0.7},
-                    "tooltip": [
-                        FIELD_DEFINITIONS["full_date"],
-                        {
-                            **event_value_field_definition,
-                            **dict(title=f"{capitalize(sensor.sensor_type)}"),
-                        },
-                        FIELD_DEFINITIONS["source_name_and_id"],
-                        FIELD_DEFINITIONS["source_model"],
-                    ],
+                    "tooltip": tooltip,
                 },
                 "transform": [
+                    {
+                        # Mask overlapping data during the fall DST transition, which we show later with a special layer
+                        "filter": "timezoneoffset(datum.event_start) >= timezoneoffset(datum.event_start + 60 * 60 * 1000) && timezoneoffset(datum.event_start) <= timezoneoffset(datum.event_start - 60 * 60 * 1000)"
+                    },
                     {
                         "calculate": "datum.source.name + ' (ID: ' + datum.source.id + ')'",
                         "as": "source_name_and_id",
@@ -189,6 +191,13 @@ def matrix_chart(
                     },
                 },
             },
+            create_fall_dst_transition_layer(
+                sensor.timezone,
+                mark,
+                event_value_field_definition,
+                event_start_field_definition,
+                tooltip,
+            ),
         ],
     }
     for k, v in override_chart_specs.items():
@@ -200,6 +209,60 @@ def matrix_chart(
     return chart_specs
 
 
+def create_fall_dst_transition_layer(
+    timezone, mark, event_value_field_definition, event_start_field_definition, tooltip
+) -> dict:
+    """Special layer for showing data during the daylight savings time transition in fall."""
+    return {
+        "mark": mark,
+        "encoding": {
+            "x": event_start_field_definition,
+            "y": {
+                "field": "dst_transition_event_start",
+                "type": "temporal",
+                "title": None,
+                "timeUnit": {"unit": "yearmonthdatehours", "step": 12},
+            },
+            "y2": {
+                "field": "dst_transition_event_start_next",
+                "timeUnit": {"unit": "yearmonthdatehours", "step": 12},
+            },
+            "color": event_value_field_definition,
+            "detail": FIELD_DEFINITIONS["source"],
+            "tooltip": [
+                {
+                    "field": "event_start",
+                    "type": "temporal",
+                    "title": "Timezone",
+                    "timeUnit": "utc",
+                    "format": [timezone],
+                    "formatType": "timezoneFormat",
+                },
+                *tooltip,
+            ],
+        },
+        "transform": [
+            {
+                "filter": "timezoneoffset(datum.event_start) < timezoneoffset(datum.event_start + 60 * 60 * 1000) || timezoneoffset(datum.event_start) > timezoneoffset(datum.event_start - 60 * 60 * 1000)"
+            },
+            {
+                # Push the more recent hour into the second 12-hour bin
+                "calculate": "timezoneoffset(datum.event_start + 60 * 60 * 1000) > timezoneoffset(datum.event_start) ? datum.event_start : datum.event_start + 12 * 60 * 60 * 1000",
+                "as": "dst_transition_event_start",
+            },
+            {
+                # Calculate a time point in the next 12-hour bin
+                "calculate": "datum.dst_transition_event_start + 12 * 60 * 60 * 1000 - 60 * 60 * 1000",
+                "as": "dst_transition_event_start_next",
+            },
+            {
+                "calculate": "datum.source.name + ' (ID: ' + datum.source.id + ')'",
+                "as": "source_name_and_id",
+            },
+        ],
+    }
+
+
 def chart_for_multiple_sensors(
     sensors_to_show: list["Sensor", list["Sensor"]],  # noqa F821
     event_starts_after: datetime | None = None,

flexmeasures/ui/static/css/flexmeasures.css

@@ -189,7 +189,7 @@ p.error {
     font-weight: 700;
 }
 
-#tzwarn {
+#tzwarn,#dstwarn {
     margin-top: 20px;
     margin-bottom: 20px;
     color: var(--secondary-color);

flexmeasures/ui/static/js/daterange-utils.js

@@ -38,4 +38,33 @@ function getTimeZoneOffset(date, timeZone) {
     // Positive values are West of GMT, opposite of ISO 8601
     // this matches the output of `Date.getTimeZoneOffset`
     return -(lie - date) / 60 / 1000;
-}
+}
+
+/**
+ * Count the number of Daylight Saving Time (DST) transitions within a given datetime range.
+ * @param {Date} startDate - The start date of the datetime range.
+ * @param {Date} endDate - The end date of the datetime range.
+ * @param {number} increment - The number of days to increment between iterations.
+ * @returns {number} The count of DST transitions within the specified range.
+ */
+export function countDSTTransitions(startDate, endDate, increment) {
+    let transitions = 0;
+    let currentDate = new Date(startDate);
+    let nextDate = new Date(startDate);
+
+    while (currentDate <= endDate) {
+        const currentOffset = currentDate.getTimezoneOffset();
+        nextDate.setDate(currentDate.getDate() + increment);
+        if (nextDate > endDate) {
+            nextDate = endDate;
+        }
+        const nextOffset = nextDate.getTimezoneOffset();
+
+        if (currentOffset !== nextOffset) {
+            transitions++;
+        }
+        currentDate.setDate(currentDate.getDate() + increment);
+    }
+
+    return transitions;
+}

flexmeasures/ui/static/js/flexmeasures.js

@@ -311,28 +311,31 @@ function submit_sensor_type() {
     $("#sensor_type-form").attr("action", empty_location).submit();
 }
 
-/** Tooltips: Register custom formatters
- *
- *          - Quantities incl. units
- *            Usage:
- *                 {
- *                     'format': [<d3-format>, <sensor unit>],
- *                     'formatType': 'quantityWithUnitFormat'
- *                 }
- *          - Timedeltas measured in human-readable quantities (usually not milliseconds)
- *            Usage:
- *                 {
- *                     'format': [<d3-format>, <breakpoint>],
- *                     'formatType': 'timedeltaFormat'
- *                 }
- *            <d3-format>  is a d3 format identifier, e.g. 'd' for decimal notation, rounded to integer.
- *                         See https://github.com/d3/d3-format for more details.
- *            <breakpoint> is a scalar that decides the breakpoint from one duration unit to the next larger unit.
-  *                        For example, a breakpoint of 4 means we format 4 days as '4 days', but 3.96 days as '95 hours'.
+/** Tooltips: Register custom formatters */
+
+/* Quantities incl. units
+ * Usage:
+ *     {
+ *         'format': [<d3-format>, <sensor unit>],
+ *         'formatType': 'quantityWithUnitFormat'
+ *     }
  */
 vega.expressionFunction('quantityWithUnitFormat', function(datum, params) {
     return d3.format(params[0])(datum) + " " + params[1];
 });
+
+/*
+ * Timedeltas measured in human-readable quantities (usually not milliseconds)
+ * Usage:
+ *     {
+ *         'format': [<d3-format>, <breakpoint>],
+ *         'formatType': 'timedeltaFormat'
+ *     }
+ * <d3-format>  is a d3 format identifier, e.g. 'd' for decimal notation, rounded to integer.
+ *              See https://github.com/d3/d3-format for more details.
+ * <breakpoint> is a scalar that decides the breakpoint from one duration unit to the next larger unit.
+ *              For example, a breakpoint of 4 means we format 4 days as '4 days', but 3.96 days as '95 hours'.
+ */
 vega.expressionFunction('timedeltaFormat', function(timedelta, params) {
     return (Math.abs(timedelta) > 1000 * 60 * 60 * 24 * 365.2425 * params[1] ? d3.format(params[0])(timedelta / (1000 * 60 * 60 * 24 * 365.2425)) + " years"
         : Math.abs(timedelta) > 1000 * 60 * 60 * 24 * params[1] ? d3.format(params[0])(timedelta / (1000 * 60 * 60 * 24)) + " days"
@@ -341,3 +344,18 @@ vega.expressionFunction('timedeltaFormat', function(timedelta, params) {
         : Math.abs(timedelta) > 1000 * params[1] ? d3.format(params[0])(timedelta / 1000) + " seconds"
         : d3.format(params[0])(timedelta) + " milliseconds");
 });
+
+/*
+ * Timezone offset including IANA timezone name
+ * Usage:
+ *     {
+ *         'format': [<IANA timezone name, e.g. 'Europe/Amsterdam'>],
+ *         'formatType': 'timezoneFormat'
+ *     }
+ */
+vega.expressionFunction('timezoneFormat', function(date, params) {
+    const timezoneString = params[0];
+    const tzOffsetNumber = date.getTimezoneOffset();
+    const tzDate = new Date(0,0,0,0,Math.abs(tzOffsetNumber));
+    return `${ tzOffsetNumber > 0 ? '-' : '+'}${("" + tzDate.getHours()).padStart(2, '0')}:${("" + tzDate.getMinutes()).padStart(2, '0')}` + ' (' + timezoneString + ')';
+});

flexmeasures/ui/templates/base.html

@@ -230,7 +230,7 @@
     <!-- Render Charts -->
     <script type="module" type="text/javascript">
 
-    import { subtract, thisMonth, lastNMonths, getOffsetBetweenTimezonesForDate } from "{{ url_for('flexmeasures_ui.static', filename='js/daterange-utils.js') }}";
+    import { subtract, thisMonth, lastNMonths, countDSTTransitions, getOffsetBetweenTimezonesForDate } from "{{ url_for('flexmeasures_ui.static', filename='js/daterange-utils.js') }}";
     import { partition, updateBeliefs, beliefTimedelta, setAbortableTimeout} from "{{ url_for('flexmeasures_ui.static', filename='js/replay-utils.js') }}";
 
     let vegaView;
@@ -242,6 +242,7 @@
     {% if event_starts_after and event_ends_before %}
         storeStartDate = new Date('{{ event_starts_after }}');
         storeEndDate = new Date('{{ event_ends_before }}');
+        checkDSTTransitions(storeStartDate, storeEndDate);
     {% endif %}
     let replaySpeed = 100
     let chartType = 'bar_chart';  // initial chart type // todo: get from session?
@@ -277,6 +278,21 @@
         }
     });
 
+    function checkDSTTransitions(startDate, endDate) {
+        var numDSTTransitions = countDSTTransitions(startDate, endDate, 90)
+        if (numDSTTransitions != 0) {
+            document.getElementById('dstwarn').style.display = 'block';
+            if (numDSTTransitions == 1) {
+                document.getElementById('dstwarn').innerHTML = 'Please note that the sensor data you are viewing includes a daylight saving time (DST) transition.';
+            } else {
+                document.getElementById('dstwarn').innerHTML = 'Please note that the sensor data you are viewing includes ' + numDSTTransitions + ' daylight saving time (DST) transitions.';
+            }
+        }
+        else {
+            document.getElementById('dstwarn').style.display = 'none';
+        }
+    }
+
     async function embedAndLoad(chartSpecsPath, elementId, datasetName, previousResult, startDate, endDate) {
 
         await vegaEmbed('#'+elementId, chartSpecsPath + 'dataset_name=' + datasetName + '&width=container&include_sensor_annotations=false&include_asset_annotations=false&chart_type=' + chartType, {{ chart_options | safe }})
@@ -369,6 +385,7 @@
         stopReplay()
 
         $("#spinner").show();
+        checkDSTTransitions(startDate, endDate)
         Promise.all([
             // Fetch time series data
             fetch(dataPath + '/chart_data/?event_starts_after=' + queryStartDate + '&event_ends_before=' + queryEndDate, {
@@ -428,8 +445,8 @@
                 var offsetDifference = getOffsetBetweenTimezonesForDate(new Date(), data.timezone, jstz.determine().name());
                 if (offsetDifference != 0) {
                     document.getElementById('tzwarn').style.display = 'block';
-                    var offsetNotice = (offsetDifference > 0) ? 'which is currently ahead by ' + offsetDifference + 'minutes' : 'which is currently behind by ' + offsetDifference + ' minutes'
-                    document.getElementById('tzwarn').innerHTML = 'Please note that the sensor data you are viewing is located in a different timezone (' + offsetNotice + ').<br/>To view the data from a local perspective, set your locale timezone to ' + data.timezone + '.'
+                    var offsetNotice = (offsetDifference > 0) ? 'which is currently ahead by ' + offsetDifference + 'minutes' : 'which is currently behind by ' + offsetDifference + ' minutes';
+                    document.getElementById('tzwarn').innerHTML = 'Please note that the sensor data you are viewing is located in a different timezone (' + offsetNotice + ').<br/>To view the data from a local perspective, set your locale timezone to ' + data.timezone + '.';
                 }
                 {% if active_page == "assets" %}
                     var timerangeVar = 'timerange_of_sensors_to_show';

flexmeasures/ui/templates/crud/asset.html

@@ -7,7 +7,10 @@
 {% block divs %}
 
 <div class="container-fluid">
-    <div class="row"><div class="alert alert-info" id="tzwarn" style="display:none;"></div></div>
+    <div class="row">
+        <div class="alert alert-info" id="tzwarn" style="display:none;"></div>
+        <div class="alert alert-info" id="dstwarn" style="display:none;"></div>
+    </div>
     <div class="row">
         <div class="col-sm-2 on-top-md">
             <div class="header-action-button">

flexmeasures/ui/templates/views/sensors.html

@@ -7,7 +7,10 @@
 {% block divs %}
 
       <div class="sensor-data charts text-center">
-          <div class="row"><div class="alert alert-info" id="tzwarn" style="display:none;"></div></div>
+          <div class="row">
+              <div class="alert alert-info" id="tzwarn" style="display:none;"></div>
+              <div class="alert alert-info" id="dstwarn" style="display:none;"></div>
+          </div>
           <div class="row on-top-md">
               <div class="col-sm-2">
                   <div class="sidepanel-container">