Merge branch 'dev/v5.8' into v5

dsmrreader · Sep 30, 2022 · 0d4cb9c · 0d4cb9c
2 parents 474b9a7 + 1f8ccee
commit 0d4cb9c
Show file tree

Hide file tree

Showing 10 changed files with 152 additions and 129 deletions.
diff --git a/.github/workflows/dependency-update-check.yml b/.github/workflows/dependency-update-check.yml
diff --git a/docs/reference/changelog.rst b/docs/reference/changelog.rst
@@ -17,6 +17,18 @@ Current version
     ℹ️ :doc:`How to update</how-to/upgrading/upgrade>` *(minor updates only)*
 
 
+v5.8.0 - 2022-10-01
+-------------------
+
+.. attention::
+
+    This release fixes a four year old bug that may have disrupted the order of your MQTT messages sent by DSMR-reader.
+
+    It only affected installations with either a high throughput of data or a delayed backend process (or both).
+    You probably only have noticed it when running an installation similar to the one above, along using per-topic data sources.
+
+- ``Fixed`` Outgoing MQTT message queue not maintaining its own order [`#1714 <https://github.com/dsmrreader/dsmr-reader/issues/1714>`_]
+
 
 v5.7.0 - 2022-09-27
 -------------------

diff --git a/dsmr_frontend/templates/dsmr_frontend/about.html b/dsmr_frontend/templates/dsmr_frontend/about.html
@@ -75,7 +75,7 @@
                             {% translate "Looking for something specific? A good place to start might be the FAQ" %}:
                         </div>
                         <div class="col-sm-6">
-                            <a href="{% url 'frontend:docs-redirect' %}" target="_blank" rel="noopener noreferrer">{% translate "View frequently asked questions (FAQ)" %} </a>
+                            <i class="fa-regular fa-circle-question"></i> &nbsp; <a href="{% url 'frontend:docs-redirect' %}" target="_blank" rel="noopener noreferrer">{% translate "View frequently asked questions (FAQ)" %} </a>
                         </div>
                     </div>
                     <div class="row">&nbsp;</div>
@@ -84,7 +84,7 @@
                             {% translate "Did something change after updating? Check out the latest changes" %}:
                         </div>
                         <div class="col-sm-6">
-                            <a href="{% url 'frontend:changelog-redirect' %}" target="_blank" rel="noopener noreferrer">{% translate "View changelog" %} </a>
+                            <i class="fa-solid fa-arrows-turn-to-dots"></i> &nbsp; <a href="{% url 'frontend:changelog-redirect' %}" target="_blank" rel="noopener noreferrer">{% translate "View changelog" %} </a>
                         </div>
                     </div>
                     <div class="row">&nbsp;</div>
@@ -93,7 +93,7 @@
                             <small>{% translate "Still need troubleshooting? Check out current issues on GitHub" %}:</small>
                         </div>
                         <div class="col-sm-6">
-                            <a href="https://github.com/dsmrreader/dsmr-reader/issues" target="_blank" rel="noopener noreferrer">{% translate "View DSMR-reader issues on GitHub (or create one)" %} </a>
+                            <i class="fa-solid fa-bullhorn"></i> &nbsp; <a href="https://github.com/dsmrreader/dsmr-reader/issues" target="_blank" rel="noopener noreferrer">{% translate "View DSMR-reader issues on GitHub (or create one)" %} </a>
                         </div>
                     </div>
                     <div class="row">&nbsp;</div>
@@ -102,6 +102,8 @@
                             <small>{% translate "For new GitHub issues, please supply a debug info dump as well:" %}</small>
                         </div>
                         <div class="col-sm-6">
+                            <i class="fa-solid fa-stethoscope"></i>
+                            &nbsp;
                             {% if user.is_authenticated %}
                             <a class="btn btn-info trigger-debug-info" href="#">{% translate "View 'dsmr_debuginfo' dump" %}</a>
                             {% else %}
@@ -122,7 +124,7 @@
                         </div>
 
                         <div class="col-sm-6">
-                            <a href="{% url 'frontend:redoc-api-docs' %}" target="_blank" rel="noopener noreferrer">{% translate "View API docs" %}</a>
+                            <i class="fa-solid fa-robot"></i> &nbsp; <a href="{% url 'frontend:redoc-api-docs' %}" target="_blank" rel="noopener noreferrer">{% translate "View API docs" %}</a>
                         </div>
                     </div>
                 </div>
@@ -142,12 +144,23 @@
                             {% translate "Originally created and authored by" %}
                         </div>
                         <div class="col-sm-6">
+                            <i class="fa-solid fa-user-ninja">
+                                &nbsp;
                             <a href="https://linkedin.com/in/dennissiemensma" target="_blank">
-                                Dennis Siemensma&nbsp; <i class="far fa-xs fa-copyright"></i> 2015 - {% now "Y" %}
+                                </i>Dennis Siemensma&nbsp; <i class="far fa-xs fa-copyright"></i> 2015 - {% now "Y" %}
                             </a>
+                        </div>
+                    </div>
+                    <div class="row">&nbsp;</div>
+                    <div class="row">
+                        <div class="col-sm-6">
+                            &nbsp;
+                        </div>
+                        <div class="col-sm-6">
+                            <i class="fas fa-heart"></i>
                             &nbsp;
                             <a href="https://dsmr-reader.readthedocs.io/{{ LANGUAGE_CODE }}/{{ DSMRREADER_MAIN_BRANCH }}/how-to/donate/thanks.html" target="_blank" rel="noopener noreferrer">
-                                <span class="unit">{% translate "want to say thanks?" %}</span>
+                                {% translate "Want to say thanks?" %}
                             </a>
                         </div>
                     </div>
@@ -157,8 +170,10 @@
                             {% translate "Improved by" %}
                         </div>
                         <div class="col-sm-6">
+                            <i class="fa-solid fa-people-group"></i>
+                            &nbsp;
                             <a href="https://dsmr-reader.readthedocs.io/{{ LANGUAGE_CODE }}/{{ DSMRREADER_MAIN_BRANCH }}/credits.html" target="_blank" rel="noopener noreferrer">
-                                {% translate "Many contributors" %} <i class="fas fa-heart"></i>
+                                {% translate "Many contributors" %}
                             </a>
                         </div>
                     </div>

diff --git a/dsmr_mqtt/services/broker.py b/dsmr_mqtt/services/broker.py
@@ -70,10 +70,29 @@ def initialize_client() -> Optional[paho.Client]:
 def run(mqtt_client: paho.Client) -> None:
     """Reads any messages from the queue and publishing them to the MQTT broker."""
 
-    # Keep batches small, only send the latest X messages. The rest will be trimmed (in case of delay).
-    message_queue = queue.Message.objects.all().order_by("-pk")[
-        0 : settings.DSMRREADER_MQTT_MAX_MESSAGES_IN_QUEUE
-    ]
+    try:
+        # Keep batches small, only send the latest X messages. So drop any excess first and fetch the remainder after.
+        lowest_pk_to_preserve = (
+            queue.Message.objects.all()
+            .order_by("-pk")[
+                settings.DSMRREADER_MQTT_MAX_MESSAGES_IN_QUEUE
+                - 1  # Zero indexed, so -1
+            ]
+            .pk
+        )
+    except IndexError:
+        # Total count within limits. No cleanup required.
+        pass
+    else:
+        deletion_count, _ = queue.Message.objects.filter(
+            pk__lt=lowest_pk_to_preserve
+        ).delete()
+        logger.warning(
+            "MQTT: Dropped %d message(s) from queue due to limit", deletion_count
+        )
+
+    # Remainder, preserving order.
+    message_queue = queue.Message.objects.all().order_by("pk")
 
     if not message_queue:
         return
@@ -121,9 +140,6 @@ def run(mqtt_client: paho.Client) -> None:
         logger.debug("MQTT: Deleting published message (#%s) from queue", current.pk)
         current.delete()
 
-    # Delete any overflow in messages.
-    queue.Message.objects.all().delete()
-
 
 def signal_reconnect() -> None:
     backend_restart_required.send_robust(None)

diff --git a/dsmr_mqtt/tests/services/test_broker.py b/dsmr_mqtt/tests/services/test_broker.py
@@ -169,11 +169,28 @@ def test_run_cleanup(self, publish_mock, loop_mock):
         self.assertEqual(publish_mock.call_count, MAX)
 
         # We assert that the LAST X messages were sent, rest is deleted.
-        for x in range(MAX + 1, MAX * 2 + 1):
+        LOWEST_PAYLOAD = MAX + 1
+        HIGHEST_PAYLOAD = MAX * 2
+
+        for x in range(LOWEST_PAYLOAD, HIGHEST_PAYLOAD + 1):
             publish_mock.assert_any_call(topic="z", payload=str(x), qos=2, retain=True)
 
         self.assertFalse(queue.Message.objects.exists())
 
+        # Make sure whatever mechanism we're using, the queue order is always preserved.
+        previous_payload = 0
+
+        for current_call in publish_mock.call_args_list:
+            current_call_payload = int(current_call[1]["payload"])
+
+            # For our sanity and to ensure the right ones are deleted.
+            self.assertGreaterEqual(current_call_payload, LOWEST_PAYLOAD)
+            self.assertLessEqual(current_call_payload, HIGHEST_PAYLOAD)
+
+            # Fails? Possible issues with queue order.
+            self.assertGreater(current_call_payload, previous_payload)
+            previous_payload = current_call_payload
+
     @mock.patch("paho.mqtt.client.Client.publish")
     @mock.patch("paho.mqtt.client.Client.loop")
     def test_run_disconnected(self, loop_mock, *mocks):

diff --git a/dsmrreader/__init__.py b/dsmrreader/__init__.py
@@ -1,5 +1,5 @@
 from django.utils.version import get_version
 
-VERSION = (5, 7, 0, "final", 0)
+VERSION = (5, 8, 0, "final", 0)
 
 __version__ = get_version(VERSION)
diff --git a/dsmrreader/locales/nl/LC_MESSAGES/django.po b/dsmrreader/locales/nl/LC_MESSAGES/django.po
@@ -2,7 +2,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: DSMR-reader\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2022-09-23 21:07+0200\n"
+"POT-Creation-Date: 2022-09-28 23:11+0200\n"
 "PO-Revision-Date: \n"
 "Last-Translator: Dennis Siemensma <github@dennissiemensma.nl>\n"
 "Language-Team: Dennis Siemensma <github@dennissiemensma.nl>\n"
@@ -1223,8 +1223,8 @@ msgstr "Credits"
 msgid "Originally created and authored by"
 msgstr "Oorspronkelijk gemaakt en onderhouden door"
 
-msgid "want to say thanks?"
-msgstr "wil je iets teruggeven?"
+msgid "Want to say thanks?"
+msgstr "Wil je iets teruggeven?"
 
 msgid "Improved by"
 msgstr "Verbeterd door"

diff --git a/dsmrreader/provisioning/downgrade/v5.8.0.sh b/dsmrreader/provisioning/downgrade/v5.8.0.sh
@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+# Dump for DSMR-reader v5.8.0
+./manage.py migrate dsmr_api 0003_create_api_user
+./manage.py migrate dsmr_backend 0015_backend_restart_required
+./manage.py migrate dsmr_backup 0015_dropbox_app_key
+./manage.py migrate dsmr_consumption 0021_schedule_quarter_hour_peaks_calculation
+./manage.py migrate dsmr_datalogger 0031_meter_statistics_meta
+./manage.py migrate dsmr_dropbox 0001_schedule_dropbox
+./manage.py migrate dsmr_frontend 0048_electricity_peaks_graph
+./manage.py migrate dsmr_influxdb 0006_influxdb_settings_field_size
+./manage.py migrate dsmr_mindergas 0005_schedule_mindergas_export
+./manage.py migrate dsmr_mqtt 0020_drop_mqtt_qos_setting
+./manage.py migrate dsmr_notification 0008_dummy_notification_provider
+./manage.py migrate dsmr_pvoutput 0004_pvoutput_setting_refactoring
+./manage.py migrate dsmr_stats 0017_day_statistics_reading_history_retroactive
+./manage.py migrate dsmr_weather 0006_schedule_weather_update
diff --git a/dsmrreader/provisioning/requirements/base.txt b/dsmrreader/provisioning/requirements/base.txt
@@ -11,7 +11,7 @@ djangorestframework==3.14.0
 django-solo==2.0
 dropbox==11.32.0
 gunicorn==20.1.0
-influxdb-client==1.32.0
+influxdb-client==1.33.0
 paho-mqtt==1.6.1
 psycopg2-binary==2.9.3
 pyserial==3.5