Fix a few MQTT issues (#855)

* Use authentication in MQTT tests

* Fix device creation in MQTT tests

* Fix some error printing

* Fix logging MQTT password

example/mqtt/docker-compose.yaml

@@ -1,5 +1,5 @@
 ---
-version: '2'
+version: "2.4"
 
 services:
   server:
@@ -29,12 +29,19 @@ services:
       - fluent
 
   broker:
-    image: eclipse-mosquitto:1.4.12
+    image: eclipse-mosquitto:2.0.15
     ports:
       - "9001:9001"
       - "1883:1883"
     volumes:
-      - "./mosquitto.conf:/mosquitto/config/mosquitto.conf"
+      - target: /mosquitto/config/mosquitto.conf
+        source: ./mosquitto.conf
+        type: bind
+        read_only: true
+      - target: /mosquitto/config/mosquitto_passwd
+        source: ./mosquitto_passwd
+        type: bind
+        read_only: true
 
   fluent:
     image: fluent/fluentd

example/mqtt/listener.py

@@ -14,6 +14,7 @@
 def get_client():
     mqtt_client = paho.Client(transport="websockets", client_id="listener")
     mqtt_client.enable_logger()
+    mqtt_client.username_pw_set(username="tavern", password="tavern")
     mqtt_client.connect_async(host="broker", port=9001)
 
     return mqtt_client

example/mqtt/mosquitto.conf

@@ -4,3 +4,7 @@ listener 9001
 protocol websockets
 
 listener 1883
+
+allow_anonymous false
+password_file  /mosquitto/config/mosquitto_passwd
+allow_zero_length_clientid false

example/mqtt/mosquitto_passwd

@@ -0,0 +1 @@
+tavern:$7$101$QTtQiXnahWIeGi4h$fgnglssgu7MQObOiQ18VtGpU45sC4pYXCH1IgykbAN6ql4U9CnGK+Q7uj/SQtGHYMPVF1SnqJtX9x9aQyE6JEw==

example/mqtt/server.py

@@ -18,6 +18,8 @@
 def get_client():
     mqtt_client = paho.Client(transport="websockets", client_id="server")
     mqtt_client.enable_logger()
+    mqtt_client.username_pw_set(username="tavern", password="tavern")
+
     mqtt_client.connect(host="broker", port=9001)
 
     try:

example/mqtt/test_mqtt.tavern.yaml

@@ -6,6 +6,9 @@ includes:
   - !include common.yaml
 
 paho-mqtt: &mqtt_spec
+  auth:
+    username: tavern
+    password: tavern
   # tls:
   #   enable: true
   connect:
@@ -466,6 +469,9 @@ includes:
   - !include common.yaml
 
 paho-mqtt:
+  auth:
+    username: tavern
+    password: tavern
   client:
     transport: websockets
     client_id: tavern-tester-{random_device_id}

example/mqtt/test_mqtt_failures.tavern.yaml

@@ -6,6 +6,9 @@ includes:
   - !include common.yaml
 
 paho-mqtt: &mqtt_spec
+  auth:
+    username: tavern
+    password: tavern
   # tls:
   #   enable: true
   connect:
@@ -19,6 +22,16 @@ paho-mqtt: &mqtt_spec
 _xfail: verify
 
 stages:
+  - &setup_device_for_test
+    name: create device
+    request:
+      url: "{host}/create_device"
+      method: PUT
+      json:
+        device_id: "{random_device_id}"
+    response:
+      status_code: 201
+
   - name: step 1 - ping/pong
     mqtt_publish:
       topic: /device/123/ping
@@ -41,6 +54,8 @@ paho-mqtt: *mqtt_spec
 _xfail: verify
 
 stages:
+  - *setup_device_for_test
+
   - name: step 1 - ping/pong
     mqtt_publish:
       topic: /device/123/ping
@@ -53,6 +68,36 @@ stages:
 
 ---
 
+test_name: Test trying to connect with an invalid username/password fails
+
+includes:
+  - !include common.yaml
+
+_xfail: run
+
+paho-mqtt:
+    <<: *mqtt_spec
+    auth:
+      username: tavern
+      password: hhehehehh
+
+stages:
+  - *setup_device_for_test
+
+  - name: Echo json
+    mqtt_publish:
+      topic: /device/{random_device_id}/echo
+      json:
+        message: hello world
+    mqtt_response:
+      topic: /device/{random_device_id}/echo/response
+      json:
+        message: hello world
+      timeout: 5
+      qos: 1
+
+---
+
 test_name: Test incorrect type token fails
 
 includes:
@@ -63,6 +108,8 @@ paho-mqtt: *mqtt_spec
 _xfail: run
 
 stages:
+  - *setup_device_for_test
+
   - name: Test checking for lights status with anystr fails
     mqtt_publish:
       topic: /device/123/status
@@ -85,6 +132,8 @@ paho-mqtt: *mqtt_spec
 _xfail: run
 
 stages:
+  - *setup_device_for_test
+
   - name: Test checking for lights status with anyint
     mqtt_publish:
       topic: /device/123/status
@@ -98,45 +147,49 @@ stages:
 test_name: Test unexpected message fails
 
 includes:
-- !include common.yaml
+  - !include common.yaml
 
 paho-mqtt: *mqtt_spec
 
 _xfail: run
 
 stages:
-- name: step 1 - ping/pong
-  mqtt_publish:
-    topic: /devices/status
-  mqtt_response:
-  - topic: /device/456/status/response
-    payload: !anything
-    timeout: 2
-    qos: 1
-  - topic: /device/123/status/response
-    payload: !anything
-    timeout: 2
-    qos: 1
-    unexpected: true
+  - *setup_device_for_test
+
+  - name: step 1 - ping/pong
+    mqtt_publish:
+      topic: /devices/status
+    mqtt_response:
+      - topic: /device/456/status/response
+        payload: !anything
+        timeout: 2
+        qos: 1
+      - topic: /device/123/status/response
+        payload: !anything
+        timeout: 2
+        qos: 1
+        unexpected: true
 
 ---
 
 test_name: Test unexpected message fails even on its own
 
 includes:
-- !include common.yaml
+  - !include common.yaml
 
 paho-mqtt: *mqtt_spec
 
 _xfail: run
 
 stages:
-- name: step 1 - ping/pong
-  mqtt_publish:
-    topic: /devices/status
-  mqtt_response:
-  - topic: /device/123/status/response
-    payload: !anything
-    timeout: 3
-    qos: 1
-    unexpected: true
+  - *setup_device_for_test
+
+  - name: step 1 - ping/pong
+    mqtt_publish:
+      topic: /devices/status
+    mqtt_response:
+      - topic: /device/123/status/response
+        payload: !anything
+        timeout: 3
+        qos: 1
+        unexpected: true

tavern/_core/pytest/error.py

@@ -210,7 +210,9 @@ def toterminal(self, tw: TerminalWriter) -> None:
         tw.line("")
 
         if not stage:
-            tw.line("Stage not found", red=True, bold=True)
+            tw.line(
+                "[Could not determine which stage was running]", red=True, bold=True
+            )
         elif missing_format_vars:
             tw.line("Missing format vars for stage", red=True, bold=True)
         else:

tavern/_plugins/mqtt/client.py

@@ -1,3 +1,4 @@
+import copy
 import dataclasses
 import logging
 import ssl
@@ -156,7 +157,12 @@ def __init__(self, **kwargs) -> None:
             },
         }
 
-        logger.debug("Initialising MQTT client with %s", kwargs)
+        sanitised_kwargs = copy.deepcopy(kwargs)
+        if auth := kwargs.get("auth"):
+            if "password" in auth:
+                sanitised_kwargs["auth"]["password"] = "******"  # noqa
+
+        logger.debug("Initialising MQTT client with %s", sanitised_kwargs)
 
         # check main block first
         check_expected_keys(expected_blocks.keys(), kwargs)
@@ -200,6 +206,7 @@ def __init__(self, **kwargs) -> None:
         self._client.enable_logger()
 
         if self._auth_args:
+            logger.debug("authenticating as '%s'", self._auth_args.get("username"))
             self._client.username_pw_set(**self._auth_args)
 
         self._client.on_message = self._on_message
@@ -313,7 +320,7 @@ def _on_message(client, userdata, message) -> None:
     @staticmethod
     def _on_connect(client, userdata, flags, rc) -> None:
         logger.debug(
-            "Client '%s' successfully connected to the broker with result code '%s'",
+            "Client '%s' connected to the broker with result code '%s'",
             client._client_id.decode(),
             paho.connack_string(rc),
         )