Fix saving in MQTT (#840)

Fix being able to save from a response/multiple responses in MQTT

example/mqtt/conftest.py

@@ -106,3 +106,8 @@ def get_response_topic_suffix():
 @pytest.fixture(scope="function", autouse=True)
 def random_device_id():
     return str(random.randint(100, 10000))
+
+
+@pytest.fixture(scope="function", autouse=True)
+def random_device_id_2():
+    return str(random.randint(100, 10000))

example/mqtt/test_mqtt.tavern.yaml

@@ -68,8 +68,8 @@ test_name: Test mqtt message echo json formatted topic name
 
 marks:
   - usefixtures:
-    - get_publish_topic
-    - get_response_topic_suffix
+      - get_publish_topic
+      - get_response_topic_suffix
 
 includes:
   - !include common.yaml
@@ -251,7 +251,7 @@ stages:
 
 ---
 
-test_name: Make sure posting publishes mqtt message
+test_name: Make sure can handle multiple types of responses
 
 includes:
   - !include common.yaml
@@ -502,7 +502,7 @@ stages:
       url: "{host}/create_device"
       method: PUT
       json:
-        device_id: "3487589234754"
+        device_id: "{random_device_id_2}"
 
   - name: step 1 - ping/pong
     mqtt_publish:
@@ -512,7 +512,7 @@ stages:
         payload: !anything
         timeout: 3
         qos: 1
-      - topic: /device/3487589234754/status/response
+      - topic: /device/{random_device_id_2}/status/response
         payload: !anything
         timeout: 3
         qos: 1
@@ -534,17 +534,209 @@ stages:
       url: "{host}/create_device"
       method: PUT
       json:
-        device_id: "43676974864576"
+        device_id: "{random_device_id_2}"
 
   - name: step 1 - ping/pong
     mqtt_publish:
       topic: /devices/status
     mqtt_response:
-      - topic: /device/43676974864576/status/response
+      - topic: /device/{random_device_id_2}/status/response
         payload: !anything
         timeout: 3
         qos: 1
       - topic: /device/{random_device_id}/status/response
         payload: !anything
         timeout: 3
         qos: 1
+
+---
+
+test_name: Save something and reuse it, one response
+
+includes:
+  - !include common.yaml
+
+paho-mqtt: *mqtt_spec
+
+stages:
+  - *setup_device_for_test
+
+  - name: step 1 - ping/pong
+    mqtt_publish:
+      topic: /devices/status
+    mqtt_response:
+      topic: /device/{random_device_id}/status/response
+      json:
+        lights: !anything
+      timeout: 3
+      qos: 1
+      save:
+        json:
+          lights_status: lights
+
+  - name: Echo text
+    mqtt_publish:
+      topic: /device/{random_device_id}/echo
+      payload: "{lights_status}"
+    mqtt_response:
+      topic: /device/{random_device_id}/echo/response
+      payload: "{lights_status}"
+      timeout: 5
+      qos: 1
+
+---
+
+test_name: Save something and reuse it, multiple responses, saved from both
+
+includes:
+  - !include common.yaml
+
+paho-mqtt: *mqtt_spec
+
+stages:
+  - *setup_device_for_test
+
+  - name: Turn lights on for first device
+    mqtt_publish:
+      topic: /device/{random_device_id}/lights
+      qos: 1
+      payload: "on"
+    delay_after: 2
+
+  - name: create device 2
+    request:
+      url: "{host}/create_device"
+      method: PUT
+      json:
+        device_id: "{random_device_id_2}"
+
+  - name: Get device statuses
+    mqtt_publish:
+      topic: /devices/status
+    mqtt_response:
+      - topic: /device/{random_device_id}/status/response
+        timeout: 3
+        qos: 1
+        json:
+          lights: 1
+        save:
+          json:
+            device_1_lights: lights
+      - topic: /device/{random_device_id_2}/status/response
+        timeout: 3
+        qos: 1
+        json:
+          lights: 0
+        save:
+          json:
+            device_2_lights: lights
+
+  - name: Ensure can use saved values 1
+    request:
+      url: "{host}/send_mqtt_message"
+      json:
+        device_id: "{random_device_id}"
+        payload: "{device_1_lights}"
+      method: POST
+      headers:
+        content-type: application/json
+    response:
+      status_code: 200
+      json:
+        topic: "/device/{random_device_id}"
+      headers:
+        content-type: application/json
+    mqtt_response:
+      topic: /device/{random_device_id}
+      payload: "1"
+      timeout: 5
+
+  - name: Ensure can use saved values 2
+    request:
+      url: "{host}/send_mqtt_message"
+      json:
+        device_id: "{random_device_id_2}"
+        payload: "{device_2_lights}"
+      method: POST
+      headers:
+        content-type: application/json
+    response:
+      status_code: 200
+      json:
+        topic: "/device/{random_device_id_2}"
+      headers:
+        content-type: application/json
+    mqtt_response:
+      topic: /device/{random_device_id_2}
+      payload: "0"
+      timeout: 5
+
+---
+
+test_name: Save something from an ext function and reuse it, one response
+
+includes:
+  - !include common.yaml
+
+paho-mqtt: *mqtt_spec
+
+stages:
+  - *setup_device_for_test
+
+  - name: step 1 - ping/pong
+    mqtt_publish:
+      topic: /devices/status
+    mqtt_response:
+      topic: /device/{random_device_id}/status/response
+      json:
+        lights: !anything
+      timeout: 3
+      qos: 1
+      save:
+        $ext:
+          function: testing_utils:return_hello
+
+  - name: Echo text
+    mqtt_publish:
+      topic: /device/{random_device_id}/echo
+      payload: "{hello}"
+    mqtt_response:
+      topic: /device/{random_device_id}/echo/response
+      payload: "there"
+      timeout: 5
+      qos: 1
+
+---
+
+test_name: Save something from an ext function and reuse it, multiple response
+
+includes:
+  - !include common.yaml
+
+paho-mqtt: *mqtt_spec
+
+stages:
+  - *setup_device_for_test
+
+  - name: step 1 - ping/pong
+    mqtt_publish:
+      topic: /devices/status
+    mqtt_response:
+      - topic: /device/{random_device_id}/status/response
+        json:
+          lights: !anything
+        timeout: 3
+        qos: 1
+        save:
+          $ext:
+            function: testing_utils:return_hello
+
+  - name: Echo text
+    mqtt_publish:
+      topic: /device/{random_device_id}/echo
+      payload: "{hello}"
+    mqtt_response:
+      topic: /device/{random_device_id}/echo/response
+      payload: "there"
+      timeout: 5
+      qos: 1

example/mqtt/testing_utils.py

@@ -1,3 +1,7 @@
 def message_says_hello(msg):
     """Make sure that the response was friendly"""
     assert msg.payload.get("message") == "hello world"
+
+
+def return_hello(_):
+    return {"hello": "there"}

scripts/smoke.bash

@@ -4,9 +4,12 @@ set -ex
 
 PYVER=3
 
+# Separate as isort can interfere with other testenvs
+tox --parallel -c tox.ini        \
+  -e py${PYVER}check
+
 tox --parallel -c tox.ini        \
   -e py${PYVER}       \
-  -e py${PYVER}check  \
   -e py${PYVER}lint   \
   -e py${PYVER}mypy
 

tavern/_core/dict_util.py

@@ -1,8 +1,9 @@
-import collections
+import collections.abc
 import logging
 import os
 import re
 import string
+from typing import Any, Dict, List, Optional, Union
 
 import box
 from box import Box
@@ -53,7 +54,7 @@ def _check_and_format_values(to_format, box_vars):
     return to_format.format(**box_vars)
 
 
-def _attempt_find_include(to_format, box_vars):
+def _attempt_find_include(to_format: str, box_vars: box.Box):
     formatter = string.Formatter()
     would_format = list(formatter.parse(to_format))
 
@@ -87,7 +88,7 @@ def _attempt_find_include(to_format, box_vars):
 
     would_replace = formatter.get_field(field_name, [], box_vars)[0]
 
-    return formatter.convert_field(would_replace, conversion)
+    return formatter.convert_field(would_replace, conversion)  # type: ignore
 
 
 def format_keys(val, variables, no_double_format=True):
@@ -224,7 +225,7 @@ def _deprecated_recurse_access_key(current_val, keys):
             raise
 
 
-def deep_dict_merge(initial_dct, merge_dct):
+def deep_dict_merge(initial_dct: Dict, merge_dct: collections.abc.Mapping) -> dict:
     """Recursive dict merge. Instead of updating only top-level keys,
     dict_merge recurses down into dicts nested to an arbitrary depth
     and returns the merged dict. Keys values present in merge_dct take
@@ -236,7 +237,7 @@ def deep_dict_merge(initial_dct, merge_dct):
         merge_dct: dct merged into dct
 
     Returns:
-        dict: recursively merged dict
+        recursively merged dict
     """
     dct = initial_dct.copy()
 
@@ -323,7 +324,12 @@ def yield_keyvals(block):
             yield [sidx], sidx, val
 
 
-def check_keys_match_recursive(expected_val, actual_val, keys, strict=True):
+def check_keys_match_recursive(
+    expected_val: Any,
+    actual_val: Any,
+    keys: List[Union[str, int]],
+    strict: Optional[Union[StrictSetting, bool]] = True,
+) -> None:
     """Utility to recursively check response values
 
     expected and actual both have to be of the same type or it will raise an
@@ -343,11 +349,11 @@ def check_keys_match_recursive(expected_val, actual_val, keys, strict=True):
         code and to remove a load of the isinstance checks
 
     Args:
-        expected_val (dict, list, str): expected value
-        actual_val (dict, list, str): actual value
-        keys (list): any keys which have been recursively parsed to get to this
+        expected_val: expected value
+        actual_val: actual value
+        keys: any keys which have been recursively parsed to get to this
             point. Used for debug output.
-        strict (bool): Whether 'strict' key checking should be done. If this is
+        strict: Whether 'strict' key checking should be done. If this is
             False, a mismatch in dictionary keys between the expected and the
             actual values will not raise an error (but a mismatch in value will
             raise an error)