Merge branch 'dev' into puddly/new-radio-settings-api

.github/workflows/ci.yml

@@ -35,7 +35,7 @@ jobs:
           key: >-
             ${{ env.CACHE_VERSION}}-${{ runner.os }}-base-venv-${{
             steps.python.outputs.python-version }}-${{
-            hashFiles('requirements_test.txt') }}
+            hashFiles('setup.py', 'requirements_test.txt') }}
           restore-keys: |
             ${{ env.CACHE_VERSION}}-${{ runner.os }}-base-venv-${{ steps.python.outputs.python-version }}-
       - name: Create Python virtual environment
@@ -67,7 +67,7 @@ jobs:
           key: >-
             ${{ env.CACHE_VERSION}}-${{ runner.os }}-base-venv-${{
             steps.python.outputs.python-version }}-${{
-            hashFiles('requirements_test.txt') }}
+            hashFiles('setup.py', 'requirements_test.txt') }}
       - name: Fail job if Python cache restore failed
         if: steps.cache-venv.outputs.cache-hit != 'true'
         run: |
@@ -108,7 +108,7 @@ jobs:
           key: >-
             ${{ env.CACHE_VERSION}}-${{ runner.os }}-base-venv-${{
             steps.python.outputs.python-version }}-${{
-            hashFiles('requirements_test.txt') }}
+            hashFiles('setup.py', 'requirements_test.txt') }}
       - name: Fail job if Python cache restore failed
         if: steps.cache-venv.outputs.cache-hit != 'true'
         run: |
@@ -151,7 +151,7 @@ jobs:
           key: >-
             ${{ env.CACHE_VERSION}}-${{ runner.os }}-base-venv-${{
             steps.python.outputs.python-version }}-${{
-            hashFiles('requirements_test.txt') }}
+            hashFiles('setup.py', 'requirements_test.txt') }}
       - name: Fail job if Python cache restore failed
         if: steps.cache-venv.outputs.cache-hit != 'true'
         run: |
@@ -197,7 +197,7 @@ jobs:
           key: >-
             ${{ env.CACHE_VERSION}}-${{ runner.os }}-base-venv-${{
             steps.python.outputs.python-version }}-${{
-            hashFiles('requirements_test.txt') }}
+            hashFiles('setup.py', 'requirements_test.txt') }}
       - name: Fail job if Python cache restore failed
         if: steps.cache-venv.outputs.cache-hit != 'true'
         run: |
@@ -240,7 +240,7 @@ jobs:
           key: >-
             ${{ env.CACHE_VERSION}}-${{ runner.os }}-base-venv-${{
             steps.python.outputs.python-version }}-${{
-            hashFiles('requirements_test.txt') }}
+            hashFiles('setup.py', 'requirements_test.txt') }}
       - name: Fail job if Python cache restore failed
         if: steps.cache-venv.outputs.cache-hit != 'true'
         run: |
@@ -286,7 +286,7 @@ jobs:
           key: >-
             ${{ env.CACHE_VERSION}}-${{ runner.os }}-base-venv-${{
             steps.python.outputs.python-version }}-${{
-            hashFiles('requirements_test.txt') }}
+            hashFiles('setup.py', 'requirements_test.txt') }}
       - name: Fail job if Python cache restore failed
         if: steps.cache-venv.outputs.cache-hit != 'true'
         run: |
@@ -334,7 +334,7 @@ jobs:
           key: >-
             ${{ env.CACHE_VERSION}}-${{ runner.os }}-base-venv-${{
             steps.python.outputs.python-version }}-${{
-            hashFiles('requirements_test.txt') }}
+            hashFiles('setup.py', 'requirements_test.txt') }}
       - name: Fail job if Python cache restore failed
         if: steps.cache-venv.outputs.cache-hit != 'true'
         run: |
@@ -396,7 +396,7 @@ jobs:
           key: >-
             ${{ env.CACHE_VERSION}}-${{ runner.os }}-base-venv-${{
             steps.python.outputs.python-version }}-${{
-            hashFiles('requirements_test.txt') }}
+            hashFiles('setup.py', 'requirements_test.txt') }}
       - name: Fail job if Python cache restore failed
         if: steps.cache-venv.outputs.cache-hit != 'true'
         run: |

setup.py

@@ -6,7 +6,7 @@
 
 import zigpy
 
-REQUIRES = ["aiohttp", "aiosqlite>=0.16.0", "crccheck", "pycryptodome", "voluptuous"]
+REQUIRES = ["aiohttp", "aiosqlite>=0.16.0", "crccheck", "cryptography", "voluptuous"]
 
 setup(
     name="zigpy",

tests/async_mock.py

@@ -3,7 +3,19 @@
 
 if sys.version_info[:2] < (3, 8):
     from asynctest.mock import *  # noqa
+    from asynctest.mock import MagicMock as _MagicMock
 
     AsyncMock = CoroutineMock  # noqa: F405
+
+    class MagicMock(_MagicMock):
+        async def __aenter__(self):
+            return self.aenter
+
+        async def __aexit__(self, *args):
+            pass
+
+        async def __aiter__(self):
+            return self.aiter
+
 else:
     from unittest.mock import *  # noqa

tests/test_appdb.py

@@ -3,6 +3,7 @@
 import sqlite3
 import sys
 import threading
+import time
 
 import aiosqlite
 import pytest
@@ -154,6 +155,12 @@ async def test_database(tmpdir):
     assert dev.get_signature()[SIG_MANUFACTURER] == "Custom"
     assert dev.get_signature()[SIG_MODEL] == "Model"
 
+    ts = time.time()
+    dev.last_seen = ts
+    dev_last_seen = dev.last_seen
+    assert isinstance(dev.last_seen, float)
+    assert abs(dev.last_seen - ts) < 0.01
+
     # Test a CustomDevice
     custom_ieee = make_ieee(1)
     app.handle_join(199, custom_ieee, 0)
@@ -174,6 +181,9 @@ async def test_database(tmpdir):
     dev.endpoints[99].level._update_attribute(0x0011, 17)
     assert dev.endpoints[1].in_clusters[0x0008]._attr_cache[0x0011] == 17
     assert dev.endpoints[99].in_clusters[0x0008]._attr_cache[0x0011] == 17
+    custom_dev_last_seen = dev.last_seen
+    assert isinstance(custom_dev_last_seen, float)
+
     await app.shutdown()
 
     # Everything should've been saved - check that it re-loads
@@ -190,13 +200,16 @@ async def test_database(tmpdir):
     assert dev.endpoints[2].out_clusters[1].cluster_id == 1
     assert dev.endpoints[3].device_type == profiles.zll.DeviceType.COLOR_LIGHT
     assert dev.relays == relays_1
+    # The timestamp won't be restored exactly but it is more than close enough
+    assert abs(dev.last_seen - dev_last_seen) < 0.01
 
     dev = app2.get_device(custom_ieee)
     # This virtual attribute is added by the quirk, there is no corresponding cluster
     # stored in the database, nor is there a corresponding endpoint 99
     assert dev.endpoints[1].in_clusters[0x0008]._attr_cache[0x0011] == 17
     assert dev.endpoints[99].in_clusters[0x0008]._attr_cache[0x0011] == 17
     assert dev.relays == relays_2
+    assert abs(dev.last_seen - custom_dev_last_seen) < 0.01
     dev.relays = None
 
     app.handle_leave(99, ieee)
@@ -741,6 +754,50 @@ async def test_load_unsupp_attr_wrong_cluster(tmpdir):
     await app.shutdown()
 
 
+async def test_last_seen(tmpdir):
+    db = os.path.join(str(tmpdir), "test.db")
+    app = await make_app(db)
+
+    ieee = make_ieee()
+    app.handle_join(99, ieee, 0)
+
+    dev = app.get_device(ieee=ieee)
+    ep = dev.add_endpoint(3)
+    ep.status = zigpy.endpoint.Status.ZDO_INIT
+    ep.profile_id = 260
+    ep.device_type = profiles.zha.DeviceType.PUMP
+    clus = ep.add_input_cluster(0)
+    ep.add_output_cluster(1)
+    clus._update_attribute(4, "Custom")
+    clus._update_attribute(5, "Model")
+    app.device_initialized(dev)
+
+    old_last_seen = dev.last_seen
+    await app.pre_shutdown()
+
+    # The `last_seen` of a joined device persists
+    app = await make_app(db)
+    dev = app.get_device(ieee=ieee)
+    await app.pre_shutdown()
+
+    next_last_seen = dev.last_seen
+    assert abs(next_last_seen - old_last_seen) < 0.01
+
+    await asyncio.sleep(0.1)
+
+    # Now the last_seen will update
+    app = await make_app(db)
+    dev = app.get_device(ieee=ieee)
+    dev.update_last_seen()
+    await app.pre_shutdown()
+
+    # And it will be updated when the database next loads
+    app = await make_app(db)
+    dev = app.get_device(ieee=ieee)
+    assert dev.last_seen > next_last_seen + 0.1
+    await app.pre_shutdown()
+
+
 @pytest.mark.parametrize(
     "stdlib_version,use_sqlite",
     [

tests/test_appdb_migration.py

@@ -9,7 +9,7 @@
 import zigpy.types as t
 from zigpy.zdo import types as zdo_t
 
-from tests.async_mock import patch
+from tests.async_mock import AsyncMock, MagicMock, patch
 from tests.test_appdb import auto_kill_aiosqlite, make_app  # noqa: F401
 
 
@@ -405,13 +405,6 @@ async def test_v4_to_v5_migration_bad_neighbors(test_db, with_bad_neighbor):
         assert num_new_neighbors == num_v4_neighbors
 
 
-async def test_v5_to_v6_migration(test_db):
-    test_db_v5 = test_db("simple_v5.sql")
-
-    app = await make_app(test_db_v5)
-    await app.shutdown()
-
-
 @pytest.mark.parametrize("with_quirk_attribute", [False, True])
 async def test_v4_to_v6_migration_missing_endpoints(test_db, with_quirk_attribute):
     """V5's schema was too rigid and failed to migrate endpoints created by quirks"""
@@ -456,4 +449,60 @@ async def test_v5_to_v7_migration(test_db):
     test_db_v5 = test_db("simple_v5.sql")
 
     app = await make_app(test_db_v5)
+    await app.pre_shutdown()
+
+
+async def test_migration_missing_tables():
+    app = MagicMock()
+    conn = MagicMock()
+    conn.close = AsyncMock()
+
+    appdb = zigpy.appdb.PersistingListener(conn, app)
+
+    appdb._get_table_versions = AsyncMock(
+        return_value={"table1_v1": "1", "table1": "", "table2_v1": "1"}
+    )
+
+    results = MagicMock()
+    results.__aiter__.return_value = results
+    results.__anext__.side_effect = StopIteration
+
+    appdb.execute = MagicMock()
+    appdb.execute.return_value.__aenter__.return_value = results
+
+    # Migrations must explicitly specify all old tables, even if they will be untouched
+    with pytest.raises(RuntimeError):
+        await appdb._migrate_tables(
+            {
+                "table1_v1": "table1_v2",
+                # "table2_v1": "table2_v2",
+            }
+        )
+
+    # The untouched table will never be queried
+    await appdb._migrate_tables({"table1_v1": "table1_v2", "table2_v1": None})
+
+    appdb.execute.assert_called_once_with("SELECT * FROM table1_v1")
+
+    with pytest.raises(AssertionError):
+        appdb.execute.assert_called_once_with("SELECT * FROM table2_v1")
+
+    await appdb.shutdown()
+
+
+async def test_last_seen_migration(test_db):
+    test_db_v5 = test_db("simple_v5.sql")
+
+    # To preserve the old behavior, `0` will not be exposed to ZHA, only `None`
+    app = await make_app(test_db_v5)
+    dev = app.get_device(nwk=0xBD4D)
+
+    assert dev.last_seen is None
+    dev.update_last_seen()
+    assert isinstance(dev.last_seen, float)
+    await app.shutdown()
+
+    # But the device's `last_seen` will still update properly when it's actually set
+    app = await make_app(test_db_v5)
+    assert isinstance(app.get_device(nwk=0xBD4D).last_seen, float)
     await app.shutdown()

tests/test_device.py

@@ -1,4 +1,5 @@
 import asyncio
+from datetime import datetime, timezone
 import logging
 
 import pytest
@@ -11,7 +12,7 @@
 import zigpy.types as t
 from zigpy.zdo import types as zdo_t
 
-from .async_mock import AsyncMock, MagicMock, patch, sentinel
+from .async_mock import ANY, AsyncMock, MagicMock, patch, sentinel
 
 
 @pytest.fixture
@@ -111,6 +112,21 @@ async def mock_req(*args, **kwargs):
     assert dev.last_seen is not None
 
 
+async def test_request_without_reply(dev):
+    seq = sentinel.tsn
+
+    async def mock_req(*args, **kwargs):
+        dev._pending[seq].result.set_result(sentinel.result)
+        return 0, sentinel.radio_status
+
+    dev.application.request.side_effect = mock_req
+    assert dev.last_seen is None
+    r = await dev.request(1, 2, 3, 3, seq, b"", expect_reply=False)
+    assert r is None
+    assert dev._application.request.call_count == 1
+    assert dev.last_seen is not None
+
+
 async def test_failed_request(dev):
     assert dev.last_seen is None
     dev._application.request = AsyncMock(return_value=(1, "error"))
@@ -414,3 +430,22 @@ def test_device_name(dev):
 
     assert dev.nwk == 0xFFFF
     assert dev.name == "0xFFFF"
+
+
+def test_device_last_seen(dev, monkeypatch):
+    """Test the device last_seen property handles updates and broadcasts events."""
+
+    monkeypatch.setattr(dev, "listener_event", MagicMock())
+    assert dev.last_seen is None
+
+    dev.last_seen = 0
+    epoch = datetime(1970, 1, 1, 0, 0, 0, 0, tzinfo=timezone.utc)
+    assert dev.last_seen == epoch.timestamp()
+
+    dev.listener_event.assert_called_once_with("device_last_seen_updated", epoch)
+    dev.listener_event.reset_mock()
+
+    dev.update_last_seen()
+    dev.listener_event.assert_called_once_with("device_last_seen_updated", ANY)
+    event_time = dev.listener_event.mock_calls[0][1][1]
+    assert (event_time - datetime.now(timezone.utc)).total_seconds() < 0.1

tests/test_zcl.py

@@ -468,13 +468,17 @@ async def test_write_attributes_cache_default_response(cluster, status):
     ),
 )
 async def test_write_attributes_cache_success(cluster, attributes, result):
+    listener = MagicMock()
+    cluster.add_listener(listener)
+
     rsp_type = t.List[foundation.WriteAttributesStatusRecord]
     write_mock = AsyncMock(return_value=[rsp_type.deserialize(result)[0]])
     with patch.object(cluster, "_write_attributes", write_mock):
         await cluster.write_attributes(attributes)
         assert cluster._write_attributes.call_count == 1
         for attr_id in attributes:
             assert cluster._attr_cache[attr_id] == attributes[attr_id]
+            listener.attribute_updated.assert_any_call(attr_id, attributes[attr_id])
 
 
 @pytest.mark.parametrize(
@@ -501,6 +505,9 @@ async def test_write_attributes_cache_success(cluster, attributes, result):
     ),
 )
 async def test_write_attributes_cache_failure(cluster, attributes, result, failed):
+    listener = MagicMock()
+    cluster.add_listener(listener)
+
     rsp_type = foundation.WriteAttributesResponse
     write_mock = AsyncMock(return_value=[rsp_type.deserialize(result)[0]])
 
@@ -510,8 +517,15 @@ async def test_write_attributes_cache_failure(cluster, attributes, result, faile
         for attr_id in attributes:
             if attr_id in failed:
                 assert attr_id not in cluster._attr_cache
+
+                # Failed writes do not propagate
+                with pytest.raises(AssertionError):
+                    listener.attribute_updated.assert_any_call(
+                        attr_id, attributes[attr_id]
+                    )
             else:
                 assert cluster._attr_cache[attr_id] == attributes[attr_id]
+                listener.attribute_updated.assert_any_call(attr_id, attributes[attr_id])
 
 
 def test_read_attributes_response(cluster):