From 6c5a8f773fa8dd93f51ee0bf0276ea9096ee88a2 Mon Sep 17 00:00:00 2001 From: puddly <32534428+puddly@users.noreply.github.com> Date: Thu, 28 Apr 2022 00:33:41 -0400 Subject: [PATCH 1/2] Migrate `last_seen` to SQLite's native `REAL` data type --- tests/databases/simple_v8.sql | 241 ++++++++++++++++++++++++++++++ tests/test_appdb_migration.py | 8 + zigpy/appdb.py | 54 +++++-- zigpy/appdb_schemas/schema_v9.sql | 201 +++++++++++++++++++++++++ 4 files changed, 488 insertions(+), 16 deletions(-) create mode 100644 tests/databases/simple_v8.sql create mode 100644 zigpy/appdb_schemas/schema_v9.sql diff --git a/tests/databases/simple_v8.sql b/tests/databases/simple_v8.sql new file mode 100644 index 000000000..76e678263 --- /dev/null +++ b/tests/databases/simple_v8.sql @@ -0,0 +1,241 @@ +PRAGMA user_version=8; +PRAGMA foreign_keys=OFF; +BEGIN TRANSACTION; +CREATE TABLE devices_v8 ( + ieee ieee NOT NULL, + nwk INTEGER NOT NULL, + status INTEGER NOT NULL, + last_seen unix_timestamp NOT NULL +); +INSERT INTO devices_v8 VALUES('00:12:4b:00:1c:a1:b8:46',0,2,1651119833288); +INSERT INTO devices_v8 VALUES('ec:1b:bd:ff:fe:2f:41:a4',44170,2,1651119836445); +INSERT INTO devices_v8 VALUES('cc:cc:cc:ff:fe:e6:8e:ca',50064,2,1651119839551); +INSERT INTO devices_v8 VALUES('00:0b:57:ff:fe:2b:d4:57',57374,2,1651119830048); +CREATE TABLE endpoints_v8 ( + ieee ieee NOT NULL, + endpoint_id INTEGER NOT NULL, + profile_id INTEGER NOT NULL, + device_type INTEGER NOT NULL, + status INTEGER NOT NULL, + + FOREIGN KEY(ieee) + REFERENCES devices_v8(ieee) + ON DELETE CASCADE +); +INSERT INTO endpoints_v8 VALUES('00:12:4b:00:1c:a1:b8:46',1,260,48879,1); +INSERT INTO endpoints_v8 VALUES('ec:1b:bd:ff:fe:2f:41:a4',1,260,268,1); +INSERT INTO endpoints_v8 VALUES('ec:1b:bd:ff:fe:2f:41:a4',242,41440,97,1); +INSERT INTO endpoints_v8 VALUES('cc:cc:cc:ff:fe:e6:8e:ca',1,260,268,1); +INSERT INTO endpoints_v8 VALUES('cc:cc:cc:ff:fe:e6:8e:ca',242,41440,97,1); +INSERT INTO endpoints_v8 VALUES('00:0b:57:ff:fe:2b:d4:57',1,260,2080,1); +CREATE TABLE in_clusters_v8 ( + ieee ieee NOT NULL, + endpoint_id INTEGER NOT NULL, + cluster INTEGER NOT NULL, + + FOREIGN KEY(ieee, endpoint_id) + REFERENCES endpoints_v8(ieee, endpoint_id) + ON DELETE CASCADE +); +INSERT INTO in_clusters_v8 VALUES('ec:1b:bd:ff:fe:2f:41:a4',1,0); +INSERT INTO in_clusters_v8 VALUES('ec:1b:bd:ff:fe:2f:41:a4',1,3); +INSERT INTO in_clusters_v8 VALUES('ec:1b:bd:ff:fe:2f:41:a4',1,4); +INSERT INTO in_clusters_v8 VALUES('ec:1b:bd:ff:fe:2f:41:a4',1,5); +INSERT INTO in_clusters_v8 VALUES('ec:1b:bd:ff:fe:2f:41:a4',1,6); +INSERT INTO in_clusters_v8 VALUES('ec:1b:bd:ff:fe:2f:41:a4',1,8); +INSERT INTO in_clusters_v8 VALUES('ec:1b:bd:ff:fe:2f:41:a4',1,768); +INSERT INTO in_clusters_v8 VALUES('ec:1b:bd:ff:fe:2f:41:a4',1,2821); +INSERT INTO in_clusters_v8 VALUES('ec:1b:bd:ff:fe:2f:41:a4',1,4096); +INSERT INTO in_clusters_v8 VALUES('ec:1b:bd:ff:fe:2f:41:a4',1,64642); +INSERT INTO in_clusters_v8 VALUES('cc:cc:cc:ff:fe:e6:8e:ca',1,0); +INSERT INTO in_clusters_v8 VALUES('cc:cc:cc:ff:fe:e6:8e:ca',1,3); +INSERT INTO in_clusters_v8 VALUES('cc:cc:cc:ff:fe:e6:8e:ca',1,4); +INSERT INTO in_clusters_v8 VALUES('cc:cc:cc:ff:fe:e6:8e:ca',1,5); +INSERT INTO in_clusters_v8 VALUES('cc:cc:cc:ff:fe:e6:8e:ca',1,6); +INSERT INTO in_clusters_v8 VALUES('cc:cc:cc:ff:fe:e6:8e:ca',1,8); +INSERT INTO in_clusters_v8 VALUES('cc:cc:cc:ff:fe:e6:8e:ca',1,768); +INSERT INTO in_clusters_v8 VALUES('cc:cc:cc:ff:fe:e6:8e:ca',1,2821); +INSERT INTO in_clusters_v8 VALUES('cc:cc:cc:ff:fe:e6:8e:ca',1,4096); +INSERT INTO in_clusters_v8 VALUES('cc:cc:cc:ff:fe:e6:8e:ca',1,64642); +INSERT INTO in_clusters_v8 VALUES('00:0b:57:ff:fe:2b:d4:57',1,0); +INSERT INTO in_clusters_v8 VALUES('00:0b:57:ff:fe:2b:d4:57',1,1); +INSERT INTO in_clusters_v8 VALUES('00:0b:57:ff:fe:2b:d4:57',1,3); +INSERT INTO in_clusters_v8 VALUES('00:0b:57:ff:fe:2b:d4:57',1,32); +INSERT INTO in_clusters_v8 VALUES('00:0b:57:ff:fe:2b:d4:57',1,4096); +CREATE TABLE neighbors_v8 ( + device_ieee ieee NOT NULL, + extended_pan_id ieee NOT NULL, + ieee ieee NOT NULL, + nwk INTEGER NOT NULL, + device_type INTEGER NOT NULL, + rx_on_when_idle INTEGER NOT NULL, + relationship INTEGER NOT NULL, + reserved1 INTEGER NOT NULL, + permit_joining INTEGER NOT NULL, + reserved2 INTEGER NOT NULL, + depth INTEGER NOT NULL, + lqi INTEGER NOT NULL, + + FOREIGN KEY(device_ieee) + REFERENCES devices_v8(ieee) + ON DELETE CASCADE +); +INSERT INTO neighbors_v8 VALUES('00:12:4b:00:1c:a1:b8:46','bd:27:0b:38:37:95:dc:87','ec:1b:bd:ff:fe:2f:41:a4',44170,1,1,2,0,2,0,15,255); +INSERT INTO neighbors_v8 VALUES('00:12:4b:00:1c:a1:b8:46','bd:27:0b:38:37:95:dc:87','cc:cc:cc:ff:fe:e6:8e:ca',50064,1,1,2,0,2,0,15,255); +INSERT INTO neighbors_v8 VALUES('00:12:4b:00:1c:a1:b8:46','bd:27:0b:38:37:95:dc:87','00:0b:57:ff:fe:2b:d4:57',57374,2,0,1,0,0,0,1,255); +INSERT INTO neighbors_v8 VALUES('ec:1b:bd:ff:fe:2f:41:a4','bd:27:0b:38:37:95:dc:87','00:12:4b:00:1c:a1:b8:46',0,0,1,2,0,2,0,0,253); +INSERT INTO neighbors_v8 VALUES('ec:1b:bd:ff:fe:2f:41:a4','bd:27:0b:38:37:95:dc:87','cc:cc:cc:ff:fe:e6:8e:ca',50064,1,1,0,0,2,0,15,255); +INSERT INTO neighbors_v8 VALUES('cc:cc:cc:ff:fe:e6:8e:ca','bd:27:0b:38:37:95:dc:87','00:12:4b:00:1c:a1:b8:46',0,0,1,0,0,2,0,0,255); +INSERT INTO neighbors_v8 VALUES('cc:cc:cc:ff:fe:e6:8e:ca','bd:27:0b:38:37:95:dc:87','ec:1b:bd:ff:fe:2f:41:a4',44170,1,1,2,0,2,0,15,255); +CREATE TABLE node_descriptors_v8 ( + ieee ieee NOT NULL, + + logical_type INTEGER NOT NULL, + complex_descriptor_available INTEGER NOT NULL, + user_descriptor_available INTEGER NOT NULL, + reserved INTEGER NOT NULL, + aps_flags INTEGER NOT NULL, + frequency_band INTEGER NOT NULL, + mac_capability_flags INTEGER NOT NULL, + manufacturer_code INTEGER NOT NULL, + maximum_buffer_size INTEGER NOT NULL, + maximum_incoming_transfer_size INTEGER NOT NULL, + server_mask INTEGER NOT NULL, + maximum_outgoing_transfer_size INTEGER NOT NULL, + descriptor_capability_field INTEGER NOT NULL, + + FOREIGN KEY(ieee) + REFERENCES devices_v8(ieee) + ON DELETE CASCADE +); +INSERT INTO node_descriptors_v8 VALUES('00:12:4b:00:1c:a1:b8:46',0,0,0,0,0,8,143,43981,82,128,11329,128,0); +INSERT INTO node_descriptors_v8 VALUES('ec:1b:bd:ff:fe:2f:41:a4',1,0,0,0,0,8,142,4688,82,82,11264,82,0); +INSERT INTO node_descriptors_v8 VALUES('cc:cc:cc:ff:fe:e6:8e:ca',1,0,0,0,0,8,142,4688,82,82,11264,82,0); +INSERT INTO node_descriptors_v8 VALUES('00:0b:57:ff:fe:2b:d4:57',2,0,0,0,0,8,128,4476,82,82,11264,82,0); +CREATE TABLE out_clusters_v8 ( + ieee ieee NOT NULL, + endpoint_id INTEGER NOT NULL, + cluster INTEGER NOT NULL, + + FOREIGN KEY(ieee, endpoint_id) + REFERENCES endpoints_v8(ieee, endpoint_id) + ON DELETE CASCADE +); +INSERT INTO out_clusters_v8 VALUES('00:12:4b:00:1c:a1:b8:46',1,1280); +INSERT INTO out_clusters_v8 VALUES('ec:1b:bd:ff:fe:2f:41:a4',1,10); +INSERT INTO out_clusters_v8 VALUES('ec:1b:bd:ff:fe:2f:41:a4',1,25); +INSERT INTO out_clusters_v8 VALUES('ec:1b:bd:ff:fe:2f:41:a4',242,33); +INSERT INTO out_clusters_v8 VALUES('cc:cc:cc:ff:fe:e6:8e:ca',1,10); +INSERT INTO out_clusters_v8 VALUES('cc:cc:cc:ff:fe:e6:8e:ca',1,25); +INSERT INTO out_clusters_v8 VALUES('cc:cc:cc:ff:fe:e6:8e:ca',242,33); +INSERT INTO out_clusters_v8 VALUES('00:0b:57:ff:fe:2b:d4:57',1,3); +INSERT INTO out_clusters_v8 VALUES('00:0b:57:ff:fe:2b:d4:57',1,4); +INSERT INTO out_clusters_v8 VALUES('00:0b:57:ff:fe:2b:d4:57',1,6); +INSERT INTO out_clusters_v8 VALUES('00:0b:57:ff:fe:2b:d4:57',1,8); +INSERT INTO out_clusters_v8 VALUES('00:0b:57:ff:fe:2b:d4:57',1,25); +INSERT INTO out_clusters_v8 VALUES('00:0b:57:ff:fe:2b:d4:57',1,4096); +CREATE TABLE attributes_cache_v8 ( + ieee ieee NOT NULL, + endpoint_id INTEGER NOT NULL, + cluster INTEGER NOT NULL, + attrid INTEGER NOT NULL, + value BLOB NOT NULL, + + -- Quirks can create "virtual" clusters and endpoints that won't be present in the + -- DB but whose values still need to be cached + FOREIGN KEY(ieee) + REFERENCES devices_v8(ieee) + ON DELETE CASCADE +); +INSERT INTO attributes_cache_v8 VALUES('ec:1b:bd:ff:fe:2f:41:a4',1,0,4,'The Home Depot'); +INSERT INTO attributes_cache_v8 VALUES('ec:1b:bd:ff:fe:2f:41:a4',1,0,5,'Ecosmart-ZBT-A19-CCT-Bulb'); +INSERT INTO attributes_cache_v8 VALUES('ec:1b:bd:ff:fe:2f:41:a4',1,6,0,1); +INSERT INTO attributes_cache_v8 VALUES('ec:1b:bd:ff:fe:2f:41:a4',1,6,16387,1); +INSERT INTO attributes_cache_v8 VALUES('ec:1b:bd:ff:fe:2f:41:a4',1,8,0,254); +INSERT INTO attributes_cache_v8 VALUES('ec:1b:bd:ff:fe:2f:41:a4',1,768,16395,153); +INSERT INTO attributes_cache_v8 VALUES('ec:1b:bd:ff:fe:2f:41:a4',1,768,16396,370); +INSERT INTO attributes_cache_v8 VALUES('ec:1b:bd:ff:fe:2f:41:a4',1,768,16394,16); +INSERT INTO attributes_cache_v8 VALUES('cc:cc:cc:ff:fe:e6:8e:ca',1,0,4,'The Home Depot'); +INSERT INTO attributes_cache_v8 VALUES('cc:cc:cc:ff:fe:e6:8e:ca',1,0,5,'Ecosmart-ZBT-A19-CCT-Bulb'); +INSERT INTO attributes_cache_v8 VALUES('ec:1b:bd:ff:fe:2f:41:a4',1,768,3,30002); +INSERT INTO attributes_cache_v8 VALUES('ec:1b:bd:ff:fe:2f:41:a4',1,768,4,26876); +INSERT INTO attributes_cache_v8 VALUES('ec:1b:bd:ff:fe:2f:41:a4',1,768,7,370); +INSERT INTO attributes_cache_v8 VALUES('cc:cc:cc:ff:fe:e6:8e:ca',1,6,0,1); +INSERT INTO attributes_cache_v8 VALUES('cc:cc:cc:ff:fe:e6:8e:ca',1,8,0,254); +INSERT INTO attributes_cache_v8 VALUES('ec:1b:bd:ff:fe:2f:41:a4',1,768,8,2); +INSERT INTO attributes_cache_v8 VALUES('cc:cc:cc:ff:fe:e6:8e:ca',1,768,8,2); +INSERT INTO attributes_cache_v8 VALUES('cc:cc:cc:ff:fe:e6:8e:ca',1,768,7,370); +INSERT INTO attributes_cache_v8 VALUES('cc:cc:cc:ff:fe:e6:8e:ca',1,768,3,30002); +INSERT INTO attributes_cache_v8 VALUES('cc:cc:cc:ff:fe:e6:8e:ca',1,768,4,26876); +INSERT INTO attributes_cache_v8 VALUES('cc:cc:cc:ff:fe:e6:8e:ca',1,6,16387,1); +INSERT INTO attributes_cache_v8 VALUES('cc:cc:cc:ff:fe:e6:8e:ca',1,768,16395,153); +INSERT INTO attributes_cache_v8 VALUES('cc:cc:cc:ff:fe:e6:8e:ca',1,768,16396,370); +INSERT INTO attributes_cache_v8 VALUES('cc:cc:cc:ff:fe:e6:8e:ca',1,768,16394,16); +INSERT INTO attributes_cache_v8 VALUES('00:0b:57:ff:fe:2b:d4:57',1,0,4,'IKEA of Sweden'); +INSERT INTO attributes_cache_v8 VALUES('00:0b:57:ff:fe:2b:d4:57',1,0,5,'TRADFRI wireless dimmer'); +CREATE TABLE groups_v8 ( + group_id INTEGER NOT NULL, + name TEXT NOT NULL +); +INSERT INTO groups_v8 VALUES(0,'Default Lightlink Group'); +CREATE TABLE group_members_v8 ( + group_id INTEGER NOT NULL, + ieee ieee NOT NULL, + endpoint_id INTEGER NOT NULL, + + FOREIGN KEY(group_id) + REFERENCES groups_v8(group_id) + ON DELETE CASCADE, + FOREIGN KEY(ieee, endpoint_id) + REFERENCES endpoints_v8(ieee, endpoint_id) + ON DELETE CASCADE +); +INSERT INTO group_members_v8 VALUES(0,'00:12:4b:00:1c:a1:b8:46',1); +CREATE TABLE relays_v8 ( + ieee ieee NOT NULL, + relays BLOB NOT NULL, + + FOREIGN KEY(ieee) + REFERENCES devices_v8(ieee) + ON DELETE CASCADE +); +INSERT INTO relays_v8 VALUES('ec:1b:bd:ff:fe:2f:41:a4',X'00'); +INSERT INTO relays_v8 VALUES('cc:cc:cc:ff:fe:e6:8e:ca',X'00'); +CREATE TABLE unsupported_attributes_v8 ( + ieee ieee NOT NULL, + endpoint_id INTEGER NOT NULL, + cluster INTEGER NOT NULL, + attrid INTEGER NOT NULL, + + FOREIGN KEY(ieee) + REFERENCES devices_v8(ieee) + ON DELETE CASCADE, + FOREIGN KEY(ieee, endpoint_id, cluster) + REFERENCES in_clusters_v8(ieee, endpoint_id, cluster) + ON DELETE CASCADE +); +INSERT INTO unsupported_attributes_v8 VALUES('ec:1b:bd:ff:fe:2f:41:a4',1,768,16386); +INSERT INTO unsupported_attributes_v8 VALUES('cc:cc:cc:ff:fe:e6:8e:ca',1,768,16386); +CREATE UNIQUE INDEX devices_idx_v8 + ON devices_v8(ieee); +CREATE UNIQUE INDEX endpoint_idx_v8 + ON endpoints_v8(ieee, endpoint_id); +CREATE UNIQUE INDEX in_clusters_idx_v8 + ON in_clusters_v8(ieee, endpoint_id, cluster); +CREATE INDEX neighbors_idx_v8 + ON neighbors_v8(device_ieee); +CREATE UNIQUE INDEX node_descriptors_idx_v8 + ON node_descriptors_v8(ieee); +CREATE UNIQUE INDEX out_clusters_idx_v8 + ON out_clusters_v8(ieee, endpoint_id, cluster); +CREATE UNIQUE INDEX attributes_idx_v8 + ON attributes_cache_v8(ieee, endpoint_id, cluster, attrid); +CREATE UNIQUE INDEX groups_idx_v8 + ON groups_v8(group_id); +CREATE UNIQUE INDEX group_members_idx_v8 + ON group_members_v8(group_id, ieee, endpoint_id); +CREATE UNIQUE INDEX relays_idx_v8 + ON relays_v8(ieee); +CREATE UNIQUE INDEX unsupported_attributes_idx_v8 + ON unsupported_attributes_v8(ieee, endpoint_id, cluster, attrid); +COMMIT; \ No newline at end of file diff --git a/tests/test_appdb_migration.py b/tests/test_appdb_migration.py index b880ae271..2b13b3ed7 100644 --- a/tests/test_appdb_migration.py +++ b/tests/test_appdb_migration.py @@ -506,3 +506,11 @@ async def test_last_seen_migration(test_db): app = await make_app(test_db_v5) assert isinstance(app.get_device(nwk=0xBD4D).last_seen, float) await app.pre_shutdown() + + +async def test_last_seen_migration_v8_to_v9(test_db): + test_db_v8 = test_db("simple_v8.sql") + + app = await make_app(test_db_v8) + assert int(app.get_device(nwk=0xE01E).last_seen) == 1651119830 + await app.pre_shutdown() diff --git a/zigpy/appdb.py b/zigpy/appdb.py index 65e13389e..bdb68b469 100644 --- a/zigpy/appdb.py +++ b/zigpy/appdb.py @@ -24,7 +24,7 @@ LOGGER = logging.getLogger(__name__) -DB_VERSION = 8 +DB_VERSION = 9 DB_V = f"_v{DB_VERSION}" MIN_SQLITE_VERSION = (3, 24, 0) @@ -77,16 +77,6 @@ def convert_ieee(s): sqlite3.register_converter("ieee", convert_ieee) - def adapt_datetime(dt): - return int(dt.timestamp() * 1000) - - sqlite3.register_adapter(datetime, adapt_datetime) - - def convert_timestamp(ts): - return datetime.fromtimestamp(int(ts.decode("ascii"), 10) / 1000, timezone.utc) - - sqlite3.register_converter("unix_timestamp", convert_timestamp) - def aiosqlite_connect( database: str, iter_chunk_size: int = 64, **kwargs @@ -228,7 +218,8 @@ def device_last_seen_updated( async def _save_device_last_seen(self, ieee: t.EUI64, last_seen: datetime) -> None: await self.execute( - f"UPDATE devices{DB_V} SET last_seen=? WHERE ieee=?", (last_seen, ieee) + f"UPDATE devices{DB_V} SET last_seen=? WHERE ieee=?", + (last_seen.timestamp(), ieee), ) await self._db.commit() @@ -372,7 +363,13 @@ async def _save_device(self, device: zigpy.typing.DeviceType) -> None: status=excluded.status, last_seen=excluded.last_seen""" await self.execute( - q, (device.ieee, device.nwk, device.status, device._last_seen or UNIX_EPOCH) + q, + ( + device.ieee, + device.nwk, + device.status, + (device._last_seen or UNIX_EPOCH).timestamp(), + ), ) if device.node_desc is not None: @@ -566,8 +563,8 @@ async def _load_devices(self) -> None: dev = self._application.add_device(ieee, nwk) dev.status = zigpy.device.Status(status) - if last_seen > UNIX_EPOCH: - dev._last_seen = last_seen + if last_seen > 0: + dev.last_seen = last_seen async def _load_node_descriptors(self) -> None: async with self.execute(f"SELECT * FROM node_descriptors{DB_V}") as cursor: @@ -686,6 +683,7 @@ async def _run_migrations(self): (self._migrate_to_v6, 6), (self._migrate_to_v7, 7), (self._migrate_to_v8, 8), + (self._migrate_to_v9, 9), ]: if db_version >= min(to_db_version, DB_VERSION): continue @@ -890,7 +888,7 @@ async def _migrate_to_v8(self): # Set the default `last_seen` to the unix epoch await self.execute( "INSERT INTO devices_v8 VALUES (?, ?, ?, ?)", - (ieee, nwk, status, UNIX_EPOCH), + (ieee, nwk, status, 0), ) # Copy the devices table first, it should have no conflicts @@ -909,3 +907,27 @@ async def _migrate_to_v8(self): "devices_v7": None, } ) + + async def _migrate_to_v9(self): + """Schema v9 changed the data type of the `devices_v8.last_seen` column.""" + + await self.execute( + """INSERT INTO devices_v9 (ieee, nwk, status, last_seen) + SELECT ieee, nwk, status, last_seen / 1000.0 FROM devices_v8""" + ) + + await self._migrate_tables( + { + "endpoints_v8": "endpoints_v9", + "in_clusters_v8": "in_clusters_v9", + "out_clusters_v8": "out_clusters_v9", + "groups_v8": "groups_v9", + "group_members_v8": "group_members_v9", + "relays_v8": "relays_v9", + "attributes_cache_v8": "attributes_cache_v9", + "neighbors_v8": "neighbors_v9", + "node_descriptors_v8": "node_descriptors_v9", + "unsupported_attributes_v8": "unsupported_attributes_v9", + "devices_v8": None, + } + ) diff --git a/zigpy/appdb_schemas/schema_v9.sql b/zigpy/appdb_schemas/schema_v9.sql new file mode 100644 index 000000000..565779279 --- /dev/null +++ b/zigpy/appdb_schemas/schema_v9.sql @@ -0,0 +1,201 @@ +PRAGMA user_version = 9; + +-- devices +DROP TABLE IF EXISTS devices_v9; +CREATE TABLE devices_v9 ( + ieee ieee NOT NULL, + nwk INTEGER NOT NULL, + status INTEGER NOT NULL, + last_seen REAL NOT NULL +); + +CREATE UNIQUE INDEX devices_idx_v9 + ON devices_v9(ieee); + + +-- endpoints +DROP TABLE IF EXISTS endpoints_v9; +CREATE TABLE endpoints_v9 ( + ieee ieee NOT NULL, + endpoint_id INTEGER NOT NULL, + profile_id INTEGER NOT NULL, + device_type INTEGER NOT NULL, + status INTEGER NOT NULL, + + FOREIGN KEY(ieee) + REFERENCES devices_v9(ieee) + ON DELETE CASCADE +); + +CREATE UNIQUE INDEX endpoint_idx_v9 + ON endpoints_v9(ieee, endpoint_id); + + +-- clusters +DROP TABLE IF EXISTS in_clusters_v9; +CREATE TABLE in_clusters_v9 ( + ieee ieee NOT NULL, + endpoint_id INTEGER NOT NULL, + cluster INTEGER NOT NULL, + + FOREIGN KEY(ieee, endpoint_id) + REFERENCES endpoints_v9(ieee, endpoint_id) + ON DELETE CASCADE +); + +CREATE UNIQUE INDEX in_clusters_idx_v9 + ON in_clusters_v9(ieee, endpoint_id, cluster); + + +-- neighbors +DROP TABLE IF EXISTS neighbors_v9; +CREATE TABLE neighbors_v9 ( + device_ieee ieee NOT NULL, + extended_pan_id ieee NOT NULL, + ieee ieee NOT NULL, + nwk INTEGER NOT NULL, + device_type INTEGER NOT NULL, + rx_on_when_idle INTEGER NOT NULL, + relationship INTEGER NOT NULL, + reserved1 INTEGER NOT NULL, + permit_joining INTEGER NOT NULL, + reserved2 INTEGER NOT NULL, + depth INTEGER NOT NULL, + lqi INTEGER NOT NULL, + + FOREIGN KEY(device_ieee) + REFERENCES devices_v9(ieee) + ON DELETE CASCADE +); + +CREATE INDEX neighbors_idx_v9 + ON neighbors_v9(device_ieee); + + +-- node descriptors +DROP TABLE IF EXISTS node_descriptors_v9; +CREATE TABLE node_descriptors_v9 ( + ieee ieee NOT NULL, + + logical_type INTEGER NOT NULL, + complex_descriptor_available INTEGER NOT NULL, + user_descriptor_available INTEGER NOT NULL, + reserved INTEGER NOT NULL, + aps_flags INTEGER NOT NULL, + frequency_band INTEGER NOT NULL, + mac_capability_flags INTEGER NOT NULL, + manufacturer_code INTEGER NOT NULL, + maximum_buffer_size INTEGER NOT NULL, + maximum_incoming_transfer_size INTEGER NOT NULL, + server_mask INTEGER NOT NULL, + maximum_outgoing_transfer_size INTEGER NOT NULL, + descriptor_capability_field INTEGER NOT NULL, + + FOREIGN KEY(ieee) + REFERENCES devices_v9(ieee) + ON DELETE CASCADE +); + +CREATE UNIQUE INDEX node_descriptors_idx_v9 + ON node_descriptors_v9(ieee); + + +-- output clusters +DROP TABLE IF EXISTS out_clusters_v9; +CREATE TABLE out_clusters_v9 ( + ieee ieee NOT NULL, + endpoint_id INTEGER NOT NULL, + cluster INTEGER NOT NULL, + + FOREIGN KEY(ieee, endpoint_id) + REFERENCES endpoints_v9(ieee, endpoint_id) + ON DELETE CASCADE +); + +CREATE UNIQUE INDEX out_clusters_idx_v9 + ON out_clusters_v9(ieee, endpoint_id, cluster); + + +-- attributes +DROP TABLE IF EXISTS attributes_cache_v9; +CREATE TABLE attributes_cache_v9 ( + ieee ieee NOT NULL, + endpoint_id INTEGER NOT NULL, + cluster INTEGER NOT NULL, + attrid INTEGER NOT NULL, + value BLOB NOT NULL, + + -- Quirks can create "virtual" clusters and endpoints that won't be present in the + -- DB but whose values still need to be cached + FOREIGN KEY(ieee) + REFERENCES devices_v9(ieee) + ON DELETE CASCADE +); + +CREATE UNIQUE INDEX attributes_idx_v9 + ON attributes_cache_v9(ieee, endpoint_id, cluster, attrid); + + +-- groups +DROP TABLE IF EXISTS groups_v9; +CREATE TABLE groups_v9 ( + group_id INTEGER NOT NULL, + name TEXT NOT NULL +); + +CREATE UNIQUE INDEX groups_idx_v9 + ON groups_v9(group_id); + + +-- group members +DROP TABLE IF EXISTS group_members_v9; +CREATE TABLE group_members_v9 ( + group_id INTEGER NOT NULL, + ieee ieee NOT NULL, + endpoint_id INTEGER NOT NULL, + + FOREIGN KEY(group_id) + REFERENCES groups_v9(group_id) + ON DELETE CASCADE, + FOREIGN KEY(ieee, endpoint_id) + REFERENCES endpoints_v9(ieee, endpoint_id) + ON DELETE CASCADE +); + +CREATE UNIQUE INDEX group_members_idx_v9 + ON group_members_v9(group_id, ieee, endpoint_id); + + +-- relays +DROP TABLE IF EXISTS relays_v9; +CREATE TABLE relays_v9 ( + ieee ieee NOT NULL, + relays BLOB NOT NULL, + + FOREIGN KEY(ieee) + REFERENCES devices_v9(ieee) + ON DELETE CASCADE +); + +CREATE UNIQUE INDEX relays_idx_v9 + ON relays_v9(ieee); + + +-- unsupported attributes +DROP TABLE IF EXISTS unsupported_attributes_v9; +CREATE TABLE unsupported_attributes_v9 ( + ieee ieee NOT NULL, + endpoint_id INTEGER NOT NULL, + cluster INTEGER NOT NULL, + attrid INTEGER NOT NULL, + + FOREIGN KEY(ieee) + REFERENCES devices_v9(ieee) + ON DELETE CASCADE, + FOREIGN KEY(ieee, endpoint_id, cluster) + REFERENCES in_clusters_v9(ieee, endpoint_id, cluster) + ON DELETE CASCADE +); + +CREATE UNIQUE INDEX unsupported_attributes_idx_v9 + ON unsupported_attributes_v9(ieee, endpoint_id, cluster, attrid); From 2eec01d813ddb9b936a6ee0fe89d5c1bd8205e90 Mon Sep 17 00:00:00 2001 From: puddly <32534428+puddly@users.noreply.github.com> Date: Thu, 28 Apr 2022 00:34:09 -0400 Subject: [PATCH 2/2] Add a unit test to ensure `DB_VERSION` matches latest schema --- tests/test_appdb_migration.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/test_appdb_migration.py b/tests/test_appdb_migration.py index 2b13b3ed7..6a1616ee4 100644 --- a/tests/test_appdb_migration.py +++ b/tests/test_appdb_migration.py @@ -6,6 +6,7 @@ import zigpy.appdb from zigpy.appdb import sqlite3 +import zigpy.appdb_schemas import zigpy.types as t from zigpy.zdo import types as zdo_t @@ -490,7 +491,7 @@ async def test_migration_missing_tables(): await appdb.shutdown() -async def test_last_seen_migration(test_db): +async def test_last_seen_initial_migration(test_db): test_db_v5 = test_db("simple_v5.sql") # To preserve the old behavior, `0` will not be exposed to ZHA, only `None` @@ -508,6 +509,10 @@ async def test_last_seen_migration(test_db): await app.pre_shutdown() +def test_db_version_is_latest_schema_version(): + assert zigpy.appdb.DB_VERSION == max(zigpy.appdb_schemas.SCHEMAS.keys()) + + async def test_last_seen_migration_v8_to_v9(test_db): test_db_v8 = test_db("simple_v8.sql")