Skip to content

Commit

Permalink
Merge pull request #983 from puddly/puddly/migrate-last_seen-to-real
Browse files Browse the repository at this point in the history 
Migrate `last_seen` to SQLite's `REAL` data type
  • Loading branch information
@puddly
puddly committed May 7, 2022
2 parents 64c2a28 + 2eec01d commit 8fefb7c
Show file tree
Hide file tree
Showing 4 changed files with 494 additions and 17 deletions.
241 changes: 241 additions & 0 deletions tests/databases/simple_v8.sql
View file
@@ -0,0 +1,241 @@
PRAGMA user_version=8;
PRAGMA foreign_keys=OFF;
BEGIN TRANSACTION;
CREATE TABLE devices_v8 (
ieee ieee NOT NULL,
nwk INTEGER NOT NULL,
status INTEGER NOT NULL,
last_seen unix_timestamp NOT NULL
);
INSERT INTO devices_v8 VALUES('00:12:4b:00:1c:a1:b8:46',0,2,1651119833288);
INSERT INTO devices_v8 VALUES('ec:1b:bd:ff:fe:2f:41:a4',44170,2,1651119836445);
INSERT INTO devices_v8 VALUES('cc:cc:cc:ff:fe:e6:8e:ca',50064,2,1651119839551);
INSERT INTO devices_v8 VALUES('00:0b:57:ff:fe:2b:d4:57',57374,2,1651119830048);
CREATE TABLE endpoints_v8 (
ieee ieee NOT NULL,
endpoint_id INTEGER NOT NULL,
profile_id INTEGER NOT NULL,
device_type INTEGER NOT NULL,
status INTEGER NOT NULL,

FOREIGN KEY(ieee)
REFERENCES devices_v8(ieee)
ON DELETE CASCADE
);
INSERT INTO endpoints_v8 VALUES('00:12:4b:00:1c:a1:b8:46',1,260,48879,1);
INSERT INTO endpoints_v8 VALUES('ec:1b:bd:ff:fe:2f:41:a4',1,260,268,1);
INSERT INTO endpoints_v8 VALUES('ec:1b:bd:ff:fe:2f:41:a4',242,41440,97,1);
INSERT INTO endpoints_v8 VALUES('cc:cc:cc:ff:fe:e6:8e:ca',1,260,268,1);
INSERT INTO endpoints_v8 VALUES('cc:cc:cc:ff:fe:e6:8e:ca',242,41440,97,1);
INSERT INTO endpoints_v8 VALUES('00:0b:57:ff:fe:2b:d4:57',1,260,2080,1);
CREATE TABLE in_clusters_v8 (
ieee ieee NOT NULL,
endpoint_id INTEGER NOT NULL,
cluster INTEGER NOT NULL,

FOREIGN KEY(ieee, endpoint_id)
REFERENCES endpoints_v8(ieee, endpoint_id)
ON DELETE CASCADE
);
INSERT INTO in_clusters_v8 VALUES('ec:1b:bd:ff:fe:2f:41:a4',1,0);
INSERT INTO in_clusters_v8 VALUES('ec:1b:bd:ff:fe:2f:41:a4',1,3);
INSERT INTO in_clusters_v8 VALUES('ec:1b:bd:ff:fe:2f:41:a4',1,4);
INSERT INTO in_clusters_v8 VALUES('ec:1b:bd:ff:fe:2f:41:a4',1,5);
INSERT INTO in_clusters_v8 VALUES('ec:1b:bd:ff:fe:2f:41:a4',1,6);
INSERT INTO in_clusters_v8 VALUES('ec:1b:bd:ff:fe:2f:41:a4',1,8);
INSERT INTO in_clusters_v8 VALUES('ec:1b:bd:ff:fe:2f:41:a4',1,768);
INSERT INTO in_clusters_v8 VALUES('ec:1b:bd:ff:fe:2f:41:a4',1,2821);
INSERT INTO in_clusters_v8 VALUES('ec:1b:bd:ff:fe:2f:41:a4',1,4096);
INSERT INTO in_clusters_v8 VALUES('ec:1b:bd:ff:fe:2f:41:a4',1,64642);
INSERT INTO in_clusters_v8 VALUES('cc:cc:cc:ff:fe:e6:8e:ca',1,0);
INSERT INTO in_clusters_v8 VALUES('cc:cc:cc:ff:fe:e6:8e:ca',1,3);
INSERT INTO in_clusters_v8 VALUES('cc:cc:cc:ff:fe:e6:8e:ca',1,4);
INSERT INTO in_clusters_v8 VALUES('cc:cc:cc:ff:fe:e6:8e:ca',1,5);
INSERT INTO in_clusters_v8 VALUES('cc:cc:cc:ff:fe:e6:8e:ca',1,6);
INSERT INTO in_clusters_v8 VALUES('cc:cc:cc:ff:fe:e6:8e:ca',1,8);
INSERT INTO in_clusters_v8 VALUES('cc:cc:cc:ff:fe:e6:8e:ca',1,768);
INSERT INTO in_clusters_v8 VALUES('cc:cc:cc:ff:fe:e6:8e:ca',1,2821);
INSERT INTO in_clusters_v8 VALUES('cc:cc:cc:ff:fe:e6:8e:ca',1,4096);
INSERT INTO in_clusters_v8 VALUES('cc:cc:cc:ff:fe:e6:8e:ca',1,64642);
INSERT INTO in_clusters_v8 VALUES('00:0b:57:ff:fe:2b:d4:57',1,0);
INSERT INTO in_clusters_v8 VALUES('00:0b:57:ff:fe:2b:d4:57',1,1);
INSERT INTO in_clusters_v8 VALUES('00:0b:57:ff:fe:2b:d4:57',1,3);
INSERT INTO in_clusters_v8 VALUES('00:0b:57:ff:fe:2b:d4:57',1,32);
INSERT INTO in_clusters_v8 VALUES('00:0b:57:ff:fe:2b:d4:57',1,4096);
CREATE TABLE neighbors_v8 (
device_ieee ieee NOT NULL,
extended_pan_id ieee NOT NULL,
ieee ieee NOT NULL,
nwk INTEGER NOT NULL,
device_type INTEGER NOT NULL,
rx_on_when_idle INTEGER NOT NULL,
relationship INTEGER NOT NULL,
reserved1 INTEGER NOT NULL,
permit_joining INTEGER NOT NULL,
reserved2 INTEGER NOT NULL,
depth INTEGER NOT NULL,
lqi INTEGER NOT NULL,

FOREIGN KEY(device_ieee)
REFERENCES devices_v8(ieee)
ON DELETE CASCADE
);
INSERT INTO neighbors_v8 VALUES('00:12:4b:00:1c:a1:b8:46','bd:27:0b:38:37:95:dc:87','ec:1b:bd:ff:fe:2f:41:a4',44170,1,1,2,0,2,0,15,255);
INSERT INTO neighbors_v8 VALUES('00:12:4b:00:1c:a1:b8:46','bd:27:0b:38:37:95:dc:87','cc:cc:cc:ff:fe:e6:8e:ca',50064,1,1,2,0,2,0,15,255);
INSERT INTO neighbors_v8 VALUES('00:12:4b:00:1c:a1:b8:46','bd:27:0b:38:37:95:dc:87','00:0b:57:ff:fe:2b:d4:57',57374,2,0,1,0,0,0,1,255);
INSERT INTO neighbors_v8 VALUES('ec:1b:bd:ff:fe:2f:41:a4','bd:27:0b:38:37:95:dc:87','00:12:4b:00:1c:a1:b8:46',0,0,1,2,0,2,0,0,253);
INSERT INTO neighbors_v8 VALUES('ec:1b:bd:ff:fe:2f:41:a4','bd:27:0b:38:37:95:dc:87','cc:cc:cc:ff:fe:e6:8e:ca',50064,1,1,0,0,2,0,15,255);
INSERT INTO neighbors_v8 VALUES('cc:cc:cc:ff:fe:e6:8e:ca','bd:27:0b:38:37:95:dc:87','00:12:4b:00:1c:a1:b8:46',0,0,1,0,0,2,0,0,255);
INSERT INTO neighbors_v8 VALUES('cc:cc:cc:ff:fe:e6:8e:ca','bd:27:0b:38:37:95:dc:87','ec:1b:bd:ff:fe:2f:41:a4',44170,1,1,2,0,2,0,15,255);
CREATE TABLE node_descriptors_v8 (
ieee ieee NOT NULL,

logical_type INTEGER NOT NULL,
complex_descriptor_available INTEGER NOT NULL,
user_descriptor_available INTEGER NOT NULL,
reserved INTEGER NOT NULL,
aps_flags INTEGER NOT NULL,
frequency_band INTEGER NOT NULL,
mac_capability_flags INTEGER NOT NULL,
manufacturer_code INTEGER NOT NULL,
maximum_buffer_size INTEGER NOT NULL,
maximum_incoming_transfer_size INTEGER NOT NULL,
server_mask INTEGER NOT NULL,
maximum_outgoing_transfer_size INTEGER NOT NULL,
descriptor_capability_field INTEGER NOT NULL,

FOREIGN KEY(ieee)
REFERENCES devices_v8(ieee)
ON DELETE CASCADE
);
INSERT INTO node_descriptors_v8 VALUES('00:12:4b:00:1c:a1:b8:46',0,0,0,0,0,8,143,43981,82,128,11329,128,0);
INSERT INTO node_descriptors_v8 VALUES('ec:1b:bd:ff:fe:2f:41:a4',1,0,0,0,0,8,142,4688,82,82,11264,82,0);
INSERT INTO node_descriptors_v8 VALUES('cc:cc:cc:ff:fe:e6:8e:ca',1,0,0,0,0,8,142,4688,82,82,11264,82,0);
INSERT INTO node_descriptors_v8 VALUES('00:0b:57:ff:fe:2b:d4:57',2,0,0,0,0,8,128,4476,82,82,11264,82,0);
CREATE TABLE out_clusters_v8 (
ieee ieee NOT NULL,
endpoint_id INTEGER NOT NULL,
cluster INTEGER NOT NULL,

FOREIGN KEY(ieee, endpoint_id)
REFERENCES endpoints_v8(ieee, endpoint_id)
ON DELETE CASCADE
);
INSERT INTO out_clusters_v8 VALUES('00:12:4b:00:1c:a1:b8:46',1,1280);
INSERT INTO out_clusters_v8 VALUES('ec:1b:bd:ff:fe:2f:41:a4',1,10);
INSERT INTO out_clusters_v8 VALUES('ec:1b:bd:ff:fe:2f:41:a4',1,25);
INSERT INTO out_clusters_v8 VALUES('ec:1b:bd:ff:fe:2f:41:a4',242,33);
INSERT INTO out_clusters_v8 VALUES('cc:cc:cc:ff:fe:e6:8e:ca',1,10);
INSERT INTO out_clusters_v8 VALUES('cc:cc:cc:ff:fe:e6:8e:ca',1,25);
INSERT INTO out_clusters_v8 VALUES('cc:cc:cc:ff:fe:e6:8e:ca',242,33);
INSERT INTO out_clusters_v8 VALUES('00:0b:57:ff:fe:2b:d4:57',1,3);
INSERT INTO out_clusters_v8 VALUES('00:0b:57:ff:fe:2b:d4:57',1,4);
INSERT INTO out_clusters_v8 VALUES('00:0b:57:ff:fe:2b:d4:57',1,6);
INSERT INTO out_clusters_v8 VALUES('00:0b:57:ff:fe:2b:d4:57',1,8);
INSERT INTO out_clusters_v8 VALUES('00:0b:57:ff:fe:2b:d4:57',1,25);
INSERT INTO out_clusters_v8 VALUES('00:0b:57:ff:fe:2b:d4:57',1,4096);
CREATE TABLE attributes_cache_v8 (
ieee ieee NOT NULL,
endpoint_id INTEGER NOT NULL,
cluster INTEGER NOT NULL,
attrid INTEGER NOT NULL,
value BLOB NOT NULL,

-- Quirks can create "virtual" clusters and endpoints that won't be present in the
-- DB but whose values still need to be cached
FOREIGN KEY(ieee)
REFERENCES devices_v8(ieee)
ON DELETE CASCADE
);
INSERT INTO attributes_cache_v8 VALUES('ec:1b:bd:ff:fe:2f:41:a4',1,0,4,'The Home Depot');
INSERT INTO attributes_cache_v8 VALUES('ec:1b:bd:ff:fe:2f:41:a4',1,0,5,'Ecosmart-ZBT-A19-CCT-Bulb');
INSERT INTO attributes_cache_v8 VALUES('ec:1b:bd:ff:fe:2f:41:a4',1,6,0,1);
INSERT INTO attributes_cache_v8 VALUES('ec:1b:bd:ff:fe:2f:41:a4',1,6,16387,1);
INSERT INTO attributes_cache_v8 VALUES('ec:1b:bd:ff:fe:2f:41:a4',1,8,0,254);
INSERT INTO attributes_cache_v8 VALUES('ec:1b:bd:ff:fe:2f:41:a4',1,768,16395,153);
INSERT INTO attributes_cache_v8 VALUES('ec:1b:bd:ff:fe:2f:41:a4',1,768,16396,370);
INSERT INTO attributes_cache_v8 VALUES('ec:1b:bd:ff:fe:2f:41:a4',1,768,16394,16);
INSERT INTO attributes_cache_v8 VALUES('cc:cc:cc:ff:fe:e6:8e:ca',1,0,4,'The Home Depot');
INSERT INTO attributes_cache_v8 VALUES('cc:cc:cc:ff:fe:e6:8e:ca',1,0,5,'Ecosmart-ZBT-A19-CCT-Bulb');
INSERT INTO attributes_cache_v8 VALUES('ec:1b:bd:ff:fe:2f:41:a4',1,768,3,30002);
INSERT INTO attributes_cache_v8 VALUES('ec:1b:bd:ff:fe:2f:41:a4',1,768,4,26876);
INSERT INTO attributes_cache_v8 VALUES('ec:1b:bd:ff:fe:2f:41:a4',1,768,7,370);
INSERT INTO attributes_cache_v8 VALUES('cc:cc:cc:ff:fe:e6:8e:ca',1,6,0,1);
INSERT INTO attributes_cache_v8 VALUES('cc:cc:cc:ff:fe:e6:8e:ca',1,8,0,254);
INSERT INTO attributes_cache_v8 VALUES('ec:1b:bd:ff:fe:2f:41:a4',1,768,8,2);
INSERT INTO attributes_cache_v8 VALUES('cc:cc:cc:ff:fe:e6:8e:ca',1,768,8,2);
INSERT INTO attributes_cache_v8 VALUES('cc:cc:cc:ff:fe:e6:8e:ca',1,768,7,370);
INSERT INTO attributes_cache_v8 VALUES('cc:cc:cc:ff:fe:e6:8e:ca',1,768,3,30002);
INSERT INTO attributes_cache_v8 VALUES('cc:cc:cc:ff:fe:e6:8e:ca',1,768,4,26876);
INSERT INTO attributes_cache_v8 VALUES('cc:cc:cc:ff:fe:e6:8e:ca',1,6,16387,1);
INSERT INTO attributes_cache_v8 VALUES('cc:cc:cc:ff:fe:e6:8e:ca',1,768,16395,153);
INSERT INTO attributes_cache_v8 VALUES('cc:cc:cc:ff:fe:e6:8e:ca',1,768,16396,370);
INSERT INTO attributes_cache_v8 VALUES('cc:cc:cc:ff:fe:e6:8e:ca',1,768,16394,16);
INSERT INTO attributes_cache_v8 VALUES('00:0b:57:ff:fe:2b:d4:57',1,0,4,'IKEA of Sweden');
INSERT INTO attributes_cache_v8 VALUES('00:0b:57:ff:fe:2b:d4:57',1,0,5,'TRADFRI wireless dimmer');
CREATE TABLE groups_v8 (
group_id INTEGER NOT NULL,
name TEXT NOT NULL
);
INSERT INTO groups_v8 VALUES(0,'Default Lightlink Group');
CREATE TABLE group_members_v8 (
group_id INTEGER NOT NULL,
ieee ieee NOT NULL,
endpoint_id INTEGER NOT NULL,

FOREIGN KEY(group_id)
REFERENCES groups_v8(group_id)
ON DELETE CASCADE,
FOREIGN KEY(ieee, endpoint_id)
REFERENCES endpoints_v8(ieee, endpoint_id)
ON DELETE CASCADE
);
INSERT INTO group_members_v8 VALUES(0,'00:12:4b:00:1c:a1:b8:46',1);
CREATE TABLE relays_v8 (
ieee ieee NOT NULL,
relays BLOB NOT NULL,

FOREIGN KEY(ieee)
REFERENCES devices_v8(ieee)
ON DELETE CASCADE
);
INSERT INTO relays_v8 VALUES('ec:1b:bd:ff:fe:2f:41:a4',X'00');
INSERT INTO relays_v8 VALUES('cc:cc:cc:ff:fe:e6:8e:ca',X'00');
CREATE TABLE unsupported_attributes_v8 (
ieee ieee NOT NULL,
endpoint_id INTEGER NOT NULL,
cluster INTEGER NOT NULL,
attrid INTEGER NOT NULL,

FOREIGN KEY(ieee)
REFERENCES devices_v8(ieee)
ON DELETE CASCADE,
FOREIGN KEY(ieee, endpoint_id, cluster)
REFERENCES in_clusters_v8(ieee, endpoint_id, cluster)
ON DELETE CASCADE
);
INSERT INTO unsupported_attributes_v8 VALUES('ec:1b:bd:ff:fe:2f:41:a4',1,768,16386);
INSERT INTO unsupported_attributes_v8 VALUES('cc:cc:cc:ff:fe:e6:8e:ca',1,768,16386);
CREATE UNIQUE INDEX devices_idx_v8
ON devices_v8(ieee);
CREATE UNIQUE INDEX endpoint_idx_v8
ON endpoints_v8(ieee, endpoint_id);
CREATE UNIQUE INDEX in_clusters_idx_v8
ON in_clusters_v8(ieee, endpoint_id, cluster);
CREATE INDEX neighbors_idx_v8
ON neighbors_v8(device_ieee);
CREATE UNIQUE INDEX node_descriptors_idx_v8
ON node_descriptors_v8(ieee);
CREATE UNIQUE INDEX out_clusters_idx_v8
ON out_clusters_v8(ieee, endpoint_id, cluster);
CREATE UNIQUE INDEX attributes_idx_v8
ON attributes_cache_v8(ieee, endpoint_id, cluster, attrid);
CREATE UNIQUE INDEX groups_idx_v8
ON groups_v8(group_id);
CREATE UNIQUE INDEX group_members_idx_v8
ON group_members_v8(group_id, ieee, endpoint_id);
CREATE UNIQUE INDEX relays_idx_v8
ON relays_v8(ieee);
CREATE UNIQUE INDEX unsupported_attributes_idx_v8
ON unsupported_attributes_v8(ieee, endpoint_id, cluster, attrid);
COMMIT;
15 changes: 14 additions & 1 deletion tests/test_appdb_migration.py
View file
Expand Up @@ -6,6 +6,7 @@

import zigpy.appdb
from zigpy.appdb import sqlite3
import zigpy.appdb_schemas
import zigpy.types as t
from zigpy.zdo import types as zdo_t

Expand Down Expand Up @@ -490,7 +491,7 @@ async def test_migration_missing_tables():
await appdb.shutdown()


async def test_last_seen_migration(test_db):
async def test_last_seen_initial_migration(test_db):
test_db_v5 = test_db("simple_v5.sql")

# To preserve the old behavior, `0` will not be exposed to ZHA, only `None`
Expand All @@ -506,3 +507,15 @@ async def test_last_seen_migration(test_db):
app = await make_app(test_db_v5)
assert isinstance(app.get_device(nwk=0xBD4D).last_seen, float)
await app.pre_shutdown()


def test_db_version_is_latest_schema_version():
assert zigpy.appdb.DB_VERSION == max(zigpy.appdb_schemas.SCHEMAS.keys())


async def test_last_seen_migration_v8_to_v9(test_db):
test_db_v8 = test_db("simple_v8.sql")

app = await make_app(test_db_v8)
assert int(app.get_device(nwk=0xE01E).last_seen) == 1651119830
await app.pre_shutdown()
54 changes: 38 additions & 16 deletions zigpy/appdb.py
View file
Expand Up @@ -24,7 +24,7 @@

LOGGER = logging.getLogger(__name__)

DB_VERSION = 8
DB_VERSION = 9
DB_V = f"_v{DB_VERSION}"
MIN_SQLITE_VERSION = (3, 24, 0)

Expand Down Expand Up @@ -77,16 +77,6 @@ def convert_ieee(s):

sqlite3.register_converter("ieee", convert_ieee)

def adapt_datetime(dt):
return int(dt.timestamp() * 1000)

sqlite3.register_adapter(datetime, adapt_datetime)

def convert_timestamp(ts):
return datetime.fromtimestamp(int(ts.decode("ascii"), 10) / 1000, timezone.utc)

sqlite3.register_converter("unix_timestamp", convert_timestamp)


def aiosqlite_connect(
database: str, iter_chunk_size: int = 64, **kwargs
Expand Down Expand Up @@ -228,7 +218,8 @@ def device_last_seen_updated(

async def _save_device_last_seen(self, ieee: t.EUI64, last_seen: datetime) -> None:
await self.execute(
f"UPDATE devices{DB_V} SET last_seen=? WHERE ieee=?", (last_seen, ieee)
f"UPDATE devices{DB_V} SET last_seen=? WHERE ieee=?",
(last_seen.timestamp(), ieee),
)
await self._db.commit()

Expand Down Expand Up @@ -372,7 +363,13 @@ async def _save_device(self, device: zigpy.typing.DeviceType) -> None:
status=excluded.status,
last_seen=excluded.last_seen"""
await self.execute(
q, (device.ieee, device.nwk, device.status, device._last_seen or UNIX_EPOCH)
q,
(
device.ieee,
device.nwk,
device.status,
(device._last_seen or UNIX_EPOCH).timestamp(),
),
)

if device.node_desc is not None:
Expand Down Expand Up @@ -566,8 +563,8 @@ async def _load_devices(self) -> None:
dev = self._application.add_device(ieee, nwk)
dev.status = zigpy.device.Status(status)

if last_seen > UNIX_EPOCH:
dev._last_seen = last_seen
if last_seen > 0:
dev.last_seen = last_seen

async def _load_node_descriptors(self) -> None:
async with self.execute(f"SELECT * FROM node_descriptors{DB_V}") as cursor:
Expand Down Expand Up @@ -686,6 +683,7 @@ async def _run_migrations(self):
(self._migrate_to_v6, 6),
(self._migrate_to_v7, 7),
(self._migrate_to_v8, 8),
(self._migrate_to_v9, 9),
]:
if db_version >= min(to_db_version, DB_VERSION):
continue
Expand Down Expand Up @@ -890,7 +888,7 @@ async def _migrate_to_v8(self):
# Set the default `last_seen` to the unix epoch
await self.execute(
"INSERT INTO devices_v8 VALUES (?, ?, ?, ?)",
(ieee, nwk, status, UNIX_EPOCH),
(ieee, nwk, status, 0),
)

# Copy the devices table first, it should have no conflicts
Expand All @@ -909,3 +907,27 @@ async def _migrate_to_v8(self):
"devices_v7": None,
}
)

async def _migrate_to_v9(self):
"""Schema v9 changed the data type of the `devices_v8.last_seen` column."""

await self.execute(
"""INSERT INTO devices_v9 (ieee, nwk, status, last_seen)
SELECT ieee, nwk, status, last_seen / 1000.0 FROM devices_v8"""
)

await self._migrate_tables(
{
"endpoints_v8": "endpoints_v9",
"in_clusters_v8": "in_clusters_v9",
"out_clusters_v8": "out_clusters_v9",
"groups_v8": "groups_v9",
"group_members_v8": "group_members_v9",
"relays_v8": "relays_v9",
"attributes_cache_v8": "attributes_cache_v9",
"neighbors_v8": "neighbors_v9",
"node_descriptors_v8": "node_descriptors_v9",
"unsupported_attributes_v8": "unsupported_attributes_v9",
"devices_v8": None,
}
)

0 comments on commit 8fefb7c

Please sign in to comment.