Remove sqlalchemy lambda_stmt usage from history and statistics

It looks like these need a bit more time to mature per
sqlalchemy/sqlalchemy#8098 (comment)
so let's rip them out for now

While this will make the queries slower, slower and correct
is better than fast and incorrect.

homeassistant/components/logbook/queries/__init__.py

@@ -2,8 +2,9 @@
 from __future__ import annotations
 
 from datetime import datetime as dt
+import json
 
-from sqlalchemy.sql.lambdas import StatementLambdaElement
+from sqlalchemy.sql.selectable import Select
 
 from homeassistant.components.recorder.filters import Filters
 
@@ -21,7 +22,7 @@ def statement_for_request(
     device_ids: list[str] | None = None,
     filters: Filters | None = None,
     context_id: str | None = None,
-) -> StatementLambdaElement:
+) -> Select:
     """Generate the logbook statement for a logbook request."""
 
     # No entities: logbook sends everything for the timeframe
@@ -38,41 +39,36 @@ def statement_for_request(
             context_id,
         )
 
-    # sqlalchemy caches object quoting, the
-    # json quotable ones must be a different
-    # object from the non-json ones to prevent
-    # sqlalchemy from quoting them incorrectly
-
     # entities and devices: logbook sends everything for the timeframe for the entities and devices
     if entity_ids and device_ids:
-        json_quotable_entity_ids = list(entity_ids)
-        json_quotable_device_ids = list(device_ids)
+        json_quoted_entity_ids = [json.dumps(entity_id) for entity_id in entity_ids]
+        json_quoted_device_ids = [json.dumps(device_id) for device_id in device_ids]
         return entities_devices_stmt(
             start_day,
             end_day,
             event_types,
             entity_ids,
-            json_quotable_entity_ids,
-            json_quotable_device_ids,
+            json_quoted_entity_ids,
+            json_quoted_device_ids,
         )
 
     # entities: logbook sends everything for the timeframe for the entities
     if entity_ids:
-        json_quotable_entity_ids = list(entity_ids)
+        json_quoted_entity_ids = [json.dumps(entity_id) for entity_id in entity_ids]
         return entities_stmt(
             start_day,
             end_day,
             event_types,
             entity_ids,
-            json_quotable_entity_ids,
+            json_quoted_entity_ids,
         )
 
     # devices: logbook sends everything for the timeframe for the devices
     assert device_ids is not None
-    json_quotable_device_ids = list(device_ids)
+    json_quoted_device_ids = [json.dumps(device_id) for device_id in device_ids]
     return devices_stmt(
         start_day,
         end_day,
         event_types,
-        json_quotable_device_ids,
+        json_quoted_device_ids,
     )

homeassistant/components/logbook/queries/all.py

@@ -3,10 +3,9 @@
 
 from datetime import datetime as dt
 
-from sqlalchemy import lambda_stmt
 from sqlalchemy.orm import Query
 from sqlalchemy.sql.elements import ClauseList
-from sqlalchemy.sql.lambdas import StatementLambdaElement
+from sqlalchemy.sql.selectable import Select
 
 from homeassistant.components.recorder.models import LAST_UPDATED_INDEX, Events, States
 
@@ -25,32 +24,29 @@ def all_stmt(
     states_entity_filter: ClauseList | None = None,
     events_entity_filter: ClauseList | None = None,
     context_id: str | None = None,
-) -> StatementLambdaElement:
+) -> Select:
     """Generate a logbook query for all entities."""
-    stmt = lambda_stmt(
-        lambda: select_events_without_states(start_day, end_day, event_types)
-    )
+    stmt = select_events_without_states(start_day, end_day, event_types)
     if context_id is not None:
         # Once all the old `state_changed` events
         # are gone from the database remove the
         # _legacy_select_events_context_id()
-        stmt += lambda s: s.where(Events.context_id == context_id).union_all(
+        stmt = stmt.where(Events.context_id == context_id).union_all(
             _states_query_for_context_id(start_day, end_day, context_id),
             legacy_select_events_context_id(start_day, end_day, context_id),
         )
     else:
         if events_entity_filter is not None:
-            stmt += lambda s: s.where(events_entity_filter)
+            stmt = stmt.where(events_entity_filter)
 
         if states_entity_filter is not None:
-            stmt += lambda s: s.union_all(
+            stmt = stmt.union_all(
                 _states_query_for_all(start_day, end_day).where(states_entity_filter)
             )
         else:
-            stmt += lambda s: s.union_all(_states_query_for_all(start_day, end_day))
+            stmt = stmt.union_all(_states_query_for_all(start_day, end_day))
 
-    stmt += lambda s: s.order_by(Events.time_fired)
-    return stmt
+    return stmt.order_by(Events.time_fired)
 
 
 def _states_query_for_all(start_day: dt, end_day: dt) -> Query:

homeassistant/components/logbook/queries/devices.py

@@ -4,11 +4,10 @@
 from collections.abc import Iterable
 from datetime import datetime as dt
 
-from sqlalchemy import lambda_stmt, select
+from sqlalchemy import select
 from sqlalchemy.orm import Query
 from sqlalchemy.sql.elements import ClauseList
-from sqlalchemy.sql.lambdas import StatementLambdaElement
-from sqlalchemy.sql.selectable import CTE, CompoundSelect
+from sqlalchemy.sql.selectable import CTE, CompoundSelect, Select
 
 from homeassistant.components.recorder.models import (
     DEVICE_ID_IN_EVENT,
@@ -31,11 +30,11 @@ def _select_device_id_context_ids_sub_query(
     start_day: dt,
     end_day: dt,
     event_types: tuple[str, ...],
-    json_quotable_device_ids: list[str],
+    json_quoted_device_ids: list[str],
 ) -> CompoundSelect:
     """Generate a subquery to find context ids for multiple devices."""
     inner = select_events_context_id_subquery(start_day, end_day, event_types).where(
-        apply_event_device_id_matchers(json_quotable_device_ids)
+        apply_event_device_id_matchers(json_quoted_device_ids)
     )
     return select(inner.c.context_id).group_by(inner.c.context_id)
 
@@ -45,14 +44,14 @@ def _apply_devices_context_union(
     start_day: dt,
     end_day: dt,
     event_types: tuple[str, ...],
-    json_quotable_device_ids: list[str],
+    json_quoted_device_ids: list[str],
 ) -> CompoundSelect:
     """Generate a CTE to find the device context ids and a query to find linked row."""
     devices_cte: CTE = _select_device_id_context_ids_sub_query(
         start_day,
         end_day,
         event_types,
-        json_quotable_device_ids,
+        json_quoted_device_ids,
     ).cte()
     return query.union_all(
         apply_events_context_hints(
@@ -72,25 +71,22 @@ def devices_stmt(
     start_day: dt,
     end_day: dt,
     event_types: tuple[str, ...],
-    json_quotable_device_ids: list[str],
-) -> StatementLambdaElement:
+    json_quoted_device_ids: list[str],
+) -> Select:
     """Generate a logbook query for multiple devices."""
-    stmt = lambda_stmt(
-        lambda: _apply_devices_context_union(
-            select_events_without_states(start_day, end_day, event_types).where(
-                apply_event_device_id_matchers(json_quotable_device_ids)
-            ),
-            start_day,
-            end_day,
-            event_types,
-            json_quotable_device_ids,
-        ).order_by(Events.time_fired)
-    )
-    return stmt
+    return _apply_devices_context_union(
+        select_events_without_states(start_day, end_day, event_types).where(
+            apply_event_device_id_matchers(json_quoted_device_ids)
+        ),
+        start_day,
+        end_day,
+        event_types,
+        json_quoted_device_ids,
+    ).order_by(Events.time_fired)
 
 
 def apply_event_device_id_matchers(
-    json_quotable_device_ids: Iterable[str],
+    json_quoted_device_ids: Iterable[str],
 ) -> ClauseList:
     """Create matchers for the device_ids in the event_data."""
-    return DEVICE_ID_IN_EVENT.in_(json_quotable_device_ids)
+    return DEVICE_ID_IN_EVENT.in_(json_quoted_device_ids)

homeassistant/components/logbook/queries/entities.py

@@ -5,10 +5,9 @@
 from datetime import datetime as dt
 
 import sqlalchemy
-from sqlalchemy import lambda_stmt, select, union_all
+from sqlalchemy import select, union_all
 from sqlalchemy.orm import Query
-from sqlalchemy.sql.lambdas import StatementLambdaElement
-from sqlalchemy.sql.selectable import CTE, CompoundSelect
+from sqlalchemy.sql.selectable import CTE, CompoundSelect, Select
 
 from homeassistant.components.recorder.models import (
     ENTITY_ID_IN_EVENT,
@@ -36,12 +35,12 @@ def _select_entities_context_ids_sub_query(
     end_day: dt,
     event_types: tuple[str, ...],
     entity_ids: list[str],
-    json_quotable_entity_ids: list[str],
+    json_quoted_entity_ids: list[str],
 ) -> CompoundSelect:
     """Generate a subquery to find context ids for multiple entities."""
     union = union_all(
         select_events_context_id_subquery(start_day, end_day, event_types).where(
-            apply_event_entity_id_matchers(json_quotable_entity_ids)
+            apply_event_entity_id_matchers(json_quoted_entity_ids)
         ),
         apply_entities_hints(select(States.context_id))
         .filter((States.last_updated > start_day) & (States.last_updated < end_day))
@@ -56,15 +55,15 @@ def _apply_entities_context_union(
     end_day: dt,
     event_types: tuple[str, ...],
     entity_ids: list[str],
-    json_quotable_entity_ids: list[str],
+    json_quoted_entity_ids: list[str],
 ) -> CompoundSelect:
     """Generate a CTE to find the entity and device context ids and a query to find linked row."""
     entities_cte: CTE = _select_entities_context_ids_sub_query(
         start_day,
         end_day,
         event_types,
         entity_ids,
-        json_quotable_entity_ids,
+        json_quoted_entity_ids,
     ).cte()
     # We used to optimize this to exclude rows we already in the union with
     # a States.entity_id.not_in(entity_ids) but that made the
@@ -91,21 +90,19 @@ def entities_stmt(
     end_day: dt,
     event_types: tuple[str, ...],
     entity_ids: list[str],
-    json_quotable_entity_ids: list[str],
-) -> StatementLambdaElement:
+    json_quoted_entity_ids: list[str],
+) -> Select:
     """Generate a logbook query for multiple entities."""
-    return lambda_stmt(
-        lambda: _apply_entities_context_union(
-            select_events_without_states(start_day, end_day, event_types).where(
-                apply_event_entity_id_matchers(json_quotable_entity_ids)
-            ),
-            start_day,
-            end_day,
-            event_types,
-            entity_ids,
-            json_quotable_entity_ids,
-        ).order_by(Events.time_fired)
-    )
+    return _apply_entities_context_union(
+        select_events_without_states(start_day, end_day, event_types).where(
+            apply_event_entity_id_matchers(json_quoted_entity_ids)
+        ),
+        start_day,
+        end_day,
+        event_types,
+        entity_ids,
+        json_quoted_entity_ids,
+    ).order_by(Events.time_fired)
 
 
 def states_query_for_entity_ids(
@@ -118,12 +115,12 @@ def states_query_for_entity_ids(
 
 
 def apply_event_entity_id_matchers(
-    json_quotable_entity_ids: Iterable[str],
+    json_quoted_entity_ids: Iterable[str],
 ) -> sqlalchemy.or_:
     """Create matchers for the entity_id in the event_data."""
-    return ENTITY_ID_IN_EVENT.in_(
-        json_quotable_entity_ids
-    ) | OLD_ENTITY_ID_IN_EVENT.in_(json_quotable_entity_ids)
+    return ENTITY_ID_IN_EVENT.in_(json_quoted_entity_ids) | OLD_ENTITY_ID_IN_EVENT.in_(
+        json_quoted_entity_ids
+    )
 
 
 def apply_entities_hints(query: Query) -> Query: