diff --git a/CHANGELOG.md b/CHANGELOG.md index 217881b..72a077c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,9 +37,32 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `description` and `default_connection_id`, so a caller holding a resolved record could not ask what was attached to it. This reads the fields that record drops. -These are Python helpers and deliberately not tools. Whether provisioning of this kind should be -agent-callable is the open question in -[#61](https://github.com/hotdata-dev/hotdata-langchain/issues/61), and nothing here settles it. +- **`hl.database_expiry` and `hl.database_expiries`.** A lifetime is *written* as a string, + either an RFC 3339 timestamp or a relative window such as `"24h"`, and the server resolves it + to an instant. So the resolved time is only knowable by reading it back, and nothing in this + package could: `ManagedDatabase` carries no `expires_at`. A caller could set a TTL and then not + learn which second it landed on, or whether a database still had one at all. + + `database_expiries` returns the workspace keyed by database id, at one request per page of the + listing rather than one per database, because the listing response already carries the field. A + database with no TTL maps to `None`, so "lives forever" stays distinguishable from "not in this + workspace". It reads the listing endpoint directly, since `client.list_managed_databases()` + drops `expires_at`, reads every database individually, and silently omits any whose detail read + fails. + + The listing is paginated and the cursor is followed, so a workspace larger than one page is not + reported as complete after one read. Two guards stop paging early and log a warning rather than + raising: a workspace past 10,000 records, and a listing that repeats a cursor it already gave. + A caller that must know the read was complete has to watch the log, not the return value. + + Reaping happens from the TTL or from an explicit cleanup step, so an unexplained timestamp in + a tool result would be surface a model can only misuse. + +Everything above is a Python helper, and none of it reaches a tool. For the attach pair, whether +provisioning should be agent-callable at all is the open question in +[#61](https://github.com/hotdata-dev/hotdata-langchain/issues/61), which this release does not +settle. For the read-back helpers it is simpler: a model has no decision to make with either +value. ## [0.15.0] - 2026-09-01 diff --git a/README.md b/README.md index 6922b0b..0d86c77 100644 --- a/README.md +++ b/README.md @@ -838,6 +838,29 @@ These are Python helpers, not tools. Whether an agent should be able to attach a is [#61](https://github.com/hotdata-dev/hotdata-langchain/issues/61)'s open question, and this does not answer it. +## Reading back when a database expires + +`expires_at` is written as a string, either an RFC 3339 timestamp or a relative window like +`"24h"`, and the server resolves it to an instant. A caller that passed `"24h"` therefore does +not know which second it lands on, and `ManagedDatabase` carries no `expires_at` to consult: + +```python +print(hl.database_expiry(client, db)) # datetime, or None when it has no TTL +print(hl.database_expiries(client)) # {database_id: datetime | None} for the workspace +``` + +`database_expiries` costs one request per page of the database listing, not one per database, +because the listing already carries the field. A database with no TTL maps to `None`, which keeps +"lives forever" distinguishable from "not in this workspace". + +The cursor is followed across pages, so one read does not pass off a subset as the whole +workspace. Two guards stop it early and log a warning instead of raising: more than 10,000 +records, or a listing that repeats a cursor. If you need certainty that the read was complete, +watch the log rather than the returned mapping. + +Neither is on a tool. Reaping runs from the TTL or from your own cleanup step, so a model has no +decision to make with the value. + ## Controlling result size Limit how many rows are returned to the LLM. Useful for keeping responses within context limits (default: 100): diff --git a/hotdata_langchain/__init__.py b/hotdata_langchain/__init__.py index 955bffd..9677c11 100644 --- a/hotdata_langchain/__init__.py +++ b/hotdata_langchain/__init__.py @@ -23,6 +23,8 @@ attach_catalog, create_managed_database, database_attachments, + database_expiries, + database_expiry, detach_catalog, list_managed_databases_json, load_managed_table, @@ -137,6 +139,8 @@ "capabilities_by_column", "create_managed_database", "database_attachments", + "database_expiries", + "database_expiry", "describe_tables_json", "detach_catalog", "engine_error_message", diff --git a/hotdata_langchain/databases.py b/hotdata_langchain/databases.py index a654f9c..098aaa8 100644 --- a/hotdata_langchain/databases.py +++ b/hotdata_langchain/databases.py @@ -7,8 +7,9 @@ import logging import socket import tempfile -from collections.abc import Sequence +from collections.abc import Iterator, Sequence from dataclasses import dataclass +from datetime import datetime from pathlib import Path from typing import Any, Literal from urllib.parse import urlsplit @@ -68,6 +69,11 @@ class CatalogAttachment: MAX_DOWNLOAD_BYTES = 1024**3 DOWNLOAD_CHUNK_BYTES = 1024 * 256 +#: A stop on paging database listings, counted in records read rather than in distinct +#: ids: a server repeating one page never grows the set of ids, so a set size would not +#: terminate. A repeated cursor is caught separately. +_MAX_DATABASES_SCANNED = 10_000 + def resolve_database_by_id( client: HotdataClient, @@ -329,6 +335,80 @@ def detach_catalog( ) +def database_expiry( + client: HotdataClient, + database_id: str | ManagedDatabase, +) -> datetime | None: + """Return when ``database_id`` is due to be reaped, or ``None`` if it has no TTL. + + A lifetime is *written* as a string, either an RFC 3339 timestamp or a relative window + such as ``"24h"``, and the server resolves it to an instant. So the resolved instant is + only ever knowable by reading it back: a caller that passed ``"24h"`` does not know + which second it lands on, and ``ManagedDatabase`` carries no ``expires_at`` to consult. + + Raises ``KeyError`` when the workspace has no database with that id. + """ + return _database_detail(client, database_id).expires_at + + +def _database_summaries(client: HotdataClient) -> Iterator[Any]: + """Yield database summaries, following the listing cursor across pages. + + ``list_databases`` is paginated, so reading one page reports a subset as if it were the + whole workspace. Two guards end paging before the listing does, each logging a warning + and returning what it has: a workspace past ``_MAX_DATABASES_SCANNED`` records, and a + cursor arriving a second time. + """ + api = DatabasesApi(client.api) + cursor: str | None = None + scanned = 0 + used: set[str] = set() + while True: + try: + listing = api.list_databases(cursor=cursor) if cursor else api.list_databases() + except ApiException as e: + raise RuntimeError(api_error_message(e)) from e + for summary in listing.databases or (): + yield summary + scanned += 1 + cursor = listing.next_cursor + if not cursor or not listing.databases: + return + if cursor in used: + logger.warning( + "database listing returned cursor %r a second time; expiries are partial", + cursor, + ) + return + used.add(cursor) + if scanned > _MAX_DATABASES_SCANNED: + logger.warning( + "stopped paging database listings after %d records; expiries are partial", + scanned, + ) + return + + +def database_expiries(client: HotdataClient) -> dict[str, datetime | None]: + """Return every instant database's expiry in the workspace, keyed by database id. + + One call per page of the listing, rather than one call per database: the listing + response already carries ``expires_at``, so nothing here needs a per-database read. + A database with no TTL maps to ``None``, so a caller can tell "lives forever" from + "not in this workspace", which a missing key would not distinguish. + + This reads the listing endpoint directly rather than going through + ``client.list_managed_databases()``, which drops ``expires_at``, fetches every database + individually, and silently omits any whose detail read fails. + + **The result can be a subset, and the only notice is a log line.** Paging stops early + on two guards: a workspace past 10,000 records, and a listing that returns a cursor it + already gave. Each logs a warning and returns what it has, so a caller that must know + the read was complete has to watch the log rather than the return value. + """ + return {str(one.id): one.expires_at for one in _database_summaries(client)} + + def list_managed_databases_json(client: HotdataClient) -> str: """List this workspace's instant databases as JSON, each with its ``id`` and ``name``. diff --git a/tests/test_expiry.py b/tests/test_expiry.py new file mode 100644 index 0000000..cd8e739 --- /dev/null +++ b/tests/test_expiry.py @@ -0,0 +1,214 @@ +"""Reading back when an instant database is due to be reaped. + +A lifetime is written as a string, either an RFC 3339 timestamp or a relative window such +as ``"24h"``, and the server resolves it to an instant. So the resolved time is only ever +knowable by reading it back, and ``ManagedDatabase`` carries no ``expires_at`` to consult. +""" + +from __future__ import annotations + +from collections.abc import Iterator +from datetime import datetime, timezone +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +import pytest +from hotdata.exceptions import ApiException +from hotdata_framework import ManagedDatabase + +from hotdata_langchain.databases import ( + _MAX_DATABASES_SCANNED, + database_expiries, + database_expiry, +) + +REAPED_AT = datetime(2026, 9, 4, 12, 0, tzinfo=timezone.utc) + + +def summary(db_id: str, expires_at: datetime | None) -> SimpleNamespace: + return SimpleNamespace(id=db_id, name=db_id, expires_at=expires_at) + + +def page(*summaries: SimpleNamespace, next_cursor: str | None = None) -> SimpleNamespace: + return SimpleNamespace( + databases=list(summaries), + next_cursor=next_cursor, + has_more=next_cursor is not None, + count=len(summaries), + limit=100, + ) + + +@pytest.fixture +def api(managed_db: ManagedDatabase) -> Iterator[MagicMock]: + with patch("hotdata_langchain.databases.DatabasesApi") as api: + api.return_value.get_database.return_value = SimpleNamespace( + id=managed_db.id, + name=managed_db.description, + default_connection_id=managed_db.default_connection_id, + attachments=[], + expires_at=None, + ) + api.return_value.list_databases.return_value = page() + yield api + + +# --- one database ------------------------------------------------------------------- + + +def test_a_resolved_record_cannot_answer_this_which_is_why_the_helper_exists( + managed_db: ManagedDatabase, +) -> None: + assert not hasattr(managed_db, "expires_at") + + +def test_expiry_reports_the_instant_the_server_resolved( + mock_client: MagicMock, managed_db: ManagedDatabase, api: MagicMock +) -> None: + api.return_value.get_database.return_value = SimpleNamespace( + id=managed_db.id, + name=None, + default_connection_id="c", + attachments=[], + expires_at=REAPED_AT, + ) + assert database_expiry(mock_client, managed_db.id) == REAPED_AT + + +def test_a_database_with_no_ttl_reports_none( + mock_client: MagicMock, managed_db: ManagedDatabase, api: MagicMock +) -> None: + assert database_expiry(mock_client, managed_db.id) is None + + +def test_expiry_raises_keyerror_for_an_unknown_database( + mock_client: MagicMock, api: MagicMock +) -> None: + api.return_value.get_database.side_effect = ApiException(status=404, reason="Not Found") + with pytest.raises(KeyError, match="no instant database"): + database_expiry(mock_client, "dbid000000000000000000000000x") + + +# --- the whole workspace ------------------------------------------------------------ + + +def test_expiries_come_from_the_listing_not_one_read_per_database( + mock_client: MagicMock, api: MagicMock +) -> None: + """The listing already carries expires_at, so a per-database read is waste.""" + api.return_value.list_databases.return_value = page( + summary("db1", REAPED_AT), summary("db2", None) + ) + + assert database_expiries(mock_client) == {"db1": REAPED_AT, "db2": None} + api.return_value.get_database.assert_not_called() + + +def test_expiries_follow_the_cursor_across_pages(mock_client: MagicMock, api: MagicMock) -> None: + """One page read as the whole workspace would report a subset as if it were complete.""" + api.return_value.list_databases.side_effect = [ + page(summary("db1", REAPED_AT), next_cursor="c1"), + page(summary("db2", None)), + ] + + assert database_expiries(mock_client) == {"db1": REAPED_AT, "db2": None} + assert api.return_value.list_databases.call_count == 2 + + +def test_the_second_page_is_requested_with_the_cursor_it_was_given( + mock_client: MagicMock, api: MagicMock +) -> None: + api.return_value.list_databases.side_effect = [ + page(summary("db1", None), next_cursor="c1"), + page(summary("db2", None)), + ] + + database_expiries(mock_client) + + assert api.return_value.list_databases.call_args_list[1].kwargs == {"cursor": "c1"} + + +def test_a_cursor_pointing_at_an_empty_page_terminates( + mock_client: MagicMock, api: MagicMock +) -> None: + """A server that keeps handing back a cursor must not spin the loop forever.""" + api.return_value.list_databases.side_effect = [ + page(summary("db1", None), next_cursor="c1"), + page(next_cursor="c2"), + ] + + assert database_expiries(mock_client) == {"db1": None} + + +def test_an_empty_workspace_reports_nothing_rather_than_failing( + mock_client: MagicMock, api: MagicMock +) -> None: + assert database_expiries(mock_client) == {} + + +def test_a_listing_failure_surfaces_the_api_message(mock_client: MagicMock, api: MagicMock) -> None: + api.return_value.list_databases.side_effect = ApiException( + status=403, reason="Forbidden", body="workspace does not permit listing" + ) + with pytest.raises(RuntimeError, match="workspace does not permit listing"): + database_expiries(mock_client) + + +# --- termination, which the cap alone did not guarantee ----------------------------- + + +def test_a_server_repeating_one_cursor_forever_terminates( + mock_client: MagicMock, api: MagicMock +) -> None: + """The same page and the same cursor on every request must not spin the loop. + + Counting distinct ids would not stop this: the id set never grows past one page. + """ + api.return_value.list_databases.return_value = page( + summary("db1", None), next_cursor="always-the-same" + ) + + assert database_expiries(mock_client) == {"db1": None} + assert api.return_value.list_databases.call_count == 2 + + +def test_a_server_cycling_between_two_cursors_terminates( + mock_client: MagicMock, api: MagicMock +) -> None: + cursors = ["a", "b", "a", "b"] + api.return_value.list_databases.side_effect = [ + page(summary(f"db{i}", None), next_cursor=c) for i, c in enumerate(cursors) + ] + + result = database_expiries(mock_client) + + assert api.return_value.list_databases.call_count == 3 + assert result == {"db0": None, "db1": None, "db2": None} + + +def test_paging_stops_once_the_record_cap_is_passed(mock_client: MagicMock, api: MagicMock) -> None: + """A fresh cursor each time, so only the record count can end this.""" + counter = iter(range(10**6)) + + def one_page(cursor: str | None = None) -> SimpleNamespace: + n = next(counter) + return page(summary(f"db{n}", None), next_cursor=f"cursor-{n}") + + api.return_value.list_databases.side_effect = one_page + + result = database_expiries(mock_client) + + assert len(result) == _MAX_DATABASES_SCANNED + 1 + + +def test_the_cursor_is_never_reused_across_requests(mock_client: MagicMock, api: MagicMock) -> None: + api.return_value.list_databases.side_effect = [ + page(summary("db1", None), next_cursor="c1"), + page(summary("db2", None), next_cursor="c2"), + page(summary("db3", None)), + ] + + database_expiries(mock_client) + + sent = [c.kwargs.get("cursor") for c in api.return_value.list_databases.call_args_list] + assert sent == [None, "c1", "c2"]