From 10d98203d173988b74e3efd1f49bffc0ff2272d9 Mon Sep 17 00:00:00 2001 From: Shefeek Jinnah Date: Fri, 10 Jul 2026 18:54:10 +0530 Subject: [PATCH 1/3] feat(managed): mode + key on managed-table loads load_managed_table gains a `mode` kwarg (replace/append/delete/update/ upsert) instead of always replacing; create_managed_database(keys=) and add_managed_table(key=) declare a table's row-identity key so the key-based modes work on it. The key path needs a hotdata client whose managed-table decl models carry `key`; the dependency floor is bumped (exact tag TBD at release). Backward compatible: omitting mode still replaces, and a client without `key` silently declares the table keyless (previous behavior). --- CHANGELOG.md | 4 ++ hotdata_framework/client.py | 24 +++++++-- hotdata_framework/managed_client.py | 2 + pyproject.toml | 5 +- tests/test_databases.py | 79 +++++++++++++++++++++++++++++ 5 files changed, 109 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ea5e935..1c43d4e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- `load_managed_table(..., mode=...)` selects the load mode (`replace` (default), `append`, `delete`, `update`, `upsert`) instead of always replacing the table. `replace`/`append` apply the upload directly; `delete`/`update`/`upsert` match rows by the table's declared key. Backward compatible — omitting `mode` still replaces. +- `create_managed_database(..., keys={table: [cols]})` and `add_managed_table(..., key=[cols])` declare a table's row-identity key, enabling the key-based load modes on it. Requires a `hotdata` client whose managed-table decl models carry `key` (see the dependency floor bump); tables declared without a key stay `replace`/`append`-only. ## [0.6.3] - 2026-07-08 diff --git a/hotdata_framework/client.py b/hotdata_framework/client.py index 54741f2..74c62c9 100644 --- a/hotdata_framework/client.py +++ b/hotdata_framework/client.py @@ -241,14 +241,26 @@ def create_managed_database( *, schema: str = DEFAULT_SCHEMA, tables: list[str] | None = None, + keys: dict[str, list[str]] | None = None, expires_at: str | None = None, ) -> ManagedDatabase: + """Create a managed database with its default-catalog tables. + + ``keys`` maps a table name to the columns that identify a row; declaring + a key enables the key-based load modes (``delete`` / ``update`` / + ``upsert``) on that table. Tables absent from ``keys`` (or with an empty + list) are declared keyless and support only ``replace`` / ``append``. + """ + keys = keys or {} schemas = None if tables: schemas = [ DatabaseDefaultSchemaDecl( name=schema, - tables=[DatabaseDefaultTableDecl(name=t) for t in tables], + tables=[ + DatabaseDefaultTableDecl(name=t, key=list(keys.get(t, []))) + for t in tables + ], ) ] request = CreateDatabaseRequest( @@ -314,6 +326,7 @@ def load_managed_table( schema: str = DEFAULT_SCHEMA, upload_id: str | None = None, file: str | None = None, + mode: str = "replace", ) -> LoadManagedTableResult: if (upload_id is None) == (file is None): raise ValueError("Exactly one of upload_id or file is required") @@ -324,7 +337,7 @@ def load_managed_table( assert file is not None resolved_upload_id = self.upload_parquet(file) request = LoadManagedTableRequest( - mode="replace", + mode=mode, upload_id=resolved_upload_id, ) try: @@ -350,15 +363,18 @@ def add_managed_table( table: str, *, schema: str = DEFAULT_SCHEMA, + key: list[str] | None = None, ) -> ManagedTable: """Declare a new table on an existing managed database. The table is added empty (declared-but-unloaded); populate it with :meth:`load_managed_table`. Use this to evolve a managed database's - schema after creation without recreating it. + schema after creation without recreating it. ``key`` declares the row + identity columns that enable the key-based load modes (``delete`` / + ``update`` / ``upsert``); omit it for a keyless table. """ db = self.resolve_managed_database(database) - request = AddManagedTableRequest(name=table) + request = AddManagedTableRequest(name=table, key=list(key or [])) try: self._databases_api().add_database_table(db.id, schema, request) except ApiException as e: diff --git a/hotdata_framework/managed_client.py b/hotdata_framework/managed_client.py index 4a2735f..376261c 100644 --- a/hotdata_framework/managed_client.py +++ b/hotdata_framework/managed_client.py @@ -203,6 +203,7 @@ def load_managed_table( *, schema: str, upload_id: str, + mode: str = "replace", ) -> LoadManagedTableResult: return self._request_with_retry( lambda: self._runtime.load_managed_table( @@ -210,6 +211,7 @@ def load_managed_table( table, schema=schema, upload_id=upload_id, + mode=mode, ) ) diff --git a/pyproject.toml b/pyproject.toml index 51cee10..25556f6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,7 +26,10 @@ classifiers = [ "Typing :: Typed", ] dependencies = [ - "hotdata>=0.6.0", + # TODO(release): pin the floor to the client version that adds `key` to the + # managed-table decl models (DatabaseDefaultTableDecl / AddManagedTableRequest). + # `create_managed_database(keys=...)` / `add_managed_table(key=...)` require it. + "hotdata>=0.7.0", "pandas>=2.0", "pyarrow>=14.0", ] diff --git a/tests/test_databases.py b/tests/test_databases.py index 994b2ce..c04b40e 100644 --- a/tests/test_databases.py +++ b/tests/test_databases.py @@ -5,8 +5,24 @@ import pytest from hotdata.exceptions import ApiException +from hotdata.models.database_default_table_decl import DatabaseDefaultTableDecl from hotdata_framework.client import HotdataClient + + +def _decl_key_supported() -> bool: + # The `key` field on the managed-table decls ships in a regenerated client; + # the key tests below activate once it's present. + try: + return DatabaseDefaultTableDecl(name="t", key=["k"]).key == ["k"] + except Exception: + return False + + +requires_key_field = pytest.mark.skipif( + not _decl_key_supported(), + reason="hotdata client without `key` on managed-table decls", +) from hotdata_framework.databases import ( is_parquet_path, managed_database_from_detail, @@ -230,6 +246,69 @@ def test_load_managed_table_requires_exactly_one_source(): ) +def _load_and_capture_request(client, **kwargs): + db = managed_database_from_detail(_detail()) + loaded = SimpleNamespace( + connection_id="conn_1", schema_name="public", table_name="orders", row_count=1 + ) + with ( + patch.object(client, "resolve_managed_database", return_value=db), + patch.object(client, "connections") as connections, + ): + connections.return_value.load_managed_table.return_value = loaded + client.load_managed_table("db_1", "orders", upload_id="upl_1", **kwargs) + return connections.return_value.load_managed_table.call_args.args[3] + + +def test_load_managed_table_defaults_to_replace(): + assert _load_and_capture_request(_client()).mode == "replace" + + +@pytest.mark.parametrize("mode", ["append", "delete", "update", "upsert"]) +def test_load_managed_table_forwards_mode(mode: str): + assert _load_and_capture_request(_client(), mode=mode).mode == mode + + +@requires_key_field +def test_create_managed_database_declares_keys(): + client = _client() + with patch.object(client, "_databases_api") as dbs: + dbs.return_value.create_database.return_value = _detail(id="db_new") + client.create_managed_database( + "mydb", tables=["orders", "events"], keys={"orders": ["id"]} + ) + req = dbs.return_value.create_database.call_args.args[0] + declared = {t.name: list(t.key) for t in req.schemas[0].tables} + assert declared == {"orders": ["id"], "events": []} + + +@requires_key_field +def test_add_managed_table_declares_key(): + client = _client() + db = managed_database_from_detail(_detail()) + with ( + patch.object(client, "resolve_managed_database", return_value=db), + patch.object(client, "_databases_api") as dbs, + ): + client.add_managed_table("db_1", "line_items", key=["order_id", "sku"]) + req = dbs.return_value.add_database_table.call_args.args[2] + assert req.name == "line_items" + assert list(req.key) == ["order_id", "sku"] + + +@requires_key_field +def test_add_managed_table_keyless_by_default(): + client = _client() + db = managed_database_from_detail(_detail()) + with ( + patch.object(client, "resolve_managed_database", return_value=db), + patch.object(client, "_databases_api") as dbs, + ): + client.add_managed_table("db_1", "orders") + req = dbs.return_value.add_database_table.call_args.args[2] + assert list(req.key) == [] + + def test_delete_managed_table_uses_default_connection_id(): client = _client() db = managed_database_from_detail(_detail()) From c711952e4580017a50f071f16d6286cecd2acbda Mon Sep 17 00:00:00 2001 From: Shefeek Jinnah Date: Tue, 14 Jul 2026 17:54:51 +0530 Subject: [PATCH 2/3] chore: lock hotdata 0.7.0 --- uv.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/uv.lock b/uv.lock index cbba0ae..43b2edf 100644 --- a/uv.lock +++ b/uv.lock @@ -86,7 +86,7 @@ wheels = [ [[package]] name = "hotdata" -version = "0.6.0" +version = "0.7.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pydantic" }, @@ -94,9 +94,9 @@ dependencies = [ { name = "typing-extensions" }, { name = "urllib3" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/d2/17/6d9e7cbe7e766cd52968653096ce4199d0cb1aa55b73fcfc2979fb9805ff/hotdata-0.6.0.tar.gz", hash = "sha256:c655fd9691a1d65dd4ed51b86f1c27bf8e73268b4c44696987a57a624da35143", size = 214162, upload-time = "2026-07-07T16:44:32.65Z" } +sdist = { url = "https://files.pythonhosted.org/packages/61/30/08681132e019f6c9ddb566dc38be6a302763c4b4848b2d97e3d7ace8d257/hotdata-0.7.0.tar.gz", hash = "sha256:ef4484a74c0f33ee543b0a0dbfb5a6c04e7b5812b277f42a6448000d41c2405d", size = 215130, upload-time = "2026-07-14T12:21:33.936Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/dc/0a/4c9f23e1fa85a8d0336681f608c13100e4ab2cf4204846d90374b1bb141b/hotdata-0.6.0-py3-none-any.whl", hash = "sha256:3324758eb0aa37bbcc505ddac524566a8c10fbe0787e226932e8b04d565200c1", size = 312820, upload-time = "2026-07-07T16:44:31.121Z" }, + { url = "https://files.pythonhosted.org/packages/dc/ef/6c4236640629688074892759b1a7593c08f30e9d916b6951c2bbfec19d7f/hotdata-0.7.0-py3-none-any.whl", hash = "sha256:d9e3008e3084d22bcc27b7bf1d08d8c2c255fe3726c252c777bb614a3cae0576", size = 314103, upload-time = "2026-07-14T12:21:32.154Z" }, ] [[package]] @@ -120,7 +120,7 @@ dev = [ [package.metadata] requires-dist = [ - { name = "hotdata", specifier = ">=0.6.0" }, + { name = "hotdata", specifier = ">=0.7.0" }, { name = "pandas", specifier = ">=2.0" }, { name = "pyarrow", specifier = ">=14.0" }, ] From 11ee270abea2a299015cd228d69ebc168e6ce94e Mon Sep 17 00:00:00 2001 From: Shefeek Jinnah Date: Tue, 14 Jul 2026 18:08:46 +0530 Subject: [PATCH 3/3] chore: finalize hotdata pin + tighten comments --- hotdata_framework/client.py | 14 ++++---------- pyproject.toml | 4 +--- tests/test_databases.py | 11 +++++------ 3 files changed, 10 insertions(+), 19 deletions(-) diff --git a/hotdata_framework/client.py b/hotdata_framework/client.py index 74c62c9..df5bf12 100644 --- a/hotdata_framework/client.py +++ b/hotdata_framework/client.py @@ -244,13 +244,8 @@ def create_managed_database( keys: dict[str, list[str]] | None = None, expires_at: str | None = None, ) -> ManagedDatabase: - """Create a managed database with its default-catalog tables. - - ``keys`` maps a table name to the columns that identify a row; declaring - a key enables the key-based load modes (``delete`` / ``update`` / - ``upsert``) on that table. Tables absent from ``keys`` (or with an empty - list) are declared keyless and support only ``replace`` / ``append``. - """ + """Create a managed database. ``keys`` maps a table to its key columns + (enabling delete/update/upsert on it); omitted tables are keyless.""" keys = keys or {} schemas = None if tables: @@ -369,9 +364,8 @@ def add_managed_table( The table is added empty (declared-but-unloaded); populate it with :meth:`load_managed_table`. Use this to evolve a managed database's - schema after creation without recreating it. ``key`` declares the row - identity columns that enable the key-based load modes (``delete`` / - ``update`` / ``upsert``); omit it for a keyless table. + schema after creation without recreating it. ``key`` sets the + row-identity columns for delete/update/upsert; omit for keyless. """ db = self.resolve_managed_database(database) request = AddManagedTableRequest(name=table, key=list(key or [])) diff --git a/pyproject.toml b/pyproject.toml index 25556f6..b87c140 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,9 +26,7 @@ classifiers = [ "Typing :: Typed", ] dependencies = [ - # TODO(release): pin the floor to the client version that adds `key` to the - # managed-table decl models (DatabaseDefaultTableDecl / AddManagedTableRequest). - # `create_managed_database(keys=...)` / `add_managed_table(key=...)` require it. + # 0.7.0 adds `key` to table decls (create_managed_database(keys=) / add_managed_table(key=)) "hotdata>=0.7.0", "pandas>=2.0", "pyarrow>=14.0", diff --git a/tests/test_databases.py b/tests/test_databases.py index c04b40e..accfcd0 100644 --- a/tests/test_databases.py +++ b/tests/test_databases.py @@ -8,11 +8,14 @@ from hotdata.models.database_default_table_decl import DatabaseDefaultTableDecl from hotdata_framework.client import HotdataClient +from hotdata_framework.databases import ( + is_parquet_path, + managed_database_from_detail, +) def _decl_key_supported() -> bool: - # The `key` field on the managed-table decls ships in a regenerated client; - # the key tests below activate once it's present. + # `key` ships with the regenerated client; the key tests activate once it does. try: return DatabaseDefaultTableDecl(name="t", key=["k"]).key == ["k"] except Exception: @@ -23,10 +26,6 @@ def _decl_key_supported() -> bool: not _decl_key_supported(), reason="hotdata client without `key` on managed-table decls", ) -from hotdata_framework.databases import ( - is_parquet_path, - managed_database_from_detail, -) def _client() -> HotdataClient: