Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

## [Unreleased]

### Added

- `load_managed_table(..., mode=...)` selects the load mode (`replace` (default), `append`, `delete`, `update`, `upsert`) instead of always replacing the table. `replace`/`append` apply the upload directly; `delete`/`update`/`upsert` match rows by the table's declared key. Backward compatible — omitting `mode` still replaces.
- `create_managed_database(..., keys={table: [cols]})` and `add_managed_table(..., key=[cols])` declare a table's row-identity key, enabling the key-based load modes on it. Requires a `hotdata` client whose managed-table decl models carry `key` (see the dependency floor bump); tables declared without a key stay `replace`/`append`-only.

## [0.6.3] - 2026-07-08

Expand Down
18 changes: 14 additions & 4 deletions hotdata_framework/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -241,14 +241,21 @@ def create_managed_database(
*,
schema: str = DEFAULT_SCHEMA,
tables: list[str] | None = None,
keys: dict[str, list[str]] | None = None,
expires_at: str | None = None,
) -> ManagedDatabase:
"""Create a managed database. ``keys`` maps a table to its key columns
(enabling delete/update/upsert on it); omitted tables are keyless."""
keys = keys or {}
schemas = None
if tables:
schemas = [
DatabaseDefaultSchemaDecl(
name=schema,
tables=[DatabaseDefaultTableDecl(name=t) for t in tables],
tables=[
DatabaseDefaultTableDecl(name=t, key=list(keys.get(t, [])))
for t in tables
],
)
]
request = CreateDatabaseRequest(
Expand Down Expand Up @@ -314,6 +321,7 @@ def load_managed_table(
schema: str = DEFAULT_SCHEMA,
upload_id: str | None = None,
file: str | None = None,
mode: str = "replace",

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

super nit: mode is typed as a bare str, so a typo (e.g. mode="upser") passes through silently and only surfaces as a server-side error. Since the SDK enumerates exactly five valid modes, a Literal["replace", "append", "delete", "update", "upsert"] here (and on the managed_client wrapper) would catch that at type-check time and self-document the accepted values. (not blocking)

) -> LoadManagedTableResult:
if (upload_id is None) == (file is None):
raise ValueError("Exactly one of upload_id or file is required")
Expand All @@ -324,7 +332,7 @@ def load_managed_table(
assert file is not None
resolved_upload_id = self.upload_parquet(file)
request = LoadManagedTableRequest(
mode="replace",
mode=mode,
upload_id=resolved_upload_id,
)
try:
Expand All @@ -350,15 +358,17 @@ def add_managed_table(
table: str,
*,
schema: str = DEFAULT_SCHEMA,
key: list[str] | None = None,
) -> ManagedTable:
"""Declare a new table on an existing managed database.

The table is added empty (declared-but-unloaded); populate it with
:meth:`load_managed_table`. Use this to evolve a managed database's
schema after creation without recreating it.
schema after creation without recreating it. ``key`` sets the
row-identity columns for delete/update/upsert; omit for keyless.
"""
db = self.resolve_managed_database(database)
request = AddManagedTableRequest(name=table)
request = AddManagedTableRequest(name=table, key=list(key or []))
try:
self._databases_api().add_database_table(db.id, schema, request)
except ApiException as e:
Expand Down
2 changes: 2 additions & 0 deletions hotdata_framework/managed_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -203,13 +203,15 @@ def load_managed_table(
*,
schema: str,
upload_id: str,
mode: str = "replace",

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: load_managed_table here goes through _request_with_retry, which retries on transient errors. Previously the mode was hardcoded to replace (idempotent), so a retry after the server already committed was harmless. With append now reachable, a transient failure after the server commits but before the response is received will re-run the append on retry, duplicating the uploaded rows. replace/delete/update/upsert are idempotent, so append is the only exposed mode with this hazard. Worth confirming the server dedupes by upload_id, or otherwise not retrying append. (not blocking)

) -> LoadManagedTableResult:
return self._request_with_retry(
lambda: self._runtime.load_managed_table(
database,
table,
schema=schema,
upload_id=upload_id,
mode=mode,
)
)

Expand Down
3 changes: 2 additions & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,8 @@ classifiers = [
"Typing :: Typed",
]
dependencies = [
"hotdata>=0.6.0",
# 0.7.0 adds `key` to table decls (create_managed_database(keys=) / add_managed_table(key=))
"hotdata>=0.7.0",
"pandas>=2.0",
"pyarrow>=14.0",
]
Expand Down
78 changes: 78 additions & 0 deletions tests/test_databases.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@

import pytest
from hotdata.exceptions import ApiException
from hotdata.models.database_default_table_decl import DatabaseDefaultTableDecl

from hotdata_framework.client import HotdataClient
from hotdata_framework.databases import (
Expand All @@ -13,6 +14,20 @@
)


def _decl_key_supported() -> bool:
# `key` ships with the regenerated client; the key tests activate once it does.
try:
return DatabaseDefaultTableDecl(name="t", key=["k"]).key == ["k"]
except Exception:
return False


requires_key_field = pytest.mark.skipif(
not _decl_key_supported(),
reason="hotdata client without `key` on managed-table decls",
)


def _client() -> HotdataClient:
return HotdataClient("k", "ws", host="https://api.hotdata.dev")

Expand Down Expand Up @@ -230,6 +245,69 @@ def test_load_managed_table_requires_exactly_one_source():
)


def _load_and_capture_request(client, **kwargs):
db = managed_database_from_detail(_detail())
loaded = SimpleNamespace(
connection_id="conn_1", schema_name="public", table_name="orders", row_count=1
)
with (
patch.object(client, "resolve_managed_database", return_value=db),
patch.object(client, "connections") as connections,
):
connections.return_value.load_managed_table.return_value = loaded
client.load_managed_table("db_1", "orders", upload_id="upl_1", **kwargs)
return connections.return_value.load_managed_table.call_args.args[3]


def test_load_managed_table_defaults_to_replace():
assert _load_and_capture_request(_client()).mode == "replace"


@pytest.mark.parametrize("mode", ["append", "delete", "update", "upsert"])
def test_load_managed_table_forwards_mode(mode: str):
assert _load_and_capture_request(_client(), mode=mode).mode == mode


@requires_key_field
def test_create_managed_database_declares_keys():
client = _client()
with patch.object(client, "_databases_api") as dbs:
dbs.return_value.create_database.return_value = _detail(id="db_new")
client.create_managed_database(
"mydb", tables=["orders", "events"], keys={"orders": ["id"]}
)
req = dbs.return_value.create_database.call_args.args[0]
declared = {t.name: list(t.key) for t in req.schemas[0].tables}
assert declared == {"orders": ["id"], "events": []}


@requires_key_field
def test_add_managed_table_declares_key():
client = _client()
db = managed_database_from_detail(_detail())
with (
patch.object(client, "resolve_managed_database", return_value=db),
patch.object(client, "_databases_api") as dbs,
):
client.add_managed_table("db_1", "line_items", key=["order_id", "sku"])
req = dbs.return_value.add_database_table.call_args.args[2]
assert req.name == "line_items"
assert list(req.key) == ["order_id", "sku"]


@requires_key_field
def test_add_managed_table_keyless_by_default():
client = _client()
db = managed_database_from_detail(_detail())
with (
patch.object(client, "resolve_managed_database", return_value=db),
patch.object(client, "_databases_api") as dbs,
):
client.add_managed_table("db_1", "orders")
req = dbs.return_value.add_database_table.call_args.args[2]
assert list(req.key) == []


def test_delete_managed_table_uses_default_connection_id():
client = _client()
db = managed_database_from_detail(_detail())
Expand Down
8 changes: 4 additions & 4 deletions uv.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading