diff --git a/CHANGELOG.md b/CHANGELOG.md index de5c8ca..b7016fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.7.2] - 2026-07-15 + +### Removed + +- The `POST /v1/files` fallback in `upload_parquet()`. Presigned upload sessions (`POST /v1/uploads`) are now required; a server that responds 501 raises a clear `RuntimeError` instead of silently falling back to the full-file-in-memory upload path. + ## [0.7.1] - 2026-07-15 ### Changed diff --git a/hotdata_framework/client.py b/hotdata_framework/client.py index 9a3ef82..543e0f8 100644 --- a/hotdata_framework/client.py +++ b/hotdata_framework/client.py @@ -321,7 +321,12 @@ def upload_parquet(self, path: str) -> str: ) except ApiException as e: if e.status == 501: - return self._upload_parquet_post(path) + raise RuntimeError( + "the server does not support presigned uploads (HTTP 501); " + "managed table loads require a storage backend that can issue " + "presigned URLs (the POST /v1/files fallback was removed in " + "hotdata-framework 0.7.2)" + ) from e raise RuntimeError(api_error_message(e)) from e http = urllib3.PoolManager() parts: list[FinalizeUploadPart] | None = None @@ -374,19 +379,6 @@ def upload_parquet(self, path: str) -> str: raise RuntimeError(api_error_message(e)) from e return finalized.upload_id - def _upload_parquet_post(self, path: str) -> str: - """Fallback for storage backends that do not support presigned URLs (501).""" - with open(path, "rb") as f: - data = f.read() - try: - uploaded = self.uploads().upload_file( - data, - _content_type="application/octet-stream", - ) - except ApiException as e: - raise RuntimeError(api_error_message(e)) from e - return uploaded.id - def load_managed_table( self, database: str, diff --git a/pyproject.toml b/pyproject.toml index 7741379..7cf5cef 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "hotdata-framework" -version = "0.7.1" +version = "0.7.2" description = "Python framework for building Hotdata integrations: workspace/session runtime, query execution, and managed databases" readme = "README.md" requires-python = ">=3.10" diff --git a/tests/test_databases.py b/tests/test_databases.py index 2a5cc33..118e318 100644 --- a/tests/test_databases.py +++ b/tests/test_databases.py @@ -278,9 +278,8 @@ def test_upload_parquet_single_put(): assert call_args.args[1] == "https://s/put" -def test_upload_parquet_fallback_on_501(): +def test_upload_parquet_raises_on_501(): client = _client() - uploaded = SimpleNamespace(id="upl_post") with ( patch("builtins.open", mock_open(read_data=b"PAR1")), @@ -289,11 +288,50 @@ def test_upload_parquet_fallback_on_501(): ): err = ApiException(status=501) uploads.return_value.create_upload_session_handler.side_effect = err - uploads.return_value.upload_file.return_value = uploaded - upload_id = client.upload_parquet("/tmp/data.parquet") + with pytest.raises(RuntimeError, match="presigned uploads"): + client.upload_parquet("/tmp/data.parquet") + + uploads.return_value.upload_file.assert_not_called() + + +def test_upload_parquet_multipart_reads_are_part_size_bounded(): + """The OOM fix: multipart mode must never read more than part_size at once.""" + client = _client() + part_size = 5 + data = b"PAR1" + b"\x00" * 9 # 13 bytes -> parts of 5, 5, 3 + read_sizes: list[int] = [] + + class _TrackingFile(io.BytesIO): + def read(self, n=-1): + read_sizes.append(n) + return super().read(n) + + bio = _TrackingFile(data) + handle = MagicMock() + handle.__enter__ = lambda s: bio + handle.__exit__ = MagicMock(return_value=False) + session = _session( + "multipart", part_size=part_size, part_urls=["https://s/1", "https://s/2", "https://s/3"] + ) + finalized = SimpleNamespace(upload_id="upl_final") + + with ( + patch("builtins.open", MagicMock(return_value=handle)), + patch("os.path.getsize", return_value=len(data)), + patch.object(client, "uploads") as uploads, + patch("hotdata_framework.client.urllib3.PoolManager") as MockPool, + ): + pool = MockPool.return_value + pool.request.return_value = _http_resp() + pool.clear.return_value = None + uploads.return_value.create_upload_session_handler.return_value = session + uploads.return_value.finalize_upload_handler.return_value = finalized + + client.upload_parquet("/tmp/data.parquet") - assert upload_id == "upl_post" + assert read_sizes, "expected chunked reads" + assert all(n == part_size for n in read_sizes) def test_load_managed_table_with_upload_id(): diff --git a/uv.lock b/uv.lock index a965abb..d9bc915 100644 --- a/uv.lock +++ b/uv.lock @@ -101,7 +101,7 @@ wheels = [ [[package]] name = "hotdata-framework" -version = "0.7.1" +version = "0.7.2" source = { editable = "." } dependencies = [ { name = "hotdata" },