BM25 and vector search on materialized entries

DESIGN.md

@@ -209,7 +209,8 @@ write-through for callers that need `.sql()` on the entry immediately.
 
 ## MaterializedFrame
 
-Implemented today: `.arrow()`, `.df()`, `.to_pylist()`, `len()`, and
+Implemented today: `.arrow()`, `.df()`, `.to_pylist()`, `len()`,
+`.search()`, `.vector_search()`, and
 `.sql()` (server-side; requires a persisted entry — on a fresh
 write-behind miss it raises until the background persist lands). The
 rest of this section is the target design.
@@ -253,6 +254,7 @@ from hotdata_materialized import materialize
     version=2,                    # bump to bust on code changes
     background=True,              # write-behind (default) | write-through
     key_fn=None,                  # stable identity for opaque arguments
+    index=BM25("notes"),          # or Vector(...); built at persist time
 )
 def daily_signups(start, end):
     return (Event.objects
@@ -261,11 +263,11 @@ def daily_signups(start, end):
             .annotate(n=Count("id")))
 ```
 
-Planned knobs (steps 4–5, do not document user-facing until they exist):
+Planned knobs (step 4, do not document user-facing until they exist):
 `mode="swr" | "strict"` (refresh behavior), `invalidate_on=[Model]`
-(signal-based staleness), `index=Vector(...)/BM25(...)` (search indexes),
-`on_error="fallback" | "raise"` (fail-open is currently always on), and a
-`from_queryset(qs, ttl=)` imperative helper for one-off QuerySets.
+(signal-based staleness), `on_error="fallback" | "raise"` (fail-open is
+currently always on), and a `from_queryset(qs, ttl=)` imperative helper
+for one-off QuerySets.
 
 ## Fingerprinting
 

README.md

@@ -98,6 +98,42 @@ Fail-open by design: if Hotdata is unreachable, the function runs and its
 result is served uncached — the cache degrades to "no cache," never to
 "no page."
 
+## Search
+
+Declare a search index on the decorator and the entry gets it when it
+persists (builds run server-side; a freshly missed entry may error on search
+for a few seconds until its index is ready):
+
+```python
+from hotdata_materialized import BM25, Vector, materialize
+
+
+@materialize(ttl=3600, index=BM25("notes"))
+def incidents():
+    return Incident.objects.values("id", "notes", "severity")
+
+
+incidents().search("checkout errors outage", column="notes", limit=5)
+```
+
+Vector (semantic) search embeds your text server-side via a workspace
+embedding provider — pass the indexed source column and a natural-language
+query:
+
+```python
+@materialize(ttl=3600, index=Vector("notes", provider="emb_...", metric="cosine"))
+def incidents_semantic():
+    return Incident.objects.values("id", "notes")
+
+
+incidents_semantic().vector_search("the site was down", column="notes", limit=5)
+```
+
+Both return the matching rows as a `pyarrow.Table` with a relevance column
+(`score` / `distance`). One platform rule to know: an embedding-backed vector
+index cannot share an entry with other indexes — the decorator rejects that
+combination up front.
+
 ## Evicting entries
 
 The decorator composes public primitives (`fingerprint_call` /
@@ -142,7 +178,8 @@ trade-offs.
 - [ ] Stale-while-revalidate refresh (rebuild protocol)
 - [ ] Sweep command for expired entries
 - [ ] Chainable queryset facade on the frame (`.filter()/.order_by()`)
-- [ ] Vector/BM25 index declarations and `frame.search()`
+- [x] Vector/BM25 index declarations, `frame.search()` and
+      `frame.vector_search()`
 - [ ] Async view support (`await frame.aarrow()`)
 - [ ] PyPI release
 

hotdata_materialized/__init__.py

@@ -15,6 +15,7 @@
     StoreError,
 )
 from .fingerprint import fingerprint_call, fingerprint_queryset
+from .indexes import BM25, Vector
 from .registry import Registry, RegistryEntry
 from .store import EntryStore
 
@@ -23,6 +24,8 @@
 __all__ = [
     "materialize",
     "MaterializedFrame",
+    "BM25",
+    "Vector",
     "Config",
     "HotdataClients",
     "get_clients",

hotdata_materialized/_sql.py

@@ -1,16 +1,26 @@
-"""SQL literal encoding for registry statements.
+"""SQL literal and identifier encoding.
 
 POST /v1/queries takes SQL text only — there are no bind parameters — so
-every value written into registry SQL must go through quote_literal().
+every value written into SQL must go through quote_literal(), and every
+caller-supplied identifier (column names in search) through quote_ident().
 """
 
 from __future__ import annotations
 
 import datetime as _dt
 import math
+import re
 
 from .exceptions import RegistryError
 
+_IDENT = re.compile(r"[A-Za-z_][A-Za-z0-9_]*\Z")
+
+
+def quote_ident(name: str) -> str:
+    if not _IDENT.match(name):
+        raise ValueError(f"invalid SQL identifier: {name!r}")
+    return name
+
 
 def quote_literal(value) -> str:
     if value is None:

hotdata_materialized/client.py

@@ -29,6 +29,7 @@ def __init__(self, config: Config):
         self.databases = hotdata.DatabasesApi(self.api_client)
         self.uploads = UploadsApi(self.api_client)
         self.results = ResultsApi(self.api_client)
+        self.indexes = hotdata.IndexesApi(self.api_client)
 
 
 _lock = threading.Lock()

hotdata_materialized/decorator.py

@@ -17,16 +17,18 @@
 import logging
 import re
 import threading
-from typing import Any, Callable, Dict, List, Optional, Tuple
+from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple, Union
 
 import pyarrow as pa
 
 from .client import get_clients
 from .conf import Config
 from .exceptions import MaterializedError
+from ._sql import quote_ident, quote_literal
 from .fingerprint import fingerprint_call
+from .indexes import BM25, Vector
 from .registry import STATUS_READY, Registry, RegistryEntry
-from .store import DATA_TABLE, EntryStore
+from .store import DATA_SCHEMA, DATA_TABLE, EntryStore
 
 logger = logging.getLogger(__name__)
 
@@ -81,6 +83,29 @@ def sql(self, sql: str) -> pa.Table:
         then; use background=False on the decorator to persist inline)."""
         return self._store.query_table(self.entry, _rewrite_this(sql))
 
+    def search(self, query: str, *, column: str, limit: int = 10) -> pa.Table:
+        """BM25 keyword search over the entry, best matches first. Requires
+        a BM25 index on the column (declare with @materialize(index=BM25(...));
+        a freshly missed entry may error until the index build finishes)."""
+        table_ref = f"default.{DATA_SCHEMA}.{DATA_TABLE}"
+        return self._store.query_table(
+            self.entry,
+            f"SELECT * FROM bm25_search({quote_literal(table_ref)}, "
+            f"{quote_literal(quote_ident(column))}, {quote_literal(query)}) "
+            f"ORDER BY score DESC LIMIT {int(limit)}",
+        )
+
+    def vector_search(self, query: str, *, column: str, limit: int = 10) -> pa.Table:
+        """Semantic search over the entry, nearest first. Requires a vector
+        index (declare with @materialize(index=Vector(...))). Pass the indexed
+        source column; the query text is embedded server-side."""
+        return self._store.query_table(
+            self.entry,
+            f"SELECT *, vector_distance({quote_ident(column)}, "
+            f"{quote_literal(query)}) AS distance "
+            f"FROM {DATA_TABLE} ORDER BY distance LIMIT {int(limit)}",
+        )
+
     def __len__(self) -> int:
         return self.arrow().num_rows
 
@@ -131,6 +156,7 @@ def materialize(
     version: int = 0,
     background: Optional[bool] = None,
     key_fn: Optional[Callable[..., Any]] = None,
+    index: Union[BM25, Vector, Sequence[Union[BM25, Vector]], None] = None,
 ) -> Callable[[Callable[..., Any]], Callable[..., MaterializedFrame]]:
     """Materialize a function's result into Hotdata.
 
@@ -140,8 +166,26 @@ def revenue_by_region():
 
     The call returns a MaterializedFrame either way; `.cached` says which path
     served it. `version=` busts the cache on code changes; `key_fn=` maps
-    non-JSON-serializable arguments to a stable identity.
+    non-JSON-serializable arguments to a stable identity. `index=` declares
+    BM25/Vector search indexes built when the entry persists.
     """
+    declarations: Tuple[Union[BM25, Vector], ...]
+    if index is None:
+        declarations = ()
+    elif isinstance(index, (BM25, Vector)):
+        declarations = (index,)
+    else:
+        declarations = tuple(index)
+    embedded = any(
+        isinstance(d, Vector) and d.provider is not None for d in declarations
+    )
+    if embedded and len(declarations) > 1:
+        # platform constraint: embedding-backed vector indexes cannot coexist
+        # with other indexes on the same table
+        raise ValueError(
+            "an embedding-backed Vector index (provider=...) cannot be "
+            "combined with other indexes on the same entry"
+        )
 
     def decorate(func: Callable[..., Any]) -> Callable[..., MaterializedFrame]:
         @functools.wraps(func)
@@ -177,6 +221,7 @@ def wrapper(*args: Any, **kwargs: Any) -> MaterializedFrame:
                     ttl=ttl,
                     version=version,
                     background=background,
+                    indexes=declarations,
                 )
             except MaterializedError:
                 logger.warning(

hotdata_materialized/indexes.py

@@ -0,0 +1,33 @@
+"""Search-index declarations for @materialize(index=...).
+
+Indexes are created right after the entry's data loads; builds run as
+background jobs on the Hotdata side, so a freshly missed entry may briefly
+error on search until its index is ready.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Optional
+
+
+@dataclass(frozen=True)
+class BM25:
+    """Keyword (BM25) index over a text column; query via frame.search()."""
+
+    column: str
+
+
+@dataclass(frozen=True)
+class Vector:
+    """Vector index; query via frame.vector_search().
+
+    With `provider` set (an embedding provider id), the column is treated
+    as text and embedded server-side; vector_search then targets the source
+    column and query text is embedded automatically. An embedding-backed
+    index cannot share a table with other indexes."""
+
+    column: str
+    provider: Optional[str] = None
+    metric: Optional[str] = None  # "l2" | "cosine" | "dot"
+    dimensions: Optional[int] = None