feat(server): in-app online-DDL lane for non-transactional/locking DDL (JEF-580)

server/src/db.rs

@@ -71,6 +71,24 @@ fn migrate_connect_options(url: &str) -> anyhow::Result<PgConnectOptions> {
         .options([("lock_timeout", "3s"), ("statement_timeout", "0")]))
 }
 
+/// Connect options for the online-DDL lane (ADR 0021, JEF-580): a dedicated
+/// connection, not the query pool, so a `CREATE`/`DROP INDEX CONCURRENTLY` build
+/// that runs for minutes on a big table is never cancelled by the pool's 60s
+/// `statement_timeout` (exactly the failure a plain, transactional `CREATE INDEX`
+/// hit on migration 0017). `lock_timeout=3s` matches [`migrate_connect_options`]:
+/// a build that can't get the lock it needs (e.g. blocked behind a long-running
+/// transaction) aborts fast rather than queuing ahead of ingest.
+/// `maintenance_work_mem` is capped explicitly rather than inherited from the
+/// server-wide default, which stays conservative for a Raspberry Pi's limited RAM;
+/// an index build can afford more without raising that default for every backend.
+pub fn online_ddl_connect_options(url: &str) -> anyhow::Result<PgConnectOptions> {
+    Ok(PgConnectOptions::from_str(url)?.options([
+        ("lock_timeout", "3s"),
+        ("statement_timeout", "0"),
+        ("maintenance_work_mem", "64MB"),
+    ]))
+}
+
 #[cfg(test)]
 mod tests {
     use super::{migrate_connect_options, PgConnection};

server/src/lib.rs

@@ -5,6 +5,7 @@ pub mod db;
 pub mod grpc;
 pub mod mcp;
 pub mod mcp_auth;
+pub mod online_ddl;
 pub mod otlp;
 pub mod retention;
 pub mod selflog;

server/src/main.rs

@@ -7,8 +7,8 @@ use tracing_subscriber::filter::dynamic_filter_fn;
 use tracing_subscriber::prelude::*;
 use tracing_subscriber::EnvFilter;
 use watcher_server::{
-    access_jwt, alerts, app_with_access, db, grpc, mcp, mcp_auth, retention, selflog, selfmon,
-    selftrace,
+    access_jwt, alerts, app_with_access, db, grpc, mcp, mcp_auth, online_ddl, retention, selflog,
+    selfmon, selftrace,
 };
 
 /// Self-instrumentation: capture watcher's own traces **in-process**, tagged
@@ -222,12 +222,26 @@ async fn main() -> anyhow::Result<()> {
         tracing::debug!("MCP server disabled (set WATCHER_MCP_ENABLED=1 to enable /mcp)");
     }
 
+    let listener = tokio::net::TcpListener::bind(&http_bind).await?;
+    tracing::info!("HTTP/OTLP + API on http://{http_bind}");
+
+    // Online-DDL lane (ADR 0021, JEF-580): spawned only now that the listener is
+    // bound and the pod is Ready, so an index build (which can take minutes on a
+    // large table) never blocks boot or /healthz. Fire-and-forget: a failure is
+    // logged, not fatal — reads fall back to whatever index already covers the
+    // query in the meantime.
+    {
+        let database_url = database_url.clone();
+        tokio::spawn(async move {
+            if let Err(e) = online_ddl::run(&database_url).await {
+                tracing::error!("online-DDL lane failed: {e:#}");
+            }
+        });
+    }
+
     let http = {
         let pool = pool.clone();
-        let bind = http_bind.clone();
         async move {
-            let listener = tokio::net::TcpListener::bind(&bind).await?;
-            tracing::info!("HTTP/OTLP + API on http://{bind}");
             axum::serve(listener, app_with_access(pool, access)).await?;
             Ok::<(), anyhow::Error>(())
         }