diff --git a/apps/dev-playground/evals.config.ts b/apps/dev-playground/evals.config.ts
new file mode 100644
index 000000000..6ad057c1b
--- /dev/null
+++ b/apps/dev-playground/evals.config.ts
@@ -0,0 +1,15 @@
+import { defineEvalConfig } from "@databricks/appkit/beta";
+
+/**
+ * Root eval config for the dev-playground. `webServer` lets `appkit agent eval`
+ * boot the app on demand (reusing an already-running dev server) instead of
+ * requiring it to be started by hand.
+ */
+export default defineEvalConfig({
+  baseUrl: "http://localhost:8000",
+  webServer: {
+    // Monorepo fixture command; a template project would use `npm run dev`.
+    command: "pnpm --filter=dev-playground dev",
+    timeoutMs: 90_000,
+  },
+});
diff --git a/docs/docs/api/appkit/Class.MlflowClient.md b/docs/docs/api/appkit/Class.MlflowClient.md
new file mode 100644
index 000000000..4b186e397
--- /dev/null
+++ b/docs/docs/api/appkit/Class.MlflowClient.md
@@ -0,0 +1,102 @@
+# Class: MlflowClient
+
+A thin client over the Databricks workspace REST API, owning the host + bearer
+token so callers (eval-run creation, assessment writes, the judge's serving
+endpoint) don't each re-derive URLs or re-attach auth. The host is normalized
+once at construction.
+
+## Constructors
+
+### Constructor
+
+```ts
+new MlflowClient(host: string, token: string): MlflowClient;
+```
+
+#### Parameters
+
+| Parameter | Type |
+| ------ | ------ |
+| `host` | `string` |
+| `token` | `string` |
+
+#### Returns
+
+`MlflowClient`
+
+## Properties
+
+### baseUrl
+
+```ts
+readonly baseUrl: string;
+```
+
+Normalized workspace base URL (scheme guaranteed, no trailing slash).
+
+## Methods
+
+### post()
+
+```ts
+post<T>(path: string, body: unknown): Promise<T>;
+```
+
+POST JSON to an MLflow REST endpoint. Returns the parsed JSON body, or
+throws with the status + response text so callers can surface a precise
+error. Use for calls whose failure should abort (e.g. `runs/create`).
+
+#### Type Parameters
+
+| Type Parameter | Default type |
+| ------ | ------ |
+| `T` | `unknown` |
+
+#### Parameters
+
+| Parameter | Type |
+| ------ | ------ |
+| `path` | `string` |
+| `body` | `unknown` |
+
+#### Returns
+
+`Promise`\<`T`\>
+
+***
+
+### postResult()
+
+```ts
+postResult(path: string, body: unknown): Promise<PostResult>;
+```
+
+POST JSON without throwing: returns `{ ok, status, error }` so best-effort
+writes (e.g. per-trace assessments) can be collected and reported without
+aborting the run.
+
+#### Parameters
+
+| Parameter | Type |
+| ------ | ------ |
+| `path` | `string` |
+| `body` | `unknown` |
+
+#### Returns
+
+`Promise`\<[`PostResult`](Interface.PostResult.md)\>
+
+***
+
+### servingEndpointsUrl()
+
+```ts
+servingEndpointsUrl(): string;
+```
+
+OpenAI-compatible base URL for Databricks Model Serving, used as the judge's
+`OPENAI_BASE_URL`. Same workspace host + token as the MLflow REST calls.
+
+#### Returns
+
+`string`
diff --git a/docs/docs/api/appkit/Function.buildAssessment.md b/docs/docs/api/appkit/Function.buildAssessment.md
deleted file mode 100644
index 62c08b2bd..000000000
--- a/docs/docs/api/appkit/Function.buildAssessment.md
+++ /dev/null
@@ -1,18 +0,0 @@
-# Function: buildAssessment()
-
-```ts
-function buildAssessment(result: EvalResult): Assessment | undefined;
-```
-
-Build the single pass/fail Feedback assessment for an eval result. Returns
-undefined when there's no trace to attach to or the eval was skipped.
-
-## Parameters
-
-| Parameter | Type |
-| ------ | ------ |
-| `result` | [`EvalResult`](Interface.EvalResult.md) |
-
-## Returns
-
-[`Assessment`](Interface.Assessment.md) \| `undefined`
diff --git a/docs/docs/api/appkit/Function.buildAssessments.md b/docs/docs/api/appkit/Function.buildAssessments.md
new file mode 100644
index 000000000..eda73df95
--- /dev/null
+++ b/docs/docs/api/appkit/Function.buildAssessments.md
@@ -0,0 +1,20 @@
+# Function: buildAssessments()
+
+```ts
+function buildAssessments(result: EvalResult): Assessment[];
+```
+
+Build the Feedback assessments for an eval result: one per assertion (judge
+assertions tagged `LLM_JUDGE` with their numeric score + rationale, so they
+render as judge feedback in MLflow) plus an overall `appkit_eval` pass/fail.
+Returns [] when there's no trace to attach to or the eval was skipped.
+
+## Parameters
+
+| Parameter | Type |
+| ------ | ------ |
+| `result` | [`EvalResult`](Interface.EvalResult.md) |
+
+## Returns
+
+[`Assessment`](Interface.Assessment.md)[]
diff --git a/docs/docs/api/appkit/Function.configureJudge.md b/docs/docs/api/appkit/Function.configureJudge.md
new file mode 100644
index 000000000..140042b14
--- /dev/null
+++ b/docs/docs/api/appkit/Function.configureJudge.md
@@ -0,0 +1,19 @@
+# Function: configureJudge()
+
+```ts
+function configureJudge(config: JudgeConfig): Promise<void>;
+```
+
+Configure the judge once. Sets the OpenAI-compatible client env autoevals
+reads and the default judge model. No-op-safe: on failure, judging stays
+disabled and [isJudgeConfigured](Function.isJudgeConfigured.md) returns false.
+
+## Parameters
+
+| Parameter | Type |
+| ------ | ------ |
+| `config` | [`JudgeConfig`](Interface.JudgeConfig.md) |
+
+## Returns
+
+`Promise`\<`void`\>
diff --git a/docs/docs/api/appkit/Function.discoverEvalConfigs.md b/docs/docs/api/appkit/Function.discoverEvalConfigs.md
new file mode 100644
index 000000000..f350af5e0
--- /dev/null
+++ b/docs/docs/api/appkit/Function.discoverEvalConfigs.md
@@ -0,0 +1,20 @@
+# Function: discoverEvalConfigs()
+
+```ts
+function discoverEvalConfigs(rootDir: string): DiscoveredEvalConfig[];
+```
+
+Discover the per-agent `evals.config.ts` (from [defineEvalConfig](Function.defineEvalConfig.md)) at
+`<rootDir>/config/agents/<agent>/evals/evals.config.ts`. Config is per-agent:
+each agent's config applies only to that agent's evals. Agents without a
+config file are omitted. Returns a stable, sorted list.
+
+## Parameters
+
+| Parameter | Type |
+| ------ | ------ |
+| `rootDir` | `string` |
+
+## Returns
+
+[`DiscoveredEvalConfig`](Interface.DiscoveredEvalConfig.md)[]
diff --git a/docs/docs/api/appkit/Function.findRootEvalConfig.md b/docs/docs/api/appkit/Function.findRootEvalConfig.md
new file mode 100644
index 000000000..2b6a4cb1b
--- /dev/null
+++ b/docs/docs/api/appkit/Function.findRootEvalConfig.md
@@ -0,0 +1,20 @@
+# Function: findRootEvalConfig()
+
+```ts
+function findRootEvalConfig(rootDir: string): string | undefined;
+```
+
+Path to the root `evals.config.ts` (from [defineEvalConfig](Function.defineEvalConfig.md)) at
+`<rootDir>/evals.config.ts`, or `undefined` when absent. The root config
+holds run-wide settings (`baseUrl`, `webServer`); it's distinct from the
+per-agent configs found by [discoverEvalConfigs](Function.discoverEvalConfigs.md).
+
+## Parameters
+
+| Parameter | Type |
+| ------ | ------ |
+| `rootDir` | `string` |
+
+## Returns
+
+`string` \| `undefined`
diff --git a/docs/docs/api/appkit/Function.formatResultsJUnit.md b/docs/docs/api/appkit/Function.formatResultsJUnit.md
new file mode 100644
index 000000000..de0e3df36
--- /dev/null
+++ b/docs/docs/api/appkit/Function.formatResultsJUnit.md
@@ -0,0 +1,20 @@
+# Function: formatResultsJUnit()
+
+```ts
+function formatResultsJUnit(results: EvalResult[]): string;
+```
+
+Render results as JUnit XML for standard CI test reporters: a single
+`<testsuite name="appkit-agent-evals">` with one `<testcase>` per result.
+Failures carry a `<failure>` (error or failing-gate summary); skips a
+`<skipped>`. All attribute/text values are XML-escaped.
+
+## Parameters
+
+| Parameter | Type |
+| ------ | ------ |
+| `results` | [`EvalResult`](Interface.EvalResult.md)[] |
+
+## Returns
+
+`string`
diff --git a/docs/docs/api/appkit/Function.formatResultsJson.md b/docs/docs/api/appkit/Function.formatResultsJson.md
new file mode 100644
index 000000000..7cf11944a
--- /dev/null
+++ b/docs/docs/api/appkit/Function.formatResultsJson.md
@@ -0,0 +1,19 @@
+# Function: formatResultsJson()
+
+```ts
+function formatResultsJson(results: EvalResult[]): string;
+```
+
+Render results as a machine-readable JSON report (2-space indented):
+`{ summary: EvalSummary, results: EvalResult[] }`. Faithful to the types —
+every field present on a result round-trips.
+
+## Parameters
+
+| Parameter | Type |
+| ------ | ------ |
+| `results` | [`EvalResult`](Interface.EvalResult.md)[] |
+
+## Returns
+
+`string`
diff --git a/docs/docs/api/appkit/Function.isJudgeConfigured.md b/docs/docs/api/appkit/Function.isJudgeConfigured.md
new file mode 100644
index 000000000..04d8f89f0
--- /dev/null
+++ b/docs/docs/api/appkit/Function.isJudgeConfigured.md
@@ -0,0 +1,9 @@
+# Function: isJudgeConfigured()
+
+```ts
+function isJudgeConfigured(): boolean;
+```
+
+## Returns
+
+`boolean`
diff --git a/docs/docs/api/appkit/Function.loadRootEvalConfig.md b/docs/docs/api/appkit/Function.loadRootEvalConfig.md
new file mode 100644
index 000000000..b3238a821
--- /dev/null
+++ b/docs/docs/api/appkit/Function.loadRootEvalConfig.md
@@ -0,0 +1,20 @@
+# Function: loadRootEvalConfig()
+
+```ts
+function loadRootEvalConfig(rootDir: string): Promise<EvalConfig | undefined>;
+```
+
+Load the root `evals.config.ts` under `rootDir` (the project root), or return
+`undefined` when there is none. This is the run-wide config carrying
+`baseUrl`/`webServer`; the CLI reads it to resolve options and manage the
+app-under-test lifecycle before calling [runEvalsInDir](Function.runEvalsInDir.md).
+
+## Parameters
+
+| Parameter | Type |
+| ------ | ------ |
+| `rootDir` | `string` |
+
+## Returns
+
+`Promise`\<[`EvalConfig`](Interface.EvalConfig.md) \| `undefined`\>
diff --git a/docs/docs/api/appkit/Function.normalizeHost.md b/docs/docs/api/appkit/Function.normalizeHost.md
new file mode 100644
index 000000000..2f4606cc6
--- /dev/null
+++ b/docs/docs/api/appkit/Function.normalizeHost.md
@@ -0,0 +1,17 @@
+# Function: normalizeHost()
+
+```ts
+function normalizeHost(raw: string): string;
+```
+
+Ensure the host has a scheme (Databricks env often lacks `https://`).
+
+## Parameters
+
+| Parameter | Type |
+| ------ | ------ |
+| `raw` | `string` |
+
+## Returns
+
+`string`
diff --git a/docs/docs/api/appkit/Function.readEvalDataset.md b/docs/docs/api/appkit/Function.readEvalDataset.md
new file mode 100644
index 000000000..5ca486b2b
--- /dev/null
+++ b/docs/docs/api/appkit/Function.readEvalDataset.md
@@ -0,0 +1,26 @@
+# Function: readEvalDataset()
+
+```ts
+function readEvalDataset(client: WorkspaceClient, options: ReadEvalDatasetOptions): Promise<DatasetRow[]>;
+```
+
+Read a Databricks managed evaluation dataset (a Unity Catalog table with
+`inputs`/`expectations` columns) into rows, over the public SQL Statement
+Execution API. Reuses SQLWarehouseConnector for submit/poll/transform
+— its result transform already JSON-parses string columns into objects, so
+`inputs`/`expectations` come back as records whether the table stores them as
+JSON strings or structs.
+
+The Python `mlflow.genai.datasets` API needs a Spark session (no TS
+equivalent), so we read the backing table directly.
+
+## Parameters
+
+| Parameter | Type |
+| ------ | ------ |
+| `client` | `WorkspaceClient` |
+| `options` | [`ReadEvalDatasetOptions`](Interface.ReadEvalDatasetOptions.md) |
+
+## Returns
+
+`Promise`\<[`DatasetRow`](Interface.DatasetRow.md)[]\>
diff --git a/docs/docs/api/appkit/Function.reportToMlflow.md b/docs/docs/api/appkit/Function.reportToMlflow.md
index 5d3c3b7b9..1b5803d24 100644
--- a/docs/docs/api/appkit/Function.reportToMlflow.md
+++ b/docs/docs/api/appkit/Function.reportToMlflow.md
@@ -1,7 +1,7 @@
 # Function: reportToMlflow()
 
 ```ts
-function reportToMlflow(results: EvalResult[], options: MlflowReportOptions): Promise<ReportOutcome>;
+function reportToMlflow(client: MlflowClient, results: EvalResult[]): Promise<ReportOutcome>;
 ```
 
 Write one pass/fail assessment per eval result to the Databricks MLflow REST
@@ -11,8 +11,8 @@ API. Never throws — failures are collected so the run still reports.
 
 | Parameter | Type |
 | ------ | ------ |
+| `client` | [`MlflowClient`](Class.MlflowClient.md) |
 | `results` | [`EvalResult`](Interface.EvalResult.md)[] |
-| `options` | [`MlflowReportOptions`](Interface.MlflowReportOptions.md) |
 
 ## Returns
 
diff --git a/docs/docs/api/appkit/Function.resolveDatabricksAuth.md b/docs/docs/api/appkit/Function.resolveDatabricksAuth.md
new file mode 100644
index 000000000..0fc2f1030
--- /dev/null
+++ b/docs/docs/api/appkit/Function.resolveDatabricksAuth.md
@@ -0,0 +1,24 @@
+# Function: resolveDatabricksAuth()
+
+```ts
+function resolveDatabricksAuth(options: ResolveDatabricksAuthOptions): Promise<DatabricksAuth | undefined>;
+```
+
+Resolve `{host, token}` for the eval runner the same way the rest of AppKit
+authenticates: construct a Databricks `WorkspaceClient` and let its config
+mint (and later refresh) an OAuth bearer from the CLI profile — no hand-set
+PAT required. An explicit host/token still wins (PAT or CI env), so the SDK
+is only consulted for whatever isn't supplied.
+
+Returns `undefined` when neither an explicit token nor a resolvable profile
+yields a bearer, so the caller can treat auth as simply unavailable.
+
+## Parameters
+
+| Parameter | Type |
+| ------ | ------ |
+| `options` | [`ResolveDatabricksAuthOptions`](Interface.ResolveDatabricksAuthOptions.md) |
+
+## Returns
+
+`Promise`\<[`DatabricksAuth`](Interface.DatabricksAuth.md) \| `undefined`\>
diff --git a/docs/docs/api/appkit/Function.resolveWorkspaceClient.md b/docs/docs/api/appkit/Function.resolveWorkspaceClient.md
new file mode 100644
index 000000000..1b80639b9
--- /dev/null
+++ b/docs/docs/api/appkit/Function.resolveWorkspaceClient.md
@@ -0,0 +1,21 @@
+# Function: resolveWorkspaceClient()
+
+```ts
+function resolveWorkspaceClient(options: ResolveDatabricksAuthOptions): WorkspaceClient | undefined;
+```
+
+Construct a Databricks `WorkspaceClient` for the eval runner — the object the
+SDK-backed connectors (e.g. `SQLWarehouseConnector`) take. An explicit
+host+token builds a PAT client; otherwise the profile (or ambient config) is
+used and the SDK resolves credentials, minting OAuth as needed. Returns
+`undefined` if construction throws (missing/invalid config).
+
+## Parameters
+
+| Parameter | Type |
+| ------ | ------ |
+| `options` | [`ResolveDatabricksAuthOptions`](Interface.ResolveDatabricksAuthOptions.md) |
+
+## Returns
+
+`WorkspaceClient` \| `undefined`
diff --git a/docs/docs/api/appkit/Function.runBounded.md b/docs/docs/api/appkit/Function.runBounded.md
new file mode 100644
index 000000000..1a08b36bf
--- /dev/null
+++ b/docs/docs/api/appkit/Function.runBounded.md
@@ -0,0 +1,34 @@
+# Function: runBounded()
+
+```ts
+function runBounded<T, R>(
+   tasks: readonly T[], 
+   limit: number, 
+worker: (task: T, index: number) => Promise<R>): Promise<R[]>;
+```
+
+Run `tasks` through a bounded worker pool and return their results in the
+SAME order as the input, regardless of completion order. Each task receives
+its input index so callers can key on it. `limit` is clamped to at least 1
+(and to the task count); at `limit === 1` this is a serial loop. Individual
+task rejections are surfaced per-slot via `settle` rather than aborting
+siblings — but eval tasks never reject (failures become results).
+
+## Type Parameters
+
+| Type Parameter |
+| ------ |
+| `T` |
+| `R` |
+
+## Parameters
+
+| Parameter | Type |
+| ------ | ------ |
+| `tasks` | readonly `T`[] |
+| `limit` | `number` |
+| `worker` | (`task`: `T`, `index`: `number`) => `Promise`\<`R`\> |
+
+## Returns
+
+`Promise`\<`R`[]\>
diff --git a/docs/docs/api/appkit/Function.runWithRetries.md b/docs/docs/api/appkit/Function.runWithRetries.md
new file mode 100644
index 000000000..5b4660a74
--- /dev/null
+++ b/docs/docs/api/appkit/Function.runWithRetries.md
@@ -0,0 +1,22 @@
+# Function: runWithRetries()
+
+```ts
+function runWithRetries(retries: number, attempt: (attemptNumber: number) => Promise<EvalResult>): Promise<EvalResult>;
+```
+
+Run `attempt` up to `1 + retries` times, stopping as soon as it returns a
+result without an `error` (infra failures — thrown errors or timeouts — set
+`error`; assertion failures do not, so a failed-but-completed eval is returned
+on the first try and never retried). Returns the last result when every
+attempt errored. `retries` below 0 is treated as 0.
+
+## Parameters
+
+| Parameter | Type |
+| ------ | ------ |
+| `retries` | `number` |
+| `attempt` | (`attemptNumber`: `number`) => `Promise`\<[`EvalResult`](Interface.EvalResult.md)\> |
+
+## Returns
+
+`Promise`\<[`EvalResult`](Interface.EvalResult.md)\>
diff --git a/docs/docs/api/appkit/Function.userTurns.md b/docs/docs/api/appkit/Function.userTurns.md
new file mode 100644
index 000000000..1ad9c8706
--- /dev/null
+++ b/docs/docs/api/appkit/Function.userTurns.md
@@ -0,0 +1,26 @@
+# Function: userTurns()
+
+```ts
+function userTurns(input: Record<string, unknown>): string[];
+```
+
+Extract every user-message content, in order, from an MLflow
+`{"messages":[{"role":"user","content":"..."}]}` input. A dataset row can
+carry a full multi-turn conversation; replaying these against one thread (one
+`t.send` per returned string) lets the agent see the accumulating history.
+
+Only `role === "user"` turns are returned — any interleaved `assistant`/
+`system` messages in the row are ignored, since the agent generates its own
+responses; you never inject the dataset's assistant turns. A single-user-turn
+row yields a one-element array (backward compatible); a row with no `messages`
+yields `[]`.
+
+## Parameters
+
+| Parameter | Type |
+| ------ | ------ |
+| `input` | `Record`\<`string`, `unknown`\> |
+
+## Returns
+
+`string`[]
diff --git a/docs/docs/api/appkit/Interface.AssertionHandle.md b/docs/docs/api/appkit/Interface.AssertionHandle.md
index 02e3c746b..0e640960b 100644
--- a/docs/docs/api/appkit/Interface.AssertionHandle.md
+++ b/docs/docs/api/appkit/Interface.AssertionHandle.md
@@ -12,7 +12,9 @@ metric; `.atLeast(n)` is a soft, score-thresholded assertion.
 atLeast(threshold: number): AssertionHandle;
 ```
 
-Soft assertion that passes only when the score is at least `threshold`.
+Set the pass threshold for a scored assertion: it passes only when the
+score is at least `threshold`. Keeps the current severity (gate unless also
+chained with `.soft()`).
 
 #### Parameters
 
diff --git a/docs/docs/api/appkit/Interface.CustomJudgeSpec.md b/docs/docs/api/appkit/Interface.CustomJudgeSpec.md
new file mode 100644
index 000000000..5bfd881e9
--- /dev/null
+++ b/docs/docs/api/appkit/Interface.CustomJudgeSpec.md
@@ -0,0 +1,27 @@
+# Interface: CustomJudgeSpec
+
+A custom LLM-judge definition: a prompt template and choice→score mapping.
+
+## Properties
+
+### choiceScores
+
+```ts
+choiceScores: Record<string, number>;
+```
+
+***
+
+### name
+
+```ts
+name: string;
+```
+
+***
+
+### promptTemplate
+
+```ts
+promptTemplate: string;
+```
diff --git a/docs/docs/api/appkit/Interface.DatabricksAuth.md b/docs/docs/api/appkit/Interface.DatabricksAuth.md
new file mode 100644
index 000000000..b1d31b35b
--- /dev/null
+++ b/docs/docs/api/appkit/Interface.DatabricksAuth.md
@@ -0,0 +1,19 @@
+# Interface: DatabricksAuth
+
+Resolved Databricks host + bearer token for the eval runner's REST calls.
+
+## Properties
+
+### host
+
+```ts
+host: string;
+```
+
+***
+
+### token
+
+```ts
+token: string;
+```
diff --git a/docs/docs/api/appkit/Interface.DatasetRow.md b/docs/docs/api/appkit/Interface.DatasetRow.md
new file mode 100644
index 000000000..32644c2dc
--- /dev/null
+++ b/docs/docs/api/appkit/Interface.DatasetRow.md
@@ -0,0 +1,22 @@
+# Interface: DatasetRow
+
+One row of a managed evaluation dataset. `inputs` are the kwargs passed to the
+agent for the turn; `expectations` (when present) is the row's ground truth /
+guidelines. Mirrors the `{inputs, expectations}` shape of `mlflow.genai`
+datasets and of the Unity Catalog table backing a managed eval dataset.
+
+## Properties
+
+### expectations?
+
+```ts
+optional expectations: Record<string, unknown>;
+```
+
+***
+
+### inputs
+
+```ts
+inputs: Record<string, unknown>;
+```
diff --git a/docs/docs/api/appkit/Interface.DiscoveredEvalConfig.md b/docs/docs/api/appkit/Interface.DiscoveredEvalConfig.md
new file mode 100644
index 000000000..7b4f2a831
--- /dev/null
+++ b/docs/docs/api/appkit/Interface.DiscoveredEvalConfig.md
@@ -0,0 +1,23 @@
+# Interface: DiscoveredEvalConfig
+
+A per-agent `evals.config.ts` found under `config/agents/<agent>/evals/`.
+
+## Properties
+
+### agent
+
+```ts
+agent: string;
+```
+
+The agent id whose evals this config applies to.
+
+***
+
+### file
+
+```ts
+file: string;
+```
+
+Absolute path to the `evals.config.ts` file.
diff --git a/docs/docs/api/appkit/Interface.DriveResult.md b/docs/docs/api/appkit/Interface.DriveResult.md
index 15625c1ac..f168e875a 100644
--- a/docs/docs/api/appkit/Interface.DriveResult.md
+++ b/docs/docs/api/appkit/Interface.DriveResult.md
@@ -34,6 +34,31 @@ Whether the turn completed without an agent/stream error.
 
 ***
 
+### toolCallDetails
+
+```ts
+toolCallDetails: {
+  args: Record<string, unknown>;
+  name: string;
+}[];
+```
+
+Tool calls with their parsed arguments, in call order.
+
+#### args
+
+```ts
+args: Record<string, unknown>;
+```
+
+#### name
+
+```ts
+name: string;
+```
+
+***
+
 ### toolCalls
 
 ```ts
diff --git a/docs/docs/api/appkit/Interface.EvalConfig.md b/docs/docs/api/appkit/Interface.EvalConfig.md
index eb8227fcd..4d8fd8519 100644
--- a/docs/docs/api/appkit/Interface.EvalConfig.md
+++ b/docs/docs/api/appkit/Interface.EvalConfig.md
@@ -1,9 +1,25 @@
 # Interface: EvalConfig
 
-Per-directory config from `evals.config.ts`.
+Eval config from `evals.config.ts` (via [defineEvalConfig](Function.defineEvalConfig.md)).
+
+Two scopes share this shape: a **root** `evals.config.ts` (project root) may
+set run-wide settings — `baseUrl` and `webServer` — plus defaults for
+`maxConcurrency`/`timeoutMs`; a **per-agent** `config/agents/<id>/evals/evals.config.ts`
+sets only that agent's `maxConcurrency`/`timeoutMs` overrides (`baseUrl`/
+`webServer` there are ignored — server lifecycle is run-wide).
 
 ## Properties
 
+### baseUrl?
+
+```ts
+optional baseUrl: string;
+```
+
+Base URL of the app to drive (root config only). Overridden by `--url`.
+
+***
+
 ### judge?
 
 ```ts
@@ -39,3 +55,13 @@ optional timeoutMs: number;
 ```
 
 Default per-eval timeout.
+
+***
+
+### webServer?
+
+```ts
+optional webServer: EvalWebServer;
+```
+
+Auto-start the app under test (root config only).
diff --git a/docs/docs/api/appkit/Interface.EvalDefinition.md b/docs/docs/api/appkit/Interface.EvalDefinition.md
index e579ce345..5420c08eb 100644
--- a/docs/docs/api/appkit/Interface.EvalDefinition.md
+++ b/docs/docs/api/appkit/Interface.EvalDefinition.md
@@ -14,6 +14,34 @@ Target agent id. Defaults to the eval's parent `config/agents/<id>` dir.
 
 ***
 
+### dataset?
+
+```ts
+optional dataset: {
+  limit?: number;
+  table: string;
+};
+```
+
+Run this eval once per row of a Databricks managed evaluation dataset (a
+Unity Catalog `catalog.schema.table` with `inputs`/`expectations` columns).
+Each row is bound to `t.input`/`t.expected`. Requires the runner to have a
+workspace client + warehouse (`--warehouse`). Omit for a single-run eval.
+
+#### limit?
+
+```ts
+optional limit: number;
+```
+
+#### table
+
+```ts
+table: string;
+```
+
+***
+
 ### description?
 
 ```ts
diff --git a/docs/docs/api/appkit/Interface.EvalDriver.md b/docs/docs/api/appkit/Interface.EvalDriver.md
index 69d81a05a..6cf961dda 100644
--- a/docs/docs/api/appkit/Interface.EvalDriver.md
+++ b/docs/docs/api/appkit/Interface.EvalDriver.md
@@ -5,6 +5,21 @@ app's agents endpoint; future drivers (in-process) implement the same shape.
 
 ## Methods
 
+### reset()?
+
+```ts
+optional reset(): void;
+```
+
+Drop the current conversation so the next `send` starts a fresh thread.
+Optional: drivers without a session concept omit it.
+
+#### Returns
+
+`void`
+
+***
+
 ### send()
 
 ```ts
diff --git a/docs/docs/api/appkit/Interface.EvalSummary.md b/docs/docs/api/appkit/Interface.EvalSummary.md
index 3c8fc1e6e..11a682148 100644
--- a/docs/docs/api/appkit/Interface.EvalSummary.md
+++ b/docs/docs/api/appkit/Interface.EvalSummary.md
@@ -28,6 +28,16 @@ passed: number;
 
 ***
 
+### passRate
+
+```ts
+passRate: number;
+```
+
+Fraction of scored (non-skipped) evals that passed, 0..1 (1 when none scored).
+
+***
+
 ### skipped
 
 ```ts
diff --git a/docs/docs/api/appkit/Interface.EvalWebServer.md b/docs/docs/api/appkit/Interface.EvalWebServer.md
new file mode 100644
index 000000000..f220ad6ae
--- /dev/null
+++ b/docs/docs/api/appkit/Interface.EvalWebServer.md
@@ -0,0 +1,49 @@
+# Interface: EvalWebServer
+
+Auto-start config for the app under test, à la Playwright's `webServer`. When
+set in a root `evals.config.ts`, the CLI boots the app before running evals
+and tears it down after — so you don't have to start the server by hand.
+
+## Properties
+
+### command
+
+```ts
+command: string;
+```
+
+Shell command that starts the app, e.g. `"npm run dev"`.
+
+***
+
+### reuseExisting?
+
+```ts
+optional reuseExisting: boolean;
+```
+
+When `true` (default), reuse a server already answering at `url` instead of
+spawning one — so a running `dev` server is used as-is. Set `false` to
+always spawn a fresh server.
+
+***
+
+### timeoutMs?
+
+```ts
+optional timeoutMs: number;
+```
+
+How long to wait for `url` to answer before giving up. Defaults to 60s.
+
+***
+
+### url?
+
+```ts
+optional url: string;
+```
+
+URL polled until it answers before evals start. Defaults to the run's
+`baseUrl` (`--url`). Readiness = any HTTP response (a 404 still proves the
+server is up).
diff --git a/docs/docs/api/appkit/Interface.JudgeConfig.md b/docs/docs/api/appkit/Interface.JudgeConfig.md
new file mode 100644
index 000000000..e20780ce0
--- /dev/null
+++ b/docs/docs/api/appkit/Interface.JudgeConfig.md
@@ -0,0 +1,31 @@
+# Interface: JudgeConfig
+
+## Properties
+
+### client
+
+```ts
+client: MlflowClient;
+```
+
+Client for the workspace hosting the judge serving endpoint.
+
+***
+
+### model
+
+```ts
+model: string;
+```
+
+Serving endpoint name used as the judge model.
+
+***
+
+### token
+
+```ts
+token: string;
+```
+
+Bearer token for the serving endpoint.
diff --git a/docs/docs/api/appkit/Interface.JudgeScore.md b/docs/docs/api/appkit/Interface.JudgeScore.md
new file mode 100644
index 000000000..d4bf0519f
--- /dev/null
+++ b/docs/docs/api/appkit/Interface.JudgeScore.md
@@ -0,0 +1,19 @@
+# Interface: JudgeScore
+
+A normalized judge result. `score` is 0..1.
+
+## Properties
+
+### rationale?
+
+```ts
+optional rationale: string;
+```
+
+***
+
+### score
+
+```ts
+score: number;
+```
diff --git a/docs/docs/api/appkit/Interface.MlflowReportOptions.md b/docs/docs/api/appkit/Interface.MlflowReportOptions.md
deleted file mode 100644
index 22733c8be..000000000
--- a/docs/docs/api/appkit/Interface.MlflowReportOptions.md
+++ /dev/null
@@ -1,21 +0,0 @@
-# Interface: MlflowReportOptions
-
-## Properties
-
-### host
-
-```ts
-host: string;
-```
-
-Databricks workspace host (scheme optional — normalized).
-
-***
-
-### token
-
-```ts
-token: string;
-```
-
-Bearer token for the MLflow REST API.
diff --git a/docs/docs/api/appkit/Interface.PostResult.md b/docs/docs/api/appkit/Interface.PostResult.md
new file mode 100644
index 000000000..35bcf684a
--- /dev/null
+++ b/docs/docs/api/appkit/Interface.PostResult.md
@@ -0,0 +1,27 @@
+# Interface: PostResult
+
+Structured result for a best-effort POST that must not throw.
+
+## Properties
+
+### error?
+
+```ts
+optional error: string;
+```
+
+***
+
+### ok
+
+```ts
+ok: boolean;
+```
+
+***
+
+### status?
+
+```ts
+optional status: number;
+```
diff --git a/docs/docs/api/appkit/Interface.ReadEvalDatasetOptions.md b/docs/docs/api/appkit/Interface.ReadEvalDatasetOptions.md
new file mode 100644
index 000000000..f6411df51
--- /dev/null
+++ b/docs/docs/api/appkit/Interface.ReadEvalDatasetOptions.md
@@ -0,0 +1,31 @@
+# Interface: ReadEvalDatasetOptions
+
+## Properties
+
+### limit?
+
+```ts
+optional limit: number;
+```
+
+Optional row cap.
+
+***
+
+### table
+
+```ts
+table: string;
+```
+
+Fully-qualified UC table: `catalog.schema.table`.
+
+***
+
+### warehouseId
+
+```ts
+warehouseId: string;
+```
+
+SQL warehouse id to run the read against.
diff --git a/docs/docs/api/appkit/Interface.ResolveDatabricksAuthOptions.md b/docs/docs/api/appkit/Interface.ResolveDatabricksAuthOptions.md
new file mode 100644
index 000000000..d13080407
--- /dev/null
+++ b/docs/docs/api/appkit/Interface.ResolveDatabricksAuthOptions.md
@@ -0,0 +1,31 @@
+# Interface: ResolveDatabricksAuthOptions
+
+## Properties
+
+### host?
+
+```ts
+optional host: string;
+```
+
+Explicit host; wins over the profile/SDK-resolved host when set.
+
+***
+
+### profile?
+
+```ts
+optional profile: string;
+```
+
+`~/.databrickscfg` profile to authenticate with (e.g. `dogfood`).
+
+***
+
+### token?
+
+```ts
+optional token: string;
+```
+
+Explicit bearer token; when set, no OAuth is minted (PAT/CI path).
diff --git a/docs/docs/api/appkit/Interface.RunEvalOptions.md b/docs/docs/api/appkit/Interface.RunEvalOptions.md
index 16a6b8190..f89837405 100644
--- a/docs/docs/api/appkit/Interface.RunEvalOptions.md
+++ b/docs/docs/api/appkit/Interface.RunEvalOptions.md
@@ -22,6 +22,16 @@ Stable id for the eval (e.g. its file path relative to the evals dir).
 
 ***
 
+### row?
+
+```ts
+optional row: DatasetRow;
+```
+
+Dataset row bound to `t.input`/`t.expected` for dataset-driven evals.
+
+***
+
 ### strict?
 
 ```ts
@@ -29,3 +39,14 @@ optional strict: boolean;
 ```
 
 When true, soft assertion failures also fail the eval.
+
+***
+
+### timeoutMs?
+
+```ts
+optional timeoutMs: number;
+```
+
+Runner-level default per-eval timeout (ms). `def.timeoutMs` wins over this;
+when both are unset the eval runs unbounded (current behavior).
diff --git a/docs/docs/api/appkit/Interface.RunEvalsOptions.md b/docs/docs/api/appkit/Interface.RunEvalsOptions.md
index 5fe5bbd9c..f88b54330 100644
--- a/docs/docs/api/appkit/Interface.RunEvalsOptions.md
+++ b/docs/docs/api/appkit/Interface.RunEvalsOptions.md
@@ -32,6 +32,51 @@ Extra request headers for the driver (e.g. auth for a deployed app).
 
 ***
 
+### judge?
+
+```ts
+optional judge: {
+  host: string;
+  model: string;
+  token: string;
+};
+```
+
+When set, enable `t.judge.*` LLM-as-judge scoring via autoevals against a
+Databricks serving endpoint (`model`).
+
+#### host
+
+```ts
+host: string;
+```
+
+#### model
+
+```ts
+model: string;
+```
+
+#### token
+
+```ts
+token: string;
+```
+
+***
+
+### maxConcurrency?
+
+```ts
+optional maxConcurrency: number;
+```
+
+Max evals/dataset rows to drive concurrently. Defaults to `1` (serial).
+Values below 1 are clamped to 1. Output order is preserved regardless.
+Wins over an agent's `evals.config.ts` `maxConcurrency`.
+
+***
+
 ### mlflow?
 
 ```ts
@@ -96,6 +141,19 @@ Progress callback, invoked as evals are discovered, started, and finished.
 
 ***
 
+### retries?
+
+```ts
+optional retries: number;
+```
+
+Re-run an eval up to this many extra times when it fails on an
+infrastructure error (a thrown error or timeout — `result.error` set), to
+absorb transient turn/stream flakiness. Assertion failures are NEVER
+retried (a wrong reply is real signal, not flake). Defaults to `0`.
+
+***
+
 ### rootDir?
 
 ```ts
@@ -113,3 +171,46 @@ optional strict: boolean;
 ```
 
 Soft assertion failures also fail the eval.
+
+***
+
+### tags?
+
+```ts
+optional tags: string[];
+```
+
+Only run evals whose `tags` intersect this list. Empty/undefined runs all.
+Tags live on the eval def, so filtering happens after each file is loaded.
+
+***
+
+### timeoutMs?
+
+```ts
+optional timeoutMs: number;
+```
+
+Default per-eval timeout (ms). A per-eval `def.timeoutMs` overrides it.
+Wins over an agent's `evals.config.ts` `timeoutMs`.
+
+***
+
+### warehouseId?
+
+```ts
+optional warehouseId: string;
+```
+
+SQL warehouse id used to read managed evaluation datasets.
+
+***
+
+### workspaceClient?
+
+```ts
+optional workspaceClient: WorkspaceClient;
+```
+
+Workspace client used to read managed evaluation datasets (for evals that
+declare `dataset`). Required alongside [warehouseId](#warehouseid) for those evals.
diff --git a/docs/docs/api/appkit/Interface.TestContext.md b/docs/docs/api/appkit/Interface.TestContext.md
index 5f249952b..ab2c549a1 100644
--- a/docs/docs/api/appkit/Interface.TestContext.md
+++ b/docs/docs/api/appkit/Interface.TestContext.md
@@ -4,6 +4,100 @@ The `t` context passed to an eval's `test` function.
 
 ## Properties
 
+### expected
+
+```ts
+readonly expected: Record<string, unknown> | undefined;
+```
+
+The current dataset row's `expectations` (ground truth / guidelines), or
+`undefined` when the row has none or the eval isn't dataset-driven.
+
+***
+
+### input
+
+```ts
+readonly input: Record<string, unknown>;
+```
+
+The current dataset row's `inputs` when the eval is dataset-driven (see
+[EvalDefinition.dataset](Interface.EvalDefinition.md#dataset)); `{}` for a plain single-run eval.
+
+***
+
+### judge
+
+```ts
+judge: {
+  closedQA: Promise<AssertionHandle>;
+  custom: Promise<AssertionHandle>;
+  factuality: Promise<AssertionHandle>;
+};
+```
+
+LLM-as-judge scoring of the last reply (via autoevals → a Databricks judge
+model). Each returns a scored assertion that gates by default (a miss fails
+the eval); chain `.atLeast(n)` to change the pass threshold or `.soft()` to
+demote to a tracked-only metric. Requires the judge to be configured
+(`--judge-model`).
+
+#### closedQA()
+
+```ts
+closedQA(criteria: string): Promise<AssertionHandle>;
+```
+
+Score whether the reply answers the question, per optional `criteria`.
+
+##### Parameters
+
+| Parameter | Type |
+| ------ | ------ |
+| `criteria` | `string` |
+
+##### Returns
+
+`Promise`\<[`AssertionHandle`](Interface.AssertionHandle.md)\>
+
+#### custom()
+
+```ts
+custom(spec: CustomJudgeSpec): Promise<AssertionHandle>;
+```
+
+A custom prompt-template judge (the TS analog of MLflow's `@scorer`).
+
+##### Parameters
+
+| Parameter | Type |
+| ------ | ------ |
+| `spec` | [`CustomJudgeSpec`](Interface.CustomJudgeSpec.md) |
+
+##### Returns
+
+`Promise`\<[`AssertionHandle`](Interface.AssertionHandle.md)\>
+
+#### factuality()
+
+```ts
+factuality(expected: string): Promise<AssertionHandle>;
+```
+
+Score factuality of the reply against an expected reference.
+
+##### Parameters
+
+| Parameter | Type |
+| ------ | ------ |
+| `expected` | `string` |
+
+##### Returns
+
+`Promise`\<[`AssertionHandle`](Interface.AssertionHandle.md)\>
+
+***
+
 ### reply
 
 ```ts
@@ -54,6 +148,29 @@ Assert a tool was called during the run (gate by default).
 
 ***
 
+### calledToolWith()
+
+```ts
+calledToolWith(name: string, expected: Record<string, unknown>): AssertionHandle;
+```
+
+Assert a tool was called with arguments that deep-contain `expected`: every
+key in `expected` must equal the actual argument (recursively for nested
+objects), so extra arguments are ignored. Gate by default.
+
+#### Parameters
+
+| Parameter | Type |
+| ------ | ------ |
+| `name` | `string` |
+| `expected` | `Record`\<`string`, `unknown`\> |
+
+#### Returns
+
+[`AssertionHandle`](Interface.AssertionHandle.md)
+
+***
+
 ### check()
 
 ```ts
@@ -75,6 +192,22 @@ Assert a value against a matcher, e.g. `t.check(t.reply, includes("Sunny"))`.
 
 ***
 
+### reset()
+
+```ts
+reset(): void;
+```
+
+Start a fresh conversation: the next `send` opens a new thread with no
+history. Use to run several independent one-shot checks in one test.
+Consecutive `send`s (without a `reset`) stay in one multi-turn conversation.
+
+#### Returns
+
+`void`
+
+***
+
 ### send()
 
 ```ts
diff --git a/docs/docs/api/appkit/index.md b/docs/docs/api/appkit/index.md
index a21025585..c3fc41620 100644
--- a/docs/docs/api/appkit/index.md
+++ b/docs/docs/api/appkit/index.md
@@ -22,6 +22,7 @@ surface with `@databricks/appkit/beta`. Not meant for application imports.
 | [DatabricksAdapter](Class.DatabricksAdapter.md) | Adapter that talks directly to Databricks Model Serving `/invocations` endpoint. |
 | [ExecutionError](Class.ExecutionError.md) | Error thrown when an operation execution fails. Use for statement failures, canceled operations, or unexpected states. |
 | [InitializationError](Class.InitializationError.md) | Error thrown when a service or component is not properly initialized. Use when accessing services before they are ready. |
+| [MlflowClient](Class.MlflowClient.md) | A thin client over the Databricks workspace REST API, owning the host + bearer token so callers (eval-run creation, assessment writes, the judge's serving endpoint) don't each re-derive URLs or re-attach auth. The host is normalized once at construction. |
 | [Plugin](Class.Plugin.md) | Base abstract class for creating AppKit plugins. |
 | [PolicyDeniedError](Class.PolicyDeniedError.md) | Thrown when a policy denies an action. |
 | [ResourceRegistry](Class.ResourceRegistry.md) | Central registry for tracking plugin resource requirements. Deduplication uses type + resourceKey (machine-stable); alias is for display only. |
@@ -45,16 +46,21 @@ surface with `@databricks/appkit/beta`. Not meant for application imports.
 | [AutoInheritToolsConfig](Interface.AutoInheritToolsConfig.md) | Auto-inherit configuration. When enabled for a given agent origin, agents with no explicit `tools:` declaration receive every registered ToolProvider plugin tool whose author marked `autoInheritable: true`. Tools without that flag — destructive, state-mutating, or privilege-sensitive — never spread automatically and must be wired via `tools:` (object or function form in code, `plugin:NAME` entries in markdown frontmatter). |
 | [BasePluginConfig](Interface.BasePluginConfig.md) | Base configuration interface for AppKit plugins |
 | [CacheConfig](Interface.CacheConfig.md) | Configuration for the CacheInterceptor. Controls TTL, size limits, storage backend, and probabilistic cleanup. |
+| [CustomJudgeSpec](Interface.CustomJudgeSpec.md) | A custom LLM-judge definition: a prompt template and choice→score mapping. |
 | [DatabaseCredential](Interface.DatabaseCredential.md) | Database credentials with OAuth token for Postgres connection |
+| [DatabricksAuth](Interface.DatabricksAuth.md) | Resolved Databricks host + bearer token for the eval runner's REST calls. |
+| [DatasetRow](Interface.DatasetRow.md) | One row of a managed evaluation dataset. `inputs` are the kwargs passed to the agent for the turn; `expectations` (when present) is the row's ground truth / guidelines. Mirrors the `{inputs, expectations}` shape of `mlflow.genai` datasets and of the Unity Catalog table backing a managed eval dataset. |
 | [DiscoveredEval](Interface.DiscoveredEval.md) | An eval file found under `config/agents/<agent>/evals/`. |
+| [DiscoveredEvalConfig](Interface.DiscoveredEvalConfig.md) | A per-agent `evals.config.ts` found under `config/agents/<agent>/evals/`. |
 | [DriveResult](Interface.DriveResult.md) | What a driver returns for a single `t.send`. |
 | [EndpointConfig](Interface.EndpointConfig.md) | - |
-| [EvalConfig](Interface.EvalConfig.md) | Per-directory config from `evals.config.ts`. |
+| [EvalConfig](Interface.EvalConfig.md) | Eval config from `evals.config.ts` (via [defineEvalConfig](Function.defineEvalConfig.md)). |
 | [EvalDefinition](Interface.EvalDefinition.md) | A single eval, default-exported from a `*.eval.ts` file. |
 | [EvalDriver](Interface.EvalDriver.md) | Abstraction over how the agent is driven. The HTTP driver posts to a running app's agents endpoint; future drivers (in-process) implement the same shape. |
 | [EvalResult](Interface.EvalResult.md) | The outcome of running one eval. |
 | [EvalRunSummary](Interface.EvalRunSummary.md) | - |
 | [EvalSummary](Interface.EvalSummary.md) | - |
+| [EvalWebServer](Interface.EvalWebServer.md) | Auto-start config for the app under test, à la Playwright's `webServer`. When set in a root `evals.config.ts`, the CLI boots the app before running evals and tears it down after — so you don't have to start the server by hand. |
 | [FilePolicyUser](Interface.FilePolicyUser.md) | Minimal user identity passed to the policy function. |
 | [FileResource](Interface.FileResource.md) | Describes the file or directory being acted upon. |
 | [FunctionTool](Interface.FunctionTool.md) | - |
@@ -65,20 +71,24 @@ surface with `@databricks/appkit/beta`. Not meant for application imports.
 | [JobAPI](Interface.JobAPI.md) | User-facing API for a single configured job. |
 | [JobConfig](Interface.JobConfig.md) | Per-job configuration options. |
 | [JobsConnectorConfig](Interface.JobsConnectorConfig.md) | - |
+| [JudgeConfig](Interface.JudgeConfig.md) | - |
+| [JudgeScore](Interface.JudgeScore.md) | A normalized judge result. `score` is 0..1. |
 | [LakebasePool](Interface.LakebasePool.md) | Subset of `pg.Pool` exposed by the Lakebase plugin. |
 | [LakebasePoolConfig](Interface.LakebasePoolConfig.md) | Configuration for creating a Lakebase connection pool |
 | [LakebasePoolManager](Interface.LakebasePoolManager.md) | Manages multiple Lakebase connection pools keyed by an identifier (e.g. userId). |
 | [MatchResult](Interface.MatchResult.md) | Result of a deterministic matcher run against a value. |
 | [McpConnectAllResult](Interface.McpConnectAllResult.md) | Per-endpoint outcome of [AppKitMcpClient.connectAll](Class.AppKitMcpClient.md#connectall). Callers (the agents plugin in particular) use the split to warn at startup when some MCP servers are unreachable without aborting boot for the rest. |
 | [Message](Interface.Message.md) | - |
-| [MlflowReportOptions](Interface.MlflowReportOptions.md) | - |
 | [PluginManifest](Interface.PluginManifest.md) | Plugin manifest that declares metadata and resource requirements. Attached to plugin classes as a static property. Extends the shared PluginManifest with strict resource types. |
 | [PluginToolkitProvider](Interface.PluginToolkitProvider.md) | Minimum shape every entry in the [Plugins](TypeAlias.Plugins.md) map must expose. Core plugins (analytics, files, genie, lakebase) implement this directly via their `.toolkit()` method. The agents plugin and standalone `runAgent` synthesize this shape for any registered plugin that doesn't implement `.toolkit()` directly (falling back to `getAgentTools()` walking). |
+| [PostResult](Interface.PostResult.md) | Structured result for a best-effort POST that must not throw. |
 | [PromptContext](Interface.PromptContext.md) | Context passed to `baseSystemPrompt` callbacks. |
+| [ReadEvalDatasetOptions](Interface.ReadEvalDatasetOptions.md) | - |
 | [RegisteredAgent](Interface.RegisteredAgent.md) | - |
 | [ReportOutcome](Interface.ReportOutcome.md) | - |
 | [RequestedClaims](Interface.RequestedClaims.md) | Optional claims for fine-grained Unity Catalog table permissions When specified, the returned token will be scoped to only the requested tables |
 | [RequestedResource](Interface.RequestedResource.md) | Resource to request permissions for in Unity Catalog |
+| [ResolveDatabricksAuthOptions](Interface.ResolveDatabricksAuthOptions.md) | - |
 | [ResourceEntry](Interface.ResourceEntry.md) | Internal representation of a resource in the registry. Extends ResourceRequirement with resolution state and plugin ownership. |
 | [ResourceRequirement](Interface.ResourceRequirement.md) | Declares a resource requirement for a plugin. Can be defined statically in a manifest or dynamically via getResourceRequirements(). |
 | [RunAgentInput](Interface.RunAgentInput.md) | - |
@@ -145,7 +155,8 @@ surface with `@databricks/appkit/beta`. Not meant for application imports.
 | [agentIdFromMarkdownPath](Function.agentIdFromMarkdownPath.md) | Derives the logical agent id from a markdown path. When the file is named `agent.md`, the id is the parent directory name (folder-based layout); otherwise the id is the file stem (e.g. legacy single-file paths). |
 | [appKitServingTypesPlugin](Function.appKitServingTypesPlugin.md) | Vite plugin to generate TypeScript types for AppKit serving endpoints. Fetches OpenAPI schemas from Databricks and generates a .d.ts with ServingEndpointRegistry module augmentation. |
 | [appKitTypesPlugin](Function.appKitTypesPlugin.md) | Vite plugin to generate types for AppKit queries. Calls generateFromEntryPoint under the hood. |
-| [buildAssessment](Function.buildAssessment.md) | Build the single pass/fail Feedback assessment for an eval result. Returns undefined when there's no trace to attach to or the eval was skipped. |
+| [buildAssessments](Function.buildAssessments.md) | Build the Feedback assessments for an eval result: one per assertion (judge assertions tagged `LLM_JUDGE` with their numeric score + rationale, so they render as judge feedback in MLflow) plus an overall `appkit_eval` pass/fail. Returns [] when there's no trace to attach to or the eval was skipped. |
+| [configureJudge](Function.configureJudge.md) | Configure the judge once. Sets the OpenAI-compatible client env autoevals reads and the default judge model. No-op-safe: on failure, judging stays disabled and [isJudgeConfigured](Function.isJudgeConfigured.md) returns false. |
 | [createAgent](Function.createAgent.md) | Pure factory for agent definitions. Returns the passed-in definition after cycle-detecting the sub-agent graph. Accepts the full `AgentDefinition` shape and is safe to call at module top-level. |
 | [createApp](Function.createApp.md) | Bootstraps AppKit with the provided configuration. |
 | [createHttpDriver](Function.createHttpDriver.md) | Drives an agent by POSTing to a running app's chat endpoint and parsing the SSE response. Keeps the thread id across `send`s so multi-turn evals share a conversation. Agent/stream errors surface as `succeeded: false` rather than throwing, so `t.succeeded()` can assert on them. |
@@ -154,15 +165,19 @@ surface with `@databricks/appkit/beta`. Not meant for application imports.
 | [defineEval](Function.defineEval.md) | Define an agent eval. Default-export the result from a `config/agents/<id>/evals/*.eval.ts` file. |
 | [defineEvalConfig](Function.defineEvalConfig.md) | Define per-directory eval config. Default-export from `evals.config.ts`. |
 | [defineTool](Function.defineTool.md) | Defines a single tool entry for a plugin's internal registry. |
+| [discoverEvalConfigs](Function.discoverEvalConfigs.md) | Discover the per-agent `evals.config.ts` (from [defineEvalConfig](Function.defineEvalConfig.md)) at `<rootDir>/config/agents/<agent>/evals/evals.config.ts`. Config is per-agent: each agent's config applies only to that agent's evals. Agents without a config file are omitted. Returns a stable, sorted list. |
 | [discoverEvalFiles](Function.discoverEvalFiles.md) | Discover evals under `<rootDir>/config/agents/<agent>/evals/`. The agent id is the directory name; the eval id is the file path relative to that evals dir with `.eval.ts` stripped. Returns a stable, sorted list. |
 | [equals](Function.equals.md) | Passes when the value equals `expected` exactly. |
 | [evalGlyph](Function.evalGlyph.md) | Status glyph for a single eval result. |
 | [executeFromRegistry](Function.executeFromRegistry.md) | Validates tool-call arguments against the entry's schema and invokes its handler. On validation failure, returns an LLM-friendly error string (matching the behavior of `tool()`) rather than throwing, so the model can self-correct on its next turn. |
 | [extractServingEndpoints](Function.extractServingEndpoints.md) | Extract serving endpoint config from a server file by AST-parsing it. Looks for `serving({ endpoints: { alias: { env: "..." }, ... } })` calls and extracts the endpoint alias names and their environment variable mappings. |
+| [findRootEvalConfig](Function.findRootEvalConfig.md) | Path to the root `evals.config.ts` (from [defineEvalConfig](Function.defineEvalConfig.md)) at `<rootDir>/evals.config.ts`, or `undefined` when absent. The root config holds run-wide settings (`baseUrl`, `webServer`); it's distinct from the per-agent configs found by [discoverEvalConfigs](Function.discoverEvalConfigs.md). |
 | [findServerFile](Function.findServerFile.md) | Find the server entry file by checking candidate paths in order. |
 | [formatEvalDetail](Function.formatEvalDetail.md) | Indented detail lines for a failing eval (error + failing assertions). |
 | [formatEvalHeadline](Function.formatEvalHeadline.md) | The one-line header for a single eval result (no failure detail). |
 | [formatEvalResults](Function.formatEvalResults.md) | Render all results as a human-readable console report (non-streaming). |
+| [formatResultsJson](Function.formatResultsJson.md) | Render results as a machine-readable JSON report (2-space indented): `{ summary: EvalSummary, results: EvalResult[] }`. Faithful to the types — every field present on a result round-trips. |
+| [formatResultsJUnit](Function.formatResultsJUnit.md) | Render results as JUnit XML for standard CI test reporters: a single `<testsuite name="appkit-agent-evals">` with one `<testcase>` per result. Failures carry a `<failure>` (error or failing-gate summary); skips a `<skipped>`. All attribute/text values are XML-escaped. |
 | [formatSummaryLine](Function.formatSummaryLine.md) | The final PASS/FAIL summary line. |
 | [functionToolToDefinition](Function.functionToolToDefinition.md) | - |
 | [generateDatabaseCredential](Function.generateDatabaseCredential.md) | Generate OAuth credentials for Postgres database connection using the proper Postgres API. |
@@ -176,18 +191,27 @@ surface with `@databricks/appkit/beta`. Not meant for application imports.
 | [includes](Function.includes.md) | Passes when the value contains `substring`. |
 | [isFunctionTool](Function.isFunctionTool.md) | - |
 | [isHostedTool](Function.isHostedTool.md) | - |
+| [isJudgeConfigured](Function.isJudgeConfigured.md) | - |
 | [isSQLTypeMarker](Function.isSQLTypeMarker.md) | Type guard to check if a value is a SQL type marker |
 | [isToolkitEntry](Function.isToolkitEntry.md) | Type guard for `ToolkitEntry` — used by the agents plugin to differentiate toolkit references from inline tools in a mixed `tools` record. |
 | [loadAgentFromFile](Function.loadAgentFromFile.md) | Loads a single markdown agent file and resolves its frontmatter against registered plugin toolkits + ambient tool library. |
 | [loadAgentsFromDir](Function.loadAgentsFromDir.md) | Scans a directory for one subdirectory per agent, each containing `agent.md` (frontmatter + body). Produces an `AgentDefinition` record keyed by agent id (folder name). Throws on frontmatter errors or unresolved references. Returns an empty map if the directory does not exist. |
+| [loadRootEvalConfig](Function.loadRootEvalConfig.md) | Load the root `evals.config.ts` under `rootDir` (the project root), or return `undefined` when there is none. This is the run-wide config carrying `baseUrl`/`webServer`; the CLI reads it to resolve options and manage the app-under-test lifecycle before calling [runEvalsInDir](Function.runEvalsInDir.md). |
 | [matches](Function.matches.md) | Passes when the value matches `pattern`. |
 | [mcpServer](Function.mcpServer.md) | Factory for declaring a custom MCP server tool. |
+| [normalizeHost](Function.normalizeHost.md) | Ensure the host has a scheme (Databricks env often lacks `https://`). |
 | [parseTextToolCalls](Function.parseTextToolCalls.md) | Parses text-based tool calls from model output. |
+| [readEvalDataset](Function.readEvalDataset.md) | Read a Databricks managed evaluation dataset (a Unity Catalog table with `inputs`/`expectations` columns) into rows, over the public SQL Statement Execution API. Reuses SQLWarehouseConnector for submit/poll/transform — its result transform already JSON-parses string columns into objects, so `inputs`/`expectations` come back as records whether the table stores them as JSON strings or structs. |
 | [reportToMlflow](Function.reportToMlflow.md) | Write one pass/fail assessment per eval result to the Databricks MLflow REST API. Never throws — failures are collected so the run still reports. |
+| [resolveDatabricksAuth](Function.resolveDatabricksAuth.md) | Resolve `{host, token}` for the eval runner the same way the rest of AppKit authenticates: construct a Databricks `WorkspaceClient` and let its config mint (and later refresh) an OAuth bearer from the CLI profile — no hand-set PAT required. An explicit host/token still wins (PAT or CI env), so the SDK is only consulted for whatever isn't supplied. |
 | [resolveHostedTools](Function.resolveHostedTools.md) | - |
+| [resolveWorkspaceClient](Function.resolveWorkspaceClient.md) | Construct a Databricks `WorkspaceClient` for the eval runner — the object the SDK-backed connectors (e.g. `SQLWarehouseConnector`) take. An explicit host+token builds a PAT client; otherwise the profile (or ambient config) is used and the SDK resolves credentials, minting OAuth as needed. Returns `undefined` if construction throws (missing/invalid config). |
 | [runAgent](Function.runAgent.md) | Standalone agent execution without `createApp`. Resolves the adapter, binds inline tools, and drives the adapter's `run()` loop to completion. |
+| [runBounded](Function.runBounded.md) | Run `tasks` through a bounded worker pool and return their results in the SAME order as the input, regardless of completion order. Each task receives its input index so callers can key on it. `limit` is clamped to at least 1 (and to the task count); at `limit === 1` this is a serial loop. Individual task rejections are surfaced per-slot via `settle` rather than aborting siblings — but eval tasks never reject (failures become results). |
 | [runEval](Function.runEval.md) | Run a single eval against a driver. Never throws for assertion or agent failures — those become a non-passing [EvalResult](Interface.EvalResult.md). Only a malformed eval definition surfaces as `result.error`. |
 | [runEvalsInDir](Function.runEvalsInDir.md) | Discover, load, and run every eval under each agent's `evals/` dir, driving the agents on a running app. Never throws for an individual eval — load/run failures become non-passing [EvalResult](Interface.EvalResult.md)s. |
+| [runWithRetries](Function.runWithRetries.md) | Run `attempt` up to `1 + retries` times, stopping as soon as it returns a result without an `error` (infra failures — thrown errors or timeouts — set `error`; assertion failures do not, so a failed-but-completed eval is returned on the first try and never retried). Returns the last result when every attempt errored. `retries` below 0 is treated as 0. |
 | [summarize](Function.summarize.md) | - |
 | [tool](Function.tool.md) | Factory for defining function tools with Zod schemas. |
 | [toolsFromRegistry](Function.toolsFromRegistry.md) | Produces the `AgentToolDefinition[]` a ToolProvider exposes to the LLM, deriving `parameters` JSON Schema from each entry's Zod schema. |
+| [userTurns](Function.userTurns.md) | Extract every user-message content, in order, from an MLflow `{"messages":[{"role":"user","content":"..."}]}` input. A dataset row can carry a full multi-turn conversation; replaying these against one thread (one `t.send` per returned string) lets the agent see the accumulating history. |
diff --git a/docs/docs/api/appkit/typedoc-sidebar.ts b/docs/docs/api/appkit/typedoc-sidebar.ts
index db2537bdb..9d93fbc69 100644
--- a/docs/docs/api/appkit/typedoc-sidebar.ts
+++ b/docs/docs/api/appkit/typedoc-sidebar.ts
@@ -61,6 +61,11 @@ const typedocSidebar: SidebarsConfig = {
           id: "api/appkit/Class.InitializationError",
           label: "InitializationError"
         },
+        {
+          type: "doc",
+          id: "api/appkit/Class.MlflowClient",
+          label: "MlflowClient"
+        },
         {
           type: "doc",
           id: "api/appkit/Class.Plugin",
@@ -157,16 +162,36 @@ const typedocSidebar: SidebarsConfig = {
           id: "api/appkit/Interface.CacheConfig",
           label: "CacheConfig"
         },
+        {
+          type: "doc",
+          id: "api/appkit/Interface.CustomJudgeSpec",
+          label: "CustomJudgeSpec"
+        },
         {
           type: "doc",
           id: "api/appkit/Interface.DatabaseCredential",
           label: "DatabaseCredential"
         },
+        {
+          type: "doc",
+          id: "api/appkit/Interface.DatabricksAuth",
+          label: "DatabricksAuth"
+        },
+        {
+          type: "doc",
+          id: "api/appkit/Interface.DatasetRow",
+          label: "DatasetRow"
+        },
         {
           type: "doc",
           id: "api/appkit/Interface.DiscoveredEval",
           label: "DiscoveredEval"
         },
+        {
+          type: "doc",
+          id: "api/appkit/Interface.DiscoveredEvalConfig",
+          label: "DiscoveredEvalConfig"
+        },
         {
           type: "doc",
           id: "api/appkit/Interface.DriveResult",
@@ -207,6 +232,11 @@ const typedocSidebar: SidebarsConfig = {
           id: "api/appkit/Interface.EvalSummary",
           label: "EvalSummary"
         },
+        {
+          type: "doc",
+          id: "api/appkit/Interface.EvalWebServer",
+          label: "EvalWebServer"
+        },
         {
           type: "doc",
           id: "api/appkit/Interface.FilePolicyUser",
@@ -257,6 +287,16 @@ const typedocSidebar: SidebarsConfig = {
           id: "api/appkit/Interface.JobsConnectorConfig",
           label: "JobsConnectorConfig"
         },
+        {
+          type: "doc",
+          id: "api/appkit/Interface.JudgeConfig",
+          label: "JudgeConfig"
+        },
+        {
+          type: "doc",
+          id: "api/appkit/Interface.JudgeScore",
+          label: "JudgeScore"
+        },
         {
           type: "doc",
           id: "api/appkit/Interface.LakebasePool",
@@ -287,11 +327,6 @@ const typedocSidebar: SidebarsConfig = {
           id: "api/appkit/Interface.Message",
           label: "Message"
         },
-        {
-          type: "doc",
-          id: "api/appkit/Interface.MlflowReportOptions",
-          label: "MlflowReportOptions"
-        },
         {
           type: "doc",
           id: "api/appkit/Interface.PluginManifest",
@@ -302,11 +337,21 @@ const typedocSidebar: SidebarsConfig = {
           id: "api/appkit/Interface.PluginToolkitProvider",
           label: "PluginToolkitProvider"
         },
+        {
+          type: "doc",
+          id: "api/appkit/Interface.PostResult",
+          label: "PostResult"
+        },
         {
           type: "doc",
           id: "api/appkit/Interface.PromptContext",
           label: "PromptContext"
         },
+        {
+          type: "doc",
+          id: "api/appkit/Interface.ReadEvalDatasetOptions",
+          label: "ReadEvalDatasetOptions"
+        },
         {
           type: "doc",
           id: "api/appkit/Interface.RegisteredAgent",
@@ -327,6 +372,11 @@ const typedocSidebar: SidebarsConfig = {
           id: "api/appkit/Interface.RequestedResource",
           label: "RequestedResource"
         },
+        {
+          type: "doc",
+          id: "api/appkit/Interface.ResolveDatabricksAuthOptions",
+          label: "ResolveDatabricksAuthOptions"
+        },
         {
           type: "doc",
           id: "api/appkit/Interface.ResourceEntry",
@@ -602,8 +652,13 @@ const typedocSidebar: SidebarsConfig = {
         },
         {
           type: "doc",
-          id: "api/appkit/Function.buildAssessment",
-          label: "buildAssessment"
+          id: "api/appkit/Function.buildAssessments",
+          label: "buildAssessments"
+        },
+        {
+          type: "doc",
+          id: "api/appkit/Function.configureJudge",
+          label: "configureJudge"
         },
         {
           type: "doc",
@@ -645,6 +700,11 @@ const typedocSidebar: SidebarsConfig = {
           id: "api/appkit/Function.defineTool",
           label: "defineTool"
         },
+        {
+          type: "doc",
+          id: "api/appkit/Function.discoverEvalConfigs",
+          label: "discoverEvalConfigs"
+        },
         {
           type: "doc",
           id: "api/appkit/Function.discoverEvalFiles",
@@ -670,6 +730,11 @@ const typedocSidebar: SidebarsConfig = {
           id: "api/appkit/Function.extractServingEndpoints",
           label: "extractServingEndpoints"
         },
+        {
+          type: "doc",
+          id: "api/appkit/Function.findRootEvalConfig",
+          label: "findRootEvalConfig"
+        },
         {
           type: "doc",
           id: "api/appkit/Function.findServerFile",
@@ -690,6 +755,16 @@ const typedocSidebar: SidebarsConfig = {
           id: "api/appkit/Function.formatEvalResults",
           label: "formatEvalResults"
         },
+        {
+          type: "doc",
+          id: "api/appkit/Function.formatResultsJson",
+          label: "formatResultsJson"
+        },
+        {
+          type: "doc",
+          id: "api/appkit/Function.formatResultsJUnit",
+          label: "formatResultsJUnit"
+        },
         {
           type: "doc",
           id: "api/appkit/Function.formatSummaryLine",
@@ -755,6 +830,11 @@ const typedocSidebar: SidebarsConfig = {
           id: "api/appkit/Function.isHostedTool",
           label: "isHostedTool"
         },
+        {
+          type: "doc",
+          id: "api/appkit/Function.isJudgeConfigured",
+          label: "isJudgeConfigured"
+        },
         {
           type: "doc",
           id: "api/appkit/Function.isSQLTypeMarker",
@@ -775,6 +855,11 @@ const typedocSidebar: SidebarsConfig = {
           id: "api/appkit/Function.loadAgentsFromDir",
           label: "loadAgentsFromDir"
         },
+        {
+          type: "doc",
+          id: "api/appkit/Function.loadRootEvalConfig",
+          label: "loadRootEvalConfig"
+        },
         {
           type: "doc",
           id: "api/appkit/Function.matches",
@@ -785,26 +870,51 @@ const typedocSidebar: SidebarsConfig = {
           id: "api/appkit/Function.mcpServer",
           label: "mcpServer"
         },
+        {
+          type: "doc",
+          id: "api/appkit/Function.normalizeHost",
+          label: "normalizeHost"
+        },
         {
           type: "doc",
           id: "api/appkit/Function.parseTextToolCalls",
           label: "parseTextToolCalls"
         },
+        {
+          type: "doc",
+          id: "api/appkit/Function.readEvalDataset",
+          label: "readEvalDataset"
+        },
         {
           type: "doc",
           id: "api/appkit/Function.reportToMlflow",
           label: "reportToMlflow"
         },
+        {
+          type: "doc",
+          id: "api/appkit/Function.resolveDatabricksAuth",
+          label: "resolveDatabricksAuth"
+        },
         {
           type: "doc",
           id: "api/appkit/Function.resolveHostedTools",
           label: "resolveHostedTools"
         },
+        {
+          type: "doc",
+          id: "api/appkit/Function.resolveWorkspaceClient",
+          label: "resolveWorkspaceClient"
+        },
         {
           type: "doc",
           id: "api/appkit/Function.runAgent",
           label: "runAgent"
         },
+        {
+          type: "doc",
+          id: "api/appkit/Function.runBounded",
+          label: "runBounded"
+        },
         {
           type: "doc",
           id: "api/appkit/Function.runEval",
@@ -815,6 +925,11 @@ const typedocSidebar: SidebarsConfig = {
           id: "api/appkit/Function.runEvalsInDir",
           label: "runEvalsInDir"
         },
+        {
+          type: "doc",
+          id: "api/appkit/Function.runWithRetries",
+          label: "runWithRetries"
+        },
         {
           type: "doc",
           id: "api/appkit/Function.summarize",
@@ -829,6 +944,11 @@ const typedocSidebar: SidebarsConfig = {
           type: "doc",
           id: "api/appkit/Function.toolsFromRegistry",
           label: "toolsFromRegistry"
+        },
+        {
+          type: "doc",
+          id: "api/appkit/Function.userTurns",
+          label: "userTurns"
         }
       ]
     }
diff --git a/packages/appkit/src/evals/discover.ts b/packages/appkit/src/evals/discover.ts
index 96bd6a74c..2ca02786b 100644
--- a/packages/appkit/src/evals/discover.ts
+++ b/packages/appkit/src/evals/discover.ts
@@ -95,6 +95,17 @@ export function discoverEvalFiles(rootDir: string): DiscoveredEval[] {
   );
 }
 
+/**
+ * Path to the root `evals.config.ts` (from {@link defineEvalConfig}) at
+ * `<rootDir>/evals.config.ts`, or `undefined` when absent. The root config
+ * holds run-wide settings (`baseUrl`, `webServer`); it's distinct from the
+ * per-agent configs found by {@link discoverEvalConfigs}.
+ */
+export function findRootEvalConfig(rootDir: string): string | undefined {
+  const file = path.join(rootDir, "evals.config.ts");
+  return isFile(file) ? file : undefined;
+}
+
 /**
  * Discover the per-agent `evals.config.ts` (from {@link defineEvalConfig}) at
  * `<rootDir>/config/agents/<agent>/evals/evals.config.ts`. Config is per-agent:
diff --git a/packages/appkit/src/evals/index.ts b/packages/appkit/src/evals/index.ts
index 0fa5aadba..b7fb84c77 100644
--- a/packages/appkit/src/evals/index.ts
+++ b/packages/appkit/src/evals/index.ts
@@ -19,6 +19,7 @@ export {
   type DiscoveredEvalConfig,
   discoverEvalConfigs,
   discoverEvalFiles,
+  findRootEvalConfig,
 } from "./discover";
 export { createHttpDriver, type HttpDriverOptions } from "./http-driver";
 export {
@@ -49,6 +50,7 @@ export { type RunEvalOptions, runEval } from "./run-eval";
 export {
   type EvalProgress,
   type EvalRunSummary,
+  loadRootEvalConfig,
   type RunEvalsOptions,
   runBounded,
   runEvalsInDir,
@@ -63,6 +65,7 @@ export type {
   EvalDefinition,
   EvalDriver,
   EvalResult,
+  EvalWebServer,
   Matcher,
   MatchResult,
   Severity,
diff --git a/packages/appkit/src/evals/run-evals.ts b/packages/appkit/src/evals/run-evals.ts
index d5347f51c..ad3b9c910 100644
--- a/packages/appkit/src/evals/run-evals.ts
+++ b/packages/appkit/src/evals/run-evals.ts
@@ -6,6 +6,7 @@ import {
   type DiscoveredEval,
   discoverEvalConfigs,
   discoverEvalFiles,
+  findRootEvalConfig,
 } from "./discover";
 import { createHttpDriver } from "./http-driver";
 import { configureJudge } from "./judge";
@@ -126,6 +127,20 @@ async function loadEvalConfig(file: string): Promise<EvalConfig | undefined> {
   return resolveConfigDefault(mod);
 }
 
+/**
+ * Load the root `evals.config.ts` under `rootDir` (the project root), or return
+ * `undefined` when there is none. This is the run-wide config carrying
+ * `baseUrl`/`webServer`; the CLI reads it to resolve options and manage the
+ * app-under-test lifecycle before calling {@link runEvalsInDir}.
+ */
+export async function loadRootEvalConfig(
+  rootDir: string,
+): Promise<EvalConfig | undefined> {
+  const file = findRootEvalConfig(rootDir);
+  if (!file) return undefined;
+  return loadEvalConfig(file);
+}
+
 /**
  * Unwrap the config default export across module-interop shapes (see
  * {@link resolveEvalDefault}). A config has no `.test`, so the first plain
diff --git a/packages/appkit/src/evals/tests/discover.test.ts b/packages/appkit/src/evals/tests/discover.test.ts
index aefc64caa..d3c55922f 100644
--- a/packages/appkit/src/evals/tests/discover.test.ts
+++ b/packages/appkit/src/evals/tests/discover.test.ts
@@ -2,7 +2,11 @@ import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
 import { tmpdir } from "node:os";
 import path from "node:path";
 import { afterEach, beforeEach, describe, expect, test } from "vitest";
-import { discoverEvalConfigs, discoverEvalFiles } from "../discover";
+import {
+  discoverEvalConfigs,
+  discoverEvalFiles,
+  findRootEvalConfig,
+} from "../discover";
 
 let root: string;
 
@@ -59,3 +63,15 @@ describe("discoverEvalConfigs", () => {
     expect(discoverEvalConfigs(root)).toEqual([]);
   });
 });
+
+describe("findRootEvalConfig", () => {
+  test("finds a root evals.config.ts", () => {
+    write("evals.config.ts");
+    expect(findRootEvalConfig(root)).toBe(path.join(root, "evals.config.ts"));
+  });
+
+  test("returns undefined when absent (and ignores per-agent configs)", () => {
+    write("config/agents/support/evals/evals.config.ts");
+    expect(findRootEvalConfig(root)).toBeUndefined();
+  });
+});
diff --git a/packages/appkit/src/evals/types.ts b/packages/appkit/src/evals/types.ts
index e65b7fc3a..a2ef151be 100644
--- a/packages/appkit/src/evals/types.ts
+++ b/packages/appkit/src/evals/types.ts
@@ -167,7 +167,39 @@ export interface EvalDefinition {
   test(t: TestContext): Promise<void> | void;
 }
 
-/** Per-directory config from `evals.config.ts`. */
+/**
+ * Auto-start config for the app under test, à la Playwright's `webServer`. When
+ * set in a root `evals.config.ts`, the CLI boots the app before running evals
+ * and tears it down after — so you don't have to start the server by hand.
+ */
+export interface EvalWebServer {
+  /** Shell command that starts the app, e.g. `"npm run dev"`. */
+  command: string;
+  /**
+   * URL polled until it answers before evals start. Defaults to the run's
+   * `baseUrl` (`--url`). Readiness = any HTTP response (a 404 still proves the
+   * server is up).
+   */
+  url?: string;
+  /** How long to wait for `url` to answer before giving up. Defaults to 60s. */
+  timeoutMs?: number;
+  /**
+   * When `true` (default), reuse a server already answering at `url` instead of
+   * spawning one — so a running `dev` server is used as-is. Set `false` to
+   * always spawn a fresh server.
+   */
+  reuseExisting?: boolean;
+}
+
+/**
+ * Eval config from `evals.config.ts` (via {@link defineEvalConfig}).
+ *
+ * Two scopes share this shape: a **root** `evals.config.ts` (project root) may
+ * set run-wide settings — `baseUrl` and `webServer` — plus defaults for
+ * `maxConcurrency`/`timeoutMs`; a **per-agent** `config/agents/<id>/evals/evals.config.ts`
+ * sets only that agent's `maxConcurrency`/`timeoutMs` overrides (`baseUrl`/
+ * `webServer` there are ignored — server lifecycle is run-wide).
+ */
 export interface EvalConfig {
   /** LLM judge config. Defaults to the agent's own serving endpoint. */
   judge?: { model?: string };
@@ -175,6 +207,10 @@ export interface EvalConfig {
   maxConcurrency?: number;
   /** Default per-eval timeout. */
   timeoutMs?: number;
+  /** Base URL of the app to drive (root config only). Overridden by `--url`. */
+  baseUrl?: string;
+  /** Auto-start the app under test (root config only). */
+  webServer?: EvalWebServer;
 }
 
 /** The outcome of running one eval. */
diff --git a/packages/shared/src/cli/commands/agent/eval.ts b/packages/shared/src/cli/commands/agent/eval.ts
index 9bc69d178..d8ece64c1 100644
--- a/packages/shared/src/cli/commands/agent/eval.ts
+++ b/packages/shared/src/cli/commands/agent/eval.ts
@@ -1,3 +1,4 @@
+import { type ChildProcess, spawn } from "node:child_process";
 import fs from "node:fs";
 import { Command, Option } from "commander";
 
@@ -48,6 +49,7 @@ interface EvalRunner {
     host?: string;
     token?: string;
   }): unknown;
+  loadRootEvalConfig(rootDir: string): Promise<EvalConfig | undefined>;
   evalGlyph(result: unknown): string;
   formatEvalDetail(result: unknown): string[];
   formatSummaryLine(results: unknown[]): string;
@@ -56,6 +58,19 @@ interface EvalRunner {
   summarize(results: unknown[]): { allPassed: boolean; passRate: number };
 }
 
+/** Subset of `@databricks/appkit/beta`'s `EvalConfig` the CLI reads. */
+interface EvalConfig {
+  maxConcurrency?: number;
+  timeoutMs?: number;
+  baseUrl?: string;
+  webServer?: {
+    command: string;
+    url?: string;
+    timeoutMs?: number;
+    reuseExisting?: boolean;
+  };
+}
+
 /**
  * Loaded at runtime from the consuming project so this command (which ships in
  * `@databricks/shared`) doesn't take a build-time dependency on appkit. The
@@ -90,8 +105,83 @@ function positiveInt(raw: string | undefined): number | undefined {
   return n > 0 ? n : undefined;
 }
 
+/** True when `url` answers with any HTTP response (a 404 still proves it's up). */
+async function isServerUp(url: string): Promise<boolean> {
+  try {
+    await fetch(url, { signal: AbortSignal.timeout(2000) });
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Start the app under test per a root config's `webServer`, à la Playwright:
+ * reuse a server already answering at `url` (unless `reuseExisting: false`),
+ * else spawn `command`, poll `url` until it answers or `timeoutMs` elapses.
+ * Returns a `stop()` that kills the spawned process group (a no-op when the
+ * server was reused). Logs go to stderr so a machine reporter's stdout stays
+ * clean. Throws if the server never comes up.
+ */
+async function startWebServer(
+  webServer: NonNullable<EvalConfig["webServer"]>,
+  baseUrl: string,
+): Promise<{ stop: () => void }> {
+  const url = webServer.url ?? baseUrl;
+  const reuse = webServer.reuseExisting !== false;
+  const noop = { stop: () => {} };
+
+  if (reuse && (await isServerUp(url))) {
+    console.error(`Reusing server already running at ${url}`);
+    return noop;
+  }
+
+  console.error(`Starting web server: ${webServer.command}`);
+  // `detached` + a negative-PID kill lets us tear down the whole process group
+  // (dev servers spawn child processes). stdout/stderr inherit so the user sees
+  // build output; the server's stdout is not our report stream.
+  const child: ChildProcess = spawn(webServer.command, {
+    shell: true,
+    detached: true,
+    stdio: "inherit",
+  });
+
+  let exited = false;
+  child.on("exit", () => {
+    exited = true;
+  });
+
+  const stop = (): void => {
+    if (exited || child.pid === undefined) return;
+    try {
+      process.kill(-child.pid, "SIGTERM");
+    } catch {
+      // Group already gone, or never became a leader — best-effort.
+    }
+  };
+
+  const deadline = Date.now() + (webServer.timeoutMs ?? 60_000);
+  try {
+    while (Date.now() < deadline) {
+      if (exited) throw new Error("web server exited before becoming ready");
+      if (await isServerUp(url)) {
+        console.error(`Web server ready at ${url}`);
+        return { stop };
+      }
+      await new Promise((r) => setTimeout(r, 500));
+    }
+  } catch (err) {
+    stop();
+    throw err;
+  }
+  stop();
+  throw new Error(
+    `web server did not respond at ${url} within ${webServer.timeoutMs ?? 60_000}ms`,
+  );
+}
+
 interface EvalOptions {
-  url: string;
+  url?: string;
   strict?: boolean;
   root?: string;
   header?: string[];
@@ -116,6 +206,15 @@ async function runAgentEval(
 ): Promise<void> {
   const runner = await loadRunner();
 
+  // Root `evals.config.ts` (project root) carries run-wide settings — baseUrl,
+  // webServer, and defaults for concurrency/timeout. A CLI flag always wins.
+  const rootDir = opts.root ?? process.cwd();
+  const config = (await runner.loadRootEvalConfig(rootDir)) ?? {};
+
+  // Base URL: --url flag > config.baseUrl > built-in default. `--url` has no
+  // commander default so an unset flag is undefined and lets config win.
+  const baseUrl = opts.url ?? config.baseUrl ?? "http://localhost:8000";
+
   // Databricks credentials shared by auth resolution and the workspace client:
   // an explicit flag/DATABRICKS_* env wins, else the SDK resolves from the CLI
   // profile.
@@ -150,11 +249,13 @@ async function runAgentEval(
   const warehouseId = opts.warehouse ?? process.env.DATABRICKS_WAREHOUSE_ID;
   const workspaceClient = runner.resolveWorkspaceClient(credentials);
 
-  // Drive up to N evals/rows concurrently (default serial). Ignore junk input.
-  const maxConcurrency = positiveInt(opts.concurrency);
+  // Drive up to N evals/rows concurrently (default serial). --concurrency flag
+  // wins over the root config; junk input is ignored.
+  const maxConcurrency = positiveInt(opts.concurrency) ?? config.maxConcurrency;
 
-  // Runner-level default per-eval timeout (ms). A per-eval `timeoutMs` wins.
-  const timeoutMs = positiveInt(opts.timeout);
+  // Runner-level default per-eval timeout (ms). --timeout flag wins over the
+  // root config; a per-eval `timeoutMs` overrides both (applied in the runner).
+  const timeoutMs = positiveInt(opts.timeout) ?? config.timeoutMs;
 
   // Extra attempts for evals that fail on an infra error (turn/timeout). Junk
   // or negative input falls back to no retries.
@@ -175,7 +276,7 @@ async function runAgentEval(
     switch (event.type) {
       case "discovered":
         info(
-          `Running ${event.total} eval${event.total === 1 ? "" : "s"} against ${opts.url}\n`,
+          `Running ${event.total} eval${event.total === 1 ? "" : "s"} against ${baseUrl}\n`,
         );
         break;
       case "run-created":
@@ -198,22 +299,33 @@ async function runAgentEval(
     }
   };
 
-  const summary = await runner.runEvalsInDir({
-    rootDir: opts.root,
-    baseUrl: opts.url,
-    filter,
-    tags: opts.tag,
-    strict: opts.strict,
-    headers: opts.header ? parseHeaders(opts.header) : undefined,
-    mlflow,
-    judge,
-    workspaceClient,
-    warehouseId,
-    maxConcurrency,
-    timeoutMs,
-    retries,
-    onEvent,
-  });
+  // Boot the app under test if the root config declares a webServer (reuses an
+  // already-running server unless told otherwise); always torn down after.
+  const server = config.webServer
+    ? await startWebServer(config.webServer, baseUrl)
+    : undefined;
+
+  let summary: EvalRunSummary;
+  try {
+    summary = await runner.runEvalsInDir({
+      rootDir,
+      baseUrl,
+      filter,
+      tags: opts.tag,
+      strict: opts.strict,
+      headers: opts.header ? parseHeaders(opts.header) : undefined,
+      mlflow,
+      judge,
+      workspaceClient,
+      warehouseId,
+      maxConcurrency,
+      timeoutMs,
+      retries,
+      onEvent,
+    });
+  } finally {
+    server?.stop();
+  }
 
   // The final human summary always shows (stderr for machine reporters so it
   // never pollutes the report on stdout/file).
@@ -288,7 +400,10 @@ export const agentEvalCommand = new Command("eval")
     "[filter]",
     "Only run evals whose <agent>/<id> contains this substring (or an exact agent id)",
   )
-  .option("--url <url>", "Base URL of the running app", "http://localhost:3000")
+  .option(
+    "--url <url>",
+    "Base URL of the app to drive (default: evals.config.ts baseUrl, else http://localhost:8000)",
+  )
   .option("--strict", "Fail on soft-assertion misses too", false)
   .option(
     "--root <dir>",