fix(ai): gateway error handling — relabel, sanitize the body, and retry transient 5xx

extensions/levelcode-ai/agent.js

@@ -719,7 +719,7 @@ async function runAgent(ctx) {
 			let streamed = false;
 			let textChars = 0;
 			const turnOpts = {
-				providerId: ctx.providerId, baseURL: ctx.baseURL,
+				providerId: ctx.providerId, baseURL: ctx.baseURL, label: ctx.label,
 				apiKey: ctx.apiKey, model: ctx.model, maxTokens: perTurnMax, system: system,
 				messages, tools: tools, signal: ctx.signal,
 				onText: (t) => { streamed = true; textChars += t.length; ctx.post({ type: 'agentDelta', text: t }); },
@@ -729,7 +729,10 @@ async function runAgent(ctx) {
 						: name === 'delete_file' ? 'deleting a file…'
 						: name === 'run_command' ? 'preparing command…' : name === 'update_plan' ? 'planning…' : 'running ' + name + '…';
 					ctx.post({ type: 'agentStatus', text: verb });
-				}
+				},
+				// A transient upstream 5xx (502/503/504) is retried once before it can fail the run — surface it
+				// as a status rather than a mystery pause, and log it. Nothing has streamed yet when this fires.
+				onRetry: (info) => { dbg('turn.retry', info); ctx.post({ type: 'agentStatus', text: 'upstream busy (' + info.status + ') — retrying…' }); }
 			};
 			let turn;
 			try {

extensions/levelcode-ai/extension.js

@@ -941,7 +941,7 @@ async function compactAgentMemory() {
 	let summary;
 	try {
 		summary = await providers.complete({
-			providerId: req.providerId, apiKey: req.apiKey, baseURL: req.baseURL,
+			providerId: req.providerId, apiKey: req.apiKey, baseURL: req.baseURL, label: req.label,
 			model: req.model, maxTokens: 1500,
 			system: COMPACT_SYSTEM,
 			messages: [{ role: 'user', content: COMPACT_INSTRUCTIONS + flat }]
@@ -1021,6 +1021,8 @@ async function agentFlow(text) {
 			messages: agentMessages, // persists across runs → the agent remembers the session
 			providerId: req.providerId,         // Anthropic native, or an OpenAI-shaped provider via translation (P2)
 			baseURL: req.baseURL,               // for the custom / Ollama endpoints
+			label: req.label,                   // route name for error attribution — "LevelCode Cloud" on the gateway,
+			                                    // else the provider's own label; keeps a 502 from being blamed on "OpenAI"
 			apiKey: req.apiKey,
 			model: req.model,
 			maxSteps: Math.max(1, cfg.get('agent.maxSteps', 25)),
@@ -1116,7 +1118,7 @@ async function handleSend(text) {
 			return;
 		}
 		const doStream = (r) => providers.streamChat({
-			providerId: r.providerId, apiKey: r.apiKey, baseURL: r.baseURL,
+			providerId: r.providerId, apiKey: r.apiKey, baseURL: r.baseURL, label: r.label,
 			model: r.model, maxTokens: r.maxTokens, system: SYSTEM_PROMPT,
 			messages: conversation, signal: abort.signal, onDelta
 		});

extensions/levelcode-ai/providers/index.js

@@ -142,8 +142,9 @@ function secretStorageKey(id) {
 /**
  * Unified streaming chat across providers. `messages` is the shared {role,content:string} shape
  * (valid for both Anthropic and OpenAI). Anthropic → native adapter; everything else → openaiCompat.
- * @param {{providerId:string, apiKey?:string, baseURL?:string, model:string, maxTokens?:number,
- *          system:string, messages:any[], signal?:AbortSignal, onDelta:(t:string)=>void}} o
+ * @param {{providerId:string, apiKey?:string, baseURL?:string, label?:string, model:string, maxTokens?:number,
+ *          system:string, messages:any[], signal?:AbortSignal, onDelta:(t:string)=>void,
+ *          onRetry?:(info:{attempt:number,retries:number,status:number})=>void}} o
  */
 async function streamChat(o) {
 	const p = getProvider(o.providerId);
@@ -155,16 +156,17 @@ async function streamChat(o) {
 		});
 	}
 	return openai.streamOpenAI({
-		baseURL: o.baseURL || p.baseURL, apiKey: o.apiKey, headers: p.headers, label: p.label,
+		baseURL: o.baseURL || p.baseURL, apiKey: o.apiKey, headers: p.headers, label: o.label || p.label,
 		model: o.model, maxTokens: o.maxTokens, system: o.system, messages: o.messages,
-		signal: o.signal, onDelta: o.onDelta
+		signal: o.signal, onDelta: o.onDelta, onRetry: o.onRetry
 	});
 }
 
 /**
  * Unified one-shot completion across providers (inline ghost-text / edit). Returns full text.
- * @param {{providerId:string, apiKey?:string, baseURL?:string, model:string, maxTokens?:number,
- *          system:string, messages:any[], stop?:string[], signal?:AbortSignal}} o
+ * @param {{providerId:string, apiKey?:string, baseURL?:string, label?:string, model:string, maxTokens?:number,
+ *          system:string, messages:any[], stop?:string[], signal?:AbortSignal,
+ *          onRetry?:(info:{attempt:number,retries:number,status:number})=>void}} o
  * @returns {Promise<string>}
  */
 async function complete(o) {
@@ -177,9 +179,9 @@ async function complete(o) {
 		});
 	}
 	return openai.completeOpenAI({
-		baseURL: o.baseURL || p.baseURL, apiKey: o.apiKey, headers: p.headers, label: p.label,
+		baseURL: o.baseURL || p.baseURL, apiKey: o.apiKey, headers: p.headers, label: o.label || p.label,
 		model: o.model, maxTokens: o.maxTokens, system: o.system, messages: o.messages,
-		stop: o.stop, signal: o.signal
+		stop: o.stop, signal: o.signal, onRetry: o.onRetry
 	});
 }
 
@@ -195,9 +197,10 @@ function supportsTools(id) {
  * translation); every other provider → the OpenAI adapter, which translates the Anthropic-shaped
  * transcript/tools in and the streamed tool-calls back out. Returns the SAME
  * {content, stop_reason, usage, malformed} shape for both, so agent.js is provider-agnostic.
- * @param {{providerId:string, apiKey?:string, baseURL?:string, model:string, maxTokens?:number,
+ * @param {{providerId:string, apiKey?:string, baseURL?:string, label?:string, model:string, maxTokens?:number,
  *          system:string, messages:any[], tools?:any[], signal?:AbortSignal,
- *          onText?:(t:string)=>void, onToolStart?:(name:string)=>void}} o
+ *          onText?:(t:string)=>void, onToolStart?:(name:string)=>void,
+ *          onRetry?:(info:{attempt:number,retries:number,status:number})=>void}} o
  */
 async function streamAgentTurn(o) {
 	const p = getProvider(o.providerId);
@@ -209,9 +212,9 @@ async function streamAgentTurn(o) {
 		});
 	}
 	return openai.streamOpenAIAgentTurn({
-		baseURL: o.baseURL || p.baseURL, apiKey: o.apiKey, headers: p.headers, label: p.label,
+		baseURL: o.baseURL || p.baseURL, apiKey: o.apiKey, headers: p.headers, label: o.label || p.label,
 		model: o.model, maxTokens: o.maxTokens, system: o.system, messages: o.messages, tools: o.tools,
-		signal: o.signal, onText: o.onText, onToolStart: o.onToolStart
+		signal: o.signal, onText: o.onText, onToolStart: o.onToolStart, onRetry: o.onRetry
 	});
 }
 

extensions/levelcode-ai/providers/openaiCompat.js

@@ -76,24 +76,111 @@ function deltaFromEvent(ev) {
 	return typeof d.content === 'string' ? d.content : '';
 }
 
+// Fallback reason phrases for when fetch leaves res.statusText empty (some HTTP/2 responses do). Not
+// exhaustive — just what a model endpoint or the proxy in front of it realistically returns.
+const STATUS_REASON = {
+	400: 'Bad Request', 401: 'Unauthorized', 403: 'Forbidden', 404: 'Not Found',
+	408: 'Request Timeout', 413: 'Payload Too Large', 429: 'Too Many Requests',
+	500: 'Internal Server Error', 502: 'Bad Gateway', 503: 'Service Unavailable', 504: 'Gateway Timeout'
+};
+
+/**
+ * Pull a human-readable message out of an error response body, or '' when there isn't one worth showing.
+ * The body is UNTRUSTED and provider-shaped: a JSON `{error:{message}}` on a normal API rejection, but a
+ * raw HTML page when a proxy IN FRONT of the model (nginx/Cloudflare) returns a 5xx — dumping that page
+ * into a chat transcript is pure noise. Return '' for HTML so the caller falls back to the status reason;
+ * cap anything else so a stray multi-KB body can't flood the UI. Pure — unit-tested.
+ */
+function extractApiError(body) {
+	const s = String(body || '').trim();
+	if (!s) { return ''; }
+	if (s[0] === '<' || /<html[\s>]/i.test(s)) { return ''; }              // HTML proxy page — no useful message
+	if (s[0] === '{' || s[0] === '[') {
+		try {
+			const j = JSON.parse(s);
+			const m = (j && j.error && (j.error.message || (typeof j.error === 'string' ? j.error : ''))) || (j && j.message) || '';
+			if (m) { return String(m).slice(0, 500); }
+		} catch { /* not valid JSON after all — fall through to the capped-text path */ }
+	}
+	return s.length > 300 ? s.slice(0, 300) + '…' : s;     // short plain text: keep it, capped
+}
+
+/**
+ * Build a clean Error for a failed (`!res.ok`) response: `"<label> API <status>: <detail>"`, where detail
+ * is the provider's own message when it gave one, else the HTTP status reason — never a dumped HTML page.
+ * `label` names the ROUTE (e.g. "LevelCode Cloud", "OpenRouter"), so the failure is attributed correctly
+ * rather than blamed on whichever adapter happens to carry it. Sets `.status` for retry/refresh logic.
+ */
+function httpError(label, res, body) {
+	const detail = extractApiError(body) || res.statusText || STATUS_REASON[res.status] || 'request failed';
+	const e = new Error(`${label} API ${res.status}: ${detail}`);
+	e.status = res.status;
+	return e;
+}
+
+// Upstream statuses worth ONE automatic retry: a proxy in front of the model (nginx/Cloudflare/the gateway)
+// briefly couldn't reach a healthy backend. These almost always clear within a second. Deliberately NOT
+// retried: 429 (rate limit — needs Retry-After, and hammering makes it worse), 500 (usually a real request
+// error, not a blip), and every other 4xx. A thrown fetch error (network drop, abort) is not retried either
+// — only an HTTP response whose status is in this set.
+const TRANSIENT_STATUS = new Set([502, 503, 504]);
+const TRANSIENT_RETRIES = 1;      // one extra attempt after the first — a single pre-stream retry
+const RETRY_DELAY_MS = 700;       // backoff before the retry (RETRY_DELAY_MS * attempt); overridable per call
+
+/**
+ * A backoff that wakes early the instant the turn is aborted, so Stop stays responsive. Resolves — never
+ * rejects: the caller's next `fetch` sees the aborted signal and rejects with the native AbortError, which
+ * is exactly how a normal aborted request already surfaces. Works with no signal too.
+ */
+function retryDelay(ms, signal) {
+	return new Promise((resolve) => {
+		if (signal && signal.aborted) { return resolve(); }
+		const timer = setTimeout(done, ms);
+		function done() { clearTimeout(timer); if (signal) { signal.removeEventListener('abort', done); } resolve(); }
+		if (signal) { signal.addEventListener('abort', done, { once: true }); }
+	});
+}
+
+/**
+ * POST /chat/completions with a single pre-stream retry on a transient upstream status (502/503/504).
+ *
+ * This is the ONLY place a chat request is retried, and it is safe precisely because it runs BEFORE any SSE
+ * line is read: on a transient status the response carries no model output, so nothing has been shown to the
+ * user or metered, and re-issuing the request cannot duplicate output or double-bill the UI. A failure that
+ * happens mid-stream is a different code path and is never retried here. A 401 is not transient, so it is
+ * thrown straight through for the agent's token-refresh path. Non-transient statuses and an exhausted retry
+ * throw a clean httpError. `opts.onRetry({attempt,retries,status})` fires just before each backoff (for a
+ * visible "retrying…" hint); `opts.retryDelayMs` overrides the backoff (0 in tests). Returns res.ok===true.
+ */
+async function postChat(opts, body) {
+	const label = opts.label || 'OpenAI-compatible';
+	const base = opts.retryDelayMs != null ? opts.retryDelayMs : RETRY_DELAY_MS;
+	const init = { method: 'POST', headers: authHeaders(opts), body: JSON.stringify(body), signal: opts.signal };
+	const url = baseUrl(opts) + '/chat/completions';
+	for (let attempt = 0; ; attempt++) {
+		const res = await fetch(url, init);
+		if (res.ok) { return res; }
+		const text = await res.text().catch(() => '');
+		if (attempt < TRANSIENT_RETRIES && TRANSIENT_STATUS.has(res.status)) {
+			if (typeof opts.onRetry === 'function') { opts.onRetry({ attempt: attempt + 1, retries: TRANSIENT_RETRIES, status: res.status }); }
+			await retryDelay(base * (attempt + 1), opts.signal);
+			continue;
+		}
+		throw httpError(label, res, text);
+	}
+}
+
 /**
  * Streaming chat over /v1/chat/completions. opts.onDelta(text) per chunk; resolves at end.
  * @param {{baseURL:string, apiKey?:string, headers?:object, label?:string, model:string,
  *          maxTokens?:number, system?:string, messages:any[], stop?:string[],
- *          signal?:AbortSignal, onDelta:(t:string)=>void}} opts
+ *          signal?:AbortSignal, onDelta:(t:string)=>void,
+ *          onRetry?:(info:{attempt:number,retries:number,status:number})=>void}} opts
  */
 async function streamOpenAI(opts) {
 	const label = opts.label || 'OpenAI-compatible';
-	const res = await fetch(baseUrl(opts) + '/chat/completions', {
-		method: 'POST',
-		headers: authHeaders(opts),
-		body: JSON.stringify(buildChatBody(Object.assign({}, opts, { stream: true }))),
-		signal: opts.signal
-	});
-	if (!res.ok || !res.body) {
-		const text = await res.text().catch(() => '');
-		throw new Error(`${label} API ${res.status}: ${text || res.statusText}`);
-	}
+	const res = await postChat(opts, buildChatBody(Object.assign({}, opts, { stream: true })));
+	if (!res.body) { throw new Error(label + ' API ' + res.status + ': empty response stream'); }
 	await readLines(res, (line) => {
 		const s = line.trim();
 		if (!s.startsWith('data:')) { return; }
@@ -110,21 +197,12 @@ async function streamOpenAI(opts) {
 /**
  * Non-streaming single completion (inline ghost-text / edit). Returns the full text.
  * @param {{baseURL:string, apiKey?:string, headers?:object, label?:string, model:string,
- *          maxTokens?:number, system?:string, messages:any[], stop?:string[], signal?:AbortSignal}} opts
+ *          maxTokens?:number, system?:string, messages:any[], stop?:string[], signal?:AbortSignal,
+ *          onRetry?:(info:{attempt:number,retries:number,status:number})=>void}} opts
  * @returns {Promise<string>}
  */
 async function completeOpenAI(opts) {
-	const label = opts.label || 'OpenAI-compatible';
-	const res = await fetch(baseUrl(opts) + '/chat/completions', {
-		method: 'POST',
-		headers: authHeaders(opts),
-		body: JSON.stringify(buildChatBody(Object.assign({}, opts, { stream: false }))),
-		signal: opts.signal
-	});
-	if (!res.ok) {
-		const text = await res.text().catch(() => '');
-		throw new Error(`${label} API ${res.status}: ${text || res.statusText}`);
-	}
+	const res = await postChat(opts, buildChatBody(Object.assign({}, opts, { stream: false })));
 	const data = await res.json();
 	const c = data && data.choices && data.choices[0];
 	return (c && c.message && typeof c.message.content === 'string') ? c.message.content : '';
@@ -172,7 +250,8 @@ async function listOpenAIModels(opts) {
  * to {type:'text'} / {type:'tool_use', id, name, input} blocks.
  * @param {{baseURL:string, apiKey?:string, headers?:object, label?:string, model:string,
  *          maxTokens?:number, system:string, messages:any[], tools?:any[], signal?:AbortSignal,
- *          onText?:(t:string)=>void, onToolStart?:(name:string)=>void}} opts
+ *          onText?:(t:string)=>void, onToolStart?:(name:string)=>void,
+ *          onRetry?:(info:{attempt:number,retries:number,status:number})=>void}} opts
  * @returns {Promise<{content:any[], stop_reason:string, usage:any, malformed:Set<string>}>}
  */
 async function streamOpenAIAgentTurn(opts) {
@@ -188,16 +267,8 @@ async function streamOpenAIAgentTurn(opts) {
 	// on OpenAI-shaped providers — they omit usage from streams unless include_usage is set. Mainstream
 	// providers (OpenAI/OpenRouter/Groq/Together/Fireworks/DeepSeek/xAI/Mistral) honor it.
 	body.stream_options = { include_usage: true };
-	const res = await fetch(baseUrl(opts) + '/chat/completions', {
-		method: 'POST',
-		headers: authHeaders(opts),
-		body: JSON.stringify(body),
-		signal: opts.signal
-	});
-	if (!res.ok || !res.body) {
-		const text = await res.text().catch(() => '');
-		throw new Error(`${label} API ${res.status}: ${text || res.statusText}`);
-	}
+	const res = await postChat(opts, body);
+	if (!res.body) { throw new Error(label + ' API ' + res.status + ': empty response stream'); }
 	let text = '';
 	/** @type {any[]} */
 	const acc = [];
@@ -240,4 +311,4 @@ async function streamOpenAIAgentTurn(opts) {
 	return { content, stop_reason: stopReason, usage, malformed };
 }
 
-module.exports = { streamOpenAI, completeOpenAI, listOpenAIModels, streamOpenAIAgentTurn, buildChatBody, deltaFromEvent, isReasoningModel, isAnthropicFamily, splitOutCachedTokens };
+module.exports = { streamOpenAI, completeOpenAI, listOpenAIModels, streamOpenAIAgentTurn, buildChatBody, deltaFromEvent, isReasoningModel, isAnthropicFamily, splitOutCachedTokens, extractApiError, httpError, postChat };