From ac13e08d69c607fdb6bf15346d4e72af268c484e Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Fri, 31 Jul 2026 12:32:51 +0100 Subject: [PATCH 001/106] feat(observability-map): package scaffold and route scanner --- .../observability-map/package.json | 21 ++++ .../observability-map/src/scan.ts | 111 ++++++++++++++++++ .../observability-map/src/types.ts | 22 ++++ .../observability-map/test/scan.test.ts | 29 +++++ .../observability-map/tsconfig.build.json | 5 + .../observability-map/tsconfig.json | 17 +++ .../observability-map/vitest.config.ts | 5 + pnpm-lock.yaml | 13 ++ 8 files changed, 223 insertions(+) create mode 100644 internal-packages/observability-map/package.json create mode 100644 internal-packages/observability-map/src/scan.ts create mode 100644 internal-packages/observability-map/src/types.ts create mode 100644 internal-packages/observability-map/test/scan.test.ts create mode 100644 internal-packages/observability-map/tsconfig.build.json create mode 100644 internal-packages/observability-map/tsconfig.json create mode 100644 internal-packages/observability-map/vitest.config.ts diff --git a/internal-packages/observability-map/package.json b/internal-packages/observability-map/package.json new file mode 100644 index 00000000000..3d6f695f67b --- /dev/null +++ b/internal-packages/observability-map/package.json @@ -0,0 +1,21 @@ +{ + "name": "@internal/observability-map", + "private": true, + "version": "0.0.1", + "main": "./dist/src/index.js", + "types": "./dist/src/index.d.ts", + "dependencies": { + "typescript": "catalog:" + }, + "devDependencies": { + "@types/node": "^24.13.3", + "rimraf": "6.0.1" + }, + "scripts": { + "clean": "rimraf dist", + "typecheck": "tsc --noEmit", + "build": "pnpm run clean && tsc -p tsconfig.build.json", + "test": "vitest run", + "test:watch": "vitest" + } +} diff --git a/internal-packages/observability-map/src/scan.ts b/internal-packages/observability-map/src/scan.ts new file mode 100644 index 00000000000..de4203e885d --- /dev/null +++ b/internal-packages/observability-map/src/scan.ts @@ -0,0 +1,111 @@ +import ts from "typescript"; +import { readdirSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +import type { EntryPoint } from "./types.js"; + +function calleeName(expr: ts.Expression): string | null { + if (ts.isIdentifier(expr)) return expr.text; + if (ts.isPropertyAccessExpression(expr)) return expr.name.text; + return null; +} + +export function scanFile(fileName: string, source: string): EntryPoint | null { + const sf = ts.createSourceFile(fileName, source, ts.ScriptTarget.Latest, true); + + const ep: EntryPoint = { + fileName, + source, + hasLoader: false, + hasAction: false, + loaderInitializerCallee: null, + actionInitializerCallee: null, + importedNames: [], + calleeNames: [], + hasTryCatch: false, + statementCount: 0, + }; + + const isExported = (n: ts.Node) => + ts.canHaveModifiers(n) && + ts.getModifiers(n)?.some((m) => m.kind === ts.SyntaxKind.ExportKeyword) === true; + + const visit = (node: ts.Node) => { + if (ts.isImportDeclaration(node) && node.importClause) { + const bindings = node.importClause.namedBindings; + if (bindings && ts.isNamedImports(bindings)) { + for (const el of bindings.elements) ep.importedNames.push(el.name.text); + } + if (node.importClause.name) ep.importedNames.push(node.importClause.name.text); + } + + if (ts.isVariableStatement(node) && isExported(node)) { + for (const decl of node.declarationList.declarations) { + const name = decl.name.getText(sf); + if (name !== "loader" && name !== "action") continue; + if (name === "loader") ep.hasLoader = true; + if (name === "action") ep.hasAction = true; + let init = decl.initializer; + if (init && ts.isPropertyAccessExpression(init)) init = init.expression; + if (init && ts.isCallExpression(init)) { + const cn = calleeName(init.expression); + if (name === "loader") ep.loaderInitializerCallee = cn; + else ep.actionInitializerCallee = cn; + } + } + } + + if (ts.isFunctionDeclaration(node) && node.name && isExported(node)) { + if (node.name.text === "loader") ep.hasLoader = true; + if (node.name.text === "action") ep.hasAction = true; + if (node.body) ep.statementCount += node.body.statements.length; + } + + if (ts.isTryStatement(node)) ep.hasTryCatch = true; + + if (ts.isCallExpression(node)) { + const cn = calleeName(node.expression); + if (cn) ep.calleeNames.push(cn); + } + + ts.forEachChild(node, visit); + }; + + visit(sf); + + if (!ep.hasLoader && !ep.hasAction) return null; + if (ep.statementCount === 0) { + ep.statementCount = countArrowBodyStatements(sf); + } + return ep; +} + +function countArrowBodyStatements(sf: ts.SourceFile): number { + let count = 0; + const visit = (n: ts.Node) => { + if ((ts.isArrowFunction(n) || ts.isFunctionExpression(n)) && n.body && ts.isBlock(n.body)) { + count += n.body.statements.length; + } + ts.forEachChild(n, visit); + }; + visit(sf); + return count; +} + +export function scanDirectory(dir: string): { + entryPoints: EntryPoint[]; + parseFailures: string[]; +} { + const entryPoints: EntryPoint[] = []; + const parseFailures: string[] = []; + const files = readdirSync(dir).filter((f) => /\.(ts|tsx)$/.test(f) && !f.endsWith(".test.ts")); + + for (const fileName of files) { + try { + const ep = scanFile(fileName, readFileSync(join(dir, fileName), "utf8")); + if (ep) entryPoints.push(ep); + } catch { + parseFailures.push(fileName); + } + } + return { entryPoints, parseFailures }; +} diff --git a/internal-packages/observability-map/src/types.ts b/internal-packages/observability-map/src/types.ts new file mode 100644 index 00000000000..7a331707e21 --- /dev/null +++ b/internal-packages/observability-map/src/types.ts @@ -0,0 +1,22 @@ +export type CheckStatus = "pass" | "fail" | "not-applicable"; + +export type CheckResult = { + id: string; + status: CheckStatus; + detail?: string; +}; + +export type EntryPoint = { + fileName: string; + source: string; + hasLoader: boolean; + hasAction: boolean; + /** Callee name when `loader`/`action` is assigned from a call, e.g. a route builder. */ + loaderInitializerCallee: string | null; + actionInitializerCallee: string | null; + importedNames: string[]; + calleeNames: string[]; + hasTryCatch: boolean; + /** Statement count across loader/action bodies, used by the triviality rule. */ + statementCount: number; +}; diff --git a/internal-packages/observability-map/test/scan.test.ts b/internal-packages/observability-map/test/scan.test.ts new file mode 100644 index 00000000000..27f233311cf --- /dev/null +++ b/internal-packages/observability-map/test/scan.test.ts @@ -0,0 +1,29 @@ +import { scanFile } from "../src/scan.js"; + +const LOADER = ` +import { json } from "@remix-run/server-runtime"; +export async function loader() { return json({}); } +`; + +const COMPONENT_ONLY = ` +export default function Page() { return null; } +`; + +describe("scanFile", () => { + it("detects an exported loader as a server entry point", () => { + const ep = scanFile("api.v1.things.ts", LOADER); + expect(ep).not.toBeNull(); + expect(ep!.hasLoader).toBe(true); + expect(ep!.hasAction).toBe(false); + }); + + it("ignores a route that only exports a component", () => { + expect(scanFile("_app.things.tsx", COMPONENT_ONLY)).toBeNull(); + }); + + it("detects a loader assigned from a call expression", () => { + const ep = scanFile("api.v1.x.ts", `export const loader = createLoaderApiRoute({});`); + expect(ep!.hasLoader).toBe(true); + expect(ep!.loaderInitializerCallee).toBe("createLoaderApiRoute"); + }); +}); diff --git a/internal-packages/observability-map/tsconfig.build.json b/internal-packages/observability-map/tsconfig.build.json new file mode 100644 index 00000000000..56ade258e18 --- /dev/null +++ b/internal-packages/observability-map/tsconfig.build.json @@ -0,0 +1,5 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { "noEmit": false, "outDir": "dist", "declaration": true }, + "exclude": ["node_modules", "dist", "test"] +} diff --git a/internal-packages/observability-map/tsconfig.json b/internal-packages/observability-map/tsconfig.json new file mode 100644 index 00000000000..61669556f76 --- /dev/null +++ b/internal-packages/observability-map/tsconfig.json @@ -0,0 +1,17 @@ +{ + "compilerOptions": { + "target": "ES2019", + "lib": ["ES2019"], + "module": "ESNext", + "moduleResolution": "Bundler", + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true, + "isolatedModules": true, + "skipLibCheck": true, + "noEmit": true, + "strict": true, + "types": ["vitest/globals", "node"], + "customConditions": ["@triggerdotdev/source"] + }, + "exclude": ["node_modules", "dist"] +} diff --git a/internal-packages/observability-map/vitest.config.ts b/internal-packages/observability-map/vitest.config.ts new file mode 100644 index 00000000000..c1680ce67f9 --- /dev/null +++ b/internal-packages/observability-map/vitest.config.ts @@ -0,0 +1,5 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { include: ["**/*.test.ts"], globals: true, isolate: true, testTimeout: 10_000 }, +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f37e3250982..b2b0c4a17c4 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1139,6 +1139,19 @@ importers: specifier: 6.0.1 version: 6.0.1 + internal-packages/observability-map: + dependencies: + typescript: + specifier: 6.0.3 + version: 6.0.3 + devDependencies: + '@types/node': + specifier: 24.13.3 + version: 24.13.3 + rimraf: + specifier: 6.0.1 + version: 6.0.1 + internal-packages/otlp-importer: dependencies: long: From 4fd2020de5a61ec7914d3350d6570d3b7f5f2620 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Fri, 31 Jul 2026 13:15:06 +0100 Subject: [PATCH 002/106] fix(observability-map): find every route entry point and scope body analysis The scanner missed 30% of server entry points in apps/webapp/app/routes. It found 299 of 427; it now finds all 427 with 0 parse failures. - detect named export clauses (export { loader }, export { h as loader }), resolving a local binding back to its declaration for the builder callee - recurse one level into flat-route directories and key entry points by a path relative to the scan root, so route.tsx files stay distinct - count statements for the loader/action bodies only, recursing through try, if, loop and switch blocks so a try-wrapped body reports its real size - scope hasTryCatch and calleeNames to the entry-point bodies, leaving importedNames file-wide - resolve the initializer callee to the root of a call chain - throw on parse diagnostics so parseFailures can actually fire - scan .test.ts route files and exclude .d.ts instead --- .../observability-map/src/scan.ts | 413 +++++++++++++++--- .../observability-map/src/types.ts | 3 + .../observability-map/test/scan.test.ts | 329 +++++++++++++- 3 files changed, 686 insertions(+), 59 deletions(-) diff --git a/internal-packages/observability-map/src/scan.ts b/internal-packages/observability-map/src/scan.ts index de4203e885d..cb506ce2cd3 100644 --- a/internal-packages/observability-map/src/scan.ts +++ b/internal-packages/observability-map/src/scan.ts @@ -3,92 +3,376 @@ import { readdirSync, readFileSync } from "node:fs"; import { join } from "node:path"; import type { EntryPoint } from "./types.js"; +/** Thrown by `scanFile` when the source does not parse cleanly. */ +export class ParseFailureError extends Error { + constructor( + readonly fileName: string, + readonly diagnostic: string + ) { + super(`${fileName}: ${diagnostic}`); + this.name = "ParseFailureError"; + } +} + +type EntryFunction = ts.FunctionDeclaration | ts.FunctionExpression | ts.ArrowFunction; + +function isEntryFunction(node: ts.Node): node is EntryFunction { + return ( + ts.isFunctionDeclaration(node) || ts.isFunctionExpression(node) || ts.isArrowFunction(node) + ); +} + +/** Strip wrappers that do not change which expression is really being referred to. */ +function unwrap(expr: ts.Expression): ts.Expression { + let current = expr; + for (;;) { + if ( + ts.isParenthesizedExpression(current) || + ts.isAwaitExpression(current) || + ts.isAsExpression(current) || + ts.isSatisfiesExpression(current) || + ts.isNonNullExpression(current) + ) { + current = current.expression; + continue; + } + return current; + } +} + +/** + * Root callee of a call, unwrapping chains: `createLoaderApiRoute({}).withCors()` + * resolves to `createLoaderApiRoute`, not `withCors`. + */ +function rootCalleeName(call: ts.CallExpression): string | null { + let current: ts.Expression = unwrap(call.expression); + for (;;) { + if (ts.isIdentifier(current)) return current.text; + if ( + ts.isCallExpression(current) || + ts.isPropertyAccessExpression(current) || + ts.isElementAccessExpression(current) + ) { + current = unwrap(current.expression); + continue; + } + return null; + } +} + +/** Callee as recorded in `calleeNames`: the identifier, or the property for a member call. */ function calleeName(expr: ts.Expression): string | null { - if (ts.isIdentifier(expr)) return expr.text; - if (ts.isPropertyAccessExpression(expr)) return expr.name.text; + const target = unwrap(expr); + if (ts.isIdentifier(target)) return target.text; + if (ts.isPropertyAccessExpression(target)) return target.name.text; return null; } +/** + * Functions on a `handler` property of an object argument, e.g. `createSSELoader({ handler })` and + * the per-method `{ POST: { handler } }` map. Only `handler` counts: the surrounding config also + * holds lambdas (`findResource`, `authorization.resource`) that are not the entry-point body. + */ +function collectNamedHandlers(object: ts.ObjectLiteralExpression, out: EntryFunction[]): void { + for (const property of object.properties) { + if (!ts.isPropertyAssignment(property) || !property.name) continue; + const value = unwrap(property.initializer); + if (ts.isObjectLiteralExpression(value)) { + collectNamedHandlers(value, out); + continue; + } + const name = + ts.isIdentifier(property.name) || ts.isStringLiteral(property.name) + ? property.name.text + : null; + if (name === "handler" && isEntryFunction(value)) out.push(value); + } +} + +/** Every function passed as an argument anywhere in a call chain, e.g. the builder's handler. */ +function collectHandlerFunctions(expr: ts.Expression, out: EntryFunction[]): void { + const target = unwrap(expr); + if (ts.isCallExpression(target)) { + for (const arg of target.arguments) { + const unwrapped = unwrap(arg); + if (isEntryFunction(unwrapped)) out.push(unwrapped); + else if (ts.isObjectLiteralExpression(unwrapped)) collectNamedHandlers(unwrapped, out); + } + collectHandlerFunctions(target.expression, out); + return; + } + if (ts.isPropertyAccessExpression(target) || ts.isElementAccessExpression(target)) { + collectHandlerFunctions(target.expression, out); + } +} + +type Initializer = { callee: string | null; functions: EntryFunction[] }; + +const NO_INITIALIZER: Initializer = { callee: null, functions: [] }; + +/** Top-level `function x` / `const x = ...` declarations, keyed by binding name. */ +type LocalDeclarations = Map; + +function analyzeInitializer( + expr: ts.Expression | undefined, + locals: LocalDeclarations, + seen: Set +): Initializer { + if (!expr) return NO_INITIALIZER; + const target = unwrap(expr); + + if (isEntryFunction(target)) return { callee: null, functions: [target] }; + + if (ts.isCallExpression(target)) { + const functions: EntryFunction[] = []; + collectHandlerFunctions(target, functions); + return { callee: rootCalleeName(target), functions }; + } + + // `export const action = route.action` where `const route = createActionApiRoute(...)`, and the + // plain alias `export const loader = h`. Resolve back to the declaration the name came from. + if (ts.isIdentifier(target)) return resolveLocal(target.text, locals, seen); + if (ts.isPropertyAccessExpression(target)) { + const root = unwrap(target.expression); + if (ts.isIdentifier(root)) return resolveLocal(root.text, locals, seen); + } + + return NO_INITIALIZER; +} + +function resolveLocal(name: string, locals: LocalDeclarations, seen: Set): Initializer { + if (seen.has(name)) return NO_INITIALIZER; + seen.add(name); + const local = locals.get(name); + if (!local) return NO_INITIALIZER; + if (ts.isFunctionDeclaration(local)) return { callee: null, functions: [local] }; + return analyzeInitializer(local, locals, seen); +} + +/** + * Statements in a statement, counting through block-bearing statements so a body wrapped in a + * single `try` reports its real size. Does not descend into nested function bodies. + */ +function countStatement(statement: ts.Statement): number { + if (ts.isBlock(statement)) { + return countStatements(statement.statements); + } + + let count = 1; + + if (ts.isTryStatement(statement)) { + count += countStatements(statement.tryBlock.statements); + if (statement.catchClause) count += countStatements(statement.catchClause.block.statements); + if (statement.finallyBlock) count += countStatements(statement.finallyBlock.statements); + return count; + } + + if (ts.isIfStatement(statement)) { + count += countStatement(statement.thenStatement); + if (statement.elseStatement) count += countStatement(statement.elseStatement); + return count; + } + + if ( + ts.isForStatement(statement) || + ts.isForInStatement(statement) || + ts.isForOfStatement(statement) || + ts.isWhileStatement(statement) || + ts.isDoStatement(statement) || + ts.isLabeledStatement(statement) || + ts.isWithStatement(statement) + ) { + count += countStatement(statement.statement); + return count; + } + + if (ts.isSwitchStatement(statement)) { + for (const clause of statement.caseBlock.clauses) { + count += countStatements(clause.statements); + } + return count; + } + + return count; +} + +function countStatements(statements: ts.NodeArray): number { + let count = 0; + for (const statement of statements) count += countStatement(statement); + return count; +} + +function countFunctionStatements(fn: EntryFunction): number { + if (!fn.body) return 0; + // A concise arrow body (`() => json({})`) is one expression, so one statement. + if (!ts.isBlock(fn.body)) return 1; + return countStatements(fn.body.statements); +} + +type EntryTarget = { + hasLoader: boolean; + hasAction: boolean; + loaderInitializerCallee: string | null; + actionInitializerCallee: string | null; + functions: Set; +}; + +/** + * Top-level `function x` / `const x = ...` declarations by binding name, so a named export clause + * (`export { action }`) can be resolved back to the initializer it came from. + */ +function collectLocalDeclarations(sf: ts.SourceFile): LocalDeclarations { + const locals: LocalDeclarations = new Map(); + + for (const statement of sf.statements) { + if (ts.isFunctionDeclaration(statement) && statement.name) { + locals.set(statement.name.text, statement); + continue; + } + if (!ts.isVariableStatement(statement)) continue; + + for (const decl of statement.declarationList.declarations) { + if (!decl.initializer) continue; + if (ts.isIdentifier(decl.name)) { + locals.set(decl.name.text, decl.initializer); + continue; + } + if (ts.isObjectBindingPattern(decl.name)) { + for (const element of decl.name.elements) { + if (ts.isIdentifier(element.name)) locals.set(element.name.text, decl.initializer); + } + } + } + } + + return locals; +} + export function scanFile(fileName: string, source: string): EntryPoint | null { const sf = ts.createSourceFile(fileName, source, ts.ScriptTarget.Latest, true); - const ep: EntryPoint = { - fileName, - source, + // `createSourceFile` recovers from malformed input instead of throwing, so the diagnostics are + // the only signal that a file did not parse. + const parseDiagnostics = (sf as ts.SourceFile & { parseDiagnostics?: ts.Diagnostic[] }) + .parseDiagnostics; + if (parseDiagnostics && parseDiagnostics.length > 0) { + const first = parseDiagnostics[0]!; + throw new ParseFailureError(fileName, ts.flattenDiagnosticMessageText(first.messageText, " ")); + } + + const importedNames: string[] = []; + for (const statement of sf.statements) { + if (!ts.isImportDeclaration(statement) || !statement.importClause) continue; + const bindings = statement.importClause.namedBindings; + if (bindings && ts.isNamedImports(bindings)) { + for (const el of bindings.elements) importedNames.push(el.name.text); + } + if (statement.importClause.name) importedNames.push(statement.importClause.name.text); + } + + const target: EntryTarget = { hasLoader: false, hasAction: false, loaderInitializerCallee: null, actionInitializerCallee: null, - importedNames: [], - calleeNames: [], - hasTryCatch: false, - statementCount: 0, + functions: new Set(), + }; + + const record = (name: string, initializer: Initializer) => { + if (name === "loader") { + target.hasLoader = true; + target.loaderInitializerCallee ??= initializer.callee; + } else { + target.hasAction = true; + target.actionInitializerCallee ??= initializer.callee; + } + for (const fn of initializer.functions) target.functions.add(fn); }; const isExported = (n: ts.Node) => ts.canHaveModifiers(n) && ts.getModifiers(n)?.some((m) => m.kind === ts.SyntaxKind.ExportKeyword) === true; - const visit = (node: ts.Node) => { - if (ts.isImportDeclaration(node) && node.importClause) { - const bindings = node.importClause.namedBindings; - if (bindings && ts.isNamedImports(bindings)) { - for (const el of bindings.elements) ep.importedNames.push(el.name.text); + const locals = collectLocalDeclarations(sf); + + for (const statement of sf.statements) { + if (ts.isFunctionDeclaration(statement) && statement.name && isExported(statement)) { + const name = statement.name.text; + if (name === "loader" || name === "action") { + record(name, { callee: null, functions: [statement] }); } - if (node.importClause.name) ep.importedNames.push(node.importClause.name.text); + continue; } - if (ts.isVariableStatement(node) && isExported(node)) { - for (const decl of node.declarationList.declarations) { - const name = decl.name.getText(sf); - if (name !== "loader" && name !== "action") continue; - if (name === "loader") ep.hasLoader = true; - if (name === "action") ep.hasAction = true; - let init = decl.initializer; - if (init && ts.isPropertyAccessExpression(init)) init = init.expression; - if (init && ts.isCallExpression(init)) { - const cn = calleeName(init.expression); - if (name === "loader") ep.loaderInitializerCallee = cn; - else ep.actionInitializerCallee = cn; + if (ts.isVariableStatement(statement) && isExported(statement)) { + for (const decl of statement.declarationList.declarations) { + if (!ts.isIdentifier(decl.name)) continue; + const name = decl.name.text; + if (name === "loader" || name === "action") { + record(name, analyzeInitializer(decl.initializer, locals, new Set())); } } + continue; } - if (ts.isFunctionDeclaration(node) && node.name && isExported(node)) { - if (node.name.text === "loader") ep.hasLoader = true; - if (node.name.text === "action") ep.hasAction = true; - if (node.body) ep.statementCount += node.body.statements.length; - } + if (ts.isExportDeclaration(statement) && statement.exportClause) { + // `export * from "./x"` has no clause and reaches nothing here; `export * as ns from "./x"` + // is a namespace clause, which cannot name a loader or action either. + if (!ts.isNamedExports(statement.exportClause)) continue; - if (ts.isTryStatement(node)) ep.hasTryCatch = true; - - if (ts.isCallExpression(node)) { - const cn = calleeName(node.expression); - if (cn) ep.calleeNames.push(cn); + for (const element of statement.exportClause.elements) { + const exportedName = element.name.text; + if (exportedName !== "loader" && exportedName !== "action") continue; + // A re-export (`export { loader } from "./x"`) has no local binding to resolve. + if (statement.moduleSpecifier) { + record(exportedName, NO_INITIALIZER); + continue; + } + const localName = element.propertyName?.text ?? exportedName; + record(exportedName, resolveLocal(localName, locals, new Set())); + } } + } - ts.forEachChild(node, visit); - }; + if (!target.hasLoader && !target.hasAction) return null; + + let statementCount = 0; + let hasTryCatch = false; + const calleeNames: string[] = []; - visit(sf); + for (const fn of target.functions) { + statementCount += countFunctionStatements(fn); - if (!ep.hasLoader && !ep.hasAction) return null; - if (ep.statementCount === 0) { - ep.statementCount = countArrowBodyStatements(sf); + if (!fn.body) continue; + const visit = (node: ts.Node) => { + if (ts.isTryStatement(node)) hasTryCatch = true; + if (ts.isCallExpression(node)) { + const cn = calleeName(node.expression); + if (cn) calleeNames.push(cn); + } + ts.forEachChild(node, visit); + }; + visit(fn.body); } - return ep; -} -function countArrowBodyStatements(sf: ts.SourceFile): number { - let count = 0; - const visit = (n: ts.Node) => { - if ((ts.isArrowFunction(n) || ts.isFunctionExpression(n)) && n.body && ts.isBlock(n.body)) { - count += n.body.statements.length; - } - ts.forEachChild(n, visit); + return { + fileName, + source, + hasLoader: target.hasLoader, + hasAction: target.hasAction, + loaderInitializerCallee: target.loaderInitializerCallee, + actionInitializerCallee: target.actionInitializerCallee, + importedNames, + calleeNames, + hasTryCatch, + statementCount, }; - visit(sf); - return count; +} + +const SOURCE_FILE = /\.tsx?$/; + +function isScannableFile(fileName: string): boolean { + return SOURCE_FILE.test(fileName) && !fileName.endsWith(".d.ts"); } export function scanDirectory(dir: string): { @@ -97,15 +381,28 @@ export function scanDirectory(dir: string): { } { const entryPoints: EntryPoint[] = []; const parseFailures: string[] = []; - const files = readdirSync(dir).filter((f) => /\.(ts|tsx)$/.test(f) && !f.endsWith(".test.ts")); - for (const fileName of files) { + const scan = (absolutePath: string, relativeName: string) => { try { - const ep = scanFile(fileName, readFileSync(join(dir, fileName), "utf8")); + const ep = scanFile(relativeName, readFileSync(absolutePath, "utf8")); if (ep) entryPoints.push(ep); } catch { - parseFailures.push(fileName); + parseFailures.push(relativeName); + } + }; + + for (const entry of readdirSync(dir, { withFileTypes: true })) { + if (entry.isDirectory()) { + // Flat-route directories hold the route module in `route.ts`/`route.tsx`. + for (const child of readdirSync(join(dir, entry.name), { withFileTypes: true })) { + if (!child.isFile() || (child.name !== "route.ts" && child.name !== "route.tsx")) continue; + scan(join(dir, entry.name, child.name), `${entry.name}/${child.name}`); + } + continue; } + if (!entry.isFile() || !isScannableFile(entry.name)) continue; + scan(join(dir, entry.name), entry.name); } + return { entryPoints, parseFailures }; } diff --git a/internal-packages/observability-map/src/types.ts b/internal-packages/observability-map/src/types.ts index 7a331707e21..fcf037305fe 100644 --- a/internal-packages/observability-map/src/types.ts +++ b/internal-packages/observability-map/src/types.ts @@ -14,8 +14,11 @@ export type EntryPoint = { /** Callee name when `loader`/`action` is assigned from a call, e.g. a route builder. */ loaderInitializerCallee: string | null; actionInitializerCallee: string | null; + /** Named and default imports, file-wide. */ importedNames: string[]; + /** Names of functions called inside the loader/action bodies. */ calleeNames: string[]; + /** Whether a `try` appears inside the loader/action bodies. */ hasTryCatch: boolean; /** Statement count across loader/action bodies, used by the triviality rule. */ statementCount: number; diff --git a/internal-packages/observability-map/test/scan.test.ts b/internal-packages/observability-map/test/scan.test.ts index 27f233311cf..758f2c35c71 100644 --- a/internal-packages/observability-map/test/scan.test.ts +++ b/internal-packages/observability-map/test/scan.test.ts @@ -1,4 +1,7 @@ -import { scanFile } from "../src/scan.js"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { scanDirectory, scanFile } from "../src/scan.js"; const LOADER = ` import { json } from "@remix-run/server-runtime"; @@ -27,3 +30,327 @@ describe("scanFile", () => { expect(ep!.loaderInitializerCallee).toBe("createLoaderApiRoute"); }); }); + +describe("scanFile: named export clauses", () => { + it("detects `export { loader }` and resolves the builder callee from the local declaration", () => { + const ep = scanFile( + "api.v1.query.ts", + ` + const { loader } = createLoaderApiRoute({ findResource: async () => 1 }, async () => { + const a = 1; + return json({ a }); + }); + export { loader }; + ` + ); + expect(ep).not.toBeNull(); + expect(ep!.hasLoader).toBe(true); + expect(ep!.loaderInitializerCallee).toBe("createLoaderApiRoute"); + expect(ep!.statementCount).toBe(2); + }); + + it("detects an aliased named export `export { h as loader }`", () => { + const ep = scanFile( + "api.v1.aliased.ts", + ` + async function h() { return json({}); } + export { h as loader }; + ` + ); + expect(ep).not.toBeNull(); + expect(ep!.hasLoader).toBe(true); + expect(ep!.statementCount).toBe(1); + }); + + it("reports both `export const loader` and a separate `export { action }`", () => { + const ep = scanFile( + "api.v1.both.ts", + ` + export const loader = createLoaderApiRoute({}, async () => json({})); + const { action } = createActionApiRoute({}, async ({ body }) => { + const x = body.x; + return json({ x }); + }); + export { action }; + ` + ); + expect(ep).not.toBeNull(); + expect(ep!.hasLoader).toBe(true); + expect(ep!.hasAction).toBe(true); + expect(ep!.actionInitializerCallee).toBe("createActionApiRoute"); + }); + + it("resolves an export assigned from a property of a local builder result", () => { + const ep = scanFile( + "api.v1.errors.$errorId.ignore.ts", + ` + const route = createActionApiRoute({ method: "POST" }, async ({ body }) => { + const a = 1; + const b = 2; + return json({ a, b }); + }); + export const action = route.action; + export const loader = route.loader; + ` + ); + expect(ep!.hasAction).toBe(true); + expect(ep!.actionInitializerCallee).toBe("createActionApiRoute"); + // Both exports share one handler, so its statements are counted once. + expect(ep!.statementCount).toBe(3); + }); + + it("counts a `handler` property body but not the surrounding builder config lambdas", () => { + const ep = scanFile( + "engine.v1.dev.presence.ts", + ` + export const loader = createSSELoader({ + timeout: 1000, + findResource: async (params) => { + const a = 1; + const b = 2; + const c = 3; + return lookup(a, b, c); + }, + handler: async ({ request }) => { + const auth = await authenticate(request); + return stream(auth); + }, + }); + ` + ); + expect(ep!.statementCount).toBe(2); + expect(ep!.calleeNames).not.toContain("lookup"); + }); + + it("detects an action-only route", () => { + const ep = scanFile( + "api.v1.action-only.ts", + `export async function action() { return json({}); }` + ); + expect(ep).not.toBeNull(); + expect(ep!.hasAction).toBe(true); + expect(ep!.hasLoader).toBe(false); + }); + + it("does not crash on a re-export or a star export", () => { + expect(() => scanFile("re-export.ts", `export { loader } from "./other";`)).not.toThrow(); + expect(() => scanFile("star.ts", `export * from "./other";`)).not.toThrow(); + const ep = scanFile("re-export.ts", `export { loader } from "./other";`); + expect(ep!.hasLoader).toBe(true); + expect(ep!.loaderInitializerCallee).toBeNull(); + }); +}); + +describe("scanFile: statement counting", () => { + it("counts only the loader's statements, not a fat exported component's", () => { + const ep = scanFile( + "route.tsx", + ` + export async function loader() { + return json({}); + } + export default function Page() { + const a = 1; + const b = 2; + const c = 3; + const d = 4; + return null; + } + export function ErrorBoundary() { + const e = 1; + return null; + } + ` + ); + expect(ep!.statementCount).toBe(1); + }); + + it("counts through a try/catch wrapper rather than reporting 1", () => { + const ep = scanFile( + "otel.v1.traces.ts", + ` + export async function action({ request }) { + try { + const body = await request.arrayBuffer(); + const result = await process(body); + return json(result); + } catch (e) { + logger.error(e); + return json({ error: true }, { status: 500 }); + } + } + ` + ); + expect(ep!.statementCount).toBeGreaterThan(4); + }); + + it("counts statements nested in if/for/while/switch blocks", () => { + const ep = scanFile( + "nested.ts", + ` + export async function loader() { + if (a) { + const x = 1; + doThing(x); + } + for (const i of list) { + use(i); + } + return json({}); + } + ` + ); + // if (1) + 2 nested + for (1) + 1 nested + return (1) + expect(ep!.statementCount).toBe(6); + }); +}); + +describe("scanFile: entry-point scoping", () => { + it("reports hasTryCatch false when the only try is in a non-entry-point export", () => { + const ep = scanFile( + "route.tsx", + ` + export async function loader() { + return json({}); + } + export default function Page() { + try { + render(); + } catch (e) { + report(e); + } + return null; + } + ` + ); + expect(ep!.hasLoader).toBe(true); + expect(ep!.hasTryCatch).toBe(false); + }); + + it("reports hasTryCatch true when the try is inside the loader", () => { + const ep = scanFile( + "route.tsx", + ` + export async function loader() { + try { + return json(await load()); + } catch (e) { + return json({}, { status: 500 }); + } + } + ` + ); + expect(ep!.hasTryCatch).toBe(true); + }); + + it("excludes callees invoked outside the loader/action body", () => { + const ep = scanFile( + "route.tsx", + ` + const schema = z.object({}); + export async function loader() { + const data = await fetchThings(); + return json(data); + } + export default function Page() { + useFancyHook(); + return null; + } + ` + ); + expect(ep!.calleeNames).toContain("fetchThings"); + expect(ep!.calleeNames).toContain("json"); + expect(ep!.calleeNames).not.toContain("useFancyHook"); + expect(ep!.calleeNames).not.toContain("object"); + }); +}); + +describe("scanFile: callee resolution", () => { + it("records the root callee of a chained builder call", () => { + const ep = scanFile( + "api.v1.cors.ts", + `export const loader = createLoaderApiRoute({}).withCors();` + ); + expect(ep!.loaderInitializerCallee).toBe("createLoaderApiRoute"); + }); + + it("leaves the callee null for a shape that cannot be named", () => { + const ep = scanFile("api.v1.anon.ts", `export const loader = async () => json({});`); + expect(ep!.hasLoader).toBe(true); + expect(ep!.loaderInitializerCallee).toBeNull(); + }); +}); + +describe("scanFile: parse failures", () => { + it("throws on a malformed source rather than returning a clean entry point", () => { + expect(() => + scanFile("broken.ts", `export async function loader() { const a = ; return json(`) + ).toThrow(); + }); + + it("does not throw on a well-formed tsx route", () => { + expect(() => + scanFile( + "route.tsx", + `export async function loader() { return json({}); } + export default function Page() { return
hi
; }` + ) + ).not.toThrow(); + }); +}); + +describe("scanDirectory", () => { + let dir: string; + + beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), "obs-map-scan-")); + }); + + afterEach(() => { + rmSync(dir, { recursive: true, force: true }); + }); + + it("recurses into route directories and keeps file names distinct", () => { + writeFileSync(join(dir, "a.ts"), `export async function loader() { return json({}); }`); + mkdirSync(join(dir, "nested.route")); + writeFileSync( + join(dir, "nested.route", "route.tsx"), + `export async function loader() { return json({}); }` + ); + mkdirSync(join(dir, "other.route")); + writeFileSync( + join(dir, "other.route", "route.tsx"), + `export async function action() { return json({}); }` + ); + + const { entryPoints, parseFailures } = scanDirectory(dir); + const names = entryPoints.map((ep) => ep.fileName).sort(); + + expect(parseFailures).toEqual([]); + expect(names).toEqual(["a.ts", "nested.route/route.tsx", "other.route/route.tsx"]); + }); + + it("records a malformed file as a parse failure instead of an entry point", () => { + writeFileSync( + join(dir, "broken.ts"), + `export async function loader() { const a = ; return json(` + ); + + const { entryPoints, parseFailures } = scanDirectory(dir); + + expect(entryPoints).toEqual([]); + expect(parseFailures).toEqual(["broken.ts"]); + }); + + it("scans a route file whose name ends in .test.ts but skips .d.ts", () => { + writeFileSync( + join(dir, "projects.v3.$projectRef.test.ts"), + `export async function loader() { return json({}); }` + ); + writeFileSync(join(dir, "types.d.ts"), `export declare const x: number;`); + + const { entryPoints } = scanDirectory(dir); + + expect(entryPoints.map((ep) => ep.fileName)).toEqual(["projects.v3.$projectRef.test.ts"]); + }); +}); From fa01e3e9adccaeaa760ae55610060adaf0115070 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Fri, 31 Jul 2026 13:22:26 +0100 Subject: [PATCH 003/106] feat(observability-map): remix route adapter --- .../observability-map/src/adapters/remix.ts | 44 ++++++++++++++ .../observability-map/test/remix.test.ts | 57 +++++++++++++++++++ 2 files changed, 101 insertions(+) create mode 100644 internal-packages/observability-map/src/adapters/remix.ts create mode 100644 internal-packages/observability-map/test/remix.test.ts diff --git a/internal-packages/observability-map/src/adapters/remix.ts b/internal-packages/observability-map/src/adapters/remix.ts new file mode 100644 index 00000000000..7815fa3e6a4 --- /dev/null +++ b/internal-packages/observability-map/src/adapters/remix.ts @@ -0,0 +1,44 @@ +export type Family = + | "api.v1" + | "api.other" + | "webhooks" + | "admin" + | "resources" + | "dashboard" + | "ingest" + | "other"; + +/** + * The name that carries routing meaning for a given `fileName`. + * + * Flat routes (`api.v1.runs.$runId.ts`) are already that name. Directory routes + * (`_app.orgs.$slug/route.tsx`) hold their module in a fixed `route.ts`/`route.tsx` file, so the + * directory segment before the slash is the meaningful name and `route.tsx` itself is not a path + * segment. + */ +function routeName(fileName: string): string { + const slashIndex = fileName.indexOf("/"); + return slashIndex === -1 ? fileName : fileName.slice(0, slashIndex); +} + +export function familyOf(fileName: string): Family { + const name = routeName(fileName); + // admin is checked first: admin.api.v1.* is an admin route, not an api.v1 one. + if (name.startsWith("admin.")) return "admin"; + if (name.startsWith("api.v1.")) return "api.v1"; + if (name.startsWith("api.")) return "api.other"; + if (name.startsWith("webhooks.")) return "webhooks"; + if (name.startsWith("resources.")) return "resources"; + if (name.startsWith("_app.")) return "dashboard"; + if (name.startsWith("otel.") || name.startsWith("engine.")) return "ingest"; + return "other"; +} + +export function routePathOf(fileName: string): string { + const withoutExt = routeName(fileName).replace(/\.(ts|tsx)$/, ""); + const segments = withoutExt + .split(".") + .filter((s) => s.length > 0) + .map((s) => (s.startsWith("$") ? `:${s.slice(1)}` : s)); + return `/${segments.join("/")}`; +} diff --git a/internal-packages/observability-map/test/remix.test.ts b/internal-packages/observability-map/test/remix.test.ts new file mode 100644 index 00000000000..4af418ef846 --- /dev/null +++ b/internal-packages/observability-map/test/remix.test.ts @@ -0,0 +1,57 @@ +import { familyOf, routePathOf } from "../src/adapters/remix.js"; + +describe("familyOf", () => { + it("classifies each family from the flat-route filename", () => { + expect(familyOf("api.v1.runs.$runId.ts")).toBe("api.v1"); + expect(familyOf("api.something.ts")).toBe("api.other"); + expect(familyOf("admin.api.v1.gc.ts")).toBe("admin"); + expect(familyOf("resources.queues.ts")).toBe("resources"); + expect(familyOf("_app.orgs.$slug.ts")).toBe("dashboard"); + expect(familyOf("otel.v1.logs.ts")).toBe("ingest"); + expect(familyOf("@.ts")).toBe("other"); + }); + + it("prefers admin over api.v1 for admin-prefixed api routes", () => { + expect(familyOf("admin.api.v1.environments.$id.ts")).toBe("admin"); + }); +}); + +describe("routePathOf", () => { + it("turns a flat-route filename into a path", () => { + expect(routePathOf("api.v1.runs.$runId.ts")).toBe("/api/v1/runs/:runId"); + }); + + it("strips a trailing method suffix", () => { + expect(routePathOf("api.v1.runs.ts")).toBe("/api/v1/runs"); + }); +}); + +// Directory routes: the scanner recurses into Remix directory routes, so `fileName` can be a +// relative path like `_app.orgs.$organizationSlug.projects.$projectParam/route.tsx` rather than a +// flat dot-separated name. The directory name (not `route.tsx`) carries the meaning. +describe("familyOf: directory routes", () => { + it("classifies a directory route by its directory name, not the flat rules", () => { + expect(familyOf("_app.orgs.$organizationSlug.projects.$projectParam/route.tsx")).toBe( + "dashboard" + ); + }); + + it("classifies each family the same whether the route is flat or a directory", () => { + expect(familyOf("api.v1.runs.$runId/route.tsx")).toBe("api.v1"); + expect(familyOf("admin.api.v1.environments.$id/route.tsx")).toBe("admin"); + expect(familyOf("resources.queues/route.tsx")).toBe("resources"); + expect(familyOf("storybook.callout/route.tsx")).toBe("other"); + }); +}); + +describe("routePathOf: directory routes", () => { + it("does not emit a literal 'route' segment for a directory route", () => { + const path = routePathOf("_app.orgs.$organizationSlug.projects.$projectParam/route.tsx"); + expect(path).not.toContain("route"); + expect(path).toBe("/_app/orgs/:organizationSlug/projects/:projectParam"); + }); + + it("produces the same path shape for a directory route as its flat equivalent", () => { + expect(routePathOf("api.v1.runs.$runId/route.tsx")).toBe(routePathOf("api.v1.runs.$runId.ts")); + }); +}); From 944f4cb3ac97aeddd33fdf13ad5f9223cc0b6009 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Fri, 31 Jul 2026 13:30:07 +0100 Subject: [PATCH 004/106] feat(observability-map): import and callee based sensitivity classification --- .../observability-map/src/sensitivity.ts | 55 ++++++++++ .../test/sensitivity.test.ts | 101 ++++++++++++++++++ 2 files changed, 156 insertions(+) create mode 100644 internal-packages/observability-map/src/sensitivity.ts create mode 100644 internal-packages/observability-map/test/sensitivity.test.ts diff --git a/internal-packages/observability-map/src/sensitivity.ts b/internal-packages/observability-map/src/sensitivity.ts new file mode 100644 index 00000000000..40f348afac7 --- /dev/null +++ b/internal-packages/observability-map/src/sensitivity.ts @@ -0,0 +1,55 @@ +import type { EntryPoint } from "./types.js"; +import { routePathOf } from "./adapters/remix.js"; + +const SENSITIVE_SYMBOLS = [ + "clearImpersonation", + "setImpersonation", + "requireAdminApiRequest", + "createPersonalAccessToken", + "regenerateApiKey", + "createJWT", + "signJWT", + "updateEnvVars", + "createAuthorizationCode", +]; + +// Whole path segments only, so "authorship" does not match "auth". +const SENSITIVE_SEGMENTS = [ + "auth", + "jwt", + "token", + "tokens", + "envvars", + "billing", + "payment", + "invoices", + "secrets", + "impersonate", + "authorization-code", + "regenerate-api-key", +]; + +export type Sensitivity = { sensitive: boolean; reasons: string[] }; + +export function classifySensitivity(ep: EntryPoint): Sensitivity { + const reasons: string[] = []; + + // importedNames is file-wide; calleeNames is scoped to the loader/action body. A sensitive + // symbol called only at module scope is caught here only if it is also imported. + const symbols = new Set([...ep.importedNames, ...ep.calleeNames]); + for (const s of SENSITIVE_SYMBOLS) { + if (symbols.has(s)) reasons.push(`calls ${s}`); + } + + // `routePathOf` turns both flat routes (`api.v1.envvars.ts`) and directory routes + // (`billing/route.tsx`) into real `/`-separated path segments, so this matches whole segments in + // either shape rather than splitting the raw fileName on ".". + const segments = routePathOf(ep.fileName) + .split("/") + .filter((s) => s.length > 0); + for (const seg of segments) { + if (SENSITIVE_SEGMENTS.includes(seg)) reasons.push(`path segment "${seg}"`); + } + + return { sensitive: reasons.length > 0, reasons }; +} diff --git a/internal-packages/observability-map/test/sensitivity.test.ts b/internal-packages/observability-map/test/sensitivity.test.ts new file mode 100644 index 00000000000..9a5c12e5f6a --- /dev/null +++ b/internal-packages/observability-map/test/sensitivity.test.ts @@ -0,0 +1,101 @@ +import { classifySensitivity } from "../src/sensitivity.js"; +import { scanFile } from "../src/scan.js"; + +const ep = (fileName: string, source: string) => scanFile(fileName, source)!; + +describe("classifySensitivity", () => { + it("flags a route whose filename says nothing but which calls a sensitive helper", () => { + const e = ep( + "@.ts", + `import { clearImpersonation } from "~/models/admin.server"; + export async function loader({ request }) { return clearImpersonation(request, "/admin"); }` + ); + const s = classifySensitivity(e); + expect(s.sensitive).toBe(true); + expect(s.reasons.some((r) => r.includes("clearImpersonation"))).toBe(true); + }); + + it("flags on the path when the filename is explicit", () => { + const e = ep("api.v1.projects.$ref.envvars.ts", `export async function loader() { return 1; }`); + expect(classifySensitivity(e).sensitive).toBe(true); + }); + + it("does not flag an ordinary read route", () => { + const e = ep( + "api.v1.timezones.ts", + `import { json } from "@remix-run/server-runtime"; + export async function loader() { return json({ timezones: [] }); }` + ); + expect(classifySensitivity(e).sensitive).toBe(false); + }); + + it("does not flag on a substring that merely contains a sensitive word", () => { + const e = ep( + "api.v1.authorship.ts", + `export async function loader() { return { author: "x" }; }` + ); + expect(classifySensitivity(e).sensitive).toBe(false); + }); +}); + +// Directory routes: `fileName` can be `dirName/route.tsx` rather than a flat dotted name. A naive +// `fileName.split(".")` treats "billing/route" as one non-matching segment because the slash never +// gets split, so it misses the directory name entirely. The classifier must derive real path +// segments (via the same route-path logic the remix adapter uses) so both shapes work alike. +describe("classifySensitivity: directory routes", () => { + it("flags a single-segment directory route by its directory name", () => { + const e = ep("billing/route.tsx", `export async function loader() { return 1; }`); + const s = classifySensitivity(e); + expect(s.sensitive).toBe(true); + expect(s.reasons.some((r) => r.includes("billing"))).toBe(true); + }); + + it("flags a multi-segment directory route via a segment among dynamic params", () => { + const e = ep( + "_app.orgs.$slug.billing/route.tsx", + `export async function loader() { return 1; }` + ); + expect(classifySensitivity(e).sensitive).toBe(true); + }); + + it("does not flag a directory route on a substring that merely contains a sensitive word", () => { + const e = ep("api.v1.authorship/route.tsx", `export async function loader() { return 1; }`); + expect(classifySensitivity(e).sensitive).toBe(false); + }); +}); + +// calleeNames is scoped to the loader/action body, unlike importedNames which is file-wide. A +// sensitive symbol called only at module scope is invisible to calleeNames, so it is only caught +// when it also shows up as an import. +describe("classifySensitivity: calleeNames is body-scoped, importedNames is file-wide", () => { + it("flags a sensitive symbol invoked only at module scope, via the import rather than the callee", () => { + const e = ep( + "api.v1.setup.ts", + `import { setImpersonation } from "~/models/admin.server"; + setImpersonation(globalThis, "seed"); + export async function loader() { return 1; }` + ); + const s = classifySensitivity(e); + expect(s.sensitive).toBe(true); + expect(s.reasons.some((r) => r.includes("setImpersonation"))).toBe(true); + }); + + it("flags a sensitive callee defined locally and invoked inside the loader, even without an import", () => { + const e = ep( + "api.v1.local-admin.ts", + `async function createJWT() { return "x"; } + export async function loader() { return createJWT(); }` + ); + expect(classifySensitivity(e).sensitive).toBe(true); + }); + + it("does not flag a sensitive-named call made only at module scope outside the loader/action body", () => { + const e = ep( + "api.v1.module-scope.ts", + `function createJWT() { return "x"; } + createJWT(); + export async function loader() { return 1; }` + ); + expect(classifySensitivity(e).sensitive).toBe(false); + }); +}); From 68e09883d19b6154156631a4ca6c90a83e0dbc12 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Fri, 31 Jul 2026 13:41:22 +0100 Subject: [PATCH 005/106] fix(observability-map): resolve one-hop helpers and narrow the scanner heuristics Follow-up to the adversarial review of the scanner fixes. - follow a call from a loader/action body to a same-file helper, one hop with a cycle guard, so a body that delegates reports the helper's statements, try/catch and callees rather than just the delegation. ph.$.ts goes from 2 statements and hasTryCatch false to 30 and true; 66 entry points gain statements, 6 gain hasTryCatch - only treat a ParseFailureError as a parse failure in scanDirectory and rethrow everything else, so an unreadable file is no longer reported as malformed source, and keep the diagnostic alongside the file name - match a builder handler only at the top of the config object or under methods..handler, not by name at any depth - read handler arguments from the root call of a builder chain only, so a callback given to a later decorator is not the route body - pin the loosened assertions and add negatives for the new resolution, the handler shapes and the chained builder --- .../observability-map/src/scan.ts | 148 +++++++++--- .../observability-map/src/types.ts | 10 +- .../observability-map/test/scan.test.ts | 225 +++++++++++++++++- 3 files changed, 344 insertions(+), 39 deletions(-) diff --git a/internal-packages/observability-map/src/scan.ts b/internal-packages/observability-map/src/scan.ts index cb506ce2cd3..2ebe03b67f5 100644 --- a/internal-packages/observability-map/src/scan.ts +++ b/internal-packages/observability-map/src/scan.ts @@ -68,41 +68,75 @@ function calleeName(expr: ts.Expression): string | null { return null; } +const HTTP_METHODS = new Set(["GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS"]); + +function propertyName(property: ts.ObjectLiteralElementLike): string | null { + if (!property.name) return null; + if (ts.isIdentifier(property.name) || ts.isStringLiteral(property.name)) + return property.name.text; + return null; +} + +/** `methods: { POST: { handler } }`, the per-method shape of `createMultiMethodApiRoute`. */ +function collectMethodHandlers(methods: ts.ObjectLiteralExpression, out: EntryFunction[]): void { + for (const method of methods.properties) { + const name = propertyName(method); + if (!name || !HTTP_METHODS.has(name)) continue; + if (!ts.isPropertyAssignment(method)) continue; + const config = unwrap(method.initializer); + if (!ts.isObjectLiteralExpression(config)) continue; + for (const property of config.properties) { + if (!ts.isPropertyAssignment(property) || propertyName(property) !== "handler") continue; + const value = unwrap(property.initializer); + if (isEntryFunction(value)) out.push(value); + } + } +} + /** - * Functions on a `handler` property of an object argument, e.g. `createSSELoader({ handler })` and - * the per-method `{ POST: { handler } }` map. Only `handler` counts: the surrounding config also - * holds lambdas (`findResource`, `authorization.resource`) that are not the entry-point body. + * The handler on an object argument, in the two shapes the route builders use: `handler` at the + * top level of the config (`createSSELoader({ handler })`) and `methods.POST.handler`. Matching by + * name at any depth would pick up an unrelated config callback that happens to be called + * `handler`, as well as the sibling lambdas (`findResource`, `authorization.resource`) that are + * not the entry-point body. */ function collectNamedHandlers(object: ts.ObjectLiteralExpression, out: EntryFunction[]): void { for (const property of object.properties) { - if (!ts.isPropertyAssignment(property) || !property.name) continue; + if (!ts.isPropertyAssignment(property)) continue; + const name = propertyName(property); const value = unwrap(property.initializer); - if (ts.isObjectLiteralExpression(value)) { - collectNamedHandlers(value, out); - continue; - } - const name = - ts.isIdentifier(property.name) || ts.isStringLiteral(property.name) - ? property.name.text - : null; if (name === "handler" && isEntryFunction(value)) out.push(value); + if (name === "methods" && ts.isObjectLiteralExpression(value)) { + collectMethodHandlers(value, out); + } } } -/** Every function passed as an argument anywhere in a call chain, e.g. the builder's handler. */ -function collectHandlerFunctions(expr: ts.Expression, out: EntryFunction[]): void { - const target = unwrap(expr); - if (ts.isCallExpression(target)) { - for (const arg of target.arguments) { - const unwrapped = unwrap(arg); - if (isEntryFunction(unwrapped)) out.push(unwrapped); - else if (ts.isObjectLiteralExpression(unwrapped)) collectNamedHandlers(unwrapped, out); +/** The innermost call of a chain: the `createLoaderApiRoute(...)` in `createLoaderApiRoute(...).withCors(...)`. */ +function rootCall(call: ts.CallExpression): ts.CallExpression { + let current = call; + for (;;) { + let next = unwrap(current.expression); + while (ts.isPropertyAccessExpression(next) || ts.isElementAccessExpression(next)) { + next = unwrap(next.expression); + } + if (ts.isCallExpression(next)) { + current = next; + continue; } - collectHandlerFunctions(target.expression, out); - return; + return current; } - if (ts.isPropertyAccessExpression(target) || ts.isElementAccessExpression(target)) { - collectHandlerFunctions(target.expression, out); +} + +/** + * The handler functions passed to a builder call. Only the root call of a chain is read: a + * callback given to a decorator further along the chain (`.withCors(cb)`) is not the route body. + */ +function collectHandlerFunctions(call: ts.CallExpression, out: EntryFunction[]): void { + for (const arg of rootCall(call).arguments) { + const unwrapped = unwrap(arg); + if (isEntryFunction(unwrapped)) out.push(unwrapped); + else if (ts.isObjectLiteralExpression(unwrapped)) collectNamedHandlers(unwrapped, out); } } @@ -248,6 +282,30 @@ function collectLocalDeclarations(sf: ts.SourceFile): LocalDeclarations { return locals; } +/** + * Top-level functions by name, for resolving a body that delegates its work to a same-file helper + * (`export async function loader({ request }) { return proxyToPostHog(request); }`). + */ +function collectLocalFunctions(sf: ts.SourceFile): Map { + const functions = new Map(); + + for (const statement of sf.statements) { + if (ts.isFunctionDeclaration(statement) && statement.name && statement.body) { + functions.set(statement.name.text, statement); + continue; + } + if (!ts.isVariableStatement(statement)) continue; + + for (const decl of statement.declarationList.declarations) { + if (!decl.initializer || !ts.isIdentifier(decl.name)) continue; + const value = unwrap(decl.initializer); + if (isEntryFunction(value)) functions.set(decl.name.text, value); + } + } + + return functions; +} + export function scanFile(fileName: string, source: string): EntryPoint | null { const sf = ts.createSourceFile(fileName, source, ts.ScriptTarget.Latest, true); @@ -340,20 +398,41 @@ export function scanFile(fileName: string, source: string): EntryPoint | null { let hasTryCatch = false; const calleeNames: string[] = []; - for (const fn of target.functions) { + const localFunctions = collectLocalFunctions(sf); + // A body that delegates to a same-file helper does the work in that helper, so the helper's + // statements, try/catch and callees belong to the entry point. One hop only: a helper's own + // helpers are not followed, and the visited set stops a cycle and any double counting. + const visited = new Set(target.functions); + const helpers: EntryFunction[] = []; + + const walkBody = (fn: EntryFunction, followHelpers: boolean) => { statementCount += countFunctionStatements(fn); - if (!fn.body) continue; + if (!fn.body) return; const visit = (node: ts.Node) => { if (ts.isTryStatement(node)) hasTryCatch = true; if (ts.isCallExpression(node)) { const cn = calleeName(node.expression); if (cn) calleeNames.push(cn); + + if (followHelpers) { + const callee = unwrap(node.expression); + if (ts.isIdentifier(callee)) { + const helper = localFunctions.get(callee.text); + if (helper && !visited.has(helper)) { + visited.add(helper); + helpers.push(helper); + } + } + } } ts.forEachChild(node, visit); }; visit(fn.body); - } + }; + + for (const fn of target.functions) walkBody(fn, true); + for (const helper of helpers) walkBody(helper, false); return { fileName, @@ -383,12 +462,19 @@ export function scanDirectory(dir: string): { const parseFailures: string[] = []; const scan = (absolutePath: string, relativeName: string) => { + let ep: EntryPoint | null; try { - const ep = scanFile(relativeName, readFileSync(absolutePath, "utf8")); - if (ep) entryPoints.push(ep); - } catch { - parseFailures.push(relativeName); + ep = scanFile(relativeName, readFileSync(absolutePath, "utf8")); + } catch (error) { + // Only a genuinely malformed source is a parse failure. An unreadable file or a bug in the + // scanner must not be laundered into the same bucket, or a non-zero count means nothing. + if (error instanceof ParseFailureError) { + parseFailures.push(`${relativeName}: ${error.diagnostic}`); + return; + } + throw error; } + if (ep) entryPoints.push(ep); }; for (const entry of readdirSync(dir, { withFileTypes: true })) { diff --git a/internal-packages/observability-map/src/types.ts b/internal-packages/observability-map/src/types.ts index fcf037305fe..1f23f0e059f 100644 --- a/internal-packages/observability-map/src/types.ts +++ b/internal-packages/observability-map/src/types.ts @@ -16,10 +16,14 @@ export type EntryPoint = { actionInitializerCallee: string | null; /** Named and default imports, file-wide. */ importedNames: string[]; - /** Names of functions called inside the loader/action bodies. */ + /** Names of functions called inside the loader/action bodies, or in a same-file helper they call. */ calleeNames: string[]; - /** Whether a `try` appears inside the loader/action bodies. */ + /** Whether a `try` appears in the loader/action bodies, or in a same-file helper they call. */ hasTryCatch: boolean; - /** Statement count across loader/action bodies, used by the triviality rule. */ + /** + * Statement count across loader/action bodies, used by the triviality rule. A body that + * delegates to a same-file helper counts that helper's statements too, one hop only: work in a + * helper's own helpers, or in an imported module, is not counted. + */ statementCount: number; }; diff --git a/internal-packages/observability-map/test/scan.test.ts b/internal-packages/observability-map/test/scan.test.ts index 758f2c35c71..c727577a78c 100644 --- a/internal-packages/observability-map/test/scan.test.ts +++ b/internal-packages/observability-map/test/scan.test.ts @@ -1,7 +1,7 @@ -import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { chmodSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { scanDirectory, scanFile } from "../src/scan.js"; +import { ParseFailureError, scanDirectory, scanFile } from "../src/scan.js"; const LOADER = ` import { json } from "@remix-run/server-runtime"; @@ -122,6 +122,69 @@ describe("scanFile: named export clauses", () => { expect(ep!.calleeNames).not.toContain("lookup"); }); + it("counts the per-method `methods.POST.handler` bodies", () => { + const ep = scanFile( + "api.v1.prompts.$slug.override.ts", + ` + const { action, loader } = createMultiMethodApiRoute({ + params: ParamsSchema, + methods: { + POST: { + body: CreateBody, + handler: async ({ body }) => { + const created = await create(body); + return json(created); + }, + }, + DELETE: { + handler: async ({ params }) => { + return json({ ok: true }); + }, + }, + }, + }); + export { action, loader }; + ` + ); + expect(ep!.statementCount).toBe(3); + expect(ep!.calleeNames).toContain("create"); + }); + + it("ignores a nested config callback that happens to be named `handler`", () => { + const ep = scanFile( + "api.v1.named-collision.ts", + ` + export const loader = build({ + onError: { + handler: async () => { + const a = 1; + const b = 2; + const c = 3; + return null; + }, + }, + handler: async () => json({}), + }); + ` + ); + expect(ep!.statementCount).toBe(1); + }); + + it("ignores a callback passed to a decorator further along the builder chain", () => { + const ep = scanFile( + "api.v1.chained.ts", + ` + export const loader = createLoaderApiRoute({}, async () => json({})).withCors(async () => { + const a = 1; + const b = 2; + return a + b; + }); + ` + ); + expect(ep!.loaderInitializerCallee).toBe("createLoaderApiRoute"); + expect(ep!.statementCount).toBe(1); + }); + it("detects an action-only route", () => { const ep = scanFile( "api.v1.action-only.ts", @@ -181,7 +244,125 @@ describe("scanFile: statement counting", () => { } ` ); - expect(ep!.statementCount).toBeGreaterThan(4); + // try (1) + 3 in the try block + 2 in the catch block + expect(ep!.statementCount).toBe(6); + }); + + it("counts a same-file helper the body delegates to", () => { + const ep = scanFile( + "ph.$.ts", + ` + async function proxyToPostHog(request) { + const url = new URL(request.url); + try { + const upstream = await fetch(url); + return new Response(upstream.body); + } catch (e) { + logger.error(e); + return new Response(null, { status: 502 }); + } + } + export async function loader({ request }) { + return proxyToPostHog(request); + } + export async function action({ request }) { + return proxyToPostHog(request); + } + ` + ); + // loader (1) + action (1) + helper: const url, try (1) + 2 + 2 + expect(ep!.statementCount).toBe(8); + expect(ep!.hasTryCatch).toBe(true); + expect(ep!.calleeNames).toContain("proxyToPostHog"); + expect(ep!.calleeNames).toContain("fetch"); + }); + + it("counts a shared helper once when both the loader and the action delegate to it", () => { + const ep = scanFile( + "shared.ts", + ` + function work() { + const a = 1; + const b = 2; + return a + b; + } + export async function loader() { return work(); } + export async function action() { return work(); } + ` + ); + // loader (1) + action (1) + helper (3), the helper counted once + expect(ep!.statementCount).toBe(5); + }); + + it("follows a delegating helper one hop only", () => { + const ep = scanFile( + "two-hop.ts", + ` + function deep() { + const a = 1; + const b = 2; + const c = 3; + return a + b + c; + } + function shallow() { + return deep(); + } + export async function loader() { return shallow(); } + ` + ); + // loader (1) + shallow (1). `deep` is a second hop and is not counted. + expect(ep!.statementCount).toBe(2); + }); + + it("terminates on a recursive helper", () => { + const ep = scanFile( + "recursive.ts", + ` + function recurse(n) { + if (n <= 0) return 0; + return recurse(n - 1); + } + export async function loader() { return recurse(3); } + ` + ); + // loader (1) + recurse: if (1) + return (1) + return (1) + expect(ep!.statementCount).toBe(4); + }); + + it("does not count an imported helper it cannot resolve", () => { + const ep = scanFile( + "imported.ts", + ` + import { proxy } from "./proxy.server"; + export async function loader({ request }) { + return proxy(request); + } + ` + ); + expect(ep!.statementCount).toBe(1); + expect(ep!.hasTryCatch).toBe(false); + }); + + it("does not count a same-file function the body never calls", () => { + const ep = scanFile( + "unused-helper.ts", + ` + function unrelated() { + try { + const a = 1; + const b = 2; + return a + b; + } catch (e) { + return null; + } + } + export async function loader() { + return json({}); + } + ` + ); + expect(ep!.statementCount).toBe(1); + expect(ep!.hasTryCatch).toBe(false); }); it("counts statements nested in if/for/while/switch blocks", () => { @@ -285,7 +466,7 @@ describe("scanFile: parse failures", () => { it("throws on a malformed source rather than returning a clean entry point", () => { expect(() => scanFile("broken.ts", `export async function loader() { const a = ; return json(`) - ).toThrow(); + ).toThrow(ParseFailureError); }); it("does not throw on a well-formed tsx route", () => { @@ -339,7 +520,41 @@ describe("scanDirectory", () => { const { entryPoints, parseFailures } = scanDirectory(dir); expect(entryPoints).toEqual([]); - expect(parseFailures).toEqual(["broken.ts"]); + expect(parseFailures).toHaveLength(1); + // The file name, then the diagnostic that made it a failure. + expect(parseFailures[0]).toMatch(/^broken\.ts: \S/); + }); + + // root ignores the mode bits, so the unreadable file would read fine. + it.skipIf(process.getuid?.() === 0)( + "rethrows an error that is not a parse failure instead of counting it as one", + () => { + const unreadable = join(dir, "unreadable.ts"); + writeFileSync(unreadable, `export async function loader() { return json({}); }`); + chmodSync(unreadable, 0o000); + + try { + expect(() => scanDirectory(dir)).toThrow(/EACCES|EPERM/); + } finally { + chmodSync(unreadable, 0o600); + } + } + ); + + it("skips a non-route file inside a route directory", () => { + mkdirSync(join(dir, "nested.route")); + writeFileSync( + join(dir, "nested.route", "route.tsx"), + `export async function loader() { return json({}); }` + ); + writeFileSync( + join(dir, "nested.route", "loaders.server.ts"), + `export async function loader() { return json({}); }` + ); + + const { entryPoints } = scanDirectory(dir); + + expect(entryPoints.map((ep) => ep.fileName)).toEqual(["nested.route/route.tsx"]); }); it("scans a route file whose name ends in .test.ts but skips .d.ts", () => { From 6ab9118c2854e4155419b26a6fc8974cf474f4b0 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Fri, 31 Jul 2026 13:52:20 +0100 Subject: [PATCH 006/106] feat(observability-map): triviality rule and suppression comments --- .../observability-map/src/suppression.ts | 13 ++ .../observability-map/src/triviality.ts | 52 ++++++ .../test/suppression.test.ts | 45 +++++ .../observability-map/test/triviality.test.ts | 167 ++++++++++++++++++ 4 files changed, 277 insertions(+) create mode 100644 internal-packages/observability-map/src/suppression.ts create mode 100644 internal-packages/observability-map/src/triviality.ts create mode 100644 internal-packages/observability-map/test/suppression.test.ts create mode 100644 internal-packages/observability-map/test/triviality.test.ts diff --git a/internal-packages/observability-map/src/suppression.ts b/internal-packages/observability-map/src/suppression.ts new file mode 100644 index 00000000000..bd807816254 --- /dev/null +++ b/internal-packages/observability-map/src/suppression.ts @@ -0,0 +1,13 @@ +// The reason runs to the end of the line: `.` does not match a newline, so a suppression on one +// line cannot pick up a reason from the next one. +const PATTERN = /obs-map-disable-next-line\s+([a-z-]+)\s+--\s+(.+)/g; + +/** Check id to reason. A suppression without a reason is ignored. */ +export function suppressedChecks(source: string): Map { + const out = new Map(); + for (const match of source.matchAll(PATTERN)) { + const [, id, reason] = match; + if (id && reason && reason.trim().length > 0) out.set(id, reason.trim()); + } + return out; +} diff --git a/internal-packages/observability-map/src/triviality.ts b/internal-packages/observability-map/src/triviality.ts new file mode 100644 index 00000000000..e36b80dc9b0 --- /dev/null +++ b/internal-packages/observability-map/src/triviality.ts @@ -0,0 +1,52 @@ +import type { EntryPoint } from "./types.js"; + +/** + * Substrings that say the route touches a service, a datastore or the network. Matched against the + * callee names and the whole file, so an import of `prisma` disqualifies the file even when the + * query itself sits somewhere the scanner does not walk. + */ +const SIDE_EFFECT_HINTS = ["prisma", "logger", "fetch", "$transaction", "redis", "engine"]; + +/** + * Calls a genuinely trivial body makes: parse the params, build a path, hand back a response. Every + * shape found in the real tree stays at or below three, so anything busier is doing work. Allowing + * a fourth admits `_app.orgs.$organizationSlug.settings/route.tsx`, which awaits two service calls. + */ +const MAX_CALLS = 3; + +/** + * Parse the params, build a path, redirect. Or an environment guard and two returns. Both real + * shapes need three. Allowing a fourth admits the routes that authenticate and then hand off to a + * presenter (`...tasks.stream/route.tsx`), which have real work behind them and belong in the + * report; allowing a fifth admits an admin route that calls a service and hand-rolls its own error + * responses. + */ +const MAX_STATEMENTS = 3; + +/** + * Nothing to instrument: a body of a statement or two that only redirects, returns a fixed + * response, or hands off in a single call. Checks report not-applicable for these rather than + * failing, which is what stops `@.ts` being a finding. + * + * Deliberately reluctant. A route wrongly called trivial is exempted and never shows up in the + * report again, so every signal that the body might be doing real work rules triviality out: + * + * - `statementCount` does not descend into inline callbacks, so a two-statement body can still hold + * a pile of work. `calleeNames` does descend, so the call count catches what the statement count + * misses. + * - An initializer callee means the route is wrapped in a builder, and the config passed to that + * builder (`findResource`, `authorization`) is work the scanner never walks. The visible body is + * not the whole route, so we cannot claim it is trivial. + * - A try/catch is exactly what the error-classification check reads, so a body with one has an + * error path worth reporting on however short it is. + */ +export function isTrivial(ep: EntryPoint): boolean { + if (ep.statementCount > MAX_STATEMENTS) return false; + if (ep.calleeNames.length > MAX_CALLS) return false; + if (ep.hasTryCatch) return false; + if (ep.loaderInitializerCallee !== null || ep.actionInitializerCallee !== null) return false; + + const callees = ep.calleeNames.join(" ").toLowerCase(); + const source = ep.source.toLowerCase(); + return !SIDE_EFFECT_HINTS.some((h) => callees.includes(h) || source.includes(h)); +} diff --git a/internal-packages/observability-map/test/suppression.test.ts b/internal-packages/observability-map/test/suppression.test.ts new file mode 100644 index 00000000000..8e8e88d5fa5 --- /dev/null +++ b/internal-packages/observability-map/test/suppression.test.ts @@ -0,0 +1,45 @@ +import { suppressedChecks } from "../src/suppression.js"; + +describe("suppressedChecks", () => { + it("reads a suppression with its reason", () => { + const m = suppressedChecks( + `// obs-map-disable-next-line error-classification -- liveness probe, deliberately silent + export async function loader() { return { ok: true }; }` + ); + expect(m.get("error-classification")).toBe("liveness probe, deliberately silent"); + }); + + it("ignores a suppression with no reason", () => { + const m = suppressedChecks(`// obs-map-disable-next-line error-classification`); + expect(m.size).toBe(0); + }); + + it("ignores a suppression whose reason is only whitespace", () => { + const m = suppressedChecks(`// obs-map-disable-next-line error-classification -- `); + expect(m.size).toBe(0); + }); + + it("reads several suppressions in one file", () => { + const m = suppressedChecks( + `// obs-map-disable-next-line error-classification -- liveness probe + // obs-map-disable-next-line request-context -- no identifiers exist here + export async function loader() { return { ok: true }; }` + ); + expect(m.size).toBe(2); + expect(m.get("error-classification")).toBe("liveness probe"); + expect(m.get("request-context")).toBe("no identifiers exist here"); + }); + + it("returns nothing for a file with no suppressions", () => { + expect(suppressedChecks(`export async function loader() { return 1; }`).size).toBe(0); + }); + + it("does not carry a reason across lines", () => { + const m = suppressedChecks( + `// obs-map-disable-next-line error-classification + // some other comment -- with a dash + export async function loader() { return 1; }` + ); + expect(m.size).toBe(0); + }); +}); diff --git a/internal-packages/observability-map/test/triviality.test.ts b/internal-packages/observability-map/test/triviality.test.ts new file mode 100644 index 00000000000..81c63b34eb7 --- /dev/null +++ b/internal-packages/observability-map/test/triviality.test.ts @@ -0,0 +1,167 @@ +import { isTrivial } from "../src/triviality.js"; +import { scanFile } from "../src/scan.js"; + +const ep = (fileName: string, source: string) => scanFile(fileName, source)!; + +describe("isTrivial", () => { + it("treats a redirect-only route as trivial", () => { + const e = ep( + "@.ts", + `import { redirect } from "@remix-run/server-runtime"; + export async function loader() { return redirect("/admin"); }` + ); + expect(isTrivial(e)).toBe(true); + }); + + it("does not treat a route that queries the database as trivial", () => { + const e = ep( + "api.v1.things.ts", + `import { prisma } from "~/db.server"; + export async function loader() { + const rows = await prisma.thing.findMany(); + return rows; + }` + ); + expect(isTrivial(e)).toBe(false); + }); + + // The motivating case from the design: four lines, one delegating call, nothing to instrument. + it("treats the impersonation-clearing route as trivial", () => { + const e = ep( + "@.ts", + `import { clearImpersonation } from "~/models/admin.server"; + export async function loader({ request }) { return clearImpersonation(request, "/admin"); }` + ); + expect(isTrivial(e)).toBe(true); + }); + + it("treats a static-response route as trivial", () => { + const e = ep( + "internal.webhooks.slack.interactivity.ts", + `export function action() { return new Response(null, { status: 200 }); }` + ); + expect(isTrivial(e)).toBe(true); + }); + + it("treats a guard and two fixed responses as trivial", () => { + const e = ep( + "api.v1.mock.ts", + `export async function action() { + if (process.env.NODE_ENV === "production") { + return new Response("Not found", { status: 404 }); + } + return new Response(JSON.stringify({ id: "123" }), { status: 200 }); + }` + ); + expect(isTrivial(e)).toBe(true); + }); + + it("treats a params-parse and redirect as trivial", () => { + const e = ep( + "orgs.$organizationSlug.billing.ts", + `import { redirect } from "@remix-run/server-runtime"; + import { OrganizationParamsSchema, v3BillingPath } from "~/utils/pathBuilder"; + export const loader = async ({ params }) => { + const { organizationSlug } = OrganizationParamsSchema.parse(params); + return redirect(v3BillingPath({ slug: organizationSlug })); + };` + ); + expect(isTrivial(e)).toBe(true); + }); +}); + +// statementCount deliberately does not descend into inline callbacks, so a two-statement body can +// still hold a pile of work. calleeNames does descend, which is what catches these. +describe("isTrivial: work hidden from the statement count", () => { + it("does not treat a short body holding a busy callback as trivial", () => { + const e = ep( + "api.v1.remote-build-provider-status.ts", + `export async function loader() { + const result = await fromPromise( + (async () => { + const response = await callProvider(); + const parsed = ProviderStatus.safeParse(await response.json()); + if (!parsed.success) return err("bad-payload"); + return ok(parsed.data); + })() + ); + return result.match(toJson, toError); + }` + ); + expect(isTrivial(e)).toBe(false); + }); + + it("does not treat a builder-wrapped route with a one-line handler as trivial", () => { + const e = ep( + "api.v1.deployments.current.ts", + `import { json } from "@remix-run/server-runtime"; + import { createLoaderApiRoute } from "~/services/routeBuilders/apiBuilder.server"; + export const loader = createLoaderApiRoute( + { findResource: async (_params, auth) => lookup(auth) }, + async ({ resource }) => { return json(resource); } + );` + ); + expect(isTrivial(e)).toBe(false); + }); + + it("does not treat a body that delegates to a same-file helper as trivial", () => { + const e = ep( + "api.v1.proxy.ts", + `export async function loader({ request }) { return proxy(request); } + async function proxy(request) { + const url = buildUrl(request); + const response = await send(url); + const body = await response.text(); + return new Response(body); + }` + ); + expect(isTrivial(e)).toBe(false); + }); +}); + +// Calibrated against apps/webapp/app/routes: three statements is the widest window that holds only +// redirects, fixed responses and single hand-offs. The fourth statement is where routes start +// authenticating and then calling a presenter, which is work worth reporting on. +describe("isTrivial: the statement boundary", () => { + it("treats a three-statement redirect as trivial", () => { + const e = ep( + "schedules._index/route.tsx", + `import { redirect } from "@remix-run/server-runtime"; + import { EnvironmentParamSchema, v3EnvironmentPath } from "~/utils/pathBuilder"; + export async function loader({ params }) { + const { organizationSlug, projectParam, envParam } = EnvironmentParamSchema.parse(params); + const tasksPath = v3EnvironmentPath({ slug: organizationSlug }, { slug: projectParam }, { slug: envParam }); + return redirect(\`\${tasksPath}?types=SCHEDULED\`); + }` + ); + expect(isTrivial(e)).toBe(true); + }); + + it("does not treat an authenticated hand-off to a presenter as trivial", () => { + const e = ep( + "tasks.stream/route.tsx", + `import { TasksStreamPresenter } from "~/presenters/v3/TasksStreamPresenter.server"; + import { requireUserId } from "~/services/session.server"; + import { EnvironmentParamSchema } from "~/utils/pathBuilder"; + export async function loader({ request, params }) { + const userId = await requireUserId(request); + const { organizationSlug, projectParam, envParam } = EnvironmentParamSchema.parse(params); + const presenter = new TasksStreamPresenter(); + return presenter.call({ request, projectParam, envParam, organizationSlug, userId }); + }` + ); + expect(isTrivial(e)).toBe(false); + }); +}); + +describe("isTrivial: an error path is something to instrument", () => { + it("does not treat a short body with a try/catch as trivial", () => { + const e = ep( + "api.v1.ping.ts", + `export async function loader() { + try { return await ping(); } catch { return null; } + }` + ); + expect(isTrivial(e)).toBe(false); + }); +}); From 75eaaa8ffc9daaef161ab17c2d1b03bd843d3cbb Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Fri, 31 Jul 2026 14:09:23 +0100 Subject: [PATCH 007/106] feat(observability-map): the four coverage checks Adds error-classification, auth-boundary, request-context and audit-trail, plus the CHECKS registry. Every check is a pure function of an EntryPoint and reads body-scoped evidence only. Two rules differ from the design. error-classification uses hasTryCatch as its gate rather than a regex over ep.source, which is the whole file including the React component; EntryPoint carries no evidence about what a catch does with the error, so the check reports the hand-rolled catch and says it has not been read. request-context looks for an identity resolved in the body rather than grepping the file for identifier names, for the same reason. Both deviations, and the calibration run over the 427 webapp entry points, are written up in the task 5 report. --- .../src/checks/auditTrail.ts | 25 ++ .../src/checks/authBoundary.ts | 42 ++ .../src/checks/errorClassification.ts | 65 ++++ .../observability-map/src/checks/index.ts | 12 + .../src/checks/requestContext.ts | 53 +++ .../observability-map/test/checks.test.ts | 360 ++++++++++++++++++ 6 files changed, 557 insertions(+) create mode 100644 internal-packages/observability-map/src/checks/auditTrail.ts create mode 100644 internal-packages/observability-map/src/checks/authBoundary.ts create mode 100644 internal-packages/observability-map/src/checks/errorClassification.ts create mode 100644 internal-packages/observability-map/src/checks/index.ts create mode 100644 internal-packages/observability-map/src/checks/requestContext.ts create mode 100644 internal-packages/observability-map/test/checks.test.ts diff --git a/internal-packages/observability-map/src/checks/auditTrail.ts b/internal-packages/observability-map/src/checks/auditTrail.ts new file mode 100644 index 00000000000..ee3dcdf45a7 --- /dev/null +++ b/internal-packages/observability-map/src/checks/auditTrail.ts @@ -0,0 +1,25 @@ +import type { CheckResult, EntryPoint } from "../types.js"; +import { classifySensitivity } from "../sensitivity.js"; + +const ID = "audit-trail"; + +const AUDIT_SYMBOLS = ["auditLog", "recordAudit", "writeAuditEvent"]; + +/** + * Whether a sensitive mutation leaves a record of who did it. Nothing in the webapp writes one + * today, so every applicable entry point fails: the check states the gap rather than measuring + * variation between routes, which is why the score leaves it out. + */ +export const auditTrail = { + id: ID, + run(ep: EntryPoint): CheckResult { + // Mutations only: a sensitive read does not need an actor record. + if (!classifySensitivity(ep).sensitive || !ep.hasAction) { + return { id: ID, status: "not-applicable", detail: "not a sensitive mutation" }; + } + const symbols = new Set([...ep.importedNames, ...ep.calleeNames]); + return AUDIT_SYMBOLS.some((s) => symbols.has(s)) + ? { id: ID, status: "pass", detail: "records an audit event" } + : { id: ID, status: "fail", detail: "sensitive mutation with no audit record" }; + }, +}; diff --git a/internal-packages/observability-map/src/checks/authBoundary.ts b/internal-packages/observability-map/src/checks/authBoundary.ts new file mode 100644 index 00000000000..11030d1d2d8 --- /dev/null +++ b/internal-packages/observability-map/src/checks/authBoundary.ts @@ -0,0 +1,42 @@ +import type { CheckResult, EntryPoint } from "../types.js"; +import { classifySensitivity } from "../sensitivity.js"; +import { usesBuilder } from "./errorClassification.js"; + +const ID = "auth-boundary"; + +/** + * The two shapes the webapp's guards take: `requireUserId`, `requireAdminApiRequest`, + * `authenticateApiRequest`, `authenticateProjectApiKey`. Matched against `calleeNames`, which is + * scoped to the loader/action bodies and follows one hop into a same-file helper, so a guard the + * route only imports and never calls does not count. + */ +const GUARD = /^(require|authenticate)/; + +/** + * Whether a route that handles credentials, tokens or money checks who is asking. + * + * The design matched `importedNames` as well as `calleeNames`. Across the 67 sensitive entry points + * in the real tree that widening changes nothing: every route with a `require*` import calls it + * from the body too. So the file-wide half only ever stood to hand out a pass for a dead import, + * and it is gone. + */ +export const authBoundary = { + id: ID, + run(ep: EntryPoint): CheckResult { + const sensitivity = classifySensitivity(ep); + if (!sensitivity.sensitive) { + return { id: ID, status: "not-applicable", detail: "not sensitive" }; + } + if (usesBuilder(ep)) { + return { id: ID, status: "pass", detail: "authenticated by the builder" }; + } + if (ep.calleeNames.some((n) => GUARD.test(n))) { + return { id: ID, status: "pass", detail: "guarded in the body" }; + } + return { + id: ID, + status: "fail", + detail: `sensitive (${sensitivity.reasons.join(", ")}) with no auth guard in the body`, + }; + }, +}; diff --git a/internal-packages/observability-map/src/checks/errorClassification.ts b/internal-packages/observability-map/src/checks/errorClassification.ts new file mode 100644 index 00000000000..637656f0d62 --- /dev/null +++ b/internal-packages/observability-map/src/checks/errorClassification.ts @@ -0,0 +1,65 @@ +import type { CheckResult, EntryPoint } from "../types.js"; +import { isTrivial } from "../triviality.js"; + +const ID = "error-classification"; + +/** + * The route builders that own the failure path: they authenticate, they catch, they pass a thrown + * `Response` through untouched and report anything else through `logBoundaryError` before + * answering 500. A route wrapped in one of these has its errors classified for it. + * + * `createSSELoader` is deliberately absent. It turns a non-Response error into a 500 but does not + * authenticate, so counting it here would hand two routes a free pass on `auth-boundary`. + * `createHybridActionApiRoute`, which the design named, exists nowhere in the tree. + */ +export const BUILDERS = new Set([ + "createLoaderApiRoute", + "createActionApiRoute", + "createLoaderPATApiRoute", + "createActionPATApiRoute", + "createMultiMethodApiRoute", + "createLoaderWorkerApiRoute", + "createActionWorkerApiRoute", + "dashboardLoader", + "dashboardAction", +]); + +export function usesBuilder(ep: EntryPoint): boolean { + return ( + (ep.loaderInitializerCallee !== null && BUILDERS.has(ep.loaderInitializerCallee)) || + (ep.actionInitializerCallee !== null && BUILDERS.has(ep.actionInitializerCallee)) + ); +} + +/** + * Who decides what a failure means. Three answers count as covered: the builder does it, the route + * declines to interfere and the error reaches the global handler, or the route is trivial. + * + * The fourth answer is the finding: a route outside the builders that catches its own errors. What + * that catch then does is the question the check would like to answer and cannot. `EntryPoint` + * carries `hasTryCatch`, a body-scoped boolean, and nothing about the catch clause itself, so a + * rethrow and a swallow look identical from here. Reading the shape of the catch out of `ep.source` + * would mean matching the whole file, React component included, which is how a component's + * try/catch ends up deciding a loader's verdict. Coarse and honest beats precise and wrong: the + * check reports the hand-rolled catch and says it has not been read. + */ +export const errorClassification = { + id: ID, + run(ep: EntryPoint): CheckResult { + if (isTrivial(ep)) { + return { id: ID, status: "not-applicable", detail: "trivial route" }; + } + if (usesBuilder(ep)) { + return { id: ID, status: "pass", detail: "classified by the builder" }; + } + if (!ep.hasTryCatch) { + return { id: ID, status: "pass", detail: "errors propagate to the global handler" }; + } + return { + id: ID, + status: "fail", + detail: + "handles its own errors outside a route builder, and the catch has not been read: check it distinguishes expected from unexpected", + }; + }, +}; diff --git a/internal-packages/observability-map/src/checks/index.ts b/internal-packages/observability-map/src/checks/index.ts new file mode 100644 index 00000000000..b94c8ad4867 --- /dev/null +++ b/internal-packages/observability-map/src/checks/index.ts @@ -0,0 +1,12 @@ +import type { CheckResult, EntryPoint } from "../types.js"; +import { errorClassification } from "./errorClassification.js"; +import { authBoundary } from "./authBoundary.js"; +import { requestContext } from "./requestContext.js"; +import { auditTrail } from "./auditTrail.js"; + +export type Check = { id: string; run: (ep: EntryPoint) => CheckResult }; + +/** audit-trail is scored separately, see score.ts. */ +export const CHECKS: Check[] = [errorClassification, authBoundary, requestContext, auditTrail]; +export const SCORED_CHECK_IDS = ["error-classification", "auth-boundary", "request-context"]; +export { usesBuilder, BUILDERS } from "./errorClassification.js"; diff --git a/internal-packages/observability-map/src/checks/requestContext.ts b/internal-packages/observability-map/src/checks/requestContext.ts new file mode 100644 index 00000000000..c224b3f413d --- /dev/null +++ b/internal-packages/observability-map/src/checks/requestContext.ts @@ -0,0 +1,53 @@ +import type { CheckResult, EntryPoint } from "../types.js"; +import { isTrivial } from "../triviality.js"; +import { usesBuilder } from "./errorClassification.js"; + +const ID = "request-context"; + +/** `requireUserId`, `authenticateApiRequest`: the request comes back with an identity attached. */ +const GUARD = /^(require|authenticate)/; + +/** `findProjectBySlug`, `loadProjectEnvironmentFromRequest`, `getUserSession`, `findWaitpoint`. */ +const RESOLVE = /^(find|get|load|resolve|lookup)/; +const SCOPE = + /(environment|project|organization|org|run|user|account|waitpoint|deployment|batch|schedule|task)/i; + +/** + * Whether the body ever works out who or what the request is for. A failure in a route that has + * resolved a user, an environment or a project can be attributed to one when it is reported; a + * route that resolves none of them has nothing to attribute a failure to, wherever it is logged. + */ +function resolvesRequestIdentity(ep: EntryPoint): boolean { + return ep.calleeNames.some((n) => GUARD.test(n) || (RESOLVE.test(n) && SCOPE.test(n))); +} + +/** + * The design asked whether an identifier reaches the failure path, and looked for one by matching + * `environmentId` and friends against `ep.source`. That is the whole file: a route whose React + * component renders `runId` would pass a check about its loader's error handling. + * + * The body-scoped substitute is weaker. `EntryPoint` records the names of the functions a body + * calls, not the arguments passed to them, so `logger.error("failed", { environmentId })` and + * `logger.error("failed")` are the same to this check. What is left is the identity the body + * resolves, which the callee names do carry. It answers a related question rather than the + * original one, and its evidence overlaps heavily with `auth-boundary`: see the task 5 report. + */ +export const requestContext = { + id: ID, + run(ep: EntryPoint): CheckResult { + if (isTrivial(ep)) { + return { id: ID, status: "not-applicable", detail: "trivial route" }; + } + if (usesBuilder(ep)) { + return { id: ID, status: "pass", detail: "attributed by the builder" }; + } + if (resolvesRequestIdentity(ep)) { + return { id: ID, status: "pass", detail: "resolves a request identity" }; + } + return { + id: ID, + status: "fail", + detail: "resolves no user, environment or project, so a failure here names nobody", + }; + }, +}; diff --git a/internal-packages/observability-map/test/checks.test.ts b/internal-packages/observability-map/test/checks.test.ts new file mode 100644 index 00000000000..57510e656ec --- /dev/null +++ b/internal-packages/observability-map/test/checks.test.ts @@ -0,0 +1,360 @@ +import { CHECKS, SCORED_CHECK_IDS } from "../src/checks/index.js"; +import { scanFile } from "../src/scan.js"; + +const run = (id: string, fileName: string, source: string) => { + const ep = scanFile(fileName, source)!; + return CHECKS.find((c) => c.id === id)!.run(ep); +}; + +/** + * The React component that lives alongside the loader in a `.tsx` route. Every check reads + * body-scoped evidence, so nothing in here may change a verdict: it try/catches, it logs, it names + * every request identifier the checks look for, and it calls an auth helper. + */ +const COMPONENT = ` + export default function Page() { + const { environmentId, organizationId, projectId, runId } = useTypedLoaderData(); + useEffect(() => { + try { + requireUserId(environmentId); + logger.error("render failed", { environmentId, organizationId, projectId, runId }); + } catch (e) { + if (e instanceof Error) return; + throw e; + } + }, [environmentId]); + return
{runId}
; + } +`; + +describe("registry", () => { + it("holds the four checks, with audit-trail left out of the score", () => { + expect(CHECKS.map((c) => c.id)).toEqual([ + "error-classification", + "auth-boundary", + "request-context", + "audit-trail", + ]); + expect(SCORED_CHECK_IDS).toEqual(["error-classification", "auth-boundary", "request-context"]); + }); +}); + +describe("error-classification", () => { + it("passes a builder-wrapped route with no local try/catch", () => { + const r = run( + "error-classification", + "api.v1.x.ts", + `import { createLoaderApiRoute } from "~/services/routeBuilders/apiBuilder.server"; + export const loader = createLoaderApiRoute({}, async () => new Response("ok"));` + ); + expect(r.status).toBe("pass"); + }); + + it("fails a raw route whose catch swallows every error identically", () => { + const r = run( + "error-classification", + "api.v1.y.ts", + `import { prisma } from "~/db.server"; + export async function loader() { + try { return await prisma.thing.findMany(); } catch (e) { return null; } + }` + ); + expect(r.status).toBe("fail"); + }); + + // The brief expected a pass here. `EntryPoint` carries `hasTryCatch` and nothing about what the + // catch does with the error, so the check cannot see the rethrow, and reading it out of + // `ep.source` would read the whole file. Known false positive, see the task 5 report. + it("also fails a raw route whose catch branches on the error", () => { + const r = run( + "error-classification", + "api.v1.z.ts", + `import { prisma } from "~/db.server"; + export async function loader() { + try { return await prisma.thing.findMany(); } + catch (e) { if (e instanceof NotFound) return null; throw e; } + }` + ); + expect(r.status).toBe("fail"); + }); + + it("passes a raw route that lets its errors propagate", () => { + const r = run( + "error-classification", + "api.v1.w.ts", + `import { prisma } from "~/db.server"; + export async function loader() { + const rows = await prisma.thing.findMany(); + return json({ rows }); + }` + ); + expect(r.status).toBe("pass"); + }); + + it("is not applicable to a trivial redirect", () => { + const r = run( + "error-classification", + "@.ts", + `import { redirect } from "@remix-run/server-runtime"; + export async function loader() { return redirect("/admin"); }` + ); + expect(r.status).toBe("not-applicable"); + }); + + // False positive fixture: the only try/catch in the file belongs to the component. + it("does not flag a route whose try/catch is in the React component", () => { + const r = run( + "error-classification", + "_app.orgs.$organizationSlug.things/route.tsx", + `import { prisma } from "~/db.server"; + export async function loader() { + const rows = await prisma.thing.findMany(); + return typedjson({ rows }); + } + ${COMPONENT}` + ); + expect(r.status).toBe("pass"); + }); +}); + +describe("auth-boundary", () => { + it("passes a sensitive route guarded by a require helper", () => { + const r = run( + "auth-boundary", + "admin.api.v1.gc.ts", + `import { requireAdminApiRequest } from "~/services/personalAccessToken.server"; + import { prisma } from "~/db.server"; + export async function loader({ request }) { + await requireAdminApiRequest(request); + return prisma.thing.findMany(); + }` + ); + expect(r.status).toBe("pass"); + }); + + it("passes a sensitive route guarded by an authenticate helper", () => { + const r = run( + "auth-boundary", + "api.v1.tokens.ts", + `import { authenticateApiRequest } from "~/services/apiAuth.server"; + import { prisma } from "~/db.server"; + export async function loader({ request }) { + const auth = await authenticateApiRequest(request); + if (!auth) throw new Response(null, { status: 401 }); + return prisma.token.findMany(); + }` + ); + expect(r.status).toBe("pass"); + }); + + it("fails a sensitive route with no guard", () => { + const r = run( + "auth-boundary", + "api.v1.tokens.ts", + `import { prisma } from "~/db.server"; + export async function loader() { + const tokens = await prisma.token.findMany(); + return json({ tokens }); + }` + ); + expect(r.status).toBe("fail"); + }); + + it("is not applicable to a non-sensitive route", () => { + const r = run( + "auth-boundary", + "api.v1.timezones.ts", + `import { prisma } from "~/db.server"; + export async function loader() { return prisma.tz.findMany(); }` + ); + expect(r.status).toBe("not-applicable"); + }); + + // False positive fixture: the guard sits one hop away, in a same-file helper. + it("does not flag a sensitive route whose guard is in a same-file helper", () => { + const r = run( + "auth-boundary", + "api.v1.tokens.ts", + `import { requireUserId } from "~/services/session.server"; + import { prisma } from "~/db.server"; + async function loadTokens(request) { + const userId = await requireUserId(request); + return prisma.token.findMany({ where: { userId } }); + } + export async function loader({ request }) { return json(await loadTokens(request)); }` + ); + expect(r.status).toBe("pass"); + }); + + // The guard has to be called, not merely imported: importedNames is file-wide. + it("fails a sensitive route that imports a guard it never calls", () => { + const r = run( + "auth-boundary", + "api.v1.tokens.ts", + `import { requireUserId } from "~/services/session.server"; + import { prisma } from "~/db.server"; + export async function loader() { + const tokens = await prisma.token.findMany(); + return json({ tokens }); + } + export function meta() { return requireUserId; }` + ); + expect(r.status).toBe("fail"); + }); +}); + +describe("request-context", () => { + it("passes a builder-wrapped route", () => { + const r = run( + "request-context", + "api.v1.q.ts", + `import { createLoaderApiRoute } from "~/services/routeBuilders/apiBuilder.server"; + import { prisma } from "~/db.server"; + export const loader = createLoaderApiRoute({}, async () => json(await prisma.thing.findMany()));` + ); + expect(r.status).toBe("pass"); + }); + + it("passes a raw route that resolves the request identity", () => { + const r = run( + "request-context", + "api.v1.q.ts", + `import { authenticateApiRequest } from "~/services/apiAuth.server"; + import { prisma } from "~/db.server"; + export async function loader({ request }) { + const auth = await authenticateApiRequest(request); + return json(await prisma.thing.findMany({ where: { environmentId: auth.environment.id } })); + }` + ); + expect(r.status).toBe("pass"); + }); + + it("passes a raw route that resolves a tenant without authenticating", () => { + const r = run( + "request-context", + "resources.things.ts", + `import { loadProjectEnvironmentFromRequest } from "~/services/environment.server"; + import { prisma } from "~/db.server"; + export async function loader({ request, params }) { + const environment = await loadProjectEnvironmentFromRequest(request, params); + return json(await prisma.thing.findMany({ where: { environmentId: environment.id } })); + }` + ); + expect(r.status).toBe("pass"); + }); + + it("fails a raw route that resolves nothing about the request", () => { + const r = run( + "request-context", + "api.v1.r.ts", + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function loader() { + try { return await prisma.thing.findMany(); } catch (e) { logger.error("failed"); throw e; } + }` + ); + expect(r.status).toBe("fail"); + }); + + // The brief expected a pass here, on the strength of the identifier in the log call. `EntryPoint` + // records callee names, not their arguments, so that identifier is not body-scoped evidence and + // the check cannot see it. Known false positive, see the task 5 report. + it("also fails a raw route that only names an identifier in a log call", () => { + const r = run( + "request-context", + "api.v1.q.ts", + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function loader({ params }) { + try { return await prisma.thing.findMany(); } + catch (e) { logger.error("failed", { environmentId: params.envId }); throw e; } + }` + ); + expect(r.status).toBe("fail"); + }); + + // False positive fixture: identifiers all over the component, none in the loader. The check must + // read the loader, so the verdict has to come from the loader's own call to requireUserId. + it("does not flag a route whose identifiers are in the React component", () => { + const r = run( + "request-context", + "_app.orgs.$organizationSlug.things/route.tsx", + `import { requireUserId } from "~/services/session.server"; + import { prisma } from "~/db.server"; + export async function loader({ request }) { + const userId = await requireUserId(request); + return typedjson(await prisma.thing.findMany({ where: { userId } })); + } + ${COMPONENT}` + ); + expect(r.status).toBe("pass"); + }); + + it("fails a route whose identifiers are only in the React component", () => { + const r = run( + "request-context", + "_app.orgs.$organizationSlug.things/route.tsx", + `import { prisma } from "~/db.server"; + export async function loader() { + return typedjson(await prisma.thing.findMany()); + } + ${COMPONENT}` + ); + expect(r.status).toBe("fail"); + }); + + it("is not applicable to a trivial redirect", () => { + const r = run( + "request-context", + "@.ts", + `import { redirect } from "@remix-run/server-runtime"; + export async function loader() { return redirect("/admin"); }` + ); + expect(r.status).toBe("not-applicable"); + }); +}); + +describe("audit-trail", () => { + it("is applicable only to sensitive mutations", () => { + const readOnly = run( + "audit-trail", + "api.v1.auth.jwt.ts", + `export async function loader() { return 1; }` + ); + expect(readOnly.status).toBe("not-applicable"); + + const mutation = run( + "audit-trail", + "api.v1.auth.jwt.ts", + `import { prisma } from "~/db.server"; + export async function action() { return prisma.token.create({ data: {} }); }` + ); + expect(mutation.status).toBe("fail"); + }); + + // False positive fixture: an ordinary mutation is not an audit target. + it("is not applicable to a non-sensitive mutation", () => { + const r = run( + "audit-trail", + "resources.things.ts", + `import { prisma } from "~/db.server"; + export async function action() { return prisma.thing.create({ data: {} }); }` + ); + expect(r.status).toBe("not-applicable"); + }); + + it("passes a sensitive mutation that records an audit event", () => { + const r = run( + "audit-trail", + "api.v1.auth.jwt.ts", + `import { auditLog } from "~/services/audit.server"; + import { prisma } from "~/db.server"; + export async function action({ request }) { + const token = await prisma.token.create({ data: {} }); + await auditLog("token.created", { tokenId: token.id }); + return json(token); + }` + ); + expect(r.status).toBe("pass"); + }); +}); From 6a07d6d2f4063109651f3c2f2200c28e4a3bbf9b Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Fri, 31 Jul 2026 14:16:40 +0100 Subject: [PATCH 008/106] feat(observability-map): record catch clause and callee evidence on entry points The error-classification and request-context checks could not tell a rethrow from a swallow, or a database call from any method with a common name. Four additive fields, all body scoped through the existing one-hop helper resolution. - catchRethrows and catchBranches: whether a catch clause in the bodies contains a throw, or branches with if, switch or instanceof. Of the 190 routes that catch, 140 do one of those and 50 take one path out - calleeTexts: the full callee path (prisma.organization.findFirst), index aligned with calleeNames, which is unchanged - logCalls: logger.* and log.* calls with their object argument field names and whether the call sits in a catch, so a check can ask whether the failure path logs an identifier No existing field changes value on any of the 427 route entry points. --- .../observability-map/src/scan.ts | 100 ++++++- .../observability-map/src/types.ts | 31 ++ .../observability-map/test/scan.test.ts | 283 ++++++++++++++++++ 3 files changed, 409 insertions(+), 5 deletions(-) diff --git a/internal-packages/observability-map/src/scan.ts b/internal-packages/observability-map/src/scan.ts index 2ebe03b67f5..4702a7d7771 100644 --- a/internal-packages/observability-map/src/scan.ts +++ b/internal-packages/observability-map/src/scan.ts @@ -1,7 +1,7 @@ import ts from "typescript"; import { readdirSync, readFileSync } from "node:fs"; import { join } from "node:path"; -import type { EntryPoint } from "./types.js"; +import type { EntryPoint, LogCall } from "./types.js"; /** Thrown by `scanFile` when the source does not parse cleanly. */ export class ParseFailureError extends Error { @@ -68,6 +68,65 @@ function calleeName(expr: ts.Expression): string | null { return null; } +/** + * Callee as recorded in `calleeTexts`: the whole path, `prisma.organization.findFirst` rather than + * `findFirst`. Null when the path runs through something with no name of its own, e.g. + * `new PromptService().createOverride`, where the caller falls back to the bare name. + */ +function calleeText(expr: ts.Expression): string | null { + const target = unwrap(expr); + if (ts.isIdentifier(target)) return target.text; + if (target.kind === ts.SyntaxKind.ThisKeyword) return "this"; + if (ts.isPropertyAccessExpression(target)) { + const base = calleeText(target.expression); + return base === null ? null : `${base}.${target.name.text}`; + } + if (ts.isCallExpression(target)) { + const base = calleeText(target.expression); + return base === null ? null : `${base}()`; + } + return null; +} + +/** `logger.error`, `log.info`, `this.logger.debug`. */ +const LOGGER_CALLEE = /(^|\.)(logger|log)\.[A-Za-z_$][\w$]*$/; + +/** Property names on the first object-literal argument, e.g. `{ environmentId, error }`. */ +function objectArgumentFields(call: ts.CallExpression): { found: boolean; fields: string[] } { + for (const arg of call.arguments) { + const target = unwrap(arg); + if (!ts.isObjectLiteralExpression(target)) continue; + const fields: string[] = []; + for (const property of target.properties) { + const name = propertyName(property); + if (name) fields.push(name); + } + return { found: true, fields }; + } + return { found: false, fields: [] }; +} + +/** What a catch clause does with the error, beyond the fact that it caught one. */ +function catchClauseEvidence(clause: ts.CatchClause): { rethrows: boolean; branches: boolean } { + let rethrows = false; + let branches = false; + + const visit = (node: ts.Node) => { + if (ts.isThrowStatement(node)) rethrows = true; + if (ts.isIfStatement(node) || ts.isSwitchStatement(node)) branches = true; + if ( + ts.isBinaryExpression(node) && + node.operatorToken.kind === ts.SyntaxKind.InstanceOfKeyword + ) { + branches = true; + } + ts.forEachChild(node, visit); + }; + visit(clause.block); + + return { rethrows, branches }; +} + const HTTP_METHODS = new Set(["GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS"]); function propertyName(property: ts.ObjectLiteralElementLike): string | null { @@ -396,7 +455,11 @@ export function scanFile(fileName: string, source: string): EntryPoint | null { let statementCount = 0; let hasTryCatch = false; + let catchRethrows = false; + let catchBranches = false; const calleeNames: string[] = []; + const calleeTexts: string[] = []; + const logCalls: LogCall[] = []; const localFunctions = collectLocalFunctions(sf); // A body that delegates to a same-file helper does the work in that helper, so the helper's @@ -409,11 +472,34 @@ export function scanFile(fileName: string, source: string): EntryPoint | null { statementCount += countFunctionStatements(fn); if (!fn.body) return; - const visit = (node: ts.Node) => { + const visit = (node: ts.Node, inCatch: boolean) => { if (ts.isTryStatement(node)) hasTryCatch = true; + + if (ts.isCatchClause(node)) { + const evidence = catchClauseEvidence(node); + catchRethrows ||= evidence.rethrows; + catchBranches ||= evidence.branches; + ts.forEachChild(node, (child) => visit(child, true)); + return; + } + if (ts.isCallExpression(node)) { const cn = calleeName(node.expression); - if (cn) calleeNames.push(cn); + if (cn) { + const text = calleeText(node.expression) ?? cn; + calleeNames.push(cn); + calleeTexts.push(text); + + if (LOGGER_CALLEE.test(text)) { + const argument = objectArgumentFields(node); + logCalls.push({ + callee: text, + hasObjectArgument: argument.found, + fields: argument.fields, + inCatch, + }); + } + } if (followHelpers) { const callee = unwrap(node.expression); @@ -426,9 +512,9 @@ export function scanFile(fileName: string, source: string): EntryPoint | null { } } } - ts.forEachChild(node, visit); + ts.forEachChild(node, (child) => visit(child, inCatch)); }; - visit(fn.body); + visit(fn.body, false); }; for (const fn of target.functions) walkBody(fn, true); @@ -443,7 +529,11 @@ export function scanFile(fileName: string, source: string): EntryPoint | null { actionInitializerCallee: target.actionInitializerCallee, importedNames, calleeNames, + calleeTexts, hasTryCatch, + catchRethrows, + catchBranches, + logCalls, statementCount, }; } diff --git a/internal-packages/observability-map/src/types.ts b/internal-packages/observability-map/src/types.ts index 1f23f0e059f..15b8d581132 100644 --- a/internal-packages/observability-map/src/types.ts +++ b/internal-packages/observability-map/src/types.ts @@ -6,6 +6,18 @@ export type CheckResult = { detail?: string; }; +/** A logging call made from a loader/action body, or from a same-file helper the body calls. */ +export type LogCall = { + /** Full callee path, e.g. `logger.error`. */ + callee: string; + /** Whether an object literal was passed as an argument. */ + hasObjectArgument: boolean; + /** Property names on that object literal, e.g. `["environmentId", "error"]`. */ + fields: string[]; + /** Whether the call sits inside a catch clause, i.e. on the failure path. */ + inCatch: boolean; +}; + export type EntryPoint = { fileName: string; source: string; @@ -18,8 +30,27 @@ export type EntryPoint = { importedNames: string[]; /** Names of functions called inside the loader/action bodies, or in a same-file helper they call. */ calleeNames: string[]; + /** + * The same calls as `calleeNames`, same order and same length, but as the whole callee path: + * `prisma.organization.findFirst` where `calleeNames` has `findFirst`. A path that runs through + * something unnameable (`new PromptService().createOverride`) falls back to the bare name. + */ + calleeTexts: string[]; /** Whether a `try` appears in the loader/action bodies, or in a same-file helper they call. */ hasTryCatch: boolean; + /** + * Whether any catch clause in those bodies contains a `throw`. A catch that rethrows has decided + * the error is not its to answer, which is a different act from swallowing it. + */ + catchRethrows: boolean; + /** + * Whether any catch clause in those bodies branches on the error: an `if`, a `switch`, or an + * `instanceof`. With `catchRethrows` both false while `hasTryCatch` is true, every catch in the + * entry point takes one path out regardless of what was thrown. + */ + catchBranches: boolean; + /** Calls to a `logger.*` or `log.*` callee in those bodies, in source order. */ + logCalls: LogCall[]; /** * Statement count across loader/action bodies, used by the triviality rule. A body that * delegates to a same-file helper counts that helper's statements too, one hop only: work in a diff --git a/internal-packages/observability-map/test/scan.test.ts b/internal-packages/observability-map/test/scan.test.ts index c727577a78c..ca92dd26182 100644 --- a/internal-packages/observability-map/test/scan.test.ts +++ b/internal-packages/observability-map/test/scan.test.ts @@ -569,3 +569,286 @@ describe("scanDirectory", () => { expect(entryPoints.map((ep) => ep.fileName)).toEqual(["projects.v3.$projectRef.test.ts"]); }); }); + +describe("scanFile: catch clause evidence", () => { + it("sets catchRethrows when a catch rethrows", () => { + const ep = scanFile( + "rethrow.ts", + ` + export async function loader({ request }) { + try { + return json(await load(request)); + } catch (e) { + logger.error(e); + throw e; + } + } + ` + ); + expect(ep!.hasTryCatch).toBe(true); + expect(ep!.catchRethrows).toBe(true); + expect(ep!.catchBranches).toBe(false); + }); + + it("leaves both flags false when the catch only returns", () => { + const ep = scanFile( + "swallow.ts", + ` + export async function loader({ request }) { + try { + return json(await load(request)); + } catch (e) { + return null; + } + } + ` + ); + expect(ep!.hasTryCatch).toBe(true); + expect(ep!.catchRethrows).toBe(false); + expect(ep!.catchBranches).toBe(false); + }); + + it("sets catchBranches for an `if` on the error", () => { + const ep = scanFile( + "branch-if.ts", + ` + export async function loader({ request }) { + try { + return json(await request.json()); + } catch (e) { + if (e instanceof SyntaxError) { + return json({ error: "bad json" }, { status: 400 }); + } + return json({ error: "failed" }, { status: 500 }); + } + } + ` + ); + expect(ep!.catchBranches).toBe(true); + expect(ep!.catchRethrows).toBe(false); + }); + + it("sets catchBranches for a bare instanceof with no `if`", () => { + const ep = scanFile( + "branch-instanceof.ts", + ` + export async function loader({ request }) { + try { + return json(await load(request)); + } catch (e) { + return e instanceof Response ? e : json({}, { status: 500 }); + } + } + ` + ); + expect(ep!.catchBranches).toBe(true); + }); + + it("sets catchBranches for a switch in the catch", () => { + const ep = scanFile( + "branch-switch.ts", + ` + export async function loader({ request }) { + try { + return json(await load(request)); + } catch (e) { + switch (e.code) { + case "P2025": + return json({}, { status: 404 }); + default: + return json({}, { status: 500 }); + } + } + } + ` + ); + expect(ep!.catchBranches).toBe(true); + }); + + it("ignores a catch that lives in the React component", () => { + const ep = scanFile( + "route.tsx", + ` + export async function loader() { + return json({}); + } + export default function Page() { + try { + render(); + } catch (e) { + if (e instanceof RenderError) throw e; + return null; + } + } + ` + ); + expect(ep!.hasTryCatch).toBe(false); + expect(ep!.catchRethrows).toBe(false); + expect(ep!.catchBranches).toBe(false); + }); + + it("reads a catch inside a same-file helper the body delegates to", () => { + const ep = scanFile( + "ph.$.ts", + ` + async function proxy(request) { + try { + return await fetch(request.url); + } catch (e) { + if (e.name === "AbortError") throw e; + return new Response(null, { status: 502 }); + } + } + export async function loader({ request }) { + return proxy(request); + } + ` + ); + expect(ep!.hasTryCatch).toBe(true); + expect(ep!.catchRethrows).toBe(true); + expect(ep!.catchBranches).toBe(true); + }); + + it("leaves both flags false for a try with no catch", () => { + const ep = scanFile( + "finally-only.ts", + ` + export async function loader() { + try { + return json(await load()); + } finally { + release(); + } + } + ` + ); + expect(ep!.hasTryCatch).toBe(true); + expect(ep!.catchRethrows).toBe(false); + expect(ep!.catchBranches).toBe(false); + }); +}); + +describe("scanFile: callee texts", () => { + it("keeps the full callee expression alongside the bare name", () => { + const ep = scanFile( + "api.v1.things.ts", + ` + export async function loader({ request }) { + const org = await prisma.organization.findFirst({ where: { id: 1 } }); + logger.error("nope", { organizationId: org.id }); + return json(org); + } + ` + ); + expect(ep!.calleeNames).toContain("findFirst"); + expect(ep!.calleeTexts).toContain("prisma.organization.findFirst"); + expect(ep!.calleeTexts).toContain("logger.error"); + expect(ep!.calleeTexts).toContain("json"); + // Index-aligned with calleeNames, so a consumer can read either. + expect(ep!.calleeTexts).toHaveLength(ep!.calleeNames.length); + }); + + it("does not leak calls made in the React component", () => { + const ep = scanFile( + "route.tsx", + ` + export async function loader() { + return json(await prisma.run.findMany()); + } + export default function Page() { + useFancyHook(); + analytics.track("viewed"); + return null; + } + ` + ); + expect(ep!.calleeTexts).toContain("prisma.run.findMany"); + expect(ep!.calleeTexts).not.toContain("analytics.track"); + expect(ep!.calleeTexts).not.toContain("useFancyHook"); + }); + + it("records callee texts from a same-file helper the body delegates to", () => { + const ep = scanFile( + "delegating.ts", + ` + async function load(id) { + return prisma.project.findUnique({ where: { id } }); + } + export async function loader({ params }) { + return json(await load(params.id)); + } + ` + ); + expect(ep!.calleeTexts).toContain("prisma.project.findUnique"); + }); + + it("falls back to the bare name for a callee it cannot render as a path", () => { + const ep = scanFile( + "new-expression.ts", + ` + export async function action({ request }) { + return json(await new PromptService().createOverride(request)); + } + ` + ); + expect(ep!.calleeNames).toContain("createOverride"); + expect(ep!.calleeTexts).toContain("createOverride"); + expect(ep!.calleeTexts).toHaveLength(ep!.calleeNames.length); + }); +}); + +describe("scanFile: log calls", () => { + it("records the fields of a log call's object argument", () => { + const ep = scanFile( + "logging.ts", + ` + export async function loader({ request }) { + try { + return json(await load(request)); + } catch (e) { + logger.error("load failed", { environmentId: env.id, error: e }); + return json({}, { status: 500 }); + } + } + ` + ); + expect(ep!.logCalls).toHaveLength(1); + expect(ep!.logCalls[0]).toEqual({ + callee: "logger.error", + hasObjectArgument: true, + fields: ["environmentId", "error"], + inCatch: true, + }); + }); + + it("records a log call with no object argument, outside a catch", () => { + const ep = scanFile( + "logging-plain.ts", + ` + export async function loader() { + log.info("starting"); + return json({}); + } + ` + ); + expect(ep!.logCalls).toEqual([ + { callee: "log.info", hasObjectArgument: false, fields: [], inCatch: false }, + ]); + }); + + it("ignores a non-logger call and a log call in the React component", () => { + const ep = scanFile( + "route.tsx", + ` + export async function loader() { + return json(await load()); + } + export default function Page() { + logger.debug("rendered", { runId: 1 }); + return null; + } + ` + ); + expect(ep!.logCalls).toEqual([]); + }); +}); From 751ea1b3b4f54942ef6e2448d9e33b01ec6f8390 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Fri, 31 Jul 2026 14:28:52 +0100 Subject: [PATCH 009/106] feat(observability-map): rework two checks on the enriched scanner error-classification now reads catchRethrows and catchBranches instead of the mere presence of a try. It fails only where every catch in the bodies takes one way out regardless of what was thrown, which drops the finding count from 130 to 50. The swallow is read before the builder is credited: a swallow inside a builder-wrapped handler never reaches the builder, and 18 of the 50 are that shape. request-context now asks whether a failure-path log names a tenant, using logCalls with inCatch and the field names. The builder pass is gone, because the builders log { error, url } at their boundary and the logger only attaches http context ambiently, so a wrapped route is not attributed either. The check no longer echoes auth-boundary: 17 entry points of 427 are scored by both, and they disagree on 10 of those. auth-boundary and audit-trail are unchanged. --- .../src/checks/errorClassification.ts | 39 +++-- .../src/checks/requestContext.ts | 62 ++++--- .../observability-map/test/checks.test.ts | 158 +++++++++++++----- 3 files changed, 164 insertions(+), 95 deletions(-) diff --git a/internal-packages/observability-map/src/checks/errorClassification.ts b/internal-packages/observability-map/src/checks/errorClassification.ts index 637656f0d62..cfa8e389a82 100644 --- a/internal-packages/observability-map/src/checks/errorClassification.ts +++ b/internal-packages/observability-map/src/checks/errorClassification.ts @@ -32,16 +32,17 @@ export function usesBuilder(ep: EntryPoint): boolean { } /** - * Who decides what a failure means. Three answers count as covered: the builder does it, the route - * declines to interfere and the error reaches the global handler, or the route is trivial. + * Who decides what a failure means, and on what evidence. * - * The fourth answer is the finding: a route outside the builders that catches its own errors. What - * that catch then does is the question the check would like to answer and cannot. `EntryPoint` - * carries `hasTryCatch`, a body-scoped boolean, and nothing about the catch clause itself, so a - * rethrow and a swallow look identical from here. Reading the shape of the catch out of `ep.source` - * would mean matching the whole file, React component included, which is how a component's - * try/catch ends up deciding a loader's verdict. Coarse and honest beats precise and wrong: the - * check reports the hand-rolled catch and says it has not been read. + * The swallow is read before the builder is credited, which looks like the wrong order until you + * read `api.v2.runs.$runParam.cancel.ts`: a `createActionApiRoute` handler wrapping its service + * call in `try { ... } catch { return 500 }`. The builder classifies what reaches it, and that + * error never does. Crediting the wrapper would hide the one case in this family worth finding. + * + * `catchRethrows` and `catchBranches` are OR-ed across every catch clause in the bodies, so both + * false means every catch in the entry point takes the same way out whatever was thrown. The + * asymmetry that buys: one good catch alongside one swallow reads as a pass. This check misses + * those rather than inventing them. */ export const errorClassification = { id: ID, @@ -49,17 +50,19 @@ export const errorClassification = { if (isTrivial(ep)) { return { id: ID, status: "not-applicable", detail: "trivial route" }; } + if (ep.hasTryCatch && !ep.catchRethrows && !ep.catchBranches) { + return { + id: ID, + status: "fail", + detail: "catches its errors and takes one way out regardless of what was thrown", + }; + } + if (ep.catchRethrows || ep.catchBranches) { + return { id: ID, status: "pass", detail: "the catch distinguishes what it caught" }; + } if (usesBuilder(ep)) { return { id: ID, status: "pass", detail: "classified by the builder" }; } - if (!ep.hasTryCatch) { - return { id: ID, status: "pass", detail: "errors propagate to the global handler" }; - } - return { - id: ID, - status: "fail", - detail: - "handles its own errors outside a route builder, and the catch has not been read: check it distinguishes expected from unexpected", - }; + return { id: ID, status: "pass", detail: "errors propagate to the global handler" }; }, }; diff --git a/internal-packages/observability-map/src/checks/requestContext.ts b/internal-packages/observability-map/src/checks/requestContext.ts index c224b3f413d..248ec1374ff 100644 --- a/internal-packages/observability-map/src/checks/requestContext.ts +++ b/internal-packages/observability-map/src/checks/requestContext.ts @@ -1,53 +1,51 @@ -import type { CheckResult, EntryPoint } from "../types.js"; -import { isTrivial } from "../triviality.js"; -import { usesBuilder } from "./errorClassification.js"; +import type { CheckResult, EntryPoint, LogCall } from "../types.js"; const ID = "request-context"; -/** `requireUserId`, `authenticateApiRequest`: the request comes back with an identity attached. */ -const GUARD = /^(require|authenticate)/; - -/** `findProjectBySlug`, `loadProjectEnvironmentFromRequest`, `getUserSession`, `findWaitpoint`. */ -const RESOLVE = /^(find|get|load|resolve|lookup)/; -const SCOPE = - /(environment|project|organization|org|run|user|account|waitpoint|deployment|batch|schedule|task)/i; - /** - * Whether the body ever works out who or what the request is for. A failure in a route that has - * resolved a user, an environment or a project can be attributed to one when it is reported; a - * route that resolves none of them has nothing to attribute a failure to, wherever it is logged. + * A field name that says which tenant, request or resource the failure belongs to. Matched on the + * suffix, in the camelCase the webapp writes: `environmentId`, `organizationSlug`, `runFriendlyId`, + * `projectParam`, `taskIdentifier`. Lowercase `id` inside a word is not a suffix, so `valid` and + * `paid` do not qualify. */ -function resolvesRequestIdentity(ep: EntryPoint): boolean { - return ep.calleeNames.some((n) => GUARD.test(n) || (RESOLVE.test(n) && SCOPE.test(n))); +const IDENTIFIER_FIELD = /^(id|ids|slug|ref)$|[a-z](Id|Ids|Slug|Ref|Param|Identifier)$/; + +function failurePathLogs(ep: EntryPoint): LogCall[] { + return ep.logCalls.filter((l) => l.inCatch); } /** - * The design asked whether an identifier reaches the failure path, and looked for one by matching - * `environmentId` and friends against `ep.source`. That is the whole file: a route whose React - * component renders `runId` would pass a check about its loader's error handling. + * Whether a failure this route reports itself can be traced to whoever it happened to. * - * The body-scoped substitute is weaker. `EntryPoint` records the names of the functions a body - * calls, not the arguments passed to them, so `logger.error("failed", { environmentId })` and - * `logger.error("failed")` are the same to this check. What is left is the identity the body - * resolves, which the callee names do carry. It answers a related question rather than the - * original one, and its evidence overlaps heavily with `auth-boundary`: see the task 5 report. + * Everything the platform attaches centrally is already accounted for, which is what makes this + * worth asking. `logger` pushes the http context (requestId, path, host, method) onto every line + * through AsyncLocalStorage, and `Logger.onError` forwards the error to Sentry. Neither carries a + * tenant: no route calls `trace({ environmentId }, ...)`, and the builders' own boundary log is + * `logBoundaryError(message, error, url)`, which is a url and an error. So a builder-wrapped route + * is not attributed either and gets no free pass here. An incident tells you which route and which + * request; whose environment it was is only ever in the fields the route passes itself. + * + * Applicable only where the route reports a failure itself, meaning it logs from inside a catch. A + * route that rethrows silently has handed the report to Sentry and there is nothing here to + * inspect. That gate has a perverse edge, noted in the task 5 report: deleting a log line moves a + * route from fail to not-applicable. */ export const requestContext = { id: ID, run(ep: EntryPoint): CheckResult { - if (isTrivial(ep)) { - return { id: ID, status: "not-applicable", detail: "trivial route" }; - } - if (usesBuilder(ep)) { - return { id: ID, status: "pass", detail: "attributed by the builder" }; + const logs = failurePathLogs(ep); + if (logs.length === 0) { + return { id: ID, status: "not-applicable", detail: "reports no failure of its own" }; } - if (resolvesRequestIdentity(ep)) { - return { id: ID, status: "pass", detail: "resolves a request identity" }; + const named = logs.find((l) => l.fields.some((f) => IDENTIFIER_FIELD.test(f))); + if (named) { + const fields = named.fields.filter((f) => IDENTIFIER_FIELD.test(f)); + return { id: ID, status: "pass", detail: `failure log names ${fields.join(", ")}` }; } return { id: ID, status: "fail", - detail: "resolves no user, environment or project, so a failure here names nobody", + detail: "logs its failure without naming an environment, project, organization, run or user", }; }, }; diff --git a/internal-packages/observability-map/test/checks.test.ts b/internal-packages/observability-map/test/checks.test.ts index 57510e656ec..cf48a61a21c 100644 --- a/internal-packages/observability-map/test/checks.test.ts +++ b/internal-packages/observability-map/test/checks.test.ts @@ -62,10 +62,8 @@ describe("error-classification", () => { expect(r.status).toBe("fail"); }); - // The brief expected a pass here. `EntryPoint` carries `hasTryCatch` and nothing about what the - // catch does with the error, so the check cannot see the rethrow, and reading it out of - // `ep.source` would read the whole file. Known false positive, see the task 5 report. - it("also fails a raw route whose catch branches on the error", () => { + // Restored from the brief: `catchBranches` sees the `instanceof` and the `if`. + it("passes a raw route whose catch branches on the error", () => { const r = run( "error-classification", "api.v1.z.ts", @@ -75,6 +73,39 @@ describe("error-classification", () => { catch (e) { if (e instanceof NotFound) return null; throw e; } }` ); + expect(r.status).toBe("pass"); + }); + + it("passes a raw route whose catch rethrows without branching", () => { + const r = run( + "error-classification", + "api.v1.v.ts", + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function loader() { + try { return await prisma.thing.findMany(); } + catch (e) { logger.error("thing lookup failed", { error: e }); throw e; } + }` + ); + expect(r.status).toBe("pass"); + }); + + // The builder only classifies what reaches it. A swallow inside the handler never does, so the + // swallow is read before the builder is credited. + it("fails a builder-wrapped route whose handler swallows", () => { + const r = run( + "error-classification", + "api.v2.runs.$runParam.cancel.ts", + `import { createActionApiRoute } from "~/services/routeBuilders/apiBuilder.server"; + import { CancelTaskRunService } from "~/services/cancelTaskRun.server"; + const { action } = createActionApiRoute({}, async ({ params }) => { + const service = new CancelTaskRunService(); + try { await service.call(params.runParam); } + catch { return json({ error: "Internal Server Error" }, { status: 500 }); } + return json({ ok: true }); + }); + export { action };` + ); expect(r.status).toBe("fail"); }); @@ -204,46 +235,55 @@ describe("auth-boundary", () => { }); describe("request-context", () => { - it("passes a builder-wrapped route", () => { + it("passes a route whose failure log names an identifier", () => { const r = run( "request-context", - "api.v1.q.ts", - `import { createLoaderApiRoute } from "~/services/routeBuilders/apiBuilder.server"; + "engine.v1.dev.config.ts", + `import { logger } from "~/services/logger.server"; import { prisma } from "~/db.server"; - export const loader = createLoaderApiRoute({}, async () => json(await prisma.thing.findMany()));` + export async function loader({ params }) { + try { return await prisma.thing.findMany(); } + catch (error) { + logger.error("dev config failed", { environmentId: params.envId, error }); + throw error; + } + }` ); expect(r.status).toBe("pass"); }); - it("passes a raw route that resolves the request identity", () => { + it("passes on a route param, which names the tenant just as well", () => { const r = run( "request-context", - "api.v1.q.ts", - `import { authenticateApiRequest } from "~/services/apiAuth.server"; + "resources.things.ts", + `import { logger } from "~/services/logger.server"; import { prisma } from "~/db.server"; - export async function loader({ request }) { - const auth = await authenticateApiRequest(request); - return json(await prisma.thing.findMany({ where: { environmentId: auth.environment.id } })); + export async function loader({ params }) { + try { return await prisma.thing.findMany(); } + catch (error) { + logger.error("lookup failed", { organizationSlug: params.organizationSlug, error }); + throw error; + } }` ); expect(r.status).toBe("pass"); }); - it("passes a raw route that resolves a tenant without authenticating", () => { + it("fails a failure log that carries only the error", () => { const r = run( "request-context", - "resources.things.ts", - `import { loadProjectEnvironmentFromRequest } from "~/services/environment.server"; + "api.v1.r.ts", + `import { logger } from "~/services/logger.server"; import { prisma } from "~/db.server"; - export async function loader({ request, params }) { - const environment = await loadProjectEnvironmentFromRequest(request, params); - return json(await prisma.thing.findMany({ where: { environmentId: environment.id } })); + export async function loader() { + try { return await prisma.thing.findMany(); } + catch (error) { logger.error("failed", { error }); throw error; } }` ); - expect(r.status).toBe("pass"); + expect(r.status).toBe("fail"); }); - it("fails a raw route that resolves nothing about the request", () => { + it("fails a bare failure log with no object argument at all", () => { const r = run( "request-context", "api.v1.r.ts", @@ -256,51 +296,79 @@ describe("request-context", () => { expect(r.status).toBe("fail"); }); - // The brief expected a pass here, on the strength of the identifier in the log call. `EntryPoint` - // records callee names, not their arguments, so that identifier is not body-scoped evidence and - // the check cannot see it. Known false positive, see the task 5 report. - it("also fails a raw route that only names an identifier in a log call", () => { + // The builder logs `{ error, url }` at its boundary and nothing that names a tenant, so being + // wrapped in one earns no pass here. This is what stops the check echoing `auth-boundary`. + it("fails a builder-wrapped route whose own failure log names nobody", () => { const r = run( "request-context", "api.v1.q.ts", - `import { logger } from "~/services/logger.server"; + `import { createLoaderApiRoute } from "~/services/routeBuilders/apiBuilder.server"; + import { logger } from "~/services/logger.server"; import { prisma } from "~/db.server"; - export async function loader({ params }) { - try { return await prisma.thing.findMany(); } - catch (e) { logger.error("failed", { environmentId: params.envId }); throw e; } - }` + export const loader = createLoaderApiRoute({}, async () => { + try { return json(await prisma.thing.findMany()); } + catch (error) { logger.error("failed", { error }); throw error; } + });` ); expect(r.status).toBe("fail"); }); - // False positive fixture: identifiers all over the component, none in the loader. The check must - // read the loader, so the verdict has to come from the loader's own call to requireUserId. - it("does not flag a route whose identifiers are in the React component", () => { + it("is not applicable to a route that logs nothing on its failure path", () => { const r = run( "request-context", - "_app.orgs.$organizationSlug.things/route.tsx", - `import { requireUserId } from "~/services/session.server"; + "api.v1.q.ts", + `import { authenticateApiRequest } from "~/services/apiAuth.server"; import { prisma } from "~/db.server"; export async function loader({ request }) { - const userId = await requireUserId(request); - return typedjson(await prisma.thing.findMany({ where: { userId } })); - } - ${COMPONENT}` + const auth = await authenticateApiRequest(request); + return json(await prisma.thing.findMany({ where: { environmentId: auth.environment.id } })); + }` ); - expect(r.status).toBe("pass"); + expect(r.status).toBe("not-applicable"); }); - it("fails a route whose identifiers are only in the React component", () => { + it("is not applicable when the only log sits outside the catch", () => { const r = run( + "request-context", + "api.v1.q.ts", + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function loader() { + logger.info("starting", { environmentId: "env_1" }); + try { return await prisma.thing.findMany(); } catch (e) { throw e; } + }` + ); + expect(r.status).toBe("not-applicable"); + }); + + // False positive fixture: the component logs every identifier there is, inside its own catch. + // Only the loader's own failure log may decide this. + it("does not read the React component's log calls", () => { + const bare = run( "request-context", "_app.orgs.$organizationSlug.things/route.tsx", - `import { prisma } from "~/db.server"; + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; export async function loader() { - return typedjson(await prisma.thing.findMany()); + try { return typedjson(await prisma.thing.findMany()); } + catch (error) { logger.error("failed", { error }); throw error; } } ${COMPONENT}` ); - expect(r.status).toBe("fail"); + expect(bare.status).toBe("fail"); + + const attributed = run( + "request-context", + "_app.orgs.$organizationSlug.things/route.tsx", + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function loader({ params }) { + try { return typedjson(await prisma.thing.findMany()); } + catch (error) { logger.error("failed", { projectParam: params.projectParam, error }); throw error; } + } + ${COMPONENT}` + ); + expect(attributed.status).toBe("pass"); }); it("is not applicable to a trivial redirect", () => { From 3b19436d649f390cecc496917cd200c2a098eb8d Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Fri, 31 Jul 2026 14:35:29 +0100 Subject: [PATCH 010/106] feat(observability-map): flag entry points whose catches guard one operation error-classification cannot tell the deliberate narrow guard, e.g. try { body = await request.json() } catch { 400 }, from a catch that swallows the whole handler. Both take one path out. catchesNarrowly is true when an entry point has at least one catch clause and no try block with a catch holds more than two statements, counted in the loader/action bodies and the same one-hop helpers as the other fields. Every catch has to qualify: one broad catch anywhere makes it false, so a route that guards a JSON.parse and also wraps its handler is still reported. Two statements lets the guarded operation bind its result and stops short of the three-statement try that covers a handler. 55 of the 427 route entry points, and 11 of the 32 error-classification failures, all eleven hand-read as the deliberate idiom. No existing field changes value. --- .../observability-map/src/scan.ts | 18 +- .../observability-map/src/types.ts | 8 + .../observability-map/test/scan.test.ts | 173 ++++++++++++++++++ 3 files changed, 198 insertions(+), 1 deletion(-) diff --git a/internal-packages/observability-map/src/scan.ts b/internal-packages/observability-map/src/scan.ts index 4702a7d7771..1cb69bcdc27 100644 --- a/internal-packages/observability-map/src/scan.ts +++ b/internal-packages/observability-map/src/scan.ts @@ -106,6 +106,13 @@ function objectArgumentFields(call: ts.CallExpression): { found: boolean; fields return { found: false, fields: [] }; } +/** + * How much a try block may guard and still count as narrow. Two, so that the guarded operation can + * bind its result (`const stripped = ...; new RegExp(stripped);`) but a third statement means the + * try has started to cover the handler rather than one operation. + */ +const NARROW_TRY_STATEMENTS = 2; + /** What a catch clause does with the error, beyond the fact that it caught one. */ function catchClauseEvidence(clause: ts.CatchClause): { rethrows: boolean; branches: boolean } { let rethrows = false; @@ -457,6 +464,8 @@ export function scanFile(fileName: string, source: string): EntryPoint | null { let hasTryCatch = false; let catchRethrows = false; let catchBranches = false; + let catchClauseCount = 0; + let broadCatch = false; const calleeNames: string[] = []; const calleeTexts: string[] = []; const logCalls: LogCall[] = []; @@ -473,7 +482,13 @@ export function scanFile(fileName: string, source: string): EntryPoint | null { if (!fn.body) return; const visit = (node: ts.Node, inCatch: boolean) => { - if (ts.isTryStatement(node)) hasTryCatch = true; + if (ts.isTryStatement(node)) { + hasTryCatch = true; + if (node.catchClause) { + catchClauseCount += 1; + if (countStatements(node.tryBlock.statements) > NARROW_TRY_STATEMENTS) broadCatch = true; + } + } if (ts.isCatchClause(node)) { const evidence = catchClauseEvidence(node); @@ -533,6 +548,7 @@ export function scanFile(fileName: string, source: string): EntryPoint | null { hasTryCatch, catchRethrows, catchBranches, + catchesNarrowly: catchClauseCount > 0 && !broadCatch, logCalls, statementCount, }; diff --git a/internal-packages/observability-map/src/types.ts b/internal-packages/observability-map/src/types.ts index 15b8d581132..5d7dc2292ae 100644 --- a/internal-packages/observability-map/src/types.ts +++ b/internal-packages/observability-map/src/types.ts @@ -49,6 +49,14 @@ export type EntryPoint = { * entry point takes one path out regardless of what was thrown. */ catchBranches: boolean; + /** + * Whether every catch in those bodies guards a specific operation rather than the handler: the + * entry point has at least one catch clause, and no try block with a catch holds more than two + * statements. The `try { body = await request.json() } catch { 400 }` idiom, which takes one path + * out and is still deliberate. False when any catch wraps the bulk of a body, and false when + * there is no catch clause at all. + */ + catchesNarrowly: boolean; /** Calls to a `logger.*` or `log.*` callee in those bodies, in source order. */ logCalls: LogCall[]; /** diff --git a/internal-packages/observability-map/test/scan.test.ts b/internal-packages/observability-map/test/scan.test.ts index ca92dd26182..943c4a56f3e 100644 --- a/internal-packages/observability-map/test/scan.test.ts +++ b/internal-packages/observability-map/test/scan.test.ts @@ -852,3 +852,176 @@ describe("scanFile: log calls", () => { expect(ep!.logCalls).toEqual([]); }); }); + +describe("scanFile: narrow catches", () => { + it("flags a try that guards a single request.json()", () => { + const ep = scanFile( + "admin.api.v1.platform-notifications.ts", + ` + export async function action({ request }) { + const user = await requireUser(request); + let body; + try { + body = await request.json(); + } catch { + return json({ error: "Invalid JSON body" }, { status: 400 }); + } + const result = await createPlatformNotification(body); + return json(result); + } + ` + ); + expect(ep!.hasTryCatch).toBe(true); + expect(ep!.catchRethrows).toBe(false); + expect(ep!.catchBranches).toBe(false); + expect(ep!.catchesNarrowly).toBe(true); + }); + + it("does not flag a catch wrapping the whole handler", () => { + const ep = scanFile( + "otel.v1.logs.ts", + ` + export async function action({ request }) { + try { + const exporter = await otlpExporter; + const contentType = request.headers.get("content-type") ?? ""; + const body = await request.json(); + await exporter.export(body); + return json({ ok: true }); + } catch (e) { + logger.error(e); + return json({}, { status: 500 }); + } + } + ` + ); + expect(ep!.hasTryCatch).toBe(true); + expect(ep!.catchesNarrowly).toBe(false); + }); + + it("does not flag a body that has both a narrow catch and a broad one", () => { + const ep = scanFile( + "mixed.ts", + ` + export async function action({ request }) { + let body; + try { + body = await request.json(); + } catch { + return json({}, { status: 400 }); + } + try { + const run = await find(body.id); + const updated = await update(run); + await notify(updated); + return json(updated); + } catch (e) { + return json({}, { status: 500 }); + } + } + ` + ); + expect(ep!.catchesNarrowly).toBe(false); + }); + + it("allows a guarded operation with its own local binding", () => { + const ep = scanFile( + "regex.ts", + ` + export async function action({ request }) { + const pattern = await patternFrom(request); + try { + const stripped = pattern.startsWith("(?i)") ? pattern.slice(4) : pattern; + new RegExp(stripped); + } catch { + return json({ error: "Invalid regex" }, { status: 400 }); + } + return json({ ok: true }); + } + ` + ); + expect(ep!.catchesNarrowly).toBe(true); + }); + + it("does not flag a try of three statements", () => { + const ep = scanFile( + "three.ts", + ` + export async function loader({ request }) { + try { + const raw = await request.json(); + const parsed = Schema.parse(raw); + return json(parsed); + } catch { + return json({}, { status: 400 }); + } + } + ` + ); + expect(ep!.catchesNarrowly).toBe(false); + }); + + it("is false when there is no try at all", () => { + const ep = scanFile("plain.ts", `export async function loader() { return json({}); }`); + expect(ep!.hasTryCatch).toBe(false); + expect(ep!.catchesNarrowly).toBe(false); + }); + + it("is false for a try with a finally and no catch", () => { + const ep = scanFile( + "finally-only.ts", + ` + export async function loader() { + try { + return json(await load()); + } finally { + release(); + } + } + ` + ); + expect(ep!.hasTryCatch).toBe(true); + expect(ep!.catchesNarrowly).toBe(false); + }); + + it("reads a narrow catch inside a same-file helper the body delegates to", () => { + const ep = scanFile( + "helper-narrow.ts", + ` + function parseTags(payload) { + try { + return JSON.parse(payload); + } catch { + return null; + } + } + export async function loader({ params }) { + return json(parseTags(params.payload)); + } + ` + ); + expect(ep!.hasTryCatch).toBe(true); + expect(ep!.catchesNarrowly).toBe(true); + }); + + it("ignores a narrow catch that lives in the React component", () => { + const ep = scanFile( + "route.tsx", + ` + export async function loader() { + return json({}); + } + export default function Page() { + try { + JSON.parse(raw); + } catch { + return null; + } + return null; + } + ` + ); + expect(ep!.hasTryCatch).toBe(false); + expect(ep!.catchesNarrowly).toBe(false); + }); +}); From 3cb31ccb4b63dd1e54de740949a9253aef78335f Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Fri, 31 Jul 2026 14:47:58 +0100 Subject: [PATCH 011/106] fix(observability-map): stop request-context excusing routes that log nothing Applicability keyed off the presence of a failure-path log, so a route that kept its errors and recorded nothing was not-applicable rather than reported, and deleting a log line took a route out of the report. Every non-trivial entry point is now judged: no catch at all passes, since the error reaches the central handler, and a catch has to name whose failure it was. 87 of the 169 failures are routes that record nothing, which is what the old gate was hiding. Verified over the real tree that removing logging cannot help: re-running all four checks against every entry point with log calls deleted, failure-path logs deleted, and log fields stripped moves 63 verdicts, none of them for the better. error-classification now uses catchesNarrowly to excuse the guard that wraps a single parse. Applied on its own the field also excuses a one-statement try around a service call, which passes the design's own swallow fixture and four findings that were hand-read as real, so the exemption also asks that the body parse something. That clears the nine verbatim request.json guards and keeps the rest: 50 failures become 35. --- .../src/checks/errorClassification.ts | 22 ++++- .../src/checks/requestContext.ts | 33 +++++-- .../observability-map/test/checks.test.ts | 87 ++++++++++++++++++- 3 files changed, 129 insertions(+), 13 deletions(-) diff --git a/internal-packages/observability-map/src/checks/errorClassification.ts b/internal-packages/observability-map/src/checks/errorClassification.ts index cfa8e389a82..caf18b3cded 100644 --- a/internal-packages/observability-map/src/checks/errorClassification.ts +++ b/internal-packages/observability-map/src/checks/errorClassification.ts @@ -24,6 +24,12 @@ export const BUILDERS = new Set([ "dashboardAction", ]); +/** + * A parse: `await request.json()`, `JSON.parse(raw)`. The dot matters, it keeps Remix's `json({})` + * response helper out. Matched against `calleeTexts`, which carries the whole callee path. + */ +const PARSE_CALL = /(^|\.)JSON\.parse$|\.json$/; + export function usesBuilder(ep: EntryPoint): boolean { return ( (ep.loaderInitializerCallee !== null && BUILDERS.has(ep.loaderInitializerCallee)) || @@ -43,6 +49,16 @@ export function usesBuilder(ep: EntryPoint): boolean { * false means every catch in the entry point takes the same way out whatever was thrown. The * asymmetry that buys: one good catch alongside one swallow reads as a pass. This check misses * those rather than inventing them. + * + * `catchesNarrowly` excuses the guard that wraps one operation and answers for that operation: + * `try { body = await request.json() } catch { 400 }` neither branches nor rethrows and does not + * need to. On its own it excuses too much, because a one-statement try around an awaited service + * call is exactly as narrow as one around a parse: applied unencumbered it passes + * `try { await service.call(run) } catch { 500 }`, and it passes the design's own swallow fixture, + * `try { return await prisma.thing.findMany() } catch { return null }`. So the exemption also asks + * that the body parse something, which is the idiom the exemption was justified by. Over the real + * tree that combination clears the nine verbatim `request.json()` guards and holds back the four + * hand-read findings, see the task 5 report. */ export const errorClassification = { id: ID, @@ -50,13 +66,17 @@ export const errorClassification = { if (isTrivial(ep)) { return { id: ID, status: "not-applicable", detail: "trivial route" }; } - if (ep.hasTryCatch && !ep.catchRethrows && !ep.catchBranches) { + const guardsAParse = ep.catchesNarrowly && ep.calleeTexts.some((t) => PARSE_CALL.test(t)); + if (ep.hasTryCatch && !ep.catchRethrows && !ep.catchBranches && !guardsAParse) { return { id: ID, status: "fail", detail: "catches its errors and takes one way out regardless of what was thrown", }; } + if (guardsAParse) { + return { id: ID, status: "pass", detail: "guards a parse, not the handler" }; + } if (ep.catchRethrows || ep.catchBranches) { return { id: ID, status: "pass", detail: "the catch distinguishes what it caught" }; } diff --git a/internal-packages/observability-map/src/checks/requestContext.ts b/internal-packages/observability-map/src/checks/requestContext.ts index 248ec1374ff..c8694b5c168 100644 --- a/internal-packages/observability-map/src/checks/requestContext.ts +++ b/internal-packages/observability-map/src/checks/requestContext.ts @@ -1,4 +1,5 @@ import type { CheckResult, EntryPoint, LogCall } from "../types.js"; +import { isTrivial } from "../triviality.js"; const ID = "request-context"; @@ -25,18 +26,31 @@ function failurePathLogs(ep: EntryPoint): LogCall[] { * is not attributed either and gets no free pass here. An incident tells you which route and which * request; whose environment it was is only ever in the fields the route passes itself. * - * Applicable only where the route reports a failure itself, meaning it logs from inside a catch. A - * route that rethrows silently has handed the report to Sentry and there is nothing here to - * inspect. That gate has a perverse edge, noted in the task 5 report: deleting a log line moves a - * route from fail to not-applicable. + * Applicability turns on whether the route keeps its own failures, never on whether it logs. The + * first version made a route not-applicable when it had no failure-path log, which excused the very + * thing the check exists to find and meant deleting a log line took a route out of the report. + * Every non-trivial entry point is now judged: + * + * - no catch at all: pass. The error reaches the central handler, which is the intended path in + * this codebase, and the tenant it does not name there is a platform-level gap reported once + * rather than against each of 222 routes. + * - a catch: the route decided the outcome itself, so it has to say whose failure it was. + * + * Deleting a log call can then only make a verdict worse or leave it alone, and adding a catch + * without a report is a regression the check reports, which is the direction the incentive should + * run in. The one move that still improves a verdict is deleting the try/catch outright, and that + * hands the error back to the central handler, which `error-classification` also treats as correct. */ export const requestContext = { id: ID, run(ep: EntryPoint): CheckResult { - const logs = failurePathLogs(ep); - if (logs.length === 0) { - return { id: ID, status: "not-applicable", detail: "reports no failure of its own" }; + if (isTrivial(ep)) { + return { id: ID, status: "not-applicable", detail: "trivial route" }; } + if (!ep.hasTryCatch) { + return { id: ID, status: "pass", detail: "hands its failures to the central handler" }; + } + const logs = failurePathLogs(ep); const named = logs.find((l) => l.fields.some((f) => IDENTIFIER_FIELD.test(f))); if (named) { const fields = named.fields.filter((f) => IDENTIFIER_FIELD.test(f)); @@ -45,7 +59,10 @@ export const requestContext = { return { id: ID, status: "fail", - detail: "logs its failure without naming an environment, project, organization, run or user", + detail: + logs.length === 0 + ? "keeps its failures and records nothing about whose they were" + : "logs its failure without naming an environment, project, organization, run or user", }; }, }; diff --git a/internal-packages/observability-map/test/checks.test.ts b/internal-packages/observability-map/test/checks.test.ts index cf48a61a21c..97fb21991e9 100644 --- a/internal-packages/observability-map/test/checks.test.ts +++ b/internal-packages/observability-map/test/checks.test.ts @@ -132,6 +132,47 @@ describe("error-classification", () => { expect(r.status).toBe("not-applicable"); }); + // A narrow guard around one operation classifies an expected failure without needing to branch + // or rethrow. `catchesNarrowly` is what tells it apart from a handler-wide catch. + it("passes a narrow guard around a single parse", () => { + const r = run( + "error-classification", + "resources.timezone.ts", + `import { prisma } from "~/db.server"; + export async function action({ request }) { + let data; + try { data = await request.json(); } + catch { return json({ error: "Invalid JSON" }, { status: 400 }); } + const saved = await prisma.preference.create({ data }); + return json({ saved }); + }` + ); + expect(r.status).toBe("pass"); + }); + + // False positive fixture for the narrow rule: a narrow parse guard must not launder the broad + // handler catch sitting next to it. `catchesNarrowly` is false when any guarded try is broad. + it("still fails when a narrow guard sits beside a handler-wide swallow", () => { + const r = run( + "error-classification", + "api.v1.thing.ts", + `import { prisma } from "~/db.server"; + export async function action({ request }) { + let data; + try { data = await request.json(); } + catch { return json({ error: "Invalid JSON" }, { status: 400 }); } + try { + const thing = await prisma.thing.create({ data }); + const audit = await prisma.audit.create({ data: { thing: thing.id } }); + return json({ thing, audit }); + } catch (error) { + return json({ error: "Something went wrong" }, { status: 500 }); + } + }` + ); + expect(r.status).toBe("fail"); + }); + // False positive fixture: the only try/catch in the file belongs to the component. it("does not flag a route whose try/catch is in the React component", () => { const r = run( @@ -313,7 +354,7 @@ describe("request-context", () => { expect(r.status).toBe("fail"); }); - it("is not applicable to a route that logs nothing on its failure path", () => { + it("passes a route that hands its failures to the central handler", () => { const r = run( "request-context", "api.v1.q.ts", @@ -324,10 +365,10 @@ describe("request-context", () => { return json(await prisma.thing.findMany({ where: { environmentId: auth.environment.id } })); }` ); - expect(r.status).toBe("not-applicable"); + expect(r.status).toBe("pass"); }); - it("is not applicable when the only log sits outside the catch", () => { + it("fails a route that catches but only names an identifier outside the catch", () => { const r = run( "request-context", "api.v1.q.ts", @@ -338,7 +379,45 @@ describe("request-context", () => { try { return await prisma.thing.findMany(); } catch (e) { throw e; } }` ); - expect(r.status).toBe("not-applicable"); + expect(r.status).toBe("fail"); + }); + + // A route that catches and reports nothing at all is the case the log-based applicability gate + // used to excuse. It is a finding, not an exemption. + it("fails a route that catches and reports nothing", () => { + const r = run( + "request-context", + "api.v1.q.ts", + `import { prisma } from "~/db.server"; + export async function loader() { + try { return await prisma.thing.findMany(); } + catch (error) { return json({ error: "Internal Server Error" }, { status: 500 }); } + }` + ); + expect(r.status).toBe("fail"); + }); + + // The incentive fixture pair. The two routes differ by one line, the log call, and nothing else. + // Deleting that line must never improve the verdict or drop the route out of the report. + it("never improves a verdict when the log call is deleted", () => { + const body = (log: string) => + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function loader({ params }) { + try { return await prisma.thing.findMany(); } + catch (error) { ${log} throw error; } + }`; + + const withLog = run( + "request-context", + "api.v1.q.ts", + body(`logger.error("failed", { environmentId: params.envId, error });`) + ); + const withoutLog = run("request-context", "api.v1.q.ts", body("")); + + expect(withLog.status).toBe("pass"); + expect(withoutLog.status).toBe("fail"); + expect(withoutLog.status).not.toBe("not-applicable"); }); // False positive fixture: the component logs every identifier there is, inside its own catch. From ed52e17277d05ea4a81a7a026a63a805d6b5be36 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Fri, 31 Jul 2026 14:56:07 +0100 Subject: [PATCH 012/106] feat(observability-map): add scoring and aggregation with an audit gap and unmeasured tracking --- .../observability-map/src/score.ts | 116 ++++++++++++++++++ .../observability-map/test/score.test.ts | 109 ++++++++++++++++ 2 files changed, 225 insertions(+) create mode 100644 internal-packages/observability-map/src/score.ts create mode 100644 internal-packages/observability-map/test/score.test.ts diff --git a/internal-packages/observability-map/src/score.ts b/internal-packages/observability-map/src/score.ts new file mode 100644 index 00000000000..542b968177c --- /dev/null +++ b/internal-packages/observability-map/src/score.ts @@ -0,0 +1,116 @@ +import type { CheckResult, EntryPoint } from "./types.js"; +import { CHECKS, SCORED_CHECK_IDS } from "./checks/index.js"; +import { suppressedChecks } from "./suppression.js"; +import { familyOf, routePathOf, type Family } from "./adapters/remix.js"; +import { classifySensitivity } from "./sensitivity.js"; + +export type ScoredEntry = { + fileName: string; + routePath: string; + family: Family; + sensitive: boolean; + checks: CheckResult[]; + /** + * Whether at least one scored check (`SCORED_CHECK_IDS`, so never `audit-trail`) was applicable. + * `false` means nothing was measured here: the 100 in `score` is a vacuous default, not a + * finding, and `buildReport` excludes an unmeasured entry from every mean it computes so that + * default cannot inflate a figure nobody checked. + */ + measured: boolean; + /** Passed over applicable, across scored checks only. 100 when nothing applies. */ + score: number; +}; + +export type MapReport = { + global: number; + /** Entry points with at least one applicable scored check, i.e. those `global` is averaged over. */ + measured: number; + /** Entry points every scored check reported not-applicable for; excluded from `global`. */ + unmeasured: number; + byFamily: Record; + sensitiveCohort: { n: number; measured: number; mean: number }; + auditGap: { sensitiveMutations: number; withAudit: number }; + entries: ScoredEntry[]; + parseFailures: string[]; +}; + +export function scoreEntry(ep: EntryPoint): ScoredEntry { + const suppressed = suppressedChecks(ep.source); + const checks = CHECKS.map((c) => { + const result = c.run(ep); + const reason = suppressed.get(c.id); + // A suppression always lands on not-applicable, never on pass: suppressing a check must remove + // it from the score, not launder it into a point in the entry's favor. + return reason + ? { id: c.id, status: "not-applicable" as const, detail: `suppressed: ${reason}` } + : result; + }); + + const scored = checks.filter((c) => SCORED_CHECK_IDS.includes(c.id)); + const applicable = scored.filter((c) => c.status !== "not-applicable"); + const passed = applicable.filter((c) => c.status === "pass").length; + + return { + fileName: ep.fileName, + routePath: routePathOf(ep.fileName), + family: familyOf(ep.fileName), + sensitive: classifySensitivity(ep).sensitive, + checks, + measured: applicable.length > 0, + score: applicable.length === 0 ? 100 : Math.round((passed / applicable.length) * 100), + }; +} + +const mean = (xs: number[]) => + xs.length === 0 ? 100 : Math.round(xs.reduce((a, b) => a + b, 0) / xs.length); + +/** + * `n` is every entry point in the group; `mean` is taken over the measured subset only, so an + * entry point nothing applied to cannot drag a family's or cohort's figure toward 100. `measured` + * is reported alongside so a reader can tell a family scoring high because it is clean apart from + * a family scoring high because most of it was never measured. + */ +function groupStats(entries: ScoredEntry[]): { n: number; measured: number; mean: number } { + const measuredEntries = entries.filter((e) => e.measured); + return { + n: entries.length, + measured: measuredEntries.length, + mean: mean(measuredEntries.map((e) => e.score)), + }; +} + +export function buildReport(eps: EntryPoint[], parseFailures: string[]): MapReport { + const entries = eps.map(scoreEntry); + const measuredEntries = entries.filter((e) => e.measured); + + const byFamily: Record = {}; + for (const family of new Set(entries.map((e) => e.family))) { + byFamily[family] = groupStats(entries.filter((e) => e.family === family)); + } + + const sensitive = entries.filter((e) => e.sensitive); + + // audit-trail is excluded from the score (see checks/index.ts and scoreEntry above), and is + // reported here as its own architectural figure instead: how many sensitive mutations have an + // audit record, out of how many. Folding it into the score would tank every sensitive route on a + // gap that is the same everywhere, and bury the routes that have their own, fixable problems. + const auditApplicable = entries.filter((e) => + e.checks.some((c) => c.id === "audit-trail" && c.status !== "not-applicable") + ); + + return { + global: mean(measuredEntries.map((e) => e.score)), + measured: measuredEntries.length, + unmeasured: entries.length - measuredEntries.length, + byFamily, + sensitiveCohort: groupStats(sensitive), + auditGap: { + sensitiveMutations: auditApplicable.length, + withAudit: auditApplicable.filter((e) => + e.checks.some((c) => c.id === "audit-trail" && c.status === "pass") + ).length, + }, + entries, + parseFailures, + }; +} diff --git a/internal-packages/observability-map/test/score.test.ts b/internal-packages/observability-map/test/score.test.ts new file mode 100644 index 00000000000..ef27f5473c4 --- /dev/null +++ b/internal-packages/observability-map/test/score.test.ts @@ -0,0 +1,109 @@ +import { scoreEntry, buildReport } from "../src/score.js"; +import { scanFile } from "../src/scan.js"; + +const BUILDER = `import { createLoaderApiRoute } from "~/services/routeBuilders/apiBuilder.server"; +export const loader = createLoaderApiRoute({}, async () => new Response("ok"));`; + +const RAW = `import { prisma } from "~/db.server"; +export async function loader() { return prisma.thing.findMany(); }`; + +/** Trivial and not sensitive: every scored check reports not-applicable. */ +const TRIVIAL = `export const loader = () => new Response("ok");`; + +/** Not trivial (touches prisma, has a try/catch) and not sensitive: swallows every error and + * records nothing about whose failure it was, so both applicable scored checks fail. */ +const BUSY_AND_FAILING = `import { prisma } from "~/db.server"; +export async function loader() { + try { return await prisma.thing.findMany(); } catch (e) { return null; } +}`; + +describe("scoreEntry", () => { + it("scores a builder route 100", () => { + expect(scoreEntry(scanFile("api.v1.a.ts", BUILDER)!).score).toBe(100); + }); + + it("excludes audit-trail from the per-entry score", () => { + const scored = scoreEntry(scanFile("api.v1.auth.jwt.ts", BUILDER)!); + expect(scored.checks.some((c) => c.id === "audit-trail")).toBe(true); + expect(scored.score).toBe(100); + }); + + it("counts a suppressed check as not-applicable", () => { + const suppressed = `// obs-map-disable-next-line error-classification -- health probe +${RAW}`; + const scored = scoreEntry(scanFile("api.v1.b.ts", suppressed)!); + const ec = scored.checks.find((c) => c.id === "error-classification")!; + expect(ec.status).toBe("not-applicable"); + }); + + it("a suppressed-to-passing entry point does not read as unmeasured", () => { + // error-classification would fail here; auth-boundary is not-applicable (not sensitive). + // Suppressing the only applicable scored check must not be indistinguishable from an entry + // point nothing applies to: it is still reported, just not scored on that axis. + const suppressed = `// obs-map-disable-next-line error-classification -- health probe +${BUSY_AND_FAILING}`; + const scored = scoreEntry(scanFile("api.v1.c.ts", suppressed)!); + expect(scored.checks.find((c) => c.id === "error-classification")!.status).toBe( + "not-applicable" + ); + }); + + it("marks an entry point with nothing applicable as unmeasured, scored 100", () => { + const scored = scoreEntry(scanFile("resources.health.ts", TRIVIAL)!); + expect(scored.checks.every((c) => c.status === "not-applicable")).toBe(true); + expect(scored.measured).toBe(false); + expect(scored.score).toBe(100); + }); + + it("marks an entry point with at least one applicable scored check as measured", () => { + const scored = scoreEntry(scanFile("api.v1.busy.ts", BUSY_AND_FAILING)!); + expect(scored.measured).toBe(true); + }); +}); + +describe("buildReport", () => { + it("reports the audit gap separately from the score", () => { + const eps = [ + scanFile("api.v1.a.ts", BUILDER)!, + scanFile( + "api.v1.auth.tokens.ts", + `import { prisma } from "~/db.server"; + export async function action() { return prisma.token.create({ data: {} }); }` + )!, + ]; + const report = buildReport(eps, []); + expect(report.auditGap.sensitiveMutations).toBe(1); + expect(report.auditGap.withAudit).toBe(0); + expect(report.global).toBeGreaterThan(0); + }); + + it("records parse failures", () => { + const report = buildReport([scanFile("api.v1.a.ts", BUILDER)!], ["broken.ts"]); + expect(report.parseFailures).toEqual(["broken.ts"]); + }); + + it("excludes an unmeasured entry point from the global mean", () => { + const trivial = scanFile("resources.health.ts", TRIVIAL)!; + const busy = scanFile("api.v1.busy.ts", BUSY_AND_FAILING)!; + + const report = buildReport([trivial, busy], []); + + expect(report.measured).toBe(1); + expect(report.unmeasured).toBe(1); + // If the trivial entry's vacuous 100 counted toward the mean, the global score would be 50 + // instead of matching the one entry that was actually measured. + expect(report.global).toBe(scoreEntry(busy).score); + }); + + it("excludes an unmeasured entry point from its family mean too", () => { + const trivial = scanFile("resources.health.ts", TRIVIAL)!; + const busy = scanFile("resources.busy.ts", BUSY_AND_FAILING)!; + + const report = buildReport([trivial, busy], []); + + const family = report.byFamily["resources"]!; + expect(family.n).toBe(2); + expect(family.measured).toBe(1); + expect(family.mean).toBe(scoreEntry(busy).score); + }); +}); From f3b582fd7cf3ef76851f2f5142b7df76587fe7f8 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Fri, 31 Jul 2026 15:06:05 +0100 Subject: [PATCH 013/106] feat(observability-map): terminal and json reports --- .../observability-map/src/report/json.ts | 5 + .../observability-map/src/report/terminal.ts | 76 ++++++++++++ .../observability-map/test/report.test.ts | 112 ++++++++++++++++++ 3 files changed, 193 insertions(+) create mode 100644 internal-packages/observability-map/src/report/json.ts create mode 100644 internal-packages/observability-map/src/report/terminal.ts create mode 100644 internal-packages/observability-map/test/report.test.ts diff --git a/internal-packages/observability-map/src/report/json.ts b/internal-packages/observability-map/src/report/json.ts new file mode 100644 index 00000000000..8a36465abd5 --- /dev/null +++ b/internal-packages/observability-map/src/report/json.ts @@ -0,0 +1,5 @@ +import type { MapReport } from "../score.js"; + +export function renderJson(report: MapReport): string { + return JSON.stringify(report, null, 2); +} diff --git a/internal-packages/observability-map/src/report/terminal.ts b/internal-packages/observability-map/src/report/terminal.ts new file mode 100644 index 00000000000..c5d19e491f9 --- /dev/null +++ b/internal-packages/observability-map/src/report/terminal.ts @@ -0,0 +1,76 @@ +import type { MapReport, ScoredEntry } from "../score.js"; +import { SCORED_CHECK_IDS } from "../checks/index.js"; + +const bar = (score: number) => { + const filled = Math.round(score / 10); + return "▰".repeat(filled) + "▱".repeat(10 - filled); +}; + +/** + * Failing checks that actually feed `score`. `audit-trail` is deliberately excluded here: it is + * excluded from the score for the same reason (see `score.ts`), and every sensitive mutation fails + * it today, so folding it in would flood this list with the same finding repeated 52 times instead + * of the fixable, route-specific gaps the list exists to surface. That gap is reported once, as + * `AUDIT`, below. + */ +const scoredFailures = (e: ScoredEntry) => + e.checks.filter((c) => SCORED_CHECK_IDS.includes(c.id) && c.status === "fail"); + +export function renderTerminal(report: MapReport): string { + const lines: string[] = []; + + lines.push( + `score ${report.global}/100 ${report.measured} measured, ${report.unmeasured} unmeasured of ${report.entries.length} entry points` + ); + lines.push(""); + lines.push("COVERAGE"); + for (const [family, stats] of Object.entries(report.byFamily).sort((a, b) => b[1].n - a[1].n)) { + lines.push( + ` ${family.padEnd(12)} ${bar(stats.mean)} ${String(stats.mean).padStart(3)} ${stats.measured}/${stats.n} entry points` + ); + } + lines.push( + ` ${"sensitive".padEnd(12)} ${bar(report.sensitiveCohort.mean)} ${String( + report.sensitiveCohort.mean + ).padStart(3)} ${report.sensitiveCohort.measured}/${report.sensitiveCohort.n} entry points` + ); + + lines.push(""); + const { sensitiveMutations, withAudit } = report.auditGap; + lines.push( + `AUDIT ${withAudit} of ${sensitiveMutations} sensitive mutations record an actor. ` + + `No audit helper exists in the webapp.` + ); + + const worst = report.entries + .filter((e) => scoredFailures(e).length > 0) + .sort( + (a, b) => + Number(b.sensitive) - Number(a.sensitive) || + a.score - b.score || + a.fileName.localeCompare(b.fileName) + ); + + lines.push(""); + lines.push("FIX FIRST"); + for (const e of worst.slice(0, 3)) { + const marks = e.sensitive ? " (sensitive)" : ""; + lines.push( + ` ${e.routePath}${marks} - ${scoredFailures(e) + .map((c) => c.id) + .join(", ")}` + ); + lines.push(` ${e.fileName}`); + } + if (worst.length > 3) { + lines.push(""); + lines.push(`THEN ${worst.length - 3} more with gaps`); + } + + lines.push(""); + lines.push(`already solid: ${report.entries.length - worst.length}`); + if (report.parseFailures.length > 0) { + lines.push(`parse failures (excluded from the score): ${report.parseFailures.join(", ")}`); + } + return lines.join("\n"); +} diff --git a/internal-packages/observability-map/test/report.test.ts b/internal-packages/observability-map/test/report.test.ts new file mode 100644 index 00000000000..be977d24d8f --- /dev/null +++ b/internal-packages/observability-map/test/report.test.ts @@ -0,0 +1,112 @@ +import { renderTerminal } from "../src/report/terminal.js"; +import { renderJson } from "../src/report/json.js"; +import { buildReport } from "../src/score.js"; +import { scanFile } from "../src/scan.js"; + +const report = () => + buildReport( + [ + scanFile( + "api.v1.a.ts", + `import { createLoaderApiRoute } from "~/services/routeBuilders/apiBuilder.server"; + export const loader = createLoaderApiRoute({}, async () => new Response("ok"));` + )!, + scanFile( + "api.v1.auth.tokens.ts", + `import { prisma } from "~/db.server"; + export async function action() { return prisma.token.create({ data: {} }); }` + )!, + ], + ["broken.ts"] + ); + +describe("renderTerminal", () => { + it("shows the global score, the audit gap and the fix list", () => { + const out = renderTerminal(report()); + expect(out).toContain("COVERAGE"); + expect(out).toContain("FIX FIRST"); + expect(out).toContain("audit"); + expect(out).toContain("/api/v1/auth/tokens"); + }); + + it("surfaces parse failures so the denominator is not silently wrong", () => { + expect(renderTerminal(report())).toContain("broken.ts"); + }); + + it("shows the unmeasured count so 415 vs 427 is not silently confusing", () => { + const trivial = scanFile( + "resources.health.ts", + `export const loader = () => new Response("ok");` + )!; + const out = renderTerminal(buildReport([trivial], [])); + expect(out).toContain("1 unmeasured"); + }); + + it("orders FIX FIRST by sensitivity first, then ascending score, and excludes audit-only gaps", () => { + // Sensitive, score 0: both applicable scored checks fail (auth-boundary, error-classification, + // request-context all fail because the catch swallows without naming who it happened to). + const sensitiveZero = scanFile( + "api.v1.envvars.ts", + `import { prisma } from "~/db.server"; + export async function action() { + try { + return await prisma.envVar.update({ where: {}, data: {} }); + } catch (e) { + return null; + } + }` + )!; + + // Sensitive, score 67: only auth-boundary fails (no guard called), the other two pass because + // there is no try/catch to fumble. + const sensitiveSixtySeven = scanFile( + "api.v1.auth.tokens.ts", + `import { prisma } from "~/db.server"; + export async function action() { return prisma.token.create({ data: {} }); }` + )!; + + // Not sensitive, score 0: worse score than sensitiveSixtySeven, but must still sort after both + // sensitive entries because sensitivity outranks raw score. + const notSensitiveZero = scanFile( + "resources.busy.ts", + `import { prisma } from "~/db.server"; + export async function loader() { + try { return await prisma.thing.findMany(); } catch (e) { return null; } + }` + )!; + + // Sensitive mutation, scored checks all pass (guarded, no try/catch): the only gap is + // audit-trail, which is a headline figure, not a per-route fix-list item. Must not appear. + const sensitiveAuditOnly = scanFile( + "api.v1.billing.ts", + `import { prisma } from "~/db.server"; + import { requireUserId } from "~/services/session.server"; + export async function action({ request }: { request: Request }) { + const userId = requireUserId(request); + return prisma.billing.update({ where: { userId }, data: {} }); + }` + )!; + + const out = renderTerminal( + buildReport([sensitiveSixtySeven, sensitiveZero, notSensitiveZero, sensitiveAuditOnly], []) + ); + + const fixFirst = out.slice(out.indexOf("FIX FIRST"), out.indexOf("already solid")); + const idxZero = fixFirst.indexOf("api.v1.envvars.ts"); + const idxSixtySeven = fixFirst.indexOf("api.v1.auth.tokens.ts"); + const idxNotSensitive = fixFirst.indexOf("resources.busy.ts"); + + expect(idxZero).toBeGreaterThan(-1); + expect(idxSixtySeven).toBeGreaterThan(idxZero); + expect(idxNotSensitive).toBeGreaterThan(idxSixtySeven); + expect(fixFirst).not.toContain("api.v1.billing.ts"); + }); +}); + +describe("renderJson", () => { + it("round-trips to an object carrying the score and entries", () => { + const parsed = JSON.parse(renderJson(report())); + expect(typeof parsed.global).toBe("number"); + expect(Array.isArray(parsed.entries)).toBe(true); + }); +}); From 718397645e3777f0f01e884bb52a3651c9e7b3a1 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Fri, 31 Jul 2026 15:19:54 +0100 Subject: [PATCH 014/106] fix(observability-map): stop auth-boundary accusing routes it cannot see into The rendered fix list opened with three auth-boundary findings and all three were wrong. Two delegate to clearImpersonation, which authenticates and writes an audit row in a file the scanner never opens, and the third is a redirect stub flagged only because its path contains billing. A fail here says the route does privileged work with no guard, which is only supportable when the body is where a guard would have to be. A trivial body cannot hold a visible privileged operation, by the triviality rule's own definition, so either nothing privileged happens or the work sits behind an import along with any guard. Those now report not-applicable with a detail saying the guard could not be verified, rather than failing. Signature checks also count as guards now, which clears the HMAC-authenticated waitpoint callback. Three findings remain and all three are genuinely unauthenticated. request-context stops treating a parse guard as the route taking over its failure path, through the same shared reading of catchesNarrowly that error-classification uses. Re-ran the incentive sweep after the change: 57 verdicts move when logging is removed, none for the better. --- .../src/checks/authBoundary.ts | 53 ++++++-- .../src/checks/errorClassification.ts | 12 +- .../src/checks/requestContext.ts | 16 ++- .../observability-map/test/checks.test.ts | 128 ++++++++++++++++++ 4 files changed, 195 insertions(+), 14 deletions(-) diff --git a/internal-packages/observability-map/src/checks/authBoundary.ts b/internal-packages/observability-map/src/checks/authBoundary.ts index 11030d1d2d8..dfe38ed1fed 100644 --- a/internal-packages/observability-map/src/checks/authBoundary.ts +++ b/internal-packages/observability-map/src/checks/authBoundary.ts @@ -1,24 +1,54 @@ import type { CheckResult, EntryPoint } from "../types.js"; import { classifySensitivity } from "../sensitivity.js"; +import { isTrivial } from "../triviality.js"; import { usesBuilder } from "./errorClassification.js"; const ID = "auth-boundary"; /** - * The two shapes the webapp's guards take: `requireUserId`, `requireAdminApiRequest`, - * `authenticateApiRequest`, `authenticateProjectApiKey`. Matched against `calleeNames`, which is + * What a guard looks like from the body. All three are matched against `calleeNames`, which is * scoped to the loader/action bodies and follows one hop into a same-file helper, so a guard the * route only imports and never calls does not count. */ -const GUARD = /^(require|authenticate)/; +const GUARDS = [ + /** `requireUserId`, `requireAdminApiRequest`, `authenticateApiRequest`. */ + /^(require|authenticate)/, + /** + * Proof of possession. A callback URL carrying an HMAC is authenticated by checking that HMAC: + * `verifyHttpCallbackHash`, `verifyWebhookSignature`. Narrowed to the signature words so that an + * unrelated `verifyEmail` does not read as an auth boundary. + */ + /^verify.*(Hash|Hmac|Signature|Webhook|Callback|Token)/, + /** `resolveAuthenticatedEnv`: the name says the identity was established. */ + /Authenticated/, +]; /** * Whether a route that handles credentials, tokens or money checks who is asking. * - * The design matched `importedNames` as well as `calleeNames`. Across the 67 sensitive entry points - * in the real tree that widening changes nothing: every route with a `require*` import calls it - * from the body too. So the file-wide half only ever stood to hand out a pass for a dead import, - * and it is gone. + * A fail here is an accusation, and it is only supportable when the body is the place a guard + * would have to be. That holds when the route does its privileged work in the open: reads the + * request, queries the datastore, mints the token. It does not hold for a trivial body, so those + * are reported not-applicable rather than failed. + * + * The reasoning is the triviality rule's own definition rather than a convenience. A trivial body + * has three statements or fewer, three calls or fewer, no try/catch, no builder, and no mention of + * prisma, redis, fetch or the engine anywhere in its source. It therefore cannot contain a visible + * privileged operation. Either it does nothing privileged at all, like the `/orgs/:slug/billing` + * redirect stub, or the privileged work sits behind an import, like `clearImpersonation`, which + * authenticates and writes an audit row in `app/models/admin.server.ts`. In the second case the + * guard is in the same unopened file as the work. Absence of evidence, and reporting it as a + * finding puts a wrong answer at the top of the fix list. + * + * This is not the rule `request-context` uses, deliberately. There the thing being looked for, a + * field on a log call inside a catch, would be in the body if it existed at all, because the catch + * is in the body. Absence of a log is evidence. Here the thing being looked for guards work that + * is not in the body either, so its absence proves nothing. The test that separates them: would + * this evidence necessarily be visible in the body if it existed? + * + * The design also matched `importedNames`. Across the 67 sensitive entry points that widening + * changes nothing, every route with a `require*` import calls it from the body too, so the + * file-wide half only ever stood to hand out a pass for a dead import. It is gone. */ export const authBoundary = { id: ID, @@ -30,9 +60,16 @@ export const authBoundary = { if (usesBuilder(ep)) { return { id: ID, status: "pass", detail: "authenticated by the builder" }; } - if (ep.calleeNames.some((n) => GUARD.test(n))) { + if (ep.calleeNames.some((n) => GUARDS.some((g) => g.test(n)))) { return { id: ID, status: "pass", detail: "guarded in the body" }; } + if (isTrivial(ep)) { + return { + id: ID, + status: "not-applicable", + detail: "cannot verify: no privileged work in the body, any guard is behind an import", + }; + } return { id: ID, status: "fail", diff --git a/internal-packages/observability-map/src/checks/errorClassification.ts b/internal-packages/observability-map/src/checks/errorClassification.ts index caf18b3cded..7320e60d092 100644 --- a/internal-packages/observability-map/src/checks/errorClassification.ts +++ b/internal-packages/observability-map/src/checks/errorClassification.ts @@ -30,6 +30,16 @@ export const BUILDERS = new Set([ */ const PARSE_CALL = /(^|\.)JSON\.parse$|\.json$/; +/** + * Whether every catch in the entry point guards a parse and nothing wider. Both checks read the + * field the same way: a guard around one parse is not the route taking charge of its failures, + * so `error-classification` does not call it a swallow and `request-context` does not ask it to + * name a tenant. + */ +export function guardsOnlyAParse(ep: EntryPoint): boolean { + return ep.catchesNarrowly && ep.calleeTexts.some((t) => PARSE_CALL.test(t)); +} + export function usesBuilder(ep: EntryPoint): boolean { return ( (ep.loaderInitializerCallee !== null && BUILDERS.has(ep.loaderInitializerCallee)) || @@ -66,7 +76,7 @@ export const errorClassification = { if (isTrivial(ep)) { return { id: ID, status: "not-applicable", detail: "trivial route" }; } - const guardsAParse = ep.catchesNarrowly && ep.calleeTexts.some((t) => PARSE_CALL.test(t)); + const guardsAParse = guardsOnlyAParse(ep); if (ep.hasTryCatch && !ep.catchRethrows && !ep.catchBranches && !guardsAParse) { return { id: ID, diff --git a/internal-packages/observability-map/src/checks/requestContext.ts b/internal-packages/observability-map/src/checks/requestContext.ts index c8694b5c168..116df0cc193 100644 --- a/internal-packages/observability-map/src/checks/requestContext.ts +++ b/internal-packages/observability-map/src/checks/requestContext.ts @@ -1,5 +1,6 @@ import type { CheckResult, EntryPoint, LogCall } from "../types.js"; import { isTrivial } from "../triviality.js"; +import { guardsOnlyAParse } from "./errorClassification.js"; const ID = "request-context"; @@ -31,10 +32,13 @@ function failurePathLogs(ep: EntryPoint): LogCall[] { * thing the check exists to find and meant deleting a log line took a route out of the report. * Every non-trivial entry point is now judged: * - * - no catch at all: pass. The error reaches the central handler, which is the intended path in - * this codebase, and the tenant it does not name there is a platform-level gap reported once - * rather than against each of 222 routes. - * - a catch: the route decided the outcome itself, so it has to say whose failure it was. + * - no catch at all, or nothing but a parse guard: pass. The route's own work still throws past it + * to the central handler, which is the intended path in this codebase, and the tenant that + * handler does not name is a platform-level gap reported once rather than against each route. A + * `try { body = await request.json() } catch { 400 }` is not a route taking over its failure + * path, and reading it as one put false positives at the top of the first rendered report. + * - a catch that covers the route's work: it decided the outcome itself, so it has to say whose + * failure it was. * * Deleting a log call can then only make a verdict worse or leave it alone, and adding a catch * without a report is a regression the check reports, which is the direction the incentive should @@ -47,7 +51,9 @@ export const requestContext = { if (isTrivial(ep)) { return { id: ID, status: "not-applicable", detail: "trivial route" }; } - if (!ep.hasTryCatch) { + // A route whose only catch guards a parse still throws its own work past it to the central + // handler. Same reading error-classification gives the field. + if (!ep.hasTryCatch || guardsOnlyAParse(ep)) { return { id: ID, status: "pass", detail: "hands its failures to the central handler" }; } const logs = failurePathLogs(ep); diff --git a/internal-packages/observability-map/test/checks.test.ts b/internal-packages/observability-map/test/checks.test.ts index 97fb21991e9..b04f7daa183 100644 --- a/internal-packages/observability-map/test/checks.test.ts +++ b/internal-packages/observability-map/test/checks.test.ts @@ -242,6 +242,73 @@ describe("auth-boundary", () => { expect(r.status).toBe("not-applicable"); }); + // False positive fixture for the delegated guard. `clearImpersonation` authenticates and writes + // an audit row, in `app/models/admin.server.ts`, which the scanner cannot open. The body shows no + // privileged work either, so there is nothing here to accuse: absence of evidence, not evidence + // of absence. + it("does not flag a sensitive route that hands its work to an imported helper", () => { + const r = run( + "auth-boundary", + "resources.impersonation.ts", + `import { clearImpersonation } from "~/models/admin.server"; + export async function action({ request }) { + return clearImpersonation(request, "/admin"); + }` + ); + expect(r.status).toBe("not-applicable"); + expect(r.detail).toMatch(/verif/i); + }); + + it("does not flag a sensitive redirect stub", () => { + const r = run( + "auth-boundary", + "orgs.$organizationSlug.billing.ts", + `import { redirect } from "@remix-run/server-runtime"; + import { OrganizationParamsSchema, v3BillingPath } from "~/utils/pathBuilder"; + export const loader = async ({ params }) => { + const { organizationSlug } = OrganizationParamsSchema.parse(params); + return redirect(v3BillingPath({ slug: organizationSlug })); + };` + ); + expect(r.status).toBe("not-applicable"); + }); + + // The gate must not swallow the real thing: a body doing its own privileged work, unguarded. + it("still fails a sensitive route whose visible body does the work unguarded", () => { + const r = run( + "auth-boundary", + "api.v1.token.ts", + `import { prisma } from "~/db.server"; + import { createPersonalAccessToken } from "~/services/personalAccessToken.server"; + export async function action({ request }) { + const body = await request.json(); + const code = await prisma.authorizationCode.findFirst({ where: { code: body.code } }); + if (!code) return json({ error: "Not found" }, { status: 404 }); + const token = await createPersonalAccessToken(code.userId); + return json({ token }); + }` + ); + expect(r.status).toBe("fail"); + }); + + // Possession of a valid signature is the auth boundary for a callback URL. + it("passes a sensitive callback guarded by a signature check", () => { + const r = run( + "auth-boundary", + "api.v1.waitpoints.tokens.$waitpointFriendlyId.callback.$hash.ts", + `import { verifyHttpCallbackHash } from "~/services/httpCallback.server"; + import { prisma } from "~/db.server"; + export async function action({ request, params }) { + const waitpoint = await prisma.waitpoint.findFirst({ where: { id: params.id } }); + if (!verifyHttpCallbackHash(params.hash, waitpoint)) { + return json({ error: "Invalid" }, { status: 401 }); + } + return json({ ok: true }); + }` + ); + expect(r.status).toBe("pass"); + }); + // False positive fixture: the guard sits one hop away, in a same-file helper. it("does not flag a sensitive route whose guard is in a same-file helper", () => { const r = run( @@ -397,6 +464,67 @@ describe("request-context", () => { expect(r.status).toBe("fail"); }); + // A guard around a parse is not the route taking over its failure path: whatever its real work + // throws still reaches the central handler. Same reading error-classification gives the field. + it("passes a route whose only catch guards a parse", () => { + const r = run( + "request-context", + "api.v1.q.ts", + `import { prisma } from "~/db.server"; + export async function loader({ request }) { + let body; + try { body = await request.json(); } + catch { return json({ error: "Invalid JSON" }, { status: 400 }); } + return json(await prisma.thing.findMany({ where: body })); + }` + ); + expect(r.status).toBe("pass"); + }); + + // Known false positive, named in the task 5 report. The only catch here guards `new URL()`, a + // constructor, which never reaches `calleeTexts`, so the parse cannot be seen and the route is + // judged as though it kept its failures. Fixing it needs the scanner, not the check. + it("fails a route whose only catch guards a constructor parse", () => { + const r = run( + "request-context", + "_app.@.orgs.$organizationSlug.$.tsx", + `import { prisma } from "~/db.server"; + function refererOrigin(request) { + const referer = request.headers.get("referer"); + try { return new URL(referer).origin; } + catch { return undefined; } + } + export async function loader({ request }) { + const origin = refererOrigin(request); + return typedjson({ origin, things: await prisma.thing.findMany() }); + }` + ); + expect(r.status).toBe("fail"); + }); + + it("still judges a route with a handler-wide catch beside a narrow guard", () => { + const r = run( + "request-context", + "api.v1.q.ts", + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function loader({ request }) { + let body; + try { body = await request.json(); } + catch { return json({ error: "Invalid JSON" }, { status: 400 }); } + try { + const rows = await prisma.thing.findMany({ where: body }); + const count = await prisma.thing.count(); + return json({ rows, count }); + } catch (error) { + logger.error("failed", { error }); + return json({ error: "Internal Server Error" }, { status: 500 }); + } + }` + ); + expect(r.status).toBe("fail"); + }); + // The incentive fixture pair. The two routes differ by one line, the log call, and nothing else. // Deleting that line must never improve the verdict or drop the route out of the report. it("never improves a verdict when the log call is deleted", () => { From a8413e1840c79a45d3cdace392804d4085373572 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Fri, 31 Jul 2026 15:26:34 +0100 Subject: [PATCH 015/106] feat(observability-map): record catch evidence per clause Whole-entry catch booleans collapse when a route has a narrow parse guard and a broad handler catch, so a check cannot reason about either. 17 route entry points are in that state. - catches: one CatchEvidence per catch clause in the bodies and the one-hop helpers, carrying narrow, rethrows, branches, guardsParse and the try block statement count - guardsParse reads constructors as well as parse calls, so new URL(referer) is visible without touching calleeTexts, which other checks read - a try/finally now yields an empty catches list. hasTryCatch keeps its meaning, a try appears, so ask catches.length whether anything is caught - catchRethrows, catchBranches and catchesNarrowly are now derived from the list and keep their values on all 427 route entry points 242 catch clauses over 189 entry points, 9 of them swallowing outright. Clears all three false positives at the top of the report. --- .../observability-map/src/scan.ts | 55 +++-- .../observability-map/src/types.ts | 33 ++- .../observability-map/test/scan.test.ts | 221 ++++++++++++++++++ 3 files changed, 294 insertions(+), 15 deletions(-) diff --git a/internal-packages/observability-map/src/scan.ts b/internal-packages/observability-map/src/scan.ts index 1cb69bcdc27..0028d55bab8 100644 --- a/internal-packages/observability-map/src/scan.ts +++ b/internal-packages/observability-map/src/scan.ts @@ -1,7 +1,7 @@ import ts from "typescript"; import { readdirSync, readFileSync } from "node:fs"; import { join } from "node:path"; -import type { EntryPoint, LogCall } from "./types.js"; +import type { CatchEvidence, EntryPoint, LogCall } from "./types.js"; /** Thrown by `scanFile` when the source does not parse cleanly. */ export class ParseFailureError extends Error { @@ -113,6 +113,32 @@ function objectArgumentFields(call: ts.CallExpression): { found: boolean; fields */ const NARROW_TRY_STATEMENTS = 2; +/** + * Calls that turn input into a value and throw when it is malformed. `parse`/`safeParse` cover + * `JSON.parse` and the zod schemas. `.json` has to be a member call, because a bare `json(...)` is + * the remix response helper, which every route calls and which parses nothing. + */ +const PARSE_CALLEE = /(^|\.)(parse|safeParse|parseAsync|safeParseAsync|decode)$|\.json$/; + +/** + * Whether the guarded region parses or constructs something. A constructor counts: `new URL(x)` is + * the commonest parse guard in the tree, and constructors are absent from `calleeTexts`. + */ +function guardsParse(tryBlock: ts.Block): boolean { + let found = false; + const visit = (node: ts.Node) => { + if (found) return; + if (ts.isNewExpression(node)) found = true; + if (ts.isCallExpression(node)) { + const text = calleeText(node.expression) ?? calleeName(node.expression); + if (text !== null && PARSE_CALLEE.test(text)) found = true; + } + ts.forEachChild(node, visit); + }; + visit(tryBlock); + return found; +} + /** What a catch clause does with the error, beyond the fact that it caught one. */ function catchClauseEvidence(clause: ts.CatchClause): { rethrows: boolean; branches: boolean } { let rethrows = false; @@ -462,10 +488,7 @@ export function scanFile(fileName: string, source: string): EntryPoint | null { let statementCount = 0; let hasTryCatch = false; - let catchRethrows = false; - let catchBranches = false; - let catchClauseCount = 0; - let broadCatch = false; + const catches: CatchEvidence[] = []; const calleeNames: string[] = []; const calleeTexts: string[] = []; const logCalls: LogCall[] = []; @@ -485,15 +508,19 @@ export function scanFile(fileName: string, source: string): EntryPoint | null { if (ts.isTryStatement(node)) { hasTryCatch = true; if (node.catchClause) { - catchClauseCount += 1; - if (countStatements(node.tryBlock.statements) > NARROW_TRY_STATEMENTS) broadCatch = true; + const tryStatementCount = countStatements(node.tryBlock.statements); + const clause = catchClauseEvidence(node.catchClause); + catches.push({ + narrow: tryStatementCount <= NARROW_TRY_STATEMENTS, + rethrows: clause.rethrows, + branches: clause.branches, + guardsParse: guardsParse(node.tryBlock), + tryStatementCount, + }); } } if (ts.isCatchClause(node)) { - const evidence = catchClauseEvidence(node); - catchRethrows ||= evidence.rethrows; - catchBranches ||= evidence.branches; ts.forEachChild(node, (child) => visit(child, true)); return; } @@ -546,9 +573,11 @@ export function scanFile(fileName: string, source: string): EntryPoint | null { calleeNames, calleeTexts, hasTryCatch, - catchRethrows, - catchBranches, - catchesNarrowly: catchClauseCount > 0 && !broadCatch, + catches, + // Kept as aggregates of `catches` so the checks can migrate one at a time. + catchRethrows: catches.some((c) => c.rethrows), + catchBranches: catches.some((c) => c.branches), + catchesNarrowly: catches.length > 0 && catches.every((c) => c.narrow), logCalls, statementCount, }; diff --git a/internal-packages/observability-map/src/types.ts b/internal-packages/observability-map/src/types.ts index 5d7dc2292ae..2e2401c87a4 100644 --- a/internal-packages/observability-map/src/types.ts +++ b/internal-packages/observability-map/src/types.ts @@ -6,6 +6,28 @@ export type CheckResult = { detail?: string; }; +/** + * One catch clause in a loader/action body, or in a same-file helper the body calls. Per clause + * rather than per entry point, so a narrow parse guard sitting beside a broad handler catch stays + * legible instead of collapsing into one boolean. + */ +export type CatchEvidence = { + /** The guarded try block holds at most two statements: one operation, not the handler. */ + narrow: boolean; + /** The clause contains a `throw`. */ + rethrows: boolean; + /** The clause branches on the error: an `if`, a `switch`, or an `instanceof`. */ + branches: boolean; + /** + * The guarded region parses or constructs something: `JSON.parse`, `request.json()`, a zod + * `parse`/`safeParse`, or any `new X(...)`. Constructors are counted here because `new URL(x)` is + * the commonest parse guard in the tree and constructors never appear in `calleeTexts`. + */ + guardsParse: boolean; + /** Statements in the guarded try block, counted as `statementCount` counts them. */ + tryStatementCount: number; +}; + /** A logging call made from a loader/action body, or from a same-file helper the body calls. */ export type LogCall = { /** Full callee path, e.g. `logger.error`. */ @@ -36,11 +58,18 @@ export type EntryPoint = { * something unnameable (`new PromptService().createOverride`) falls back to the bare name. */ calleeTexts: string[]; - /** Whether a `try` appears in the loader/action bodies, or in a same-file helper they call. */ + /** + * Whether a `try` appears in the loader/action bodies, or in a same-file helper they call. Note + * that this says a `try`, not a catch: a `try`/`finally` sets it while `catches` stays empty and + * every catch-shaped field stays false. Read `catches.length` to ask whether anything is caught. + */ hasTryCatch: boolean; + /** One entry per catch clause in those bodies, in source order. */ + catches: CatchEvidence[]; /** * Whether any catch clause in those bodies contains a `throw`. A catch that rethrows has decided - * the error is not its to answer, which is a different act from swallowing it. + * the error is not its to answer, which is a different act from swallowing it. Aggregate of + * `catches`, kept so existing consumers keep working. */ catchRethrows: boolean; /** diff --git a/internal-packages/observability-map/test/scan.test.ts b/internal-packages/observability-map/test/scan.test.ts index 943c4a56f3e..e4e2388fd35 100644 --- a/internal-packages/observability-map/test/scan.test.ts +++ b/internal-packages/observability-map/test/scan.test.ts @@ -1025,3 +1025,224 @@ describe("scanFile: narrow catches", () => { expect(ep!.catchesNarrowly).toBe(false); }); }); + +describe("scanFile: per-catch evidence", () => { + it("records one entry per catch clause, keeping a narrow guard distinct from a broad catch", () => { + const ep = scanFile( + "two-catches.ts", + ` + export async function action({ request }) { + let body; + try { + body = await request.json(); + } catch { + return json({}, { status: 400 }); + } + try { + const run = await find(body.id); + const updated = await update(run); + await notify(updated); + return json(updated); + } catch (e) { + return json({}, { status: 500 }); + } + } + ` + ); + expect(ep!.catches).toHaveLength(2); + expect(ep!.catches[0]).toEqual({ + narrow: true, + rethrows: false, + branches: false, + guardsParse: true, + tryStatementCount: 1, + }); + expect(ep!.catches[1]).toMatchObject({ + narrow: false, + guardsParse: false, + tryStatementCount: 4, + }); + // The aggregate still collapses, which is why the per-catch list exists. + expect(ep!.catchesNarrowly).toBe(false); + }); + + it("leaves catches empty for a try/finally with no catch clause", () => { + const ep = scanFile( + "runs-replication.status.ts", + ` + export async function loader() { + const redis = createRedis(); + try { + for (const source of sources) { + const exists = await redis.exists(source.slotName); + leaders.set(source.id, exists === 1); + } + } finally { + await redis.quit(); + } + return json({}); + } + ` + ); + expect(ep!.catches).toEqual([]); + // hasTryCatch keeps its meaning: a `try` appears. Nothing is caught here. + expect(ep!.hasTryCatch).toBe(true); + expect(ep!.catchRethrows).toBe(false); + expect(ep!.catchBranches).toBe(false); + expect(ep!.catchesNarrowly).toBe(false); + }); + + it("sees a constructor as a guarded parse", () => { + const ep = scanFile( + "_app.@.orgs.$organizationSlug.$.tsx", + ` + function refererOrigin(request) { + const referer = request.headers.get("referer"); + if (!referer) return undefined; + try { + return new URL(referer).origin; + } catch { + return undefined; + } + } + export async function action({ request }) { + const origin = refererOrigin(request); + return json({ origin }); + } + ` + ); + expect(ep!.catches).toHaveLength(1); + expect(ep!.catches[0]!.guardsParse).toBe(true); + expect(ep!.catches[0]!.narrow).toBe(true); + }); + + it("sees a parse in a try that grew past the narrowness threshold", () => { + const ep = scanFile( + "admin.api.v1.orgs.$organizationId.stream-basin.ts", + ` + export async function action({ request }) { + let parsed; + try { + const text = await request.text(); + const raw = text.length > 0 ? JSON.parse(text) : {}; + const result = BodySchema.safeParse(raw); + if (!result.success) { + return json({ ok: false }, { status: 400 }); + } + parsed = result.data; + } catch { + return json({ ok: false, error: "Invalid JSON body" }, { status: 400 }); + } + return json(parsed); + } + ` + ); + expect(ep!.catches).toHaveLength(1); + expect(ep!.catches[0]!.narrow).toBe(false); + expect(ep!.catches[0]!.guardsParse).toBe(true); + expect(ep!.catches[0]!.tryStatementCount).toBe(6); + }); + + it("does not call a broad catch over database work a parse guard", () => { + const ep = scanFile( + "broad.ts", + ` + export async function loader({ params }) { + try { + const run = await prisma.run.findFirst({ where: { id: params.id } }); + const events = await prisma.event.findMany({ where: { runId: run.id } }); + await touch(run); + return json({ run, events }); + } catch (e) { + return json({}, { status: 500 }); + } + } + ` + ); + expect(ep!.catches).toHaveLength(1); + expect(ep!.catches[0]).toEqual({ + narrow: false, + rethrows: false, + branches: false, + guardsParse: false, + tryStatementCount: 4, + }); + }); + + it("keeps rethrow and branch evidence per clause", () => { + const ep = scanFile( + "mixed-clauses.ts", + ` + export async function loader({ request }) { + try { + return json(await load(request)); + } catch (e) { + if (e instanceof Response) throw e; + return json({}, { status: 500 }); + } + } + export async function action({ request }) { + try { + return json(await save(request)); + } catch (e) { + return null; + } + } + ` + ); + expect(ep!.catches).toHaveLength(2); + expect(ep!.catches.filter((c) => c.rethrows && c.branches)).toHaveLength(1); + expect(ep!.catches.filter((c) => !c.rethrows && !c.branches)).toHaveLength(1); + // Aggregates stay as they are: any clause sets them. + expect(ep!.catchRethrows).toBe(true); + expect(ep!.catchBranches).toBe(true); + }); + + it("includes a catch from a same-file helper and excludes one from the React component", () => { + const ep = scanFile( + "route.tsx", + ` + function parseTags(payload) { + try { + return JSON.parse(payload); + } catch { + return null; + } + } + export async function loader({ params }) { + return json(parseTags(params.payload)); + } + export default function Page() { + try { + render(); + } catch (e) { + throw e; + } + return null; + } + ` + ); + expect(ep!.catches).toHaveLength(1); + expect(ep!.catches[0]!.guardsParse).toBe(true); + expect(ep!.catchRethrows).toBe(false); + }); + + it("keeps the aggregates derivable from the per-catch list", () => { + const ep = scanFile( + "aggregate.ts", + ` + export async function loader({ request }) { + try { + return json(await request.json()); + } catch (e) { + if (e instanceof SyntaxError) return json({}, { status: 400 }); + return json({}, { status: 500 }); + } + } + ` + ); + expect(ep!.catchRethrows).toBe(ep!.catches.some((c) => c.rethrows)); + expect(ep!.catchBranches).toBe(ep!.catches.some((c) => c.branches)); + expect(ep!.catchesNarrowly).toBe(ep!.catches.length > 0 && ep!.catches.every((c) => c.narrow)); + }); +}); From 093d447d0dfd66a827ac45f5155cd770de092773 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Fri, 31 Jul 2026 15:35:59 +0100 Subject: [PATCH 016/106] fix(observability-map): judge catch clauses one at a time Both checks now read EntryPoint.catches instead of the aggregate booleans, so an entry point is only as good as its worst catch. 39 routes have more than one catch and 17 mix a narrow guard with a broad handler, and a single well-behaved catch used to speak for the swallow beside it. Neither check reads hasTryCatch any more. A try/finally leaves it true with no catch clause at all, which is what put runs-replication.status at the top of the first rendered fix list; the question is now catches.length. A parse guard is recognised when it covers less than half the body, which keeps otel.v1.logs reported, where the catch covers 15 of 18 statements and merely contains a request.json. The narrow limb of the proposed rule is left out. A one-statement try around an awaited service call is as narrow as one around a parse. Taking it clears eleven more routes and reading all eleven says six are real, including a silent run cancellation and two credential paths that report a database failure to the browser as a 400 with the internal message in it. FIX FIRST now reads account.tokens, api.v1.authorization-code and api.v1.token, all three genuine. Global score 83. --- .../src/checks/errorClassification.ts | 70 ++++++------ .../src/checks/requestContext.ts | 11 +- .../observability-map/test/checks.test.ts | 101 +++++++++++++++++- 3 files changed, 141 insertions(+), 41 deletions(-) diff --git a/internal-packages/observability-map/src/checks/errorClassification.ts b/internal-packages/observability-map/src/checks/errorClassification.ts index 7320e60d092..9b6e856d1ed 100644 --- a/internal-packages/observability-map/src/checks/errorClassification.ts +++ b/internal-packages/observability-map/src/checks/errorClassification.ts @@ -1,4 +1,4 @@ -import type { CheckResult, EntryPoint } from "../types.js"; +import type { CatchEvidence, CheckResult, EntryPoint } from "../types.js"; import { isTrivial } from "../triviality.js"; const ID = "error-classification"; @@ -25,19 +25,33 @@ export const BUILDERS = new Set([ ]); /** - * A parse: `await request.json()`, `JSON.parse(raw)`. The dot matters, it keeps Remix's `json({})` - * response helper out. Matched against `calleeTexts`, which carries the whole callee path. + * Whether a catch clause is a guard rather than the route's error handling: it wraps a parse and + * covers less than half the entry point's statements. Both halves matter. `guardsParse` alone lets + * `otel.v1.logs.ts` off, whose catch covers 15 of its 18 statements and merely happens to contain a + * `request.json()`, and that is a real swallow. The coverage test is relative to the body rather + * than a second absolute threshold, so it holds for a three-statement route and a fifty-statement + * one alike. */ -const PARSE_CALL = /(^|\.)JSON\.parse$|\.json$/; +export function isParseGuard(clause: CatchEvidence, ep: EntryPoint): boolean { + return clause.guardsParse && clause.tryStatementCount * 2 < ep.statementCount; +} /** - * Whether every catch in the entry point guards a parse and nothing wider. Both checks read the - * field the same way: a guard around one parse is not the route taking charge of its failures, - * so `error-classification` does not call it a swallow and `request-context` does not ask it to - * name a tenant. + * Whether a clause has decided what the error means. Rethrowing is a decision, branching is a + * decision, and guarding a parse answers for the one thing the guard covers. + * + * `narrow` is deliberately not a fourth way to qualify, which is where this differs from the rule + * the scanner work proposed. A one-statement try around `await service.call(run)` is narrow and is + * still a swallow. Taking the narrow limb clears eleven more entry points, and reading all eleven + * says six are real: the silent cancel in `api.v2.runs.$runParam.cancel.ts`, the PAT revoke in + * `account.tokens/route.tsx` and the invite revoke, both of which report a database failure to the + * browser as a 400 with an internal message in it, a `.map` that drops a broken dashboard on the + * floor, and two more. The four it would rightly clear are all the same deliberate shape, best + * effort side work that logs and carries on, and all four are non-sensitive so they sort to the + * bottom of the fix list. See the task 5 report; switching is one limb in this function. */ -export function guardsOnlyAParse(ep: EntryPoint): boolean { - return ep.catchesNarrowly && ep.calleeTexts.some((t) => PARSE_CALL.test(t)); +export function accountedFor(clause: CatchEvidence, ep: EntryPoint): boolean { + return clause.rethrows || clause.branches || isParseGuard(clause, ep); } export function usesBuilder(ep: EntryPoint): boolean { @@ -55,20 +69,15 @@ export function usesBuilder(ep: EntryPoint): boolean { * call in `try { ... } catch { return 500 }`. The builder classifies what reaches it, and that * error never does. Crediting the wrapper would hide the one case in this family worth finding. * - * `catchRethrows` and `catchBranches` are OR-ed across every catch clause in the bodies, so both - * false means every catch in the entry point takes the same way out whatever was thrown. The - * asymmetry that buys: one good catch alongside one swallow reads as a pass. This check misses - * those rather than inventing them. + * Judged per catch clause, so an entry point is only as good as its worst one. That is the whole + * point of the per-clause evidence: 39 routes have more than one catch and 17 mix a narrow guard + * with a broad handler, and under the old aggregate booleans a single well-behaved catch spoke for + * the swallow next to it. * - * `catchesNarrowly` excuses the guard that wraps one operation and answers for that operation: - * `try { body = await request.json() } catch { 400 }` neither branches nor rethrows and does not - * need to. On its own it excuses too much, because a one-statement try around an awaited service - * call is exactly as narrow as one around a parse: applied unencumbered it passes - * `try { await service.call(run) } catch { 500 }`, and it passes the design's own swallow fixture, - * `try { return await prisma.thing.findMany() } catch { return null }`. So the exemption also asks - * that the body parse something, which is the idiom the exemption was justified by. Over the real - * tree that combination clears the nine verbatim `request.json()` guards and holds back the four - * hand-read findings, see the task 5 report. + * "Does this route catch anything" is `catches.length`, never `hasTryCatch`. A try/finally with no + * catch leaves `hasTryCatch` true and `catches` empty: nothing is swallowed there, the error + * propagates once the cleanup has run, and reading the old flag as a catch put + * `admin.api.v1.runs-replication.status.ts` at the top of the first rendered fix list. */ export const errorClassification = { id: ID, @@ -76,19 +85,18 @@ export const errorClassification = { if (isTrivial(ep)) { return { id: ID, status: "not-applicable", detail: "trivial route" }; } - const guardsAParse = guardsOnlyAParse(ep); - if (ep.hasTryCatch && !ep.catchRethrows && !ep.catchBranches && !guardsAParse) { + const unaccounted = ep.catches.filter((c) => !accountedFor(c, ep)); + if (unaccounted.length > 0) { + const which = + ep.catches.length > 1 ? ` (${unaccounted.length} of ${ep.catches.length} catches)` : ""; return { id: ID, status: "fail", - detail: "catches its errors and takes one way out regardless of what was thrown", + detail: `catches its errors and takes one way out regardless of what was thrown${which}`, }; } - if (guardsAParse) { - return { id: ID, status: "pass", detail: "guards a parse, not the handler" }; - } - if (ep.catchRethrows || ep.catchBranches) { - return { id: ID, status: "pass", detail: "the catch distinguishes what it caught" }; + if (ep.catches.length > 0) { + return { id: ID, status: "pass", detail: "every catch decides what it caught" }; } if (usesBuilder(ep)) { return { id: ID, status: "pass", detail: "classified by the builder" }; diff --git a/internal-packages/observability-map/src/checks/requestContext.ts b/internal-packages/observability-map/src/checks/requestContext.ts index 116df0cc193..3ab19bc4695 100644 --- a/internal-packages/observability-map/src/checks/requestContext.ts +++ b/internal-packages/observability-map/src/checks/requestContext.ts @@ -1,6 +1,6 @@ import type { CheckResult, EntryPoint, LogCall } from "../types.js"; import { isTrivial } from "../triviality.js"; -import { guardsOnlyAParse } from "./errorClassification.js"; +import { isParseGuard } from "./errorClassification.js"; const ID = "request-context"; @@ -32,7 +32,7 @@ function failurePathLogs(ep: EntryPoint): LogCall[] { * thing the check exists to find and meant deleting a log line took a route out of the report. * Every non-trivial entry point is now judged: * - * - no catch at all, or nothing but a parse guard: pass. The route's own work still throws past it + * - no catch at all, or nothing but parse guards: pass. The route's own work still throws past it * to the central handler, which is the intended path in this codebase, and the tenant that * handler does not name is a platform-level gap reported once rather than against each route. A * `try { body = await request.json() } catch { 400 }` is not a route taking over its failure @@ -51,9 +51,10 @@ export const requestContext = { if (isTrivial(ep)) { return { id: ID, status: "not-applicable", detail: "trivial route" }; } - // A route whose only catch guards a parse still throws its own work past it to the central - // handler. Same reading error-classification gives the field. - if (!ep.hasTryCatch || guardsOnlyAParse(ep)) { + // `catches`, not `hasTryCatch`: a try/finally catches nothing, so its errors reach the central + // handler like any other. A route whose every clause guards a parse is in the same position, + // its own work still throws past those guards. Same reading error-classification gives them. + if (ep.catches.every((c) => isParseGuard(c, ep))) { return { id: ID, status: "pass", detail: "hands its failures to the central handler" }; } const logs = failurePathLogs(ep); diff --git a/internal-packages/observability-map/test/checks.test.ts b/internal-packages/observability-map/test/checks.test.ts index b04f7daa183..26dc4bc3f2c 100644 --- a/internal-packages/observability-map/test/checks.test.ts +++ b/internal-packages/observability-map/test/checks.test.ts @@ -173,6 +173,79 @@ describe("error-classification", () => { expect(r.status).toBe("fail"); }); + // try/finally with no catch clause. `hasTryCatch` is true here and `catches` is empty, and it is + // `catches` that answers "does this route catch anything". Nothing is swallowed: the error + // propagates once the connection is closed. + it("passes a try/finally that catches nothing", () => { + const r = run( + "error-classification", + "admin.api.v1.runs-replication.status.ts", + `import Redis from "ioredis"; + export async function loader() { + const redis = new Redis({ host: "localhost" }); + try { + const exists = await redis.exists("some-key"); + const other = await redis.exists("other-key"); + return json({ exists, other }); + } finally { + await redis.quit(); + } + }` + ); + expect(r.status).toBe("pass"); + }); + + // Multi-catch, the case the aggregate booleans could not describe. Judged per clause: the parse + // guard is a guard, the handler catch rethrows, so both are accounted for. + it("passes a parse guard sitting beside a handler catch that rethrows", () => { + const r = run( + "error-classification", + "api.v1.thing.ts", + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function action({ request }) { + let data; + try { data = await request.json(); } + catch { return json({ error: "Invalid JSON" }, { status: 400 }); } + try { + const thing = await prisma.thing.create({ data }); + const audit = await prisma.audit.create({ data: { thing: thing.id } }); + const count = await prisma.thing.count(); + return json({ thing, audit, count }); + } catch (error) { + logger.error("create failed", { error }); + throw error; + } + }` + ); + expect(r.status).toBe("pass"); + }); + + // A parse guard that has grown to cover the handler is not a guard any more. `otel.v1.logs.ts` + // catches 15 of its 18 statements around a `request.json()` and answers 500 for all of them. + it("fails a parse guard that covers most of the body", () => { + const r = run( + "error-classification", + "otel.v1.logs.ts", + `import { otlpExporter } from "~/v3/otlpExporter.server"; + export async function action({ request }) { + try { + const exporter = await otlpExporter; + const contentType = request.headers.get("content-type"); + const body = await request.json(); + const result = await exporter.exportLogs(body); + const encoded = encodeResponse(result); + const headers = buildHeaders(contentType); + return new Response(encoded, { status: 200, headers }); + } catch (error) { + console.error(error); + return new Response("Internal Server Error", { status: 500 }); + } + }` + ); + expect(r.status).toBe("fail"); + }); + // False positive fixture: the only try/catch in the file belongs to the component. it("does not flag a route whose try/catch is in the React component", () => { const r = run( @@ -481,10 +554,10 @@ describe("request-context", () => { expect(r.status).toBe("pass"); }); - // Known false positive, named in the task 5 report. The only catch here guards `new URL()`, a - // constructor, which never reaches `calleeTexts`, so the parse cannot be seen and the route is - // judged as though it kept its failures. Fixing it needs the scanner, not the check. - it("fails a route whose only catch guards a constructor parse", () => { + // Was a known false positive: `new URL()` is a constructor, so the parse was invisible while the + // evidence came from `calleeTexts`. `CatchEvidence.guardsParse` covers constructors, so the guard + // is legible now and the route is no longer judged as though it kept its failures. + it("passes a route whose only catch guards a constructor parse", () => { const r = run( "request-context", "_app.@.orgs.$organizationSlug.$.tsx", @@ -499,7 +572,25 @@ describe("request-context", () => { return typedjson({ origin, things: await prisma.thing.findMany() }); }` ); - expect(r.status).toBe("fail"); + expect(r.status).toBe("pass"); + }); + + it("passes a try/finally that catches nothing", () => { + const r = run( + "request-context", + "admin.api.v1.runs-replication.status.ts", + `import Redis from "ioredis"; + export async function loader() { + const redis = new Redis({ host: "localhost" }); + try { + const exists = await redis.exists("some-key"); + return json({ exists }); + } finally { + await redis.quit(); + } + }` + ); + expect(r.status).toBe("pass"); }); it("still judges a route with a handler-wide catch beside a narrow guard", () => { From 1dbb49a7a5c5f88f1c540e9bfc6682263428f969 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Fri, 31 Jul 2026 15:45:38 +0100 Subject: [PATCH 017/106] feat(observability-map): cli, single entry inspection and integration smoke Adds pnpm run map (repo root and package script), single-entry inspection mode, and index.ts exports. The routes directory now resolves against the repo root found by walking up to pnpm-workspace.yaml, not process.cwd(), so the CLI works from both the repo root and the package directory. Single-entry mode notes when an entry has no applicable scored checks rather than printing a bare 100/100. Gitignores the generated observability-map.json artifact. --- .gitignore | 3 + .../observability-map/package.json | 6 +- .../observability-map/src/cli.ts | 67 +++++++++++++++++++ .../observability-map/src/index.ts | 6 ++ .../test/integration.test.ts | 47 +++++++++++++ package.json | 1 + pnpm-lock.yaml | 3 + 7 files changed, 131 insertions(+), 2 deletions(-) create mode 100644 internal-packages/observability-map/src/cli.ts create mode 100644 internal-packages/observability-map/src/index.ts create mode 100644 internal-packages/observability-map/test/integration.test.ts diff --git a/.gitignore b/.gitignore index 7d9dc169042..f540927e32b 100644 --- a/.gitignore +++ b/.gitignore @@ -82,3 +82,6 @@ ailogger-output.log # local planning/design docs, not committed **/docs/superpowers/ + +# observability-map CLI output artifact, not committed +observability-map.json diff --git a/internal-packages/observability-map/package.json b/internal-packages/observability-map/package.json index 3d6f695f67b..03321ad7d1f 100644 --- a/internal-packages/observability-map/package.json +++ b/internal-packages/observability-map/package.json @@ -9,13 +9,15 @@ }, "devDependencies": { "@types/node": "^24.13.3", - "rimraf": "6.0.1" + "rimraf": "6.0.1", + "tsx": "^4.19.2" }, "scripts": { "clean": "rimraf dist", "typecheck": "tsc --noEmit", "build": "pnpm run clean && tsc -p tsconfig.build.json", "test": "vitest run", - "test:watch": "vitest" + "test:watch": "vitest", + "map": "tsx src/cli.ts" } } diff --git a/internal-packages/observability-map/src/cli.ts b/internal-packages/observability-map/src/cli.ts new file mode 100644 index 00000000000..360f39a84f6 --- /dev/null +++ b/internal-packages/observability-map/src/cli.ts @@ -0,0 +1,67 @@ +import { existsSync, writeFileSync } from "node:fs"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { scanDirectory } from "./scan.js"; +import { buildReport, scoreEntry } from "./score.js"; +import { renderTerminal } from "./report/terminal.js"; +import { renderJson } from "./report/json.js"; + +const DEFAULT_ROUTES = "apps/webapp/app/routes"; + +/** + * Walks up from this file looking for `pnpm-workspace.yaml`, so the routes directory resolves + * correctly whether `map` is run from the repo root or from the package directory (where + * `pnpm --filter` puts you). Resolving `DEFAULT_ROUTES` against `process.cwd()` instead would only + * work from the repo root. + */ +function findRepoRoot(startDir: string): string { + let dir = startDir; + for (let i = 0; i < 10; i++) { + if (existsSync(resolve(dir, "pnpm-workspace.yaml"))) return dir; + const parent = dirname(dir); + if (parent === dir) break; + dir = parent; + } + throw new Error("could not find repo root (no pnpm-workspace.yaml in any parent directory)"); +} + +export function main(argv: string[]): number { + const args = argv.slice(2); + const asJson = args.includes("--json"); + const noWrite = args.includes("--no-write"); + const target = args.find((a) => !a.startsWith("--")); + + const repoRoot = findRepoRoot(dirname(fileURLToPath(import.meta.url))); + const routesDir = resolve(repoRoot, DEFAULT_ROUTES); + const { entryPoints, parseFailures } = scanDirectory(routesDir); + + if (target) { + const match = entryPoints.find( + (e) => e.fileName === target || e.fileName.startsWith(target.replace(/^\//, "")) + ); + if (!match) { + process.stderr.write(`no entry point matching "${target}"\n`); + return 1; + } + const scored = scoreEntry(match); + const measuredNote = scored.measured ? "" : " (not measured: no applicable checks)"; + process.stdout.write( + `${scored.routePath} ${scored.score}/100${measuredNote}\n${scored.fileName}\n\nCHECKS\n` + ); + for (const c of scored.checks) { + const mark = c.status === "pass" ? "PASS" : c.status === "fail" ? "FAIL" : "n/a "; + process.stdout.write(` ${mark} ${c.id}${c.detail ? ` (${c.detail})` : ""}\n`); + } + return 0; + } + + const report = buildReport(entryPoints, parseFailures); + process.stdout.write(asJson ? renderJson(report) : renderTerminal(report)); + process.stdout.write("\n"); + if (!noWrite) { + writeFileSync(resolve(repoRoot, "observability-map.json"), renderJson(report)); + } + return 0; +} + +process.exitCode = main(process.argv); diff --git a/internal-packages/observability-map/src/index.ts b/internal-packages/observability-map/src/index.ts new file mode 100644 index 00000000000..1a8e9904931 --- /dev/null +++ b/internal-packages/observability-map/src/index.ts @@ -0,0 +1,6 @@ +export { scanDirectory, scanFile } from "./scan.js"; +export { buildReport, scoreEntry } from "./score.js"; +export { renderTerminal } from "./report/terminal.js"; +export { renderJson } from "./report/json.js"; +export type { MapReport, ScoredEntry } from "./score.js"; +export type { EntryPoint, CheckResult, CheckStatus } from "./types.js"; diff --git a/internal-packages/observability-map/test/integration.test.ts b/internal-packages/observability-map/test/integration.test.ts new file mode 100644 index 00000000000..f628c5a1dfd --- /dev/null +++ b/internal-packages/observability-map/test/integration.test.ts @@ -0,0 +1,47 @@ +import { existsSync, readdirSync } from "node:fs"; +import { join, resolve } from "node:path"; +import { scanDirectory } from "../src/scan.js"; +import { buildReport } from "../src/score.js"; + +const ROUTES = resolve(__dirname, "../../../apps/webapp/app/routes"); + +/** + * Counts route module candidates the same way `scanDirectory` walks the tree, without scanning + * their contents: a flat `.ts`/`.tsx` file, or one `route.ts`/`route.tsx` per directory. This is a + * structural upper bound, not a golden number - not every candidate exports a loader or action, so + * `entryPoints.length` must stay below it, and the route set is free to grow or shrink over time + * without breaking this test. + */ +function countCandidates(dir: string): number { + let count = 0; + for (const entry of readdirSync(dir, { withFileTypes: true })) { + if (entry.isDirectory()) { + for (const child of readdirSync(join(dir, entry.name), { withFileTypes: true })) { + if (child.isFile() && (child.name === "route.ts" || child.name === "route.tsx")) count++; + } + continue; + } + if (entry.isFile() && /\.tsx?$/.test(entry.name) && !entry.name.endsWith(".d.ts")) count++; + } + return count; +} + +describe("scanning the real webapp routes", () => { + it("parses every route file without crashing", () => { + if (!existsSync(ROUTES)) return; + const { entryPoints, parseFailures } = scanDirectory(ROUTES); + // Invariants, not exact numbers: the route set changes constantly. + expect(entryPoints.length).toBeGreaterThan(200); + expect(entryPoints.length).toBeLessThan(countCandidates(ROUTES)); + expect(parseFailures).toEqual([]); + }); + + it("produces a report with a score in range", () => { + if (!existsSync(ROUTES)) return; + const { entryPoints, parseFailures } = scanDirectory(ROUTES); + const report = buildReport(entryPoints, parseFailures); + expect(report.global).toBeGreaterThanOrEqual(0); + expect(report.global).toBeLessThanOrEqual(100); + expect(Object.keys(report.byFamily).length).toBeGreaterThan(1); + }); +}); diff --git a/package.json b/package.json index 2708b06be73..9fad94dd9d4 100644 --- a/package.json +++ b/package.json @@ -40,6 +40,7 @@ "clean": "turbo run clean", "clean:node_modules": "find . -name 'node_modules' -type d -prune -exec rm -rf '{}' +", "typecheck": "turbo run typecheck", + "map": "pnpm --filter @internal/observability-map run map", "test:e2e": "playwright test", "test:e2e:ui": "playwright test --ui", "test:e2e:dev": "turbo run test:e2e:dev", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b2b0c4a17c4..d5fdb570112 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1151,6 +1151,9 @@ importers: rimraf: specifier: 6.0.1 version: 6.0.1 + tsx: + specifier: ^4.19.2 + version: 4.22.4 internal-packages/otlp-importer: dependencies: From ca8b9160e6b9f1caaee5e0b77b6accbdfee95115 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Fri, 31 Jul 2026 20:25:16 +0100 Subject: [PATCH 018/106] fix(observability-map): stop guardsParse and branches firing on lookalikes Two catch-evidence fields matched shapes that resemble the thing they detect, which excused catches the checks exist to find. - guardsParse took any new X(), so new BranchesPresenter() or new Set() excused a catch over ordinary work. It now needs a parsing constructor, URL, URLSearchParams or RegExp, chosen from what the route tree actually constructs inside try blocks. 60 of 242 clauses change, 141 true to 81 - branches took an instanceof anywhere in the clause, including the error instanceof Error ? error.message : String(error) idiom, which words a message rather than picking a path. It now needs an if, a switch, or a conditional that is the whole return or throw. 29 of 242 clauses change, 134 true to 105. All 33 bare instanceof uses in the tree are the formatting idiom Clauses with no evidence at all go from 9 to 37. error-classification will need recalibrating: on this evidence it reports 64 routes rather than 28, and nothing it reported before stops being reported. --- .../observability-map/src/scan.ts | 46 ++++-- .../observability-map/src/types.ts | 14 +- .../observability-map/test/scan.test.ts | 149 +++++++++++++++++- 3 files changed, 194 insertions(+), 15 deletions(-) diff --git a/internal-packages/observability-map/src/scan.ts b/internal-packages/observability-map/src/scan.ts index 0028d55bab8..6301f8ed6f8 100644 --- a/internal-packages/observability-map/src/scan.ts +++ b/internal-packages/observability-map/src/scan.ts @@ -121,14 +121,27 @@ const NARROW_TRY_STATEMENTS = 2; const PARSE_CALLEE = /(^|\.)(parse|safeParse|parseAsync|safeParseAsync|decode)$|\.json$/; /** - * Whether the guarded region parses or constructs something. A constructor counts: `new URL(x)` is - * the commonest parse guard in the tree, and constructors are absent from `calleeTexts`. + * Constructors that parse untrusted input and throw when it is malformed. Deliberately short: any + * constructor at all would mean `new BranchesPresenter()` or `new Set(...)` excuses a catch that + * guards ordinary work, which was true of 77 try blocks in the route tree. + */ +const PARSE_CONSTRUCTORS = new Set(["URL", "URLSearchParams", "RegExp"]); + +/** + * Whether the guarded region parses something. A `new URL(x)` counts, and has to be read here + * because constructors are absent from `calleeTexts`. */ function guardsParse(tryBlock: ts.Block): boolean { let found = false; const visit = (node: ts.Node) => { if (found) return; - if (ts.isNewExpression(node)) found = true; + if ( + ts.isNewExpression(node) && + ts.isIdentifier(node.expression) && + PARSE_CONSTRUCTORS.has(node.expression.text) + ) { + found = true; + } if (ts.isCallExpression(node)) { const text = calleeText(node.expression) ?? calleeName(node.expression); if (text !== null && PARSE_CALLEE.test(text)) found = true; @@ -139,6 +152,26 @@ function guardsParse(tryBlock: ts.Block): boolean { return found; } +function containsInstanceOf(node: ts.Node): boolean { + if (ts.isBinaryExpression(node) && node.operatorToken.kind === ts.SyntaxKind.InstanceOfKeyword) { + return true; + } + return ts.forEachChild(node, containsInstanceOf) === true; +} + +/** + * Whether a conditional expression tests the error to pick what the clause does, rather than to + * word what it says. It counts only when the whole `return`/`throw` is the conditional, so + * `return e instanceof Response ? e : json({}, { status: 500 })` counts and + * `return json({ error: e instanceof Error ? e.message : String(e) }, { status: 400 })` does not. + * The second is message formatting: every error leaves by the same path. + */ +function selectsAnErrorPath(node: ts.ConditionalExpression): boolean { + if (!containsInstanceOf(node.condition)) return false; + const parent = node.parent; + return parent !== undefined && (ts.isReturnStatement(parent) || ts.isThrowStatement(parent)); +} + /** What a catch clause does with the error, beyond the fact that it caught one. */ function catchClauseEvidence(clause: ts.CatchClause): { rethrows: boolean; branches: boolean } { let rethrows = false; @@ -147,12 +180,7 @@ function catchClauseEvidence(clause: ts.CatchClause): { rethrows: boolean; branc const visit = (node: ts.Node) => { if (ts.isThrowStatement(node)) rethrows = true; if (ts.isIfStatement(node) || ts.isSwitchStatement(node)) branches = true; - if ( - ts.isBinaryExpression(node) && - node.operatorToken.kind === ts.SyntaxKind.InstanceOfKeyword - ) { - branches = true; - } + if (ts.isConditionalExpression(node) && selectsAnErrorPath(node)) branches = true; ts.forEachChild(node, visit); }; visit(clause.block); diff --git a/internal-packages/observability-map/src/types.ts b/internal-packages/observability-map/src/types.ts index 2e2401c87a4..4a9eec56b69 100644 --- a/internal-packages/observability-map/src/types.ts +++ b/internal-packages/observability-map/src/types.ts @@ -16,12 +16,18 @@ export type CatchEvidence = { narrow: boolean; /** The clause contains a `throw`. */ rethrows: boolean; - /** The clause branches on the error: an `if`, a `switch`, or an `instanceof`. */ + /** + * The clause picks what to do from what it caught: an `if`, a `switch`, or a conditional that is + * the whole `return`/`throw`. An `instanceof` used only to word a message, + * `json({ error: e instanceof Error ? e.message : String(e) })`, does not count: every error + * still leaves by the same path. + */ branches: boolean; /** - * The guarded region parses or constructs something: `JSON.parse`, `request.json()`, a zod - * `parse`/`safeParse`, or any `new X(...)`. Constructors are counted here because `new URL(x)` is - * the commonest parse guard in the tree and constructors never appear in `calleeTexts`. + * The guarded region parses something: `JSON.parse`, `request.json()`, a zod `parse`/`safeParse`, + * a `decode`, or a `new URL`/`URLSearchParams`/`RegExp`. Those three constructors are read here + * because constructors never appear in `calleeTexts`; other constructors do not count, or every + * `new SomePresenter()` in a try would excuse its catch. */ guardsParse: boolean; /** Statements in the guarded try block, counted as `statementCount` counts them. */ diff --git a/internal-packages/observability-map/test/scan.test.ts b/internal-packages/observability-map/test/scan.test.ts index e4e2388fd35..6625adf9343 100644 --- a/internal-packages/observability-map/test/scan.test.ts +++ b/internal-packages/observability-map/test/scan.test.ts @@ -628,7 +628,7 @@ describe("scanFile: catch clause evidence", () => { expect(ep!.catchRethrows).toBe(false); }); - it("sets catchBranches for a bare instanceof with no `if`", () => { + it("sets catchBranches for an instanceof conditional that is the whole returned expression", () => { const ep = scanFile( "branch-instanceof.ts", ` @@ -1092,7 +1092,7 @@ describe("scanFile: per-catch evidence", () => { expect(ep!.catchesNarrowly).toBe(false); }); - it("sees a constructor as a guarded parse", () => { + it("sees a URL constructor as a guarded parse", () => { const ep = scanFile( "_app.@.orgs.$organizationSlug.$.tsx", ` @@ -1246,3 +1246,148 @@ describe("scanFile: per-catch evidence", () => { expect(ep!.catchesNarrowly).toBe(ep!.catches.length > 0 && ep!.catches.every((c) => c.narrow)); }); }); + +describe("scanFile: guardsParse is limited to parsing constructors", () => { + it("does not treat a presenter construction as a parse guard", () => { + const ep = scanFile( + "_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.branches/route.tsx", + ` + export async function loader({ request, params }) { + try { + const presenter = new BranchesPresenter(); + const result = await presenter.call({ userId: 1, projectSlug: params.projectParam }); + return typedjson(result); + } catch (error) { + logger.error("Error loading preview branches page", { error }); + throw new Response(undefined, { status: 400 }); + } + } + ` + ); + expect(ep!.catches).toHaveLength(1); + expect(ep!.catches[0]!.guardsParse).toBe(false); + }); + + it("does not treat a collection construction as a parse guard", () => { + const ep = scanFile( + "account.tokens/route.tsx", + ` + export async function action({ request }) { + try { + const roles = await loadRoles(request); + const names = new Set(roles.map((r) => r.name)); + return json({ names: [...names] }); + } catch (error) { + return json({ error: "failed" }, { status: 400 }); + } + } + ` + ); + expect(ep!.catches[0]!.guardsParse).toBe(false); + }); + + it("treats RegExp and URLSearchParams construction as a parse guard", () => { + const regexp = scanFile( + "regexp.ts", + ` + export async function action({ request }) { + const pattern = await patternFrom(request); + try { + new RegExp(pattern); + } catch { + return json({ error: "Invalid regex" }, { status: 400 }); + } + return json({ ok: true }); + } + ` + ); + expect(regexp!.catches[0]!.guardsParse).toBe(true); + + const search = scanFile( + "search-params.ts", + ` + export async function loader({ request }) { + try { + return json(Object.fromEntries(new URLSearchParams(request.url))); + } catch { + return json({}, { status: 400 }); + } + } + ` + ); + expect(search!.catches[0]!.guardsParse).toBe(true); + }); + + it("still sees a parse call in a try that also constructs something ordinary", () => { + const ep = scanFile( + "parse-and-construct.ts", + ` + export async function action({ request }) { + try { + const body = JSON.parse(await request.text()); + const service = new PromptService(); + return json(await service.create(body)); + } catch { + return json({}, { status: 400 }); + } + } + ` + ); + expect(ep!.catches[0]!.guardsParse).toBe(true); + }); +}); + +describe("scanFile: branches ignores the error-stringifying ternary", () => { + it("does not count an instanceof nested inside a returned call argument", () => { + const ep = scanFile( + "admin.api.v1.runs-replication.start.ts", + ` + export async function action({ request }) { + try { + return json(await start(request)); + } catch (error) { + return json({ error: error instanceof Error ? error.message : error }, { status: 400 }); + } + } + ` + ); + expect(ep!.catches).toHaveLength(1); + expect(ep!.catches[0]!.branches).toBe(false); + expect(ep!.catchBranches).toBe(false); + }); + + it("does not count an instanceof used to build a logged message", () => { + const ep = scanFile( + "admin.api.v1.feature-flags.ts", + ` + export async function action({ request }) { + try { + return json(await setFlag(request)); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + logger.error("flag update failed", { message }); + return json({ error: message }, { status: 400 }); + } + } + ` + ); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + it("still counts an instanceof in an `if`", () => { + const ep = scanFile( + "branch-if-instanceof.ts", + ` + export async function loader({ request }) { + try { + return json(await load(request)); + } catch (e) { + if (e instanceof Response) return e; + return json({}, { status: 500 }); + } + } + ` + ); + expect(ep!.catches[0]!.branches).toBe(true); + }); +}); From e4eaa7caf511e47aeef11882e5600ab2fbd06ffd Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Fri, 31 Jul 2026 20:29:13 +0100 Subject: [PATCH 019/106] chore(observability-map): declare vitest and scope the build The build config had no include and no rootDir, so tsc inferred the package root because vitest.config.ts happened to be inside the compilation. That is the only reason dist/src/index.js landed where the package's main points, and excluding the config would have silently moved the entry point. Scope the build to src and pin rootDir so the layout is intentional. vitest was resolving from the root workspace by hoisting despite being the test runner and supplying the global types. Declare it at the version the other internal packages use. --- internal-packages/observability-map/package.json | 3 ++- .../observability-map/tsconfig.build.json | 10 ++++++++-- pnpm-lock.yaml | 3 +++ 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/internal-packages/observability-map/package.json b/internal-packages/observability-map/package.json index 03321ad7d1f..776ec458963 100644 --- a/internal-packages/observability-map/package.json +++ b/internal-packages/observability-map/package.json @@ -10,7 +10,8 @@ "devDependencies": { "@types/node": "^24.13.3", "rimraf": "6.0.1", - "tsx": "^4.19.2" + "tsx": "^4.19.2", + "vitest": "4.1.7" }, "scripts": { "clean": "rimraf dist", diff --git a/internal-packages/observability-map/tsconfig.build.json b/internal-packages/observability-map/tsconfig.build.json index 56ade258e18..254fcfbca49 100644 --- a/internal-packages/observability-map/tsconfig.build.json +++ b/internal-packages/observability-map/tsconfig.build.json @@ -1,5 +1,11 @@ { "extends": "./tsconfig.json", - "compilerOptions": { "noEmit": false, "outDir": "dist", "declaration": true }, - "exclude": ["node_modules", "dist", "test"] + "include": ["src/**/*.ts"], + "compilerOptions": { + "noEmit": false, + "declaration": true, + "outDir": "dist", + "rootDir": ".", + "types": ["node"] + } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d5fdb570112..64f6fe98b18 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1154,6 +1154,9 @@ importers: tsx: specifier: ^4.19.2 version: 4.22.4 + vitest: + specifier: 4.1.7 + version: 4.1.7(@opentelemetry/api@1.9.1)(@types/node@24.13.3)(@vitest/coverage-v8@4.1.7)(vite@6.4.2(@types/node@24.13.3)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.22.4)(yaml@2.9.0)) internal-packages/otlp-importer: dependencies: From 8232d110b932a135a055daa90f438a2f1ab4180f Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Fri, 31 Jul 2026 20:36:41 +0100 Subject: [PATCH 020/106] fix(observability-map): stop paying routes for having no error handling Emptying every catch clause in the tree scored it 100. Both scored checks passed on the single fact that a route has no catch: error-classification credited it as propagating to the global handler, request-context treated it as having handed its failures over. So the gradient rewarded deleting error handling, and 222 of 412 entries scored 100 on that one shared fact. error-classification now reports not-applicable for a route with no catch, since there is no classification decision to judge, and no longer credits a builder wrapper for error handling the route does not do. request-context fails it instead: the global handler carries requestId, path, host and method and no tenant, so such a route genuinely cannot name whose request broke. Excusing it would reinstate the perverse incentive. Score falls from 76 to 22, which is the honest reading. Deleting all error handling now takes it to 7 rather than 100. The two checks decorrelate: kappa on error-classification against request-context moves from +0.231 to -0.032, and the other two pairs stay near zero. --- .../src/checks/errorClassification.ts | 40 ++++++----- .../src/checks/requestContext.ts | 66 ++++++++----------- .../observability-map/test/checks.test.ts | 36 +++++----- .../observability-map/test/report.test.ts | 22 +++++-- .../observability-map/test/score.test.ts | 47 +++++++++---- 5 files changed, 118 insertions(+), 93 deletions(-) diff --git a/internal-packages/observability-map/src/checks/errorClassification.ts b/internal-packages/observability-map/src/checks/errorClassification.ts index 9b6e856d1ed..f4a8ce5e8b2 100644 --- a/internal-packages/observability-map/src/checks/errorClassification.ts +++ b/internal-packages/observability-map/src/checks/errorClassification.ts @@ -4,9 +4,10 @@ import { isTrivial } from "../triviality.js"; const ID = "error-classification"; /** - * The route builders that own the failure path: they authenticate, they catch, they pass a thrown - * `Response` through untouched and report anything else through `logBoundaryError` before - * answering 500. A route wrapped in one of these has its errors classified for it. + * The route builders that authenticate the request, which is what `auth-boundary` reads them for. + * They also catch and classify, passing a thrown `Response` through untouched and reporting + * anything else through `logBoundaryError`, but `error-classification` no longer credits that: a + * route with no catch of its own is judged on nothing, wrapper or not. * * `createSSELoader` is deliberately absent. It turns a non-Response error into a 500 but does not * authenticate, so counting it here would hand two routes a free pass on `auth-boundary`. @@ -64,15 +65,17 @@ export function usesBuilder(ep: EntryPoint): boolean { /** * Who decides what a failure means, and on what evidence. * - * The swallow is read before the builder is credited, which looks like the wrong order until you - * read `api.v2.runs.$runParam.cancel.ts`: a `createActionApiRoute` handler wrapping its service - * call in `try { ... } catch { return 500 }`. The builder classifies what reaches it, and that - * error never does. Crediting the wrapper would hide the one case in this family worth finding. + * Judged per catch clause, so an entry point is only as good as its worst one. That is the point of + * the per-clause evidence: 39 routes have more than one catch and 17 mix a narrow guard with a + * broad handler, and under the old aggregate booleans a single well-behaved catch spoke for the + * swallow next to it. * - * Judged per catch clause, so an entry point is only as good as its worst one. That is the whole - * point of the per-clause evidence: 39 routes have more than one catch and 17 mix a narrow guard - * with a broad handler, and under the old aggregate booleans a single well-behaved catch spoke for - * the swallow next to it. + * A route with no catch is not-applicable, not a pass. It makes no classification decision, so + * there is nothing here to judge and nothing to credit. Crediting it was worse than merely + * generous: with `request-context` also passing the same routes, emptying every catch clause in the + * tree scored it 100, so the metric paid you for deleting error handling. Out of the denominator + * is the honest place for it, and it takes the builder credit with it: a builder-wrapped route with + * no catch of its own now sits out too, rather than collecting a point for the wrapper. * * "Does this route catch anything" is `catches.length`, never `hasTryCatch`. A try/finally with no * catch leaves `hasTryCatch` true and `catches` empty: nothing is swallowed there, the error @@ -85,6 +88,13 @@ export const errorClassification = { if (isTrivial(ep)) { return { id: ID, status: "not-applicable", detail: "trivial route" }; } + if (ep.catches.length === 0) { + return { + id: ID, + status: "not-applicable", + detail: "catches nothing, so it classifies nothing", + }; + } const unaccounted = ep.catches.filter((c) => !accountedFor(c, ep)); if (unaccounted.length > 0) { const which = @@ -95,12 +105,6 @@ export const errorClassification = { detail: `catches its errors and takes one way out regardless of what was thrown${which}`, }; } - if (ep.catches.length > 0) { - return { id: ID, status: "pass", detail: "every catch decides what it caught" }; - } - if (usesBuilder(ep)) { - return { id: ID, status: "pass", detail: "classified by the builder" }; - } - return { id: ID, status: "pass", detail: "errors propagate to the global handler" }; + return { id: ID, status: "pass", detail: "every catch decides what it caught" }; }, }; diff --git a/internal-packages/observability-map/src/checks/requestContext.ts b/internal-packages/observability-map/src/checks/requestContext.ts index 3ab19bc4695..041a9515257 100644 --- a/internal-packages/observability-map/src/checks/requestContext.ts +++ b/internal-packages/observability-map/src/checks/requestContext.ts @@ -1,6 +1,5 @@ -import type { CheckResult, EntryPoint, LogCall } from "../types.js"; +import type { CheckResult, EntryPoint } from "../types.js"; import { isTrivial } from "../triviality.js"; -import { isParseGuard } from "./errorClassification.js"; const ID = "request-context"; @@ -12,38 +11,27 @@ const ID = "request-context"; */ const IDENTIFIER_FIELD = /^(id|ids|slug|ref)$|[a-z](Id|Ids|Slug|Ref|Param|Identifier)$/; -function failurePathLogs(ep: EntryPoint): LogCall[] { - return ep.logCalls.filter((l) => l.inCatch); -} - /** - * Whether a failure this route reports itself can be traced to whoever it happened to. + * Whether a failure here can be traced to whoever it happened to. * - * Everything the platform attaches centrally is already accounted for, which is what makes this - * worth asking. `logger` pushes the http context (requestId, path, host, method) onto every line + * Everything the platform attaches centrally is accounted for, which is what makes this worth + * asking. `logger` pushes the http context, `{ requestId, path, host, method }`, onto every line * through AsyncLocalStorage, and `Logger.onError` forwards the error to Sentry. Neither carries a * tenant: no route calls `trace({ environmentId }, ...)`, and the builders' own boundary log is - * `logBoundaryError(message, error, url)`, which is a url and an error. So a builder-wrapped route - * is not attributed either and gets no free pass here. An incident tells you which route and which - * request; whose environment it was is only ever in the fields the route passes itself. - * - * Applicability turns on whether the route keeps its own failures, never on whether it logs. The - * first version made a route not-applicable when it had no failure-path log, which excused the very - * thing the check exists to find and meant deleting a log line took a route out of the report. - * Every non-trivial entry point is now judged: + * `logBoundaryError(message, error, url)`, a url and an error. So an incident tells you which route + * and which request failed, and never whose environment it was, unless the route passed the field + * itself. 21 of 427 entry points do. * - * - no catch at all, or nothing but parse guards: pass. The route's own work still throws past it - * to the central handler, which is the intended path in this codebase, and the tenant that - * handler does not name is a platform-level gap reported once rather than against each route. A - * `try { body = await request.json() } catch { 400 }` is not a route taking over its failure - * path, and reading it as one put false positives at the top of the first rendered report. - * - a catch that covers the route's work: it decided the outcome itself, so it has to say whose - * failure it was. + * Every non-trivial entry point is judged, and a route that never catches fails like any other. + * That is the whole point rather than an oversight: its failures go to the global handler, which + * names no tenant, so it genuinely cannot say whose request broke. Passing those routes, as this + * check used to, meant deleting every catch clause in the tree scored it 100. Excusing them as + * not-applicable would be the same mistake in quieter clothes, since it would once again reward + * having no failure handling to inspect. * - * Deleting a log call can then only make a verdict worse or leave it alone, and adding a catch - * without a report is a regression the check reports, which is the direction the incentive should - * run in. The one move that still improves a verdict is deleting the try/catch outright, and that - * hands the error back to the central handler, which `error-classification` also treats as correct. + * The consequence is a check that fails 90% of what it looks at, which is an honest reading of a + * codebase where the fix is one platform change, tenant fields through `trace(...)` in the auth + * path, rather than 300 route edits. Weight it accordingly, but do not read the count as noise. */ export const requestContext = { id: ID, @@ -51,25 +39,27 @@ export const requestContext = { if (isTrivial(ep)) { return { id: ID, status: "not-applicable", detail: "trivial route" }; } - // `catches`, not `hasTryCatch`: a try/finally catches nothing, so its errors reach the central - // handler like any other. A route whose every clause guards a parse is in the same position, - // its own work still throws past those guards. Same reading error-classification gives them. - if (ep.catches.every((c) => isParseGuard(c, ep))) { - return { id: ID, status: "pass", detail: "hands its failures to the central handler" }; - } - const logs = failurePathLogs(ep); - const named = logs.find((l) => l.fields.some((f) => IDENTIFIER_FIELD.test(f))); + const failurePathLogs = ep.logCalls.filter((l) => l.inCatch); + const named = failurePathLogs.find((l) => l.fields.some((f) => IDENTIFIER_FIELD.test(f))); if (named) { const fields = named.fields.filter((f) => IDENTIFIER_FIELD.test(f)); return { id: ID, status: "pass", detail: `failure log names ${fields.join(", ")}` }; } + if (failurePathLogs.length > 0) { + return { + id: ID, + status: "fail", + detail: + "logs its failure without naming an environment, project, organization, run or user", + }; + } return { id: ID, status: "fail", detail: - logs.length === 0 + ep.catches.length > 0 ? "keeps its failures and records nothing about whose they were" - : "logs its failure without naming an environment, project, organization, run or user", + : "leaves its failures to the central handler, which names no tenant", }; }, }; diff --git a/internal-packages/observability-map/test/checks.test.ts b/internal-packages/observability-map/test/checks.test.ts index 26dc4bc3f2c..856a221c6b6 100644 --- a/internal-packages/observability-map/test/checks.test.ts +++ b/internal-packages/observability-map/test/checks.test.ts @@ -40,14 +40,16 @@ describe("registry", () => { }); describe("error-classification", () => { - it("passes a builder-wrapped route with no local try/catch", () => { + // C1. A route with no catch makes no classification decision, so there is nothing here to judge + // and nothing to credit. Crediting it made deleting error handling raise the score. + it("is not applicable to a builder-wrapped route with no local try/catch", () => { const r = run( "error-classification", "api.v1.x.ts", `import { createLoaderApiRoute } from "~/services/routeBuilders/apiBuilder.server"; export const loader = createLoaderApiRoute({}, async () => new Response("ok"));` ); - expect(r.status).toBe("pass"); + expect(r.status).toBe("not-applicable"); }); it("fails a raw route whose catch swallows every error identically", () => { @@ -109,7 +111,7 @@ describe("error-classification", () => { expect(r.status).toBe("fail"); }); - it("passes a raw route that lets its errors propagate", () => { + it("is not applicable to a raw route that lets its errors propagate", () => { const r = run( "error-classification", "api.v1.w.ts", @@ -119,7 +121,7 @@ describe("error-classification", () => { return json({ rows }); }` ); - expect(r.status).toBe("pass"); + expect(r.status).toBe("not-applicable"); }); it("is not applicable to a trivial redirect", () => { @@ -176,7 +178,7 @@ describe("error-classification", () => { // try/finally with no catch clause. `hasTryCatch` is true here and `catches` is empty, and it is // `catches` that answers "does this route catch anything". Nothing is swallowed: the error // propagates once the connection is closed. - it("passes a try/finally that catches nothing", () => { + it("is not applicable to a try/finally that catches nothing", () => { const r = run( "error-classification", "admin.api.v1.runs-replication.status.ts", @@ -192,7 +194,7 @@ describe("error-classification", () => { } }` ); - expect(r.status).toBe("pass"); + expect(r.status).toBe("not-applicable"); }); // Multi-catch, the case the aggregate booleans could not describe. Judged per clause: the parse @@ -247,7 +249,7 @@ describe("error-classification", () => { }); // False positive fixture: the only try/catch in the file belongs to the component. - it("does not flag a route whose try/catch is in the React component", () => { + it("does not judge a route whose try/catch is in the React component", () => { const r = run( "error-classification", "_app.orgs.$organizationSlug.things/route.tsx", @@ -258,7 +260,7 @@ describe("error-classification", () => { } ${COMPONENT}` ); - expect(r.status).toBe("pass"); + expect(r.status).toBe("not-applicable"); }); }); @@ -494,7 +496,9 @@ describe("request-context", () => { expect(r.status).toBe("fail"); }); - it("passes a route that hands its failures to the central handler", () => { + // C1. The global handler carries requestId, path, host and method, and no tenant. A route that + // never catches cannot name one, so it fails rather than being credited or excused. + it("fails a route that leaves everything to the central handler", () => { const r = run( "request-context", "api.v1.q.ts", @@ -505,7 +509,7 @@ describe("request-context", () => { return json(await prisma.thing.findMany({ where: { environmentId: auth.environment.id } })); }` ); - expect(r.status).toBe("pass"); + expect(r.status).toBe("fail"); }); it("fails a route that catches but only names an identifier outside the catch", () => { @@ -539,7 +543,7 @@ describe("request-context", () => { // A guard around a parse is not the route taking over its failure path: whatever its real work // throws still reaches the central handler. Same reading error-classification gives the field. - it("passes a route whose only catch guards a parse", () => { + it("fails a route whose only catch guards a parse", () => { const r = run( "request-context", "api.v1.q.ts", @@ -551,13 +555,13 @@ describe("request-context", () => { return json(await prisma.thing.findMany({ where: body })); }` ); - expect(r.status).toBe("pass"); + expect(r.status).toBe("fail"); }); // Was a known false positive: `new URL()` is a constructor, so the parse was invisible while the // evidence came from `calleeTexts`. `CatchEvidence.guardsParse` covers constructors, so the guard // is legible now and the route is no longer judged as though it kept its failures. - it("passes a route whose only catch guards a constructor parse", () => { + it("fails a route whose only catch guards a constructor parse", () => { const r = run( "request-context", "_app.@.orgs.$organizationSlug.$.tsx", @@ -572,10 +576,10 @@ describe("request-context", () => { return typedjson({ origin, things: await prisma.thing.findMany() }); }` ); - expect(r.status).toBe("pass"); + expect(r.status).toBe("fail"); }); - it("passes a try/finally that catches nothing", () => { + it("fails a try/finally that catches nothing", () => { const r = run( "request-context", "admin.api.v1.runs-replication.status.ts", @@ -590,7 +594,7 @@ describe("request-context", () => { } }` ); - expect(r.status).toBe("pass"); + expect(r.status).toBe("fail"); }); it("still judges a route with a handler-wide catch beside a narrow guard", () => { diff --git a/internal-packages/observability-map/test/report.test.ts b/internal-packages/observability-map/test/report.test.ts index be977d24d8f..c41dbf6136a 100644 --- a/internal-packages/observability-map/test/report.test.ts +++ b/internal-packages/observability-map/test/report.test.ts @@ -57,12 +57,18 @@ describe("renderTerminal", () => { }` )!; - // Sensitive, score 67: only auth-boundary fails (no guard called), the other two pass because - // there is no try/catch to fumble. + // Sensitive, score 67: guarded and classifies what it catches, but its failure log names + // nobody, so only request-context fails. const sensitiveSixtySeven = scanFile( "api.v1.auth.tokens.ts", - `import { prisma } from "~/db.server"; - export async function action() { return prisma.token.create({ data: {} }); }` + `import { requireUserId } from "~/services/session.server"; + import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function action({ request }) { + const userId = await requireUserId(request); + try { return await prisma.token.create({ data: { userId } }); } + catch (error) { logger.error("failed", { error }); throw error; } + }` )!; // Not sensitive, score 0: worse score than sensitiveSixtySeven, but must still sort after both @@ -80,10 +86,12 @@ describe("renderTerminal", () => { const sensitiveAuditOnly = scanFile( "api.v1.billing.ts", `import { prisma } from "~/db.server"; + import { logger } from "~/services/logger.server"; import { requireUserId } from "~/services/session.server"; - export async function action({ request }: { request: Request }) { - const userId = requireUserId(request); - return prisma.billing.update({ where: { userId }, data: {} }); + export async function action({ request }) { + const userId = await requireUserId(request); + try { return await prisma.billing.update({ where: { userId }, data: {} }); } + catch (error) { logger.error("billing update failed", { userId, error }); throw error; } }` )!; diff --git a/internal-packages/observability-map/test/score.test.ts b/internal-packages/observability-map/test/score.test.ts index ef27f5473c4..01f90ff72d6 100644 --- a/internal-packages/observability-map/test/score.test.ts +++ b/internal-packages/observability-map/test/score.test.ts @@ -17,14 +17,39 @@ export async function loader() { try { return await prisma.thing.findMany(); } catch (e) { return null; } }`; +/** Guarded, classifies what it catches, and names the tenant on the failure path. */ +const CLEAN = `import { requireUserId } from "~/services/session.server"; +import { logger } from "~/services/logger.server"; +import { prisma } from "~/db.server"; +export async function action({ request, params }) { + const userId = await requireUserId(request); + try { + return await prisma.token.create({ data: { userId } }); + } catch (error) { + logger.error("token create failed", { userId, environmentId: params.envId, error }); + throw error; + } +}`; + describe("scoreEntry", () => { - it("scores a builder route 100", () => { - expect(scoreEntry(scanFile("api.v1.a.ts", BUILDER)!).score).toBe(100); + it("scores an entry that passes every applicable check 100", () => { + expect(scoreEntry(scanFile("api.v1.auth.tokens.ts", CLEAN)!).score).toBe(100); + }); + + // A builder wrapper classifies errors for the route, but the route itself catches nothing and + // names nobody on its failure path, so there is one applicable check and it fails. + it("does not credit a builder route for the error handling it does not do", () => { + const scored = scoreEntry(scanFile("api.v1.a.ts", BUILDER)!); + expect(scored.checks.find((c) => c.id === "error-classification")!.status).toBe( + "not-applicable" + ); + expect(scored.checks.find((c) => c.id === "request-context")!.status).toBe("fail"); + expect(scored.score).toBe(0); }); it("excludes audit-trail from the per-entry score", () => { - const scored = scoreEntry(scanFile("api.v1.auth.jwt.ts", BUILDER)!); - expect(scored.checks.some((c) => c.id === "audit-trail")).toBe(true); + const scored = scoreEntry(scanFile("api.v1.auth.jwt.ts", CLEAN)!); + expect(scored.checks.find((c) => c.id === "audit-trail")!.status).toBe("fail"); expect(scored.score).toBe(100); }); @@ -63,18 +88,12 @@ ${BUSY_AND_FAILING}`; describe("buildReport", () => { it("reports the audit gap separately from the score", () => { - const eps = [ - scanFile("api.v1.a.ts", BUILDER)!, - scanFile( - "api.v1.auth.tokens.ts", - `import { prisma } from "~/db.server"; - export async function action() { return prisma.token.create({ data: {} }); }` - )!, - ]; - const report = buildReport(eps, []); + // A sensitive mutation with no audit record, but nothing else wrong: the audit gap is reported + // as its own figure and must not pull the score down with it. + const report = buildReport([scanFile("api.v1.auth.tokens.ts", CLEAN)!], []); expect(report.auditGap.sensitiveMutations).toBe(1); expect(report.auditGap.withAudit).toBe(0); - expect(report.global).toBeGreaterThan(0); + expect(report.global).toBe(100); }); it("records parse failures", () => { From bacda64b3b8fbf54f548ef14073402267d7dfe8a Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Fri, 31 Jul 2026 22:09:31 +0100 Subject: [PATCH 021/106] fix(observability-map): stop a suppression buying a better score score was passed over applicable, and a suppressed check left the denominator, so removing a failing check raised the entry: 33 became 50 became 100 as checks were suppressed. The comment above the code claimed the opposite of what it did. A suppression now takes its check out of the numerator and the denominator, and the result is capped by what the entry would score unsuppressed, so it can only ever hold the number still or lower it. What a suppression buys is removal from the worklist, with a reason on the record. An entry whose every scored check is suppressed no longer reads as measured, and the report prints a suppression count so laundering is visible rather than silent. Tests assert the score effect, not just the check status, which is why the original slipped through. --- .../observability-map/src/report/terminal.ts | 10 +++ .../observability-map/src/score.ts | 42 +++++++++--- .../observability-map/test/report.test.ts | 18 +++++ .../observability-map/test/score.test.ts | 66 +++++++++++++++++++ 4 files changed, 126 insertions(+), 10 deletions(-) diff --git a/internal-packages/observability-map/src/report/terminal.ts b/internal-packages/observability-map/src/report/terminal.ts index c5d19e491f9..312b7b8a16c 100644 --- a/internal-packages/observability-map/src/report/terminal.ts +++ b/internal-packages/observability-map/src/report/terminal.ts @@ -42,6 +42,16 @@ export function renderTerminal(report: MapReport): string { `No audit helper exists in the webapp.` ); + if (report.suppressions.checks > 0) { + const { entries, checks } = report.suppressions; + lines.push( + `SUPPRESSED ${checks} check${checks === 1 ? "" : "s"} across ${entries} entry point${ + entries === 1 ? "" : "s" + }, each with a reason on the record. A suppression removes a finding from this list, ` + + `it does not raise a score.` + ); + } + const worst = report.entries .filter((e) => scoredFailures(e).length > 0) .sort( diff --git a/internal-packages/observability-map/src/score.ts b/internal-packages/observability-map/src/score.ts index 542b968177c..65782c79d05 100644 --- a/internal-packages/observability-map/src/score.ts +++ b/internal-packages/observability-map/src/score.ts @@ -17,6 +17,8 @@ export type ScoredEntry = { * default cannot inflate a figure nobody checked. */ measured: boolean; + /** Scored checks a comment in the source suppressed, in `SCORED_CHECK_IDS` order. */ + suppressed: string[]; /** Passed over applicable, across scored checks only. 100 when nothing applies. */ score: number; }; @@ -27,6 +29,8 @@ export type MapReport = { measured: number; /** Entry points every scored check reported not-applicable for; excluded from `global`. */ unmeasured: number; + /** Suppressions in force: how many entry points carry one, and how many scored checks in total. */ + suppressions: { entries: number; checks: number }; byFamily: Record; sensitiveCohort: { n: number; measured: number; mean: number }; auditGap: { sensitiveMutations: number; withAudit: number }; @@ -36,19 +40,25 @@ export type MapReport = { export function scoreEntry(ep: EntryPoint): ScoredEntry { const suppressed = suppressedChecks(ep.source); - const checks = CHECKS.map((c) => { - const result = c.run(ep); - const reason = suppressed.get(c.id); - // A suppression always lands on not-applicable, never on pass: suppressing a check must remove - // it from the score, not launder it into a point in the entry's favor. + const raw = CHECKS.map((c) => c.run(ep)); + const checks = raw.map((result) => { + const reason = suppressed.get(result.id); return reason - ? { id: c.id, status: "not-applicable" as const, detail: `suppressed: ${reason}` } + ? { id: result.id, status: "not-applicable" as const, detail: `suppressed: ${reason}` } : result; }); - const scored = checks.filter((c) => SCORED_CHECK_IDS.includes(c.id)); - const applicable = scored.filter((c) => c.status !== "not-applicable"); - const passed = applicable.filter((c) => c.status === "pass").length; + const scored = raw.filter((c) => SCORED_CHECK_IDS.includes(c.id)); + const ratio = (of: CheckResult[]) => { + const applicable = of.filter((c) => c.status !== "not-applicable"); + if (applicable.length === 0) return 100; + return Math.round( + (applicable.filter((c) => c.status === "pass").length / applicable.length) * 100 + ); + }; + + const visible = scored.filter((c) => !suppressed.has(c.id)); + const applicable = visible.filter((c) => c.status !== "not-applicable"); return { fileName: ep.fileName, @@ -56,8 +66,14 @@ export function scoreEntry(ep: EntryPoint): ScoredEntry { family: familyOf(ep.fileName), sensitive: classifySensitivity(ep).sensitive, checks, + suppressed: scored.filter((c) => suppressed.has(c.id)).map((c) => c.id), + // Suppressing a check takes it out of the numerator and the denominator, and the result is + // capped by what the entry would have scored unsuppressed. Otherwise removing a failing check + // shrinks the denominator and the ratio climbs, which is how 33 became 50 became 100: the + // suppression comment laundered the finding into a point. What a suppression buys is removal + // from the worklist, with a reason on the record. It cannot buy a better number. measured: applicable.length > 0, - score: applicable.length === 0 ? 100 : Math.round((passed / applicable.length) * 100), + score: Math.min(ratio(visible), ratio(scored)), }; } @@ -98,10 +114,16 @@ export function buildReport(eps: EntryPoint[], parseFailures: string[]): MapRepo e.checks.some((c) => c.id === "audit-trail" && c.status !== "not-applicable") ); + const suppressing = entries.filter((e) => e.suppressed.length > 0); + return { global: mean(measuredEntries.map((e) => e.score)), measured: measuredEntries.length, unmeasured: entries.length - measuredEntries.length, + suppressions: { + entries: suppressing.length, + checks: suppressing.reduce((n, e) => n + e.suppressed.length, 0), + }, byFamily, sensitiveCohort: groupStats(sensitive), auditGap: { diff --git a/internal-packages/observability-map/test/report.test.ts b/internal-packages/observability-map/test/report.test.ts index c41dbf6136a..39fef9730e4 100644 --- a/internal-packages/observability-map/test/report.test.ts +++ b/internal-packages/observability-map/test/report.test.ts @@ -29,6 +29,24 @@ describe("renderTerminal", () => { expect(out).toContain("/api/v1/auth/tokens"); }); + it("surfaces suppressions so laundering is visible rather than silent", () => { + const suppressed = scanFile( + "api.v1.d.ts", + `// obs-map-disable-next-line error-classification -- deliberate, see ticket + import { prisma } from "~/db.server"; + export async function loader() { + try { return await prisma.thing.findMany(); } catch (e) { return null; } + }` + )!; + const out = renderTerminal(buildReport([suppressed], [])); + expect(out).toMatch(/suppress/i); + expect(out).toContain("1"); + }); + + it("does not mention suppressions when there are none", () => { + expect(renderTerminal(report())).not.toMatch(/suppress/i); + }); + it("surfaces parse failures so the denominator is not silently wrong", () => { expect(renderTerminal(report())).toContain("broken.ts"); }); diff --git a/internal-packages/observability-map/test/score.test.ts b/internal-packages/observability-map/test/score.test.ts index 01f90ff72d6..f962aa2b8c8 100644 --- a/internal-packages/observability-map/test/score.test.ts +++ b/internal-packages/observability-map/test/score.test.ts @@ -73,6 +73,57 @@ ${BUSY_AND_FAILING}`; ); }); + // I1. `score = passed / applicable` meant removing a failing check from the denominator raised + // the entry's score, so suppression laundered findings into points. A suppression buys removal + // from the worklist, never a better number. + it("does not raise the score when a failing check is suppressed", () => { + const source = `import { requireUserId } from "~/services/session.server"; +import { prisma } from "~/db.server"; +export async function action({ request }) { + const userId = await requireUserId(request); + try { return await prisma.token.create({ data: { userId } }); } + catch (error) { return null; } +}`; + const plain = scoreEntry(scanFile("api.v1.auth.tokens.ts", source)!); + const suppressed = scoreEntry( + scanFile( + "api.v1.auth.tokens.ts", + `// obs-map-disable-next-line error-classification -- deliberate, see ticket +${source}` + )! + ); + + expect(plain.checks.find((c) => c.id === "error-classification")!.status).toBe("fail"); + expect(suppressed.checks.find((c) => c.id === "error-classification")!.status).toBe( + "not-applicable" + ); + expect(suppressed.score).toBeLessThanOrEqual(plain.score); + }); + + it("records which scored checks were suppressed", () => { + const suppressed = scoreEntry( + scanFile( + "api.v1.b.ts", + `// obs-map-disable-next-line error-classification -- health probe +// obs-map-disable-next-line request-context -- nothing to name here +${BUSY_AND_FAILING}` + )! + ); + expect(suppressed.suppressed).toEqual(["error-classification", "request-context"]); + }); + + it("does not let an entry whose every scored check is suppressed read as measured", () => { + const suppressed = scoreEntry( + scanFile( + "api.v1.b.ts", + `// obs-map-disable-next-line error-classification -- health probe +// obs-map-disable-next-line request-context -- nothing to name here +${BUSY_AND_FAILING}` + )! + ); + expect(suppressed.measured).toBe(false); + }); + it("marks an entry point with nothing applicable as unmeasured, scored 100", () => { const scored = scoreEntry(scanFile("resources.health.ts", TRIVIAL)!); expect(scored.checks.every((c) => c.status === "not-applicable")).toBe(true); @@ -87,6 +138,21 @@ ${BUSY_AND_FAILING}`; }); describe("buildReport", () => { + it("counts suppressions so laundering is visible in the report", () => { + const report = buildReport( + [ + scanFile( + "api.v1.b.ts", + `// obs-map-disable-next-line error-classification -- health probe +${BUSY_AND_FAILING}` + )!, + scanFile("api.v1.c.ts", BUSY_AND_FAILING)!, + ], + [] + ); + expect(report.suppressions).toEqual({ entries: 1, checks: 1 }); + }); + it("reports the audit gap separately from the score", () => { // A sensitive mutation with no audit record, but nothing else wrong: the audit gap is reported // as its own figure and must not pull the score down with it. From 8631151b64d4d2dde9546f0b76c0b5dee0a20cba Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Fri, 31 Jul 2026 22:12:11 +0100 Subject: [PATCH 022/106] fix(observability-map): break the circularity in sensitivity requireAdminApiRequest was on the sensitive-symbol list, so 34 of the 67 sensitive entry points were sensitive only because they call the admin guard, which auth-boundary then passed them for. A mitigation cannot be what makes a route risky, and this list is the fix list's primary sort key. The tokens path segment was matching waitpoint routes: seven of the eight matches were run coordination handles rather than credentials. A token segment directly under waitpoints no longer counts. The cohort falls from 67 to 26, and the two remaining token matches are the personal access token routes the segment was meant to find. --- .../observability-map/src/sensitivity.ts | 16 +++- .../observability-map/test/checks.test.ts | 18 +++-- .../test/sensitivity.test.ts | 73 +++++++++++++++++++ 3 files changed, 96 insertions(+), 11 deletions(-) diff --git a/internal-packages/observability-map/src/sensitivity.ts b/internal-packages/observability-map/src/sensitivity.ts index 40f348afac7..b80beee65e5 100644 --- a/internal-packages/observability-map/src/sensitivity.ts +++ b/internal-packages/observability-map/src/sensitivity.ts @@ -1,10 +1,15 @@ import type { EntryPoint } from "./types.js"; import { routePathOf } from "./adapters/remix.js"; +/** + * Symbols whose presence says the route does something risky. Calling a guard is not one of them: + * `requireAdminApiRequest` was on this list and made 34 of the 67 sensitive entry points sensitive + * purely because they were guarded, which `auth-boundary` then passed them for. A mitigation + * cannot be the hazard, and this list feeds the fix list's primary sort key. + */ const SENSITIVE_SYMBOLS = [ "clearImpersonation", "setImpersonation", - "requireAdminApiRequest", "createPersonalAccessToken", "regenerateApiKey", "createJWT", @@ -47,8 +52,13 @@ export function classifySensitivity(ep: EntryPoint): Sensitivity { const segments = routePathOf(ep.fileName) .split("/") .filter((s) => s.length > 0); - for (const seg of segments) { - if (SENSITIVE_SEGMENTS.includes(seg)) reasons.push(`path segment "${seg}"`); + for (const [i, seg] of segments.entries()) { + if (!SENSITIVE_SEGMENTS.includes(seg)) continue; + // A waitpoint token is a handle for resuming a run, not a credential. Seven of the eight + // `tokens` matches in the tree were waitpoint routes, so the segment on its own was mostly + // finding the wrong thing. + if ((seg === "token" || seg === "tokens") && segments[i - 1] === "waitpoints") continue; + reasons.push(`path segment "${seg}"`); } return { sensitive: reasons.length > 0, reasons }; diff --git a/internal-packages/observability-map/test/checks.test.ts b/internal-packages/observability-map/test/checks.test.ts index 856a221c6b6..081db1661ed 100644 --- a/internal-packages/observability-map/test/checks.test.ts +++ b/internal-packages/observability-map/test/checks.test.ts @@ -266,14 +266,16 @@ describe("error-classification", () => { describe("auth-boundary", () => { it("passes a sensitive route guarded by a require helper", () => { + // Sensitive on the impersonation call, not on the guard: calling a guard is not what makes a + // route sensitive, see sensitivity.test.ts. const r = run( "auth-boundary", - "admin.api.v1.gc.ts", + "admin.api.v1.impersonate.ts", `import { requireAdminApiRequest } from "~/services/personalAccessToken.server"; - import { prisma } from "~/db.server"; - export async function loader({ request }) { + import { setImpersonation } from "~/models/admin.server"; + export async function action({ request }) { await requireAdminApiRequest(request); - return prisma.thing.findMany(); + return setImpersonation(request, "user_1"); }` ); expect(r.status).toBe("pass"); @@ -370,12 +372,12 @@ describe("auth-boundary", () => { it("passes a sensitive callback guarded by a signature check", () => { const r = run( "auth-boundary", - "api.v1.waitpoints.tokens.$waitpointFriendlyId.callback.$hash.ts", - `import { verifyHttpCallbackHash } from "~/services/httpCallback.server"; + "webhooks.v1.billing.$hash.ts", + `import { verifyWebhookSignature } from "~/services/webhooks.server"; import { prisma } from "~/db.server"; export async function action({ request, params }) { - const waitpoint = await prisma.waitpoint.findFirst({ where: { id: params.id } }); - if (!verifyHttpCallbackHash(params.hash, waitpoint)) { + const invoice = await prisma.invoice.findFirst({ where: { id: params.id } }); + if (!verifyWebhookSignature(params.hash, invoice)) { return json({ error: "Invalid" }, { status: 401 }); } return json({ ok: true }); diff --git a/internal-packages/observability-map/test/sensitivity.test.ts b/internal-packages/observability-map/test/sensitivity.test.ts index 9a5c12e5f6a..d7ff63edde3 100644 --- a/internal-packages/observability-map/test/sensitivity.test.ts +++ b/internal-packages/observability-map/test/sensitivity.test.ts @@ -99,3 +99,76 @@ describe("classifySensitivity: calleeNames is body-scoped, importedNames is file expect(classifySensitivity(e).sensitive).toBe(false); }); }); + +describe("what sensitivity must not mean", () => { + const ep = (fileName: string, source: string) => scanFile(fileName, source)!; + + // C4. Calling the admin guard cannot be what makes a route risky: it is the mitigation, not the + // hazard. Counting it made 34 of 67 sensitive entries sensitive only because they were guarded, + // and `auth-boundary` then passed every one of them on the same call. Circular, and it was the + // fix list's primary sort key. + it("does not treat calling the admin guard as what makes a route sensitive", () => { + const s = classifySensitivity( + ep( + "admin.api.v1.queue-metrics.ts", + `import { requireAdminApiRequest } from "~/services/personalAccessToken.server"; + import { prisma } from "~/db.server"; + export async function loader({ request }) { + await requireAdminApiRequest(request); + return json(await prisma.queueMetric.findMany()); + }` + ) + ); + expect(s.sensitive).toBe(false); + }); + + it("still flags an admin route that does something sensitive on its own account", () => { + const s = classifySensitivity( + ep( + "admin.api.v1.impersonate.ts", + `import { requireAdminApiRequest } from "~/services/personalAccessToken.server"; + import { setImpersonation } from "~/models/admin.server"; + export async function action({ request }) { + await requireAdminApiRequest(request); + return setImpersonation(request, "user_1"); + }` + ) + ); + expect(s.sensitive).toBe(true); + expect(s.reasons).toContain("calls setImpersonation"); + }); + + // A waitpoint token is a run coordination handle, not a credential. Seven of the eight routes + // matching the `tokens` segment were waitpoint routes. + it("does not treat a waitpoint token route as a credential route", () => { + const s = classifySensitivity( + ep( + "api.v1.waitpoints.tokens.$waitpointFriendlyId.complete.ts", + `import { prisma } from "~/db.server"; + export async function action() { return prisma.waitpoint.update({ where: {}, data: {} }); }` + ) + ); + expect(s.sensitive).toBe(false); + }); + + it("still flags the personal access token routes", () => { + expect( + classifySensitivity( + ep( + "account.tokens/route.tsx", + `import { prisma } from "~/db.server"; + export async function loader() { return prisma.personalAccessToken.findMany(); }` + ) + ).reasons + ).toContain('path segment "tokens"'); + expect( + classifySensitivity( + ep( + "api.v1.token.ts", + `import { prisma } from "~/db.server"; + export async function action() { return prisma.token.create({ data: {} }); }` + ) + ).reasons + ).toContain('path segment "token"'); + }); +}); From fe629644413df77024af7f6fcf381cd4a98b26a3 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Fri, 31 Jul 2026 22:18:55 +0100 Subject: [PATCH 023/106] fix(observability-map): make the report and the CLI say what they mean An empty group meant mean 100, so a family with nothing measured rendered a full green bar; it renders as not measured now, and so does a global score with nothing behind it. The audit sentence claiming no helper exists is only printed when the figure in front of it says zero, and the whole line is skipped when no sensitive mutation was found. 'already solid' counted entries that pass because they do nothing alongside entries nothing applied to, so it now reports the two separately. The suppression directive is read from comments only, line by line. Matching the raw source meant a string literal quoting the directive switched a real check off. The CLI accepts the route paths the report prints, not just file names, so the identifier on screen can be pasted back in. An exact match wins over the routes it is a prefix of, and an ambiguous prefix warns instead of silently taking the first. cli.ts had no tests and now has six, which needed the entry point guarded so importing the module does not scan the tree. Also unexports BUILDERS, isParseGuard and accountedFor, none of which had a caller outside their own module. --- .../src/checks/errorClassification.ts | 6 +- .../observability-map/src/checks/index.ts | 1 - .../observability-map/src/cli.ts | 64 +++++++++++++---- .../observability-map/src/report/terminal.ts | 43 ++++++++---- .../observability-map/src/score.ts | 21 ++++-- .../observability-map/src/suppression.ts | 42 +++++++++-- .../observability-map/test/checks.test.ts | 8 ++- .../observability-map/test/cli.test.ts | 60 ++++++++++++++++ .../observability-map/test/report.test.ts | 70 +++++++++++++++++++ 9 files changed, 271 insertions(+), 44 deletions(-) create mode 100644 internal-packages/observability-map/test/cli.test.ts diff --git a/internal-packages/observability-map/src/checks/errorClassification.ts b/internal-packages/observability-map/src/checks/errorClassification.ts index f4a8ce5e8b2..c2d46b3af5b 100644 --- a/internal-packages/observability-map/src/checks/errorClassification.ts +++ b/internal-packages/observability-map/src/checks/errorClassification.ts @@ -13,7 +13,7 @@ const ID = "error-classification"; * authenticate, so counting it here would hand two routes a free pass on `auth-boundary`. * `createHybridActionApiRoute`, which the design named, exists nowhere in the tree. */ -export const BUILDERS = new Set([ +const BUILDERS = new Set([ "createLoaderApiRoute", "createActionApiRoute", "createLoaderPATApiRoute", @@ -33,7 +33,7 @@ export const BUILDERS = new Set([ * than a second absolute threshold, so it holds for a three-statement route and a fifty-statement * one alike. */ -export function isParseGuard(clause: CatchEvidence, ep: EntryPoint): boolean { +function isParseGuard(clause: CatchEvidence, ep: EntryPoint): boolean { return clause.guardsParse && clause.tryStatementCount * 2 < ep.statementCount; } @@ -51,7 +51,7 @@ export function isParseGuard(clause: CatchEvidence, ep: EntryPoint): boolean { * effort side work that logs and carries on, and all four are non-sensitive so they sort to the * bottom of the fix list. See the task 5 report; switching is one limb in this function. */ -export function accountedFor(clause: CatchEvidence, ep: EntryPoint): boolean { +function accountedFor(clause: CatchEvidence, ep: EntryPoint): boolean { return clause.rethrows || clause.branches || isParseGuard(clause, ep); } diff --git a/internal-packages/observability-map/src/checks/index.ts b/internal-packages/observability-map/src/checks/index.ts index b94c8ad4867..73025d0c5c8 100644 --- a/internal-packages/observability-map/src/checks/index.ts +++ b/internal-packages/observability-map/src/checks/index.ts @@ -9,4 +9,3 @@ export type Check = { id: string; run: (ep: EntryPoint) => CheckResult }; /** audit-trail is scored separately, see score.ts. */ export const CHECKS: Check[] = [errorClassification, authBoundary, requestContext, auditTrail]; export const SCORED_CHECK_IDS = ["error-classification", "auth-boundary", "request-context"]; -export { usesBuilder, BUILDERS } from "./errorClassification.js"; diff --git a/internal-packages/observability-map/src/cli.ts b/internal-packages/observability-map/src/cli.ts index 360f39a84f6..6feef381e33 100644 --- a/internal-packages/observability-map/src/cli.ts +++ b/internal-packages/observability-map/src/cli.ts @@ -3,6 +3,7 @@ import { dirname, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { scanDirectory } from "./scan.js"; import { buildReport, scoreEntry } from "./score.js"; +import { routePathOf } from "./adapters/remix.js"; import { renderTerminal } from "./report/terminal.js"; import { renderJson } from "./report/json.js"; @@ -25,7 +26,36 @@ function findRepoRoot(startDir: string): string { throw new Error("could not find repo root (no pnpm-workspace.yaml in any parent directory)"); } -export function main(argv: string[]): number { +/** Where output goes. Injectable so the tests can read it without spawning a process. */ +export type Io = { out: (s: string) => void; err: (s: string) => void }; + +const processIo: Io = { + out: (s) => process.stdout.write(s), + err: (s) => process.stderr.write(s), +}; + +/** + * Entry points matching what the user typed, by file name or by route path, exact first. + * + * Route paths matter because they are what the report prints: `map /api/v1/token` used to exit 1 + * because only the file name was matched, so the identifier on screen was not one you could paste + * back in. + */ +function findMatches(entryPoints: { fileName: string }[], target: string) { + const asPath = target.startsWith("/") ? target : `/${target}`; + const asFile = target.replace(/^\//, ""); + + const exact = entryPoints.filter( + (e) => e.fileName === target || routePathOf(e.fileName) === asPath + ); + if (exact.length > 0) return exact; + + return entryPoints.filter( + (e) => e.fileName.startsWith(asFile) || routePathOf(e.fileName).startsWith(asPath) + ); +} + +export function main(argv: string[], io: Io = processIo): number { const args = argv.slice(2); const asJson = args.includes("--json"); const noWrite = args.includes("--no-write"); @@ -36,32 +66,42 @@ export function main(argv: string[]): number { const { entryPoints, parseFailures } = scanDirectory(routesDir); if (target) { - const match = entryPoints.find( - (e) => e.fileName === target || e.fileName.startsWith(target.replace(/^\//, "")) - ); - if (!match) { - process.stderr.write(`no entry point matching "${target}"\n`); + const matches = findMatches(entryPoints, target); + if (matches.length === 0) { + io.err(`no entry point matching "${target}"\n`); return 1; } - const scored = scoreEntry(match); + if (matches.length > 1) { + const others = matches.slice(1, 4).map((m) => routePathOf(m.fileName)); + const rest = matches.length - 1 - others.length; + io.err( + `"${target}" matches ${matches.length} entry points, showing the first. ` + + `Others: ${others.join(", ")}${rest > 0 ? `, and ${rest} more` : ""}\n` + ); + } + const scored = scoreEntry(matches[0]!); const measuredNote = scored.measured ? "" : " (not measured: no applicable checks)"; - process.stdout.write( + io.out( `${scored.routePath} ${scored.score}/100${measuredNote}\n${scored.fileName}\n\nCHECKS\n` ); for (const c of scored.checks) { const mark = c.status === "pass" ? "PASS" : c.status === "fail" ? "FAIL" : "n/a "; - process.stdout.write(` ${mark} ${c.id}${c.detail ? ` (${c.detail})` : ""}\n`); + io.out(` ${mark} ${c.id}${c.detail ? ` (${c.detail})` : ""}\n`); } return 0; } const report = buildReport(entryPoints, parseFailures); - process.stdout.write(asJson ? renderJson(report) : renderTerminal(report)); - process.stdout.write("\n"); + io.out(asJson ? renderJson(report) : renderTerminal(report)); + io.out("\n"); if (!noWrite) { writeFileSync(resolve(repoRoot, "observability-map.json"), renderJson(report)); } return 0; } -process.exitCode = main(process.argv); +// Only when run as a program. Importing the module, which the tests do, must not scan the tree or +// write a report. +const invokedDirectly = + process.argv[1] !== undefined && resolve(process.argv[1]) === fileURLToPath(import.meta.url); +if (invokedDirectly) process.exitCode = main(process.argv); diff --git a/internal-packages/observability-map/src/report/terminal.ts b/internal-packages/observability-map/src/report/terminal.ts index 312b7b8a16c..21facadb21c 100644 --- a/internal-packages/observability-map/src/report/terminal.ts +++ b/internal-packages/observability-map/src/report/terminal.ts @@ -1,9 +1,13 @@ import type { MapReport, ScoredEntry } from "../score.js"; import { SCORED_CHECK_IDS } from "../checks/index.js"; -const bar = (score: number) => { +const NOT_MEASURED = "not measured".padEnd(15); + +/** A bar and a figure, or a plain "not measured" where there is no figure to draw. */ +const gauge = (score: number | null) => { + if (score === null) return NOT_MEASURED; const filled = Math.round(score / 10); - return "▰".repeat(filled) + "▱".repeat(10 - filled); + return `${"▰".repeat(filled)}${"▱".repeat(10 - filled)} ${String(score).padStart(3)}`; }; /** @@ -19,28 +23,36 @@ const scoredFailures = (e: ScoredEntry) => export function renderTerminal(report: MapReport): string { const lines: string[] = []; + const headline = report.global === null ? "score not measured" : `score ${report.global}/100`; lines.push( - `score ${report.global}/100 ${report.measured} measured, ${report.unmeasured} unmeasured of ${report.entries.length} entry points` + `${headline} ${report.measured} measured, ${report.unmeasured} unmeasured of ${report.entries.length} entry points` ); lines.push(""); lines.push("COVERAGE"); for (const [family, stats] of Object.entries(report.byFamily).sort((a, b) => b[1].n - a[1].n)) { lines.push( - ` ${family.padEnd(12)} ${bar(stats.mean)} ${String(stats.mean).padStart(3)} ${stats.measured}/${stats.n} entry points` + ` ${family.padEnd(12)} ${gauge(stats.mean)} ${stats.measured}/${stats.n} entry points` ); } lines.push( - ` ${"sensitive".padEnd(12)} ${bar(report.sensitiveCohort.mean)} ${String( - report.sensitiveCohort.mean - ).padStart(3)} ${report.sensitiveCohort.measured}/${report.sensitiveCohort.n} entry points` + ` ${"sensitive".padEnd(12)} ${gauge(report.sensitiveCohort.mean)} ${ + report.sensitiveCohort.measured + }/${report.sensitiveCohort.n} entry points` ); - lines.push(""); const { sensitiveMutations, withAudit } = report.auditGap; - lines.push( - `AUDIT ${withAudit} of ${sensitiveMutations} sensitive mutations record an actor. ` + - `No audit helper exists in the webapp.` - ); + if (sensitiveMutations > 0) { + lines.push(""); + // The closing sentence is a claim about the codebase, so it is only made when the figure in + // front of it supports it. It was printed unconditionally, including next to a non-zero count. + const gap = + withAudit === 0 + ? " No audit helper exists in the webapp." + : ` ${sensitiveMutations - withAudit} without one.`; + lines.push( + `AUDIT ${withAudit} of ${sensitiveMutations} sensitive mutations record an actor.${gap}` + ); + } if (report.suppressions.checks > 0) { const { entries, checks } = report.suppressions; @@ -78,7 +90,12 @@ export function renderTerminal(report: MapReport): string { } lines.push(""); - lines.push(`already solid: ${report.entries.length - worst.length}`); + // Not one flattering number: an entry with nothing applicable is not the same as an entry that + // passed, and lumping them together counted routes as solid for doing nothing. + const clean = report.entries.filter((e) => e.measured && scoredFailures(e).length === 0).length; + lines.push( + `no findings: ${clean} passed every applicable check, ${report.unmeasured} had none to apply` + ); if (report.parseFailures.length > 0) { lines.push(`parse failures (excluded from the score): ${report.parseFailures.join(", ")}`); } diff --git a/internal-packages/observability-map/src/score.ts b/internal-packages/observability-map/src/score.ts index 65782c79d05..27aaa680950 100644 --- a/internal-packages/observability-map/src/score.ts +++ b/internal-packages/observability-map/src/score.ts @@ -24,15 +24,16 @@ export type ScoredEntry = { }; export type MapReport = { - global: number; + /** Null when no entry point had an applicable scored check: an absent figure, not a perfect one. */ + global: number | null; /** Entry points with at least one applicable scored check, i.e. those `global` is averaged over. */ measured: number; /** Entry points every scored check reported not-applicable for; excluded from `global`. */ unmeasured: number; /** Suppressions in force: how many entry points carry one, and how many scored checks in total. */ suppressions: { entries: number; checks: number }; - byFamily: Record; - sensitiveCohort: { n: number; measured: number; mean: number }; + byFamily: Record; + sensitiveCohort: { n: number; measured: number; mean: number | null }; auditGap: { sensitiveMutations: number; withAudit: number }; entries: ScoredEntry[]; parseFailures: string[]; @@ -77,8 +78,12 @@ export function scoreEntry(ep: EntryPoint): ScoredEntry { }; } -const mean = (xs: number[]) => - xs.length === 0 ? 100 : Math.round(xs.reduce((a, b) => a + b, 0) / xs.length); +/** + * Null for an empty group rather than 100. A family nothing was measured in has no score, and + * rendering the absence as a full green bar said the opposite of what the data said. + */ +const mean = (xs: number[]): number | null => + xs.length === 0 ? null : Math.round(xs.reduce((a, b) => a + b, 0) / xs.length); /** * `n` is every entry point in the group; `mean` is taken over the measured subset only, so an @@ -86,7 +91,11 @@ const mean = (xs: number[]) => * is reported alongside so a reader can tell a family scoring high because it is clean apart from * a family scoring high because most of it was never measured. */ -function groupStats(entries: ScoredEntry[]): { n: number; measured: number; mean: number } { +function groupStats(entries: ScoredEntry[]): { + n: number; + measured: number; + mean: number | null; +} { const measuredEntries = entries.filter((e) => e.measured); return { n: entries.length, diff --git a/internal-packages/observability-map/src/suppression.ts b/internal-packages/observability-map/src/suppression.ts index bd807816254..8f7352f6a46 100644 --- a/internal-packages/observability-map/src/suppression.ts +++ b/internal-packages/observability-map/src/suppression.ts @@ -1,13 +1,43 @@ -// The reason runs to the end of the line: `.` does not match a newline, so a suppression on one -// line cannot pick up a reason from the next one. -const PATTERN = /obs-map-disable-next-line\s+([a-z-]+)\s+--\s+(.+)/g; +/** + * The directive, and the reason that must follow it. The reason runs to the end of the line: `.` + * does not match a newline, so a suppression on one line cannot pick up a reason from the next. + * A trailing block-comment terminator is trimmed off so it does not end up inside the reason. + */ +const PATTERN = /obs-map-disable-next-line\s+([a-z-]+)\s+--\s+(.+)/; -/** Check id to reason. A suppression without a reason is ignored. */ +/** + * The comment part of a line, or null if there is none. + * + * Line-scoped and comment-only, because the directive is a comment directive. Matching the raw + * source meant a string literal that merely quotes the directive, in a test fixture or an error + * message, silently switched a real check off. Handles line comments, block comments and the + * leading star of a jsdoc block; a line-comment marker inside a string on the same line can still + * be misread, which costs a suppression that was never written rather than hiding one that was. + */ +function commentPart(line: string): string | null { + const slashes = line.indexOf("//"); + if (slashes !== -1) return line.slice(slashes + 2); + + const block = line.indexOf("/*"); + if (block !== -1) return line.slice(block + 2).replace(/\*\/\s*$/, ""); + + const trimmed = line.trimStart(); + if (trimmed.startsWith("*")) return trimmed.slice(1); + + return null; +} + +/** Check id to reason. A suppression without a reason, or outside a comment, is ignored. */ export function suppressedChecks(source: string): Map { const out = new Map(); - for (const match of source.matchAll(PATTERN)) { + for (const line of source.split("\n")) { + const comment = commentPart(line); + if (comment === null) continue; + const match = PATTERN.exec(comment); + if (!match) continue; const [, id, reason] = match; - if (id && reason && reason.trim().length > 0) out.set(id, reason.trim()); + const trimmedReason = reason?.replace(/\*\/\s*$/, "").trim(); + if (id && trimmedReason && trimmedReason.length > 0) out.set(id, trimmedReason); } return out; } diff --git a/internal-packages/observability-map/test/checks.test.ts b/internal-packages/observability-map/test/checks.test.ts index 081db1661ed..6aeb75a7ecb 100644 --- a/internal-packages/observability-map/test/checks.test.ts +++ b/internal-packages/observability-map/test/checks.test.ts @@ -64,7 +64,7 @@ describe("error-classification", () => { expect(r.status).toBe("fail"); }); - // Restored from the brief: `catchBranches` sees the `instanceof` and the `if`. + // Restored from the brief: the clause branches, on the `instanceof` and the `if`. it("passes a raw route whose catch branches on the error", () => { const r = run( "error-classification", @@ -135,7 +135,8 @@ describe("error-classification", () => { }); // A narrow guard around one operation classifies an expected failure without needing to branch - // or rethrow. `catchesNarrowly` is what tells it apart from a handler-wide catch. + // or rethrow. Guarding a parse over a small part of the body is what tells it apart from a + // handler-wide catch. it("passes a narrow guard around a single parse", () => { const r = run( "error-classification", @@ -153,7 +154,8 @@ describe("error-classification", () => { }); // False positive fixture for the narrow rule: a narrow parse guard must not launder the broad - // handler catch sitting next to it. `catchesNarrowly` is false when any guarded try is broad. + // handler catch sitting next to it. Clauses are judged one at a time, so the broad one still + // counts against the entry point. it("still fails when a narrow guard sits beside a handler-wide swallow", () => { const r = run( "error-classification", diff --git a/internal-packages/observability-map/test/cli.test.ts b/internal-packages/observability-map/test/cli.test.ts new file mode 100644 index 00000000000..2dec21ca9a8 --- /dev/null +++ b/internal-packages/observability-map/test/cli.test.ts @@ -0,0 +1,60 @@ +import { main, type Io } from "../src/cli.js"; + +const capture = () => { + const out: string[] = []; + const err: string[] = []; + const io: Io = { out: (s) => out.push(s), err: (s) => err.push(s) }; + return { io, out: () => out.join(""), err: () => err.join("") }; +}; + +const run = (...args: string[]) => { + const c = capture(); + const code = main(["node", "cli.js", ...args], c.io); + return { code, out: c.out(), err: c.err() }; +}; + +describe("map ", () => { + // I8. The report prints route paths, so the identifier on screen has to be one you can paste + // back in. Matching file names only meant `map /api/v1/token` exited 1. + it("accepts the route path the report prints", () => { + const r = run("/api/v1/token"); + expect(r.code).toBe(0); + expect(r.out).toContain("/api/v1/token"); + expect(r.out).toContain("CHECKS"); + }); + + it("accepts a file name too", () => { + const r = run("api.v1.token.ts"); + expect(r.code).toBe(0); + expect(r.out).toContain("api.v1.token.ts"); + }); + + it("exits 1 with a message when nothing matches", () => { + const r = run("/api/v1/does-not-exist"); + expect(r.code).toBe(1); + expect(r.err).toContain("no entry point matching"); + }); + + it("warns when a prefix matches more than one route rather than silently taking the first", () => { + const r = run("/admin/api/v1/runs-replication"); + expect(r.code).toBe(0); + expect(r.err).toMatch(/matches \d+ entry points, showing the first/); + expect(r.err).toContain("Others:"); + }); + + // `/api/v1/runs` is a prefix of a dozen others, and also a route in its own right. + it("prefers an exact match over the routes it is a prefix of", () => { + const r = run("/api/v1/runs"); + expect(r.err).toBe(""); + expect(r.out.split("\n")[1]).toBe("api.v1.runs.ts"); + }); +}); + +describe("map", () => { + it("renders the whole report without writing when asked not to", () => { + const r = run("--no-write"); + expect(r.code).toBe(0); + expect(r.out).toContain("COVERAGE"); + expect(r.out).toContain("FIX FIRST"); + }); +}); diff --git a/internal-packages/observability-map/test/report.test.ts b/internal-packages/observability-map/test/report.test.ts index 39fef9730e4..7a28c7a4b9c 100644 --- a/internal-packages/observability-map/test/report.test.ts +++ b/internal-packages/observability-map/test/report.test.ts @@ -136,3 +136,73 @@ describe("renderJson", () => { expect(Array.isArray(parsed.entries)).toBe(true); }); }); + +describe("rendering honestly when there is nothing to say", () => { + // I4. mean([]) returned 100, so a family with nothing measured rendered a full green bar. + it("renders a family with nothing measured as not measured, not as 100", () => { + const trivial = scanFile( + "resources.health.ts", + `export const loader = () => new Response("ok");` + )!; + const out = renderTerminal(buildReport([trivial], [])); + const line = out.split("\n").find((l) => l.includes("resources"))!; + expect(line).not.toMatch(/100/); + expect(line).toMatch(/not measured/i); + }); + + it("renders the global score as not measured when nothing was measured", () => { + const trivial = scanFile( + "resources.health.ts", + `export const loader = () => new Response("ok");` + )!; + expect(renderTerminal(buildReport([trivial], []))).toMatch(/score not measured/i); + }); + + // I11. The audit sentence was printed unconditionally, including when the figure said otherwise. + it("does not claim no audit helper exists when one is in use", () => { + const audited = scanFile( + "api.v1.auth.tokens.ts", + `import { auditLog } from "~/services/audit.server"; + import { prisma } from "~/db.server"; + export async function action() { + const token = await prisma.token.create({ data: {} }); + await auditLog("token.created", { tokenId: token.id }); + return json(token); + }` + )!; + const out = renderTerminal(buildReport([audited], [])); + expect(out).toContain("1 of 1"); + expect(out).not.toContain("No audit helper exists"); + }); + + it("says nothing about audit when no sensitive mutation was found", () => { + const plain = scanFile( + "resources.things.ts", + `import { prisma } from "~/db.server"; + export async function loader() { return prisma.thing.findMany(); }` + )!; + expect(renderTerminal(buildReport([plain], []))).not.toMatch(/AUDIT/); + }); + + // I5. "already solid" counted entries that are clean because they do nothing, alongside entries + // nothing applied to, in one flattering number. + it("separates entries that passed from entries nothing applied to", () => { + const clean = scanFile( + "api.v1.clean.ts", + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function loader({ params }) { + try { return await prisma.thing.findMany(); } + catch (error) { logger.error("failed", { environmentId: params.envId, error }); throw error; } + }` + )!; + const trivial = scanFile( + "resources.health.ts", + `export const loader = () => new Response("ok");` + )!; + const out = renderTerminal(buildReport([clean, trivial], [])); + expect(out).not.toMatch(/already solid/i); + expect(out).toMatch(/1 passed every applicable check/i); + expect(out).toMatch(/1 had none to apply/i); + }); +}); From 2f4823a7af12e36ed02810aec19601046208f1b7 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Fri, 31 Jul 2026 22:19:17 +0100 Subject: [PATCH 024/106] test(observability-map): cover the comment-only suppression rule The directive inside a string literal must not suppress anything, and block comments and jsdoc lines must be read like line comments. --- .../test/suppression.test.ts | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/internal-packages/observability-map/test/suppression.test.ts b/internal-packages/observability-map/test/suppression.test.ts index 8e8e88d5fa5..0bb6fcc5f9b 100644 --- a/internal-packages/observability-map/test/suppression.test.ts +++ b/internal-packages/observability-map/test/suppression.test.ts @@ -42,4 +42,40 @@ describe("suppressedChecks", () => { ); expect(m.size).toBe(0); }); + + // I2. The directive is a comment directive. Matching it file-wide meant a string literal that + // merely quotes it, in a test fixture or an error message, silently suppressed a real check. + it("ignores the directive inside a string literal", () => { + const m = suppressedChecks( + `const example = "obs-map-disable-next-line error-classification -- not a real suppression"; + export async function loader() { return 1; }` + ); + expect(m.size).toBe(0); + }); + + it("reads the directive from a block comment", () => { + const m = suppressedChecks( + `/* obs-map-disable-next-line auth-boundary -- public by design, see ADR 12 */ + export async function loader() { return 1; }` + ); + expect(m.get("auth-boundary")).toBe("public by design, see ADR 12"); + }); + + it("reads the directive from a jsdoc line", () => { + const m = suppressedChecks( + `/** + * obs-map-disable-next-line request-context -- nothing tenant-scoped here + */ + export async function loader() { return 1; }` + ); + expect(m.get("request-context")).toBe("nothing tenant-scoped here"); + }); + + it("ignores code that happens to follow a comment on the same line", () => { + const m = suppressedChecks( + `const x = 1; // obs-map-disable-next-line error-classification -- fine + export async function loader() { return x; }` + ); + expect(m.get("error-classification")).toBe("fine"); + }); }); From ec5922a32b2c9f19c1f0e8c7a49c916d46fb14de Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Fri, 31 Jul 2026 22:27:27 +0100 Subject: [PATCH 025/106] fix(observability-map): report the request-context gap as a figure, not a list request-context fails 391 of the 412 entry points it applies to, so listing each one turned the fix list into a single house-style finding repeated. That is the same reason audit-trail is kept out of the list, at 52 rather than 391. An entry whose only finding is request-context now collapses into a CONTEXT line reading how many entry points name a tenant on a failure path, and the line says how many appear only there so they are not mistaken for having disappeared. An entry that fails something else keeps its full set of findings, so /account/tokens still shows the request-context gap alongside the swallow. The check stays fully in the score. The gap is real and the score is meant to show it; only the presentation changes. --- .../observability-map/src/report/terminal.ts | 26 ++++++- .../observability-map/src/score.ts | 14 ++++ .../observability-map/test/report.test.ts | 68 +++++++++++++++---- .../observability-map/test/score.test.ts | 24 +++++++ 4 files changed, 118 insertions(+), 14 deletions(-) diff --git a/internal-packages/observability-map/src/report/terminal.ts b/internal-packages/observability-map/src/report/terminal.ts index 21facadb21c..868acf284d6 100644 --- a/internal-packages/observability-map/src/report/terminal.ts +++ b/internal-packages/observability-map/src/report/terminal.ts @@ -20,6 +20,18 @@ const gauge = (score: number | null) => { const scoredFailures = (e: ScoredEntry) => e.checks.filter((c) => SCORED_CHECK_IDS.includes(c.id) && c.status === "fail"); +/** + * An entry whose only finding is `request-context`. 391 of 412 entry points fail that check, so + * listing each one turns the fix list into a single house-style finding repeated, which is the + * reason `audit-trail` is kept out of the list too. Collapsed into the `CONTEXT` figure instead. + * An entry that fails something else as well stays in the list with all of its findings, so a + * route like `/account/tokens` still shows the request-context gap alongside the rest. + */ +const contextOnly = (e: ScoredEntry) => { + const failures = scoredFailures(e); + return failures.length === 1 && failures[0]!.id === "request-context"; +}; + export function renderTerminal(report: MapReport): string { const lines: string[] = []; @@ -54,6 +66,18 @@ export function renderTerminal(report: MapReport): string { ); } + const { applicable, naming } = report.contextGap; + if (applicable > 0) { + const collapsed = report.entries.filter(contextOnly).length; + lines.push(""); + lines.push( + `CONTEXT ${naming} of ${applicable} entry points name a tenant on a failure path.` + + (collapsed > 0 + ? ` ${collapsed} appear${collapsed === 1 ? "s" : ""} only here, not in the list below.` + : "") + ); + } + if (report.suppressions.checks > 0) { const { entries, checks } = report.suppressions; lines.push( @@ -65,7 +89,7 @@ export function renderTerminal(report: MapReport): string { } const worst = report.entries - .filter((e) => scoredFailures(e).length > 0) + .filter((e) => scoredFailures(e).length > 0 && !contextOnly(e)) .sort( (a, b) => Number(b.sensitive) - Number(a.sensitive) || diff --git a/internal-packages/observability-map/src/score.ts b/internal-packages/observability-map/src/score.ts index 27aaa680950..d9493517864 100644 --- a/internal-packages/observability-map/src/score.ts +++ b/internal-packages/observability-map/src/score.ts @@ -35,6 +35,12 @@ export type MapReport = { byFamily: Record; sensitiveCohort: { n: number; measured: number; mean: number | null }; auditGap: { sensitiveMutations: number; withAudit: number }; + /** + * `request-context` fails 391 of the 412 entry points it applies to, so it is reported as a + * figure rather than as hundreds of identical list entries, the same treatment `audit-trail` + * gets. It stays fully in the score: the gap is real and the score is meant to show it. + */ + contextGap: { applicable: number; naming: number }; entries: ScoredEntry[]; parseFailures: string[]; }; @@ -119,6 +125,10 @@ export function buildReport(eps: EntryPoint[], parseFailures: string[]): MapRepo // reported here as its own architectural figure instead: how many sensitive mutations have an // audit record, out of how many. Folding it into the score would tank every sensitive route on a // gap that is the same everywhere, and bury the routes that have their own, fixable problems. + const contextChecks = entries + .map((e) => e.checks.find((c) => c.id === "request-context")) + .filter((c) => c !== undefined && c.status !== "not-applicable"); + const auditApplicable = entries.filter((e) => e.checks.some((c) => c.id === "audit-trail" && c.status !== "not-applicable") ); @@ -141,6 +151,10 @@ export function buildReport(eps: EntryPoint[], parseFailures: string[]): MapRepo e.checks.some((c) => c.id === "audit-trail" && c.status === "pass") ).length, }, + contextGap: { + applicable: contextChecks.length, + naming: contextChecks.filter((c) => c.status === "pass").length, + }, entries, parseFailures, }; diff --git a/internal-packages/observability-map/test/report.test.ts b/internal-packages/observability-map/test/report.test.ts index 7a28c7a4b9c..76680fb6072 100644 --- a/internal-packages/observability-map/test/report.test.ts +++ b/internal-packages/observability-map/test/report.test.ts @@ -75,21 +75,19 @@ describe("renderTerminal", () => { }` )!; - // Sensitive, score 67: guarded and classifies what it catches, but its failure log names - // nobody, so only request-context fails. - const sensitiveSixtySeven = scanFile( + // Sensitive, score 33: classifies what it catches, but has no guard and names nobody. Fails + // more than request-context, so it stays in the list rather than collapsing into the figure. + const sensitiveThirtyThree = scanFile( "api.v1.auth.tokens.ts", - `import { requireUserId } from "~/services/session.server"; - import { logger } from "~/services/logger.server"; + `import { logger } from "~/services/logger.server"; import { prisma } from "~/db.server"; - export async function action({ request }) { - const userId = await requireUserId(request); - try { return await prisma.token.create({ data: { userId } }); } + export async function action() { + try { return await prisma.token.create({ data: {} }); } catch (error) { logger.error("failed", { error }); throw error; } }` )!; - // Not sensitive, score 0: worse score than sensitiveSixtySeven, but must still sort after both + // Not sensitive, score 0: worse score than sensitiveThirtyThree, but must still sort after both // sensitive entries because sensitivity outranks raw score. const notSensitiveZero = scanFile( "resources.busy.ts", @@ -114,17 +112,17 @@ describe("renderTerminal", () => { )!; const out = renderTerminal( - buildReport([sensitiveSixtySeven, sensitiveZero, notSensitiveZero, sensitiveAuditOnly], []) + buildReport([sensitiveThirtyThree, sensitiveZero, notSensitiveZero, sensitiveAuditOnly], []) ); const fixFirst = out.slice(out.indexOf("FIX FIRST"), out.indexOf("already solid")); const idxZero = fixFirst.indexOf("api.v1.envvars.ts"); - const idxSixtySeven = fixFirst.indexOf("api.v1.auth.tokens.ts"); + const idxThirtyThree = fixFirst.indexOf("api.v1.auth.tokens.ts"); const idxNotSensitive = fixFirst.indexOf("resources.busy.ts"); expect(idxZero).toBeGreaterThan(-1); - expect(idxSixtySeven).toBeGreaterThan(idxZero); - expect(idxNotSensitive).toBeGreaterThan(idxSixtySeven); + expect(idxThirtyThree).toBeGreaterThan(idxZero); + expect(idxNotSensitive).toBeGreaterThan(idxThirtyThree); expect(fixFirst).not.toContain("api.v1.billing.ts"); }); }); @@ -206,3 +204,47 @@ describe("rendering honestly when there is nothing to say", () => { expect(out).toMatch(/1 had none to apply/i); }); }); + +describe("collapsing the house-style finding", () => { + const namesNobody = () => + scanFile( + "api.v1.silent.ts", + `import { prisma } from "~/db.server"; + export async function loader() { return prisma.thing.findMany(); }` + )!; + + const namesNobodyAndSwallows = () => + scanFile( + "api.v1.auth.tokens.ts", + `import { prisma } from "~/db.server"; + export async function action() { + try { return await prisma.token.create({ data: {} }); } catch (e) { return null; } + }` + )!; + + // request-context fails 391 of 412 entry points, so listing each one turns the fix list into a + // single finding repeated. Same reasoning that keeps audit-trail out of the list. + it("keeps an entry whose only finding is request-context out of the fix list", () => { + const out = renderTerminal(buildReport([namesNobody()], [])); + const fixFirst = out.slice(out.indexOf("FIX FIRST")); + expect(fixFirst).not.toContain("api.v1.silent.ts"); + }); + + it("reports the gap as a headline figure instead", () => { + const out = renderTerminal(buildReport([namesNobody()], [])); + expect(out).toMatch(/CONTEXT\s+0 of 1 entry points name a tenant on a failure path/); + }); + + it("says how many entries the collapse took out of the list", () => { + const out = renderTerminal(buildReport([namesNobody(), namesNobodyAndSwallows()], [])); + expect(out).toMatch(/1 appears? only here/i); + }); + + it("still lists request-context when the entry fails something else too", () => { + const out = renderTerminal(buildReport([namesNobodyAndSwallows()], [])); + const fixFirst = out.slice(out.indexOf("FIX FIRST")); + expect(fixFirst).toContain("api.v1.auth.tokens.ts"); + expect(fixFirst).toContain("request-context"); + expect(fixFirst).toContain("error-classification"); + }); +}); diff --git a/internal-packages/observability-map/test/score.test.ts b/internal-packages/observability-map/test/score.test.ts index f962aa2b8c8..85cee860676 100644 --- a/internal-packages/observability-map/test/score.test.ts +++ b/internal-packages/observability-map/test/score.test.ts @@ -138,6 +138,30 @@ ${BUSY_AND_FAILING}` }); describe("buildReport", () => { + it("reports the request-context gap as a figure, like the audit gap", () => { + const naming = scanFile( + "api.v1.named.ts", + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function loader({ params }) { + try { return await prisma.thing.findMany(); } + catch (error) { logger.error("failed", { environmentId: params.envId, error }); throw error; } + }` + )!; + const silent = scanFile( + "api.v1.silent.ts", + `import { prisma } from "~/db.server"; + export async function loader() { return prisma.thing.findMany(); }` + )!; + const trivial = scanFile( + "resources.health.ts", + `export const loader = () => new Response("ok");` + )!; + + const report = buildReport([naming, silent, trivial], []); + expect(report.contextGap).toEqual({ applicable: 2, naming: 1 }); + }); + it("counts suppressions so laundering is visible in the report", () => { const report = buildReport( [ From 99e14ca1a9666f5eb32dd360678df0b52405b811 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Fri, 31 Jul 2026 22:29:43 +0100 Subject: [PATCH 026/106] docs(observability-map): explain what the number means For a tool whose only value is being believed, there was nothing in the repo explaining the score, the checks, or where they are wrong. Covers both CLI modes, what 19 means and why it is deliberately unflattering, the four checks a line each, why the audit and context gaps are figures rather than list entries, why an unmeasured entry is excluded rather than scored, the visibility test every applicability decision follows, that a suppression needs a reason and cannot raise a score, and the known limits: one hop and same file only, loggers matched by spelling, only the first object-literal argument read, inline callbacks not descended into. --- internal-packages/observability-map/README.md | 143 ++++++++++++++++++ 1 file changed, 143 insertions(+) create mode 100644 internal-packages/observability-map/README.md diff --git a/internal-packages/observability-map/README.md b/internal-packages/observability-map/README.md new file mode 100644 index 00000000000..c4d2041868e --- /dev/null +++ b/internal-packages/observability-map/README.md @@ -0,0 +1,143 @@ +# @internal/observability-map + +Scores every webapp entry point on whether it could explain itself during an incident, and prints +the ones worth fixing. An entry point is a Remix `loader` or `action` under +`apps/webapp/app/routes`, 427 of them at the time of writing. + +The number it prints today is 19 out of 100. That is not a bug, and the rest of this file is mostly +about why you should believe it. + +## Running it + +```bash +pnpm --filter @internal/observability-map run map # the whole tree +pnpm --filter @internal/observability-map run map --json # same, as JSON on stdout +pnpm --filter @internal/observability-map run map /api/v1/token # one route, with its check results +``` + +The whole-tree run also writes `observability-map.json` at the repo root, which `--no-write` +suppresses. The single-route mode takes either the route path the report prints (`/api/v1/token`) or +the file name (`api.v1.token.ts`). An exact match wins over the routes it is a prefix of, and an +ambiguous prefix warns and names the alternatives rather than silently picking one. + +## What 19 means + +It is the mean score of the 412 entry points that had at least one applicable check, where an +entry's score is the share of its applicable checks that passed. It is low because the webapp does +not attach tenant identity to its failures: **21 of 412 entry points name an environment, project, +organization, run or user on a failure path.** Everything else, when it breaks at 3am, tells you the +route and the request id and nothing about whose request it was. + +The score was 76 until we stopped crediting routes for the error handling they do not do. Emptying +every catch clause in the tree used to score it 100, which meant the metric paid you for deleting +error handling. Now removing the catches takes 19 down to 8, and removing the logs as well takes it +to 2. If you change this package, keep that property: mutate the tree to remove error handling and +check the score falls. + +So the number is deliberately unflattering, and one platform change would move most of it. Nothing +central attaches a tenant: `logger` pushes `{ requestId, path, host, method }` onto every line +through AsyncLocalStorage and forwards errors to Sentry, and the route builders log +`logBoundaryError(message, error, url)`. If the auth path ever pushed `environmentId` through +`trace(...)`, several hundred entry points would flip at once, and this check would want rethinking +rather than celebrating. + +## The four checks + +- **error-classification**: does every catch clause decide what it caught, by rethrowing, by + branching on the error, or by guarding a parse it can answer for. +- **auth-boundary**: does a route handling credentials, tokens, billing or impersonation check who + is asking. +- **request-context**: when this entry point's failure is reported, is the tenant named. +- **audit-trail**: does a sensitive mutation leave a record of who did it. Nothing in the webapp + writes one, so every applicable entry point fails. + +`audit-trail` is excluded from the score. `request-context` is in it. + +## Two findings are headlines, not list entries + +`audit-trail` fails 19 of 19, and `request-context` fails 391 of 412. Printing either one per route +would bury the route-specific findings under the same sentence repeated hundreds of times, so both +are reported as a figure: the `AUDIT` and `CONTEXT` lines. 333 entry points fail nothing except +`request-context` and appear only in that figure, which leaves 67 in the fix list. An entry that +fails `request-context` *and* something else keeps both findings and stays in the list, so +`/account/tokens` still shows the whole picture. + +`request-context` is still scored, unlike `audit-trail`. The gap it measures is real and the score +is meant to show it. Only the presentation collapses. + +## Not applicable is not a pass + +An entry with no applicable scored check is `measured: false`, and it is left out of every mean the +report computes. Its `score` field reads 100, which is a placeholder for "nothing was measured +here", not a verdict, and nothing averages it. This matters because the alternative, letting +unmeasured entries into the mean at 100, would let the tool look better the less it understood. The +header prints both counts (`412 measured, 15 unmeasured`) so the denominator is never hidden, and a +family with nothing measured renders as `not measured` rather than as a full green bar. + +## When a check declines to judge + +The rule every applicability decision follows: **would this evidence necessarily be visible in the +body if it existed?** + +A log call inside a catch would be, because the catch is right there in the body being read. So its +absence is evidence of absence and `request-context` fails the route. A guard on work that happens +inside an imported helper would not be, because neither the work nor the guard is in the body. So +`auth-boundary` reports not-applicable with a detail saying it could not verify, rather than +accusing the route of being unguarded. `resources.impersonation.ts` is the worked example: it calls +`clearImpersonation`, which authenticates and writes an audit row in `app/models/admin.server.ts`, +a file this tool never opens. + +The failure mode this rule exists to prevent is a fix list whose top three entries are all wrong. +That happened, twice, and both times the cause was a check asserting something the evidence did not +support. + +## Suppression + +```ts +// obs-map-disable-next-line auth-boundary -- public by design, see ADR 12 +``` + +The reason is mandatory: a suppression without one is ignored. The directive is read from comments +only, line by line, so a string literal quoting it does not switch a check off. + +A suppression cannot raise a score. The suppressed check leaves the numerator and the denominator, +and the result is capped by what the entry would have scored unsuppressed, so suppressing a failing +check holds the number still rather than improving it. What you buy is removal from the worklist +with a reason on the record. The report prints how many suppressions are in force so the practice +stays visible. + +## Known limits + +Read these before trusting a specific verdict. + +- **One hop, same file only.** If a loader delegates to a helper in the same file, that helper's + statements, catches and calls count as the route's. A helper's own helpers do not, and nothing + imported from another module is ever opened. Most of what a route does is behind an import, which + is why `auth-boundary` applies to 26 entry points rather than 427. +- **Loggers are matched by spelling.** A call counts as logging when the callee reads `logger.*` or + `log.*`. An aliased logger, one wrapped in a helper, or `console.error` is invisible, so a route + can be reported as recording nothing while it records plenty. +- **Only the first object-literal argument is read** for identifier fields, and only its property + names. `logger.error("failed", ctx)` where `ctx` is a variable contributes nothing, and neither + does a second object. +- **Inline callbacks are not descended into** when counting statements, so a two-statement body can + hold a pile of work inside a `.map()`. The call count is what catches those cases, imperfectly. +- **Sensitivity is a heuristic**: a symbol list plus path segments. It was circular until recently, + counting `requireAdminApiRequest` as a hazard when it is a mitigation, which made 34 of 67 + sensitive routes sensitive purely for being guarded. Expect it to need pruning again as routes + move. +- **The score is a mean of means over a heuristic.** Read the fix list and the two headline figures. + Watching the single number for small movements will mislead you. + +## Layout + +`scan.ts` walks the routes directory and produces an `EntryPoint` per module, carrying only +body-scoped evidence. `checks/` holds the four checks, each a pure function of an `EntryPoint`. +`score.ts` turns checks into an entry score and a report, `report/` renders it, `cli.ts` is the +entry point. `sensitivity.ts`, `triviality.ts` and `suppression.ts` are the three inputs the checks +share. + +Tests sit next to their subject in `test/`. Every check has a false-positive fixture, something it +must not flag, alongside the positive one. Keep that: most of the bugs this package has had were +checks that fired on the wrong thing, and a test that only proves the heuristic fires would have +caught none of them. From 3c748691efbc8843a4e2caa0b2aea5e0a6cc76e5 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Fri, 31 Jul 2026 22:48:02 +0100 Subject: [PATCH 027/106] fix(observability-map): make the package typecheck and build again Three type errors from the last wave broke the root typecheck workflow and the internal-package build that gates the unit test job. findMatches was typed on a structural subset of EntryPoint and handed the result to scoreEntry, the local byFamily type had not followed groupStats to number or null, and the request-context filter was not a type predicate so the element stayed possibly undefined. Also sets noEmitOnError on the build config. tsc was exiting 2 and emitting anyway, so a broken build left a usable dist behind and only the exit code showed it. --- internal-packages/observability-map/src/cli.ts | 3 ++- internal-packages/observability-map/src/score.ts | 4 ++-- internal-packages/observability-map/tsconfig.build.json | 3 +++ 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/internal-packages/observability-map/src/cli.ts b/internal-packages/observability-map/src/cli.ts index 6feef381e33..4be0ff59193 100644 --- a/internal-packages/observability-map/src/cli.ts +++ b/internal-packages/observability-map/src/cli.ts @@ -1,6 +1,7 @@ import { existsSync, writeFileSync } from "node:fs"; import { dirname, resolve } from "node:path"; import { fileURLToPath } from "node:url"; +import type { EntryPoint } from "./types.js"; import { scanDirectory } from "./scan.js"; import { buildReport, scoreEntry } from "./score.js"; import { routePathOf } from "./adapters/remix.js"; @@ -41,7 +42,7 @@ const processIo: Io = { * because only the file name was matched, so the identifier on screen was not one you could paste * back in. */ -function findMatches(entryPoints: { fileName: string }[], target: string) { +function findMatches(entryPoints: EntryPoint[], target: string): EntryPoint[] { const asPath = target.startsWith("/") ? target : `/${target}`; const asFile = target.replace(/^\//, ""); diff --git a/internal-packages/observability-map/src/score.ts b/internal-packages/observability-map/src/score.ts index d9493517864..1203f730158 100644 --- a/internal-packages/observability-map/src/score.ts +++ b/internal-packages/observability-map/src/score.ts @@ -114,7 +114,7 @@ export function buildReport(eps: EntryPoint[], parseFailures: string[]): MapRepo const entries = eps.map(scoreEntry); const measuredEntries = entries.filter((e) => e.measured); - const byFamily: Record = {}; + const byFamily: MapReport["byFamily"] = {}; for (const family of new Set(entries.map((e) => e.family))) { byFamily[family] = groupStats(entries.filter((e) => e.family === family)); } @@ -127,7 +127,7 @@ export function buildReport(eps: EntryPoint[], parseFailures: string[]): MapRepo // gap that is the same everywhere, and bury the routes that have their own, fixable problems. const contextChecks = entries .map((e) => e.checks.find((c) => c.id === "request-context")) - .filter((c) => c !== undefined && c.status !== "not-applicable"); + .filter((c): c is CheckResult => c !== undefined && c.status !== "not-applicable"); const auditApplicable = entries.filter((e) => e.checks.some((c) => c.id === "audit-trail" && c.status !== "not-applicable") diff --git a/internal-packages/observability-map/tsconfig.build.json b/internal-packages/observability-map/tsconfig.build.json index 254fcfbca49..6b6b2863f1f 100644 --- a/internal-packages/observability-map/tsconfig.build.json +++ b/internal-packages/observability-map/tsconfig.build.json @@ -3,6 +3,9 @@ "include": ["src/**/*.ts"], "compilerOptions": { "noEmit": false, + // A build that does not compile must not leave a dist behind: three type errors shipped in the + // last wave while `build` still emitted, so the failure was only visible in the exit code. + "noEmitOnError": true, "declaration": true, "outDir": "dist", "rootDir": ".", From 655b3caef0db14d57149222eb208a0b6c54a63d8 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Fri, 31 Jul 2026 22:52:13 +0100 Subject: [PATCH 028/106] fix(observability-map): stop paying for a catch that only rethrows Wrapping a body in try { ... } catch (e) { throw e } moved error-classification from not-applicable to pass, worth 50 points a route. Applied to the tree it took the global from 19 to 46, 223 entries up and none down, for a change that does nothing: the error propagates either way, so scoring the two differently paid for a semantic no-op. A clause now qualifies by branching or by guarding a parse. One that only rethrows is inert and reads exactly as no catch does, not-applicable. The same mutation is now flat at 18, no entry moves in either direction, while deleting error handling still costs, 18 down to 8 and to 2 with the logs as well. The cost is that a catch which logs and rethrows also reads as inert, since the clause evidence cannot say whether it does anything besides rethrow. That withholds credit rather than granting it, and crediting it would reopen the hole a single logger.error line wide. request-context still asks whether that log names a tenant. Both directions of the invariant are asserted in tests now, not just measured once: no-op error handling must not raise a score, deleting error handling must not raise a score. --- .../src/checks/errorClassification.ts | 65 ++++++++++++------ .../observability-map/test/checks.test.ts | 7 +- .../observability-map/test/report.test.ts | 13 ++-- .../observability-map/test/score.test.ts | 68 +++++++++++++++++++ 4 files changed, 124 insertions(+), 29 deletions(-) diff --git a/internal-packages/observability-map/src/checks/errorClassification.ts b/internal-packages/observability-map/src/checks/errorClassification.ts index c2d46b3af5b..2c48f40dded 100644 --- a/internal-packages/observability-map/src/checks/errorClassification.ts +++ b/internal-packages/observability-map/src/checks/errorClassification.ts @@ -38,21 +38,39 @@ function isParseGuard(clause: CatchEvidence, ep: EntryPoint): boolean { } /** - * Whether a clause has decided what the error means. Rethrowing is a decision, branching is a - * decision, and guarding a parse answers for the one thing the guard covers. + * Whether a clause decides anything about the error it caught. Two ways to qualify: it branches, on + * an `if`, a `switch` or an `instanceof`, or it guards a parse it can answer for. * - * `narrow` is deliberately not a fourth way to qualify, which is where this differs from the rule - * the scanner work proposed. A one-statement try around `await service.call(run)` is narrow and is - * still a swallow. Taking the narrow limb clears eleven more entry points, and reading all eleven - * says six are real: the silent cancel in `api.v2.runs.$runParam.cancel.ts`, the PAT revoke in - * `account.tokens/route.tsx` and the invite revoke, both of which report a database failure to the - * browser as a 400 with an internal message in it, a `.map` that drops a broken dashboard on the - * floor, and two more. The four it would rightly clear are all the same deliberate shape, best - * effort side work that logs and carries on, and all four are non-sensitive so they sort to the - * bottom of the fix list. See the task 5 report; switching is one limb in this function. + * Rethrowing is not a third way, which is the correction from the last wave. A clause whose only + * effect is `throw e` leaves the error propagating exactly as it would with no catch at all, so + * treating that as a pass while no catch is not-applicable paid 50 points a route for wrapping a + * body in `try { ... } catch (e) { throw e }`, and 27 across the tree. The two are observationally + * identical and are now scored identically. + * + * The cost is real and worth stating: `catch (e) { logger.error(...); throw e }` also reads as + * inert, because `CatchEvidence` cannot say whether a clause does anything besides rethrow. That + * withholds credit from a route that reports before propagating, which is the safe direction to be + * wrong in, since crediting it would reopen the hole a bare `logger.error` line wide. + * `request-context` still reads that log and asks whether it names a tenant, so the reporting is + * unrewarded here rather than unmeasured. + * + * `narrow` is not a way to qualify either. A one-statement try around `await service.call(run)` is + * narrow and is still a swallow: reading all eleven entry points that limb would clear said six + * were real, including a silent run cancellation and two credential paths that report a database + * failure to the browser as a 400 with an internal message in it. */ -function accountedFor(clause: CatchEvidence, ep: EntryPoint): boolean { - return clause.rethrows || clause.branches || isParseGuard(clause, ep); +function decides(clause: CatchEvidence, ep: EntryPoint): boolean { + return clause.branches || isParseGuard(clause, ep); +} + +/** Passes the error through unchanged, which is the same outcome as not catching it. */ +function inert(clause: CatchEvidence, ep: EntryPoint): boolean { + return clause.rethrows && !decides(clause, ep); +} + +/** The error stops here and nothing chose what it meant. */ +function swallows(clause: CatchEvidence, ep: EntryPoint): boolean { + return !decides(clause, ep) && !inert(clause, ep); } export function usesBuilder(ep: EntryPoint): boolean { @@ -88,21 +106,24 @@ export const errorClassification = { if (isTrivial(ep)) { return { id: ID, status: "not-applicable", detail: "trivial route" }; } - if (ep.catches.length === 0) { + const swallowed = ep.catches.filter((c) => swallows(c, ep)); + if (swallowed.length > 0) { + const which = + ep.catches.length > 1 ? ` (${swallowed.length} of ${ep.catches.length} catches)` : ""; return { id: ID, - status: "not-applicable", - detail: "catches nothing, so it classifies nothing", + status: "fail", + detail: `catches its errors and takes one way out regardless of what was thrown${which}`, }; } - const unaccounted = ep.catches.filter((c) => !accountedFor(c, ep)); - if (unaccounted.length > 0) { - const which = - ep.catches.length > 1 ? ` (${unaccounted.length} of ${ep.catches.length} catches)` : ""; + if (!ep.catches.some((c) => decides(c, ep))) { return { id: ID, - status: "fail", - detail: `catches its errors and takes one way out regardless of what was thrown${which}`, + status: "not-applicable", + detail: + ep.catches.length === 0 + ? "catches nothing, so it classifies nothing" + : "every catch rethrows and nothing else, so it classifies nothing", }; } return { id: ID, status: "pass", detail: "every catch decides what it caught" }; diff --git a/internal-packages/observability-map/test/checks.test.ts b/internal-packages/observability-map/test/checks.test.ts index 6aeb75a7ecb..cbc0c73a00d 100644 --- a/internal-packages/observability-map/test/checks.test.ts +++ b/internal-packages/observability-map/test/checks.test.ts @@ -78,7 +78,10 @@ describe("error-classification", () => { expect(r.status).toBe("pass"); }); - it("passes a raw route whose catch rethrows without branching", () => { + // A clause that only rethrows makes no classification decision: the error propagates exactly as + // it would with no catch at all, so it is read as no catch at all. Scoring the two differently + // paid 50 points a route for wrapping a body in `try { ... } catch (e) { throw e }`. + it("is not applicable to a raw route whose catch only rethrows", () => { const r = run( "error-classification", "api.v1.v.ts", @@ -89,7 +92,7 @@ describe("error-classification", () => { catch (e) { logger.error("thing lookup failed", { error: e }); throw e; } }` ); - expect(r.status).toBe("pass"); + expect(r.status).toBe("not-applicable"); }); // The builder only classifies what reaches it. A swallow inside the handler never does, so the diff --git a/internal-packages/observability-map/test/report.test.ts b/internal-packages/observability-map/test/report.test.ts index 76680fb6072..bb80ca1bb6c 100644 --- a/internal-packages/observability-map/test/report.test.ts +++ b/internal-packages/observability-map/test/report.test.ts @@ -75,15 +75,18 @@ describe("renderTerminal", () => { }` )!; - // Sensitive, score 33: classifies what it catches, but has no guard and names nobody. Fails - // more than request-context, so it stays in the list rather than collapsing into the figure. + // Sensitive, score 33: its catch decides something, telling a bad request apart from the rest, + // but it has no guard and names nobody. Fails more than request-context, so it stays in the + // list rather than collapsing into the figure. const sensitiveThirtyThree = scanFile( "api.v1.auth.tokens.ts", - `import { logger } from "~/services/logger.server"; - import { prisma } from "~/db.server"; + `import { prisma } from "~/db.server"; export async function action() { try { return await prisma.token.create({ data: {} }); } - catch (error) { logger.error("failed", { error }); throw error; } + catch (error) { + if (error instanceof BadRequest) return json({ error: "bad" }, { status: 400 }); + throw error; + } }` )!; diff --git a/internal-packages/observability-map/test/score.test.ts b/internal-packages/observability-map/test/score.test.ts index 85cee860676..e677d95f3f1 100644 --- a/internal-packages/observability-map/test/score.test.ts +++ b/internal-packages/observability-map/test/score.test.ts @@ -216,3 +216,71 @@ ${BUSY_AND_FAILING}` expect(family.mean).toBe(scoreEntry(busy).score); }); }); + +/** + * The invariant, both ways round. The README states one direction, removing error handling must + * lower the score, and that alone could not see the free-points path: adding a catch that only + * rethrows changes nothing about how the route behaves, and used to move it from not-applicable to + * pass, worth 50 points a route and 27 points across the tree. + */ +describe("no-op error handling must not pay", () => { + const BODY = `const rows = await prisma.thing.findMany(); + return json({ rows });`; + + const plain = `import { prisma } from "~/db.server"; +export async function loader() { + ${BODY} +}`; + + const wrappedInARethrow = `import { prisma } from "~/db.server"; +export async function loader() { + try { + ${BODY} + } catch (e) { + throw e; + } +}`; + + const handled = `import { logger } from "~/services/logger.server"; +import { prisma } from "~/db.server"; +export async function loader({ params }) { + try { + ${BODY} + } catch (error) { + if (error instanceof NotFoundError) return json({ error: "not found" }, { status: 404 }); + logger.error("thing lookup failed", { environmentId: params.envId, error }); + throw error; + } +}`; + + it("does not pay for wrapping a body in a catch that only rethrows", () => { + const before = scoreEntry(scanFile("api.v1.x.ts", plain)!); + const after = scoreEntry(scanFile("api.v1.x.ts", wrappedInARethrow)!); + expect(after.score).toBeLessThanOrEqual(before.score); + }); + + it("does not pay for wrapping a whole tree in catches that only rethrow", () => { + const before = buildReport( + [scanFile("api.v1.x.ts", plain)!, scanFile("api.v1.y.ts", plain)!], + [] + ); + const after = buildReport( + [scanFile("api.v1.x.ts", wrappedInARethrow)!, scanFile("api.v1.y.ts", wrappedInARethrow)!], + [] + ); + expect(after.global!).toBeLessThanOrEqual(before.global!); + }); + + it("does not pay for deleting error handling either", () => { + const before = scoreEntry(scanFile("api.v1.x.ts", handled)!); + const after = scoreEntry(scanFile("api.v1.x.ts", plain)!); + expect(after.score).toBeLessThanOrEqual(before.score); + // And the handled version is genuinely better, so the invariant is not holding by both being 0. + expect(before.score).toBeGreaterThan(after.score); + }); + + it("still credits a catch that decides something on its way through", () => { + const scored = scoreEntry(scanFile("api.v1.x.ts", handled)!); + expect(scored.checks.find((c) => c.id === "error-classification")!.status).toBe("pass"); + }); +}); From 7754cac6b503f07bc15864010e2fa3834e6e4523 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Fri, 31 Jul 2026 22:57:08 +0100 Subject: [PATCH 029/106] fix(observability-map): name the suppression directive honestly, and tighten the tests obs-map-disable-next-line applied to the whole entry point, so a directive on the last line of a file switched a check off for everything above it. Scoping it to a line is not available, since a finding is attached to an entry point and carries no line number to match against, and inventing a proximity rule would silently drop legitimate suppressions. So the name is now obs-map-disable, which is what it does. The old spelling is not honoured and a test says so. The CONTEXT line now says how many of the collapsed entries are sensitive, 18 of 333, so a reader knows to open the JSON rather than trusting the list. Three tests were passing by luck. The FIX FIRST ordering test sliced on a string the report no longer prints, so its assertions ran against the whole tail. The --no-write test never checked that no file was written, so it would have written into the repo root if the flag broke. A suppression test asserted toContain('1') against a report full of digits, and another never asserted the measured flag it was named for. README corrected: the score is 18, auth-boundary applies to 23 entry points rather than 26 and gates on sensitivity before the one-hop limit, and the invariant section now states both directions. --- internal-packages/observability-map/README.md | 48 ++++++++++++++----- .../observability-map/src/report/terminal.ts | 8 ++-- .../observability-map/src/suppression.ts | 10 +++- .../observability-map/test/cli.test.ts | 11 +++++ .../observability-map/test/report.test.ts | 27 +++++++++-- .../observability-map/test/score.test.ts | 20 ++++---- .../test/suppression.test.ts | 31 ++++++++---- 7 files changed, 118 insertions(+), 37 deletions(-) diff --git a/internal-packages/observability-map/README.md b/internal-packages/observability-map/README.md index c4d2041868e..457c22ff390 100644 --- a/internal-packages/observability-map/README.md +++ b/internal-packages/observability-map/README.md @@ -4,7 +4,7 @@ Scores every webapp entry point on whether it could explain itself during an inc the ones worth fixing. An entry point is a Remix `loader` or `action` under `apps/webapp/app/routes`, 427 of them at the time of writing. -The number it prints today is 19 out of 100. That is not a bug, and the rest of this file is mostly +The number it prints today is 18 out of 100. That is not a bug, and the rest of this file is mostly about why you should believe it. ## Running it @@ -20,7 +20,7 @@ suppresses. The single-route mode takes either the route path the report prints the file name (`api.v1.token.ts`). An exact match wins over the routes it is a prefix of, and an ambiguous prefix warns and names the alternatives rather than silently picking one. -## What 19 means +## What 18 means It is the mean score of the 412 entry points that had at least one applicable check, where an entry's score is the share of its applicable checks that passed. It is low because the webapp does @@ -30,9 +30,18 @@ route and the request id and nothing about whose request it was. The score was 76 until we stopped crediting routes for the error handling they do not do. Emptying every catch clause in the tree used to score it 100, which meant the metric paid you for deleting -error handling. Now removing the catches takes 19 down to 8, and removing the logs as well takes it -to 2. If you change this package, keep that property: mutate the tree to remove error handling and -check the score falls. +error handling. + +Two invariants hold now, and both are asserted in `test/score.test.ts` rather than measured once: + +- **Removing error handling must not raise the score.** Deleting every catch clause takes 18 to 8, + and deleting the logs as well takes it to 2. +- **Adding error handling that does nothing must not raise the score.** Wrapping every body in + `try { ... } catch (e) { throw e }` leaves it at 18, with no entry moving in either direction. + That mutation used to be worth 27 points across the tree, because a rethrow-only clause counted + as a pass while no catch at all was not-applicable, and the two are observationally identical. + +If you change this package, check both directions still hold. So the number is deliberately unflattering, and one platform change would move most of it. Nothing central attaches a tenant: `logger` pushes `{ requestId, path, host, method }` onto every line @@ -43,8 +52,9 @@ rather than celebrating. ## The four checks -- **error-classification**: does every catch clause decide what it caught, by rethrowing, by - branching on the error, or by guarding a parse it can answer for. +- **error-classification**: does every catch clause decide what it caught, by branching on the + error or by guarding a parse it can answer for. A clause that only rethrows decides nothing and + is read as though there were no catch, so it neither passes nor fails. - **auth-boundary**: does a route handling credentials, tokens, billing or impersonation check who is asking. - **request-context**: when this entry point's failure is reported, is the tenant named. @@ -62,6 +72,11 @@ are reported as a figure: the `AUDIT` and `CONTEXT` lines. 333 entry points fail fails `request-context` *and* something else keeps both findings and stays in the list, so `/account/tokens` still shows the whole picture. +18 of those 333 are sensitive, including `/admin/impersonate`, the API-key regeneration route and +four envvars routes, so the `CONTEXT` line says how many. Read them out of +`observability-map.json`, where every entry keeps its full check results, rather than assuming the +list is the whole story. + `request-context` is still scored, unlike `audit-trail`. The gap it measures is real and the score is meant to show it. Only the presentation collapses. @@ -94,11 +109,17 @@ support. ## Suppression ```ts -// obs-map-disable-next-line auth-boundary -- public by design, see ADR 12 +// obs-map-disable auth-boundary -- public by design, see ADR 12 ``` The reason is mandatory: a suppression without one is ignored. The directive is read from comments -only, line by line, so a string literal quoting it does not switch a check off. +only, so a string literal quoting it does not switch a check off. + +It applies to the whole entry point, not to the line under it. It was called +`obs-map-disable-next-line`, which was untrue in a way that mattered: a directive on the last line +of a file switched a check off for everything above it. Genuine line scoping is not available, +because a finding is attached to an entry point and carries no line number to match against, so the +name was corrected instead. The old spelling is not honoured, and there is a test saying so. A suppression cannot raise a score. The suppressed check leaves the numerator and the denominator, and the result is capped by what the entry would have scored unsuppressed, so suppressing a failing @@ -112,11 +133,16 @@ Read these before trusting a specific verdict. - **One hop, same file only.** If a loader delegates to a helper in the same file, that helper's statements, catches and calls count as the route's. A helper's own helpers do not, and nothing - imported from another module is ever opened. Most of what a route does is behind an import, which - is why `auth-boundary` applies to 26 entry points rather than 427. + imported from another module is ever opened. `auth-boundary` applies to 23 entry points: it gates + on sensitivity first, which is 26 routes, and the one-hop limit accounts for the other 3, which + hand their work to an imported helper and are reported as unverified rather than unguarded. - **Loggers are matched by spelling.** A call counts as logging when the callee reads `logger.*` or `log.*`. An aliased logger, one wrapped in a helper, or `console.error` is invisible, so a route can be reported as recording nothing while it records plenty. +- **A catch that logs and rethrows reads as though it only rethrows.** The clause evidence cannot + say whether a clause does anything besides rethrow, so `error-classification` withholds credit + rather than granting it. Crediting it would reopen the free-points path a single `logger.error` + line wide. - **Only the first object-literal argument is read** for identifier fields, and only its property names. `logger.error("failed", ctx)` where `ctx` is a variable contributes nothing, and neither does a second object. diff --git a/internal-packages/observability-map/src/report/terminal.ts b/internal-packages/observability-map/src/report/terminal.ts index 868acf284d6..eaf0bd5ded1 100644 --- a/internal-packages/observability-map/src/report/terminal.ts +++ b/internal-packages/observability-map/src/report/terminal.ts @@ -68,12 +68,14 @@ export function renderTerminal(report: MapReport): string { const { applicable, naming } = report.contextGap; if (applicable > 0) { - const collapsed = report.entries.filter(contextOnly).length; + const collapsed = report.entries.filter(contextOnly); + const sensitive = collapsed.filter((e) => e.sensitive).length; lines.push(""); lines.push( `CONTEXT ${naming} of ${applicable} entry points name a tenant on a failure path.` + - (collapsed > 0 - ? ` ${collapsed} appear${collapsed === 1 ? "s" : ""} only here, not in the list below.` + (collapsed.length > 0 + ? ` ${collapsed.length} appear${collapsed.length === 1 ? "s" : ""} only here, ` + + `${sensitive} of them sensitive, in the JSON rather than the list below.` : "") ); } diff --git a/internal-packages/observability-map/src/suppression.ts b/internal-packages/observability-map/src/suppression.ts index 8f7352f6a46..cdf2957d38a 100644 --- a/internal-packages/observability-map/src/suppression.ts +++ b/internal-packages/observability-map/src/suppression.ts @@ -2,8 +2,16 @@ * The directive, and the reason that must follow it. The reason runs to the end of the line: `.` * does not match a newline, so a suppression on one line cannot pick up a reason from the next. * A trailing block-comment terminator is trimmed off so it does not end up inside the reason. + * + * It was `obs-map-disable-next-line`, which was a lie: a check applies to a whole entry point, so + * the directive did too, and one on the last line of a file switched a check off for everything + * above it. The honest options were to scope it to a line or to rename it, and scoping is not + * available: a `CheckResult` carries no line number, and neither does an `EntryPoint`, so there is + * nothing to match a line against. Scoping it would mean inventing a proximity rule that silently + * drops legitimate suppressions. So the name now says what it does. Real line scoping needs + * positions on the findings, which is scanner work. */ -const PATTERN = /obs-map-disable-next-line\s+([a-z-]+)\s+--\s+(.+)/; +const PATTERN = /obs-map-disable\s+([a-z-]+)\s+--\s+(.+)/; /** * The comment part of a line, or null if there is none. diff --git a/internal-packages/observability-map/test/cli.test.ts b/internal-packages/observability-map/test/cli.test.ts index 2dec21ca9a8..f97aa2269ed 100644 --- a/internal-packages/observability-map/test/cli.test.ts +++ b/internal-packages/observability-map/test/cli.test.ts @@ -1,5 +1,9 @@ +import { existsSync, rmSync } from "node:fs"; +import { resolve } from "node:path"; import { main, type Io } from "../src/cli.js"; +const REPORT_FILE = resolve(__dirname, "../../../observability-map.json"); + const capture = () => { const out: string[] = []; const err: string[] = []; @@ -51,10 +55,17 @@ describe("map ", () => { }); describe("map", () => { + // The flag is the only thing standing between a test run and a file written into the repo root, + // so the test has to check the file, not just the exit code. it("renders the whole report without writing when asked not to", () => { + const existedBefore = existsSync(REPORT_FILE); + if (existedBefore) rmSync(REPORT_FILE); + const r = run("--no-write"); + expect(r.code).toBe(0); expect(r.out).toContain("COVERAGE"); expect(r.out).toContain("FIX FIRST"); + expect(existsSync(REPORT_FILE)).toBe(false); }); }); diff --git a/internal-packages/observability-map/test/report.test.ts b/internal-packages/observability-map/test/report.test.ts index bb80ca1bb6c..d287e19b259 100644 --- a/internal-packages/observability-map/test/report.test.ts +++ b/internal-packages/observability-map/test/report.test.ts @@ -32,15 +32,14 @@ describe("renderTerminal", () => { it("surfaces suppressions so laundering is visible rather than silent", () => { const suppressed = scanFile( "api.v1.d.ts", - `// obs-map-disable-next-line error-classification -- deliberate, see ticket + `// obs-map-disable error-classification -- deliberate, see ticket import { prisma } from "~/db.server"; export async function loader() { try { return await prisma.thing.findMany(); } catch (e) { return null; } }` )!; const out = renderTerminal(buildReport([suppressed], [])); - expect(out).toMatch(/suppress/i); - expect(out).toContain("1"); + expect(out).toMatch(/SUPPRESSED\s+1 check across 1 entry point/); }); it("does not mention suppressions when there are none", () => { @@ -118,7 +117,11 @@ describe("renderTerminal", () => { buildReport([sensitiveThirtyThree, sensitiveZero, notSensitiveZero, sensitiveAuditOnly], []) ); - const fixFirst = out.slice(out.indexOf("FIX FIRST"), out.indexOf("already solid")); + // Slice to the end of the list, not to a string the I5 fix deleted: `indexOf` returned -1 for + // "already solid" and the assertions were quietly running against the whole tail. + const listEnd = out.indexOf("no findings:"); + expect(listEnd).toBeGreaterThan(-1); + const fixFirst = out.slice(out.indexOf("FIX FIRST"), listEnd); const idxZero = fixFirst.indexOf("api.v1.envvars.ts"); const idxThirtyThree = fixFirst.indexOf("api.v1.auth.tokens.ts"); const idxNotSensitive = fixFirst.indexOf("resources.busy.ts"); @@ -238,6 +241,22 @@ describe("collapsing the house-style finding", () => { expect(out).toMatch(/CONTEXT\s+0 of 1 entry points name a tenant on a failure path/); }); + // NEW-3. 18 of the collapsed entries are sensitive, including /admin/impersonate and the envvars + // routes, so the line has to say a reader should go and look at them. + it("says how many of the collapsed entries are sensitive", () => { + const sensitiveAndSilent = scanFile( + "api.v1.auth.jwt.ts", + `import { requireUserId } from "~/services/session.server"; + import { prisma } from "~/db.server"; + export async function loader({ request }) { + const userId = await requireUserId(request); + return prisma.token.findMany({ where: { userId } }); + }` + )!; + const out = renderTerminal(buildReport([sensitiveAndSilent], [])); + expect(out).toMatch(/1 appears? only here[^\n]*1 of them sensitive/i); + }); + it("says how many entries the collapse took out of the list", () => { const out = renderTerminal(buildReport([namesNobody(), namesNobodyAndSwallows()], [])); expect(out).toMatch(/1 appears? only here/i); diff --git a/internal-packages/observability-map/test/score.test.ts b/internal-packages/observability-map/test/score.test.ts index e677d95f3f1..37b8eafec47 100644 --- a/internal-packages/observability-map/test/score.test.ts +++ b/internal-packages/observability-map/test/score.test.ts @@ -54,7 +54,7 @@ describe("scoreEntry", () => { }); it("counts a suppressed check as not-applicable", () => { - const suppressed = `// obs-map-disable-next-line error-classification -- health probe + const suppressed = `// obs-map-disable error-classification -- health probe ${RAW}`; const scored = scoreEntry(scanFile("api.v1.b.ts", suppressed)!); const ec = scored.checks.find((c) => c.id === "error-classification")!; @@ -65,12 +65,16 @@ ${RAW}`; // error-classification would fail here; auth-boundary is not-applicable (not sensitive). // Suppressing the only applicable scored check must not be indistinguishable from an entry // point nothing applies to: it is still reported, just not scored on that axis. - const suppressed = `// obs-map-disable-next-line error-classification -- health probe + const suppressed = `// obs-map-disable error-classification -- health probe ${BUSY_AND_FAILING}`; const scored = scoreEntry(scanFile("api.v1.c.ts", suppressed)!); expect(scored.checks.find((c) => c.id === "error-classification")!.status).toBe( "not-applicable" ); + // The point of the test, which it did not previously assert: request-context still applies, so + // the entry is still measured and still counted in the mean. + expect(scored.checks.find((c) => c.id === "request-context")!.status).toBe("fail"); + expect(scored.measured).toBe(true); }); // I1. `score = passed / applicable` meant removing a failing check from the denominator raised @@ -88,7 +92,7 @@ export async function action({ request }) { const suppressed = scoreEntry( scanFile( "api.v1.auth.tokens.ts", - `// obs-map-disable-next-line error-classification -- deliberate, see ticket + `// obs-map-disable error-classification -- deliberate, see ticket ${source}` )! ); @@ -104,8 +108,8 @@ ${source}` const suppressed = scoreEntry( scanFile( "api.v1.b.ts", - `// obs-map-disable-next-line error-classification -- health probe -// obs-map-disable-next-line request-context -- nothing to name here + `// obs-map-disable error-classification -- health probe +// obs-map-disable request-context -- nothing to name here ${BUSY_AND_FAILING}` )! ); @@ -116,8 +120,8 @@ ${BUSY_AND_FAILING}` const suppressed = scoreEntry( scanFile( "api.v1.b.ts", - `// obs-map-disable-next-line error-classification -- health probe -// obs-map-disable-next-line request-context -- nothing to name here + `// obs-map-disable error-classification -- health probe +// obs-map-disable request-context -- nothing to name here ${BUSY_AND_FAILING}` )! ); @@ -167,7 +171,7 @@ describe("buildReport", () => { [ scanFile( "api.v1.b.ts", - `// obs-map-disable-next-line error-classification -- health probe + `// obs-map-disable error-classification -- health probe ${BUSY_AND_FAILING}` )!, scanFile("api.v1.c.ts", BUSY_AND_FAILING)!, diff --git a/internal-packages/observability-map/test/suppression.test.ts b/internal-packages/observability-map/test/suppression.test.ts index 0bb6fcc5f9b..8a01a5953c1 100644 --- a/internal-packages/observability-map/test/suppression.test.ts +++ b/internal-packages/observability-map/test/suppression.test.ts @@ -3,26 +3,26 @@ import { suppressedChecks } from "../src/suppression.js"; describe("suppressedChecks", () => { it("reads a suppression with its reason", () => { const m = suppressedChecks( - `// obs-map-disable-next-line error-classification -- liveness probe, deliberately silent + `// obs-map-disable error-classification -- liveness probe, deliberately silent export async function loader() { return { ok: true }; }` ); expect(m.get("error-classification")).toBe("liveness probe, deliberately silent"); }); it("ignores a suppression with no reason", () => { - const m = suppressedChecks(`// obs-map-disable-next-line error-classification`); + const m = suppressedChecks(`// obs-map-disable error-classification`); expect(m.size).toBe(0); }); it("ignores a suppression whose reason is only whitespace", () => { - const m = suppressedChecks(`// obs-map-disable-next-line error-classification -- `); + const m = suppressedChecks(`// obs-map-disable error-classification -- `); expect(m.size).toBe(0); }); it("reads several suppressions in one file", () => { const m = suppressedChecks( - `// obs-map-disable-next-line error-classification -- liveness probe - // obs-map-disable-next-line request-context -- no identifiers exist here + `// obs-map-disable error-classification -- liveness probe + // obs-map-disable request-context -- no identifiers exist here export async function loader() { return { ok: true }; }` ); expect(m.size).toBe(2); @@ -36,7 +36,7 @@ describe("suppressedChecks", () => { it("does not carry a reason across lines", () => { const m = suppressedChecks( - `// obs-map-disable-next-line error-classification + `// obs-map-disable error-classification // some other comment -- with a dash export async function loader() { return 1; }` ); @@ -47,7 +47,7 @@ describe("suppressedChecks", () => { // merely quotes it, in a test fixture or an error message, silently suppressed a real check. it("ignores the directive inside a string literal", () => { const m = suppressedChecks( - `const example = "obs-map-disable-next-line error-classification -- not a real suppression"; + `const example = "obs-map-disable error-classification -- not a real suppression"; export async function loader() { return 1; }` ); expect(m.size).toBe(0); @@ -55,7 +55,7 @@ describe("suppressedChecks", () => { it("reads the directive from a block comment", () => { const m = suppressedChecks( - `/* obs-map-disable-next-line auth-boundary -- public by design, see ADR 12 */ + `/* obs-map-disable auth-boundary -- public by design, see ADR 12 */ export async function loader() { return 1; }` ); expect(m.get("auth-boundary")).toBe("public by design, see ADR 12"); @@ -64,7 +64,7 @@ describe("suppressedChecks", () => { it("reads the directive from a jsdoc line", () => { const m = suppressedChecks( `/** - * obs-map-disable-next-line request-context -- nothing tenant-scoped here + * obs-map-disable request-context -- nothing tenant-scoped here */ export async function loader() { return 1; }` ); @@ -73,9 +73,20 @@ describe("suppressedChecks", () => { it("ignores code that happens to follow a comment on the same line", () => { const m = suppressedChecks( - `const x = 1; // obs-map-disable-next-line error-classification -- fine + `const x = 1; // obs-map-disable error-classification -- fine export async function loader() { return x; }` ); expect(m.get("error-classification")).toBe("fine"); }); + + // The directive was called `-next-line` while applying to the whole entry point, so a comment on + // the last line of a file switched a check off for everything above it. Renamed rather than + // scoped, because a finding has no line number to scope it to. The old spelling is not honoured. + it("does not honour the old -next-line spelling", () => { + const m = suppressedChecks( + `// obs-map-disable-next-line error-classification -- stale directive + export async function loader() { return 1; }` + ); + expect(m.size).toBe(0); + }); }); From 2cb36aa90e8c65e15b861daf0d482146e0d31861 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Sat, 1 Aug 2026 00:08:48 +0100 Subject: [PATCH 030/106] chore(observability-map): remove dead scanner surface EntryPoint.calleeTexts, catchRethrows, catchBranches, catchesNarrowly and LogCall.hasObjectArgument were written by the scanner and read by nothing: errorClassification reads the per-clause CatchEvidence fields, sensitivity and triviality read calleeNames, not calleeTexts. Removed the fields, their derivations, and the tests that asserted them directly; reworded doc comments that referenced them. --- .../observability-map/src/scan.ts | 28 ++- .../observability-map/src/types.ts | 44 +---- .../observability-map/test/checks.test.ts | 7 +- .../observability-map/test/scan.test.ts | 168 ++++-------------- 4 files changed, 56 insertions(+), 191 deletions(-) diff --git a/internal-packages/observability-map/src/scan.ts b/internal-packages/observability-map/src/scan.ts index 6301f8ed6f8..0f10138e627 100644 --- a/internal-packages/observability-map/src/scan.ts +++ b/internal-packages/observability-map/src/scan.ts @@ -69,9 +69,10 @@ function calleeName(expr: ts.Expression): string | null { } /** - * Callee as recorded in `calleeTexts`: the whole path, `prisma.organization.findFirst` rather than - * `findFirst`. Null when the path runs through something with no name of its own, e.g. - * `new PromptService().createOverride`, where the caller falls back to the bare name. + * The whole callee path of a call, `prisma.organization.findFirst` rather than `findFirst`. Used to + * match a call against `LOGGER_CALLEE` and `PARSE_CALLEE`. Null when the path runs through something + * with no name of its own, e.g. `new PromptService().createOverride`, where the caller falls back to + * the bare name. */ function calleeText(expr: ts.Expression): string | null { const target = unwrap(expr); @@ -92,7 +93,7 @@ function calleeText(expr: ts.Expression): string | null { const LOGGER_CALLEE = /(^|\.)(logger|log)\.[A-Za-z_$][\w$]*$/; /** Property names on the first object-literal argument, e.g. `{ environmentId, error }`. */ -function objectArgumentFields(call: ts.CallExpression): { found: boolean; fields: string[] } { +function objectArgumentFields(call: ts.CallExpression): string[] { for (const arg of call.arguments) { const target = unwrap(arg); if (!ts.isObjectLiteralExpression(target)) continue; @@ -101,9 +102,9 @@ function objectArgumentFields(call: ts.CallExpression): { found: boolean; fields const name = propertyName(property); if (name) fields.push(name); } - return { found: true, fields }; + return fields; } - return { found: false, fields: [] }; + return []; } /** @@ -128,8 +129,8 @@ const PARSE_CALLEE = /(^|\.)(parse|safeParse|parseAsync|safeParseAsync|decode)$| const PARSE_CONSTRUCTORS = new Set(["URL", "URLSearchParams", "RegExp"]); /** - * Whether the guarded region parses something. A `new URL(x)` counts, and has to be read here - * because constructors are absent from `calleeTexts`. + * Whether the guarded region parses something. A `new URL(x)` counts, and has to be read here as a + * `ts.isNewExpression`, because the call-callee scan that builds `calleeNames` never sees it. */ function guardsParse(tryBlock: ts.Block): boolean { let found = false; @@ -518,7 +519,6 @@ export function scanFile(fileName: string, source: string): EntryPoint | null { let hasTryCatch = false; const catches: CatchEvidence[] = []; const calleeNames: string[] = []; - const calleeTexts: string[] = []; const logCalls: LogCall[] = []; const localFunctions = collectLocalFunctions(sf); @@ -558,14 +558,11 @@ export function scanFile(fileName: string, source: string): EntryPoint | null { if (cn) { const text = calleeText(node.expression) ?? cn; calleeNames.push(cn); - calleeTexts.push(text); if (LOGGER_CALLEE.test(text)) { - const argument = objectArgumentFields(node); logCalls.push({ callee: text, - hasObjectArgument: argument.found, - fields: argument.fields, + fields: objectArgumentFields(node), inCatch, }); } @@ -599,13 +596,8 @@ export function scanFile(fileName: string, source: string): EntryPoint | null { actionInitializerCallee: target.actionInitializerCallee, importedNames, calleeNames, - calleeTexts, hasTryCatch, catches, - // Kept as aggregates of `catches` so the checks can migrate one at a time. - catchRethrows: catches.some((c) => c.rethrows), - catchBranches: catches.some((c) => c.branches), - catchesNarrowly: catches.length > 0 && catches.every((c) => c.narrow), logCalls, statementCount, }; diff --git a/internal-packages/observability-map/src/types.ts b/internal-packages/observability-map/src/types.ts index 4a9eec56b69..ff26c7174fe 100644 --- a/internal-packages/observability-map/src/types.ts +++ b/internal-packages/observability-map/src/types.ts @@ -17,17 +17,19 @@ export type CatchEvidence = { /** The clause contains a `throw`. */ rethrows: boolean; /** - * The clause picks what to do from what it caught: an `if`, a `switch`, or a conditional that is - * the whole `return`/`throw`. An `instanceof` used only to word a message, - * `json({ error: e instanceof Error ? e.message : String(e) })`, does not count: every error - * still leaves by the same path. + * The clause picks what to do from what it caught: an `if` or `switch` whose condition references + * the caught error binding, or a conditional that is the whole `return`/`throw`. `if (retries > 0)` + * does not count, and a bindingless `catch { ... }` cannot count at all. An `instanceof` used only + * to word a message, `json({ error: e instanceof Error ? e.message : String(e) })`, does not + * count either: every error still leaves by the same path. */ branches: boolean; /** * The guarded region parses something: `JSON.parse`, `request.json()`, a zod `parse`/`safeParse`, * a `decode`, or a `new URL`/`URLSearchParams`/`RegExp`. Those three constructors are read here - * because constructors never appear in `calleeTexts`; other constructors do not count, or every - * `new SomePresenter()` in a try would excuse its catch. + * because a `new` expression is not a call, so the call-callee scan that feeds this check never + * sees them; other constructors do not count, or every `new SomePresenter()` in a try would excuse + * its catch. */ guardsParse: boolean; /** Statements in the guarded try block, counted as `statementCount` counts them. */ @@ -38,9 +40,7 @@ export type CatchEvidence = { export type LogCall = { /** Full callee path, e.g. `logger.error`. */ callee: string; - /** Whether an object literal was passed as an argument. */ - hasObjectArgument: boolean; - /** Property names on that object literal, e.g. `["environmentId", "error"]`. */ + /** Property names on the first object-literal argument, e.g. `["environmentId", "error"]`. */ fields: string[]; /** Whether the call sits inside a catch clause, i.e. on the failure path. */ inCatch: boolean; @@ -58,12 +58,6 @@ export type EntryPoint = { importedNames: string[]; /** Names of functions called inside the loader/action bodies, or in a same-file helper they call. */ calleeNames: string[]; - /** - * The same calls as `calleeNames`, same order and same length, but as the whole callee path: - * `prisma.organization.findFirst` where `calleeNames` has `findFirst`. A path that runs through - * something unnameable (`new PromptService().createOverride`) falls back to the bare name. - */ - calleeTexts: string[]; /** * Whether a `try` appears in the loader/action bodies, or in a same-file helper they call. Note * that this says a `try`, not a catch: a `try`/`finally` sets it while `catches` stays empty and @@ -72,26 +66,6 @@ export type EntryPoint = { hasTryCatch: boolean; /** One entry per catch clause in those bodies, in source order. */ catches: CatchEvidence[]; - /** - * Whether any catch clause in those bodies contains a `throw`. A catch that rethrows has decided - * the error is not its to answer, which is a different act from swallowing it. Aggregate of - * `catches`, kept so existing consumers keep working. - */ - catchRethrows: boolean; - /** - * Whether any catch clause in those bodies branches on the error: an `if`, a `switch`, or an - * `instanceof`. With `catchRethrows` both false while `hasTryCatch` is true, every catch in the - * entry point takes one path out regardless of what was thrown. - */ - catchBranches: boolean; - /** - * Whether every catch in those bodies guards a specific operation rather than the handler: the - * entry point has at least one catch clause, and no try block with a catch holds more than two - * statements. The `try { body = await request.json() } catch { 400 }` idiom, which takes one path - * out and is still deliberate. False when any catch wraps the bulk of a body, and false when - * there is no catch clause at all. - */ - catchesNarrowly: boolean; /** Calls to a `logger.*` or `log.*` callee in those bodies, in source order. */ logCalls: LogCall[]; /** diff --git a/internal-packages/observability-map/test/checks.test.ts b/internal-packages/observability-map/test/checks.test.ts index cbc0c73a00d..eff2617e755 100644 --- a/internal-packages/observability-map/test/checks.test.ts +++ b/internal-packages/observability-map/test/checks.test.ts @@ -565,9 +565,10 @@ describe("request-context", () => { expect(r.status).toBe("fail"); }); - // Was a known false positive: `new URL()` is a constructor, so the parse was invisible while the - // evidence came from `calleeTexts`. `CatchEvidence.guardsParse` covers constructors, so the guard - // is legible now and the route is no longer judged as though it kept its failures. + // Was a known false positive: `new URL()` is a constructor, so the parse was invisible to the + // call-callee scan the evidence used to come from. `CatchEvidence.guardsParse` covers + // constructors, so the guard is legible now and the route is no longer judged as though it kept + // its failures. it("fails a route whose only catch guards a constructor parse", () => { const r = run( "request-context", diff --git a/internal-packages/observability-map/test/scan.test.ts b/internal-packages/observability-map/test/scan.test.ts index 6625adf9343..7f764e98981 100644 --- a/internal-packages/observability-map/test/scan.test.ts +++ b/internal-packages/observability-map/test/scan.test.ts @@ -571,7 +571,7 @@ describe("scanDirectory", () => { }); describe("scanFile: catch clause evidence", () => { - it("sets catchRethrows when a catch rethrows", () => { + it("sets rethrows on the clause when a catch rethrows", () => { const ep = scanFile( "rethrow.ts", ` @@ -586,8 +586,8 @@ describe("scanFile: catch clause evidence", () => { ` ); expect(ep!.hasTryCatch).toBe(true); - expect(ep!.catchRethrows).toBe(true); - expect(ep!.catchBranches).toBe(false); + expect(ep!.catches[0]!.rethrows).toBe(true); + expect(ep!.catches[0]!.branches).toBe(false); }); it("leaves both flags false when the catch only returns", () => { @@ -604,11 +604,11 @@ describe("scanFile: catch clause evidence", () => { ` ); expect(ep!.hasTryCatch).toBe(true); - expect(ep!.catchRethrows).toBe(false); - expect(ep!.catchBranches).toBe(false); + expect(ep!.catches[0]!.rethrows).toBe(false); + expect(ep!.catches[0]!.branches).toBe(false); }); - it("sets catchBranches for an `if` on the error", () => { + it("sets branches for an `if` on the error", () => { const ep = scanFile( "branch-if.ts", ` @@ -624,11 +624,11 @@ describe("scanFile: catch clause evidence", () => { } ` ); - expect(ep!.catchBranches).toBe(true); - expect(ep!.catchRethrows).toBe(false); + expect(ep!.catches[0]!.branches).toBe(true); + expect(ep!.catches[0]!.rethrows).toBe(false); }); - it("sets catchBranches for an instanceof conditional that is the whole returned expression", () => { + it("sets branches for an instanceof conditional that is the whole returned expression", () => { const ep = scanFile( "branch-instanceof.ts", ` @@ -641,10 +641,10 @@ describe("scanFile: catch clause evidence", () => { } ` ); - expect(ep!.catchBranches).toBe(true); + expect(ep!.catches[0]!.branches).toBe(true); }); - it("sets catchBranches for a switch in the catch", () => { + it("sets branches for a switch on the error", () => { const ep = scanFile( "branch-switch.ts", ` @@ -662,7 +662,7 @@ describe("scanFile: catch clause evidence", () => { } ` ); - expect(ep!.catchBranches).toBe(true); + expect(ep!.catches[0]!.branches).toBe(true); }); it("ignores a catch that lives in the React component", () => { @@ -683,8 +683,7 @@ describe("scanFile: catch clause evidence", () => { ` ); expect(ep!.hasTryCatch).toBe(false); - expect(ep!.catchRethrows).toBe(false); - expect(ep!.catchBranches).toBe(false); + expect(ep!.catches).toEqual([]); }); it("reads a catch inside a same-file helper the body delegates to", () => { @@ -705,11 +704,11 @@ describe("scanFile: catch clause evidence", () => { ` ); expect(ep!.hasTryCatch).toBe(true); - expect(ep!.catchRethrows).toBe(true); - expect(ep!.catchBranches).toBe(true); + expect(ep!.catches[0]!.rethrows).toBe(true); + expect(ep!.catches[0]!.branches).toBe(true); }); - it("leaves both flags false for a try with no catch", () => { + it("leaves catches empty for a try with no catch", () => { const ep = scanFile( "finally-only.ts", ` @@ -723,77 +722,7 @@ describe("scanFile: catch clause evidence", () => { ` ); expect(ep!.hasTryCatch).toBe(true); - expect(ep!.catchRethrows).toBe(false); - expect(ep!.catchBranches).toBe(false); - }); -}); - -describe("scanFile: callee texts", () => { - it("keeps the full callee expression alongside the bare name", () => { - const ep = scanFile( - "api.v1.things.ts", - ` - export async function loader({ request }) { - const org = await prisma.organization.findFirst({ where: { id: 1 } }); - logger.error("nope", { organizationId: org.id }); - return json(org); - } - ` - ); - expect(ep!.calleeNames).toContain("findFirst"); - expect(ep!.calleeTexts).toContain("prisma.organization.findFirst"); - expect(ep!.calleeTexts).toContain("logger.error"); - expect(ep!.calleeTexts).toContain("json"); - // Index-aligned with calleeNames, so a consumer can read either. - expect(ep!.calleeTexts).toHaveLength(ep!.calleeNames.length); - }); - - it("does not leak calls made in the React component", () => { - const ep = scanFile( - "route.tsx", - ` - export async function loader() { - return json(await prisma.run.findMany()); - } - export default function Page() { - useFancyHook(); - analytics.track("viewed"); - return null; - } - ` - ); - expect(ep!.calleeTexts).toContain("prisma.run.findMany"); - expect(ep!.calleeTexts).not.toContain("analytics.track"); - expect(ep!.calleeTexts).not.toContain("useFancyHook"); - }); - - it("records callee texts from a same-file helper the body delegates to", () => { - const ep = scanFile( - "delegating.ts", - ` - async function load(id) { - return prisma.project.findUnique({ where: { id } }); - } - export async function loader({ params }) { - return json(await load(params.id)); - } - ` - ); - expect(ep!.calleeTexts).toContain("prisma.project.findUnique"); - }); - - it("falls back to the bare name for a callee it cannot render as a path", () => { - const ep = scanFile( - "new-expression.ts", - ` - export async function action({ request }) { - return json(await new PromptService().createOverride(request)); - } - ` - ); - expect(ep!.calleeNames).toContain("createOverride"); - expect(ep!.calleeTexts).toContain("createOverride"); - expect(ep!.calleeTexts).toHaveLength(ep!.calleeNames.length); + expect(ep!.catches).toEqual([]); }); }); @@ -815,7 +744,6 @@ describe("scanFile: log calls", () => { expect(ep!.logCalls).toHaveLength(1); expect(ep!.logCalls[0]).toEqual({ callee: "logger.error", - hasObjectArgument: true, fields: ["environmentId", "error"], inCatch: true, }); @@ -831,9 +759,7 @@ describe("scanFile: log calls", () => { } ` ); - expect(ep!.logCalls).toEqual([ - { callee: "log.info", hasObjectArgument: false, fields: [], inCatch: false }, - ]); + expect(ep!.logCalls).toEqual([{ callee: "log.info", fields: [], inCatch: false }]); }); it("ignores a non-logger call and a log call in the React component", () => { @@ -872,9 +798,9 @@ describe("scanFile: narrow catches", () => { ` ); expect(ep!.hasTryCatch).toBe(true); - expect(ep!.catchRethrows).toBe(false); - expect(ep!.catchBranches).toBe(false); - expect(ep!.catchesNarrowly).toBe(true); + expect(ep!.catches[0]!.rethrows).toBe(false); + expect(ep!.catches[0]!.branches).toBe(false); + expect(ep!.catches[0]!.narrow).toBe(true); }); it("does not flag a catch wrapping the whole handler", () => { @@ -896,10 +822,10 @@ describe("scanFile: narrow catches", () => { ` ); expect(ep!.hasTryCatch).toBe(true); - expect(ep!.catchesNarrowly).toBe(false); + expect(ep!.catches[0]!.narrow).toBe(false); }); - it("does not flag a body that has both a narrow catch and a broad one", () => { + it("keeps a narrow catch and a broad one distinct", () => { const ep = scanFile( "mixed.ts", ` @@ -921,7 +847,8 @@ describe("scanFile: narrow catches", () => { } ` ); - expect(ep!.catchesNarrowly).toBe(false); + expect(ep!.catches[0]!.narrow).toBe(true); + expect(ep!.catches[1]!.narrow).toBe(false); }); it("allows a guarded operation with its own local binding", () => { @@ -940,7 +867,7 @@ describe("scanFile: narrow catches", () => { } ` ); - expect(ep!.catchesNarrowly).toBe(true); + expect(ep!.catches[0]!.narrow).toBe(true); }); it("does not flag a try of three statements", () => { @@ -958,16 +885,16 @@ describe("scanFile: narrow catches", () => { } ` ); - expect(ep!.catchesNarrowly).toBe(false); + expect(ep!.catches[0]!.narrow).toBe(false); }); - it("is false when there is no try at all", () => { + it("is empty when there is no try at all", () => { const ep = scanFile("plain.ts", `export async function loader() { return json({}); }`); expect(ep!.hasTryCatch).toBe(false); - expect(ep!.catchesNarrowly).toBe(false); + expect(ep!.catches).toEqual([]); }); - it("is false for a try with a finally and no catch", () => { + it("is empty for a try with a finally and no catch", () => { const ep = scanFile( "finally-only.ts", ` @@ -981,7 +908,7 @@ describe("scanFile: narrow catches", () => { ` ); expect(ep!.hasTryCatch).toBe(true); - expect(ep!.catchesNarrowly).toBe(false); + expect(ep!.catches).toEqual([]); }); it("reads a narrow catch inside a same-file helper the body delegates to", () => { @@ -1001,7 +928,7 @@ describe("scanFile: narrow catches", () => { ` ); expect(ep!.hasTryCatch).toBe(true); - expect(ep!.catchesNarrowly).toBe(true); + expect(ep!.catches[0]!.narrow).toBe(true); }); it("ignores a narrow catch that lives in the React component", () => { @@ -1022,7 +949,7 @@ describe("scanFile: narrow catches", () => { ` ); expect(ep!.hasTryCatch).toBe(false); - expect(ep!.catchesNarrowly).toBe(false); + expect(ep!.catches).toEqual([]); }); }); @@ -1062,8 +989,6 @@ describe("scanFile: per-catch evidence", () => { guardsParse: false, tryStatementCount: 4, }); - // The aggregate still collapses, which is why the per-catch list exists. - expect(ep!.catchesNarrowly).toBe(false); }); it("leaves catches empty for a try/finally with no catch clause", () => { @@ -1087,9 +1012,6 @@ describe("scanFile: per-catch evidence", () => { expect(ep!.catches).toEqual([]); // hasTryCatch keeps its meaning: a `try` appears. Nothing is caught here. expect(ep!.hasTryCatch).toBe(true); - expect(ep!.catchRethrows).toBe(false); - expect(ep!.catchBranches).toBe(false); - expect(ep!.catchesNarrowly).toBe(false); }); it("sees a URL constructor as a guarded parse", () => { @@ -1193,9 +1115,6 @@ describe("scanFile: per-catch evidence", () => { expect(ep!.catches).toHaveLength(2); expect(ep!.catches.filter((c) => c.rethrows && c.branches)).toHaveLength(1); expect(ep!.catches.filter((c) => !c.rethrows && !c.branches)).toHaveLength(1); - // Aggregates stay as they are: any clause sets them. - expect(ep!.catchRethrows).toBe(true); - expect(ep!.catchBranches).toBe(true); }); it("includes a catch from a same-file helper and excludes one from the React component", () => { @@ -1224,26 +1143,6 @@ describe("scanFile: per-catch evidence", () => { ); expect(ep!.catches).toHaveLength(1); expect(ep!.catches[0]!.guardsParse).toBe(true); - expect(ep!.catchRethrows).toBe(false); - }); - - it("keeps the aggregates derivable from the per-catch list", () => { - const ep = scanFile( - "aggregate.ts", - ` - export async function loader({ request }) { - try { - return json(await request.json()); - } catch (e) { - if (e instanceof SyntaxError) return json({}, { status: 400 }); - return json({}, { status: 500 }); - } - } - ` - ); - expect(ep!.catchRethrows).toBe(ep!.catches.some((c) => c.rethrows)); - expect(ep!.catchBranches).toBe(ep!.catches.some((c) => c.branches)); - expect(ep!.catchesNarrowly).toBe(ep!.catches.length > 0 && ep!.catches.every((c) => c.narrow)); }); }); @@ -1353,7 +1252,6 @@ describe("scanFile: branches ignores the error-stringifying ternary", () => { ); expect(ep!.catches).toHaveLength(1); expect(ep!.catches[0]!.branches).toBe(false); - expect(ep!.catchBranches).toBe(false); }); it("does not count an instanceof used to build a logged message", () => { From 751af3feb85e91bc4687ab5afe86a600cd4a3a18 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Sat, 1 Aug 2026 00:20:07 +0100 Subject: [PATCH 031/106] fix(observability-map): stop crediting an if or switch that never looks at the error catchClauseEvidence counted any if/switch inside a catch as branching, whether or not its condition read the caught error. catch (_e) { if (organization) ... } and catch (error: any) { if (wantsJson) ... } both counted, crediting a route that always takes the same path regardless of what was thrown. The if condition (or switch discriminant) must now reference the caught error binding; a bindingless catch { ... } cannot qualify at all. Ternary handling is unchanged. Measured on the real tree: 5 of 242 catch clauses lose branches, 4 routes flip from pass to fail on error-classification, and the global score moves from 18 to 17. Hand-read all 4: each was a false positive, branching on an unrelated local rather than the error. --- .../observability-map/src/scan.ts | 34 ++++++- .../observability-map/test/scan.test.ts | 98 +++++++++++++++++++ 2 files changed, 127 insertions(+), 5 deletions(-) diff --git a/internal-packages/observability-map/src/scan.ts b/internal-packages/observability-map/src/scan.ts index 0f10138e627..9cb19407c01 100644 --- a/internal-packages/observability-map/src/scan.ts +++ b/internal-packages/observability-map/src/scan.ts @@ -153,11 +153,28 @@ function guardsParse(tryBlock: ts.Block): boolean { return found; } +/** Whether some node in the tree rooted at `node` matches `predicate`. */ +function someNode(node: ts.Node, predicate: (n: ts.Node) => boolean): boolean { + if (predicate(node)) return true; + return ts.forEachChild(node, (child) => someNode(child, predicate)) === true; +} + function containsInstanceOf(node: ts.Node): boolean { - if (ts.isBinaryExpression(node) && node.operatorToken.kind === ts.SyntaxKind.InstanceOfKeyword) { - return true; - } - return ts.forEachChild(node, containsInstanceOf) === true; + return someNode( + node, + (n) => ts.isBinaryExpression(n) && n.operatorToken.kind === ts.SyntaxKind.InstanceOfKeyword + ); +} + +/** Whether `node` reads the given catch binding anywhere, e.g. `e` in `e instanceof X` or `error.code`. */ +function referencesBinding(node: ts.Node, bindingName: string): boolean { + return someNode(node, (n) => ts.isIdentifier(n) && n.text === bindingName); +} + +/** The catch binding's name, or null for a bindingless `catch { ... }` or a destructured one. */ +function catchBindingName(clause: ts.CatchClause): string | null { + const decl = clause.variableDeclaration; + return decl && ts.isIdentifier(decl.name) ? decl.name.text : null; } /** @@ -177,10 +194,17 @@ function selectsAnErrorPath(node: ts.ConditionalExpression): boolean { function catchClauseEvidence(clause: ts.CatchClause): { rethrows: boolean; branches: boolean } { let rethrows = false; let branches = false; + const bindingName = catchBindingName(clause); const visit = (node: ts.Node) => { if (ts.isThrowStatement(node)) rethrows = true; - if (ts.isIfStatement(node) || ts.isSwitchStatement(node)) branches = true; + if ( + bindingName !== null && + ((ts.isIfStatement(node) && referencesBinding(node.expression, bindingName)) || + (ts.isSwitchStatement(node) && referencesBinding(node.expression, bindingName))) + ) { + branches = true; + } if (ts.isConditionalExpression(node) && selectsAnErrorPath(node)) branches = true; ts.forEachChild(node, visit); }; diff --git a/internal-packages/observability-map/test/scan.test.ts b/internal-packages/observability-map/test/scan.test.ts index 7f764e98981..db1ff081952 100644 --- a/internal-packages/observability-map/test/scan.test.ts +++ b/internal-packages/observability-map/test/scan.test.ts @@ -1289,3 +1289,101 @@ describe("scanFile: branches ignores the error-stringifying ternary", () => { expect(ep!.catches[0]!.branches).toBe(true); }); }); + +describe("scanFile: branches requires the if/switch condition to examine the error", () => { + it("does not set branches for an `if` on an unrelated variable", () => { + const ep = scanFile( + "retry-count.ts", + ` + export async function action({ request }) { + let attempt = 0; + try { + return json(await load(request)); + } catch (e) { + if (attempt > 3) return json({}, { status: 503 }); + return json({}, { status: 500 }); + } + } + ` + ); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + it("sets branches for an `if` whose condition references the caught error", () => { + const ep = scanFile( + "branch-if-e.ts", + ` + export async function loader({ request }) { + try { + return json(await load(request)); + } catch (e) { + if (e instanceof ApiError) return json({}, { status: e.status }); + return json({}, { status: 500 }); + } + } + ` + ); + expect(ep!.catches[0]!.branches).toBe(true); + }); + + it("sets branches for a `switch` on a property of the caught error", () => { + const ep = scanFile( + "branch-switch-error-code.ts", + ` + export async function loader({ request }) { + try { + return json(await load(request)); + } catch (error) { + switch (error.code) { + case "P2025": + return json({}, { status: 404 }); + default: + return json({}, { status: 500 }); + } + } + } + ` + ); + expect(ep!.catches[0]!.branches).toBe(true); + }); + + it("does not set branches for a `switch` on an unrelated discriminant", () => { + const ep = scanFile( + "branch-switch-unrelated.ts", + ` + export async function action({ request }) { + const mode = "strict"; + try { + return json(await load(request)); + } catch (error) { + switch (mode) { + case "strict": + return json({}, { status: 400 }); + default: + return json({}, { status: 500 }); + } + } + } + ` + ); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + it("cannot set branches for a bindingless catch, even with an `if` inside it", () => { + const ep = scanFile( + "bindingless-if.ts", + ` + export async function loader({ request }) { + let attempt = 0; + try { + return json(await load(request)); + } catch { + if (attempt > 3) return json({}, { status: 503 }); + return json({}, { status: 500 }); + } + } + ` + ); + expect(ep!.catches[0]!.branches).toBe(false); + }); +}); From 11c590890ede45f21d0394a896f10d61a53d38b7 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Sat, 1 Aug 2026 00:20:20 +0100 Subject: [PATCH 032/106] docs(observability-map): explain triviality's exclusion and the gaming boundary Two gaps: nothing said what makes a route trivial or why exclusion beats a vacuous 100, and nothing said the tool verifies presence, not meaning. Documents isTrivial's rule and adds a gaming-boundary section, measured on the real tree: a synthetic environmentId on every in-catch log call, with no other change, moves the global score from 17 to 27. Also updates the current-score figures for the branches fix in the prior commit. --- internal-packages/observability-map/README.md | 39 +++++++++++++++---- 1 file changed, 32 insertions(+), 7 deletions(-) diff --git a/internal-packages/observability-map/README.md b/internal-packages/observability-map/README.md index 457c22ff390..703404aef9f 100644 --- a/internal-packages/observability-map/README.md +++ b/internal-packages/observability-map/README.md @@ -4,7 +4,7 @@ Scores every webapp entry point on whether it could explain itself during an inc the ones worth fixing. An entry point is a Remix `loader` or `action` under `apps/webapp/app/routes`, 427 of them at the time of writing. -The number it prints today is 18 out of 100. That is not a bug, and the rest of this file is mostly +The number it prints today is 17 out of 100. That is not a bug, and the rest of this file is mostly about why you should believe it. ## Running it @@ -20,7 +20,7 @@ suppresses. The single-route mode takes either the route path the report prints the file name (`api.v1.token.ts`). An exact match wins over the routes it is a prefix of, and an ambiguous prefix warns and names the alternatives rather than silently picking one. -## What 18 means +## What 17 means It is the mean score of the 412 entry points that had at least one applicable check, where an entry's score is the share of its applicable checks that passed. It is low because the webapp does @@ -34,12 +34,12 @@ error handling. Two invariants hold now, and both are asserted in `test/score.test.ts` rather than measured once: -- **Removing error handling must not raise the score.** Deleting every catch clause takes 18 to 8, - and deleting the logs as well takes it to 2. +- **Removing error handling must not raise the score.** Deleting every catch clause drops it to 8, + and deleting the logs as well drops it to 2. - **Adding error handling that does nothing must not raise the score.** Wrapping every body in - `try { ... } catch (e) { throw e }` leaves it at 18, with no entry moving in either direction. - That mutation used to be worth 27 points across the tree, because a rethrow-only clause counted - as a pass while no catch at all was not-applicable, and the two are observationally identical. + `try { ... } catch (e) { throw e }` leaves the score unchanged. That mutation used to be worth 27 + points across the tree, because a rethrow-only clause counted as a pass while no catch at all was + not-applicable, and the two are observationally identical. If you change this package, check both directions still hold. @@ -89,6 +89,16 @@ unmeasured entries into the mean at 100, would let the tool look better the less header prints both counts (`412 measured, 15 unmeasured`) so the denominator is never hidden, and a family with nothing measured renders as `not measured` rather than as a full green bar. +15 of those 427 routes are unmeasured because `isTrivial` (`src/triviality.ts`) rules them out before +any check runs. Trivial means a body of three statements or fewer, three or fewer calls, no +try/catch, no builder wrapping it, and nothing in the calls or the source naming a datastore or a +service (`prisma`, `logger`, `fetch`, `redis`, and the like). Parse the params, build a path, +redirect: nothing there for a check to find evidence in either way. Exclusion is a denominator exit, +not a credit: a trivial route's `score` is the same placeholder 100 that an unmeasured entry always +carries, and it is left out of every mean for the same reason. Scoring it a pass instead would say a +route earned a clean result by never doing anything a check could look at, which is the same vacuous +100 the header already refuses to average in. + ## When a check declines to judge The rule every applicability decision follows: **would this evidence necessarily be visible in the @@ -127,6 +137,21 @@ check holds the number still rather than improving it. What you buy is removal f with a reason on the record. The report prints how many suppressions are in force so the practice stays visible. +## The gaming boundary + +`request-context` checks that a failure-path log names a tenant field. It does not check that the +value is real. A codemod that added `environmentId` to every in-catch `logger.error` call, wiring it +up to the wrong variable or a constant, would move the score exactly as far as one that wired it up +correctly. Measured on the real tree: adding a synthetic `environmentId` field to every in-catch log +call, with no other change, takes the global score from 17 to 27. + +That is the tool verifying presence, not meaning, and it is not a bug to fix. Every check here reads +syntax: a field name, a call, a binding reference. None of them can tell a genuine tenant id from a +hardcoded string with the right key. What a reviewer owns is whether the value behind the field is +real, the same way a Lighthouse accessibility score checks that an `alt` attribute exists and not +that its text describes the image. The number tells you where to look. It does not tell you what +you will find there. + ## Known limits Read these before trusting a specific verdict. From b8971419ff4cb37778e32031c1110a062a80b079 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Sat, 1 Aug 2026 00:27:12 +0100 Subject: [PATCH 033/106] docs(observability-map): refresh the collapse figures and say audit-trail collapses too --- internal-packages/observability-map/README.md | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/internal-packages/observability-map/README.md b/internal-packages/observability-map/README.md index 703404aef9f..0c0fe56d2d8 100644 --- a/internal-packages/observability-map/README.md +++ b/internal-packages/observability-map/README.md @@ -67,12 +67,14 @@ rather than celebrating. `audit-trail` fails 19 of 19, and `request-context` fails 391 of 412. Printing either one per route would bury the route-specific findings under the same sentence repeated hundreds of times, so both -are reported as a figure: the `AUDIT` and `CONTEXT` lines. 333 entry points fail nothing except -`request-context` and appear only in that figure, which leaves 67 in the fix list. An entry that -fails `request-context` *and* something else keeps both findings and stays in the list, so -`/account/tokens` still shows the whole picture. - -18 of those 333 are sensitive, including `/admin/impersonate`, the API-key regeneration route and +are reported as a figure: the `AUDIT` and `CONTEXT` lines. 329 entry points fail nothing except +`request-context` and appear only in that figure, which leaves 71 in the fix list. An entry that +fails `request-context` *and* another scored check keeps both findings and stays in the list, so +`/account/tokens` still shows the whole picture. `audit-trail` does not count as "another" for this +purpose: it is already a headline, so a route failing only `request-context` and `audit-trail` +collapses too (13 do today, all sensitive, which is most of the 18 below). + +18 of those 329 are sensitive, including `/admin/impersonate`, the API-key regeneration route and four envvars routes, so the `CONTEXT` line says how many. Read them out of `observability-map.json`, where every entry keeps its full check results, rather than assuming the list is the whole story. From dd3c2f6a567d7c909033605da6671f9e6e21d025 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Sat, 1 Aug 2026 00:51:30 +0100 Subject: [PATCH 034/106] fix(observability-map): remove the last dead scanner field and pin the callee-path fallback CatchEvidence.narrow was the same species as the fields removed earlier: written by scan.ts, asserted in tests, read by nothing. error-classification computes narrowness itself inline through isParseGuard and never reads clause.narrow. Removed the field, its NARROW_TRY_STATEMENTS derivation, and every test asserting it; kept tryStatementCount, which isParseGuard reads. Also pins the two callee-path shapes the earlier callee-texts test deletion left unexercised: a three-level property chain (prisma.organization.findFirst arriving as findFirst in calleeNames) and the constructor-fallback shape (new PromptService().createOverride), plus a logger call in the same fixture to confirm calleeText's normal chain-building still feeds LogCall.callee. --- .../src/checks/errorClassification.ts | 8 +- .../observability-map/src/scan.ts | 8 - .../observability-map/src/types.ts | 2 - .../observability-map/test/scan.test.ts | 230 ++++-------------- 4 files changed, 56 insertions(+), 192 deletions(-) diff --git a/internal-packages/observability-map/src/checks/errorClassification.ts b/internal-packages/observability-map/src/checks/errorClassification.ts index 2c48f40dded..a78559eefba 100644 --- a/internal-packages/observability-map/src/checks/errorClassification.ts +++ b/internal-packages/observability-map/src/checks/errorClassification.ts @@ -54,10 +54,10 @@ function isParseGuard(clause: CatchEvidence, ep: EntryPoint): boolean { * `request-context` still reads that log and asks whether it names a tenant, so the reporting is * unrewarded here rather than unmeasured. * - * `narrow` is not a way to qualify either. A one-statement try around `await service.call(run)` is - * narrow and is still a swallow: reading all eleven entry points that limb would clear said six - * were real, including a silent run cancellation and two credential paths that report a database - * failure to the browser as a 400 with an internal message in it. + * A narrow guard is not a way to qualify either. A one-statement try around `await + * service.call(run)` is narrow and is still a swallow: reading all eleven entry points that limb + * would clear said six were real, including a silent run cancellation and two credential paths + * that report a database failure to the browser as a 400 with an internal message in it. */ function decides(clause: CatchEvidence, ep: EntryPoint): boolean { return clause.branches || isParseGuard(clause, ep); diff --git a/internal-packages/observability-map/src/scan.ts b/internal-packages/observability-map/src/scan.ts index 9cb19407c01..3454d0ad4c5 100644 --- a/internal-packages/observability-map/src/scan.ts +++ b/internal-packages/observability-map/src/scan.ts @@ -107,13 +107,6 @@ function objectArgumentFields(call: ts.CallExpression): string[] { return []; } -/** - * How much a try block may guard and still count as narrow. Two, so that the guarded operation can - * bind its result (`const stripped = ...; new RegExp(stripped);`) but a third statement means the - * try has started to cover the handler rather than one operation. - */ -const NARROW_TRY_STATEMENTS = 2; - /** * Calls that turn input into a value and throw when it is malformed. `parse`/`safeParse` cover * `JSON.parse` and the zod schemas. `.json` has to be a member call, because a bare `json(...)` is @@ -563,7 +556,6 @@ export function scanFile(fileName: string, source: string): EntryPoint | null { const tryStatementCount = countStatements(node.tryBlock.statements); const clause = catchClauseEvidence(node.catchClause); catches.push({ - narrow: tryStatementCount <= NARROW_TRY_STATEMENTS, rethrows: clause.rethrows, branches: clause.branches, guardsParse: guardsParse(node.tryBlock), diff --git a/internal-packages/observability-map/src/types.ts b/internal-packages/observability-map/src/types.ts index ff26c7174fe..a1d3be0b2db 100644 --- a/internal-packages/observability-map/src/types.ts +++ b/internal-packages/observability-map/src/types.ts @@ -12,8 +12,6 @@ export type CheckResult = { * legible instead of collapsing into one boolean. */ export type CatchEvidence = { - /** The guarded try block holds at most two statements: one operation, not the handler. */ - narrow: boolean; /** The clause contains a `throw`. */ rethrows: boolean; /** diff --git a/internal-packages/observability-map/test/scan.test.ts b/internal-packages/observability-map/test/scan.test.ts index db1ff081952..d9af8df904c 100644 --- a/internal-packages/observability-map/test/scan.test.ts +++ b/internal-packages/observability-map/test/scan.test.ts @@ -460,6 +460,31 @@ describe("scanFile: callee resolution", () => { expect(ep!.hasLoader).toBe(true); expect(ep!.loaderInitializerCallee).toBeNull(); }); + + it("resolves a multi-level call and falls back to the bare name past an unnameable one", () => { + const ep = scanFile( + "api.v1.things.ts", + ` + export async function loader({ request }) { + try { + const org = await prisma.organization.findFirst({ where: { id: 1 } }); + return json(await new PromptService().createOverride(org)); + } catch (e) { + logger.error("nope", { error: e }); + return json({}, { status: 500 }); + } + } + ` + ); + // A three-level property chain still lands on its bare method name. + expect(ep!.calleeNames).toContain("findFirst"); + // A chain through a `new` expression has no name of its own, so this also falls back to the + // bare name rather than losing the call. + expect(ep!.calleeNames).toContain("createOverride"); + // The full path still builds where nothing unnameable sits in it, which is what `LogCall.callee` + // depends on. + expect(ep!.logCalls[0]!.callee).toBe("logger.error"); + }); }); describe("scanFile: parse failures", () => { @@ -724,62 +749,7 @@ describe("scanFile: catch clause evidence", () => { expect(ep!.hasTryCatch).toBe(true); expect(ep!.catches).toEqual([]); }); -}); -describe("scanFile: log calls", () => { - it("records the fields of a log call's object argument", () => { - const ep = scanFile( - "logging.ts", - ` - export async function loader({ request }) { - try { - return json(await load(request)); - } catch (e) { - logger.error("load failed", { environmentId: env.id, error: e }); - return json({}, { status: 500 }); - } - } - ` - ); - expect(ep!.logCalls).toHaveLength(1); - expect(ep!.logCalls[0]).toEqual({ - callee: "logger.error", - fields: ["environmentId", "error"], - inCatch: true, - }); - }); - - it("records a log call with no object argument, outside a catch", () => { - const ep = scanFile( - "logging-plain.ts", - ` - export async function loader() { - log.info("starting"); - return json({}); - } - ` - ); - expect(ep!.logCalls).toEqual([{ callee: "log.info", fields: [], inCatch: false }]); - }); - - it("ignores a non-logger call and a log call in the React component", () => { - const ep = scanFile( - "route.tsx", - ` - export async function loader() { - return json(await load()); - } - export default function Page() { - logger.debug("rendered", { runId: 1 }); - return null; - } - ` - ); - expect(ep!.logCalls).toEqual([]); - }); -}); - -describe("scanFile: narrow catches", () => { it("flags a try that guards a single request.json()", () => { const ep = scanFile( "admin.api.v1.platform-notifications.ts", @@ -800,92 +770,6 @@ describe("scanFile: narrow catches", () => { expect(ep!.hasTryCatch).toBe(true); expect(ep!.catches[0]!.rethrows).toBe(false); expect(ep!.catches[0]!.branches).toBe(false); - expect(ep!.catches[0]!.narrow).toBe(true); - }); - - it("does not flag a catch wrapping the whole handler", () => { - const ep = scanFile( - "otel.v1.logs.ts", - ` - export async function action({ request }) { - try { - const exporter = await otlpExporter; - const contentType = request.headers.get("content-type") ?? ""; - const body = await request.json(); - await exporter.export(body); - return json({ ok: true }); - } catch (e) { - logger.error(e); - return json({}, { status: 500 }); - } - } - ` - ); - expect(ep!.hasTryCatch).toBe(true); - expect(ep!.catches[0]!.narrow).toBe(false); - }); - - it("keeps a narrow catch and a broad one distinct", () => { - const ep = scanFile( - "mixed.ts", - ` - export async function action({ request }) { - let body; - try { - body = await request.json(); - } catch { - return json({}, { status: 400 }); - } - try { - const run = await find(body.id); - const updated = await update(run); - await notify(updated); - return json(updated); - } catch (e) { - return json({}, { status: 500 }); - } - } - ` - ); - expect(ep!.catches[0]!.narrow).toBe(true); - expect(ep!.catches[1]!.narrow).toBe(false); - }); - - it("allows a guarded operation with its own local binding", () => { - const ep = scanFile( - "regex.ts", - ` - export async function action({ request }) { - const pattern = await patternFrom(request); - try { - const stripped = pattern.startsWith("(?i)") ? pattern.slice(4) : pattern; - new RegExp(stripped); - } catch { - return json({ error: "Invalid regex" }, { status: 400 }); - } - return json({ ok: true }); - } - ` - ); - expect(ep!.catches[0]!.narrow).toBe(true); - }); - - it("does not flag a try of three statements", () => { - const ep = scanFile( - "three.ts", - ` - export async function loader({ request }) { - try { - const raw = await request.json(); - const parsed = Schema.parse(raw); - return json(parsed); - } catch { - return json({}, { status: 400 }); - } - } - ` - ); - expect(ep!.catches[0]!.narrow).toBe(false); }); it("is empty when there is no try at all", () => { @@ -893,68 +777,63 @@ describe("scanFile: narrow catches", () => { expect(ep!.hasTryCatch).toBe(false); expect(ep!.catches).toEqual([]); }); +}); - it("is empty for a try with a finally and no catch", () => { +describe("scanFile: log calls", () => { + it("records the fields of a log call's object argument", () => { const ep = scanFile( - "finally-only.ts", + "logging.ts", ` - export async function loader() { + export async function loader({ request }) { try { - return json(await load()); - } finally { - release(); + return json(await load(request)); + } catch (e) { + logger.error("load failed", { environmentId: env.id, error: e }); + return json({}, { status: 500 }); } } ` ); - expect(ep!.hasTryCatch).toBe(true); - expect(ep!.catches).toEqual([]); + expect(ep!.logCalls).toHaveLength(1); + expect(ep!.logCalls[0]).toEqual({ + callee: "logger.error", + fields: ["environmentId", "error"], + inCatch: true, + }); }); - it("reads a narrow catch inside a same-file helper the body delegates to", () => { + it("records a log call with no object argument, outside a catch", () => { const ep = scanFile( - "helper-narrow.ts", + "logging-plain.ts", ` - function parseTags(payload) { - try { - return JSON.parse(payload); - } catch { - return null; - } - } - export async function loader({ params }) { - return json(parseTags(params.payload)); + export async function loader() { + log.info("starting"); + return json({}); } ` ); - expect(ep!.hasTryCatch).toBe(true); - expect(ep!.catches[0]!.narrow).toBe(true); + expect(ep!.logCalls).toEqual([{ callee: "log.info", fields: [], inCatch: false }]); }); - it("ignores a narrow catch that lives in the React component", () => { + it("ignores a non-logger call and a log call in the React component", () => { const ep = scanFile( "route.tsx", ` export async function loader() { - return json({}); + return json(await load()); } export default function Page() { - try { - JSON.parse(raw); - } catch { - return null; - } + logger.debug("rendered", { runId: 1 }); return null; } ` ); - expect(ep!.hasTryCatch).toBe(false); - expect(ep!.catches).toEqual([]); + expect(ep!.logCalls).toEqual([]); }); }); describe("scanFile: per-catch evidence", () => { - it("records one entry per catch clause, keeping a narrow guard distinct from a broad catch", () => { + it("records one entry per catch clause, keeping a parse guard distinct from a broad catch", () => { const ep = scanFile( "two-catches.ts", ` @@ -978,14 +857,12 @@ describe("scanFile: per-catch evidence", () => { ); expect(ep!.catches).toHaveLength(2); expect(ep!.catches[0]).toEqual({ - narrow: true, rethrows: false, branches: false, guardsParse: true, tryStatementCount: 1, }); expect(ep!.catches[1]).toMatchObject({ - narrow: false, guardsParse: false, tryStatementCount: 4, }); @@ -1035,10 +912,9 @@ describe("scanFile: per-catch evidence", () => { ); expect(ep!.catches).toHaveLength(1); expect(ep!.catches[0]!.guardsParse).toBe(true); - expect(ep!.catches[0]!.narrow).toBe(true); }); - it("sees a parse in a try that grew past the narrowness threshold", () => { + it("sees a parse in a try that outgrows a single statement", () => { const ep = scanFile( "admin.api.v1.orgs.$organizationId.stream-basin.ts", ` @@ -1060,7 +936,6 @@ describe("scanFile: per-catch evidence", () => { ` ); expect(ep!.catches).toHaveLength(1); - expect(ep!.catches[0]!.narrow).toBe(false); expect(ep!.catches[0]!.guardsParse).toBe(true); expect(ep!.catches[0]!.tryStatementCount).toBe(6); }); @@ -1083,7 +958,6 @@ describe("scanFile: per-catch evidence", () => { ); expect(ep!.catches).toHaveLength(1); expect(ep!.catches[0]).toEqual({ - narrow: false, rethrows: false, branches: false, guardsParse: false, From 1df57d4d7c1840b3db685349c546ccf0cd0ceed2 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Sat, 1 Aug 2026 14:24:46 +0100 Subject: [PATCH 035/106] feat(observability-map): add --routes flag and a PR-comment renderer for CI --- internal-packages/observability-map/README.md | 7 + .../observability-map/src/cli.ts | 22 ++- .../observability-map/src/report/prComment.ts | 179 ++++++++++++++++++ .../src/report/prCommentCli.ts | 39 ++++ .../observability-map/src/report/terminal.ts | 61 +++--- .../observability-map/test/cli.test.ts | 33 +++- .../observability-map/test/prComment.test.ts | 119 ++++++++++++ .../test/prCommentCli.test.ts | 57 ++++++ 8 files changed, 489 insertions(+), 28 deletions(-) create mode 100644 internal-packages/observability-map/src/report/prComment.ts create mode 100644 internal-packages/observability-map/src/report/prCommentCli.ts create mode 100644 internal-packages/observability-map/test/prComment.test.ts create mode 100644 internal-packages/observability-map/test/prCommentCli.test.ts diff --git a/internal-packages/observability-map/README.md b/internal-packages/observability-map/README.md index 0c0fe56d2d8..76638b7e7ab 100644 --- a/internal-packages/observability-map/README.md +++ b/internal-packages/observability-map/README.md @@ -20,6 +20,13 @@ suppresses. The single-route mode takes either the route path the report prints the file name (`api.v1.token.ts`). An exact match wins over the routes it is a prefix of, and an ambiguous prefix warns and names the alternatives rather than silently picking one. +## CI + +A PR that touches `apps/webapp/app/routes` or this package gets a sticky comment scanning head +against the PR's merge base, with the score, what changed, and the current fix list. It is +report-only: nothing here fails the build or blocks a merge, and the gate stays deferred until a +later phase decides to add one. See `.github/workflows/observability-map.yml`. + ## What 17 means It is the mean score of the 412 entry points that had at least one applicable check, where an diff --git a/internal-packages/observability-map/src/cli.ts b/internal-packages/observability-map/src/cli.ts index 4be0ff59193..cb9c9b6017a 100644 --- a/internal-packages/observability-map/src/cli.ts +++ b/internal-packages/observability-map/src/cli.ts @@ -1,4 +1,4 @@ -import { existsSync, writeFileSync } from "node:fs"; +import { existsSync, statSync, writeFileSync } from "node:fs"; import { dirname, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import type { EntryPoint } from "./types.js"; @@ -63,7 +63,25 @@ export function main(argv: string[], io: Io = processIo): number { const target = args.find((a) => !a.startsWith("--")); const repoRoot = findRepoRoot(dirname(fileURLToPath(import.meta.url))); - const routesDir = resolve(repoRoot, DEFAULT_ROUTES); + const routesFlag = args.find((a) => a.startsWith("--routes=")); + + let routesDir: string; + if (routesFlag) { + routesDir = resolve(process.cwd(), routesFlag.slice("--routes=".length)); + let isDir = false; + try { + isDir = statSync(routesDir).isDirectory(); + } catch { + isDir = false; + } + if (!isDir) { + io.err(`--routes: not a readable directory: ${routesDir}\n`); + return 1; + } + } else { + routesDir = resolve(repoRoot, DEFAULT_ROUTES); + } + const { entryPoints, parseFailures } = scanDirectory(routesDir); if (target) { diff --git a/internal-packages/observability-map/src/report/prComment.ts b/internal-packages/observability-map/src/report/prComment.ts new file mode 100644 index 00000000000..ed4480959a5 --- /dev/null +++ b/internal-packages/observability-map/src/report/prComment.ts @@ -0,0 +1,179 @@ +import type { MapReport, ScoredEntry } from "../score.js"; +import { auditLine, contextLine, contextOnly, scoredFailures } from "./terminal.js"; + +/** First line of every comment this job posts, so the upsert step can find its own comment again. */ +export const MARKER = ""; + +const MAX_CHANGED_ROWS = 15; + +const failingIds = (e: ScoredEntry) => e.checks.filter((c) => c.status === "fail").map((c) => c.id); + +function scoreLine(head: MapReport, base: MapReport | null): string { + const headline = + head.global === null + ? `not measured over ${head.measured} measured of ${head.entries.length} entry points` + : `**${head.global}/100** over ${head.measured} measured of ${head.entries.length} entry points`; + + if (!base) return headline; + if (base.global === null || head.global === null) return `${headline} (base not measured)`; + + const diff = head.global - base.global; + const comparison = + diff === 0 + ? `(base ${base.global}, no change)` + : diff > 0 + ? `(base ${base.global}, up ${diff})` + : `(base ${base.global}, down ${-diff})`; + return `${headline} ${comparison}`; +} + +type ChangedRow = { + routePath: string; + sensitive: boolean; + baseScore: number | "new"; + headScore: number; + nowFailing: string[]; + /** + * How much the entry got worse, used to sort the table. A new entry has no base score to + * subtract from, so it is scored against a perfect 100: a new entry landing at 60 sorts the + * same as an existing one that dropped 40 points, which is the ordering "what needs fixing + * first" implies. + */ + drop: number; +}; + +function changedRows(head: MapReport, base: MapReport): { rows: ChangedRow[]; removed: number } { + const baseByFile = new Map(base.entries.map((e) => [e.fileName, e])); + const headFiles = new Set(head.entries.map((e) => e.fileName)); + + const rows: ChangedRow[] = []; + for (const h of head.entries) { + const b = baseByFile.get(h.fileName); + if (!b) { + rows.push({ + routePath: h.routePath, + sensitive: h.sensitive, + baseScore: "new", + headScore: h.score, + nowFailing: failingIds(h), + drop: 100 - h.score, + }); + continue; + } + if (b.score === h.score) continue; + const baseFailing = new Set(failingIds(b)); + rows.push({ + routePath: h.routePath, + sensitive: h.sensitive, + baseScore: b.score, + headScore: h.score, + nowFailing: failingIds(h).filter((id) => !baseFailing.has(id)), + drop: b.score - h.score, + }); + } + + rows.sort( + (a, b) => + Number(b.sensitive) - Number(a.sensitive) || + b.drop - a.drop || + a.routePath.localeCompare(b.routePath) + ); + + const removed = base.entries.filter((e) => !headFiles.has(e.fileName)).length; + return { rows, removed }; +} + +function whatChangedSection(head: MapReport, base: MapReport | null): string[] { + const lines = ["**What this PR changed**"]; + + if (!base) { + lines.push("Base comparison unavailable."); + return lines; + } + + const { rows, removed } = changedRows(head, base); + + if (rows.length === 0 && removed === 0) { + lines.push("No entry point this PR touches changed its score."); + return lines; + } + + if (rows.length > 0) { + lines.push(""); + lines.push("| route | base | head | now failing |"); + lines.push("| --- | --- | --- | --- |"); + for (const row of rows.slice(0, MAX_CHANGED_ROWS)) { + lines.push( + `| ${row.routePath} | ${row.baseScore} | ${row.headScore} | ${row.nowFailing.join(", ")} |` + ); + } + if (rows.length > MAX_CHANGED_ROWS) { + lines.push(""); + lines.push(`and ${rows.length - MAX_CHANGED_ROWS} more`); + } + } + + if (removed > 0) { + lines.push(""); + lines.push(`${removed} entries removed`); + } + + return lines; +} + +function fixFirstSection(head: MapReport): string[] { + const lines = ["FIX FIRST"]; + const worst = head.entries + .filter((e) => scoredFailures(e).length > 0 && !contextOnly(e)) + .sort( + (a, b) => + Number(b.sensitive) - Number(a.sensitive) || + a.score - b.score || + a.fileName.localeCompare(b.fileName) + ); + + for (const e of worst.slice(0, 3)) { + const marks = e.sensitive ? " (sensitive)" : ""; + lines.push( + `- ${e.routePath}${marks} - ${scoredFailures(e) + .map((c) => c.id) + .join(", ")}` + ); + } + return lines; +} + +/** + * Pure function, no I/O: `head` and `base` are already-built reports. Matches entries across the + * two by `fileName`, the same identifier `renderJson` carries. + */ +export function renderPrComment(head: MapReport, base: MapReport | null): string { + const lines = [MARKER, "", "## Observability map", "", scoreLine(head, base), ""]; + + lines.push(...whatChangedSection(head, base), ""); + lines.push(...fixFirstSection(head), ""); + + const audit = auditLine(head); + if (audit) lines.push(audit); + const context = contextLine(head); + if (context) lines.push(context); + if (audit || context) lines.push(""); + + lines.push( + "Report only, nothing here gates the merge. The rules and their reasons: " + + "internal-packages/observability-map/README.md." + ); + + const headFailures = head.parseFailures.length; + const baseFailures = base?.parseFailures.length ?? 0; + if (headFailures > 0 || baseFailures > 0) { + const parts: string[] = []; + if (headFailures > 0) parts.push(`${headFailures} at head`); + if (baseFailures > 0) parts.push(`${baseFailures} at base`); + lines.push( + `Warning: parse failures (${parts.join(", ")}) are excluded from the score, shrinking the denominator.` + ); + } + + return lines.join("\n"); +} diff --git a/internal-packages/observability-map/src/report/prCommentCli.ts b/internal-packages/observability-map/src/report/prCommentCli.ts new file mode 100644 index 00000000000..0fa1d7786e7 --- /dev/null +++ b/internal-packages/observability-map/src/report/prCommentCli.ts @@ -0,0 +1,39 @@ +import { readFileSync } from "node:fs"; +import { resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import type { MapReport } from "../score.js"; +import { renderPrComment } from "./prComment.js"; + +/** Where output goes. Injectable so tests can read it without spawning a process. */ +export type Io = { out: (s: string) => void; err: (s: string) => void }; + +const processIo: Io = { + out: (s) => process.stdout.write(s), + err: (s) => process.stderr.write(s), +}; + +/** `-` or a missing second arg means no base: the CI job falls back to this when the base scan + * itself failed, so the comment still renders rather than the job going red. */ +export function main(argv: string[], io: Io = processIo): number { + const args = argv.slice(2); + const headPath = args[0]; + const basePath = args[1]; + + if (!headPath) { + io.err("usage: prCommentCli.ts [base.json|-]\n"); + return 1; + } + + const head = JSON.parse(readFileSync(headPath, "utf8")) as MapReport; + const base: MapReport | null = + !basePath || basePath === "-" ? null : JSON.parse(readFileSync(basePath, "utf8")); + + io.out(renderPrComment(head, base)); + io.out("\n"); + return 0; +} + +// Only when run as a program. Importing the module, which the tests do, must not read a file. +const invokedDirectly = + process.argv[1] !== undefined && resolve(process.argv[1]) === fileURLToPath(import.meta.url); +if (invokedDirectly) process.exitCode = main(process.argv); diff --git a/internal-packages/observability-map/src/report/terminal.ts b/internal-packages/observability-map/src/report/terminal.ts index eaf0bd5ded1..6175a93a88c 100644 --- a/internal-packages/observability-map/src/report/terminal.ts +++ b/internal-packages/observability-map/src/report/terminal.ts @@ -17,7 +17,7 @@ const gauge = (score: number | null) => { * of the fixable, route-specific gaps the list exists to surface. That gap is reported once, as * `AUDIT`, below. */ -const scoredFailures = (e: ScoredEntry) => +export const scoredFailures = (e: ScoredEntry) => e.checks.filter((c) => SCORED_CHECK_IDS.includes(c.id) && c.status === "fail"); /** @@ -27,11 +27,40 @@ const scoredFailures = (e: ScoredEntry) => * An entry that fails something else as well stays in the list with all of its findings, so a * route like `/account/tokens` still shows the request-context gap alongside the rest. */ -const contextOnly = (e: ScoredEntry) => { +export const contextOnly = (e: ScoredEntry) => { const failures = scoredFailures(e); return failures.length === 1 && failures[0]!.id === "request-context"; }; +/** The AUDIT figure, shared with `prComment.ts` so both renderers say the same thing. Null when + * there is nothing to report, i.e. no sensitive mutation exists. */ +export function auditLine(report: MapReport): string | null { + const { sensitiveMutations, withAudit } = report.auditGap; + if (sensitiveMutations === 0) return null; + // The closing sentence is a claim about the codebase, so it is only made when the figure in + // front of it supports it. It was printed unconditionally, including next to a non-zero count. + const gap = + withAudit === 0 + ? " No audit helper exists in the webapp." + : ` ${sensitiveMutations - withAudit} without one.`; + return `AUDIT ${withAudit} of ${sensitiveMutations} sensitive mutations record an actor.${gap}`; +} + +/** The CONTEXT figure, shared with `prComment.ts`. Null when nothing is applicable. */ +export function contextLine(report: MapReport): string | null { + const { applicable, naming } = report.contextGap; + if (applicable === 0) return null; + const collapsed = report.entries.filter(contextOnly); + const sensitive = collapsed.filter((e) => e.sensitive).length; + return ( + `CONTEXT ${naming} of ${applicable} entry points name a tenant on a failure path.` + + (collapsed.length > 0 + ? ` ${collapsed.length} appear${collapsed.length === 1 ? "s" : ""} only here, ` + + `${sensitive} of them sensitive, in the JSON rather than the list below.` + : "") + ); +} + export function renderTerminal(report: MapReport): string { const lines: string[] = []; @@ -52,32 +81,16 @@ export function renderTerminal(report: MapReport): string { }/${report.sensitiveCohort.n} entry points` ); - const { sensitiveMutations, withAudit } = report.auditGap; - if (sensitiveMutations > 0) { + const audit = auditLine(report); + if (audit) { lines.push(""); - // The closing sentence is a claim about the codebase, so it is only made when the figure in - // front of it supports it. It was printed unconditionally, including next to a non-zero count. - const gap = - withAudit === 0 - ? " No audit helper exists in the webapp." - : ` ${sensitiveMutations - withAudit} without one.`; - lines.push( - `AUDIT ${withAudit} of ${sensitiveMutations} sensitive mutations record an actor.${gap}` - ); + lines.push(audit); } - const { applicable, naming } = report.contextGap; - if (applicable > 0) { - const collapsed = report.entries.filter(contextOnly); - const sensitive = collapsed.filter((e) => e.sensitive).length; + const context = contextLine(report); + if (context) { lines.push(""); - lines.push( - `CONTEXT ${naming} of ${applicable} entry points name a tenant on a failure path.` + - (collapsed.length > 0 - ? ` ${collapsed.length} appear${collapsed.length === 1 ? "s" : ""} only here, ` + - `${sensitive} of them sensitive, in the JSON rather than the list below.` - : "") - ); + lines.push(context); } if (report.suppressions.checks > 0) { diff --git a/internal-packages/observability-map/test/cli.test.ts b/internal-packages/observability-map/test/cli.test.ts index f97aa2269ed..2fd28955d85 100644 --- a/internal-packages/observability-map/test/cli.test.ts +++ b/internal-packages/observability-map/test/cli.test.ts @@ -1,5 +1,6 @@ -import { existsSync, rmSync } from "node:fs"; -import { resolve } from "node:path"; +import { existsSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; import { main, type Io } from "../src/cli.js"; const REPORT_FILE = resolve(__dirname, "../../../observability-map.json"); @@ -69,3 +70,31 @@ describe("map", () => { expect(existsSync(REPORT_FILE)).toBe(false); }); }); + +describe("map --routes=", () => { + it("scans the directory it names instead of the repo's routes tree", () => { + const dir = mkdtempSync(join(tmpdir(), "obs-map-routes-")); + writeFileSync( + join(dir, "resources.only.ts"), + `export const loader = () => new Response("ok");` + ); + + const r = run("--routes=" + dir, "--json", "--no-write"); + + expect(r.code).toBe(0); + const parsed = JSON.parse(r.out); + expect(parsed.entries).toHaveLength(1); + expect(parsed.entries[0].fileName).toBe("resources.only.ts"); + + rmSync(dir, { recursive: true }); + }); + + it("exits 1 with a message when the directory does not exist", () => { + const dir = join(tmpdir(), "obs-map-routes-does-not-exist"); + const r = run("--routes=" + dir); + + expect(r.code).toBe(1); + expect(r.err).toContain("not a readable directory"); + expect(r.out).toBe(""); + }); +}); diff --git a/internal-packages/observability-map/test/prComment.test.ts b/internal-packages/observability-map/test/prComment.test.ts new file mode 100644 index 00000000000..9a115fc79d7 --- /dev/null +++ b/internal-packages/observability-map/test/prComment.test.ts @@ -0,0 +1,119 @@ +import { renderPrComment } from "../src/report/prComment.js"; +import { buildReport } from "../src/score.js"; +import { scanFile } from "../src/scan.js"; + +const cleanSource = ` + import { requireUserId } from "~/services/session.server"; + import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function action({ request }) { + const userId = await requireUserId(request); + try { return await prisma.token.create({ data: { userId } }); } + catch (error) { logger.error("token create failed", { userId, error }); throw error; } + }`; + +const brokenSource = ` + import { prisma } from "~/db.server"; + export async function action() { + try { return await prisma.token.create({ data: {} }); } catch (e) { return null; } + }`; + +describe("renderPrComment", () => { + it("puts the upsert marker on the first line, always", () => { + const head = buildReport([scanFile("api.v1.a.ts", brokenSource)!], []); + expect(renderPrComment(head, null).split("\n")[0]).toBe(""); + expect(renderPrComment(head, head).split("\n")[0]).toBe(""); + }); + + it("says the comparison is unavailable when base is null", () => { + const head = buildReport([scanFile("api.v1.a.ts", brokenSource)!], []); + const out = renderPrComment(head, null); + expect(out).toContain("Base comparison unavailable."); + expect(out).not.toContain("no change"); + }); + + it("reports a score drop and the newly failing checks", () => { + const head = buildReport([scanFile("api.v1.auth.tokens.ts", brokenSource)!], []); + const base = buildReport([scanFile("api.v1.auth.tokens.ts", cleanSource)!], []); + const out = renderPrComment(head, base); + + expect(out).toMatch(/\(base \d+, down \d+\)/); + expect(out).toContain("| /api/v1/auth/tokens |"); + // request-context and error-classification regress; auth-boundary is not applicable here + // (no sensitivity signal on this route), so it must not show up as newly failing. + expect(out).toMatch(/\| \/api\/v1\/auth\/tokens \| \d+ \| \d+ \|[^|]*error-classification/); + }); + + it("reports a score improvement the other way round", () => { + const head = buildReport([scanFile("api.v1.auth.tokens.ts", cleanSource)!], []); + const base = buildReport([scanFile("api.v1.auth.tokens.ts", brokenSource)!], []); + const out = renderPrComment(head, base); + expect(out).toMatch(/\(base \d+, up \d+\)/); + }); + + it("says nothing changed when every score matches", () => { + const head = buildReport([scanFile("api.v1.auth.tokens.ts", cleanSource)!], []); + const base = buildReport([scanFile("api.v1.auth.tokens.ts", cleanSource)!], []); + const out = renderPrComment(head, base); + expect(out).toContain("No entry point this PR touches changed its score."); + expect(out).toContain("(base 100, no change)"); + }); + + it("shows a new entry with its base column as 'new' and lists its failing checks", () => { + const head = buildReport( + [scanFile("api.v1.auth.tokens.ts", cleanSource)!, scanFile("api.v1.new.ts", brokenSource)!], + [] + ); + const base = buildReport([scanFile("api.v1.auth.tokens.ts", cleanSource)!], []); + const out = renderPrComment(head, base); + + expect(out).toMatch(/\| \/api\/v1\/new \| new \| \d+ \|/); + }); + + it("reports a removed entry as a count line, not a row", () => { + const head = buildReport([scanFile("api.v1.auth.tokens.ts", cleanSource)!], []); + const base = buildReport( + [scanFile("api.v1.auth.tokens.ts", cleanSource)!, scanFile("api.v1.gone.ts", brokenSource)!], + [] + ); + const out = renderPrComment(head, base); + + expect(out).toContain("1 entries removed"); + expect(out).not.toContain("/api/v1/gone"); + }); + + it("caps the changed-entries table at 15 rows and says how many more", () => { + const headEntries = []; + const baseEntries = []; + for (let i = 0; i < 20; i++) { + headEntries.push(scanFile(`api.v1.route${i}.ts`, brokenSource)!); + baseEntries.push(scanFile(`api.v1.route${i}.ts`, cleanSource)!); + } + const head = buildReport(headEntries, []); + const base = buildReport(baseEntries, []); + const out = renderPrComment(head, base); + + const rows = out.split("\n").filter((l) => l.startsWith("| /api/v1/route")); + expect(rows).toHaveLength(15); + expect(out).toContain("and 5 more"); + }); + + it("warns about parse failures in either report, since they shrink the denominator", () => { + const head = buildReport([scanFile("api.v1.a.ts", cleanSource)!], ["broken-head.ts"]); + const base = buildReport([scanFile("api.v1.a.ts", cleanSource)!], ["broken-base.ts"]); + const out = renderPrComment(head, base); + expect(out).toMatch(/Warning: parse failures \(1 at head, 1 at base\)/); + }); + + it("does not warn about parse failures when there are none", () => { + const head = buildReport([scanFile("api.v1.a.ts", cleanSource)!], []); + expect(renderPrComment(head, null)).not.toContain("Warning: parse failures"); + }); + + it("footer names the report-only rule and the readme", () => { + const head = buildReport([scanFile("api.v1.a.ts", cleanSource)!], []); + const out = renderPrComment(head, null); + expect(out).toContain("Report only, nothing here gates the merge."); + expect(out).toContain("internal-packages/observability-map/README.md"); + }); +}); diff --git a/internal-packages/observability-map/test/prCommentCli.test.ts b/internal-packages/observability-map/test/prCommentCli.test.ts new file mode 100644 index 00000000000..8e687bbf936 --- /dev/null +++ b/internal-packages/observability-map/test/prCommentCli.test.ts @@ -0,0 +1,57 @@ +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { main, type Io } from "../src/report/prCommentCli.js"; +import { buildReport } from "../src/score.js"; +import { renderJson } from "../src/report/json.js"; +import { scanFile } from "../src/scan.js"; + +const capture = () => { + const out: string[] = []; + const err: string[] = []; + const io: Io = { out: (s) => out.push(s), err: (s) => err.push(s) }; + return { io, out: () => out.join(""), err: () => err.join("") }; +}; + +const run = (...args: string[]) => { + const c = capture(); + const code = main(["node", "prCommentCli.js", ...args], c.io); + return { code, out: c.out(), err: c.err() }; +}; + +describe("prCommentCli", () => { + const dir = mkdtempSync(join(tmpdir(), "obs-map-cli-")); + const headPath = join(dir, "head.json"); + const basePath = join(dir, "base.json"); + + const source = `export const loader = () => new Response("ok");`; + const report = buildReport([scanFile("resources.a.ts", source)!], []); + writeFileSync(headPath, renderJson(report)); + writeFileSync(basePath, renderJson(report)); + + afterAll(() => rmSync(dir, { recursive: true })); + + it("renders the markdown comment for head and base file arguments", () => { + const r = run(headPath, basePath); + expect(r.code).toBe(0); + expect(r.out.split("\n")[0]).toBe(""); + }); + + it("treats '-' as no base", () => { + const r = run(headPath, "-"); + expect(r.code).toBe(0); + expect(r.out).toContain("Base comparison unavailable."); + }); + + it("treats a missing second argument as no base", () => { + const r = run(headPath); + expect(r.code).toBe(0); + expect(r.out).toContain("Base comparison unavailable."); + }); + + it("exits 1 with a usage message when head.json is missing", () => { + const r = run(); + expect(r.code).toBe(1); + expect(r.err).toContain("usage:"); + }); +}); From 902c0136c20debae0d70fa97584815a64aba0fcc Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Sat, 1 Aug 2026 14:24:56 +0100 Subject: [PATCH 036/106] ci(observability-map): scan head vs merge base and upsert a sticky PR comment --- .github/workflows/observability-map.yml | 78 +++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 .github/workflows/observability-map.yml diff --git a/.github/workflows/observability-map.yml b/.github/workflows/observability-map.yml new file mode 100644 index 00000000000..460d0d5f884 --- /dev/null +++ b/.github/workflows/observability-map.yml @@ -0,0 +1,78 @@ +name: 🗺️ Observability Map + +on: + pull_request: + types: [opened, synchronize, reopened] + paths: + - "apps/webapp/app/routes/**" + - "internal-packages/observability-map/**" + +concurrency: + group: observability-map-${{ github.event.pull_request.number }} + cancel-in-progress: true + +permissions: + contents: read + pull-requests: write + +jobs: + report: + runs-on: warp-ubuntu-latest-x64-4x + # Fork PRs get a read-only token, so the comment cannot post. Skipping the job beats a red x. + if: github.event.pull_request.head.repo.full_name == github.repository + steps: + - name: ⬇️ Checkout repo + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + fetch-depth: 0 + persist-credentials: false + + - name: ⎔ Setup pnpm + uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0 + with: + version: 10.33.2 + + - name: ⎔ Setup node + uses: WarpBuilds/setup-node@bc639b444d583175926b588962199c247d23e8d3 # v6 + with: + node-version: 24.18.0 + cache: "pnpm" + + - name: 📥 Download deps + run: pnpm install --frozen-lockfile + + - name: 🔎 Scan head + run: | + pnpm --filter @internal/observability-map exec tsx src/cli.ts --json --no-write > /tmp/head.json + + - name: 🔎 Scan base with the head's scanner + run: | + if git worktree add /tmp/base-tree ${{ github.event.pull_request.base.sha }} \ + && pnpm --filter @internal/observability-map exec tsx src/cli.ts --json --no-write \ + --routes=/tmp/base-tree/apps/webapp/app/routes > /tmp/base.json; then + : + else + echo "-" > /tmp/base.json || true + echo "base scan failed or the worktree could not be added; falling back to no base" >&2 + fi + + - name: 📝 Render comment + run: | + if [ "$(cat /tmp/base.json)" = "-" ]; then + pnpm --filter @internal/observability-map exec tsx src/report/prCommentCli.ts /tmp/head.json - > /tmp/comment.md + else + pnpm --filter @internal/observability-map exec tsx src/report/prCommentCli.ts /tmp/head.json /tmp/base.json > /tmp/comment.md + fi + + - name: 💬 Upsert PR comment + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} + run: | + existing=$(gh api "repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/comments" --paginate \ + --jq '[.[] | select(.body | startswith(""))][0].id // empty') + if [ -n "$existing" ]; then + gh api -X PATCH "repos/${GITHUB_REPOSITORY}/issues/comments/${existing}" -F body=@/tmp/comment.md + else + gh api "repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/comments" -F body=@/tmp/comment.md + fi From df0333eb581ab6192f4f6371aae0562129699419 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Sat, 1 Aug 2026 14:36:57 +0100 Subject: [PATCH 037/106] fix(observability-map): stop audit-trail leaking into the PR-comment fix list and never fail the CI job on a render or upsert error --- .github/workflows/observability-map.yml | 7 ++ .../observability-map/src/report/prComment.ts | 7 +- .../src/report/prCommentCli.ts | 28 +++++++- .../observability-map/test/prComment.test.ts | 68 +++++++++++++++++++ .../test/prCommentCli.test.ts | 26 +++++++ 5 files changed, 132 insertions(+), 4 deletions(-) diff --git a/.github/workflows/observability-map.yml b/.github/workflows/observability-map.yml index 460d0d5f884..f63409adb96 100644 --- a/.github/workflows/observability-map.yml +++ b/.github/workflows/observability-map.yml @@ -56,7 +56,11 @@ jobs: echo "base scan failed or the worktree could not be added; falling back to no base" >&2 fi + # continue-on-error: this job must never block a PR. A malformed head.json or a rendering + # bug would otherwise turn the job red the same way a base-scan failure would not, since that + # step already falls back to "-" instead of failing. - name: 📝 Render comment + continue-on-error: true run: | if [ "$(cat /tmp/base.json)" = "-" ]; then pnpm --filter @internal/observability-map exec tsx src/report/prCommentCli.ts /tmp/head.json - > /tmp/comment.md @@ -64,7 +68,10 @@ jobs: pnpm --filter @internal/observability-map exec tsx src/report/prCommentCli.ts /tmp/head.json /tmp/base.json > /tmp/comment.md fi + # continue-on-error for the same reason: a transient gh api failure (rate limit, network) + # must not fail the job either. Worst case, the PR gets no comment this run. - name: 💬 Upsert PR comment + continue-on-error: true env: GH_TOKEN: ${{ github.token }} PR_NUMBER: ${{ github.event.pull_request.number }} diff --git a/internal-packages/observability-map/src/report/prComment.ts b/internal-packages/observability-map/src/report/prComment.ts index ed4480959a5..8e857b8c6d9 100644 --- a/internal-packages/observability-map/src/report/prComment.ts +++ b/internal-packages/observability-map/src/report/prComment.ts @@ -1,4 +1,5 @@ import type { MapReport, ScoredEntry } from "../score.js"; +import { SCORED_CHECK_IDS } from "../checks/index.js"; import { auditLine, contextLine, contextOnly, scoredFailures } from "./terminal.js"; /** First line of every comment this job posts, so the upsert step can find its own comment again. */ @@ -6,7 +7,11 @@ export const MARKER = ""; const MAX_CHANGED_ROWS = 15; -const failingIds = (e: ScoredEntry) => e.checks.filter((c) => c.status === "fail").map((c) => c.id); +// Scored checks only, same exclusion terminal.ts's scoredFailures makes: audit-trail fails almost +// every sensitive mutation today, so listing it per route would nag with something unfixable +// instead of surfacing the route-specific gaps this column exists for. +const failingIds = (e: ScoredEntry) => + e.checks.filter((c) => SCORED_CHECK_IDS.includes(c.id) && c.status === "fail").map((c) => c.id); function scoreLine(head: MapReport, base: MapReport | null): string { const headline = diff --git a/internal-packages/observability-map/src/report/prCommentCli.ts b/internal-packages/observability-map/src/report/prCommentCli.ts index 0fa1d7786e7..0283eb96591 100644 --- a/internal-packages/observability-map/src/report/prCommentCli.ts +++ b/internal-packages/observability-map/src/report/prCommentCli.ts @@ -12,6 +12,22 @@ const processIo: Io = { err: (s) => process.stderr.write(s), }; +/** Reads and parses one report file, raising a message naming the file rather than letting an + * unreadable path or malformed JSON surface as a stack trace. */ +function readReport(path: string, label: string): MapReport { + let raw: string; + try { + raw = readFileSync(path, "utf8"); + } catch { + throw new Error(`cannot read ${label}: ${path}`); + } + try { + return JSON.parse(raw) as MapReport; + } catch { + throw new Error(`${label} is not valid JSON: ${path}`); + } +} + /** `-` or a missing second arg means no base: the CI job falls back to this when the base scan * itself failed, so the comment still renders rather than the job going red. */ export function main(argv: string[], io: Io = processIo): number { @@ -24,9 +40,15 @@ export function main(argv: string[], io: Io = processIo): number { return 1; } - const head = JSON.parse(readFileSync(headPath, "utf8")) as MapReport; - const base: MapReport | null = - !basePath || basePath === "-" ? null : JSON.parse(readFileSync(basePath, "utf8")); + let head: MapReport; + let base: MapReport | null; + try { + head = readReport(headPath, "head report"); + base = !basePath || basePath === "-" ? null : readReport(basePath, "base report"); + } catch (error) { + io.err(`${error instanceof Error ? error.message : String(error)}\n`); + return 1; + } io.out(renderPrComment(head, base)); io.out("\n"); diff --git a/internal-packages/observability-map/test/prComment.test.ts b/internal-packages/observability-map/test/prComment.test.ts index 9a115fc79d7..7f651d57dc2 100644 --- a/internal-packages/observability-map/test/prComment.test.ts +++ b/internal-packages/observability-map/test/prComment.test.ts @@ -70,6 +70,74 @@ describe("renderPrComment", () => { expect(out).toMatch(/\| \/api\/v1\/new \| new \| \d+ \|/); }); + // Mirrors the guard report.test.ts has for the terminal renderer: audit-trail fails almost + // every sensitive mutation today (no audit helper exists), so it is a headline figure, not a + // per-route nag. A regression here previously let it leak into the "now failing" column. + it("does not list audit-trail among a new sensitive entry's failing checks", () => { + const sensitiveMutation = scanFile( + "api.v1.envvars.ts", + `import { prisma } from "~/db.server"; + export async function action() { + try { + return await prisma.envVar.update({ where: {}, data: {} }); + } catch (e) { + return null; + } + }` + )!; + const head = buildReport([sensitiveMutation], []); + const base = buildReport([], []); + const out = renderPrComment(head, base); + + const row = out.split("\n").find((l) => l.startsWith("| /api/v1/envvars |"))!; + expect(row).toBeDefined(); + expect(row).toContain("new"); + expect(row).not.toContain("audit-trail"); + expect(row).toMatch(/error-classification|auth-boundary|request-context/); + }); + + it("sorts a sensitive entry with a small drop above a non-sensitive entry with a large drop", () => { + const sensitiveSmallDropBase = scanFile("api.v1.auth.tokens.ts", cleanSource)!; + const sensitiveSmallDropHead = scanFile( + "api.v1.auth.tokens.ts", + `import { requireUserId } from "~/services/session.server"; + import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function action({ request }) { + const userId = await requireUserId(request); + try { return await prisma.token.create({ data: { userId } }); } + catch (error) { logger.error("token create failed", { error }); throw error; } + }` + )!; + + const notSensitiveLargeDropBase = scanFile( + "resources.busy.ts", + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function loader({ params }) { + try { return await prisma.thing.findMany(); } + catch (error) { logger.error("failed", { environmentId: params.envId, error }); throw error; } + }` + )!; + const notSensitiveLargeDropHead = scanFile( + "resources.busy.ts", + `import { prisma } from "~/db.server"; + export async function loader() { + try { return await prisma.thing.findMany(); } catch (e) { return null; } + }` + )!; + + const head = buildReport([sensitiveSmallDropHead, notSensitiveLargeDropHead], []); + const base = buildReport([sensitiveSmallDropBase, notSensitiveLargeDropBase], []); + const out = renderPrComment(head, base); + + const sensitiveIndex = out.indexOf("/api/v1/auth/tokens"); + const notSensitiveIndex = out.indexOf("/resources/busy"); + expect(sensitiveIndex).toBeGreaterThan(-1); + expect(notSensitiveIndex).toBeGreaterThan(-1); + expect(sensitiveIndex).toBeLessThan(notSensitiveIndex); + }); + it("reports a removed entry as a count line, not a row", () => { const head = buildReport([scanFile("api.v1.auth.tokens.ts", cleanSource)!], []); const base = buildReport( diff --git a/internal-packages/observability-map/test/prCommentCli.test.ts b/internal-packages/observability-map/test/prCommentCli.test.ts index 8e687bbf936..b464725ea77 100644 --- a/internal-packages/observability-map/test/prCommentCli.test.ts +++ b/internal-packages/observability-map/test/prCommentCli.test.ts @@ -54,4 +54,30 @@ describe("prCommentCli", () => { expect(r.code).toBe(1); expect(r.err).toContain("usage:"); }); + + it("exits 1 with a one-line message, not a stack trace, when head.json does not exist", () => { + const r = run(join(dir, "does-not-exist.json")); + expect(r.code).toBe(1); + expect(r.err.split("\n").filter(Boolean)).toHaveLength(1); + expect(r.err).toContain("cannot read head report"); + expect(r.err).not.toContain(" at "); + }); + + it("exits 1 with a one-line message, not a stack trace, when head.json is malformed", () => { + const malformedPath = join(dir, "malformed.json"); + writeFileSync(malformedPath, "{ not json"); + const r = run(malformedPath); + expect(r.code).toBe(1); + expect(r.err.split("\n").filter(Boolean)).toHaveLength(1); + expect(r.err).toContain("head report is not valid JSON"); + expect(r.err).not.toContain(" at "); + }); + + it("exits 1 with a one-line message when base.json is malformed", () => { + const malformedBasePath = join(dir, "malformed-base.json"); + writeFileSync(malformedBasePath, "not json at all"); + const r = run(headPath, malformedBasePath); + expect(r.code).toBe(1); + expect(r.err).toContain("base report is not valid JSON"); + }); }); From 960a568cdbaa7ddc17561ee1fd5a852cd49aaa91 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Sat, 1 Aug 2026 14:44:33 +0100 Subject: [PATCH 038/106] fix(observability-map): make the CONTEXT line true in both renderers --- internal-packages/observability-map/src/report/terminal.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal-packages/observability-map/src/report/terminal.ts b/internal-packages/observability-map/src/report/terminal.ts index 6175a93a88c..05cc7d6a248 100644 --- a/internal-packages/observability-map/src/report/terminal.ts +++ b/internal-packages/observability-map/src/report/terminal.ts @@ -56,7 +56,7 @@ export function contextLine(report: MapReport): string | null { `CONTEXT ${naming} of ${applicable} entry points name a tenant on a failure path.` + (collapsed.length > 0 ? ` ${collapsed.length} appear${collapsed.length === 1 ? "s" : ""} only here, ` + - `${sensitive} of them sensitive, in the JSON rather than the list below.` + `${sensitive} of them sensitive, in the JSON rather than the fix list.` : "") ); } From 08f6be57eb3fb4fe3aa64023c372a3c5362a3252 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Sat, 1 Aug 2026 16:07:03 +0100 Subject: [PATCH 039/106] fix(observability-map): derive measured from pre-suppression applicability score.ts computed measured from visible (post-suppression) applicability, so suppressing an entry's only applicable check set measured: false and dropped it, still failing, out of the global mean, every family mean and the sensitive cohort. Tree-wide, prepending a zero-behaviour suppression comment moved the global from 17 to 33 and measured from 412 to 176. measured now reads scored (pre-suppression) applicability instead, so a fully-suppressed entry stays in the denominator at its capped score. unmeasured still means no applicable check at all. The per-entry Math.min cap was already correct and is untouched. --- .../observability-map/src/score.ts | 12 +++- .../test/integration.test.ts | 23 +++++++- .../observability-map/test/score.test.ts | 55 ++++++++++++++++++- 3 files changed, 84 insertions(+), 6 deletions(-) diff --git a/internal-packages/observability-map/src/score.ts b/internal-packages/observability-map/src/score.ts index 1203f730158..a39a1ab7c82 100644 --- a/internal-packages/observability-map/src/score.ts +++ b/internal-packages/observability-map/src/score.ts @@ -11,7 +11,9 @@ export type ScoredEntry = { sensitive: boolean; checks: CheckResult[]; /** - * Whether at least one scored check (`SCORED_CHECK_IDS`, so never `audit-trail`) was applicable. + * Whether at least one scored check (`SCORED_CHECK_IDS`, so never `audit-trail`) was applicable + * before suppression. A fully-suppressed entry stays measured, at its capped score, so a + * suppression cannot buy removal from every mean by way of removal from this one. * `false` means nothing was measured here: the 100 in `score` is a vacuous default, not a * finding, and `buildReport` excludes an unmeasured entry from every mean it computes so that * default cannot inflate a figure nobody checked. @@ -65,7 +67,11 @@ export function scoreEntry(ep: EntryPoint): ScoredEntry { }; const visible = scored.filter((c) => !suppressed.has(c.id)); - const applicable = visible.filter((c) => c.status !== "not-applicable"); + // Pre-suppression: an entry whose only applicable check gets suppressed still had something to + // measure, and must stay in the denominator at its capped score rather than vanish as if nothing + // ever applied. `unmeasured` is reserved for entries with no applicable check at all, suppression + // or no suppression. + const scoredApplicable = scored.filter((c) => c.status !== "not-applicable"); return { fileName: ep.fileName, @@ -79,7 +85,7 @@ export function scoreEntry(ep: EntryPoint): ScoredEntry { // shrinks the denominator and the ratio climbs, which is how 33 became 50 became 100: the // suppression comment laundered the finding into a point. What a suppression buys is removal // from the worklist, with a reason on the record. It cannot buy a better number. - measured: applicable.length > 0, + measured: scoredApplicable.length > 0, score: Math.min(ratio(visible), ratio(scored)), }; } diff --git a/internal-packages/observability-map/test/integration.test.ts b/internal-packages/observability-map/test/integration.test.ts index f628c5a1dfd..032f960fb5e 100644 --- a/internal-packages/observability-map/test/integration.test.ts +++ b/internal-packages/observability-map/test/integration.test.ts @@ -1,7 +1,8 @@ import { existsSync, readdirSync } from "node:fs"; import { join, resolve } from "node:path"; -import { scanDirectory } from "../src/scan.js"; +import { scanDirectory, scanFile } from "../src/scan.js"; import { buildReport } from "../src/score.js"; +import { SCORED_CHECK_IDS } from "../src/checks/index.js"; const ROUTES = resolve(__dirname, "../../../apps/webapp/app/routes"); @@ -44,4 +45,24 @@ describe("scanning the real webapp routes", () => { expect(report.global).toBeLessThanOrEqual(100); expect(Object.keys(report.byFamily).length).toBeGreaterThan(1); }); + + // A1, exhaustive: every scored check suppressed on every real route, zero behavioural change. + // The old measured-from-visible logic took this global from 17 to 33 and measured from 412 to + // 176, because every entry whose only applicable checks were suppressed dropped out of the + // mean. Measured must not move: every entry point that had something applicable still does. + it("suppressing every scored check on every real route does not raise the global", () => { + if (!existsSync(ROUTES)) return; + const { entryPoints, parseFailures } = scanDirectory(ROUTES); + const before = buildReport(entryPoints, parseFailures); + + const directive = SCORED_CHECK_IDS.map( + (id) => `// obs-map-disable ${id} -- exhaustive sweep\n` + ).join(""); + const suppressed = entryPoints.map((ep) => scanFile(ep.fileName, directive + ep.source)!); + const after = buildReport(suppressed, parseFailures); + + expect(after.measured).toBe(before.measured); + expect(after.unmeasured).toBe(before.unmeasured); + expect(after.global).not.toBeGreaterThan(before.global!); + }); }); diff --git a/internal-packages/observability-map/test/score.test.ts b/internal-packages/observability-map/test/score.test.ts index 37b8eafec47..ccbc24b973e 100644 --- a/internal-packages/observability-map/test/score.test.ts +++ b/internal-packages/observability-map/test/score.test.ts @@ -116,7 +116,11 @@ ${BUSY_AND_FAILING}` expect(suppressed.suppressed).toEqual(["error-classification", "request-context"]); }); - it("does not let an entry whose every scored check is suppressed read as measured", () => { + // A1. `measured` reads pre-suppression applicability. Before the fix it read `visible` + // (post-suppression) applicability, so suppressing an entry's only applicable checks flipped + // `measured` to false and dropped the entry, still failing, out of the global mean, every family + // mean and the sensitive cohort. `unmeasured` stays for entries nothing was ever applicable to. + it("still reads as measured when every applicable scored check is suppressed", () => { const suppressed = scoreEntry( scanFile( "api.v1.b.ts", @@ -125,7 +129,8 @@ ${BUSY_AND_FAILING}` ${BUSY_AND_FAILING}` )! ); - expect(suppressed.measured).toBe(false); + expect(suppressed.measured).toBe(true); + expect(suppressed.score).toBe(0); }); it("marks an entry point with nothing applicable as unmeasured, scored 100", () => { @@ -221,6 +226,52 @@ ${BUSY_AND_FAILING}` }); }); +// A1. `measured` used to read post-suppression (`visible`) applicability, so suppressing an +// entry's only applicable checks removed it from the global mean, every family mean and the +// sensitive cohort, rather than keeping it at its capped score. That is how a suppression with no +// behavioural change moved the global from 17 to 33 tree-wide. +describe("A1: a suppression cannot raise the global", () => { + it("scoring 100 and 0, suppressing every check on the failing entry leaves the global at 50", () => { + const passing = scanFile("api.v1.auth.tokens.ts", CLEAN)!; + const failing = scanFile("api.v1.busy.ts", BUSY_AND_FAILING)!; + + const before = buildReport([passing, failing], []); + expect(before.global).toBe(50); + + const failingSuppressed = scanFile( + "api.v1.busy.ts", + `// obs-map-disable error-classification -- silence +// obs-map-disable request-context -- silence +${BUSY_AND_FAILING}` + )!; + const after = buildReport([passing, failingSuppressed], []); + + expect(after.global).toBe(50); + expect(after.measured).toBe(2); + }); + + it("suppressing a passing check does not raise the score either", () => { + // error-classification branches on the error (pass); request-context never names a tenant + // (fail). Not sensitive, so auth-boundary sits out. + const source = `import { prisma } from "~/db.server"; +export async function loader() { + try { return await prisma.thing.findMany(); } + catch (e) { if (e instanceof Error) return null; throw e; } +}`; + const plain = scoreEntry(scanFile("api.v1.mixed.ts", source)!); + expect(plain.checks.find((c) => c.id === "error-classification")!.status).toBe("pass"); + + const suppressed = scoreEntry( + scanFile( + "api.v1.mixed.ts", + `// obs-map-disable error-classification -- silence a pass +${source}` + )! + ); + expect(suppressed.score).toBeLessThanOrEqual(plain.score); + }); +}); + /** * The invariant, both ways round. The README states one direction, removing error handling must * lower the score, and that alone could not see the free-points path: adding a catch that only From e88f04d01bae7655dabb287942cf07ab634536d4 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Sat, 1 Aug 2026 16:11:08 +0100 Subject: [PATCH 040/106] fix(observability-map): read suppression comments from real comment ranges commentPart used line.indexOf("//") and indexOf("/*") against the raw text, so a string or template literal that merely quoted the directive granted a suppression nobody wrote and silenced a real check: "see // obs-map-disable error-classification -- because reasons" and a URL string containing "obs-map-disable" both suppressed for real. Replaced the hand-rolled text scan with the TypeScript scanner's own token stream. A string or template literal is one token there, consumed in a single step, and the scanner never emits comment trivia for what is inside it, so there is no substring left to fool. Line comments, block comments and jsdoc lines are read the same as before; no textual path remains. Also corrects the docstring's claimed failure direction: the old comment said misreading a string cost a suppression that was never written, when the real effect was the opposite, granting one nobody wrote. --- .../observability-map/src/suppression.ts | 55 ++++++++++++------- .../test/suppression.test.ts | 29 ++++++++++ 2 files changed, 64 insertions(+), 20 deletions(-) diff --git a/internal-packages/observability-map/src/suppression.ts b/internal-packages/observability-map/src/suppression.ts index cdf2957d38a..6012ee09d4e 100644 --- a/internal-packages/observability-map/src/suppression.ts +++ b/internal-packages/observability-map/src/suppression.ts @@ -1,7 +1,8 @@ +import ts from "typescript"; + /** * The directive, and the reason that must follow it. The reason runs to the end of the line: `.` * does not match a newline, so a suppression on one line cannot pick up a reason from the next. - * A trailing block-comment terminator is trimmed off so it does not end up inside the reason. * * It was `obs-map-disable-next-line`, which was a lie: a check applies to a whole entry point, so * the directive did too, and one on the last line of a file switched a check off for everything @@ -14,37 +15,51 @@ const PATTERN = /obs-map-disable\s+([a-z-]+)\s+--\s+(.+)/; /** - * The comment part of a line, or null if there is none. + * Every physical line of genuine comment content in the source, line comments and block comments + * alike, one entry per line, with the `//`, `/*`, `*​/` and a jsdoc `*` prefix stripped. * - * Line-scoped and comment-only, because the directive is a comment directive. Matching the raw - * source meant a string literal that merely quotes the directive, in a test fixture or an error - * message, silently switched a real check off. Handles line comments, block comments and the - * leading star of a jsdoc block; a line-comment marker inside a string on the same line can still - * be misread, which costs a suppression that was never written rather than hiding one that was. + * Read from the TypeScript scanner's own token stream rather than `indexOf("//")` against the raw + * text. The old text-matching read the directive out of a string or template literal that merely + * quoted it, so `"see // obs-map-disable auth-boundary -- nope"` granted a suppression nobody + * wrote, silencing a real check. The scanner already knows the difference: a string or template + * literal is one token, consumed in a single step, and never yields comment trivia for what is + * inside it, so there is no substring rule left to fool. */ -function commentPart(line: string): string | null { - const slashes = line.indexOf("//"); - if (slashes !== -1) return line.slice(slashes + 2); +function commentLines(source: string): string[] { + const scanner = ts.createScanner( + ts.ScriptTarget.Latest, + /* skipTrivia */ false, + ts.LanguageVariant.Standard, + source + ); + const lines: string[] = []; - const block = line.indexOf("/*"); - if (block !== -1) return line.slice(block + 2).replace(/\*\/\s*$/, ""); + for (let kind = scanner.scan(); kind !== ts.SyntaxKind.EndOfFileToken; kind = scanner.scan()) { + if (kind === ts.SyntaxKind.SingleLineCommentTrivia) { + lines.push(scanner.getTokenText().slice(2)); + continue; + } + if (kind !== ts.SyntaxKind.MultiLineCommentTrivia) continue; - const trimmed = line.trimStart(); - if (trimmed.startsWith("*")) return trimmed.slice(1); + const text = scanner.getTokenText(); + const body = text.slice(2, text.length - 2); // drop the leading /* and the closing */ + for (const rawLine of body.split("\n")) { + const trimmed = rawLine.trimStart(); + lines.push(trimmed.startsWith("*") ? trimmed.slice(1) : rawLine); + } + } - return null; + return lines; } /** Check id to reason. A suppression without a reason, or outside a comment, is ignored. */ export function suppressedChecks(source: string): Map { const out = new Map(); - for (const line of source.split("\n")) { - const comment = commentPart(line); - if (comment === null) continue; - const match = PATTERN.exec(comment); + for (const line of commentLines(source)) { + const match = PATTERN.exec(line); if (!match) continue; const [, id, reason] = match; - const trimmedReason = reason?.replace(/\*\/\s*$/, "").trim(); + const trimmedReason = reason?.trim(); if (id && trimmedReason && trimmedReason.length > 0) out.set(id, trimmedReason); } return out; diff --git a/internal-packages/observability-map/test/suppression.test.ts b/internal-packages/observability-map/test/suppression.test.ts index 8a01a5953c1..3259278a594 100644 --- a/internal-packages/observability-map/test/suppression.test.ts +++ b/internal-packages/observability-map/test/suppression.test.ts @@ -89,4 +89,33 @@ describe("suppressedChecks", () => { ); expect(m.size).toBe(0); }); + + // A2. `indexOf("//")` against the raw text found the marker inside a string literal too, so a + // string that merely quotes the directive granted a suppression nobody wrote and silenced a real + // check. Reading genuine comment ranges from the TypeScript scanner closes both shapes: the + // scanner consumes a string or template literal as one token and never emits comment trivia for + // what is inside it. + it("does not suppress from a directive quoted inside a string literal", () => { + const m = suppressedChecks( + `const msg = "see // obs-map-disable error-classification -- because reasons"; + export async function loader() { return 1; }` + ); + expect(m.size).toBe(0); + }); + + it("does not suppress from a directive quoted inside a string with no real comment marker", () => { + const m = suppressedChecks( + `const u = "https://example.com obs-map-disable auth-boundary -- nope"; + export async function loader() { return 1; }` + ); + expect(m.size).toBe(0); + }); + + it("does not suppress from a directive inside a template literal", () => { + const m = suppressedChecks( + "const msg = `see // obs-map-disable error-classification -- template literal`;\n" + + "export async function loader() { return 1; }" + ); + expect(m.size).toBe(0); + }); }); From b3200c1e7d1e561dca2d4daee94b1defdfc21aef Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Sat, 1 Aug 2026 16:20:17 +0100 Subject: [PATCH 041/106] fix(observability-map): require a genuine read of the catch binding referencesBinding matched any ts.Identifier whose text equalled the catch binding, so a property name in a member expression (fallback.error), an object literal key ({ error: false }), and a name re-declared in a nested scope all counted as reading the error. A clause whose only if tests fallback.error passed error-classification without ever inspecting what it caught. An identifier only counts now when it is a real reference: excluded are the name side of a property access, a property assignment name, and any position inside a scope (function or catch parameter, or a var/let/const/function/ class declared in a block) that re-declares the same name. Ternaries route through the same predicate instead of accepting any instanceof in the condition, so the fix lands once at the root rather than twice. Measured on the real routes tree by re-scanning every file with the old and new logic and diffing catches[].branches directly: zero clauses change and the global stays at 17. The bug is real and reproduces synthetically, it just has no live occurrence in the current tree to hand-read. --- .../observability-map/src/scan.ts | 62 +++++++++++- .../observability-map/test/checks.test.ts | 19 ++++ .../observability-map/test/scan.test.ts | 99 +++++++++++++++++++ 3 files changed, 176 insertions(+), 4 deletions(-) diff --git a/internal-packages/observability-map/src/scan.ts b/internal-packages/observability-map/src/scan.ts index 3454d0ad4c5..54b58d0aa23 100644 --- a/internal-packages/observability-map/src/scan.ts +++ b/internal-packages/observability-map/src/scan.ts @@ -159,9 +159,57 @@ function containsInstanceOf(node: ts.Node): boolean { ); } -/** Whether `node` reads the given catch binding anywhere, e.g. `e` in `e instanceof X` or `error.code`. */ +/** Whether `name` is declared by a var/let/const, function or class statement directly in this + * statement list. Not recursive: a nested block's own declarations are handled when the walk + * reaches that block. */ +function declaresInScope(statements: readonly ts.Statement[], name: string): boolean { + for (const statement of statements) { + if (ts.isFunctionDeclaration(statement) && statement.name?.text === name) return true; + if (ts.isClassDeclaration(statement) && statement.name?.text === name) return true; + if ( + ts.isVariableStatement(statement) && + statement.declarationList.declarations.some( + (d) => ts.isIdentifier(d.name) && d.name.text === name + ) + ) { + return true; + } + } + return false; +} + +/** + * Whether `node` contains a genuine read of the given catch binding, e.g. `e` in `e instanceof X` + * or `error.code`. An identifier only counts when it is a real reference. Two shapes share the + * binding's text without reading it: the property side of a member expression (`fallback.error`) + * and an object literal key (`{ error: true }`), both excluded by checking which side of the + * parent node the identifier sits on. A name re-declared in a nested scope, as a function or catch + * parameter or as a var/let/const/function/class in a block, refers to that declaration instead, + * so the walk stops at the boundary that re-declares it rather than crediting the outer binding. + */ function referencesBinding(node: ts.Node, bindingName: string): boolean { - return someNode(node, (n) => ts.isIdentifier(n) && n.text === bindingName); + if (ts.isIdentifier(node) && node.text === bindingName) { + const parent = node.parent; + if (ts.isPropertyAccessExpression(parent) && parent.name === node) return false; + if (ts.isPropertyAssignment(parent) && parent.name === node) return false; + return true; + } + + if ( + ts.isFunctionLike(node) && + node.parameters.some((p) => ts.isIdentifier(p.name) && p.name.text === bindingName) + ) { + return false; + } + + if (ts.isCatchClause(node)) { + const decl = node.variableDeclaration; + if (decl && ts.isIdentifier(decl.name) && decl.name.text === bindingName) return false; + } + + if (ts.isBlock(node) && declaresInScope(node.statements, bindingName)) return false; + + return ts.forEachChild(node, (child) => referencesBinding(child, bindingName)) === true; } /** The catch binding's name, or null for a bindingless `catch { ... }` or a destructured one. */ @@ -176,9 +224,15 @@ function catchBindingName(clause: ts.CatchClause): string | null { * `return e instanceof Response ? e : json({}, { status: 500 })` counts and * `return json({ error: e instanceof Error ? e.message : String(e) }, { status: 400 })` does not. * The second is message formatting: every error leaves by the same path. + * + * Goes through `referencesBinding`, the same predicate the `if`/`switch` check uses, rather than + * accepting any `instanceof` in the condition: an `instanceof` that never reads the caught binding + * is not a decision made on the error, and a bindingless catch has nothing here to reference. */ -function selectsAnErrorPath(node: ts.ConditionalExpression): boolean { +function selectsAnErrorPath(node: ts.ConditionalExpression, bindingName: string | null): boolean { + if (bindingName === null) return false; if (!containsInstanceOf(node.condition)) return false; + if (!referencesBinding(node.condition, bindingName)) return false; const parent = node.parent; return parent !== undefined && (ts.isReturnStatement(parent) || ts.isThrowStatement(parent)); } @@ -198,7 +252,7 @@ function catchClauseEvidence(clause: ts.CatchClause): { rethrows: boolean; branc ) { branches = true; } - if (ts.isConditionalExpression(node) && selectsAnErrorPath(node)) branches = true; + if (ts.isConditionalExpression(node) && selectsAnErrorPath(node, bindingName)) branches = true; ts.forEachChild(node, visit); }; visit(clause.block); diff --git a/internal-packages/observability-map/test/checks.test.ts b/internal-packages/observability-map/test/checks.test.ts index eff2617e755..e69d53b7f60 100644 --- a/internal-packages/observability-map/test/checks.test.ts +++ b/internal-packages/observability-map/test/checks.test.ts @@ -253,6 +253,25 @@ describe("error-classification", () => { expect(r.status).toBe("fail"); }); + // A3. `referencesBinding` used to match any identifier with the binding's text, including a + // property name in a member expression. A catch whose only `if` tests `fallback.error`, never the + // caught binding itself, was credited with classifying an error it never inspected. + it("fails a catch whose only if tests a same-named property, not the caught error", () => { + const r = run( + "error-classification", + "api.v1.y.ts", + `import { prisma } from "~/db.server"; + export async function loader() { + try { return await prisma.thing.findMany(); } + catch (error) { + if (fallback.error) return json({}, { status: 500 }); + return json({}, { status: 500 }); + } + }` + ); + expect(r.status).toBe("fail"); + }); + // False positive fixture: the only try/catch in the file belongs to the component. it("does not judge a route whose try/catch is in the React component", () => { const r = run( diff --git a/internal-packages/observability-map/test/scan.test.ts b/internal-packages/observability-map/test/scan.test.ts index d9af8df904c..5fea2292f81 100644 --- a/internal-packages/observability-map/test/scan.test.ts +++ b/internal-packages/observability-map/test/scan.test.ts @@ -1261,3 +1261,102 @@ describe("scanFile: branches requires the if/switch condition to examine the err expect(ep!.catches[0]!.branches).toBe(false); }); }); + +// A3. `referencesBinding` matched any identifier with the binding's text, including a property +// name, an object literal key and a name re-declared in a nested scope. So a clause that never +// really inspects the error still counted as deciding on it. +describe("scanFile: branches requires a genuine read of the binding, not a lookalike", () => { + it("does not set branches for an `if` that only reads a same-named property", () => { + const ep = scanFile( + "branch-property-name.ts", + ` + export async function loader({ request }) { + try { + return json(await load(request)); + } catch (error) { + if (fallback.error) return json({}, { status: 500 }); + return json({}, { status: 500 }); + } + } + ` + ); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + it("does not set branches for an `if` that only reads a same-named object literal key", () => { + const ep = scanFile( + "branch-object-key.ts", + ` + export async function loader({ request }) { + try { + return json(await load(request)); + } catch (error) { + if (buildOptions({ error: false }).ok) return json({}, { status: 500 }); + return json({}, { status: 500 }); + } + } + ` + ); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + it("does not set branches for an `if` whose only reference is inside a callback that re-declares the name", () => { + const ep = scanFile( + "branch-shadowed-param.ts", + ` + export async function loader({ request }) { + try { + return json(await load(request)); + } catch (error) { + if (items.some(function (error) { return error.code === 1; })) { + return json({}, { status: 500 }); + } + return json({}, { status: 500 }); + } + } + ` + ); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + it("does not set branches for an `if` whose only reference is inside a block that re-declares the name", () => { + const ep = scanFile( + "branch-shadowed-block.ts", + ` + export async function loader({ request }) { + try { + return json(await load(request)); + } catch (error) { + if ( + (() => { + const error = 1; + return error > 0; + })() + ) { + return json({}, { status: 500 }); + } + return json({}, { status: 500 }); + } + } + ` + ); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + it("still sets branches for an `if` that genuinely reads the binding beside a lookalike", () => { + const ep = scanFile( + "branch-genuine-and-lookalike.ts", + ` + export async function loader({ request }) { + try { + return json(await load(request)); + } catch (error) { + if (fallback.error || error instanceof NotFound) return json({}, { status: 404 }); + return json({}, { status: 500 }); + } + } + ` + ); + expect(ep!.catches[0]!.branches).toBe(true); + }); +}); From 46c5ad011a2a94dc4f26eed65d7fc5ca09587897 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Sat, 1 Aug 2026 16:25:06 +0100 Subject: [PATCH 042/106] fix(observability-map): stop a dead throw counting as a rethrow rethrows was set by any ThrowStatement anywhere in a catch clause, reachable or not, so appending throw e; after a return flipped a swallowing catch from a fail (swallows, no decision made) to not-applicable (read as inert, rethrow-only), with zero behavioural change. Worth about 3 points tree-wide on the real routes tree, confirmed by simulating the mutation against every catch clause's real evidence. catchClauseEvidence now walks each Block, CaseClause and DefaultClause's own statement list only up to the first statement that unconditionally exits (return, throw, continue or break); anything after that is dead and is never visited. This is not full flow analysis, an if/else where both branches return is not itself recognised as an exit, but it covers the mutation this check was measured against. --- .../observability-map/src/scan.ts | 27 +++++++++++++++++ .../observability-map/test/scan.test.ts | 21 +++++++++++++ .../observability-map/test/score.test.ts | 30 +++++++++++++++++++ 3 files changed, 78 insertions(+) diff --git a/internal-packages/observability-map/src/scan.ts b/internal-packages/observability-map/src/scan.ts index 54b58d0aa23..aeb58f232be 100644 --- a/internal-packages/observability-map/src/scan.ts +++ b/internal-packages/observability-map/src/scan.ts @@ -237,6 +237,29 @@ function selectsAnErrorPath(node: ts.ConditionalExpression, bindingName: string return parent !== undefined && (ts.isReturnStatement(parent) || ts.isThrowStatement(parent)); } +/** A statement that unconditionally leaves the statement list it sits in, so anything after it in + * the same list never runs. */ +function isDefiniteExit(statement: ts.Statement): boolean { + return ( + ts.isReturnStatement(statement) || + ts.isThrowStatement(statement) || + ts.isContinueStatement(statement) || + ts.isBreakStatement(statement) + ); +} + +/** + * `statements` up to and including the first one that definitely exits. Not full flow analysis: + * an `if`/`else` where both branches return is not itself recognised as an exit, only a bare + * `return`, `throw`, `continue` or `break` is. That is enough to make a `throw e;` appended after + * a `return` dead code rather than evidence the clause rethrows, which is the one shape a mutation + * testing this check actually produced. + */ +function reachableStatements(statements: readonly ts.Statement[]): readonly ts.Statement[] { + const index = statements.findIndex(isDefiniteExit); + return index === -1 ? statements : statements.slice(0, index + 1); +} + /** What a catch clause does with the error, beyond the fact that it caught one. */ function catchClauseEvidence(clause: ts.CatchClause): { rethrows: boolean; branches: boolean } { let rethrows = false; @@ -244,6 +267,10 @@ function catchClauseEvidence(clause: ts.CatchClause): { rethrows: boolean; branc const bindingName = catchBindingName(clause); const visit = (node: ts.Node) => { + if (ts.isBlock(node) || ts.isCaseClause(node) || ts.isDefaultClause(node)) { + for (const statement of reachableStatements(node.statements)) visit(statement); + return; + } if (ts.isThrowStatement(node)) rethrows = true; if ( bindingName !== null && diff --git a/internal-packages/observability-map/test/scan.test.ts b/internal-packages/observability-map/test/scan.test.ts index 5fea2292f81..ae67d598bc7 100644 --- a/internal-packages/observability-map/test/scan.test.ts +++ b/internal-packages/observability-map/test/scan.test.ts @@ -615,6 +615,27 @@ describe("scanFile: catch clause evidence", () => { expect(ep!.catches[0]!.branches).toBe(false); }); + // A4. `rethrows` used to be set by any `ThrowStatement` in the clause, reachable or not, so a + // `throw e;` appended after a `return` flipped a swallowing catch from rethrows: false to true + // with no behavioural change, which read as inert instead of a swallow. + it("does not set rethrows for a throw that is dead code after a return", () => { + const ep = scanFile( + "dead-throw.ts", + ` + export async function loader({ request }) { + try { + return json(await load(request)); + } catch (e) { + return null; + throw e; + } + } + ` + ); + expect(ep!.catches[0]!.rethrows).toBe(false); + expect(ep!.catches[0]!.branches).toBe(false); + }); + it("leaves both flags false when the catch only returns", () => { const ep = scanFile( "swallow.ts", diff --git a/internal-packages/observability-map/test/score.test.ts b/internal-packages/observability-map/test/score.test.ts index ccbc24b973e..05386622993 100644 --- a/internal-packages/observability-map/test/score.test.ts +++ b/internal-packages/observability-map/test/score.test.ts @@ -314,6 +314,36 @@ export async function loader({ params }) { expect(after.score).toBeLessThanOrEqual(before.score); }); + // A4. rethrows used to be set by any ThrowStatement anywhere in the clause, dead code included, + // so appending `throw e;` after a `return` in a swallowing catch flipped error-classification + // from fail to not-applicable: a mutation with no behavioural effect that hid the swallow. + it("does not improve the verdict when a dead throw follows a return in a swallowing catch", () => { + const swallows = `import { prisma } from "~/db.server"; +export async function loader() { + try { + ${BODY} + } catch (e) { + return null; + } +}`; + const swallowsWithDeadThrow = `import { prisma } from "~/db.server"; +export async function loader() { + try { + ${BODY} + } catch (e) { + return null; + throw e; + } +}`; + + const before = scoreEntry(scanFile("api.v1.x.ts", swallows)!); + const after = scoreEntry(scanFile("api.v1.x.ts", swallowsWithDeadThrow)!); + + expect(before.checks.find((c) => c.id === "error-classification")!.status).toBe("fail"); + expect(after.checks.find((c) => c.id === "error-classification")!.status).toBe("fail"); + expect(after.score).toBeLessThanOrEqual(before.score); + }); + it("does not pay for wrapping a whole tree in catches that only rethrow", () => { const before = buildReport( [scanFile("api.v1.x.ts", plain)!, scanFile("api.v1.y.ts", plain)!], From 433ebec31209f19532ddb3f5274a80c04edb82cc Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Sat, 1 Aug 2026 16:33:27 +0100 Subject: [PATCH 043/106] fix(observability-map): require request-context to name a tenant, at a level worth reading request-context filtered failure-path logs on inCatch only, never on level, and IDENTIFIER_FIELD matched any suffix shape (Id/Ids/Slug/Ref/Param/ Identifier) regardless of the root word. So logger.debug("cache miss", { id: 1 }) passed inside a catch, and 10 of the 21 passes named a resource (batchId, notificationId, chatId, spanParam, runFriendlyId, taskIdentifier, runId, waitpointId, sessionId) rather than a tenant. TENANT_FIELD now anchors on the root word: environment, organization, project or user, in the full and abbreviated forms the webapp actually writes (environmentId/envId, organizationId/organizationSlug/orgId, projectId/ projectParam, userId), derived by grepping the routes tree rather than invented. QUALIFYING_LEVELS restricts the log itself to error, warn and fatal; debug and trace are routinely dropped before an incident is read, and info is excluded too since it is not reserved for failure reporting. Measured on the real tree: CONTEXT moves from 21 of 412 to 11 of 412, all ten lost to the field-name fix (every flip was already at error or warn level, so the level restriction changes nothing on the current tree; it still closes the exact logger.debug shape in the brief). Global moves from 17 to 16. The 11 that still pass all name a genuine environment, organization or project field. --- .../src/checks/requestContext.ts | 42 +++++-- .../observability-map/src/report/terminal.ts | 2 +- .../observability-map/test/checks.test.ts | 110 ++++++++++++++++++ .../observability-map/test/report.test.ts | 2 +- 4 files changed, 143 insertions(+), 13 deletions(-) diff --git a/internal-packages/observability-map/src/checks/requestContext.ts b/internal-packages/observability-map/src/checks/requestContext.ts index 041a9515257..f2d37aa600e 100644 --- a/internal-packages/observability-map/src/checks/requestContext.ts +++ b/internal-packages/observability-map/src/checks/requestContext.ts @@ -4,12 +4,30 @@ import { isTrivial } from "../triviality.js"; const ID = "request-context"; /** - * A field name that says which tenant, request or resource the failure belongs to. Matched on the - * suffix, in the camelCase the webapp writes: `environmentId`, `organizationSlug`, `runFriendlyId`, - * `projectParam`, `taskIdentifier`. Lowercase `id` inside a word is not a suffix, so `valid` and - * `paid` do not qualify. + * A field name that plausibly names a TENANT: environment, organization, project or user, the four + * things every entry point ultimately belongs to. Anchored on the root word, not just the suffix, + * in the full and abbreviated camelCase the webapp actually writes for each: + * `environmentId`/`envId`, `organizationId`/`organizationSlug`/`orgId`, `projectId`/`projectParam`, + * `userId`. A bare `id`, and a resource id that happens to share the same `Id`/`Param` suffix, + * `batchId`, `notificationId`, `chatId`, `spanParam`, `runFriendlyId`, `taskIdentifier`, does not + * qualify: those name a resource the failure touched, not who it happened to. */ -const IDENTIFIER_FIELD = /^(id|ids|slug|ref)$|[a-z](Id|Ids|Slug|Ref|Param|Identifier)$/; +const TENANT_FIELD = + /^(environment|env|organization|org|project|user)(Id|Ids|Slug|Ref|Param|Identifier)?$/; + +/** + * error, warn and fatal are levels an incident is actually read at; debug and trace are routinely + * dropped or sampled out before anyone looks, so a tenant field logged only at that level is not + * really recorded on the failure path. info is deliberately excluded too: it is not reserved for + * failure reporting, so a route can log an info line inside a catch that says nothing about the + * catch actually handling anything, and crediting it would launder the same gap `debug` closes. + */ +const QUALIFYING_LEVELS = new Set(["error", "warn", "fatal"]); + +/** The level a `LogCall`'s callee was made at, e.g. `"error"` from `logger.error`. */ +function logLevel(callee: string): string { + return callee.slice(callee.lastIndexOf(".") + 1); +} /** * Whether a failure here can be traced to whoever it happened to. @@ -20,7 +38,8 @@ const IDENTIFIER_FIELD = /^(id|ids|slug|ref)$|[a-z](Id|Ids|Slug|Ref|Param|Identi * tenant: no route calls `trace({ environmentId }, ...)`, and the builders' own boundary log is * `logBoundaryError(message, error, url)`, a url and an error. So an incident tells you which route * and which request failed, and never whose environment it was, unless the route passed the field - * itself. 21 of 427 entry points do. + * itself. 11 of 427 entry points do, naming an environment, organization, project or user; the + * other 10 that used to be counted here only named a resource the failure touched, not a tenant. * * Every non-trivial entry point is judged, and a route that never catches fails like any other. * That is the whole point rather than an oversight: its failures go to the global handler, which @@ -39,18 +58,19 @@ export const requestContext = { if (isTrivial(ep)) { return { id: ID, status: "not-applicable", detail: "trivial route" }; } - const failurePathLogs = ep.logCalls.filter((l) => l.inCatch); - const named = failurePathLogs.find((l) => l.fields.some((f) => IDENTIFIER_FIELD.test(f))); + const failurePathLogs = ep.logCalls.filter( + (l) => l.inCatch && QUALIFYING_LEVELS.has(logLevel(l.callee)) + ); + const named = failurePathLogs.find((l) => l.fields.some((f) => TENANT_FIELD.test(f))); if (named) { - const fields = named.fields.filter((f) => IDENTIFIER_FIELD.test(f)); + const fields = named.fields.filter((f) => TENANT_FIELD.test(f)); return { id: ID, status: "pass", detail: `failure log names ${fields.join(", ")}` }; } if (failurePathLogs.length > 0) { return { id: ID, status: "fail", - detail: - "logs its failure without naming an environment, project, organization, run or user", + detail: "logs its failure without naming an environment, organization, project or user", }; } return { diff --git a/internal-packages/observability-map/src/report/terminal.ts b/internal-packages/observability-map/src/report/terminal.ts index 05cc7d6a248..e85c629bb2d 100644 --- a/internal-packages/observability-map/src/report/terminal.ts +++ b/internal-packages/observability-map/src/report/terminal.ts @@ -21,7 +21,7 @@ export const scoredFailures = (e: ScoredEntry) => e.checks.filter((c) => SCORED_CHECK_IDS.includes(c.id) && c.status === "fail"); /** - * An entry whose only finding is `request-context`. 391 of 412 entry points fail that check, so + * An entry whose only finding is `request-context`. 401 of 412 entry points fail that check, so * listing each one turns the fix list into a single house-style finding repeated, which is the * reason `audit-trail` is kept out of the list too. Collapsed into the `CONTEXT` figure instead. * An entry that fails something else as well stays in the list with all of its findings, so a diff --git a/internal-packages/observability-map/test/checks.test.ts b/internal-packages/observability-map/test/checks.test.ts index e69d53b7f60..f4377e930ab 100644 --- a/internal-packages/observability-map/test/checks.test.ts +++ b/internal-packages/observability-map/test/checks.test.ts @@ -709,6 +709,116 @@ describe("request-context", () => { ); expect(r.status).toBe("not-applicable"); }); + + // A5. IDENTIFIER_FIELD matched any suffix, so a resource id (a run, a batch, a notification, a + // chat, a span) that shares the same `Id`/`Param` shape as a tenant field passed, and a bare `id` + // passed too. TENANT_FIELD requires the root word itself to be environment, organization, project + // or user. + it("does not pass a failure log that only names a resource, not a tenant", () => { + const r = run( + "request-context", + "api.v3.batches.$batchId.ts", + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function loader({ params }) { + try { return await prisma.thing.findMany(); } + catch (error) { + logger.error("batch lookup failed", { batchId: params.batchId, error }); + throw error; + } + }` + ); + expect(r.status).toBe("fail"); + }); + + it("does not pass a failure log that only names a bare id", () => { + const r = run( + "request-context", + "api.v1.q.ts", + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function loader() { + try { return await prisma.thing.findMany(); } + catch (error) { + logger.debug("cache miss", { id: 1 }); + logger.error("lookup failed", { id: 1, error }); + throw error; + } + }` + ); + expect(r.status).toBe("fail"); + }); + + it("passes on the abbreviated envId/orgId forms the webapp also writes", () => { + const r = run( + "request-context", + "api.v1.q.ts", + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function loader({ params }) { + try { return await prisma.thing.findMany(); } + catch (error) { + logger.error("lookup failed", { envId: params.envId, orgId: params.orgId, error }); + throw error; + } + }` + ); + expect(r.status).toBe("pass"); + }); + + // A5. request-context filtered failure-path logs on inCatch only, not on level, so a debug log + // naming a tenant field passed the check even though debug lines are routinely dropped or + // sampled out before an incident is read. + it("does not pass a debug-level failure log, even one that names a tenant", () => { + const r = run( + "request-context", + "api.v1.q.ts", + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function loader({ params }) { + try { return await prisma.thing.findMany(); } + catch (error) { + logger.debug("lookup failed", { environmentId: params.envId, error }); + throw error; + } + }` + ); + expect(r.status).toBe("fail"); + }); + + it("does not pass an info-level failure log either", () => { + const r = run( + "request-context", + "api.v1.q.ts", + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function loader({ params }) { + try { return await prisma.thing.findMany(); } + catch (error) { + logger.info("lookup failed", { environmentId: params.envId, error }); + throw error; + } + }` + ); + expect(r.status).toBe("fail"); + }); + + it("still passes a warn-level failure log that names a tenant", () => { + const r = run( + "request-context", + "api.v1.q.ts", + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function loader({ params }) { + try { return await prisma.thing.findMany(); } + catch (error) { + logger.warn("lookup failed", { environmentId: params.envId, error }); + throw error; + } + }` + ); + expect(r.status).toBe("pass"); + }); }); describe("audit-trail", () => { diff --git a/internal-packages/observability-map/test/report.test.ts b/internal-packages/observability-map/test/report.test.ts index d287e19b259..7beed431da1 100644 --- a/internal-packages/observability-map/test/report.test.ts +++ b/internal-packages/observability-map/test/report.test.ts @@ -228,7 +228,7 @@ describe("collapsing the house-style finding", () => { }` )!; - // request-context fails 391 of 412 entry points, so listing each one turns the fix list into a + // request-context fails 401 of 412 entry points, so listing each one turns the fix list into a // single finding repeated. Same reasoning that keeps audit-trail out of the list. it("keeps an entry whose only finding is request-context out of the fix list", () => { const out = renderTerminal(buildReport([namesNobody()], [])); From 2061d999a53b8dbacf8e534050dab0e4a057c53e Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Sat, 1 Aug 2026 16:41:42 +0100 Subject: [PATCH 044/106] fix(observability-map): compare a parse guard against its own body, not the entry point isParseGuard divided tryStatementCount by ep.statementCount, which sums the loader, the action and every one-hop helper together. Adding an unrelated sibling handler or a fat helper to the same file, with zero change to the clause itself, diluted that denominator and relabelled a broad swallow as a narrow parse guard. CatchEvidence now carries enclosingStatementCount, the statement count of the specific loader, action or helper body the clause is directly inside, set once per walkBody call in scan.ts. isParseGuard compares against that instead, and no longer needs the EntryPoint threaded through decides/inert/ swallows just to reach ep.statementCount. Measured on the real tree: two routes flip from pass to fail, both genuine swallows that a same-file sibling or a followed helper had been diluting into an apparent narrow guard. Global stays at 16 (rounds the same either way). A byte-identical swallowing action test proves the verdict no longer depends on what else shares the file. --- .../src/checks/errorClassification.ts | 35 ++++++------ .../observability-map/src/scan.ts | 4 +- .../observability-map/src/types.ts | 8 +++ .../observability-map/test/checks.test.ts | 54 +++++++++++++++++++ .../observability-map/test/scan.test.ts | 2 + 5 files changed, 87 insertions(+), 16 deletions(-) diff --git a/internal-packages/observability-map/src/checks/errorClassification.ts b/internal-packages/observability-map/src/checks/errorClassification.ts index a78559eefba..7d5f24f5dc0 100644 --- a/internal-packages/observability-map/src/checks/errorClassification.ts +++ b/internal-packages/observability-map/src/checks/errorClassification.ts @@ -27,14 +27,19 @@ const BUILDERS = new Set([ /** * Whether a catch clause is a guard rather than the route's error handling: it wraps a parse and - * covers less than half the entry point's statements. Both halves matter. `guardsParse` alone lets - * `otel.v1.logs.ts` off, whose catch covers 15 of its 18 statements and merely happens to contain a - * `request.json()`, and that is a real swallow. The coverage test is relative to the body rather - * than a second absolute threshold, so it holds for a three-statement route and a fifty-statement - * one alike. + * covers less than half the statements of the body it is actually in. Both halves matter. + * `guardsParse` alone lets `otel.v1.logs.ts` off, whose catch covers 15 of its 18 statements and + * merely happens to contain a `request.json()`, and that is a real swallow. The coverage test is + * relative to the enclosing body rather than a second absolute threshold, so it holds for a + * three-statement route and a fifty-statement one alike. + * + * Compared against `clause.enclosingStatementCount`, never `ep.statementCount`: the entry point's + * total sums the loader, the action and every one-hop helper together, so an unrelated sibling + * handler or a fat helper in the same file diluted the denominator and relabelled a broad swallow + * as a narrow parse guard, with no change to the clause itself. */ -function isParseGuard(clause: CatchEvidence, ep: EntryPoint): boolean { - return clause.guardsParse && clause.tryStatementCount * 2 < ep.statementCount; +function isParseGuard(clause: CatchEvidence): boolean { + return clause.guardsParse && clause.tryStatementCount * 2 < clause.enclosingStatementCount; } /** @@ -59,18 +64,18 @@ function isParseGuard(clause: CatchEvidence, ep: EntryPoint): boolean { * would clear said six were real, including a silent run cancellation and two credential paths * that report a database failure to the browser as a 400 with an internal message in it. */ -function decides(clause: CatchEvidence, ep: EntryPoint): boolean { - return clause.branches || isParseGuard(clause, ep); +function decides(clause: CatchEvidence): boolean { + return clause.branches || isParseGuard(clause); } /** Passes the error through unchanged, which is the same outcome as not catching it. */ -function inert(clause: CatchEvidence, ep: EntryPoint): boolean { - return clause.rethrows && !decides(clause, ep); +function inert(clause: CatchEvidence): boolean { + return clause.rethrows && !decides(clause); } /** The error stops here and nothing chose what it meant. */ -function swallows(clause: CatchEvidence, ep: EntryPoint): boolean { - return !decides(clause, ep) && !inert(clause, ep); +function swallows(clause: CatchEvidence): boolean { + return !decides(clause) && !inert(clause); } export function usesBuilder(ep: EntryPoint): boolean { @@ -106,7 +111,7 @@ export const errorClassification = { if (isTrivial(ep)) { return { id: ID, status: "not-applicable", detail: "trivial route" }; } - const swallowed = ep.catches.filter((c) => swallows(c, ep)); + const swallowed = ep.catches.filter(swallows); if (swallowed.length > 0) { const which = ep.catches.length > 1 ? ` (${swallowed.length} of ${ep.catches.length} catches)` : ""; @@ -116,7 +121,7 @@ export const errorClassification = { detail: `catches its errors and takes one way out regardless of what was thrown${which}`, }; } - if (!ep.catches.some((c) => decides(c, ep))) { + if (!ep.catches.some(decides)) { return { id: ID, status: "not-applicable", diff --git a/internal-packages/observability-map/src/scan.ts b/internal-packages/observability-map/src/scan.ts index aeb58f232be..4f43e70745d 100644 --- a/internal-packages/observability-map/src/scan.ts +++ b/internal-packages/observability-map/src/scan.ts @@ -627,7 +627,8 @@ export function scanFile(fileName: string, source: string): EntryPoint | null { const helpers: EntryFunction[] = []; const walkBody = (fn: EntryFunction, followHelpers: boolean) => { - statementCount += countFunctionStatements(fn); + const enclosingStatementCount = countFunctionStatements(fn); + statementCount += enclosingStatementCount; if (!fn.body) return; const visit = (node: ts.Node, inCatch: boolean) => { @@ -641,6 +642,7 @@ export function scanFile(fileName: string, source: string): EntryPoint | null { branches: clause.branches, guardsParse: guardsParse(node.tryBlock), tryStatementCount, + enclosingStatementCount, }); } } diff --git a/internal-packages/observability-map/src/types.ts b/internal-packages/observability-map/src/types.ts index a1d3be0b2db..bf6e6a79022 100644 --- a/internal-packages/observability-map/src/types.ts +++ b/internal-packages/observability-map/src/types.ts @@ -32,6 +32,14 @@ export type CatchEvidence = { guardsParse: boolean; /** Statements in the guarded try block, counted as `statementCount` counts them. */ tryStatementCount: number; + /** + * Statements in the body the clause actually sits in: the loader, the action, or the one-hop + * helper it delegates to, whichever function this `try` is directly inside. Never the entry + * point's combined total, which sums the loader, the action and every helper together: comparing + * a clause against that whole-entry-point figure let an unrelated sibling handler or a fat helper + * in the same file relabel a broad swallow as a narrow parse guard. + */ + enclosingStatementCount: number; }; /** A logging call made from a loader/action body, or from a same-file helper the body calls. */ diff --git a/internal-packages/observability-map/test/checks.test.ts b/internal-packages/observability-map/test/checks.test.ts index f4377e930ab..b958cd7a487 100644 --- a/internal-packages/observability-map/test/checks.test.ts +++ b/internal-packages/observability-map/test/checks.test.ts @@ -253,6 +253,60 @@ describe("error-classification", () => { expect(r.status).toBe("fail"); }); + // A6. isParseGuard compared the clause against ep.statementCount, the loader and the action and + // every one-hop helper summed together, rather than the statements of the body the clause is + // actually in. So an unrelated sibling handler or a fat helper in the same file diluted the + // denominator and relabelled the same broad swallow as a narrow parse guard. Byte-identical + // action, verdict must not move. + it("gives the same verdict to a byte-identical swallow whether or not an unrelated sibling and helper share the file", () => { + const action = `import { otlpExporter } from "~/v3/otlpExporter.server"; + export async function action({ request }) { + try { + const exporter = await otlpExporter; + const contentType = request.headers.get("content-type"); + const body = await request.json(); + const result = await exporter.exportLogs(body); + const encoded = encodeResponse(result); + const headers = buildHeaders(contentType); + return new Response(encoded, { status: 200, headers }); + } catch (error) { + console.error(error); + return new Response("Internal Server Error", { status: 500 }); + } + }`; + + const withUnrelatedSiblingAndHelper = `${action} + function unrelatedHelper() { + let total = 0; + total += 1; + total += 2; + total += 3; + total += 4; + total += 5; + total += 6; + total += 7; + total += 8; + total += 9; + total += 10; + total += 11; + return total; + } + export async function loader() { + const helperTotal = unrelatedHelper(); + return new Response(String(helperTotal)); + }`; + + const alone = run("error-classification", "otel.v1.logs.ts", action); + const withSiblingAndHelper = run( + "error-classification", + "otel.v1.logs.ts", + withUnrelatedSiblingAndHelper + ); + + expect(alone.status).toBe("fail"); + expect(withSiblingAndHelper.status).toBe("fail"); + }); + // A3. `referencesBinding` used to match any identifier with the binding's text, including a // property name in a member expression. A catch whose only `if` tests `fallback.error`, never the // caught binding itself, was credited with classifying an error it never inspected. diff --git a/internal-packages/observability-map/test/scan.test.ts b/internal-packages/observability-map/test/scan.test.ts index ae67d598bc7..af022e34974 100644 --- a/internal-packages/observability-map/test/scan.test.ts +++ b/internal-packages/observability-map/test/scan.test.ts @@ -882,6 +882,7 @@ describe("scanFile: per-catch evidence", () => { branches: false, guardsParse: true, tryStatementCount: 1, + enclosingStatementCount: 10, }); expect(ep!.catches[1]).toMatchObject({ guardsParse: false, @@ -983,6 +984,7 @@ describe("scanFile: per-catch evidence", () => { branches: false, guardsParse: false, tryStatementCount: 4, + enclosingStatementCount: 6, }); }); From 9e3b21a65a30feb1de9af05248b9bf35e0bec2a8 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Sat, 1 Aug 2026 16:48:47 +0100 Subject: [PATCH 045/106] fix(observability-map): stop attributing an inline callback's catch to the route The catch collector descended into inline callbacks (Promise.all(items.map(async i => { try { ... } catch { ... } }))) while countStatement, by design, does not, so the two disagreed about what the entry point even is: tryStatementCount could exceed the entry's own statementCount, and a per-item error boundary was judged as though it were the route's own catch. Chose not to descend into an inline callback for catch evidence either, the smaller change: it matches the existing statement-counting decision rather than reopening the over-counting a callback walk was fixed for earlier. calleeNames and logCalls still descend into callbacks, unchanged, since isTrivial deliberately relies on calleeNames seeing work a short statement count hides. hasTryCatch also stays true for a callback-nested try, since it is only a "some try appeared" signal for triviality, not an attribution claim. Measured on the real tree: five routes move, four from fail to not-applicable and one from pass to not-applicable, all genuinely catching nothing at the route's own level once a callback's catch is no longer credited to it. Global moves from 16 to 15. --- .../observability-map/src/scan.ts | 21 ++++++++++--- .../observability-map/test/checks.test.ts | 25 +++++++++++++++ .../observability-map/test/scan.test.ts | 31 +++++++++++++++++++ 3 files changed, 72 insertions(+), 5 deletions(-) diff --git a/internal-packages/observability-map/src/scan.ts b/internal-packages/observability-map/src/scan.ts index 4f43e70745d..72b6b35ba16 100644 --- a/internal-packages/observability-map/src/scan.ts +++ b/internal-packages/observability-map/src/scan.ts @@ -631,10 +631,21 @@ export function scanFile(fileName: string, source: string): EntryPoint | null { statementCount += enclosingStatementCount; if (!fn.body) return; - const visit = (node: ts.Node, inCatch: boolean) => { + // `inCallback` is true once the walk has entered an inline function (`.map((item) => { ... })`), + // never reset back to false: a callback nested inside another callback is still a callback. + // `calleeNames` and `logCalls` keep descending in there regardless, which is what lets + // `isTrivial` see work a short statement count hides. A try/catch does not: it is not part of + // this body's own statement list, and `countStatement` already stops at a nested function + // boundary, so counting it here let `tryStatementCount` exceed the entry point's whole + // `statementCount` and judged a per-item error boundary as though it were the route's own. + const visit = (node: ts.Node, inCatch: boolean, inCallback: boolean) => { + if (ts.isFunctionLike(node)) { + ts.forEachChild(node, (child) => visit(child, inCatch, true)); + return; + } if (ts.isTryStatement(node)) { hasTryCatch = true; - if (node.catchClause) { + if (node.catchClause && !inCallback) { const tryStatementCount = countStatements(node.tryBlock.statements); const clause = catchClauseEvidence(node.catchClause); catches.push({ @@ -648,7 +659,7 @@ export function scanFile(fileName: string, source: string): EntryPoint | null { } if (ts.isCatchClause(node)) { - ts.forEachChild(node, (child) => visit(child, true)); + ts.forEachChild(node, (child) => visit(child, true, inCallback)); return; } @@ -678,9 +689,9 @@ export function scanFile(fileName: string, source: string): EntryPoint | null { } } } - ts.forEachChild(node, (child) => visit(child, inCatch)); + ts.forEachChild(node, (child) => visit(child, inCatch, inCallback)); }; - visit(fn.body, false); + visit(fn.body, false, false); }; for (const fn of target.functions) walkBody(fn, true); diff --git a/internal-packages/observability-map/test/checks.test.ts b/internal-packages/observability-map/test/checks.test.ts index b958cd7a487..11c606639e2 100644 --- a/internal-packages/observability-map/test/checks.test.ts +++ b/internal-packages/observability-map/test/checks.test.ts @@ -340,6 +340,31 @@ describe("error-classification", () => { ); expect(r.status).toBe("not-applicable"); }); + + // A7. A per-item error boundary inside a `.map()` callback used to be judged as the route's own + // catch. The route's own visible body catches nothing, so it is not-applicable, not a pass or a + // fail on the strength of a swallow one level of nesting away. + it("is not applicable to a route whose only catch is inside a Promise.all(items.map(...)) callback", () => { + const r = run( + "error-classification", + "batch.process.ts", + `import { prisma } from "~/db.server"; + export async function action({ request }) { + const items = await prisma.item.findMany(); + await Promise.all( + items.map(async (item) => { + try { + await processItem(item); + } catch { + return null; + } + }) + ); + return json({ ok: true }); + }` + ); + expect(r.status).toBe("not-applicable"); + }); }); describe("auth-boundary", () => { diff --git a/internal-packages/observability-map/test/scan.test.ts b/internal-packages/observability-map/test/scan.test.ts index af022e34974..0f0151df9d7 100644 --- a/internal-packages/observability-map/test/scan.test.ts +++ b/internal-packages/observability-map/test/scan.test.ts @@ -798,6 +798,37 @@ describe("scanFile: catch clause evidence", () => { expect(ep!.hasTryCatch).toBe(false); expect(ep!.catches).toEqual([]); }); + + // A7. The catch collector descended into inline callbacks while `countStatement`, by design, + // does not, so a per-item error boundary inside a `.map()` was judged as the route's own catch + // and `tryStatementCount` could exceed the entry point's whole `statementCount`. Fixed by not + // descending into inline callbacks for catch evidence either, the smaller change: it matches the + // existing statement-counting decision rather than reopening the over-counting a callback walk + // was fixed for earlier. `calleeNames` and `logCalls` still descend, unchanged, which is what lets + // `isTrivial` see work a short statement count hides. + it("does not attribute a catch inside an inline callback (Promise.all(items.map(...))) to the route", () => { + const ep = scanFile( + "batch.process.ts", + ` + export async function action({ request }) { + const items = await loadItems(request); + await Promise.all( + items.map(async (item) => { + try { + await processItem(item); + } catch { + return null; + } + }) + ); + return json({ ok: true }); + } + ` + ); + expect(ep!.catches).toEqual([]); + // A try/catch appeared somewhere in the body, which is still a real signal for triviality. + expect(ep!.hasTryCatch).toBe(true); + }); }); describe("scanFile: log calls", () => { From 470422889355aeaa5b8002a0315288d1fe54b7d9 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Sat, 1 Aug 2026 17:28:14 +0100 Subject: [PATCH 046/106] fix(observability-map): root the callback-boundary rule for catch attribution C1: the A7 fix stopped at ANY function-like node, purely lexical, so wrapping a whole route body in a single-shot callback (trace(async () => {...}), mutateWithFallback({ pgMutation }), new ReadableStream({ start })) deleted its catches and error-classification went not-applicable, worth up to 50 points for a syntactic no-op. That is the A1 defect class, reintroduced. Confirmed wrong on three real routes: api.v2.runs.$runParam.cancel.ts (the main mutation path, inside pgMutation), the presence.tsx SSE tick handler, and the ai-generate.tsx streaming body inside ReadableStream.start. The real distinction is per-item iteration versus everything else. walkBody now only treats a callback passed to map/forEach/filter/reduce/reduceRight/ flatMap/some/every as a boundary; every other function-like node, including the three wrapper shapes above, is descended into and its catch attributed. Verified against all five real routes the previous fix moved: the three wrongly-affected ones are fail again (their own genuine swallow), the two genuine .map() per-item boundaries stay not-applicable. C2: catchClauseEvidence still descended into every function body unconditionally, so a throw or an error test merely constructed inside a callback the clause registers (queue.push(() => { throw e }), setTimeout) was credited to the clause, and reachableStatements only handled dead code from statement ordering, not a statically-false condition (if (false), while (false)). Both closed: the clause-level walk stops at every function-like node (broader than walkBody's boundary on purpose, since here we are already inside a catch the route owns, and constructing a further callback is deferred work, not the clause's own decision), and isFalsyLiteral recognises false/null/undefined/0 so a dead branch is never visited. A do/while, which runs its body once regardless of the trailing condition, still registers, proving the fix is not over-broad. Real-tree effect: global unchanged at 15, measured unchanged at 412. Three routes move from not-applicable back to fail (their own catch, correctly attributed again); two more show additional catches now discovered (previously wrongly excluded) with no change to their overall verdict. --- .../observability-map/src/scan.ts | 93 +++++++- .../observability-map/test/scan.test.ts | 215 ++++++++++++++++-- 2 files changed, 275 insertions(+), 33 deletions(-) diff --git a/internal-packages/observability-map/src/scan.ts b/internal-packages/observability-map/src/scan.ts index 72b6b35ba16..a3bf3456363 100644 --- a/internal-packages/observability-map/src/scan.ts +++ b/internal-packages/observability-map/src/scan.ts @@ -260,17 +260,45 @@ function reachableStatements(statements: readonly ts.Statement[]): readonly ts.S return index === -1 ? statements : statements.slice(0, index + 1); } -/** What a catch clause does with the error, beyond the fact that it caught one. */ +/** + * A literal that is always falsy: `false`, `null`, `undefined`, `0`. Not general constant folding, + * on purpose: `!true`, `1 === 2` and a reference to a `const` declared elsewhere are not covered, so + * `if (false) { throw e; }` and `while (false) { throw e; }` are recognised as dead and nothing more + * elaborate is claimed to be. + */ +function isFalsyLiteral(expr: ts.Expression): boolean { + if (expr.kind === ts.SyntaxKind.FalseKeyword || expr.kind === ts.SyntaxKind.NullKeyword) { + return true; + } + if (ts.isIdentifier(expr) && expr.text === "undefined") return true; + return ts.isNumericLiteral(expr) && expr.text === "0"; +} + +/** + * What a catch clause does with the error, beyond the fact that it caught one. + * + * Stops at every function-like node, not only an iteration callback (contrast `walkBody`'s + * boundary, which lets a route's own single-shot wrapper, `trace(async () => {...})`, + * `mutateWithFallback({ pgMutation })`, `new ReadableStream({ start })`, through so the route's own + * catch is found at all). Here the walk is already inside a catch clause that `walkBody` decided + * belongs to the route; anything the clause does by constructing a further callback, + * `queue.push(() => { throw e; })`, a `.then`, a `setTimeout`, is deferred work the clause merely + * registers, not a decision it makes on its own execution. Both walks refuse a per-item iteration + * callback; this one refuses every other kind of callback too, for that reason. + */ function catchClauseEvidence(clause: ts.CatchClause): { rethrows: boolean; branches: boolean } { let rethrows = false; let branches = false; const bindingName = catchBindingName(clause); const visit = (node: ts.Node) => { + if (ts.isFunctionLike(node)) return; + if (ts.isBlock(node) || ts.isCaseClause(node) || ts.isDefaultClause(node)) { for (const statement of reachableStatements(node.statements)) visit(statement); return; } + if (ts.isThrowStatement(node)) rethrows = true; if ( bindingName !== null && @@ -280,6 +308,20 @@ function catchClauseEvidence(clause: ts.CatchClause): { rethrows: boolean; branc branches = true; } if (ts.isConditionalExpression(node) && selectsAnErrorPath(node, bindingName)) branches = true; + + // A statically-false condition makes the guarded statement dead code: it can set `branches` + // above (the clause still decided to test the error, even if the arm never runs), but nothing + // inside it can set `rethrows` or a nested `branches`, so it is not visited at all. + if (ts.isIfStatement(node)) { + if (!isFalsyLiteral(node.expression)) visit(node.thenStatement); + if (node.elseStatement) visit(node.elseStatement); + return; + } + if (ts.isWhileStatement(node)) { + if (!isFalsyLiteral(node.expression)) visit(node.statement); + return; + } + ts.forEachChild(node, visit); }; visit(clause.block); @@ -287,6 +329,35 @@ function catchClauseEvidence(clause: ts.CatchClause): { rethrows: boolean; branc return { rethrows, branches }; } +/** + * Array methods that invoke their callback once per element, never once as a whole. The one + * structural signal that separates a per-item boundary (`items.map((item) => { try {...} })`, a + * fresh catch for every element) from a route's own body expressed through one more layer of + * function nesting (`trace(async () => {...})`, `mutateWithFallback({ pgMutation: async (t) => + * {...} })`, `new ReadableStream({ start: async (c) => {...} })`), all of which invoke their + * callback exactly once, as the route's own continuation. + */ +const ITERATION_METHODS = new Set([ + "map", + "forEach", + "filter", + "reduce", + "reduceRight", + "flatMap", + "some", + "every", +]); + +/** Whether the function-like `node` is the callback argument of a call to one of + * `ITERATION_METHODS`, e.g. the arrow function in `items.map((item) => ...)`. */ +function isIterationCallback(node: ts.Node): boolean { + const parent = node.parent; + if (!parent || !ts.isCallExpression(parent)) return false; + if (!parent.arguments.includes(node as ts.Expression)) return false; + const callee = unwrap(parent.expression); + return ts.isPropertyAccessExpression(callee) && ITERATION_METHODS.has(callee.name.text); +} + const HTTP_METHODS = new Set(["GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS"]); function propertyName(property: ts.ObjectLiteralElementLike): string | null { @@ -631,16 +702,22 @@ export function scanFile(fileName: string, source: string): EntryPoint | null { statementCount += enclosingStatementCount; if (!fn.body) return; - // `inCallback` is true once the walk has entered an inline function (`.map((item) => { ... })`), - // never reset back to false: a callback nested inside another callback is still a callback. - // `calleeNames` and `logCalls` keep descending in there regardless, which is what lets - // `isTrivial` see work a short statement count hides. A try/catch does not: it is not part of - // this body's own statement list, and `countStatement` already stops at a nested function - // boundary, so counting it here let `tryStatementCount` exceed the entry point's whole + // `inCallback` is true once the walk has entered a per-item iteration callback + // (`items.map((item) => { ... })`), never reset back to false: nesting deeper inside one is + // still inside it. `calleeNames` and `logCalls` keep descending regardless, which is what lets + // `isTrivial` see work a short statement count hides. A try/catch does not: a per-item catch is + // not part of this body's own statement list, and `countStatement` already stops at a nested + // function boundary, so counting it here let `tryStatementCount` exceed the entry point's whole // `statementCount` and judged a per-item error boundary as though it were the route's own. + // + // Only an iteration callback is a boundary, not every function-like node: a route's own body + // wrapped in `trace(async () => {...})`, `mutateWithFallback({ pgMutation: async (t) => {...} })` + // or `new ReadableStream({ start: async (c) => {...} })` still runs exactly once, as the route's + // own continuation one layer of nesting away, and its catch is the route's own error handling. const visit = (node: ts.Node, inCatch: boolean, inCallback: boolean) => { if (ts.isFunctionLike(node)) { - ts.forEachChild(node, (child) => visit(child, inCatch, true)); + const entersIterationCallback = inCallback || isIterationCallback(node); + ts.forEachChild(node, (child) => visit(child, inCatch, entersIterationCallback)); return; } if (ts.isTryStatement(node)) { diff --git a/internal-packages/observability-map/test/scan.test.ts b/internal-packages/observability-map/test/scan.test.ts index 0f0151df9d7..aafc72438db 100644 --- a/internal-packages/observability-map/test/scan.test.ts +++ b/internal-packages/observability-map/test/scan.test.ts @@ -636,6 +636,67 @@ describe("scanFile: catch clause evidence", () => { expect(ep!.catches[0]!.branches).toBe(false); }); + // C2. `reachableStatements` only handled dead code from statement ordering (A4), not a + // statically-false condition, and `catchClauseEvidence`'s walk descended into every function + // body unconditionally, so a throw or an error test merely REGISTERED in a callback the clause + // constructs (never executed as part of the clause's own synchronous handling) was credited to + // it. All four shapes below must leave a plain swallow (`catch (e) { return null; }`) inert. + describe("dead and deferred code inside a catch does not count as evidence", () => { + const swallow = (mutation: string) => ` + export async function loader() { + try { + return await prisma.thing.findMany(); + } catch (e) { + ${mutation} + return null; + } + } + `; + + it("is inert as a baseline with no mutation", () => { + const ep = scanFile("x.ts", swallow("")); + expect(ep!.catches[0]).toMatchObject({ rethrows: false, branches: false }); + }); + + it("does not set rethrows for a throw inside a statically-false if", () => { + const ep = scanFile("x.ts", swallow("if (false) { throw e; }")); + expect(ep!.catches[0]).toMatchObject({ rethrows: false, branches: false }); + }); + + it("does not set rethrows for a throw inside a statically-false while", () => { + const ep = scanFile("x.ts", swallow("while (false) { throw e; }")); + expect(ep!.catches[0]).toMatchObject({ rethrows: false, branches: false }); + }); + + it("does not set rethrows for a throw merely registered in a constructed callback", () => { + const ep = scanFile("x.ts", swallow("queue.push(() => { throw e; });")); + expect(ep!.catches[0]).toMatchObject({ rethrows: false, branches: false }); + }); + + it("does not set branches for an error test merely registered in a constructed callback", () => { + const ep = scanFile( + "x.ts", + swallow("queue.push(() => { if (e instanceof Error) { doThing(); } });") + ); + expect(ep!.catches[0]).toMatchObject({ rethrows: false, branches: false }); + }); + + // Extra input beyond the brief's four, exercising the same "merely registered" mechanism with + // a different callback-taking call, to check the fix is not scoped to `.push` specifically. + it("does not set rethrows for a throw registered in a setTimeout callback", () => { + const ep = scanFile("x.ts", swallow("setTimeout(() => { throw e; }, 0);")); + expect(ep!.catches[0]).toMatchObject({ rethrows: false, branches: false }); + }); + + // Positive control: a do/while runs its body at least once regardless of the trailing + // condition, so a throw in one is genuinely unconditional and must still register. This checks + // the while(false) fix was not implemented broadly enough to swallow a real rethrow too. + it("still sets rethrows for a throw in a do/while, which runs its body once regardless", () => { + const ep = scanFile("x.ts", swallow("do { throw e; } while (false);")); + expect(ep!.catches[0]!.rethrows).toBe(true); + }); + }); + it("leaves both flags false when the catch only returns", () => { const ep = scanFile( "swallow.ts", @@ -799,35 +860,139 @@ describe("scanFile: catch clause evidence", () => { expect(ep!.catches).toEqual([]); }); - // A7. The catch collector descended into inline callbacks while `countStatement`, by design, - // does not, so a per-item error boundary inside a `.map()` was judged as the route's own catch - // and `tryStatementCount` could exceed the entry point's whole `statementCount`. Fixed by not - // descending into inline callbacks for catch evidence either, the smaller change: it matches the - // existing statement-counting decision rather than reopening the over-counting a callback walk - // was fixed for earlier. `calleeNames` and `logCalls` still descend, unchanged, which is what lets - // `isTrivial` see work a short statement count hides. - it("does not attribute a catch inside an inline callback (Promise.all(items.map(...))) to the route", () => { - const ep = scanFile( - "batch.process.ts", - ` - export async function action({ request }) { - const items = await loadItems(request); - await Promise.all( - items.map(async (item) => { + // A7 / C1. The first fix stopped at ANY function-like node, purely lexical, which correctly + // excludes a per-item `.map()` boundary but also deletes the route's own catch when the whole + // body is wrapped in a single-shot callback: `trace(async () => { ...whole body... })`, + // `mutateWithFallback({ pgMutation: async (t) => {...} })`, `new ReadableStream({ start: async (c) + // => {...} })`. All three invoke their callback exactly once, as the route's own continuation. + // The real distinction is per-item iteration versus everything else, so only a callback passed to + // `map`/`forEach`/`filter`/`reduce`/`reduceRight`/`flatMap`/`some`/`every` is a boundary now. + describe("inline single-shot wrappers are attributed to the route", () => { + it("attributes a catch wrapped in trace(async () => {...})", () => { + const ep = scanFile( + "x.ts", + ` + export async function action({ request }) { + return trace("update", async () => { + try { + await doWork(request); + return json({ ok: true }); + } catch { + return json({ error: "failed" }, { status: 500 }); + } + }); + } + ` + ); + expect(ep!.catches).toHaveLength(1); + }); + + it("attributes a catch inside a pgMutation callback passed as an object property", () => { + const ep = scanFile( + "x.ts", + ` + export async function action({ request }) { + const outcome = await mutateWithFallback({ + pgMutation: async (taskRun) => { + try { + await doWork(taskRun); + } catch { + return json({ error: "Internal Server Error" }, { status: 500 }); + } + }, + }); + return outcome; + } + ` + ); + expect(ep!.catches).toHaveLength(1); + }); + + it("attributes a catch inside new ReadableStream({ start })", () => { + const ep = scanFile( + "x.ts", + ` + export async function action({ request }) { + const stream = new ReadableStream({ + async start(controller) { + try { + await doWork(controller); + } catch { + controller.error("failed"); + } + }, + }); + return new Response(stream); + } + ` + ); + expect(ep!.catches).toHaveLength(1); + }); + }); + + describe("per-item iteration callbacks are not attributed to the route", () => { + it("does not attribute a catch inside items.map(...)", () => { + const ep = scanFile( + "x.ts", + ` + export async function action({ request }) { + const items = await load(request); + return items.map((item) => { try { - await processItem(item); + return process(item); } catch { return null; } - }) - ); - return json({ ok: true }); - } - ` - ); - expect(ep!.catches).toEqual([]); - // A try/catch appeared somewhere in the body, which is still a real signal for triviality. - expect(ep!.hasTryCatch).toBe(true); + }); + } + ` + ); + expect(ep!.catches).toEqual([]); + }); + + it("does not attribute a catch inside Promise.all(items.map(...))", () => { + const ep = scanFile( + "batch.process.ts", + ` + export async function action({ request }) { + const items = await loadItems(request); + await Promise.all( + items.map(async (item) => { + try { + await processItem(item); + } catch { + return null; + } + }) + ); + return json({ ok: true }); + } + ` + ); + expect(ep!.catches).toEqual([]); + // A try/catch appeared somewhere in the body, which is still a real signal for triviality. + expect(ep!.hasTryCatch).toBe(true); + }); + + it("does not attribute a catch inside items.forEach(...)", () => { + const ep = scanFile( + "x.ts", + ` + export async function action({ request }) { + const items = await load(request); + items.forEach((item) => { + try { + process(item); + } catch { + return; + } + }); + return json({ ok: true }); + } + ` + ); + expect(ep!.catches).toEqual([]); + }); }); }); From 255accbc4fa7e945c0d7500251f612f238d23ba0 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Sat, 1 Aug 2026 17:36:21 +0100 Subject: [PATCH 047/106] fix(observability-map): read suppression comments off the parsed source, not a bare scanner I3: a standalone ts.createScanner has no parser state, so it still granted suppressions nobody wrote in two shapes, both reproduced end to end: a template literal WITH a substitution is never rescanned as a continuation after the `}`, so text following it lexes as ordinary code and a `//` in it is a real comment; and a scanner created in LanguageVariant.Standard has no JSX context, so `//` inside JSX text (e.g. a URL in prose) lexes as a line comment. The prior docstring claimed a template literal was one token with "no substring rule left to fool", which was false for both shapes; this is the second docstring in a row on this function to overclaim a safety property, so this one describes the mechanism and names the residual instead. Replaced the scanner with comment ranges read off the actual parsed ts.SourceFile: getLeadingCommentRanges and getTrailingCommentRanges at every leaf token's boundary (walking .getChildren(), which reaches bare punctuation and keyword tokens that ts.forEachChild silently skips). A template's literal segments and a JSX text node are real nodes with their own span there, never trivia, so a directive inside either is content the parser already claimed. suppressedChecks now takes the fileName too, to pick ScriptKind.TSX vs TS: parsing every file as TSX would misparse a generic arrow function ((x) => x) as JSX in a plain .ts file. Verified against the two brief shapes plus four extra inputs: a second substitution, a string literal nested in a JSX expression container, a genuine directive in a .tsx file, and a generic arrow function in a .ts file with a real trailing comment, to check the script-kind change did not break ordinary parsing. Zero real-tree effect (no live suppressions exist today). --- .../observability-map/src/score.ts | 2 +- .../observability-map/src/suppression.ts | 97 ++++++++++++++----- .../test/suppression.test.ts | 72 +++++++++++++- 3 files changed, 141 insertions(+), 30 deletions(-) diff --git a/internal-packages/observability-map/src/score.ts b/internal-packages/observability-map/src/score.ts index a39a1ab7c82..66ed594e2d7 100644 --- a/internal-packages/observability-map/src/score.ts +++ b/internal-packages/observability-map/src/score.ts @@ -48,7 +48,7 @@ export type MapReport = { }; export function scoreEntry(ep: EntryPoint): ScoredEntry { - const suppressed = suppressedChecks(ep.source); + const suppressed = suppressedChecks(ep.source, ep.fileName); const raw = CHECKS.map((c) => c.run(ep)); const checks = raw.map((result) => { const reason = suppressed.get(result.id); diff --git a/internal-packages/observability-map/src/suppression.ts b/internal-packages/observability-map/src/suppression.ts index 6012ee09d4e..4740594746d 100644 --- a/internal-packages/observability-map/src/suppression.ts +++ b/internal-packages/observability-map/src/suppression.ts @@ -15,47 +15,92 @@ import ts from "typescript"; const PATTERN = /obs-map-disable\s+([a-z-]+)\s+--\s+(.+)/; /** - * Every physical line of genuine comment content in the source, line comments and block comments - * alike, one entry per line, with the `//`, `/*`, `*​/` and a jsdoc `*` prefix stripped. + * Every leaf token in the parsed source: keeps descending through `.getChildren()` rather than + * `ts.forEachChild`, which only returns the child nodes a statement or expression models as its + * own properties and silently skips a bare punctuation or keyword token (a closing brace, a + * semicolon). A comment can sit directly before one of those with nothing else following it, the + * last line inside a block, and `.getChildren()` still reaches it because the token itself is + * still a node with a position. + */ +function leafTokens(node: ts.Node): ts.Node[] { + const children = node.getChildren(); + return children.length === 0 ? [node] : children.flatMap(leafTokens); +} + +/** + * Every genuine comment range in the source, read off a real parsed `ts.SourceFile` rather than a + * standalone `ts.createScanner`. A bare scanner has no parser state behind it, and that is exactly + * what let two shapes through: + * + * - a template literal WITH a substitution never gets rescanned as a template continuation by a + * scanner running on its own, so the text after `${x}` reads as ordinary code and a `//` in it is + * a real comment to the scanner, though it never leaves the template literal to the parser. + * - JSX text has no comment syntax at all, but a scanner created in `LanguageVariant.Standard` + * does not know it is looking at JSX text, so a `//` inside `

see https://x

` reads as a + * line comment starting mid-URL. * - * Read from the TypeScript scanner's own token stream rather than `indexOf("//")` against the raw - * text. The old text-matching read the directive out of a string or template literal that merely - * quoted it, so `"see // obs-map-disable auth-boundary -- nope"` granted a suppression nobody - * wrote, silencing a real check. The scanner already knows the difference: a string or template - * literal is one token, consumed in a single step, and never yields comment trivia for what is - * inside it, so there is no substring rule left to fool. + * The actual parser closes both: a template's literal segments and a JSX text node are real nodes + * with their own span here, never trivia, so a directive inside either is content the parser + * already claimed, not a comment. `getLeadingCommentRanges` and `getTrailingCommentRanges` are both + * needed at every token boundary, because which one returns a given comment depends on whether it + * shares a line with the token before it (trailing) or comes after a line break (leading), not on + * which directive it happens to be. */ -function commentLines(source: string): string[] { - const scanner = ts.createScanner( - ts.ScriptTarget.Latest, - /* skipTrivia */ false, - ts.LanguageVariant.Standard, - source - ); - const lines: string[] = []; +function commentRanges(source: string, sf: ts.SourceFile): ts.CommentRange[] { + const seen = new Set(); + const ranges: ts.CommentRange[] = []; + const add = (found: ts.CommentRange[] | undefined) => { + for (const range of found ?? []) { + if (seen.has(range.pos)) continue; + seen.add(range.pos); + ranges.push(range); + } + }; + for (const token of leafTokens(sf)) { + add(ts.getLeadingCommentRanges(source, token.getFullStart())); + add(ts.getTrailingCommentRanges(source, token.getEnd())); + } + return ranges; +} - for (let kind = scanner.scan(); kind !== ts.SyntaxKind.EndOfFileToken; kind = scanner.scan()) { - if (kind === ts.SyntaxKind.SingleLineCommentTrivia) { - lines.push(scanner.getTokenText().slice(2)); +/** One physical line of comment content per range, the `//`, `/*`, `*​/` and a jsdoc `*` prefix + * stripped, so a multi-line block comment still matches the directive one line at a time. */ +function commentLines(source: string, sf: ts.SourceFile): string[] { + const lines: string[] = []; + for (const range of commentRanges(source, sf)) { + const text = source.slice(range.pos, range.end); + if (range.kind === ts.SyntaxKind.SingleLineCommentTrivia) { + lines.push(text.slice(2)); continue; } - if (kind !== ts.SyntaxKind.MultiLineCommentTrivia) continue; - - const text = scanner.getTokenText(); const body = text.slice(2, text.length - 2); // drop the leading /* and the closing */ for (const rawLine of body.split("\n")) { const trimmed = rawLine.trimStart(); lines.push(trimmed.startsWith("*") ? trimmed.slice(1) : rawLine); } } - return lines; } -/** Check id to reason. A suppression without a reason, or outside a comment, is ignored. */ -export function suppressedChecks(source: string): Map { +/** + * Check id to reason. A suppression without a reason, or outside a comment, is ignored. + * + * `fileName` picks the parser's script kind: JSX syntax is only legal, and only correctly + * distinguished from a generic type argument list (`(x) => x`), when the file is really a + * `.tsx`. Defaults to a plain `.ts` for callers that only have source text. + */ +export function suppressedChecks(source: string, fileName = "check.ts"): Map { + const scriptKind = fileName.endsWith(".tsx") ? ts.ScriptKind.TSX : ts.ScriptKind.TS; + const sf = ts.createSourceFile( + fileName, + source, + ts.ScriptTarget.Latest, + /* setParentNodes */ true, + scriptKind + ); + const out = new Map(); - for (const line of commentLines(source)) { + for (const line of commentLines(source, sf)) { const match = PATTERN.exec(line); if (!match) continue; const [, id, reason] = match; diff --git a/internal-packages/observability-map/test/suppression.test.ts b/internal-packages/observability-map/test/suppression.test.ts index 3259278a594..7ef32f9f94e 100644 --- a/internal-packages/observability-map/test/suppression.test.ts +++ b/internal-packages/observability-map/test/suppression.test.ts @@ -92,9 +92,8 @@ describe("suppressedChecks", () => { // A2. `indexOf("//")` against the raw text found the marker inside a string literal too, so a // string that merely quotes the directive granted a suppression nobody wrote and silenced a real - // check. Reading genuine comment ranges from the TypeScript scanner closes both shapes: the - // scanner consumes a string or template literal as one token and never emits comment trivia for - // what is inside it. + // check. Reading comment ranges off the parsed source closes this: a string literal is one node + // with its own span, never trivia, so a directive inside it is content, not a comment. it("does not suppress from a directive quoted inside a string literal", () => { const m = suppressedChecks( `const msg = "see // obs-map-disable error-classification -- because reasons"; @@ -118,4 +117,71 @@ describe("suppressedChecks", () => { ); expect(m.size).toBe(0); }); + + // I3. A standalone `ts.createScanner` has no parser state, so it still granted two suppressions + // nobody wrote: it never rescans a template as a continuation after a `${...}` substitution, so + // text after the `}` reads as ordinary code and a `//` in it is a real comment to the scanner; + // and a scanner created in `LanguageVariant.Standard` has no JSX context, so a `//` inside JSX + // text reads as a line comment mid-URL. Reading comments off the actual parsed tree closes both: + // a template's literal segments and a JSX text node are real nodes, never trivia. + it("does not suppress from a directive after a template substitution", () => { + const m = suppressedChecks( + "const msg = `${name} // obs-map-disable error-classification -- via substitution`;\n" + + "export async function loader() { return 1; }" + ); + expect(m.size).toBe(0); + }); + + it("does not suppress from a directive inside JSX text", () => { + const m = suppressedChecks( + `export default function Page() { + return

docs at https://example.com obs-map-disable error-classification -- x

; + }`, + "route.tsx" + ); + expect(m.size).toBe(0); + }); + + // Extra inputs beyond the brief's two, exercising the same "not a real parse position" mechanism + // differently: a second substitution, and a string literal nested inside a JSX expression + // container, which is a different node kind again from either hole above. + it("does not suppress from a directive after a second template substitution", () => { + const m = suppressedChecks( + "const msg = `${a}${b} // obs-map-disable auth-boundary -- nested substitution`;\n" + + "export async function loader() { return 1; }" + ); + expect(m.size).toBe(0); + }); + + it("does not suppress from a directive inside a string literal nested in a JSX expression container", () => { + const m = suppressedChecks( + `export default function Page() { + return

{"see // obs-map-disable request-context -- nested string"}

; + }`, + "route.tsx" + ); + expect(m.size).toBe(0); + }); + + // Positive control: a genuine directive still works in a .tsx file, and a directive after a + // template with no substitution (already covered above) is not the only shape that must survive. + it("still reads a genuine directive in a .tsx file", () => { + const m = suppressedChecks( + `// obs-map-disable error-classification -- liveness probe + export default function Page() { return

hi

; }`, + "route.tsx" + ); + expect(m.get("error-classification")).toBe("liveness probe"); + }); + + // Regression control: a generic arrow function is only unambiguous when the file is parsed as + // plain TypeScript, not TSX (`` would otherwise start a JSX element). A .ts file must still + // parse sanely and keep reading a genuine trailing comment correctly. + it("still reads a genuine directive beside a generic arrow function in a .ts file", () => { + const m = suppressedChecks( + "const identity = (x: T): T => x; // obs-map-disable auth-boundary -- generic helper\n" + + "export async function loader() { return identity(1); }" + ); + expect(m.get("auth-boundary")).toBe("generic helper"); + }); }); From fc73a2e485c04353f4ad1d2586fdc1c2cd401088 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Sat, 1 Aug 2026 17:43:08 +0100 Subject: [PATCH 048/106] fix(observability-map): make every published figure pre-suppression I4: measured was fixed to read pre-suppression applicability, but contextGap and auditGap still read `checks`, the post-suppression display view, so suppressing the one request-context finding on an entry shrank CONTEXT's denominator too and raised its printed percentage, on the same screen as the SUPPRESSED line's claim that suppression cannot raise a score. Reproduced exactly: two routes, one failing request-context, suppressing it moved CONTEXT from 1 of 2 (50%) to 1 of 1 (100%). Second half: audit-trail is not in SCORED_CHECK_IDS, so `suppressed` never recorded an audit-trail suppression, so a written directive silently disappeared from the SUPPRESSED line while the audit denominator shrank underneath it, confirmed with a standalone audit-trail suppression: old code counted zero suppressions while auditGap still moved 2/1 to 1/1. ScoredEntry now carries rawChecks, every check exactly as it ran before a suppression can turn a result into not-applicable. contextGap and auditGap both read it instead of `checks`. `suppressed` now reads every check a directive named, scored or not, so audit-trail suppressions are counted too. Also collapsed three copies of the same suppression/denominator explanation (M10) down to one, on the field that actually needs it. Zero real-tree effect (no live suppressions exist today); both repros verified directly against old and new code side by side. --- .../observability-map/src/score.ts | 36 ++++++---- .../observability-map/test/score.test.ts | 71 +++++++++++++++++++ 2 files changed, 93 insertions(+), 14 deletions(-) diff --git a/internal-packages/observability-map/src/score.ts b/internal-packages/observability-map/src/score.ts index 66ed594e2d7..7a5c6924bda 100644 --- a/internal-packages/observability-map/src/score.ts +++ b/internal-packages/observability-map/src/score.ts @@ -9,7 +9,15 @@ export type ScoredEntry = { routePath: string; family: Family; sensitive: boolean; + /** Post-suppression: a suppressed check reads `not-applicable` here, with the reason in + * `detail`. This is the display view; every denominator below reads `rawChecks` instead, so a + * suppression is never invisible to a published figure just because its check is not scored. */ checks: CheckResult[]; + /** Every check exactly as it ran, before a suppression comment can turn a result into + * `not-applicable`. The one true source for any figure that counts applicability: `measured` + * below, and `contextGap`/`auditGap` in `MapReport`, which read this rather than `checks` for + * exactly that reason. */ + rawChecks: CheckResult[]; /** * Whether at least one scored check (`SCORED_CHECK_IDS`, so never `audit-trail`) was applicable * before suppression. A fully-suppressed entry stays measured, at its capped score, so a @@ -19,7 +27,8 @@ export type ScoredEntry = { * default cannot inflate a figure nobody checked. */ measured: boolean; - /** Scored checks a comment in the source suppressed, in `SCORED_CHECK_IDS` order. */ + /** Every check a comment in the source suppressed, scored or not, in `CHECKS` order. Includes + * `audit-trail`: a suppression is real regardless of whether its check feeds the score. */ suppressed: string[]; /** Passed over applicable, across scored checks only. 100 when nothing applies. */ score: number; @@ -67,10 +76,6 @@ export function scoreEntry(ep: EntryPoint): ScoredEntry { }; const visible = scored.filter((c) => !suppressed.has(c.id)); - // Pre-suppression: an entry whose only applicable check gets suppressed still had something to - // measure, and must stay in the denominator at its capped score rather than vanish as if nothing - // ever applied. `unmeasured` is reserved for entries with no applicable check at all, suppression - // or no suppression. const scoredApplicable = scored.filter((c) => c.status !== "not-applicable"); return { @@ -79,13 +84,12 @@ export function scoreEntry(ep: EntryPoint): ScoredEntry { family: familyOf(ep.fileName), sensitive: classifySensitivity(ep).sensitive, checks, - suppressed: scored.filter((c) => suppressed.has(c.id)).map((c) => c.id), - // Suppressing a check takes it out of the numerator and the denominator, and the result is - // capped by what the entry would have scored unsuppressed. Otherwise removing a failing check - // shrinks the denominator and the ratio climbs, which is how 33 became 50 became 100: the - // suppression comment laundered the finding into a point. What a suppression buys is removal - // from the worklist, with a reason on the record. It cannot buy a better number. + rawChecks: raw, + suppressed: raw.filter((c) => suppressed.has(c.id)).map((c) => c.id), measured: scoredApplicable.length > 0, + // Capped by the pre-suppression ratio: removing a failing check from both the numerator and + // the denominator otherwise raises the ratio, which is how 33 became 50 became 100 before this + // cap existed. See ScoredEntry.measured for why the denominator itself is pre-suppression too. score: Math.min(ratio(visible), ratio(scored)), }; } @@ -131,12 +135,16 @@ export function buildReport(eps: EntryPoint[], parseFailures: string[]): MapRepo // reported here as its own architectural figure instead: how many sensitive mutations have an // audit record, out of how many. Folding it into the score would tank every sensitive route on a // gap that is the same everywhere, and bury the routes that have their own, fixable problems. + // + // Both gaps read `rawChecks`, pre-suppression, the same reason `measured` does: suppressing the + // one request-context or audit-trail finding on an entry must not shrink these denominators and + // raise the printed percentage, on the same screen as a claim that suppression cannot do that. const contextChecks = entries - .map((e) => e.checks.find((c) => c.id === "request-context")) + .map((e) => e.rawChecks.find((c) => c.id === "request-context")) .filter((c): c is CheckResult => c !== undefined && c.status !== "not-applicable"); const auditApplicable = entries.filter((e) => - e.checks.some((c) => c.id === "audit-trail" && c.status !== "not-applicable") + e.rawChecks.some((c) => c.id === "audit-trail" && c.status !== "not-applicable") ); const suppressing = entries.filter((e) => e.suppressed.length > 0); @@ -154,7 +162,7 @@ export function buildReport(eps: EntryPoint[], parseFailures: string[]): MapRepo auditGap: { sensitiveMutations: auditApplicable.length, withAudit: auditApplicable.filter((e) => - e.checks.some((c) => c.id === "audit-trail" && c.status === "pass") + e.rawChecks.some((c) => c.id === "audit-trail" && c.status === "pass") ).length, }, contextGap: { diff --git a/internal-packages/observability-map/test/score.test.ts b/internal-packages/observability-map/test/score.test.ts index 05386622993..c64c75bbed8 100644 --- a/internal-packages/observability-map/test/score.test.ts +++ b/internal-packages/observability-map/test/score.test.ts @@ -186,6 +186,77 @@ ${BUSY_AND_FAILING}` expect(report.suppressions).toEqual({ entries: 1, checks: 1 }); }); + // I4. contextGap and auditGap read `checks` (post-suppression), so suppressing the one failing + // request-context on an entry removed it from the denominator too, moving CONTEXT from 1 of 2 + // (50%) to 1 of 1 (100%) printed on the same screen as "a suppression does not raise a score". + // Both gaps now read `rawChecks`, pre-suppression, exactly like `measured`. + it("does not let a suppressed request-context finding shrink the context gap's denominator", () => { + const failing = `import { prisma } from "~/db.server"; +export async function loader() { + try { return await prisma.thing.findMany(); } catch (e) { return null; } +}`; + const passing = `import { logger } from "~/services/logger.server"; +import { prisma } from "~/db.server"; +export async function loader({ params }) { + try { return await prisma.thing.findMany(); } + catch (error) { logger.error("failed", { environmentId: params.envId, error }); throw error; } +}`; + + const before = buildReport([scanFile("a.ts", failing)!, scanFile("b.ts", passing)!], []); + expect(before.contextGap).toEqual({ applicable: 2, naming: 1 }); + + const after = buildReport( + [ + scanFile( + "a.ts", + `// obs-map-disable request-context -- silence +${failing}` + )!, + scanFile("b.ts", passing)!, + ], + [] + ); + expect(after.contextGap).toEqual({ applicable: 2, naming: 1 }); + }); + + // I4, second half. A suppressed audit-trail directive never showed up in `suppressed` because + // audit-trail is not in SCORED_CHECK_IDS, so the SUPPRESSED line undercounted while the audit + // denominator silently shrank underneath it. Every suppression is now counted, scored or not. + it("counts an audit-trail suppression and does not let it shrink the audit gap's denominator", () => { + const missingAudit = `import { prisma } from "~/db.server"; +export async function action() { return prisma.token.create({ data: {} }); }`; + const withAudit = `import { auditLog } from "~/services/audit.server"; +import { prisma } from "~/db.server"; +export async function action({ request }) { + const token = await prisma.token.create({ data: {} }); + await auditLog("token.created", { tokenId: token.id }); + return json(token); +}`; + + const before = buildReport( + [ + scanFile("api.v1.auth.tokens.ts", missingAudit)!, + scanFile("api.v1.auth.jwt.ts", withAudit)!, + ], + [] + ); + expect(before.auditGap).toEqual({ sensitiveMutations: 2, withAudit: 1 }); + + const after = buildReport( + [ + scanFile( + "api.v1.auth.tokens.ts", + `// obs-map-disable audit-trail -- accepted risk +${missingAudit}` + )!, + scanFile("api.v1.auth.jwt.ts", withAudit)!, + ], + [] + ); + expect(after.auditGap).toEqual({ sensitiveMutations: 2, withAudit: 1 }); + expect(after.suppressions).toEqual({ entries: 1, checks: 1 }); + }); + it("reports the audit gap separately from the score", () => { // A sensitive mutation with no audit record, but nothing else wrong: the audit gap is reported // as its own figure and must not pull the score down with it. From 9e300335efc8196ffe1e914041afb55c94c4ee5b Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Sat, 1 Aug 2026 17:51:29 +0100 Subject: [PATCH 049/106] fix(observability-map): shadow-check a binding from any enclosing scope, not just a nested one I5: referencesBinding's shadow logic only fires while searching downward from an if/switch's own condition, so it caught shadowing NESTED inside that condition (what the existing tests exercise) but not a scope that instead WRAPS the if: a for...of loop declaring the same name, or a nested catch clause with the same binding. Confirmed both credited a branch that should not count, plus two more of the same shape (for...in, a classic for with a let-declared shadow) and a destructured const inside a plain block. catchClauseEvidence now tracks shadowing as it descends, the same pattern `inCallback` already uses: once a scope re-declares the binding, a for/ for-of/for-in loop's own variable, a nested catch with the same name, or a block that declares it, everything nested inside stays shadowed and an if/ switch referencing the name there no longer sets branches. Positive controls confirm a genuine reference, including through a for-of loop with a DIFFERENT loop variable, still counts. Second bug found while fixing the first: the function-parameter and catch-clause shadow checks inside referencesBinding only recognised ts.isIdentifier, so a destructured parameter ({ error }) was invisible to them too, even directly inside an if's own condition (items.some(({ error }) => error > 0)), a shape C1/C2 do not otherwise cover. Added a recursive bindingDeclares helper, reused across the function-parameter check, the catch-clause check, declaresInScope, and a new declaresLoopVariable, so every declaration-shadow check in the file recognises a destructured name the same way. Zero real-tree effect, same as A3's original fix: reproduces synthetically, no live occurrence in the current tree. --- .../observability-map/src/scan.ts | 86 ++++++++++++++--- .../observability-map/test/scan.test.ts | 95 +++++++++++++++++++ 2 files changed, 166 insertions(+), 15 deletions(-) diff --git a/internal-packages/observability-map/src/scan.ts b/internal-packages/observability-map/src/scan.ts index a3bf3456363..d8b10b5da55 100644 --- a/internal-packages/observability-map/src/scan.ts +++ b/internal-packages/observability-map/src/scan.ts @@ -159,6 +159,21 @@ function containsInstanceOf(node: ts.Node): boolean { ); } +/** + * Whether a binding name pattern declares `target`, recursively: a plain `error`, a destructured + * `{ error }` (shorthand) or `{ code: error }` (renamed), an array pattern `[error]`, and any of + * those nested inside another. A destructured parameter or declaration re-declares the name just as + * completely as a plain one does, so a shadow check that only recognised `ts.isIdentifier` missed + * every destructured shape, function parameters and variable declarations alike. + */ +function bindingDeclares(name: ts.BindingName, target: string): boolean { + if (ts.isIdentifier(name)) return name.text === target; + for (const element of name.elements) { + if (!ts.isOmittedExpression(element) && bindingDeclares(element.name, target)) return true; + } + return false; +} + /** Whether `name` is declared by a var/let/const, function or class statement directly in this * statement list. Not recursive: a nested block's own declarations are handled when the walk * reaches that block. */ @@ -168,9 +183,7 @@ function declaresInScope(statements: readonly ts.Statement[], name: string): boo if (ts.isClassDeclaration(statement) && statement.name?.text === name) return true; if ( ts.isVariableStatement(statement) && - statement.declarationList.declarations.some( - (d) => ts.isIdentifier(d.name) && d.name.text === name - ) + statement.declarationList.declarations.some((d) => bindingDeclares(d.name, name)) ) { return true; } @@ -178,14 +191,29 @@ function declaresInScope(statements: readonly ts.Statement[], name: string): boo return false; } +/** Whether a `for`/`for...of`/`for...in` loop's own declared variable is `name`, so its body + * shadows the rest of the enclosing scope for that name. */ +function declaresLoopVariable( + node: ts.ForStatement | ts.ForOfStatement | ts.ForInStatement, + name: string +): boolean { + const initializer = node.initializer; + return ( + initializer !== undefined && + ts.isVariableDeclarationList(initializer) && + initializer.declarations.some((d) => bindingDeclares(d.name, name)) + ); +} + /** * Whether `node` contains a genuine read of the given catch binding, e.g. `e` in `e instanceof X` * or `error.code`. An identifier only counts when it is a real reference. Two shapes share the * binding's text without reading it: the property side of a member expression (`fallback.error`) * and an object literal key (`{ error: true }`), both excluded by checking which side of the * parent node the identifier sits on. A name re-declared in a nested scope, as a function or catch - * parameter or as a var/let/const/function/class in a block, refers to that declaration instead, - * so the walk stops at the boundary that re-declares it rather than crediting the outer binding. + * parameter (including a destructured one) or as a var/let/const/function/class in a block, refers + * to that declaration instead, so the walk stops at the boundary that re-declares it rather than + * crediting the outer binding. */ function referencesBinding(node: ts.Node, bindingName: string): boolean { if (ts.isIdentifier(node) && node.text === bindingName) { @@ -197,14 +225,14 @@ function referencesBinding(node: ts.Node, bindingName: string): boolean { if ( ts.isFunctionLike(node) && - node.parameters.some((p) => ts.isIdentifier(p.name) && p.name.text === bindingName) + node.parameters.some((p) => bindingDeclares(p.name, bindingName)) ) { return false; } if (ts.isCatchClause(node)) { const decl = node.variableDeclaration; - if (decl && ts.isIdentifier(decl.name) && decl.name.text === bindingName) return false; + if (decl && bindingDeclares(decl.name, bindingName)) return false; } if (ts.isBlock(node) && declaresInScope(node.statements, bindingName)) return false; @@ -291,40 +319,68 @@ function catchClauseEvidence(clause: ts.CatchClause): { rethrows: boolean; branc let branches = false; const bindingName = catchBindingName(clause); - const visit = (node: ts.Node) => { + // `shadowed` is true once the walk has passed a scope that re-declares `bindingName`: a nested + // catch clause with the same name, a `for`/`for...of`/`for...in` loop declaring it, or a block + // that does (reusing the same `declaresInScope` a bare block already checks). From there on, an + // `if`/`switch` that references the name textually is referencing the SHADOWING declaration, not + // this clause's own binding, so it must not count as this clause deciding anything. Never reset + // back to false, the same rule `inCallback` follows: once shadowed, everything nested inside is + // still shadowed. + const visit = (node: ts.Node, shadowed: boolean) => { if (ts.isFunctionLike(node)) return; + if (bindingName !== null && ts.isCatchClause(node)) { + const decl = node.variableDeclaration; + const shadowsHere = decl !== undefined && bindingDeclares(decl.name, bindingName); + ts.forEachChild(node, (child) => visit(child, shadowed || shadowsHere)); + return; + } + + if ( + bindingName !== null && + (ts.isForStatement(node) || ts.isForOfStatement(node) || ts.isForInStatement(node)) && + declaresLoopVariable(node, bindingName) + ) { + visit(node.statement, true); + return; + } + if (ts.isBlock(node) || ts.isCaseClause(node) || ts.isDefaultClause(node)) { - for (const statement of reachableStatements(node.statements)) visit(statement); + const shadowedHere = + shadowed || (bindingName !== null && declaresInScope(node.statements, bindingName)); + for (const statement of reachableStatements(node.statements)) visit(statement, shadowedHere); return; } if (ts.isThrowStatement(node)) rethrows = true; if ( + !shadowed && bindingName !== null && ((ts.isIfStatement(node) && referencesBinding(node.expression, bindingName)) || (ts.isSwitchStatement(node) && referencesBinding(node.expression, bindingName))) ) { branches = true; } - if (ts.isConditionalExpression(node) && selectsAnErrorPath(node, bindingName)) branches = true; + if (!shadowed && ts.isConditionalExpression(node) && selectsAnErrorPath(node, bindingName)) { + branches = true; + } // A statically-false condition makes the guarded statement dead code: it can set `branches` // above (the clause still decided to test the error, even if the arm never runs), but nothing // inside it can set `rethrows` or a nested `branches`, so it is not visited at all. if (ts.isIfStatement(node)) { - if (!isFalsyLiteral(node.expression)) visit(node.thenStatement); - if (node.elseStatement) visit(node.elseStatement); + if (!isFalsyLiteral(node.expression)) visit(node.thenStatement, shadowed); + if (node.elseStatement) visit(node.elseStatement, shadowed); return; } if (ts.isWhileStatement(node)) { - if (!isFalsyLiteral(node.expression)) visit(node.statement); + if (!isFalsyLiteral(node.expression)) visit(node.statement, shadowed); return; } - ts.forEachChild(node, visit); + ts.forEachChild(node, (child) => visit(child, shadowed)); }; - visit(clause.block); + visit(clause.block, false); return { rethrows, branches }; } diff --git a/internal-packages/observability-map/test/scan.test.ts b/internal-packages/observability-map/test/scan.test.ts index aafc72438db..92f6a093bdb 100644 --- a/internal-packages/observability-map/test/scan.test.ts +++ b/internal-packages/observability-map/test/scan.test.ts @@ -1579,3 +1579,98 @@ describe("scanFile: branches requires a genuine read of the binding, not a looka expect(ep!.catches[0]!.branches).toBe(true); }); }); + +// I5. The shadow check only fired inside `referencesBinding`'s own top-down search from an +// if/switch's condition, so it only caught shadowing NESTED inside that condition, exactly what +// the tests above exercise. A shadowing scope that instead WRAPS the if (a for-of loop, a nested +// catch with the same name) was invisible, because nothing walked up from the if to notice it. +// catchClauseEvidence now tracks shadowing as it descends, the same way `inCallback` tracks a +// callback boundary: once a scope re-declares the binding, everything nested inside stays shadowed. +describe("scanFile: a binding shadowed by an enclosing scope, not just a nested one", () => { + const swallow = (mutation: string) => ` + export async function loader() { + try { + return await prisma.thing.findMany(); + } catch (error) { + ${mutation} + return null; + } + } + `; + + it("does not credit an if inside a for...of loop that re-declares the binding", () => { + const ep = scanFile( + "x.ts", + swallow("for (const error of errors) { if (error.code === 1) { doThing(); } }") + ); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + it("does not credit an if inside a for...in loop that re-declares the binding", () => { + const ep = scanFile( + "x.ts", + swallow("for (const error in errorsByKey) { if (error) { doThing(); } }") + ); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + it("does not credit an if inside a classic for loop that re-declares the binding", () => { + const ep = scanFile( + "x.ts", + swallow("for (let error = 0; error < 10; error++) { if (error) { doThing(); } }") + ); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + it("does not credit an if inside a nested catch clause with the same binding name", () => { + const ep = scanFile( + "x.ts", + swallow("try { doWork(); } catch (error) { if (error) { doThing(); } }") + ); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + it("does not credit an if inside a block whose own destructured const shadows the binding", () => { + const ep = scanFile( + "x.ts", + swallow(` + if (attempt > 0) { + const { error } = computeSomething(); + if (error) { doThing(); } + } + `) + ); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + it("does not credit an if reading a destructured parameter inside the if's own condition", () => { + const ep = scanFile( + "x.ts", + swallow("if (items.some(({ error }) => error > 0)) { doThing(); }") + ); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + it("does not credit an if shadowed by an array-destructured declaration", () => { + const ep = scanFile( + "x.ts", + swallow("const [error] = getErrors();\n if (error) { doThing(); }") + ); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + // Positive controls: a genuine reference to the real binding must still be credited, including + // through an enclosing loop whose OWN variable has a different name. + it("still credits an if that genuinely reads the outer binding directly", () => { + const ep = scanFile("x.ts", swallow("if (error instanceof Error) { doThing(); }")); + expect(ep!.catches[0]!.branches).toBe(true); + }); + + it("still credits an if inside a for...of loop with a different loop variable", () => { + const ep = scanFile( + "x.ts", + swallow("for (const item of items) { if (error.code === item) { doThing(); } }") + ); + expect(ep!.catches[0]!.branches).toBe(true); + }); +}); From e2e9ab38ea84ac7a8008acd7ef22f567f4373fc4 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Sat, 1 Aug 2026 18:07:58 +0100 Subject: [PATCH 050/106] fix(observability-map): drop the parse-guard ratio for an absolute, unpaddable count I6: moving the denominator from the entry point to the enclosing function (A6) closed cross-body dilution but the rule was still a ratio, "unrelated statements dilute", wherever the padding lives. Reproduced: padding the SAME action with 11 inert const statements after the try relabelled the identical broad otel.v1.logs.ts-shaped swallow from fail to pass, with zero change to the clause itself. Went back to the absolute rule this codebase already had and hand-read once, before the ratio replaced it: NARROW_TRY_STATEMENTS = 2, removed earlier only because it had become dead code once isParseGuard switched to the ratio, never because the count itself was found wrong. isParseGuard is now guardsParse && tryStatementCount <= 2, over the try block alone. Nothing outside the try can move that number, in the same body or another. Removed CatchEvidence.enclosingStatementCount, dead now that nothing reads it. Measured on the real tree: 7 clauses stop counting as a narrow guard; 2 of those move the overall verdict (5 have other evidence keeping the route's verdict where it was). Both hand-read: bulk-actions/route.tsx pass to not-applicable, a 9-statement catch that does a full presenter call and RBAC check was wrongly excused as a narrow guard and only rethrows, so it is correctly inert now, not credited; stream-basin.ts pass to fail, a 6-statement zod-validation catch that swallows a parse failure with no classification is correctly a swallow, not a guard. Global unchanged at 15. --- .../src/checks/errorClassification.ts | 31 ++++--- .../observability-map/src/scan.ts | 4 +- .../observability-map/src/types.ts | 8 -- .../observability-map/test/checks.test.ts | 91 +++++++++++++++++++ .../observability-map/test/scan.test.ts | 2 - 5 files changed, 112 insertions(+), 24 deletions(-) diff --git a/internal-packages/observability-map/src/checks/errorClassification.ts b/internal-packages/observability-map/src/checks/errorClassification.ts index 7d5f24f5dc0..6bdfc661474 100644 --- a/internal-packages/observability-map/src/checks/errorClassification.ts +++ b/internal-packages/observability-map/src/checks/errorClassification.ts @@ -26,20 +26,29 @@ const BUILDERS = new Set([ ]); /** - * Whether a catch clause is a guard rather than the route's error handling: it wraps a parse and - * covers less than half the statements of the body it is actually in. Both halves matter. - * `guardsParse` alone lets `otel.v1.logs.ts` off, whose catch covers 15 of its 18 statements and - * merely happens to contain a `request.json()`, and that is a real swallow. The coverage test is - * relative to the enclosing body rather than a second absolute threshold, so it holds for a - * three-statement route and a fifty-statement one alike. + * How much a try block may guard and still count as narrow. Two, so the guarded operation can bind + * its result (`const stripped = ...; new RegExp(stripped);`), but a third statement means the try + * has started to cover the handler rather than one operation. The idiom this was chosen for and + * hand-read against originally: 55 of 427 entry points, 11 of the failures at the time, all eleven + * the deliberate `try { body = await request.json() } catch { 400 }` shape. * - * Compared against `clause.enclosingStatementCount`, never `ep.statementCount`: the entry point's - * total sums the loader, the action and every one-hop helper together, so an unrelated sibling - * handler or a fat helper in the same file diluted the denominator and relabelled a broad swallow - * as a narrow parse guard, with no change to the clause itself. + * An absolute count, not a ratio against the enclosing body. A ratio is diluted by anything else in + * the same body: padding the action with unrelated statements after the try relabelled the same + * broad swallow as a narrow guard, moving the denominator without touching the clause at all. An + * absolute count over the try block alone cannot be diluted by anything outside it. + */ +const NARROW_TRY_STATEMENTS = 2; + +/** + * Whether a catch clause is a guard rather than the route's error handling: it wraps a parse and + * holds at most `NARROW_TRY_STATEMENTS`. Both halves matter. `guardsParse` alone lets + * `otel.v1.logs.ts` off, whose catch covers 7 statements and merely happens to contain a + * `request.json()`, and that is a real swallow; a validating zod `.safeParse` followed by an issue + * check and a bespoke error response is real handling too, not a bind-and-return guard, and stays + * excluded at three statements or more for the same reason. */ function isParseGuard(clause: CatchEvidence): boolean { - return clause.guardsParse && clause.tryStatementCount * 2 < clause.enclosingStatementCount; + return clause.guardsParse && clause.tryStatementCount <= NARROW_TRY_STATEMENTS; } /** diff --git a/internal-packages/observability-map/src/scan.ts b/internal-packages/observability-map/src/scan.ts index d8b10b5da55..77246ca082a 100644 --- a/internal-packages/observability-map/src/scan.ts +++ b/internal-packages/observability-map/src/scan.ts @@ -754,8 +754,7 @@ export function scanFile(fileName: string, source: string): EntryPoint | null { const helpers: EntryFunction[] = []; const walkBody = (fn: EntryFunction, followHelpers: boolean) => { - const enclosingStatementCount = countFunctionStatements(fn); - statementCount += enclosingStatementCount; + statementCount += countFunctionStatements(fn); if (!fn.body) return; // `inCallback` is true once the walk has entered a per-item iteration callback @@ -786,7 +785,6 @@ export function scanFile(fileName: string, source: string): EntryPoint | null { branches: clause.branches, guardsParse: guardsParse(node.tryBlock), tryStatementCount, - enclosingStatementCount, }); } } diff --git a/internal-packages/observability-map/src/types.ts b/internal-packages/observability-map/src/types.ts index bf6e6a79022..a1d3be0b2db 100644 --- a/internal-packages/observability-map/src/types.ts +++ b/internal-packages/observability-map/src/types.ts @@ -32,14 +32,6 @@ export type CatchEvidence = { guardsParse: boolean; /** Statements in the guarded try block, counted as `statementCount` counts them. */ tryStatementCount: number; - /** - * Statements in the body the clause actually sits in: the loader, the action, or the one-hop - * helper it delegates to, whichever function this `try` is directly inside. Never the entry - * point's combined total, which sums the loader, the action and every helper together: comparing - * a clause against that whole-entry-point figure let an unrelated sibling handler or a fat helper - * in the same file relabel a broad swallow as a narrow parse guard. - */ - enclosingStatementCount: number; }; /** A logging call made from a loader/action body, or from a same-file helper the body calls. */ diff --git a/internal-packages/observability-map/test/checks.test.ts b/internal-packages/observability-map/test/checks.test.ts index 11c606639e2..3187c916b6c 100644 --- a/internal-packages/observability-map/test/checks.test.ts +++ b/internal-packages/observability-map/test/checks.test.ts @@ -156,6 +156,48 @@ describe("error-classification", () => { expect(r.status).toBe("pass"); }); + // I6. NARROW_TRY_STATEMENTS is an absolute count over the try block alone (guardsParse still + // required), not a ratio against the enclosing body, so it holds the exact boundary regardless of + // how big or small the rest of the function is: two statements binds the parsed result and still + // passes, a third means the try has started to cover the handler and fails, even though both + // guard the same parse. + it("passes a parse guard that binds its result in exactly two statements", () => { + const r = run( + "error-classification", + "resources.pattern.ts", + `export async function action({ request }) { + let parsed; + try { + const raw = await request.text(); + parsed = new RegExp(raw); + } catch { + return json({ error: "Invalid pattern" }, { status: 400 }); + } + return json({ parsed: parsed.source }); + }` + ); + expect(r.status).toBe("pass"); + }); + + it("fails a parse guard that takes a third statement beyond binding the result", () => { + const r = run( + "error-classification", + "resources.pattern.ts", + `export async function action({ request }) { + let parsed; + try { + const raw = await request.text(); + const trimmed = raw.trim(); + parsed = new RegExp(trimmed); + } catch { + return json({ error: "Invalid pattern" }, { status: 400 }); + } + return json({ parsed: parsed.source }); + }` + ); + expect(r.status).toBe("fail"); + }); + // False positive fixture for the narrow rule: a narrow parse guard must not launder the broad // handler catch sitting next to it. Clauses are judged one at a time, so the broad one still // counts against the entry point. @@ -307,6 +349,55 @@ describe("error-classification", () => { expect(withSiblingAndHelper.status).toBe("fail"); }); + // I6. Moving the denominator from the entry point to the enclosing body (A6) closed + // cross-body dilution but not same-body dilution: the rule was still a ratio, "unrelated + // statements dilute", wherever the unrelated statements live. Padding the SAME action with 11 + // inert statements after the try relabelled the identical broad swallow from fail to pass. + // isParseGuard is now an absolute count over the try block alone (NARROW_TRY_STATEMENTS), + // which nothing outside the try can dilute, in the same body or another. + it("gives the same verdict to a byte-identical swallow whether or not it is padded with inert statements in the same body", () => { + const action = `import { otlpExporter } from "~/v3/otlpExporter.server"; + export async function action({ request }) { + try { + const exporter = await otlpExporter; + const contentType = request.headers.get("content-type"); + const body = await request.json(); + const result = await exporter.exportLogs(body); + const encoded = encodeResponse(result); + const headers = buildHeaders(contentType); + return new Response(encoded, { status: 200, headers }); + } catch (error) { + console.error(error); + return new Response("Internal Server Error", { status: 500 }); + } + }`; + + const padding = Array.from({ length: 11 }, (_, i) => `const pad${i} = ${i};`).join("\n"); + const paddedInSameBody = `import { otlpExporter } from "~/v3/otlpExporter.server"; + export async function action({ request }) { + try { + const exporter = await otlpExporter; + const contentType = request.headers.get("content-type"); + const body = await request.json(); + const result = await exporter.exportLogs(body); + const encoded = encodeResponse(result); + const headers = buildHeaders(contentType); + return new Response(encoded, { status: 200, headers }); + } catch (error) { + console.error(error); + return new Response("Internal Server Error", { status: 500 }); + } + ${padding} + return new Response("unreachable", { status: 200 }); + }`; + + const alone = run("error-classification", "otel.v1.logs.ts", action); + const padded = run("error-classification", "otel.v1.logs.ts", paddedInSameBody); + + expect(alone.status).toBe("fail"); + expect(padded.status).toBe("fail"); + }); + // A3. `referencesBinding` used to match any identifier with the binding's text, including a // property name in a member expression. A catch whose only `if` tests `fallback.error`, never the // caught binding itself, was credited with classifying an error it never inspected. diff --git a/internal-packages/observability-map/test/scan.test.ts b/internal-packages/observability-map/test/scan.test.ts index 92f6a093bdb..4d1c56fb917 100644 --- a/internal-packages/observability-map/test/scan.test.ts +++ b/internal-packages/observability-map/test/scan.test.ts @@ -1078,7 +1078,6 @@ describe("scanFile: per-catch evidence", () => { branches: false, guardsParse: true, tryStatementCount: 1, - enclosingStatementCount: 10, }); expect(ep!.catches[1]).toMatchObject({ guardsParse: false, @@ -1180,7 +1179,6 @@ describe("scanFile: per-catch evidence", () => { branches: false, guardsParse: false, tryStatementCount: 4, - enclosingStatementCount: 6, }); }); From 4e4dd812d6a01d047b7982f6b22bd4f218d54756 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Sat, 1 Aug 2026 18:15:46 +0100 Subject: [PATCH 051/106] fix(observability-map): tighten the minors (level set, bare env, a dead test, stale figures) M7: the qualifying level set claimed fatal and implicitly trace, neither of which the real logger (packages/core/src/logger.ts) has. It has log, error, warn, info, debug, verbose, and log is level 0, the one TRIGGER_LOG_LEVEL never filters out, so it belongs in the set and was wrongly excluded. QUALIFYING_LEVELS is now { log, error, warn }, correcting the docstring against the actual level list rather than an assumed one. M8: TENANT_FIELD accepted a bare env, so { env: process.env.NODE_ENV } would have read as naming a tenant. env and org now require a real suffix (envId, orgSlug); the full words (environment, organization, project, user) still qualify bare, since those are unambiguous on their own. M9: deleted a test asserting suppressed.score <= plain.score for a suppressed passing check. Removing a passing (maximal) result from a ratio can only lower or hold it, at the entry level and the aggregate level, with or without A1's fix, so the assertion passed unchanged against the pre-fix code and could not discriminate anything. The two tests beside it already exercise "does not raise the score" with real discriminating power, on the failing-check direction that actually distinguishes old code from new. M10: score.ts had a third stale "391 of 412" request-context figure (MapReport.contextGap's docstring) missed when A1's terminal.ts and report.test.ts occurrences were corrected; now 401, matching every other copy in the package. Zero real-tree effect from M7/M8, confirmed directly. Full suite green. --- .../src/checks/requestContext.ts | 29 +++++--- .../observability-map/src/score.ts | 2 +- .../observability-map/test/checks.test.ts | 74 +++++++++++++++++++ .../observability-map/test/score.test.ts | 27 ++----- 4 files changed, 100 insertions(+), 32 deletions(-) diff --git a/internal-packages/observability-map/src/checks/requestContext.ts b/internal-packages/observability-map/src/checks/requestContext.ts index f2d37aa600e..f54dc8d5f9e 100644 --- a/internal-packages/observability-map/src/checks/requestContext.ts +++ b/internal-packages/observability-map/src/checks/requestContext.ts @@ -6,23 +6,30 @@ const ID = "request-context"; /** * A field name that plausibly names a TENANT: environment, organization, project or user, the four * things every entry point ultimately belongs to. Anchored on the root word, not just the suffix, - * in the full and abbreviated camelCase the webapp actually writes for each: - * `environmentId`/`envId`, `organizationId`/`organizationSlug`/`orgId`, `projectId`/`projectParam`, - * `userId`. A bare `id`, and a resource id that happens to share the same `Id`/`Param` suffix, - * `batchId`, `notificationId`, `chatId`, `spanParam`, `runFriendlyId`, `taskIdentifier`, does not - * qualify: those name a resource the failure touched, not who it happened to. + * in the full and abbreviated camelCase the webapp actually writes for each: `environmentId`/ + * `envId`, `organizationId`/`organizationSlug`/`orgId`, `projectId`/`projectParam`, `userId`. A bare + * `id`, and a resource id that happens to share the same `Id`/`Param` suffix, `batchId`, + * `notificationId`, `chatId`, `spanParam`, `runFriendlyId`, `taskIdentifier`, does not qualify: + * those name a resource the failure touched, not who it happened to. + * + * The abbreviated roots, `env` and `org`, require a suffix; the full words do not. A bare `env` is + * ambiguous with a deployment environment name (`{ env: process.env.NODE_ENV }`), which is not a + * tenant, and nothing in the tree relies on it being bare, so the field alone cannot qualify. */ const TENANT_FIELD = - /^(environment|env|organization|org|project|user)(Id|Ids|Slug|Ref|Param|Identifier)?$/; + /^(environment|organization|project|user)(Id|Ids|Slug|Ref|Param|Identifier)?$|^(env|org)(Id|Ids|Slug|Ref|Param|Identifier)$/; /** - * error, warn and fatal are levels an incident is actually read at; debug and trace are routinely - * dropped or sampled out before anyone looks, so a tenant field logged only at that level is not - * really recorded on the failure path. info is deliberately excluded too: it is not reserved for + * Levels against the real logger (`packages/core/src/logger.ts`): `log`, `error`, `warn`, `info`, + * `debug`, `verbose`, in that order, no `fatal` and no `trace` (the `trace` in + * `apps/webapp/app/services/logger.server.ts` is the AsyncLocalStorage field helper, unrelated to + * log level). `log` is level 0, the level `TRIGGER_LOG_LEVEL` never filters out, so it qualifies + * alongside `error` and `warn`. `info`, `debug` and `verbose` do not: `info` is not reserved for * failure reporting, so a route can log an info line inside a catch that says nothing about the - * catch actually handling anything, and crediting it would launder the same gap `debug` closes. + * catch actually handling anything, and `debug`/`verbose` are routinely dropped or sampled out + * before anyone reads an incident. */ -const QUALIFYING_LEVELS = new Set(["error", "warn", "fatal"]); +const QUALIFYING_LEVELS = new Set(["log", "error", "warn"]); /** The level a `LogCall`'s callee was made at, e.g. `"error"` from `logger.error`. */ function logLevel(callee: string): string { diff --git a/internal-packages/observability-map/src/score.ts b/internal-packages/observability-map/src/score.ts index 7a5c6924bda..420a07b19f4 100644 --- a/internal-packages/observability-map/src/score.ts +++ b/internal-packages/observability-map/src/score.ts @@ -47,7 +47,7 @@ export type MapReport = { sensitiveCohort: { n: number; measured: number; mean: number | null }; auditGap: { sensitiveMutations: number; withAudit: number }; /** - * `request-context` fails 391 of the 412 entry points it applies to, so it is reported as a + * `request-context` fails 401 of the 412 entry points it applies to, so it is reported as a * figure rather than as hundreds of identical list entries, the same treatment `audit-trail` * gets. It stays fully in the score: the gap is real and the score is meant to show it. */ diff --git a/internal-packages/observability-map/test/checks.test.ts b/internal-packages/observability-map/test/checks.test.ts index 3187c916b6c..5a73eca50e5 100644 --- a/internal-packages/observability-map/test/checks.test.ts +++ b/internal-packages/observability-map/test/checks.test.ts @@ -989,6 +989,80 @@ describe("request-context", () => { ); expect(r.status).toBe("pass"); }); + + // M7. The real logger (packages/core/src/logger.ts) has log/error/warn/info/debug/verbose, no + // fatal and no trace, and log is level 0, never filtered by TRIGGER_LOG_LEVEL, so it must + // qualify. verbose is the actual noisiest level this codebase has, not trace. + it("passes a log-level failure log, which the real logger never filters", () => { + const r = run( + "request-context", + "api.v1.q.ts", + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function loader({ params }) { + try { return await prisma.thing.findMany(); } + catch (error) { + logger.log("lookup failed", { environmentId: params.envId, error }); + throw error; + } + }` + ); + expect(r.status).toBe("pass"); + }); + + it("does not pass a verbose-level failure log", () => { + const r = run( + "request-context", + "api.v1.q.ts", + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function loader({ params }) { + try { return await prisma.thing.findMany(); } + catch (error) { + logger.verbose("lookup failed", { environmentId: params.envId, error }); + throw error; + } + }` + ); + expect(r.status).toBe("fail"); + }); + + // M8. A bare `env` field is ambiguous with a deployment environment name + // (`{ env: process.env.NODE_ENV }`), not a tenant, so it must not qualify on its own; the + // abbreviated root still works with a real suffix. + it("does not pass a failure log that only names a bare env field", () => { + const r = run( + "request-context", + "api.v1.q.ts", + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function loader() { + try { return await prisma.thing.findMany(); } + catch (error) { + logger.error("lookup failed", { env: process.env.NODE_ENV, error }); + throw error; + } + }` + ); + expect(r.status).toBe("fail"); + }); + + it("still passes envId, the abbreviated root with a real suffix", () => { + const r = run( + "request-context", + "api.v1.q.ts", + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function loader({ params }) { + try { return await prisma.thing.findMany(); } + catch (error) { + logger.error("lookup failed", { envId: params.envId, error }); + throw error; + } + }` + ); + expect(r.status).toBe("pass"); + }); }); describe("audit-trail", () => { diff --git a/internal-packages/observability-map/test/score.test.ts b/internal-packages/observability-map/test/score.test.ts index c64c75bbed8..d0877736af1 100644 --- a/internal-packages/observability-map/test/score.test.ts +++ b/internal-packages/observability-map/test/score.test.ts @@ -321,26 +321,13 @@ ${BUSY_AND_FAILING}` expect(after.measured).toBe(2); }); - it("suppressing a passing check does not raise the score either", () => { - // error-classification branches on the error (pass); request-context never names a tenant - // (fail). Not sensitive, so auth-boundary sits out. - const source = `import { prisma } from "~/db.server"; -export async function loader() { - try { return await prisma.thing.findMany(); } - catch (e) { if (e instanceof Error) return null; throw e; } -}`; - const plain = scoreEntry(scanFile("api.v1.mixed.ts", source)!); - expect(plain.checks.find((c) => c.id === "error-classification")!.status).toBe("pass"); - - const suppressed = scoreEntry( - scanFile( - "api.v1.mixed.ts", - `// obs-map-disable error-classification -- silence a pass -${source}` - )! - ); - expect(suppressed.score).toBeLessThanOrEqual(plain.score); - }); + // M9. A prior version of this test asserted suppressed.score <= plain.score for a check that was + // passing, which the pre-existing per-entry Math.min cap already guarantees on its own: removing + // a passing (maximal) result from a ratio can only lower or hold it, at every level, with or + // without A1's fix, so the assertion passed unchanged against the pre-fix code and proved + // nothing about the aggregate mechanism A1 actually changed. Deleted rather than kept as + // decoration; "does not raise the score" for a failing suppression is exercised, with real + // discriminating power, by the two tests above. }); /** From 28c034db2161eb4efaadfa9173829448aadca1a1 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Sat, 1 Aug 2026 19:42:24 +0100 Subject: [PATCH 052/106] fix(observability-map): read a directive off a comment, not off JSX text getLeadingCommentRanges and getTrailingCommentRanges are raw lexers over source text from an offset and never consult the parse tree, so at a leaf token boundary they read the start of a JSX text node as a comment whenever it begins with // or /*. Three shapes suppressed a check that way, and the tree already contains one: resources.branches.create.tsx's // was read as a comment. A range whose start falls inside a JsxText, string literal, template segment or regex literal is now dropped. Measured over the whole route tree: 3344 comment ranges across 480 files, exactly one removed, the known bad one. A [fullStart, getStart) gap filter was tried instead and rejected, it loses same-line trailing comments and JSX expression containers. The docstring said a JSX text node is 'never trivia' to these two functions, which was false. It now says what the code does. --- .../observability-map/src/suppression.ts | 61 ++++++++++++++----- 1 file changed, 46 insertions(+), 15 deletions(-) diff --git a/internal-packages/observability-map/src/suppression.ts b/internal-packages/observability-map/src/suppression.ts index 4740594746d..26a51e13085 100644 --- a/internal-packages/observability-map/src/suppression.ts +++ b/internal-packages/observability-map/src/suppression.ts @@ -28,31 +28,62 @@ function leafTokens(node: ts.Node): ts.Node[] { } /** - * Every genuine comment range in the source, read off a real parsed `ts.SourceFile` rather than a - * standalone `ts.createScanner`. A bare scanner has no parser state behind it, and that is exactly - * what let two shapes through: + * Node kinds whose text the parser has already claimed as content, so nothing inside their span can + * be trivia however it is spelled. `getLeadingCommentRanges` and `getTrailingCommentRanges` are raw + * lexers over source text from an offset and consult no parse tree at all, so at a leaf-token + * boundary they will happily lex the inside of one of these as a comment: a JSX text node that + * BEGINS with `//` or `/*` is the shape that reached the real tree, in + * `resources.branches.create.tsx`'s `//`. * - * - a template literal WITH a substitution never gets rescanned as a template continuation by a - * scanner running on its own, so the text after `${x}` reads as ordinary code and a `//` in it is - * a real comment to the scanner, though it never leaves the template literal to the parser. - * - JSX text has no comment syntax at all, but a scanner created in `LanguageVariant.Standard` - * does not know it is looking at JSX text, so a `//` inside `

see https://x

` reads as a - * line comment starting mid-URL. + * `content-is-not-a-comment` in `test/suppression.test.ts` covers each kind, and + * `jsx-text-line-directive`, `jsx-text-after-expression` and `jsx-text-block-directive` in the + * mutation corpus cover the JSX shapes over the whole route tree. + */ +function isClaimedContent(node: ts.Node): boolean { + return ( + ts.isJsxText(node) || + ts.isStringLiteral(node) || + ts.isNoSubstitutionTemplateLiteral(node) || + ts.isTemplateHead(node) || + ts.isTemplateMiddle(node) || + ts.isTemplateTail(node) || + ts.isRegularExpressionLiteral(node) + ); +} + +/** + * Every comment range in the source, read off a real parsed `ts.SourceFile` rather than a + * standalone `ts.createScanner`, and then filtered against the spans above. * - * The actual parser closes both: a template's literal segments and a JSX text node are real nodes - * with their own span here, never trivia, so a directive inside either is content the parser - * already claimed, not a comment. `getLeadingCommentRanges` and `getTrailingCommentRanges` are both - * needed at every token boundary, because which one returns a given comment depends on whether it - * shares a line with the token before it (trailing) or comes after a line break (leading), not on - * which directive it happens to be. + * Both halves are needed. Parsing rather than scanning is what stops a template literal WITH a + * substitution being rescanned as ordinary code after `${x}`, and what makes JSX text a node at all. + * Filtering by span is what stops the two comment-range lexers reading the start of such a node as + * a comment anyway, which they do because they never see the tree the parser built. + * + * The filter is on the range's start offset falling inside a claimed span, not on the gap between a + * token's full start and its start. A gap filter was tried and rejected: it loses a same-line + * trailing comment and a comment inside a JSX expression container, both of which are real. + * + * Both lexers are called at every token boundary, because which one returns a given comment depends + * on whether it shares a line with the token before it (trailing) or comes after a line break + * (leading), not on which directive it happens to be. */ function commentRanges(source: string, sf: ts.SourceFile): ts.CommentRange[] { + const claimed: ts.TextRange[] = []; + const collectClaimed = (node: ts.Node) => { + if (isClaimedContent(node)) claimed.push({ pos: node.getStart(sf), end: node.end }); + ts.forEachChild(node, collectClaimed); + }; + collectClaimed(sf); + const inClaimedSpan = (pos: number) => claimed.some((s) => pos >= s.pos && pos < s.end); + const seen = new Set(); const ranges: ts.CommentRange[] = []; const add = (found: ts.CommentRange[] | undefined) => { for (const range of found ?? []) { if (seen.has(range.pos)) continue; seen.add(range.pos); + if (inClaimedSpan(range.pos)) continue; ranges.push(range); } }; From 68b838061606844a6a3a94aeb8f59387bc5597b9 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Sat, 1 Aug 2026 19:42:53 +0100 Subject: [PATCH 053/106] fix(observability-map): judge a catch on its own path, and stop excusing a hidden one Four changes to the catch evidence, each measured against the whole route tree. Global 15 before, 15 after; no route's score rose. rethrows and branches are now read off the clause's straight-line path: its statements, cut at the first definite exit, recursing into a bare block and a do body and nothing else. That replaces the list of statically-false shapes the previous round kept extending. The list recognised if (false) and while (false) and lost to for (;false;), if (true) else, switch (1) { case 2 }, try {} catch, for (const x of []), for (const k in {}), if (""), if (!true) and if (1 === 2), each worth 50 points a route. Asking for the throw to be unconditional refuses all eleven without naming any. A branch also has to send an arm somewhere the others do not go, by returning or throwing. if (e instanceof Error) { } was the cheapest no-op in the tool. The cost is three routes that decorate a Response and rethrow, which now read as inert rather than deciding: auth.github.ts, auth.google.ts and login.magic/route.tsx. A catch the boundary rule refuses is counted rather than dropped, and a route whose only catches were refused now fails instead of sitting out. Reading that as not-applicable was worth 50 points to anything that could get the rule to refuse the route's real catch, which Promise.all([0].map(async () => { whole body })) did. A single-element array literal receiver is also no longer read as iteration. Two routes move to fail, both genuine silent swallows: _app.orgs.$organizationSlug/route.tsx ignores JSON parse failures and api.v1.query.dashboards._index.ts drops unserializable dashboards. Statements inside inline callbacks now count towards statementCount, so wrapping a body in trace("x", async () => { ... }) no longer collapses the route to one statement and takes it out of the report as trivial. The parse-guard rule gains awaitsOnlyParse and countStatement counts declarators and comma operands rather than semicolons. Both close ways to make a broad handler look like a narrow guard: merging a seven-statement try into one declaration list, and writing the handler inside the try next to the parse. The design's suggestion of requiring a 4xx was measured first and is not used, it credits the widest swallows in the tree on its own and costs three real guards on top of the other conditions. No route moved from either change. --- .../src/checks/errorClassification.ts | 60 ++- .../observability-map/src/scan.ts | 343 +++++++++++------- .../observability-map/src/types.ts | 41 ++- .../observability-map/test/checks.test.ts | 178 ++++++++- .../observability-map/test/scan.test.ts | 66 +++- 5 files changed, 519 insertions(+), 169 deletions(-) diff --git a/internal-packages/observability-map/src/checks/errorClassification.ts b/internal-packages/observability-map/src/checks/errorClassification.ts index 6bdfc661474..b377e577cf2 100644 --- a/internal-packages/observability-map/src/checks/errorClassification.ts +++ b/internal-packages/observability-map/src/checks/errorClassification.ts @@ -34,21 +34,49 @@ const BUILDERS = new Set([ * * An absolute count, not a ratio against the enclosing body. A ratio is diluted by anything else in * the same body: padding the action with unrelated statements after the try relabelled the same - * broad swallow as a narrow guard, moving the denominator without touching the clause at all. An - * absolute count over the try block alone cannot be diluted by anything outside it. + * broad swallow as a narrow guard, moving the denominator without touching the clause at all. + * `inert-statements-after-try` in the mutation corpus is that shape, and it holds. + * + * What the count is NOT is unpaddable, which an earlier docstring and commit subject both claimed. + * `countStatement` now counts declarators and comma operands rather than semicolons, so the two + * known ways to pack a try into fewer statements move the number the same as writing it out; that + * is what `merge-declarations` and `merge-comma-expressions` in the corpus check. A third + * way nobody has written down would work, which is why the count is no longer the only condition + * and no longer the load-bearing one. */ const NARROW_TRY_STATEMENTS = 2; /** - * Whether a catch clause is a guard rather than the route's error handling: it wraps a parse and - * holds at most `NARROW_TRY_STATEMENTS`. Both halves matter. `guardsParse` alone lets - * `otel.v1.logs.ts` off, whose catch covers 7 statements and merely happens to contain a - * `request.json()`, and that is a real swallow; a validating zod `.safeParse` followed by an issue - * check and a bespoke error response is real handling too, not a bind-and-return guard, and stays - * excluded at three statements or more for the same reason. + * Whether a catch clause is a guard rather than the route's error handling: the try block parses, + * waits for nothing except that parse, and is short. + * + * `awaitsOnlyParse` is the condition the previous wave was missing, and it is the one a statement + * count cannot express. `try { const body = await request.json(); return await handleEverything(body); } + * catch { return 500; }` is two statements, one of them a parse, and the whole handler inside it: + * the count reads it as narrow and it is the `otel.v1.logs.ts` swallow written compactly. Asking + * what the block waits for separates them, and unlike the count it does not care how the statements + * are punctuated or how deeply the work is nested inside one of them. + * + * The design's own suggestion, requiring the clause to answer with a 4xx, was measured first and is + * not used. On its own it credits 11 clauses guarding four to thirty statements, the widest swallows + * in the tree, including `admin.api.v1.workers.ts`, whose 28-statement try answers every failure + * with a 400 carrying the internal error message. Added on top it costs three routes their pass, + * all three narrow parse guards that compute a fallback value rather than answering a request + * (`try { return new URL(referer).origin; } catch { return undefined; }`), and it buys only the case + * of a narrow parse guard answering 500. Requiring every CALL to be a parse, rather than every + * await, was measured too and is worse still: it refuses the four `matchPattern.slice(4); new + * RegExp(...)` guards, because preparing a parse's input is ordinary synchronous string work. + * + * The residual, since awaiting is the signal: a try block that does its non-parse work + * synchronously still reads as a guard. Nothing in the tree does, and it is written down in the + * round A fix 2 report rather than defended. */ function isParseGuard(clause: CatchEvidence): boolean { - return clause.guardsParse && clause.tryStatementCount <= NARROW_TRY_STATEMENTS; + return ( + clause.guardsParse && + clause.awaitsOnlyParse && + clause.tryStatementCount <= NARROW_TRY_STATEMENTS + ); } /** @@ -113,6 +141,13 @@ export function usesBuilder(ep: EntryPoint): boolean { * catch leaves `hasTryCatch` true and `catches` empty: nothing is swallowed there, the error * propagates once the cleanup has run, and reading the old flag as a catch put * `admin.api.v1.runs-replication.status.ts` at the top of the first rendered fix list. + * + * `callbackCatches` is the third case, and it is what stops "no catch is not-applicable" from being + * a payout. A route whose catches all sat inside a callback the scanner refused to attribute has + * error handling, the scanner just could not read it as the route's; excusing that is worth 50 + * points to anyone who wraps a body in something the boundary rule refuses, which + * `Promise.all([0].map(async () => { ... }))` did. It fails instead. The precision cost is a route + * that genuinely only handles errors per item, which now fails rather than sitting out. */ export const errorClassification = { id: ID, @@ -130,6 +165,13 @@ export const errorClassification = { detail: `catches its errors and takes one way out regardless of what was thrown${which}`, }; } + if (ep.catches.length === 0 && ep.callbackCatches > 0) { + return { + id: ID, + status: "fail", + detail: "its only error handling sits in a callback the route does not own", + }; + } if (!ep.catches.some(decides)) { return { id: ID, diff --git a/internal-packages/observability-map/src/scan.ts b/internal-packages/observability-map/src/scan.ts index 77246ca082a..da46cb4e54f 100644 --- a/internal-packages/observability-map/src/scan.ts +++ b/internal-packages/observability-map/src/scan.ts @@ -121,29 +121,64 @@ const PARSE_CALLEE = /(^|\.)(parse|safeParse|parseAsync|safeParseAsync|decode)$| */ const PARSE_CONSTRUCTORS = new Set(["URL", "URLSearchParams", "RegExp"]); +function isParseCall(node: ts.Node): boolean { + if (ts.isNewExpression(node)) { + return ts.isIdentifier(node.expression) && PARSE_CONSTRUCTORS.has(node.expression.text); + } + if (!ts.isCallExpression(node)) return false; + const text = calleeText(node.expression) ?? calleeName(node.expression); + return text !== null && PARSE_CALLEE.test(text); +} + /** - * Whether the guarded region parses something. A `new URL(x)` counts, and has to be read here as a - * `ts.isNewExpression`, because the call-callee scan that builds `calleeNames` never sees it. + * Body reads: the thing a parse guard waits for before it parses. `request.json()` is in + * `PARSE_CALLEE` already because it reads and parses in one call, and these are the same operation + * with the parse written separately, `const raw = await request.text(); new RegExp(raw);`. + * + * Only consulted for `awaitsOnlyParse`, never for `guardsParse`, which is what bounds it: a body + * read on its own still does not make a try block a parse guard, so the widest this list can do is + * let a block that already parses also read the thing it parses. */ -function guardsParse(tryBlock: ts.Block): boolean { - let found = false; +const BODY_READ_METHODS = new Set(["text", "formData", "arrayBuffer", "blob", "bytes"]); + +function isBodyRead(node: ts.Node): boolean { + if (!ts.isCallExpression(node)) return false; + const callee = unwrap(node.expression); + return ts.isPropertyAccessExpression(callee) && BODY_READ_METHODS.has(callee.name.text); +} + +/** + * What the guarded region does, in the two terms `error-classification` needs to tell a parse guard + * from a handler wrapped around a parse. + * + * `guardsParse` is whether anything in it parses at all. A `new URL(x)` counts and has to be read + * as a `ts.isNewExpression` here, because the call-callee scan that builds `calleeNames` never sees + * it. + * + * `awaitsOnlyParse` is whether everything the block waits for is a parse or a read of the body it + * parses. Awaiting is the signal, not calling: the calls that prepare a parse's input are ordinary + * synchronous string work (`matchPattern.startsWith("(?i)")`, `.slice(4)` before a `new RegExp`), + * and refusing those refuses four of the tree's clearest guards, while the swallow this has to + * catch reaches a service: `try { const body = await request.json(); return await + * handleEverything(body); }`. + * + * Nested function bodies are skipped: a callback written inside the try is not work the try is + * guarding on this pass through. + */ +function guardedWork(tryBlock: ts.Block): { guardsParse: boolean; awaitsOnlyParse: boolean } { + let guardsParse = false; + let awaitsOnlyParse = true; const visit = (node: ts.Node) => { - if (found) return; - if ( - ts.isNewExpression(node) && - ts.isIdentifier(node.expression) && - PARSE_CONSTRUCTORS.has(node.expression.text) - ) { - found = true; - } - if (ts.isCallExpression(node)) { - const text = calleeText(node.expression) ?? calleeName(node.expression); - if (text !== null && PARSE_CALLEE.test(text)) found = true; + if (ts.isFunctionLike(node)) return; + if (isParseCall(node)) guardsParse = true; + if (ts.isAwaitExpression(node)) { + const awaited = unwrap(node.expression); + if (!isParseCall(awaited) && !isBodyRead(awaited)) awaitsOnlyParse = false; } ts.forEachChild(node, visit); }; visit(tryBlock); - return found; + return { guardsParse, awaitsOnlyParse }; } /** Whether some node in the tree rooted at `node` matches `predicate`. */ @@ -191,20 +226,6 @@ function declaresInScope(statements: readonly ts.Statement[], name: string): boo return false; } -/** Whether a `for`/`for...of`/`for...in` loop's own declared variable is `name`, so its body - * shadows the rest of the enclosing scope for that name. */ -function declaresLoopVariable( - node: ts.ForStatement | ts.ForOfStatement | ts.ForInStatement, - name: string -): boolean { - const initializer = node.initializer; - return ( - initializer !== undefined && - ts.isVariableDeclarationList(initializer) && - initializer.declarations.some((d) => bindingDeclares(d.name, name)) - ); -} - /** * Whether `node` contains a genuine read of the given catch binding, e.g. `e` in `e instanceof X` * or `error.code`. An identifier only counts when it is a real reference. Two shapes share the @@ -248,8 +269,8 @@ function catchBindingName(clause: ts.CatchClause): string | null { /** * Whether a conditional expression tests the error to pick what the clause does, rather than to - * word what it says. It counts only when the whole `return`/`throw` is the conditional, so - * `return e instanceof Response ? e : json({}, { status: 500 })` counts and + * word what it says. The caller only offers it the whole value of a `return`/`throw`, so + * `return e instanceof Response ? e : json({}, { status: 500 })` reaches here and * `return json({ error: e instanceof Error ? e.message : String(e) }, { status: 400 })` does not. * The second is message formatting: every error leaves by the same path. * @@ -260,9 +281,7 @@ function catchBindingName(clause: ts.CatchClause): string | null { function selectsAnErrorPath(node: ts.ConditionalExpression, bindingName: string | null): boolean { if (bindingName === null) return false; if (!containsInstanceOf(node.condition)) return false; - if (!referencesBinding(node.condition, bindingName)) return false; - const parent = node.parent; - return parent !== undefined && (ts.isReturnStatement(parent) || ts.isThrowStatement(parent)); + return referencesBinding(node.condition, bindingName); } /** A statement that unconditionally leaves the statement list it sits in, so anything after it in @@ -288,110 +307,119 @@ function reachableStatements(statements: readonly ts.Statement[]): readonly ts.S return index === -1 ? statements : statements.slice(0, index + 1); } +/** Whether the tree rooted at `node` contains a `return` or a `throw` of its own, not counting one + * inside a nested function. What separates an arm that takes the error somewhere from an arm that + * runs and falls back into the clause's single common exit. */ +function containsExit(node: ts.Node): boolean { + if (ts.isFunctionLike(node)) return false; + if (ts.isReturnStatement(node) || ts.isThrowStatement(node)) return true; + return ts.forEachChild(node, containsExit) === true; +} + /** - * A literal that is always falsy: `false`, `null`, `undefined`, `0`. Not general constant folding, - * on purpose: `!true`, `1 === 2` and a reference to a `const` declared elsewhere are not covered, so - * `if (false) { throw e; }` and `while (false) { throw e; }` are recognised as dead and nothing more - * elaborate is claimed to be. + * Whether an `if`/`switch` sends at least one arm somewhere the others do not go, by returning or + * throwing from inside it. `if (e instanceof Error) { }` and `if (e instanceof Error) { log(e); }` + * both fail this: every error still leaves the clause by the same path afterwards, so the test + * changed the wording and not the outcome. The empty-body form was the cheapest no-op in the tool, + * worth 50 points a route; `empty-instanceof-if` in the mutation corpus is the tree-scale version. + * + * Two arms that return the SAME value pass this and should not. That residual is written down in + * the round A fix 2 report rather than defended: telling two returns apart needs the values + * compared, which is a different kind of analysis from anything else here. */ -function isFalsyLiteral(expr: ts.Expression): boolean { - if (expr.kind === ts.SyntaxKind.FalseKeyword || expr.kind === ts.SyntaxKind.NullKeyword) { - return true; +function selectsADistinctPath(statement: ts.IfStatement | ts.SwitchStatement): boolean { + if (ts.isIfStatement(statement)) { + return ( + containsExit(statement.thenStatement) || + (statement.elseStatement !== undefined && containsExit(statement.elseStatement)) + ); } - if (ts.isIdentifier(expr) && expr.text === "undefined") return true; - return ts.isNumericLiteral(expr) && expr.text === "0"; + return statement.caseBlock.clauses.some((clause) => clause.statements.some(containsExit)); } /** * What a catch clause does with the error, beyond the fact that it caught one. * - * Stops at every function-like node, not only an iteration callback (contrast `walkBody`'s - * boundary, which lets a route's own single-shot wrapper, `trace(async () => {...})`, - * `mutateWithFallback({ pgMutation })`, `new ReadableStream({ start })`, through so the route's own - * catch is found at all). Here the walk is already inside a catch clause that `walkBody` decided - * belongs to the route; anything the clause does by constructing a further callback, - * `queue.push(() => { throw e; })`, a `.then`, a `setTimeout`, is deferred work the clause merely - * registers, not a decision it makes on its own execution. Both walks refuse a per-item iteration - * callback; this one refuses every other kind of callback too, for that reason. + * Both answers are read off the clause's own straight-line path: the statements of its block, cut + * at the first one that definitely exits, recursing into a bare nested block and nothing else. A + * `throw` or a test that sits inside an `if`, a loop, a `switch`, a nested `try` or a callback is + * not on that path, so it does not count. + * + * That is the whole dead-code defence, and it replaces the list of statically-false shapes the + * previous round kept extending. The list was losing: `if (false)` and `while (false)` were + * recognised, and `for (;false;)`, `if (true) {} else`, `switch (1) { case 2: }`, `try {} catch`, + * `for (const x of [])`, `for (const k in {})`, `if ("")`, `if (!true)` and `if (1 === 2)` were not, + * each worth 50 points a route. Asking for the throw to be unconditional refuses all eleven without + * naming any of them, and refuses the twelfth nobody has written yet. `dead-*` in the mutation + * corpus is the tree-scale proof, one entry per shape. + * + * The cost is real: `catch (e) { if (transient) throw e; return null; }` no longer reads as a + * rethrow, so it reads as a swallow and fails rather than sitting out. That is the direction to be + * wrong in, since the reverse hands out points. */ function catchClauseEvidence(clause: ts.CatchClause): { rethrows: boolean; branches: boolean } { let rethrows = false; let branches = false; const bindingName = catchBindingName(clause); - // `shadowed` is true once the walk has passed a scope that re-declares `bindingName`: a nested - // catch clause with the same name, a `for`/`for...of`/`for...in` loop declaring it, or a block - // that does (reusing the same `declaresInScope` a bare block already checks). From there on, an - // `if`/`switch` that references the name textually is referencing the SHADOWING declaration, not - // this clause's own binding, so it must not count as this clause deciding anything. Never reset - // back to false, the same rule `inCallback` follows: once shadowed, everything nested inside is - // still shadowed. - const visit = (node: ts.Node, shadowed: boolean) => { - if (ts.isFunctionLike(node)) return; - - if (bindingName !== null && ts.isCatchClause(node)) { - const decl = node.variableDeclaration; - const shadowsHere = decl !== undefined && bindingDeclares(decl.name, bindingName); - ts.forEachChild(node, (child) => visit(child, shadowed || shadowsHere)); - return; - } - - if ( - bindingName !== null && - (ts.isForStatement(node) || ts.isForOfStatement(node) || ts.isForInStatement(node)) && - declaresLoopVariable(node, bindingName) - ) { - visit(node.statement, true); - return; - } - - if (ts.isBlock(node) || ts.isCaseClause(node) || ts.isDefaultClause(node)) { - const shadowedHere = - shadowed || (bindingName !== null && declaresInScope(node.statements, bindingName)); - for (const statement of reachableStatements(node.statements)) visit(statement, shadowedHere); - return; - } - - if (ts.isThrowStatement(node)) rethrows = true; - if ( - !shadowed && - bindingName !== null && - ((ts.isIfStatement(node) && referencesBinding(node.expression, bindingName)) || - (ts.isSwitchStatement(node) && referencesBinding(node.expression, bindingName))) - ) { - branches = true; - } - if (!shadowed && ts.isConditionalExpression(node) && selectsAnErrorPath(node, bindingName)) { - branches = true; - } + const walk = (statements: readonly ts.Statement[]) => { + // A block that re-declares the binding name means an `if` below it referencing that name is + // referencing the shadowing declaration, not this clause's error. Nothing in such a block can + // speak for the clause, so the whole list is skipped for branch purposes. + const shadowed = bindingName !== null && declaresInScope(statements, bindingName); - // A statically-false condition makes the guarded statement dead code: it can set `branches` - // above (the clause still decided to test the error, even if the arm never runs), but nothing - // inside it can set `rethrows` or a nested `branches`, so it is not visited at all. - if (ts.isIfStatement(node)) { - if (!isFalsyLiteral(node.expression)) visit(node.thenStatement, shadowed); - if (node.elseStatement) visit(node.elseStatement, shadowed); - return; - } - if (ts.isWhileStatement(node)) { - if (!isFalsyLiteral(node.expression)) visit(node.statement, shadowed); - return; + for (const statement of reachableStatements(statements)) { + if (ts.isThrowStatement(statement)) { + rethrows = true; + continue; + } + if (ts.isBlock(statement)) { + walk(statement.statements); + continue; + } + // A `do` body runs before its condition is ever read, so it is on the straight-line path + // whatever the condition says. The only loop form that is. + if (ts.isDoStatement(statement)) { + const body = statement.statement; + walk(ts.isBlock(body) ? body.statements : [body]); + continue; + } + if (bindingName === null || shadowed) continue; + + if ( + (ts.isIfStatement(statement) || ts.isSwitchStatement(statement)) && + referencesBinding(statement.expression, bindingName) && + selectsADistinctPath(statement) + ) { + branches = true; + continue; + } + if ( + (ts.isReturnStatement(statement) || ts.isThrowStatement(statement)) && + statement.expression !== undefined + ) { + const value = unwrap(statement.expression); + if (ts.isConditionalExpression(value) && selectsAnErrorPath(value, bindingName)) { + branches = true; + } + } } - - ts.forEachChild(node, (child) => visit(child, shadowed)); }; - visit(clause.block, false); + walk(clause.block.statements); return { rethrows, branches }; } /** - * Array methods that invoke their callback once per element, never once as a whole. The one - * structural signal that separates a per-item boundary (`items.map((item) => { try {...} })`, a - * fresh catch for every element) from a route's own body expressed through one more layer of - * function nesting (`trace(async () => {...})`, `mutateWithFallback({ pgMutation: async (t) => - * {...} })`, `new ReadableStream({ start: async (c) => {...} })`), all of which invoke their - * callback exactly once, as the route's own continuation. + * Method names that invoke their callback once per element, never once as a whole. The structural + * signal that separates a per-item boundary (`items.map((item) => { try {...} })`, a fresh catch + * for every element) from a route's own body expressed through one more layer of function nesting + * (`trace(async () => {...})`, `mutateWithFallback({ pgMutation: async (t) => {...} })`, + * `new ReadableStream({ start: async (c) => {...} })`), all of which invoke their callback exactly + * once. + * + * A name list, because nothing in a syntactic scan can tell `users.map` from `Result.map`. The + * consequence is written down where it matters, on `isIterationCallback`. */ const ITERATION_METHODS = new Set([ "map", @@ -404,14 +432,39 @@ const ITERATION_METHODS = new Set([ "every", ]); -/** Whether the function-like `node` is the callback argument of a call to one of - * `ITERATION_METHODS`, e.g. the arrow function in `items.map((item) => ...)`. */ +/** Whether an expression is an array literal of fewer than two elements, the one receiver shape + * that cannot be a per-item iteration however the method is named. */ +function isAtMostSingletonArray(expr: ts.Expression): boolean { + const target = unwrap(expr); + return ts.isArrayLiteralExpression(target) && target.elements.length < 2; +} + +/** + * Whether the function-like `node` is the callback argument of a per-item iteration, e.g. the arrow + * function in `items.map((item) => ...)`. + * + * Being wrong here is asymmetric. Calling a per-item callback the route's own continuation + * mis-attributes a per-element catch to the route, which was the bug the boundary was added for. + * Calling the route's own continuation a per-item callback hides the route's catch, and + * `error-classification` used to read a route with no catch as not-applicable, which is 50 points + * more than the swallow it was hiding. So the second direction paid, and `[0].map(async () => { + * whole body })` collected it. + * + * Two things changed. A receiver that is an array literal of one element or none is refused here, + * because it cannot iterate. And the direction that pays no longer pays: `walkBody` counts the + * catches it refuses, and `error-classification` fails a route whose only catches were refused + * rather than excusing it. So a wrong answer here costs precision, not points. That is what makes + * the name list survivable, and it is why `Result.map(...)`, which no name list can tell from + * `users.map(...)`, is a corpus entry that passes rather than a hole. + */ function isIterationCallback(node: ts.Node): boolean { const parent = node.parent; if (!parent || !ts.isCallExpression(parent)) return false; if (!parent.arguments.includes(node as ts.Expression)) return false; const callee = unwrap(parent.expression); - return ts.isPropertyAccessExpression(callee) && ITERATION_METHODS.has(callee.name.text); + if (!ts.isPropertyAccessExpression(callee)) return false; + if (!ITERATION_METHODS.has(callee.name.text)) return false; + return !isAtMostSingletonArray(callee.expression); } const HTTP_METHODS = new Set(["GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS"]); @@ -529,15 +582,42 @@ function resolveLocal(name: string, locals: LocalDeclarations, seen: Set return analyzeInitializer(local, locals, seen); } +/** + * How many operands a comma expression has, so `a(), b(), c()` is three and not one. Anything else + * is one. + */ +function commaOperands(expr: ts.Expression): number { + const target = unwrap(expr); + if (ts.isBinaryExpression(target) && target.operatorToken.kind === ts.SyntaxKind.CommaToken) { + return commaOperands(target.left) + commaOperands(target.right); + } + return 1; +} + /** * Statements in a statement, counting through block-bearing statements so a body wrapped in a * single `try` reports its real size. Does not descend into nested function bodies. + * + * Counts bindings and comma operands rather than semicolons, which is what makes the number mean + * something. `const a = f(), b = g(), c = h();` is three initializers however it is punctuated, and + * `a(), b(), c()` is three calls: scoring either as one let a seven-statement try be rewritten into + * a two-statement one with no change to what it runs, which took `error-classification` from fail + * to pass. `merge-declarations` and `merge-comma-expressions` in the mutation corpus are + * the tree-scale versions. */ function countStatement(statement: ts.Statement): number { if (ts.isBlock(statement)) { return countStatements(statement.statements); } + if (ts.isVariableStatement(statement)) { + return statement.declarationList.declarations.length; + } + + if (ts.isExpressionStatement(statement)) { + return commaOperands(statement.expression); + } + let count = 1; if (ts.isTryStatement(statement)) { @@ -742,6 +822,7 @@ export function scanFile(fileName: string, source: string): EntryPoint | null { let statementCount = 0; let hasTryCatch = false; + let callbackCatches = 0; const catches: CatchEvidence[] = []; const calleeNames: string[] = []; const logCalls: LogCall[] = []; @@ -759,31 +840,40 @@ export function scanFile(fileName: string, source: string): EntryPoint | null { if (!fn.body) return; // `inCallback` is true once the walk has entered a per-item iteration callback // (`items.map((item) => { ... })`), never reset back to false: nesting deeper inside one is - // still inside it. `calleeNames` and `logCalls` keep descending regardless, which is what lets - // `isTrivial` see work a short statement count hides. A try/catch does not: a per-item catch is - // not part of this body's own statement list, and `countStatement` already stops at a nested - // function boundary, so counting it here let `tryStatementCount` exceed the entry point's whole - // `statementCount` and judged a per-item error boundary as though it were the route's own. + // still inside it. `calleeNames` and `logCalls` keep descending regardless. A try/catch does + // not: a per-item catch is not part of this body's own statement list, and `countStatement` + // already stops at a nested function boundary, so counting it here let `tryStatementCount` + // exceed the entry point's whole `statementCount` and judged a per-item error boundary as + // though it were the route's own. What is refused is counted in `callbackCatches` instead of + // dropped, so a route whose only error handling was refused is failed rather than excused. // // Only an iteration callback is a boundary, not every function-like node: a route's own body // wrapped in `trace(async () => {...})`, `mutateWithFallback({ pgMutation: async (t) => {...} })` // or `new ReadableStream({ start: async (c) => {...} })` still runs exactly once, as the route's // own continuation one layer of nesting away, and its catch is the route's own error handling. + // + // A nested function's statements count towards `statementCount` too, whichever kind it is. + // They are work the route does, and leaving them out let `trace("x", async () => { whole body + // })` collapse a route to one statement, which is inside the triviality rule's limit: the route + // then read as trivial and every check reported not-applicable for it. `wrap-body-in-trace` in + // the mutation corpus is that shape. const visit = (node: ts.Node, inCatch: boolean, inCallback: boolean) => { if (ts.isFunctionLike(node)) { + if (isEntryFunction(node)) statementCount += countFunctionStatements(node); const entersIterationCallback = inCallback || isIterationCallback(node); ts.forEachChild(node, (child) => visit(child, inCatch, entersIterationCallback)); return; } if (ts.isTryStatement(node)) { hasTryCatch = true; + if (node.catchClause && inCallback) callbackCatches++; if (node.catchClause && !inCallback) { const tryStatementCount = countStatements(node.tryBlock.statements); const clause = catchClauseEvidence(node.catchClause); catches.push({ rethrows: clause.rethrows, branches: clause.branches, - guardsParse: guardsParse(node.tryBlock), + ...guardedWork(node.tryBlock), tryStatementCount, }); } @@ -839,6 +929,7 @@ export function scanFile(fileName: string, source: string): EntryPoint | null { calleeNames, hasTryCatch, catches, + callbackCatches, logCalls, statementCount, }; diff --git a/internal-packages/observability-map/src/types.ts b/internal-packages/observability-map/src/types.ts index a1d3be0b2db..ff0ad245660 100644 --- a/internal-packages/observability-map/src/types.ts +++ b/internal-packages/observability-map/src/types.ts @@ -12,14 +12,20 @@ export type CheckResult = { * legible instead of collapsing into one boolean. */ export type CatchEvidence = { - /** The clause contains a `throw`. */ + /** + * The clause throws on its own straight-line path: a `throw` among its statements, or among a + * bare nested block's, reached before anything that definitely exits. A throw guarded by an `if`, + * a loop, a `switch`, a nested `try` or a callback does not count, however the guard is spelled. + */ rethrows: boolean; /** - * The clause picks what to do from what it caught: an `if` or `switch` whose condition references - * the caught error binding, or a conditional that is the whole `return`/`throw`. `if (retries > 0)` - * does not count, and a bindingless `catch { ... }` cannot count at all. An `instanceof` used only - * to word a message, `json({ error: e instanceof Error ? e.message : String(e) })`, does not - * count either: every error still leaves by the same path. + * The clause picks what to do from what it caught, on that same straight-line path: an `if` or + * `switch` whose condition references the caught error binding AND at least one of whose arms + * returns or throws, or a conditional that is the whole value of a `return`/`throw`. + * `if (retries > 0)` does not count, `if (e instanceof Error) { }` does not count, and a + * bindingless `catch { ... }` cannot count at all. An `instanceof` used only to word a message, + * `json({ error: e instanceof Error ? e.message : String(e) })`, does not count either: every + * error still leaves by the same path. */ branches: boolean; /** @@ -30,6 +36,15 @@ export type CatchEvidence = { * its catch. */ guardsParse: boolean; + /** + * Everything the guarded region waits for is one of those parses. What separates + * `try { const body = await request.json(); } catch { 400 }` from + * `try { const body = await request.json(); return await handleEverything(body); } catch { 500 }`, + * which the statement count reads as the same size. Synchronous work is not counted here: the + * calls that prepare a parse's input are synchronous, and the swallows this has to catch wait on + * a service. + */ + awaitsOnlyParse: boolean; /** Statements in the guarded try block, counted as `statementCount` counts them. */ tryStatementCount: number; }; @@ -64,12 +79,20 @@ export type EntryPoint = { hasTryCatch: boolean; /** One entry per catch clause in those bodies, in source order. */ catches: CatchEvidence[]; + /** + * Catch clauses the scan found but refused to attribute to the route, because they sit inside a + * per-item iteration callback. Kept rather than dropped so `error-classification` can tell "this + * route catches nothing" from "this route's only error handling was refused", which are 50 points + * apart and used to read the same. + */ + callbackCatches: number; /** Calls to a `logger.*` or `log.*` callee in those bodies, in source order. */ logCalls: LogCall[]; /** - * Statement count across loader/action bodies, used by the triviality rule. A body that - * delegates to a same-file helper counts that helper's statements too, one hop only: work in a - * helper's own helpers, or in an imported module, is not counted. + * Statement count across loader/action bodies, used by the triviality rule. Includes the + * statements of functions written inline in those bodies, so wrapping a body in a callback does + * not shrink it. A body that delegates to a same-file helper counts that helper's statements too, + * one hop only: work in a helper's own helpers, or in an imported module, is not counted. */ statementCount: number; }; diff --git a/internal-packages/observability-map/test/checks.test.ts b/internal-packages/observability-map/test/checks.test.ts index 5a73eca50e5..21150799ab4 100644 --- a/internal-packages/observability-map/test/checks.test.ts +++ b/internal-packages/observability-map/test/checks.test.ts @@ -179,6 +179,75 @@ describe("error-classification", () => { expect(r.status).toBe("pass"); }); + // C5. The count is not the only condition any more, and this is the shape that showed why: two + // statements, one of them a parse, and the whole handler inside the try. Before `awaitsOnlyParse` + // the count read it as a narrow guard and passed it, which is the `otel.v1.logs.ts` swallow + // written compactly. Three spellings of the same thing, all of which the count reads as narrow. + const COMPACT_SWALLOWS: Array<[string, string]> = [ + [ + "two statements", + `try { const body = await request.json(); return await handleEverything(body); } + catch (error) { return new Response("Internal Server Error", { status: 500 }); }`, + ], + [ + "one statement, the parse nested inside the call", + `try { return await handleEverything(await request.json()); } + catch (error) { return new Response("Internal Server Error", { status: 500 }); }`, + ], + [ + "one statement, merged into a declaration list", + `try { const body = await request.json(), out = await handleEverything(body); return out; } + catch (error) { return new Response("Internal Server Error", { status: 500 }); }`, + ], + ]; + + for (const [label, body] of COMPACT_SWALLOWS) { + it(`fails a whole handler wrapped in a parse-guard-shaped try (${label})`, () => { + const r = run( + "error-classification", + "otel.v1.logs.ts", + `export async function action({ request }) {\n${body}\n}` + ); + expect(r.status).toBe("fail"); + }); + } + + // The counterpart: the same route with the handler moved out of the try is a real guard and + // still passes, so the rule above is not just "any try containing an await fails". + it("passes the same route once the handler moves out of the try", () => { + const r = run( + "error-classification", + "otel.v1.logs.ts", + `export async function action({ request }) { + let body; + try { body = await request.json(); } + catch { return json({ error: "bad json" }, { status: 400 }); } + return await handleEverything(body); + }` + ); + expect(r.status).toBe("pass"); + }); + + // Synchronous string work preparing a parse's input is not what `awaitsOnlyParse` refuses. Four + // real routes are this shape, `admin.llm-models.new.tsx` among them. + it("passes a guard that prepares its input synchronously before parsing", () => { + const r = run( + "error-classification", + "admin.llm-models.new.tsx", + `export async function action({ request }) { + const matchPattern = String(await request.text()); + try { + const testPattern = matchPattern.startsWith("(?i)") ? matchPattern.slice(4) : matchPattern; + new RegExp(testPattern); + } catch { + return json({ error: "Invalid regex" }, { status: 400 }); + } + return await save(matchPattern); + }` + ); + expect(r.status).toBe("pass"); + }); + it("fails a parse guard that takes a third statement beyond binding the result", () => { const r = run( "error-classification", @@ -432,14 +501,13 @@ describe("error-classification", () => { expect(r.status).toBe("not-applicable"); }); - // A7. A per-item error boundary inside a `.map()` callback used to be judged as the route's own - // catch. The route's own visible body catches nothing, so it is not-applicable, not a pass or a - // fail on the strength of a swallow one level of nesting away. - it("is not applicable to a route whose only catch is inside a Promise.all(items.map(...)) callback", () => { - const r = run( - "error-classification", - "batch.process.ts", - `import { prisma } from "~/db.server"; + // A7 as revised. A per-item error boundary inside a `.map()` callback is still not judged as the + // route's own catch, so it never sets `catches` and never speaks for the route's `tryStatementCount`. + // It is no longer excused either. Reading "no catch of its own" as not-applicable was worth 50 + // points to anything that could get the boundary rule to refuse the route's real catch, which + // `[0].map(...)` did, so a refused catch now fails instead of sitting out. + it("fails a route whose only catch is inside a Promise.all(items.map(...)) callback", () => { + const source = `import { prisma } from "~/db.server"; export async function action({ request }) { const items = await prisma.item.findMany(); await Promise.all( @@ -452,9 +520,103 @@ describe("error-classification", () => { }) ); return json({ ok: true }); + }`; + const ep = scanFile("batch.process.ts", source)!; + expect(ep.catches).toEqual([]); + expect(ep.callbackCatches).toBe(1); + const r = run("error-classification", "batch.process.ts", source); + expect(r.status).toBe("fail"); + expect(r.detail).toContain("callback the route does not own"); + }); + + // The same route with nothing caught anywhere stays not-applicable, so the fail above is + // attributable to the refused catch and not to the check having stopped excusing anything. + it("is not applicable to a route that catches nothing at all", () => { + const r = run( + "error-classification", + "batch.process.ts", + `import { prisma } from "~/db.server"; + export async function action({ request }) { + const items = await prisma.item.findMany(); + await Promise.all(items.map(async (item) => processItem(item))); + return json({ ok: true }); }` ); expect(r.status).toBe("not-applicable"); + expect(r.detail).toContain("catches nothing"); + }); + + // C2. A single-element array cannot iterate, so `[0].map(async () => { whole body })` is not a + // per-item boundary and the route's own catch is found where it always was. Before this, the + // wrapper deleted the route's catches and took a swallow from fail to not-applicable. + it("still fails a swallow wrapped in Promise.all([0].map(...))", () => { + const r = run( + "error-classification", + "wrapped.ts", + `import { prisma } from "~/db.server"; + export async function action({ request }) { + const [result] = await Promise.all([0].map(async () => { + try { + const body = await request.json(); + const a = await stepOne(body); + const b = await stepTwo(a); + return json({ b }); + } catch (e) { + return new Response("nope", { status: 500 }); + } + })); + return result; + }` + ); + expect(r.status).toBe("fail"); + }); + + // A second receiver exercising the same mechanism: an empty array literal, which no name list + // would treat differently from a populated one. + it("still fails a swallow wrapped in [].flatMap(...)", () => { + const r = run( + "error-classification", + "wrapped-empty.ts", + `import { prisma } from "~/db.server"; + export async function action({ request }) { + return [].flatMap(async () => { + try { + const body = await request.json(); + const a = await stepOne(body); + const b = await stepTwo(a); + return json({ b }); + } catch (e) { + return new Response("nope", { status: 500 }); + } + }); + }` + ); + expect(r.status).toBe("fail"); + }); + + // A third, where the name list cannot help at all: a non-array receiver whose method is called + // `map`. The boundary rule still refuses the callback, so the route has no catch of its own, and + // the refusal now fails rather than excusing. This is the shape the name list cannot tell from + // `users.map(...)`, and it is why the refusal had to stop paying. + it("still fails a swallow wrapped in a non-array receiver's .map(...)", () => { + const r = run( + "error-classification", + "wrapped-result.ts", + `import { prisma } from "~/db.server"; + export async function action({ request }) { + return await Result.map(async () => { + try { + const body = await request.json(); + const a = await stepOne(body); + const b = await stepTwo(a); + return json({ b }); + } catch (e) { + return new Response("nope", { status: 500 }); + } + }); + }` + ); + expect(r.status).toBe("fail"); }); }); diff --git a/internal-packages/observability-map/test/scan.test.ts b/internal-packages/observability-map/test/scan.test.ts index 4d1c56fb917..9c5bc9e468c 100644 --- a/internal-packages/observability-map/test/scan.test.ts +++ b/internal-packages/observability-map/test/scan.test.ts @@ -658,15 +658,31 @@ describe("scanFile: catch clause evidence", () => { expect(ep!.catches[0]).toMatchObject({ rethrows: false, branches: false }); }); - it("does not set rethrows for a throw inside a statically-false if", () => { - const ep = scanFile("x.ts", swallow("if (false) { throw e; }")); - expect(ep!.catches[0]).toMatchObject({ rethrows: false, branches: false }); - }); - - it("does not set rethrows for a throw inside a statically-false while", () => { - const ep = scanFile("x.ts", swallow("while (false) { throw e; }")); - expect(ep!.catches[0]).toMatchObject({ rethrows: false, branches: false }); - }); + // Every shape anyone has found that puts a `throw` somewhere it can never run. The previous + // round recognised the first two by folding the literal `false`, and lost to the other nine. + // None of them is named in the rule now: a throw counts when it is unconditional, and every + // one of these is guarded by something. `dead-*` in the mutation corpus runs the same list over + // the whole route tree. + const DEAD_SHAPES: Array<[string, string]> = [ + ["if (false)", "if (false) { throw e; }"], + ["while (false)", "while (false) { throw e; }"], + ["for (;false;)", "for (;false;) { throw e; }"], + ["if (true) else", "if (true) { doThing(); } else { throw e; }"], + ["switch with no matching case", "switch (1) { case 2: throw e; }"], + ["inner try/catch", "try { doThing(); } catch { throw e; }"], + ["for...of an empty array", "for (const item of []) { throw e; }"], + ["for...in an empty object", "for (const key in {}) { throw e; }"], + ["if on an empty string", 'if ("") { throw e; }'], + ["if on a negated literal", "if (!true) { throw e; }"], + ["if on a constant comparison", "if (1 === 2) { throw e; }"], + ]; + + for (const [label, shape] of DEAD_SHAPES) { + it(`does not set rethrows for a throw inside ${label}`, () => { + const ep = scanFile("x.ts", swallow(shape)); + expect(ep!.catches[0]).toMatchObject({ rethrows: false, branches: false }); + }); + } it("does not set rethrows for a throw merely registered in a constructed callback", () => { const ep = scanFile("x.ts", swallow("queue.push(() => { throw e; });")); @@ -811,7 +827,10 @@ describe("scanFile: catch clause evidence", () => { ` ); expect(ep!.hasTryCatch).toBe(true); - expect(ep!.catches[0]!.rethrows).toBe(true); + // The `throw e` here is guarded by an `if`, so it is not on the clause's straight-line path and + // does not read as a rethrow. The `if` itself does: it reads the binding and one arm throws, so + // the clause decides. The verdict the checks care about is unchanged. + expect(ep!.catches[0]!.rethrows).toBe(false); expect(ep!.catches[0]!.branches).toBe(true); }); @@ -1077,6 +1096,7 @@ describe("scanFile: per-catch evidence", () => { rethrows: false, branches: false, guardsParse: true, + awaitsOnlyParse: true, tryStatementCount: 1, }); expect(ep!.catches[1]).toMatchObject({ @@ -1178,6 +1198,7 @@ describe("scanFile: per-catch evidence", () => { rethrows: false, branches: false, guardsParse: false, + awaitsOnlyParse: false, tryStatementCount: 4, }); }); @@ -1204,7 +1225,10 @@ describe("scanFile: per-catch evidence", () => { ` ); expect(ep!.catches).toHaveLength(2); - expect(ep!.catches.filter((c) => c.rethrows && c.branches)).toHaveLength(1); + // `if (e instanceof Response) throw e;` branches (it reads the binding and one arm throws) and + // does not rethrow (the throw is guarded, so it is not on the clause's own path). The action's + // catch does neither. What this test is for is that the two clauses stay separate. + expect(ep!.catches.filter((c) => !c.rethrows && c.branches)).toHaveLength(1); expect(ep!.catches.filter((c) => !c.rethrows && !c.branches)).toHaveLength(1); }); @@ -1657,18 +1681,26 @@ describe("scanFile: a binding shadowed by an enclosing scope, not just a nested expect(ep!.catches[0]!.branches).toBe(false); }); - // Positive controls: a genuine reference to the real binding must still be credited, including - // through an enclosing loop whose OWN variable has a different name. + // Positive control: a genuine reference to the real binding, on the clause's own path, with an + // arm that takes the error somewhere the other arm does not go. it("still credits an if that genuinely reads the outer binding directly", () => { - const ep = scanFile("x.ts", swallow("if (error instanceof Error) { doThing(); }")); + const ep = scanFile("x.ts", swallow("if (error instanceof Error) { return badRequest(); }")); expect(ep!.catches[0]!.branches).toBe(true); }); - it("still credits an if inside a for...of loop with a different loop variable", () => { + // Two shapes that read the real binding and are still not credited, for reasons that are not + // shadowing. Both are precision the straight-line rule gives up on purpose, and both are + // recorded here so a later reader can tell a deliberate limit from a bug. + it("does not credit an if whose arm does not take the error anywhere", () => { + const ep = scanFile("x.ts", swallow("if (error instanceof Error) { doThing(); }")); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + it("does not credit an if nested inside a loop, even with a different loop variable", () => { const ep = scanFile( "x.ts", - swallow("for (const item of items) { if (error.code === item) { doThing(); } }") + swallow("for (const item of items) { if (error.code === item) { return item; } }") ); - expect(ep!.catches[0]!.branches).toBe(true); + expect(ep!.catches[0]!.branches).toBe(false); }); }); From fb8ca4dee79cc19afd32dbe3b6701532f65aef38 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Sat, 1 Aug 2026 19:43:13 +0100 Subject: [PATCH 054/106] test(observability-map): make the score-cannot-rise property a corpus Three rounds argued the property shape by shape and each was broken by a shape nobody had thought of. This replaces the argument with evidence. test/mutationCorpus.test.ts applies 30 whole-tree rewrites to a temp copy of apps/webapp/app/routes and holds each to three assertions: the published global does not rise, the mean over the routes measured in both runs does not rise, and for a semantics-preserving rewrite no route's score rises and no measured route drops out of the measured set. The third is what catches a mutation that raises one route and lowers another; the second holds the population fixed so the comparison is about scores rather than denominators. test/mutations.ts holds the rewrites, each labelled preserving or deleting. Every laundering shape found on this branch is there, plus five dead-code shapes and two iteration receivers found while writing it. They are text splices at AST positions rather than reprints, so a failure diffs down to the one construct that moved. A mutation that touches fewer than 20 files, breaks parsing, or loses routes fails rather than passing vacuously. All 30 hold. Against the code as it was at the start of this round, 16 did not. It takes about three minutes, so it is gated behind OBS_MAP_MUTATION_CORPUS=1 and runs as its own job in the observability-map workflow rather than in pnpm test. That job is allowed to fail the build, and unlike the report job it runs for fork PRs. README figures refreshed to the current tree and the two-invariant section replaced by a pointer to the corpus. --- .github/workflows/observability-map.yml | 35 + internal-packages/observability-map/README.md | 45 +- .../test/mutationCorpus.test.ts | 255 ++++++++ .../observability-map/test/mutations.ts | 605 ++++++++++++++++++ 4 files changed, 924 insertions(+), 16 deletions(-) create mode 100644 internal-packages/observability-map/test/mutationCorpus.test.ts create mode 100644 internal-packages/observability-map/test/mutations.ts diff --git a/.github/workflows/observability-map.yml b/.github/workflows/observability-map.yml index f63409adb96..0406f1c3730 100644 --- a/.github/workflows/observability-map.yml +++ b/.github/workflows/observability-map.yml @@ -16,6 +16,41 @@ permissions: pull-requests: write jobs: + # The tree-scale mutation corpus: every known laundering shape applied to the whole route tree, + # asserting the score does not rise. Roughly three minutes, which is why it is gated out of the + # package's default `pnpm test` and run here instead. Unlike the report job below it has no token + # to lose, so it runs for fork PRs too, and unlike the report job it is allowed to fail the build. + mutation-corpus: + name: 🧬 Mutation corpus + runs-on: warp-ubuntu-latest-x64-4x + steps: + - name: ⬇️ Checkout repo + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + fetch-depth: 1 + persist-credentials: false + + - name: ⎔ Setup pnpm + uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0 + with: + version: 10.33.2 + + - name: ⎔ Setup node + uses: WarpBuilds/setup-node@bc639b444d583175926b588962199c247d23e8d3 # v6 + with: + node-version: 24.18.0 + cache: "pnpm" + + - name: 📥 Download deps + run: pnpm install --frozen-lockfile + + - name: 🧬 Run the corpus + env: + OBS_MAP_MUTATION_CORPUS: "1" + run: | + pnpm --filter @internal/observability-map exec vitest run \ + test/mutationCorpus.test.ts --testTimeout=120000 --disable-console-intercept + report: runs-on: warp-ubuntu-latest-x64-4x # Fork PRs get a read-only token, so the comment cannot post. Skipping the job beats a red x. diff --git a/internal-packages/observability-map/README.md b/internal-packages/observability-map/README.md index 76638b7e7ab..a058282c727 100644 --- a/internal-packages/observability-map/README.md +++ b/internal-packages/observability-map/README.md @@ -4,7 +4,7 @@ Scores every webapp entry point on whether it could explain itself during an inc the ones worth fixing. An entry point is a Remix `loader` or `action` under `apps/webapp/app/routes`, 427 of them at the time of writing. -The number it prints today is 17 out of 100. That is not a bug, and the rest of this file is mostly +The number it prints today is 15 out of 100. That is not a bug, and the rest of this file is mostly about why you should believe it. ## Running it @@ -27,28 +27,38 @@ against the PR's merge base, with the score, what changed, and the current fix l report-only: nothing here fails the build or blocks a merge, and the gate stays deferred until a later phase decides to add one. See `.github/workflows/observability-map.yml`. -## What 17 means +## What 15 means It is the mean score of the 412 entry points that had at least one applicable check, where an entry's score is the share of its applicable checks that passed. It is low because the webapp does -not attach tenant identity to its failures: **21 of 412 entry points name an environment, project, -organization, run or user on a failure path.** Everything else, when it breaks at 3am, tells you the +not attach tenant identity to its failures: **11 of 412 entry points name an environment, project, +organization or user on a failure path.** Everything else, when it breaks at 3am, tells you the route and the request id and nothing about whose request it was. The score was 76 until we stopped crediting routes for the error handling they do not do. Emptying every catch clause in the tree used to score it 100, which meant the metric paid you for deleting error handling. -Two invariants hold now, and both are asserted in `test/score.test.ts` rather than measured once: +The property behind that is now a test corpus rather than a claim. `test/mutationCorpus.test.ts` +applies 30 semantics-preserving or handling-deleting rewrites to the whole route tree in a temp copy +and asserts three things for each: the published global does not rise, the mean over the routes +measured in both runs does not rise, and for a semantics-preserving rewrite no individual route's +score rises or drops out of the measured set. Every laundering shape a reviewer has found on this +branch is an entry in it, `test/mutations.ts` holds them, and each entry says which it is. -- **Removing error handling must not raise the score.** Deleting every catch clause drops it to 8, - and deleting the logs as well drops it to 2. -- **Adding error handling that does nothing must not raise the score.** Wrapping every body in - `try { ... } catch (e) { throw e }` leaves the score unchanged. That mutation used to be worth 27 - points across the tree, because a rethrow-only clause counted as a pass while no catch at all was - not-applicable, and the two are observationally identical. +Two of them are worth naming because they are the ones the design turns on. Deleting every catch +clause in the tree drops the score from 15 to 2, so the metric does not pay you for removing error +handling. Wrapping every body in `try { ... } catch (e) { throw e }` leaves it unchanged, so it does +not pay you for adding error handling that does nothing either. -If you change this package, check both directions still hold. +The honest statement is "these 30 rewrites are defended, and here they are", not "unpaddable". The +corpus takes about three minutes, so it is gated behind `OBS_MAP_MUTATION_CORPUS=1` and run as its +own CI job rather than in `pnpm test`. If you change this package, run it: + +```bash +OBS_MAP_MUTATION_CORPUS=1 pnpm --filter @internal/observability-map exec vitest run \ + test/mutationCorpus.test.ts --testTimeout=120000 --disable-console-intercept +``` So the number is deliberately unflattering, and one platform change would move most of it. Nothing central attaches a tenant: `logger` pushes `{ requestId, path, host, method }` onto every line @@ -72,7 +82,7 @@ rather than celebrating. ## Two findings are headlines, not list entries -`audit-trail` fails 19 of 19, and `request-context` fails 391 of 412. Printing either one per route +`audit-trail` fails 19 of 19, and `request-context` fails 401 of 412. Printing either one per route would bury the route-specific findings under the same sentence repeated hundreds of times, so both are reported as a figure: the `AUDIT` and `CONTEXT` lines. 329 entry points fail nothing except `request-context` and appear only in that figure, which leaves 71 in the fix list. An entry that @@ -152,7 +162,7 @@ stays visible. value is real. A codemod that added `environmentId` to every in-catch `logger.error` call, wiring it up to the wrong variable or a constant, would move the score exactly as far as one that wired it up correctly. Measured on the real tree: adding a synthetic `environmentId` field to every in-catch log -call, with no other change, takes the global score from 17 to 27. +call, with no other change, takes the global score from 15 to 26. That is the tool verifying presence, not meaning, and it is not a bug to fix. Every check here reads syntax: a field name, a call, a binding reference. None of them can tell a genuine tenant id from a @@ -180,8 +190,11 @@ Read these before trusting a specific verdict. - **Only the first object-literal argument is read** for identifier fields, and only its property names. `logger.error("failed", ctx)` where `ctx` is a variable contributes nothing, and neither does a second object. -- **Inline callbacks are not descended into** when counting statements, so a two-statement body can - hold a pile of work inside a `.map()`. The call count is what catches those cases, imperfectly. +- **A catch inside a per-item callback is not the route's.** `items.map((item) => { try {...} })` + is a fresh boundary per element, so its clause is not read as the route's own error handling. The + test is the method name, which cannot tell `users.map` from `Result.map`. Being wrong there costs + precision rather than points: a refused catch fails the route rather than excusing it, so no + wrapper can turn a swallow into a not-applicable by getting the boundary rule to refuse it. - **Sensitivity is a heuristic**: a symbol list plus path segments. It was circular until recently, counting `requireAdminApiRequest` as a hazard when it is a mitigation, which made 34 of 67 sensitive routes sensitive purely for being guarded. Expect it to need pruning again as routes diff --git a/internal-packages/observability-map/test/mutationCorpus.test.ts b/internal-packages/observability-map/test/mutationCorpus.test.ts new file mode 100644 index 00000000000..f74076c1d5b --- /dev/null +++ b/internal-packages/observability-map/test/mutationCorpus.test.ts @@ -0,0 +1,255 @@ +import { existsSync, mkdirSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { scanDirectory } from "../src/scan.js"; +import { buildReport } from "../src/score.js"; +import { MUTATIONS, type Mutation } from "./mutations.js"; + +/** + * The tree-scale mutation corpus. + * + * The tool's central claim is that no semantics-preserving edit to a route raises its score. Three + * rounds argued that claim shape by shape and lost each time. This file turns it into evidence + * instead: every laundering shape anyone has found is a corpus entry, and each entry rewrites the + * whole real route tree in a temp copy and is held to three assertions. + * + * - the published global does not rise. That is the figure the claim is about. + * - the mean over the routes measured in BOTH runs does not rise. Same comparison at full + * precision, with the population held fixed so it measures scores rather than denominators. + * - for a semantics-preserving rewrite, no individual route's score rises and no measured route + * drops out of the measured set. The tree mean can hide a route going up by taking another down; + * `[0].map(...)` is exactly that shape. + * + * Tree scale, not per-fixture, because that is where laundering pays. A shape that moves one + * hand-written fixture by 50 points may move the tree by nothing; a shape that moves the tree is the + * one worth defending. + * + * The honest statement this file supports is "these N mutations are defended, and here they are", + * never "unpaddable". + * + * Runtime is roughly six seconds per entry, which is why the whole file is gated behind + * `OBS_MAP_MUTATION_CORPUS=1`. The `observability-map` workflow sets it, so the gate keeps the + * default suite fast without making this the thing nobody runs. + */ + +const ROUTES = resolve(__dirname, "../../../apps/webapp/app/routes"); +const ENABLED = process.env.OBS_MAP_MUTATION_CORPUS === "1"; + +/** + * Where a corpus entry goes when the tool does not defend it. Empty as of round A fix 2: all 30 + * entries hold. When a reviewer finds a shape that cannot be defended, add it to the corpus and + * name it here rather than leaving it out, and write down why in the round's report. `it.fails` + * keeps such an entry running, so closing the hole later turns this file red until the entry is + * moved back out deliberately. + */ +const KNOWN_GAPS = new Set([]); + +type SourceFile = { relativeName: string; source: string }; + +/** Route modules exactly as `scanDirectory` enumerates them: flat files, plus one `route.ts(x)` per + * directory. Read once; every mutation rewrites this list rather than the tree on disk. */ +function readTree(dir: string): SourceFile[] { + const files: SourceFile[] = []; + const take = (absolutePath: string, relativeName: string) => { + files.push({ relativeName, source: readFileSync(absolutePath, "utf8") }); + }; + for (const entry of readdirSync(dir, { withFileTypes: true })) { + if (entry.isDirectory()) { + for (const child of readdirSync(join(dir, entry.name), { withFileTypes: true })) { + if (!child.isFile() || (child.name !== "route.ts" && child.name !== "route.tsx")) continue; + take(join(dir, entry.name, child.name), `${entry.name}/${child.name}`); + } + continue; + } + if (!entry.isFile() || !/\.tsx?$/.test(entry.name) || entry.name.endsWith(".d.ts")) continue; + take(join(dir, entry.name), entry.name); + } + return files; +} + +function materialize(files: SourceFile[]): string { + const root = join( + tmpdir(), + `obs-map-corpus-${process.pid}-${Math.random().toString(36).slice(2)}` + ); + for (const file of files) { + const target = join(root, file.relativeName); + mkdirSync(dirname(target), { recursive: true }); + writeFileSync(target, file.source); + } + return root; +} + +type Measurement = { + global: number | null; + /** The unrounded mean the global is a rounding of. A rise of less than half a point is invisible + * in `global` and is still a rise, so the assertions read this. */ + exactMean: number; + measured: number; + entryPoints: number; + parseFailures: number; + /** Per route file, so a mutation that raises one route while lowering the tree is still caught. + * `[0].map(...)` is exactly that shape: it deletes a route's catches, which takes a failing route + * to 100 and a passing one to nothing, and the two cancel in the global. */ + perEntry: Map; +}; + +function measure(files: SourceFile[]): Measurement { + const root = materialize(files); + try { + const { entryPoints, parseFailures } = scanDirectory(root); + const report = buildReport(entryPoints, parseFailures); + const scores = report.entries.filter((e) => e.measured).map((e) => e.score); + const perEntry = new Map(); + for (const entry of report.entries) { + perEntry.set(entry.fileName, { + score: entry.score, + measured: entry.measured, + checks: entry.rawChecks.map((c) => `${c.id}=${c.status}`).join(" "), + }); + } + return { + global: report.global, + exactMean: scores.length === 0 ? 0 : scores.reduce((a, b) => a + b, 0) / scores.length, + measured: report.measured, + entryPoints: entryPoints.length, + parseFailures: parseFailures.length, + perEntry, + }; + } finally { + rmSync(root, { recursive: true, force: true }); + } +} + +type Rise = { fileName: string; from: number; to: number; before: string; after: string }; + +/** + * Route files the mutation made look better, worst first. Two ways to qualify, both counted: the + * score went up, or a route that was being measured stopped being measured. The second is a rise + * too. An unmeasured route's score is the vacuous 100 and it leaves every mean, so dropping out of + * the measured set is the most complete form of the thing the property forbids. + * + * A route the mutation removed from the report entirely is not counted here; the entry-point guard + * catches that instead. + */ +function risesIn(baseline: Measurement, after: Measurement): Rise[] { + const rises: Rise[] = []; + for (const [fileName, before] of baseline.perEntry) { + const now = after.perEntry.get(fileName); + if (!now) continue; + const droppedOut = before.measured && !now.measured; + if (!droppedOut && now.score <= before.score) continue; + rises.push({ + fileName, + from: before.score, + to: now.score, + before: before.checks, + after: now.checks, + }); + } + return rises.sort((a, b) => b.to - b.from - (a.to - a.from)); +} + +/** Mean score over the routes measured in BOTH runs. The plain mean moves when the measured + * population moves, which a mutation can do without making any route look better: an inert + * try/catch takes 15 trivial routes off the exemption list and into the report, and a route joining + * at 50 raises a tree averaging 15 while itself having gone from an unmeasured 100 to a measured 50. + * Holding the population fixed is what makes the comparison about the scores. */ +function commonMean(baseline: Measurement, after: Measurement): { before: number; after: number } { + let sumBefore = 0; + let sumAfter = 0; + let n = 0; + for (const [fileName, before] of baseline.perEntry) { + const now = after.perEntry.get(fileName); + if (!before.measured || !now || !now.measured) continue; + sumBefore += before.score; + sumAfter += now.score; + n++; + } + return n === 0 ? { before: 0, after: 0 } : { before: sumBefore / n, after: sumAfter / n }; +} + +function mutate(files: SourceFile[], mutation: Mutation): { files: SourceFile[]; changed: number } { + let changed = 0; + const out = files.map((file) => { + const source = mutation.apply(file.relativeName, file.source); + if (source === null || source === file.source) return file; + changed++; + return { relativeName: file.relativeName, source }; + }); + return { files: out, changed }; +} + +const describeCorpus = ENABLED && existsSync(ROUTES) ? describe : describe.skip; + +describeCorpus("mutation corpus over the real route tree", () => { + const files = ENABLED && existsSync(ROUTES) ? readTree(ROUTES) : []; + const baseline = ENABLED && existsSync(ROUTES) ? measure(files) : null; + + it("has a baseline worth mutating", () => { + expect(baseline).not.toBeNull(); + expect(baseline!.entryPoints).toBeGreaterThan(300); + expect(baseline!.global).not.toBeNull(); + console.log( + `[corpus] baseline global=${baseline!.global} mean=${baseline!.exactMean.toFixed(3)} ` + + `measured=${baseline!.measured} eps=${baseline!.entryPoints} files=${files.length}` + ); + }); + + /** + * The number of files a mutation must touch before its result means anything. A mutation that + * silently matched nothing would otherwise "pass" by leaving the tree alone, which is the exact + * failure mode that let earlier rounds believe a shape was defended. + */ + const MINIMUM_FILES_TOUCHED = 20; + + for (const mutation of MUTATIONS) { + const run = KNOWN_GAPS.has(mutation.id) ? it.fails : it; + + run(`${mutation.kind}: ${mutation.what} (${mutation.id})`, () => { + const { files: mutated, changed } = mutate(files, mutation); + expect(changed).toBeGreaterThanOrEqual(MINIMUM_FILES_TOUCHED); + + const after = measure(mutated); + + // A mutation that stops the tree parsing, or that hides most of the routes from the scanner, + // has not tested the property: whatever the score does afterwards is measuring a different + // tree. Both guards fail loudly rather than letting such a mutation report a pass. + expect(after.parseFailures).toBe(baseline!.parseFailures); + expect(after.entryPoints).toBeGreaterThanOrEqual(baseline!.entryPoints - 5); + + const rises = risesIn(baseline!, after); + const common = commonMean(baseline!, after); + console.log( + `[corpus] ${mutation.id}: global ${baseline!.global} -> ${after.global} ` + + `(mean ${baseline!.exactMean.toFixed(3)} -> ${after.exactMean.toFixed(3)}, ` + + `common mean ${common.before.toFixed(3)} -> ${common.after.toFixed(3)}, ` + + `measured ${baseline!.measured} -> ${after.measured}, files ${changed}, ` + + `routes raised ${rises.length})` + + rises + .slice(0, 3) + .map( + (r) => + `\n ${r.fileName} ${r.from}->${r.to}\n was: ${r.before}\n now: ${r.after}` + ) + .join("") + ); + + // The published figure, which is what the claim is about. + expect(after.global!).toBeLessThanOrEqual(baseline!.global!); + // The same comparison at full precision, over a fixed population so it measures the scores + // and not who is in the denominator. + expect(common.after).toBeLessThanOrEqual(common.before + 1e-9); + + // Per route, for the preserving half of the corpus. This is the property as stated: an edit + // that does not change what a route does must not make that route look better, whatever it + // does to the tree's mean. The deleting half is exempt on purpose: a route whose only failing + // check was error-classification really does leave the denominator when its catch goes, which + // the design chose over crediting a route for deleting its error handling, and the global + // figure above is where that trade is held to account. + if (mutation.kind === "preserving") { + expect(rises.map((r) => `${r.fileName} ${r.from}->${r.to}`)).toEqual([]); + } + }); + } +}); diff --git a/internal-packages/observability-map/test/mutations.ts b/internal-packages/observability-map/test/mutations.ts new file mode 100644 index 00000000000..4b8f6df4671 --- /dev/null +++ b/internal-packages/observability-map/test/mutations.ts @@ -0,0 +1,605 @@ +import ts from "typescript"; + +/** + * Source-to-source mutations for the tree-scale corpus in `mutationCorpus.test.ts`. + * + * Every mutation here is a *text* rewrite driven by AST positions, never a reprint. A reprint would + * change formatting everywhere and make a failure impossible to read; splicing at node positions + * leaves the rest of the file byte-identical, so a corpus failure can be diffed down to the one + * construct that moved. + * + * Two kinds of entry live in the corpus and they are labelled `preserving` and `deleting`: + * + * - `preserving`: the rewrite does not change what the route does. Dead code that can never run, + * a wrapper that runs the same statements once, a comment, a merge of adjacent `const`s. The + * property under test is the one the tool claims: no such edit may raise the score. + * - `deleting`: the rewrite removes error handling or logging. The route is worse afterwards, so + * the score must not rise either, for a different and simpler reason. + * + * Neither kind is ever executed. "Semantics-preserving" here means preserving the observable + * behaviour of the route as written, which is what the scanner claims to measure; it is not a + * claim that the mutated tree compiles against its real types. + */ + +export type MutationKind = "preserving" | "deleting"; + +export type Mutation = { + id: string; + kind: MutationKind; + /** What the rewrite does, in one line, for the corpus table in the report. */ + what: string; + /** The mutated source, or null when this file has nothing for the mutation to touch. */ + apply(fileName: string, source: string): string | null; +}; + +type Edit = { start: number; end: number; text: string }; + +function parse(fileName: string, source: string): ts.SourceFile { + return ts.createSourceFile( + fileName, + source, + ts.ScriptTarget.Latest, + /* setParentNodes */ true, + fileName.endsWith(".tsx") ? ts.ScriptKind.TSX : ts.ScriptKind.TS + ); +} + +/** + * Splice edits into `source`, right to left so earlier offsets stay valid. + * + * An edit that falls inside an earlier edit's range is dropped rather than applied: a mutation that + * deletes a catch clause and one that rewrites a statement inside that clause would otherwise + * produce overlapping splices. Dropping the inner one is what "the outer rewrite won" means. + */ +function applyEdits(source: string, edits: Edit[]): string | null { + if (edits.length === 0) return null; + const sorted = [...edits].sort((a, b) => a.start - b.start || a.end - b.end); + const kept: Edit[] = []; + for (const edit of sorted) { + const last = kept[kept.length - 1]; + if (last && edit.start < last.end) continue; + kept.push(edit); + } + let out = source; + for (let i = kept.length - 1; i >= 0; i--) { + const edit = kept[i]!; + out = out.slice(0, edit.start) + edit.text + out.slice(edit.end); + } + return out === source ? null : out; +} + +function insert(at: number, text: string): Edit { + return { start: at, end: at, text }; +} + +function forEachNode(node: ts.Node, visit: (n: ts.Node) => void): void { + visit(node); + ts.forEachChild(node, (child) => { + forEachNode(child, visit); + }); +} + +// -- entry points --------------------------------------------------------------------------- + +type EntryFunction = ts.FunctionDeclaration | ts.FunctionExpression | ts.ArrowFunction; + +function isEntryFunction(node: ts.Node): node is EntryFunction { + return ( + ts.isFunctionDeclaration(node) || ts.isFunctionExpression(node) || ts.isArrowFunction(node) + ); +} + +function unwrap(expr: ts.Expression): ts.Expression { + let current = expr; + for (;;) { + if ( + ts.isParenthesizedExpression(current) || + ts.isAwaitExpression(current) || + ts.isAsExpression(current) || + ts.isSatisfiesExpression(current) || + ts.isNonNullExpression(current) + ) { + current = current.expression; + continue; + } + return current; + } +} + +function propertyNameOf(property: ts.ObjectLiteralElementLike): string | null { + if (!property.name) return null; + if (ts.isIdentifier(property.name) || ts.isStringLiteral(property.name)) { + return property.name.text; + } + return null; +} + +/** + * Handler functions on a builder's object argument, in the two shapes the route builders use: + * `handler` at the top level and `methods.POST.handler`. + * + * Deliberately a copy of the same shapes `src/scan.ts` recognises rather than an import of them. + * The harness has to be able to disagree with the scanner about where a route body is; sharing the + * scanner's own notion would let a bug in that notion hide a laundering shape from the corpus. + */ +function collectNamedHandlers(object: ts.ObjectLiteralExpression, out: EntryFunction[]): void { + for (const property of object.properties) { + if (!ts.isPropertyAssignment(property)) continue; + const name = propertyNameOf(property); + const value = unwrap(property.initializer); + if (name === "handler" && isEntryFunction(value)) out.push(value); + if (name === "methods" && ts.isObjectLiteralExpression(value)) { + for (const method of value.properties) { + if (!ts.isPropertyAssignment(method)) continue; + const config = unwrap(method.initializer); + if (ts.isObjectLiteralExpression(config)) collectNamedHandlers(config, out); + } + } + } +} + +function rootCall(call: ts.CallExpression): ts.CallExpression { + let current = call; + for (;;) { + let next = unwrap(current.expression); + while (ts.isPropertyAccessExpression(next) || ts.isElementAccessExpression(next)) { + next = unwrap(next.expression); + } + if (ts.isCallExpression(next)) { + current = next; + continue; + } + return current; + } +} + +function fromInitializer(expr: ts.Expression, out: EntryFunction[]): void { + const target = unwrap(expr); + if (isEntryFunction(target)) { + out.push(target); + return; + } + if (!ts.isCallExpression(target)) return; + for (const arg of rootCall(target).arguments) { + const unwrapped = unwrap(arg); + if (isEntryFunction(unwrapped)) out.push(unwrapped); + else if (ts.isObjectLiteralExpression(unwrapped)) collectNamedHandlers(unwrapped, out); + } +} + +const ENTRY_NAMES = new Set(["loader", "action"]); + +function isExported(node: ts.Node): boolean { + return ( + ts.canHaveModifiers(node) && + ts.getModifiers(node)?.some((m) => m.kind === ts.SyntaxKind.ExportKeyword) === true + ); +} + +/** Block bodies of the exported `loader`/`action` handlers, the region a whole-body wrapper wraps. */ +function entryBodies(sf: ts.SourceFile): ts.Block[] { + const functions: EntryFunction[] = []; + for (const statement of sf.statements) { + if (!isExported(statement)) continue; + if (ts.isFunctionDeclaration(statement) && statement.name) { + if (ENTRY_NAMES.has(statement.name.text)) functions.push(statement); + continue; + } + if (!ts.isVariableStatement(statement)) continue; + for (const decl of statement.declarationList.declarations) { + if (!decl.initializer || !ts.isIdentifier(decl.name)) continue; + if (ENTRY_NAMES.has(decl.name.text)) fromInitializer(decl.initializer, functions); + } + } + const bodies: ts.Block[] = []; + for (const fn of functions) { + if (fn.body && ts.isBlock(fn.body)) bodies.push(fn.body); + } + return bodies; +} + +// -- generic mutation shapes ---------------------------------------------------------------- + +function catchClauses(sf: ts.SourceFile): ts.CatchClause[] { + const out: ts.CatchClause[] = []; + forEachNode(sf, (node) => { + if (ts.isCatchClause(node)) out.push(node); + }); + return out; +} + +function bindingNameOf(clause: ts.CatchClause): string | null { + const decl = clause.variableDeclaration; + return decl && ts.isIdentifier(decl.name) ? decl.name.text : null; +} + +/** + * Append a statement at the end of every catch clause that names its binding. `snippet` receives + * the binding name. Appending is the position that matters: a shape spliced in after a `return` is + * already unreachable and proves nothing. + */ +function appendToEveryCatch( + id: string, + kind: MutationKind, + what: string, + snippet: (binding: string) => string +): Mutation { + return { + id, + kind, + what, + apply(fileName, source) { + const sf = parse(fileName, source); + const edits: Edit[] = []; + for (const clause of catchClauses(sf)) { + const binding = bindingNameOf(clause); + if (binding === null) continue; + edits.push(insert(clause.block.end - 1, `\n${snippet(binding)}\n`)); + } + return applyEdits(source, edits); + }, + }; +} + +/** Wrap every route body in a single-shot wrapper, `open` before its statements and `close` after. */ +function wrapEveryBody(id: string, what: string, open: string, close: string): Mutation { + return { + id, + kind: "preserving", + what, + apply(fileName, source) { + const sf = parse(fileName, source); + const edits: Edit[] = []; + for (const body of entryBodies(sf)) { + edits.push(insert(body.getStart() + 1, `\n${open}\n`)); + edits.push(insert(body.end - 1, `\n${close}\n`)); + } + return applyEdits(source, edits); + }, + }; +} + +/** Prepend text at the very top of every file, before the first token's leading trivia. */ +function prependToEveryFile(id: string, what: string, text: string): Mutation { + return { + id, + kind: "preserving", + what, + apply(_fileName, source) { + return `${text}\n${source}`; + }, + }; +} + +const LOGGER_CALLEE = /(^|\.)(logger|log)\.[A-Za-z_$][\w$]*$/; + +function calleeText(expr: ts.Expression): string | null { + const target = unwrap(expr); + if (ts.isIdentifier(target)) return target.text; + if (target.kind === ts.SyntaxKind.ThisKeyword) return "this"; + if (ts.isPropertyAccessExpression(target)) { + const base = calleeText(target.expression); + return base === null ? null : `${base}.${target.name.text}`; + } + if (ts.isCallExpression(target)) { + const base = calleeText(target.expression); + return base === null ? null : `${base}()`; + } + return null; +} + +function logStatementEdits(sf: ts.SourceFile): Edit[] { + const edits: Edit[] = []; + forEachNode(sf, (node) => { + if (!ts.isExpressionStatement(node)) return; + const call = unwrap(node.expression); + if (!ts.isCallExpression(call)) return; + const text = calleeText(call.expression); + if (text !== null && LOGGER_CALLEE.test(text)) { + edits.push({ start: node.getStart(), end: node.end, text: ";" }); + } + }); + return edits; +} + +/** + * Remove the catch clause from every `try`. With a `finally` present the clause alone goes and the + * `try`/`finally` stands; without one the whole `try` collapses to the bare block it guarded, which + * is still a legal statement. Point edits either way, so a nested rewrite inside the clause is + * simply dropped by `applyEdits` rather than colliding. + */ +function catchDeletionEdits(sf: ts.SourceFile): Edit[] { + const edits: Edit[] = []; + forEachNode(sf, (node) => { + if (!ts.isTryStatement(node) || !node.catchClause) return; + if (node.finallyBlock) { + edits.push({ start: node.catchClause.getStart(), end: node.catchClause.end, text: " " }); + return; + } + edits.push({ start: node.getStart(), end: node.tryBlock.getStart(), text: "" }); + edits.push({ start: node.tryBlock.end, end: node.end, text: "" }); + }); + return edits; +} + +// -- the corpus ------------------------------------------------------------------------------- + +/** + * Every laundering shape found by a reviewer on this branch, plus the five extra dead-code shapes + * and two extra iteration receivers found while writing this file. Each entry is a whole-tree + * rewrite; `mutationCorpus.test.ts` asserts the global score does not rise for any of them. + */ +export const MUTATIONS: Mutation[] = [ + prependToEveryFile( + "suppress-every-check", + "prepend an obs-map-disable directive for every check to every file", + [ + "// obs-map-disable error-classification -- mutation corpus", + "// obs-map-disable request-context -- mutation corpus", + "// obs-map-disable auth-boundary -- mutation corpus", + "// obs-map-disable audit-trail -- mutation corpus", + ].join("\n") + ), + + { + id: "jsx-text-line-directive", + kind: "preserving", + what: "add a component whose JSX text begins with a // directive", + apply(fileName, source) { + if (!fileName.endsWith(".tsx")) return null; + return `${source}\nexport function ObsMapMutationA() {\n return

// obs-map-disable error-classification -- mutation corpus

;\n}\n`; + }, + }, + { + id: "jsx-text-after-expression", + kind: "preserving", + what: "add a component whose JSX text starts a // directive right after an expression container", + apply(fileName, source) { + if (!fileName.endsWith(".tsx")) return null; + return `${source}\nexport function ObsMapMutationB({ name }: { name: string }) {\n return

{name}// obs-map-disable request-context -- mutation corpus

;\n}\n`; + }, + }, + { + id: "jsx-text-block-directive", + kind: "preserving", + what: "add a component whose JSX text is a /* */ directive", + apply(fileName, source) { + if (!fileName.endsWith(".tsx")) return null; + return `${source}\nexport function ObsMapMutationC() {\n return

/* obs-map-disable audit-trail -- mutation corpus */

;\n}\n`; + }, + }, + + { + id: "delete-every-catch", + kind: "deleting", + what: "remove every catch clause", + apply(fileName, source) { + return applyEdits(source, catchDeletionEdits(parse(fileName, source))); + }, + }, + { + id: "delete-every-log", + kind: "deleting", + what: "remove every logger call statement", + apply(fileName, source) { + return applyEdits(source, logStatementEdits(parse(fileName, source))); + }, + }, + { + id: "delete-every-catch-and-log", + kind: "deleting", + what: "remove every catch clause and every logger call statement", + apply(fileName, source) { + const sf = parse(fileName, source); + return applyEdits(source, [...catchDeletionEdits(sf), ...logStatementEdits(sf)]); + }, + }, + + wrapEveryBody( + "wrap-body-in-rethrow", + "wrap every route body in try { ... } catch (e) { throw e }", + "try {", + "} catch (obsMapMutationError) { throw obsMapMutationError; }" + ), + wrapEveryBody( + "wrap-body-in-trace", + 'wrap every route body in trace("x", async () => { ... })', + 'return obsMapTrace("obs-map-mutation", async () => {', + "});" + ), + wrapEveryBody( + "wrap-body-in-single-element-map", + "wrap every route body in Promise.all([0].map(async () => { ... }))", + "return Promise.all([0].map(async () => {", + "})).then((obsMapResults) => obsMapResults[0]);" + ), + wrapEveryBody( + "wrap-body-in-single-element-flatmap", + "wrap every route body in Promise.all([0].flatMap(async () => { ... }))", + "return Promise.all([0].flatMap(async () => {", + "})).then((obsMapResults) => obsMapResults[0]);" + ), + wrapEveryBody( + "wrap-body-in-non-array-map", + "wrap every route body in a non-array receiver's .map(...)", + "return obsMapResult.map(async () => {", + "});" + ), + wrapEveryBody( + "wrap-body-in-non-array-filter", + "wrap every route body in a non-array receiver's .filter(...)", + "return obsMapPipe.filter(async () => {", + "});" + ), + + { + id: "throw-after-return-in-catch", + kind: "preserving", + what: "append throw e; after the first return in every catch", + apply(fileName, source) { + const sf = parse(fileName, source); + const edits: Edit[] = []; + for (const clause of catchClauses(sf)) { + const binding = bindingNameOf(clause); + if (binding === null) continue; + const returned = clause.block.statements.find(ts.isReturnStatement); + if (!returned) continue; + edits.push(insert(returned.end, ` throw ${binding};`)); + } + return applyEdits(source, edits); + }, + }, + + appendToEveryCatch( + "dead-if-false", + "preserving", + "append if (false) { throw e; } to every catch", + (e) => `if (false) { throw ${e}; }` + ), + appendToEveryCatch( + "dead-while-false", + "preserving", + "append while (false) { throw e; } to every catch", + (e) => `while (false) { throw ${e}; }` + ), + appendToEveryCatch( + "dead-for-false", + "preserving", + "append for (;false;) { throw e; } to every catch", + (e) => `for (;false;) { throw ${e}; }` + ), + appendToEveryCatch( + "dead-if-true-else", + "preserving", + "append if (true) { 0; } else { throw e; } to every catch", + (e) => `if (true) { 0; } else { throw ${e}; }` + ), + appendToEveryCatch( + "dead-switch-no-case", + "preserving", + "append switch (1) { case 2: throw e; } to every catch", + (e) => `switch (1) { case 2: throw ${e}; }` + ), + appendToEveryCatch( + "dead-inner-try", + "preserving", + "append try { 0; } catch { throw e; } to every catch", + (e) => `try { 0; } catch { throw ${e}; }` + ), + appendToEveryCatch( + "dead-for-of-empty", + "preserving", + "append for (const x of []) { throw e; } to every catch", + (e) => `for (const obsMapItem of []) { throw ${e}; }` + ), + appendToEveryCatch( + "dead-for-in-empty", + "preserving", + "append for (const k in {}) { throw e; } to every catch", + (e) => `for (const obsMapKey in {}) { throw ${e}; }` + ), + appendToEveryCatch( + "dead-if-empty-string", + "preserving", + 'append if ("") { throw e; } to every catch', + (e) => `if ("") { throw ${e}; }` + ), + appendToEveryCatch( + "dead-if-not-true", + "preserving", + "append if (!true) { throw e; } to every catch", + (e) => `if (!true) { throw ${e}; }` + ), + appendToEveryCatch( + "dead-if-const-compare", + "preserving", + "append if (1 === 2) { throw e; } to every catch", + (e) => `if (1 === 2) { throw ${e}; }` + ), + appendToEveryCatch( + "registered-throw", + "preserving", + "append [].push(() => { throw e; }) to every catch", + (e) => `[].push(() => { throw ${e}; });` + ), + appendToEveryCatch( + "empty-instanceof-if", + "preserving", + "append if (e instanceof Error) { } to every catch", + (e) => `if (${e} instanceof Error) { }` + ), + + { + id: "merge-declarations", + kind: "preserving", + what: "merge adjacent const statements into one declaration list", + apply(fileName, source) { + const sf = parse(fileName, source); + const edits: Edit[] = []; + forEachNode(sf, (node) => { + if (!ts.isBlock(node)) return; + const statements = node.statements; + for (let i = 1; i < statements.length; i++) { + const previous = statements[i - 1]!; + const current = statements[i]!; + if (!isSingleConst(previous) || !isSingleConst(current)) continue; + if (source[previous.end - 1] !== ";") continue; + const declaration = current.declarationList.declarations[0]!; + edits.push({ start: previous.end - 1, end: declaration.getStart(), text: ", " }); + } + }); + return applyEdits(source, edits); + }, + }, + + { + id: "merge-comma-expressions", + kind: "preserving", + what: "merge adjacent expression statements into one comma expression", + apply(fileName, source) { + const sf = parse(fileName, source); + const edits: Edit[] = []; + forEachNode(sf, (node) => { + if (!ts.isBlock(node) && !ts.isSourceFile(node)) return; + const statements = node.statements; + for (let i = 1; i < statements.length; i++) { + const previous = statements[i - 1]!; + const current = statements[i]!; + if (!ts.isExpressionStatement(previous) || !ts.isExpressionStatement(current)) continue; + if (source[previous.end - 1] !== ";") continue; + edits.push({ start: previous.end - 1, end: current.getStart(), text: ", " }); + } + }); + return applyEdits(source, edits); + }, + }, + + { + id: "inert-statements-after-try", + kind: "preserving", + what: "append five unused const declarations after every try statement in a route body", + apply(fileName, source) { + const sf = parse(fileName, source); + const edits: Edit[] = []; + let n = 0; + for (const body of entryBodies(sf)) { + forEachNode(body, (node) => { + if (!ts.isTryStatement(node)) return; + if (!node.parent || !ts.isBlock(node.parent)) return; + const filler = Array.from({ length: 5 }, () => `const obsMapInert${n++} = 1;`).join(" "); + edits.push(insert(node.end, `\n${filler}\n`)); + }); + } + return applyEdits(source, edits); + }, + }, +]; + +function isSingleConst(statement: ts.Statement): statement is ts.VariableStatement { + return ( + ts.isVariableStatement(statement) && + (statement.declarationList.flags & ts.NodeFlags.Const) !== 0 && + statement.declarationList.declarations.length === 1 && + statement.declarationList.declarations[0]!.initializer !== undefined + ); +} From eaafa3ddb9b593e65f54aff04a0824602dd0a14b Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Sat, 1 Aug 2026 20:54:15 +0100 Subject: [PATCH 055/106] test(observability-map): check that a docstring names a test that exists Six docstrings on this branch have asserted something unverifiable or false, and the sixth named `content-is-not-a-comment`, a test nobody ever wrote. The rule has been asked for in prose six times, so it is mechanical now. test/docstringReferences.test.ts scans src for three kinds of reference and asserts each resolves: a backticked kebab-case token, which in this package is always a check id, a corpus id or a test name; a backticked glob like dead-*, which must match a corpus entry by prefix; and a backticked prose phrase long enough and clean enough of code punctuation to be a test title. Three more cases run the same predicates over invented references, so the checker is known to be able to fail rather than merely green. What it does not check is written down beside it: a reference without backticks, a title of fewer than five words, and anything outside src. The kebab half is the half that has actually failed. It caught three references in this very change before they shipped. --- .../test/docstringReferences.test.ts | 185 ++++++++++++++++++ 1 file changed, 185 insertions(+) create mode 100644 internal-packages/observability-map/test/docstringReferences.test.ts diff --git a/internal-packages/observability-map/test/docstringReferences.test.ts b/internal-packages/observability-map/test/docstringReferences.test.ts new file mode 100644 index 00000000000..3800066b2c4 --- /dev/null +++ b/internal-packages/observability-map/test/docstringReferences.test.ts @@ -0,0 +1,185 @@ +import ts from "typescript"; +import { readFileSync, readdirSync } from "node:fs"; +import { join, resolve } from "node:path"; +import { CHECKS } from "../src/checks/index.js"; +import { MUTATIONS } from "./mutations.js"; + +/** + * Every test name a docstring in `src/` claims to be covered by must exist. + * + * The rule this enforces has been asked for six times in prose and broken six times, most recently + * by a docstring naming `content-is-not-a-comment`, a test that was never written. Prose cannot + * enforce itself, so this does. + * + * What is checked, precisely, because a checker that overstates its reach is the same defect again: + * + * - every backticked kebab-case token in a `src/` comment, e.g. `empty-instanceof-if`. Those are + * never valid JavaScript identifiers, so in this package they are always a check id, a mutation + * corpus id, a test name, or one of the handful of domain words in `NOT_A_TEST_NAME` below. + * - a backticked glob, `dead-*`, which must match at least one corpus id by prefix. + * - every backticked prose phrase of `MINIMUM_TITLE_WORDS` words or more that contains no code + * punctuation, e.g. `jsx text is content, not a comment`. That is what a test title looks like + * and what a code sample does not. + * + * What is NOT checked: a reference written without backticks, a test title of fewer than + * `MINIMUM_TITLE_WORDS` words (`throw e` and `new URL` are code, and telling a short title from + * short code needs more than punctuation), and anything outside `src/`. A docstring can still name + * a nonexistent short test. The kebab half is the half that has actually failed. + */ + +const SRC = resolve(__dirname, "../src"); +const TESTS = resolve(__dirname); + +/** Kebab-case tokens that are domain vocabulary rather than a test or corpus name. Anything added + * here is a deliberate statement that the token names no test, and shows up in review as such. */ +const NOT_A_TEST_NAME = new Set([ + // A `CheckStatus` value. + "not-applicable", + // The directive spelling that was retired, named in `suppression.ts` to say it is not honoured. + "obs-map-disable-next-line", +]); + +/** A backticked phrase this long or longer, with no code punctuation, is read as a test title. */ +const MINIMUM_TITLE_WORDS = 5; + +/** Characters that mean a backticked phrase is a code sample rather than a test title. */ +const CODE_PUNCTUATION = /[{}()[\];=<>"'`|&$/\\]|\.\.\.|\.tsx?\b/; + +function walkFiles(dir: string, suffix: string, out: string[] = []): string[] { + for (const entry of readdirSync(dir, { withFileTypes: true })) { + const path = join(dir, entry.name); + if (entry.isDirectory()) walkFiles(path, suffix, out); + else if (entry.name.endsWith(suffix)) out.push(path); + } + return out; +} + +/** Comment text with jsdoc line prefixes removed, so a backticked phrase that wrapped across two + * lines reads as one phrase rather than one with a stray asterisk in it. */ +function commentText(file: string): string { + const source = readFileSync(file, "utf8"); + const sf = ts.createSourceFile(file, source, ts.ScriptTarget.Latest, true); + const seen = new Set(); + const parts: string[] = []; + const visit = (node: ts.Node) => { + for (const range of ts.getLeadingCommentRanges(source, node.getFullStart()) ?? []) { + if (seen.has(range.pos)) continue; + seen.add(range.pos); + parts.push(source.slice(range.pos, range.end)); + } + ts.forEachChild(node, visit); + }; + visit(sf); + return parts.join("\n").replace(/\n\s*\*\s?/g, " "); +} + +/** Static titles from every `it`/`test`/`describe` call, including the literal chunks of a + * template-literal title, so a reference to part of a generated name still resolves. */ +function testTitles(): Set { + const titles = new Set(); + const add = (value: string) => { + const trimmed = value.replace(/\s+/g, " ").trim(); + if (trimmed.length > 0) titles.add(trimmed); + }; + for (const file of walkFiles(TESTS, ".test.ts")) { + const source = readFileSync(file, "utf8"); + const sf = ts.createSourceFile(file, source, ts.ScriptTarget.Latest, true); + const visit = (node: ts.Node) => { + if (ts.isCallExpression(node)) { + const callee = node.expression; + const root = ts.isPropertyAccessExpression(callee) ? callee.expression : callee; + if (ts.isIdentifier(root) && ["it", "test", "describe"].includes(root.text)) { + const first = node.arguments[0]; + if (first) { + if (ts.isStringLiteralLike(first)) add(first.text); + if (ts.isTemplateExpression(first)) { + add(first.head.text); + for (const span of first.templateSpans) add(span.literal.text); + } + } + } + } + ts.forEachChild(node, visit); + }; + visit(sf); + } + return titles; +} + +describe("docstrings in src name things that exist", () => { + const known = new Set([ + ...CHECKS.map((c) => c.id), + ...MUTATIONS.map((m) => m.id), + ...NOT_A_TEST_NAME, + ]); + const titles = testTitles(); + const corpusIds = MUTATIONS.map((m) => m.id); + const files = walkFiles(SRC, ".ts"); + + it("finds source files and test titles to check against", () => { + expect(files.length).toBeGreaterThan(5); + expect(titles.size).toBeGreaterThan(50); + expect(corpusIds.length).toBeGreaterThan(20); + }); + + it("every backticked kebab-case token names a check, a corpus entry or a test", () => { + const unknown: string[] = []; + for (const file of files) { + for (const match of commentText(file).matchAll(/`([a-z][a-z0-9]*(?:-[a-z0-9]+)+)`/g)) { + const token = match[1]!; + if (known.has(token)) continue; + if ([...titles].some((t) => t.includes(token))) continue; + unknown.push(`${file}: ${token}`); + } + } + expect(unknown).toEqual([]); + }); + + it("every backticked glob matches at least one corpus entry", () => { + const unmatched: string[] = []; + for (const file of files) { + for (const match of commentText(file).matchAll(/`([a-z][a-z0-9-]*)-\*`/g)) { + const prefix = `${match[1]!}-`; + if (corpusIds.some((id) => id.startsWith(prefix))) continue; + unmatched.push(`${file}: ${prefix}*`); + } + } + expect(unmatched).toEqual([]); + }); + + it("every backticked prose phrase long enough to be a test title is one", () => { + const unknown: string[] = []; + for (const file of files) { + for (const match of commentText(file).matchAll(/`([a-z][^`\n]*)`/g)) { + const phrase = match[1]!.replace(/\s+/g, " ").trim(); + if (phrase.split(" ").length < MINIMUM_TITLE_WORDS) continue; + if (CODE_PUNCTUATION.test(phrase)) continue; + if (titles.has(phrase)) continue; + unknown.push(`${file}: ${phrase}`); + } + } + expect(unknown).toEqual([]); + }); + + // The checker has to be able to fail, or it is decoration. These run the same predicates over an + // invented docstring rather than over `src/`, so the guarantee does not rest on `src/` currently + // happening to contain a bad reference. + it("would reject a docstring naming a test that does not exist", () => { + const invented = "see `content-is-not-a-comment` for the proof"; + const token = /`([a-z][a-z0-9]*(?:-[a-z0-9]+)+)`/.exec(invented)![1]!; + expect(known.has(token)).toBe(false); + expect([...titles].some((t) => t.includes(token))).toBe(false); + }); + + it("would reject a docstring naming a corpus glob that matches nothing", () => { + const prefix = /`([a-z][a-z0-9-]*)-\*`/.exec("covered by `no-such-family-*` above")![1]!; + expect(corpusIds.some((id) => id.startsWith(`${prefix}-`))).toBe(false); + }); + + it("would reject a docstring naming a prose test title that does not exist", () => { + const phrase = "reads a directive that nobody ever wrote down anywhere"; + expect(phrase.split(" ").length).toBeGreaterThanOrEqual(MINIMUM_TITLE_WORDS); + expect(CODE_PUNCTUATION.test(phrase)).toBe(false); + expect(titles.has(phrase)).toBe(false); + }); +}); From 3a95d7e9b15bee5e93e4d77a4768153b58efbc7a Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Sat, 1 Aug 2026 20:54:32 +0100 Subject: [PATCH 056/106] test(observability-map): test the JSX suppression fix, and name a test that exists The fix shipped last round with no test that fails without it. The one test that mentioned JSX used text with the // mid-string, which the comment-range lexers never read as a comment anyway, so removing ts.isJsxText left the whole suite green. The corpus cannot cover this by construction: scoreEntry caps an entry at its pre-suppression ratio, so a suppression can only lower a score, and a harness that watches for the score rising is blind to it. Four cases now cover the shapes that needed the fix, JSX text that BEGINS with // or /*, in four tree positions. Verified by removing ts.isJsxText: those four fail and nothing else does. A positive control beside them keeps a real comment in a JSX expression container working, so the filter cannot be widened until it eats real comments. The docstring named content-is-not-a-comment, which does not exist. It now names the tests that do. --- .../observability-map/src/suppression.ts | 13 ++++- .../test/suppression.test.ts | 58 +++++++++++++++++++ 2 files changed, 68 insertions(+), 3 deletions(-) diff --git a/internal-packages/observability-map/src/suppression.ts b/internal-packages/observability-map/src/suppression.ts index 26a51e13085..476c2a5a2ee 100644 --- a/internal-packages/observability-map/src/suppression.ts +++ b/internal-packages/observability-map/src/suppression.ts @@ -35,9 +35,16 @@ function leafTokens(node: ts.Node): ts.Node[] { * BEGINS with `//` or `/*` is the shape that reached the real tree, in * `resources.branches.create.tsx`'s `//`. * - * `content-is-not-a-comment` in `test/suppression.test.ts` covers each kind, and - * `jsx-text-line-directive`, `jsx-text-after-expression` and `jsx-text-block-directive` in the - * mutation corpus cover the JSX shapes over the whole route tree. + * The four cases in `jsx text is content, not a comment` (`test/suppression.test.ts`) are the ones + * that fail without `ts.isJsxText` here; the positive control beside them, `still reads a directive + * from a comment in a JSX expression container`, is what stops the filter being widened until it + * eats real comments. `does not suppress from a directive inside a template literal` and the two + * substitution cases cover the template kinds, and `ignores the directive inside a string literal` + * covers the string kind. + * + * The mutation corpus does NOT cover any of this, and cannot: a suppression can only lower an + * entry's score, because `scoreEntry` caps it at the pre-suppression ratio. Suppression bugs are + * invisible to a harness that watches for the score rising, so they need ordinary unit tests. */ function isClaimedContent(node: ts.Node): boolean { return ( diff --git a/internal-packages/observability-map/test/suppression.test.ts b/internal-packages/observability-map/test/suppression.test.ts index 7ef32f9f94e..c40d1332894 100644 --- a/internal-packages/observability-map/test/suppression.test.ts +++ b/internal-packages/observability-map/test/suppression.test.ts @@ -142,6 +142,64 @@ describe("suppressedChecks", () => { expect(m.size).toBe(0); }); + // S4. The case above passes without any JSX handling at all, because the comment-range lexers + // only find a comment at the exact offset they are asked about and the `//` there is mid-text. + // These are the shapes that actually needed the fix: JSX text that BEGINS with a comment marker, + // which is what the lexers see when they are pointed at the start of a JsxText node. Removing + // `ts.isJsxText` from `isClaimedContent` makes all four fail and nothing else in the suite. + describe("jsx text is content, not a comment", () => { + const page = (body: string) => `const name = "x"; + export default function Page() { + return ${body}; + }`; + + it("does not suppress from JSX text beginning with a line comment marker", () => { + const m = suppressedChecks( + page(`

// obs-map-disable error-classification -- jsx line

`), + "route.tsx" + ); + expect(m.size).toBe(0); + }); + + it("does not suppress from JSX text beginning with a block comment marker", () => { + const m = suppressedChecks( + page(`

/* obs-map-disable audit-trail -- jsx block */

`), + "route.tsx" + ); + expect(m.size).toBe(0); + }); + + it("does not suppress from JSX text starting right after an expression container", () => { + const m = suppressedChecks( + page(`

{name}// obs-map-disable request-context -- after expression

`), + "route.tsx" + ); + expect(m.size).toBe(0); + }); + + // A fourth input, exercising the same mechanism at a different tree position: the text is not + // the first child of the outermost element, so the token boundary the lexer is pointed at is a + // different one again. + it("does not suppress from JSX text nested several elements deep", () => { + const m = suppressedChecks( + page(`
// obs-map-disable auth-boundary -- nested jsx
`), + "route.tsx" + ); + expect(m.size).toBe(0); + }); + + // Positive control for the same code path: a real comment inside a JSX expression container is + // not JSX text and must survive. A filter that dropped it would pass every test above for the + // wrong reason. + it("still reads a directive from a comment in a JSX expression container", () => { + const m = suppressedChecks( + page(`

{/* obs-map-disable audit-trail -- real comment */}

`), + "route.tsx" + ); + expect(m.get("audit-trail")).toBe("real comment"); + }); + }); + // Extra inputs beyond the brief's two, exercising the same "not a real parse position" mechanism // differently: a second substitution, and a string literal nested inside a JSX expression // container, which is a different node kind again from either hole above. From 07dc007e63cbca774ce0f7495ea85f7834dbf974 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Sat, 1 Aug 2026 20:54:55 +0100 Subject: [PATCH 057/106] fix(observability-map): refuse three ways of adding error handling that does nothing Every laundering shape found before this round took real signal away or moved it about. These three add fake signal, which is what someone reaches for when a CI comment nags them, and two of them are the largest holes ever found here. Global 15 before and after; no route's score rose from any of them. A catch whose try block cannot raise is no longer read as error handling. Prepending try { 0; } catch (e) { if (e instanceof Error) { return 400; } throw e; } to every body took the tree from 15 to 42 and raised 224 routes, because 261 routes catch nothing and sat at not-applicable, and a dead clause moved every one of them to pass. guardedWork now reports whether the block does anything that could reach the clause: a call, a construction, an await, a member access, a throw, an iteration, an instanceof. What is not on that list, a temporal-dead-zone read and a coercion that raises, is written down. A throw written after something that already exited no longer reads as a rethrow. definitelyExits sees through a bare block, a do body, an if/else where both arms exit, a switch with a returning default and a try/finally, where before it recognised only a bare return or throw. The if (true) form needs constant folding this file will not do, and is answered instead by rethrows requiring the clause to contain no reachable return at all: the claim rethrows feeds is that the error leaves the way it arrived, which is only true when throwing is the only way out. A ternary on the error has to send its arms somewhere different, which the if/switch path has required since the round before. return e instanceof Error ? (X) : (X) was worth 50 points a route for a test that decides nothing, and the same comparison now applies to an if/else with identical arms. The 4xx and pMap claims in two docstrings were false in one direction each and now say what is true. Each fix has a test that fails without it, listed in the round A fix 3 report. --- .../src/checks/errorClassification.ts | 32 ++- .../observability-map/src/scan.ts | 218 +++++++++++++---- .../observability-map/src/types.ts | 16 +- .../observability-map/test/checks.test.ts | 57 +++++ .../test/integration.test.ts | 3 +- .../observability-map/test/scan.test.ts | 221 +++++++++++++++++- 6 files changed, 488 insertions(+), 59 deletions(-) diff --git a/internal-packages/observability-map/src/checks/errorClassification.ts b/internal-packages/observability-map/src/checks/errorClassification.ts index b377e577cf2..22a431ab764 100644 --- a/internal-packages/observability-map/src/checks/errorClassification.ts +++ b/internal-packages/observability-map/src/checks/errorClassification.ts @@ -67,9 +67,12 @@ const NARROW_TRY_STATEMENTS = 2; * await, was measured too and is worse still: it refuses the four `matchPattern.slice(4); new * RegExp(...)` guards, because preparing a parse's input is ordinary synchronous string work. * - * The residual, since awaiting is the signal: a try block that does its non-parse work - * synchronously still reads as a guard. Nothing in the tree does, and it is written down in the - * round A fix 2 report rather than defended. + * Two residuals, since awaiting is the signal. A try block that does its non-parse work + * synchronously still reads as a guard. And `guardedWork` looks for a `ts.AwaitExpression`, which + * `for await (const chunk of work(await request.json()))` and `await using` are not, so a block + * whose only non-parse work is one of those reads as a guard too. Neither occurs in the tree and + * neither is reachable by rewriting a real route, since both need work that is not there to begin + * with. Both are in the round A fix 3 report. */ function isParseGuard(clause: CatchEvidence): boolean { return ( @@ -148,6 +151,14 @@ export function usesBuilder(ep: EntryPoint): boolean { * points to anyone who wraps a body in something the boundary rule refuses, which * `Promise.all([0].map(async () => { ... }))` did. It fails instead. The precision cost is a route * that genuinely only handles errors per item, which now fails rather than sitting out. + * + * A clause whose try block cannot raise is read as no clause at all, `guardCanRaise` on the + * evidence. It is not error handling, and crediting one was the largest hole ever found here: + * prepending `try { 0; } catch (e) { if (e instanceof Error) { return json(x, { status: 400 }); } + * throw e; }` to every body took the tree from 15 to 42 and raised 224 routes, because the 261 + * routes that catch nothing were sitting at not-applicable and a dead clause moved each of them to + * pass. Dropping it here rather than in the scan keeps the evidence honest about what is written + * and puts the judgement where the other judgements are. */ export const errorClassification = { id: ID, @@ -155,30 +166,33 @@ export const errorClassification = { if (isTrivial(ep)) { return { id: ID, status: "not-applicable", detail: "trivial route" }; } - const swallowed = ep.catches.filter(swallows); + const reachable = ep.catches.filter((c) => c.guardCanRaise); + const swallowed = reachable.filter(swallows); if (swallowed.length > 0) { const which = - ep.catches.length > 1 ? ` (${swallowed.length} of ${ep.catches.length} catches)` : ""; + reachable.length > 1 ? ` (${swallowed.length} of ${reachable.length} catches)` : ""; return { id: ID, status: "fail", detail: `catches its errors and takes one way out regardless of what was thrown${which}`, }; } - if (ep.catches.length === 0 && ep.callbackCatches > 0) { + if (reachable.length === 0 && ep.callbackCatches > 0) { return { id: ID, status: "fail", detail: "its only error handling sits in a callback the route does not own", }; } - if (!ep.catches.some(decides)) { + if (!reachable.some(decides)) { return { id: ID, status: "not-applicable", detail: - ep.catches.length === 0 - ? "catches nothing, so it classifies nothing" + reachable.length === 0 + ? ep.catches.length === 0 + ? "catches nothing, so it classifies nothing" + : "guards nothing that can throw, so it classifies nothing" : "every catch rethrows and nothing else, so it classifies nothing", }; } diff --git a/internal-packages/observability-map/src/scan.ts b/internal-packages/observability-map/src/scan.ts index da46cb4e54f..eaf14cd4aa6 100644 --- a/internal-packages/observability-map/src/scan.ts +++ b/internal-packages/observability-map/src/scan.ts @@ -148,8 +148,35 @@ function isBodyRead(node: ts.Node): boolean { } /** - * What the guarded region does, in the two terms `error-classification` needs to tell a parse guard - * from a handler wrapped around a parse. + * Syntax that can raise. Everything a try block might do that produces something for a catch clause + * to catch: a call, a construction, a tagged template, an `await` or `yield` (the awaited promise + * rejects), a member access (the base may be null or undefined), a `throw`, an iteration (the + * iterator protocol raises on a non-iterable), and `instanceof`/`in` (a TypeError on a non-object + * right side). + * + * See `guardedWork` for what is NOT on this list and why that is a disclosed residual rather than + * an oversight. + */ +function canRaise(node: ts.Node): boolean { + return ( + ts.isCallExpression(node) || + ts.isNewExpression(node) || + ts.isTaggedTemplateExpression(node) || + ts.isAwaitExpression(node) || + ts.isYieldExpression(node) || + ts.isPropertyAccessExpression(node) || + ts.isElementAccessExpression(node) || + ts.isThrowStatement(node) || + ts.isForOfStatement(node) || + ts.isForInStatement(node) || + (ts.isBinaryExpression(node) && + (node.operatorToken.kind === ts.SyntaxKind.InstanceOfKeyword || + node.operatorToken.kind === ts.SyntaxKind.InKeyword)) + ); +} + +/** + * What the guarded region does, in the three terms `error-classification` needs. * * `guardsParse` is whether anything in it parses at all. A `new URL(x)` counts and has to be read * as a `ts.isNewExpression` here, because the call-callee scan that builds `calleeNames` never sees @@ -162,15 +189,36 @@ function isBodyRead(node: ts.Node): boolean { * catch reaches a service: `try { const body = await request.json(); return await * handleEverything(body); }`. * - * Nested function bodies are skipped: a callback written inside the try is not work the try is - * guarding on this pass through. + * `canRaise` is whether the block does anything at all that could reach the clause. A clause whose + * try block cannot raise is not error handling, and reading one as classification paid 50 points a + * route to anyone willing to prepend `try { 0; } catch (e) { if (e instanceof Error) { return + * json(x, { status: 400 }); } throw e; }` to a body: on the real tree that took the global from 15 + * to 42 and raised 224 routes, which is more than every other shape found on this branch put + * together. `dead-classifying-try` in the mutation corpus is the tree-scale version. + * + * The alternative considered was proving unreachability the other way round, by ruling out every + * expression that could throw. It is the same predicate read backwards and it fails the same way, + * so the cheaper direction won: ask what the block DOES, and require it to do something. The + * residual is what `canRaise` does not list. A temporal-dead-zone read (`try { const x = later; }`), + * a coercion that raises (`try { const x = 1 + someSymbol; }`) and a `delete` on a frozen object are + * all treated as unable to raise. None is expressible as a preserving rewrite of a real route, and + * all of them would need types the scanner does not have. + * + * Nested function bodies are skipped throughout: a callback written inside the try is not work the + * try is guarding on this pass through. A `throw` inside one is not either, which is deliberate. */ -function guardedWork(tryBlock: ts.Block): { guardsParse: boolean; awaitsOnlyParse: boolean } { +function guardedWork(tryBlock: ts.Block): { + guardsParse: boolean; + awaitsOnlyParse: boolean; + guardCanRaise: boolean; +} { let guardsParse = false; let awaitsOnlyParse = true; + let guardCanRaise = false; const visit = (node: ts.Node) => { if (ts.isFunctionLike(node)) return; if (isParseCall(node)) guardsParse = true; + if (canRaise(node)) guardCanRaise = true; if (ts.isAwaitExpression(node)) { const awaited = unwrap(node.expression); if (!isParseCall(awaited) && !isBodyRead(awaited)) awaitsOnlyParse = false; @@ -178,7 +226,7 @@ function guardedWork(tryBlock: ts.Block): { guardsParse: boolean; awaitsOnlyPars ts.forEachChild(node, visit); }; visit(tryBlock); - return { guardsParse, awaitsOnlyParse }; + return { guardsParse, awaitsOnlyParse, guardCanRaise }; } /** Whether some node in the tree rooted at `node` matches `predicate`. */ @@ -267,6 +315,11 @@ function catchBindingName(clause: ts.CatchClause): string | null { return decl && ts.isIdentifier(decl.name) ? decl.name.text : null; } +/** A node's source text with all whitespace removed, for comparing two branch arms. */ +function normalizedText(node: ts.Node): string { + return node.getText().replace(/\s+/g, ""); +} + /** * Whether a conditional expression tests the error to pick what the clause does, rather than to * word what it says. The caller only offers it the whole value of a `return`/`throw`, so @@ -277,36 +330,88 @@ function catchBindingName(clause: ts.CatchClause): string | null { * Goes through `referencesBinding`, the same predicate the `if`/`switch` check uses, rather than * accepting any `instanceof` in the condition: an `instanceof` that never reads the caught binding * is not a decision made on the error, and a bindingless catch has nothing here to reference. + * + * The two arms also have to differ, which is the same requirement `selectsADistinctPath` makes of + * an `if`. `return e instanceof Error ? (X) : (X)` is a test whose outcome is the same either way, + * and it was worth 50 points a route; `same-arms-ternary` in the mutation corpus is the tree-scale + * version, and `test/scan.test.ts` has the unit case. Parentheses and whitespace are stripped + * before the comparison, so the shape has to differ in something a reader would call a difference. + * The residual both branch tests share is stated once, on `selectsADistinctPath`. */ function selectsAnErrorPath(node: ts.ConditionalExpression, bindingName: string | null): boolean { if (bindingName === null) return false; if (!containsInstanceOf(node.condition)) return false; - return referencesBinding(node.condition, bindingName); + if (!referencesBinding(node.condition, bindingName)) return false; + return normalizedText(unwrap(node.whenTrue)) !== normalizedText(unwrap(node.whenFalse)); } -/** A statement that unconditionally leaves the statement list it sits in, so anything after it in - * the same list never runs. */ -function isDefiniteExit(statement: ts.Statement): boolean { - return ( +/** + * A statement that leaves the statement list it sits in on every path through itself, so anything + * after it in the same list never runs. + * + * Recognises a nested construct, not only a bare `return`/`throw`/`break`/`continue`. Recognising + * only the bare form is what let a dead `throw error;` count as a rethrow when the statement before + * it was a block, a `do` body or an `if`/`else` that returned; `dead-throw-after-*` in the mutation + * corpus is that family, and `test/scan.test.ts` has one case per construct. + * + * A sound under-approximation. `if` without an `else`, a labelled statement (a `break` to the label + * escapes it) and every other loop form answer false, because none of them is guaranteed to run its + * body. Saying false when the truth is true only leaves a later statement in the list, which is the + * direction that withholds evidence rather than inventing it. + */ +function definitelyExits(statement: ts.Statement): boolean { + if ( ts.isReturnStatement(statement) || ts.isThrowStatement(statement) || ts.isContinueStatement(statement) || ts.isBreakStatement(statement) - ); + ) { + return true; + } + if (ts.isBlock(statement)) return statement.statements.some(definitelyExits); + // A `do` body runs before its condition is ever read. + if (ts.isDoStatement(statement)) return definitelyExits(statement.statement); + if (ts.isIfStatement(statement)) { + return ( + statement.elseStatement !== undefined && + definitelyExits(statement.thenStatement) && + definitelyExits(statement.elseStatement) + ); + } + if (ts.isTryStatement(statement)) { + if (statement.finallyBlock && definitelyExits(statement.finallyBlock)) return true; + if (!definitelyExits(statement.tryBlock)) return false; + return statement.catchClause === undefined || definitelyExits(statement.catchClause.block); + } + if (ts.isSwitchStatement(statement)) { + const clauses = statement.caseBlock.clauses; + const last = clauses[clauses.length - 1]; + if (!clauses.some(ts.isDefaultClause) || last === undefined) return false; + // An empty clause falls through to the next one, so it does not have to exit itself; the last + // clause has nothing to fall through to and does. + return ( + clauses.every((c) => c.statements.length === 0 || c.statements.some(definitelyExits)) && + last.statements.some(definitelyExits) + ); + } + return false; } -/** - * `statements` up to and including the first one that definitely exits. Not full flow analysis: - * an `if`/`else` where both branches return is not itself recognised as an exit, only a bare - * `return`, `throw`, `continue` or `break` is. That is enough to make a `throw e;` appended after - * a `return` dead code rather than evidence the clause rethrows, which is the one shape a mutation - * testing this check actually produced. - */ +/** `statements` up to and including the first one that definitely exits. */ function reachableStatements(statements: readonly ts.Statement[]): readonly ts.Statement[] { - const index = statements.findIndex(isDefiniteExit); + const index = statements.findIndex(definitelyExits); return index === -1 ? statements : statements.slice(0, index + 1); } +/** Whether the clause returns anywhere at all, nested functions excluded. A clause with a `return` + * on any path has an exit that is not the throw, so the error does not leave it the way it arrived. + */ +function containsReturn(node: ts.Node): boolean { + if (ts.isFunctionLike(node)) return false; + if (ts.isReturnStatement(node)) return true; + return ts.forEachChild(node, containsReturn) === true; +} + /** Whether the tree rooted at `node` contains a `return` or a `throw` of its own, not counting one * inside a nested function. What separates an arm that takes the error somewhere from an arm that * runs and falls back into the clause's single common exit. */ @@ -323,16 +428,25 @@ function containsExit(node: ts.Node): boolean { * changed the wording and not the outcome. The empty-body form was the cheapest no-op in the tool, * worth 50 points a route; `empty-instanceof-if` in the mutation corpus is the tree-scale version. * - * Two arms that return the SAME value pass this and should not. That residual is written down in - * the round A fix 2 report rather than defended: telling two returns apart needs the values - * compared, which is a different kind of analysis from anything else here. + * An `if`/`else` whose two arms are textually identical does not count, the same comparison + * `selectsAnErrorPath` makes of a ternary's arms. + * + * The residual both branch tests share, stated here once for both: two arms that produce the same + * outcome by different spellings still read as a real decision. + * `if (e instanceof Error) { return json(x); } return Response.json(x);` counts and decides + * nothing, and so does the `if` with no `else` whose arm returns what the statement after it + * returns. Telling those apart needs the produced values compared for meaning rather than for text, + * which is a different kind of analysis from anything else in this file. The textual comparison is + * the cheapest thing that catches the copy-paste form, which is the one a mutation produces. */ function selectsADistinctPath(statement: ts.IfStatement | ts.SwitchStatement): boolean { if (ts.isIfStatement(statement)) { - return ( - containsExit(statement.thenStatement) || - (statement.elseStatement !== undefined && containsExit(statement.elseStatement)) - ); + const otherwise = statement.elseStatement; + if (otherwise !== undefined) { + if (normalizedText(statement.thenStatement) === normalizedText(otherwise)) return false; + return containsExit(statement.thenStatement) || containsExit(otherwise); + } + return containsExit(statement.thenStatement); } return statement.caseBlock.clauses.some((clause) => clause.statements.some(containsExit)); } @@ -345,20 +459,29 @@ function selectsADistinctPath(statement: ts.IfStatement | ts.SwitchStatement): b * `throw` or a test that sits inside an `if`, a loop, a `switch`, a nested `try` or a callback is * not on that path, so it does not count. * - * That is the whole dead-code defence, and it replaces the list of statically-false shapes the - * previous round kept extending. The list was losing: `if (false)` and `while (false)` were + * That is the whole dead-code defence, and it replaces the list of statically-false shapes an + * earlier round kept extending. The list was losing: `if (false)` and `while (false)` were * recognised, and `for (;false;)`, `if (true) {} else`, `switch (1) { case 2: }`, `try {} catch`, * `for (const x of [])`, `for (const k in {})`, `if ("")`, `if (!true)` and `if (1 === 2)` were not, * each worth 50 points a route. Asking for the throw to be unconditional refuses all eleven without - * naming any of them, and refuses the twelfth nobody has written yet. `dead-*` in the mutation - * corpus is the tree-scale proof, one entry per shape. + * naming any of them. `dead-*` in the mutation corpus is the tree-scale proof, one entry per shape. * - * The cost is real: `catch (e) { if (transient) throw e; return null; }` no longer reads as a - * rethrow, so it reads as a swallow and fails rather than sitting out. That is the direction to be - * wrong in, since the reverse hands out points. + * `rethrows` asks for one thing more: that the clause contains no `return` at all. The claim it + * feeds is that the clause passes the error through unchanged, which is only true when throwing is + * the ONLY way out. Without it a `throw error;` written after a statement that already exited read + * as a rethrow, in seven spellings: after a bare block, a `do` body, an `if (true)`, an `if`/`else` + * where both arms return, a `switch` with a returning default, and a `try`/`finally` that returns. + * `definitelyExits` handles most of those on its own, and the no-return rule handles the rest + * without any constant folding. `dead-throw-after-*` in the mutation corpus covers them. + * + * The cost is real, in both rules. `catch (e) { if (transient) throw e; return null; }` no longer + * reads as a rethrow, so it reads as a swallow and fails rather than sitting out, and neither does + * `catch (e) { if (e instanceof Response) return e; throw e; }`, which passes on its branch instead. + * That is the direction to be wrong in, since the reverse hands out points. */ function catchClauseEvidence(clause: ts.CatchClause): { rethrows: boolean; branches: boolean } { let rethrows = false; + let returns = false; let branches = false; const bindingName = catchBindingName(clause); @@ -378,12 +501,16 @@ function catchClauseEvidence(clause: ts.CatchClause): { rethrows: boolean; branc continue; } // A `do` body runs before its condition is ever read, so it is on the straight-line path - // whatever the condition says. The only loop form that is. + // whatever the condition says. The only loop form that is; `definitelyExits` agrees. if (ts.isDoStatement(statement)) { const body = statement.statement; walk(ts.isBlock(body) ? body.statements : [body]); continue; } + // Any other reachable statement that could return means throwing is not the only way out. + // Read here rather than over the whole clause so a `return` the walk has already cut as dead + // does not count, which is what a `do { throw e; } while (false); return null;` produces. + if (containsReturn(statement)) returns = true; if (bindingName === null || shadowed) continue; if ( @@ -407,7 +534,7 @@ function catchClauseEvidence(clause: ts.CatchClause): { rethrows: boolean; branc }; walk(clause.block.statements); - return { rethrows, branches }; + return { rethrows: rethrows && !returns, branches }; } /** @@ -451,11 +578,20 @@ function isAtMostSingletonArray(expr: ts.Expression): boolean { * whole body })` collected it. * * Two things changed. A receiver that is an array literal of one element or none is refused here, - * because it cannot iterate. And the direction that pays no longer pays: `walkBody` counts the - * catches it refuses, and `error-classification` fails a route whose only catches were refused - * rather than excusing it. So a wrong answer here costs precision, not points. That is what makes - * the name list survivable, and it is why `Result.map(...)`, which no name list can tell from - * `users.map(...)`, is a corpus entry that passes rather than a hole. + * because it cannot iterate. And the direction that used to pay no longer pays: `walkBody` counts + * the catches it refuses, and `error-classification` fails a route whose only catches were refused + * rather than excusing it. That is what makes the name list survivable, and it is why + * `Result.map(...)`, which no name list can tell from `users.map(...)`, is a corpus entry that + * passes rather than a hole. + * + * The other direction still costs points and the earlier version of this comment said otherwise. + * A per-item callback under a callee the name list does not know, `pMap(items, cb)` or + * `Array.prototype.map.call(items, cb)`, is attributed to the route, so a per-element catch that + * decides can carry the route to `pass`. No mutation of a real route produces it: the reviewer + * tried `Array.prototype.map.call` over the tree and it moved nothing, because a route has to + * already be iterating for the shape to exist. It is a wrong verdict waiting for a route to be + * written that way, not a laundering path, and it is why this list is worth extending when a new + * iteration helper shows up in the tree. */ function isIterationCallback(node: ts.Node): boolean { const parent = node.parent; diff --git a/internal-packages/observability-map/src/types.ts b/internal-packages/observability-map/src/types.ts index ff0ad245660..e05d5e7595d 100644 --- a/internal-packages/observability-map/src/types.ts +++ b/internal-packages/observability-map/src/types.ts @@ -13,9 +13,12 @@ export type CheckResult = { */ export type CatchEvidence = { /** - * The clause throws on its own straight-line path: a `throw` among its statements, or among a - * bare nested block's, reached before anything that definitely exits. A throw guarded by an `if`, - * a loop, a `switch`, a nested `try` or a callback does not count, however the guard is spelled. + * Throwing is the clause's only way out. Two conditions: a `throw` is reached on its + * straight-line path (its own statements and a bare nested block's or a `do` body's, cut at the + * first statement that definitely exits, see `definitelyExits`), and the clause contains no + * `return` anywhere. A throw guarded by an `if`, a loop, a `switch`, a nested `try` or a callback + * does not count, however the guard is spelled, and neither does one written after something that + * has already returned. */ rethrows: boolean; /** @@ -36,6 +39,13 @@ export type CatchEvidence = { * its catch. */ guardsParse: boolean; + /** + * The guarded region does something that could raise at all: a call, a construction, an `await`, + * a member access, a `throw`, an iteration, an `instanceof`. A clause whose try block cannot + * raise is unreachable, so it is not error handling and `error-classification` reads no evidence + * off it. See `canRaise` in `scan.ts` for what is not on that list. + */ + guardCanRaise: boolean; /** * Everything the guarded region waits for is one of those parses. What separates * `try { const body = await request.json(); } catch { 400 }` from diff --git a/internal-packages/observability-map/test/checks.test.ts b/internal-packages/observability-map/test/checks.test.ts index 21150799ab4..bca5e4f6f4c 100644 --- a/internal-packages/observability-map/test/checks.test.ts +++ b/internal-packages/observability-map/test/checks.test.ts @@ -546,6 +546,63 @@ describe("error-classification", () => { expect(r.detail).toContain("catches nothing"); }); + // S2 at the check level. The evidence tests in `scan.test.ts` pin `guardCanRaise` itself; these + // pin the check reading it, which is where the 50 points were. Prepending this to a route that + // catches nothing took it from not-applicable to pass, and 224 routes were in exactly that state. + it("is not applicable to a route whose only catch guards a try that cannot throw", () => { + const r = run( + "error-classification", + "prepended.ts", + `import { prisma } from "~/db.server"; + export async function action({ request }) { + try { 0; } catch (e) { + if (e instanceof Error) { return new Response(null, { status: 400 }); } + throw e; + } + const rows = await prisma.thing.findMany(); + return json({ rows }); + }` + ); + expect(r.status).toBe("not-applicable"); + expect(r.detail).toContain("guards nothing that can throw"); + }); + + it("still fails a swallow that a dead classifying catch was prepended to", () => { + const r = run( + "error-classification", + "prepended-swallow.ts", + `import { prisma } from "~/db.server"; + export async function action({ request }) { + try { 0; } catch (e) { + if (e instanceof Error) { return new Response(null, { status: 400 }); } + throw e; + } + try { + return json(await prisma.thing.findMany()); + } catch (error) { + return new Response(null, { status: 500 }); + } + }` + ); + expect(r.status).toBe("fail"); + }); + + it("still passes the same classifying catch once its try does real work", () => { + const r = run( + "error-classification", + "live.ts", + `import { prisma } from "~/db.server"; + export async function action({ request }) { + try { await prisma.thing.findMany(); } catch (e) { + if (e instanceof Error) { return new Response(null, { status: 400 }); } + throw e; + } + return json({ ok: true }); + }` + ); + expect(r.status).toBe("pass"); + }); + // C2. A single-element array cannot iterate, so `[0].map(async () => { whole body })` is not a // per-item boundary and the route's own catch is found where it always was. Before this, the // wrapper deleted the route's catches and took a swallow from fail to not-applicable. diff --git a/internal-packages/observability-map/test/integration.test.ts b/internal-packages/observability-map/test/integration.test.ts index 032f960fb5e..4db7dfc223a 100644 --- a/internal-packages/observability-map/test/integration.test.ts +++ b/internal-packages/observability-map/test/integration.test.ts @@ -64,5 +64,6 @@ describe("scanning the real webapp routes", () => { expect(after.measured).toBe(before.measured); expect(after.unmeasured).toBe(before.unmeasured); expect(after.global).not.toBeGreaterThan(before.global!); - }); + // Two full tree scans plus a re-scan of every source, which does not fit the suite default. + }, 60_000); }); diff --git a/internal-packages/observability-map/test/scan.test.ts b/internal-packages/observability-map/test/scan.test.ts index 9c5bc9e468c..5e08f0f3a5a 100644 --- a/internal-packages/observability-map/test/scan.test.ts +++ b/internal-packages/observability-map/test/scan.test.ts @@ -658,11 +658,13 @@ describe("scanFile: catch clause evidence", () => { expect(ep!.catches[0]).toMatchObject({ rethrows: false, branches: false }); }); - // Every shape anyone has found that puts a `throw` somewhere it can never run. The previous - // round recognised the first two by folding the literal `false`, and lost to the other nine. - // None of them is named in the rule now: a throw counts when it is unconditional, and every - // one of these is guarded by something. `dead-*` in the mutation corpus runs the same list over - // the whole route tree. + // Eleven shapes that put a `throw` somewhere it can never run, all of them found by review + // rather than by this suite. An earlier round recognised the first two by folding the literal + // `false` and lost to the other nine. None of them is named in the rule now: a throw counts + // when it is unconditional, and every one of these is guarded by something. There is no claim + // that the list is complete, and a twelfth family arrived the round after it was written, see + // `dead throw written after something that already exited`. `dead-*` in the mutation corpus + // runs the same list over the whole route tree. const DEAD_SHAPES: Array<[string, string]> = [ ["if (false)", "if (false) { throw e; }"], ["while (false)", "while (false) { throw e; }"], @@ -713,6 +715,159 @@ describe("scanFile: catch clause evidence", () => { }); }); + // S1. The other end of the same problem. `reachableStatements` used to cut the statement list + // only on a BARE `return`/`throw`, while the walk descended into blocks and `do` bodies, so a + // `throw e;` written after a nested construct that had already returned was still read as the + // clause rethrowing. Every one of these takes a swallow from `fail` to `not-applicable`, worth 50 + // points a route, and they are semantics-preserving because the throw cannot run. + // + // Two rules answer them together. `definitelyExits` sees through the block, the `do` and the + // `if`/`else`, the `switch` and the `try`/`finally`; the `if (true)` form needs constant folding + // that this file deliberately does not do, and is answered instead by `rethrows` requiring the + // clause to contain no reachable `return` at all. `dead-throw-after-*` in the mutation corpus + // runs all six over the whole route tree. + describe("dead throw written after something that already exited", () => { + const exiting = (wrapped: string) => ` + export async function loader() { + try { + return await prisma.thing.findMany(); + } catch (e) { + ${wrapped} + throw e; + } + } + `; + + const EXITED: Array<[string, string]> = [ + ["a bare block", "{ logger.error(e); return null; }"], + ["a do body", "do { return null; } while (false);"], + ["an if (true)", "if (true) { return null; }"], + ["an if/else where both arms return", "if (pick()) { return null; } else { return 0; }"], + ["a switch with a returning default", "switch (1) { default: return null; }"], + ["a try/finally that returns", "try { return null; } finally { }"], + ]; + + for (const [label, wrapped] of EXITED) { + it(`does not set rethrows for a throw after ${label}`, () => { + const ep = scanFile("x.ts", exiting(wrapped)); + expect(ep!.catches[0]!.rethrows).toBe(false); + }); + } + + // The same six wrappers on the branches side, which is what pins `definitelyExits` itself: the + // no-return rule above says nothing about branch credit, so only the cut sees these. An error + // test written after a construct that already returned is dead code and must not read as the + // clause deciding anything. + // + // `an if (true)` is absent from this list on purpose. Nothing here evaluates a condition, so + // the cut cannot see that the wrapper always exits, and the error test after it is still + // credited. That residual is `dead-branch-after-if-true` in the mutation corpus, which runs as + // an expected failure with the two rejected alternatives written out beside it. + const BRANCH_EXITED = EXITED.filter(([label]) => label !== "an if (true)"); + + for (const [label, wrapped] of BRANCH_EXITED) { + it(`does not credit an error test written after ${label}`, () => { + const ep = scanFile( + "x.ts", + `export async function loader() { + try { return await prisma.thing.findMany(); } + catch (e) { + ${wrapped} + if (e instanceof Error) { return json({ a: 1 }); } + } + }` + ); + expect(ep!.catches[0]!.branches).toBe(false); + }); + } + + it("still credits an error test with nothing exiting before it", () => { + const ep = scanFile( + "x.ts", + `export async function loader() { + try { return await prisma.thing.findMany(); } + catch (e) { + logger.error(e); + if (e instanceof Error) { return json({ a: 1 }); } + } + }` + ); + expect(ep!.catches[0]!.branches).toBe(true); + }); + + // Positive control: nothing before the throw exits, so the throw is real and the clause has no + // other way out. + it("still sets rethrows when nothing before the throw returns", () => { + const ep = scanFile("x.ts", exiting("logger.error(e);")); + expect(ep!.catches[0]!.rethrows).toBe(true); + }); + + // Second positive control, for the no-return half specifically: a `return` the walk has already + // cut as dead must not count against the rethrow. + it("still sets rethrows when the only return is dead code after the throw", () => { + const ep = scanFile( + "x.ts", + `export async function loader() { + try { return await prisma.thing.findMany(); } + catch (e) { throw e; return null; } + }` + ); + expect(ep!.catches[0]!.rethrows).toBe(true); + }); + }); + + // S2. A clause whose try block cannot throw is unreachable, so it is not error handling and + // nothing should be read off it. Crediting one was the largest hole ever found here: prepending + // this to a body took the real tree from 15 to 42 and raised 224 routes, because the 261 routes + // that catch nothing sat at `not-applicable` and a dead clause moved every one of them to `pass`. + // `dead-classifying-try` in the mutation corpus is the tree-scale version. + describe("a catch over a try block that cannot throw", () => { + const guarding = (guarded: string) => ` + export async function loader({ request }) { + try { ${guarded} } catch (e) { + if (e instanceof Error) { return new Response(null, { status: 400 }); } + throw e; + } + return await prisma.thing.findMany(); + } + `; + + const INERT: Array<[string, string]> = [ + ["an empty block", ""], + ["a literal expression statement", "0;"], + ["a literal declaration", "const x = 1;"], + ["arithmetic on literals", "const x = 1 + 2 * 3;"], + ["a bare identifier read", "const x = someLocal;"], + ]; + + for (const [label, guarded] of INERT) { + it(`is not read as error handling when the try holds only ${label}`, () => { + const ep = scanFile("x.ts", guarding(guarded)); + expect(ep!.catches[0]!.guardCanRaise).toBe(false); + }); + } + + // Positive controls, one per reason `canRaise` recognises, so the predicate is not passing the + // cases above by being false for everything. + const LIVE: Array<[string, string]> = [ + ["a call", "doThing();"], + ["a construction", "new Thing();"], + ["an await", "await later;"], + ["a member access", "const x = thing.value;"], + ["an element access", "const x = thing[0];"], + ["a throw", "throw new Error('x');"], + ["an iteration", "for (const item of items) { }"], + ["an instanceof", "const x = thing instanceof Error;"], + ]; + + for (const [label, guarded] of LIVE) { + it(`is read as error handling when the try holds ${label}`, () => { + const ep = scanFile("x.ts", guarding(guarded)); + expect(ep!.catches[0]!.guardCanRaise).toBe(true); + }); + } + }); + it("leaves both flags false when the catch only returns", () => { const ep = scanFile( "swallow.ts", @@ -1097,6 +1252,7 @@ describe("scanFile: per-catch evidence", () => { branches: false, guardsParse: true, awaitsOnlyParse: true, + guardCanRaise: true, tryStatementCount: 1, }); expect(ep!.catches[1]).toMatchObject({ @@ -1199,6 +1355,7 @@ describe("scanFile: per-catch evidence", () => { branches: false, guardsParse: false, awaitsOnlyParse: false, + guardCanRaise: true, tryStatementCount: 4, }); }); @@ -1704,3 +1861,57 @@ describe("scanFile: a binding shadowed by an enclosing scope, not just a nested expect(ep!.catches[0]!.branches).toBe(false); }); }); + +// S3. The ternary path checked only that the condition tested the error, never that the two arms +// went anywhere different, while the `if`/`switch` path had checked exactly that since the round +// before. Rewriting `return X;` as `return e instanceof Error ? (X) : (X)` was therefore worth 50 +// points a route for a change that decides nothing, and it is semantics-preserving. +// `same-arms-ternary` in the mutation corpus is the tree-scale version. +describe("a ternary on the error has to send its arms somewhere different", () => { + const returning = (value: string) => ` + export async function loader() { + try { + return await prisma.thing.findMany(); + } catch (error) { + return ${value}; + } + } + `; + + it("does not credit a ternary whose arms are identical", () => { + const ep = scanFile("x.ts", returning("error instanceof Error ? (json({})) : (json({}))")); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + it("does not credit a ternary whose arms differ only in whitespace", () => { + const ep = scanFile("x.ts", returning("error instanceof Error ? (json( {} )) : (json({}))")); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + it("does not credit a ternary whose arms differ only in parentheses", () => { + const ep = scanFile("x.ts", returning("error instanceof Error ? ((json({}))) : (json({}))")); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + it("still credits a ternary whose arms go somewhere different", () => { + const ep = scanFile( + "x.ts", + returning("error instanceof Response ? error : json({}, { status: 500 })") + ); + expect(ep!.catches[0]!.branches).toBe(true); + }); + + // The same comparison on the `if` path, which had the exit test but not the arm test. + it("does not credit an if/else whose two arms are identical", () => { + const ep = scanFile( + "x.ts", + `export async function loader() { + try { return await prisma.thing.findMany(); } + catch (error) { + if (error instanceof Error) { return json({}); } else { return json({}); } + } + }` + ); + expect(ep!.catches[0]!.branches).toBe(false); + }); +}); From d919ee2567edce07219a01c61604ce8d4a723064 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Sat, 1 Aug 2026 20:55:19 +0100 Subject: [PATCH 058/106] test(observability-map): add the additive half of the corpus Every entry was subtractive: delete the catches, wrap the body, merge the statements, splice in dead code. None added fake signal, and both of the shapes that got past the corpus this round lived in that blind spot. Nine entries now cover it, ADDITIVE_IDS names them, and the harness asserts the class is not empty so it cannot quietly go away again. Three other things the corpus was getting wrong. The anti-vacuity guard counted files, which says a rewrite touched a file and not that it reached anything inside it. prependToEveryCatch used to splice at the END of a clause, and 234 of the tree's 260 clauses end in a return or a throw, so eleven entries reported 172 files while landing somewhere that mattered for 26 clauses. The splice moved to the head, where every clause is reachable, and mutations now report the number of sites they edited so the threshold is on something real. The guard the design asked for, verdict movement, cannot be used: a defended mutation moves no verdict anywhere, which is what defended means. Routes are no longer allowed to disappear. risesIn and commonMean both skip a route the mutated scan cannot see, and the entry-point guard tolerated five going missing. It is exact now, with an explicit check that every baseline route is still there. dead-branch-after-if-true is in the corpus as an expected failure. It wraps a catch body in if (true) { ... } and writes a dead error test after it, taking the tree from 15 to 23 and raising 68 routes. Both ways to close it are worse than the hole: folding the literal true starts the list an earlier round lost with false, and cutting branch credit at the first statement that MIGHT exit is sound but accuses 78 real routes and takes the tree to 6. Measured both, chose neither, said so. --- internal-packages/observability-map/README.md | 14 +- .../test/mutationCorpus.test.ts | 79 +++-- .../observability-map/test/mutations.ts | 299 +++++++++++++++--- 3 files changed, 328 insertions(+), 64 deletions(-) diff --git a/internal-packages/observability-map/README.md b/internal-packages/observability-map/README.md index a058282c727..c62c4ad4360 100644 --- a/internal-packages/observability-map/README.md +++ b/internal-packages/observability-map/README.md @@ -40,7 +40,7 @@ every catch clause in the tree used to score it 100, which meant the metric paid error handling. The property behind that is now a test corpus rather than a claim. `test/mutationCorpus.test.ts` -applies 30 semantics-preserving or handling-deleting rewrites to the whole route tree in a temp copy +applies 39 semantics-preserving or handling-deleting rewrites to the whole route tree in a temp copy and asserts three things for each: the published global does not rise, the mean over the routes measured in both runs does not rise, and for a semantics-preserving rewrite no individual route's score rises or drops out of the measured set. Every laundering shape a reviewer has found on this @@ -51,8 +51,16 @@ clause in the tree drops the score from 15 to 2, so the metric does not pay you handling. Wrapping every body in `try { ... } catch (e) { throw e }` leaves it unchanged, so it does not pay you for adding error handling that does nothing either. -The honest statement is "these 30 rewrites are defended, and here they are", not "unpaddable". The -corpus takes about three minutes, so it is gated behind `OBS_MAP_MUTATION_CORPUS=1` and run as its +The rewrites come in two directions and both matter. A subtractive one takes real signal away or +moves it about: delete the catches, wrap the body, merge the statements. An additive one puts fake +signal in: a classifying catch over a try that cannot throw, a test whose two arms are the same, a +rethrow that can never run. The corpus had only the subtractive half for a while, and the two +largest holes ever found here were both additive. + +The honest statement is "these 38 rewrites are defended, here they are, and here is the one that is +not", not "unpaddable". One entry, `dead-branch-after-if-true`, runs as an expected failure with the +residual written out beside it. The corpus takes about four minutes, so it is gated behind +`OBS_MAP_MUTATION_CORPUS=1` and run as its own CI job rather than in `pnpm test`. If you change this package, run it: ```bash diff --git a/internal-packages/observability-map/test/mutationCorpus.test.ts b/internal-packages/observability-map/test/mutationCorpus.test.ts index f74076c1d5b..8dd03ee6ded 100644 --- a/internal-packages/observability-map/test/mutationCorpus.test.ts +++ b/internal-packages/observability-map/test/mutationCorpus.test.ts @@ -3,7 +3,7 @@ import { tmpdir } from "node:os"; import { dirname, join, resolve } from "node:path"; import { scanDirectory } from "../src/scan.js"; import { buildReport } from "../src/score.js"; -import { MUTATIONS, type Mutation } from "./mutations.js"; +import { ADDITIVE_IDS, MUTATIONS, type Mutation } from "./mutations.js"; /** * The tree-scale mutation corpus. @@ -36,13 +36,20 @@ const ROUTES = resolve(__dirname, "../../../apps/webapp/app/routes"); const ENABLED = process.env.OBS_MAP_MUTATION_CORPUS === "1"; /** - * Where a corpus entry goes when the tool does not defend it. Empty as of round A fix 2: all 30 - * entries hold. When a reviewer finds a shape that cannot be defended, add it to the corpus and - * name it here rather than leaving it out, and write down why in the round's report. `it.fails` - * keeps such an entry running, so closing the hole later turns this file red until the entry is - * moved back out deliberately. + * Where a corpus entry goes when the tool does not defend it. `it.fails` keeps the entry running, + * so closing the hole later turns this file red until the entry is moved back out deliberately. + * + * `dead-branch-after-if-true` wraps a catch clause's body in `if (true) { ... }` and writes an + * error test after it. The test can never run, and it takes the tree from 15 to 23 and raises 68 + * routes. Nothing in the scanner evaluates a condition, so `definitelyExits` cannot see that the + * wrapper always exits, and the two ways to make it see are both worse than the hole. Folding the + * literal `true` starts the same list an earlier round lost with `false`, where every spelling + * nobody thought of pays. Cutting the branch credit at the first statement that MIGHT exit is + * sound and folding-free, and it was measured: it takes the tree from 15 to 6 and accuses 78 real + * routes, because `catch (e) { if (rare) return null; if (e instanceof X) return y; }` is an + * ordinary shape. See the round A fix 3 report. */ -const KNOWN_GAPS = new Set([]); +const KNOWN_GAPS = new Set(["dead-branch-after-if-true"]); type SourceFile = { relativeName: string; source: string }; @@ -169,15 +176,20 @@ function commonMean(baseline: Measurement, after: Measurement): { before: number return n === 0 ? { before: 0, after: 0 } : { before: sumBefore / n, after: sumAfter / n }; } -function mutate(files: SourceFile[], mutation: Mutation): { files: SourceFile[]; changed: number } { +function mutate( + files: SourceFile[], + mutation: Mutation +): { files: SourceFile[]; changed: number; sites: number } { let changed = 0; + let sites = 0; const out = files.map((file) => { - const source = mutation.apply(file.relativeName, file.source); - if (source === null || source === file.source) return file; + const result = mutation.apply(file.relativeName, file.source); + if (result === null || result.source === file.source) return file; changed++; - return { relativeName: file.relativeName, source }; + sites += result.sites; + return { relativeName: file.relativeName, source: result.source }; }); - return { files: out, changed }; + return { files: out, changed, sites }; } const describeCorpus = ENABLED && existsSync(ROUTES) ? describe : describe.skip; @@ -186,6 +198,15 @@ describeCorpus("mutation corpus over the real route tree", () => { const files = ENABLED && existsSync(ROUTES) ? readTree(ROUTES) : []; const baseline = ENABLED && existsSync(ROUTES) ? measure(files) : null; + it("covers the additive direction, not only the subtractive one", () => { + // Every corpus entry once removed or restructured real signal, and none added fake signal. The + // two largest holes ever found here lived in that blind spot, so the class is asserted rather + // than left to whoever edits the list next. + const ids = new Set(MUTATIONS.map((m) => m.id)); + expect(ADDITIVE_IDS.filter((id) => !ids.has(id))).toEqual([]); + expect(ADDITIVE_IDS.length).toBeGreaterThanOrEqual(8); + }); + it("has a baseline worth mutating", () => { expect(baseline).not.toBeNull(); expect(baseline!.entryPoints).toBeGreaterThan(300); @@ -197,26 +218,42 @@ describeCorpus("mutation corpus over the real route tree", () => { }); /** - * The number of files a mutation must touch before its result means anything. A mutation that - * silently matched nothing would otherwise "pass" by leaving the tree alone, which is the exact - * failure mode that let earlier rounds believe a shape was defended. + * How much a mutation must reach before its result means anything. A mutation that silently + * matched nothing would otherwise "pass" by leaving the tree alone, which is the exact failure + * mode that let earlier rounds believe a shape was defended. + * + * Sites, not only files, and sites are what the threshold is really on. A file count says a + * rewrite touched a file, not that it reached anything inside it: eleven entries reported 172 + * files while landing in a position that mattered for 26 of the tree's 260 catch clauses, because + * the splice went after statements that had already returned. `prependToEveryCatch` now splices + * at the head of the clause, so all 260 count, and this threshold is what would notice if a later + * change quietly took that back. + * + * The guard the design asked for, verdict movement, cannot be used: a defended mutation moves no + * verdict anywhere, and that is precisely what "defended" means, so requiring movement would fail + * every entry that works. Site count is the reachable version of the same intent. */ const MINIMUM_FILES_TOUCHED = 20; + const MINIMUM_SITES_TOUCHED = 40; for (const mutation of MUTATIONS) { const run = KNOWN_GAPS.has(mutation.id) ? it.fails : it; run(`${mutation.kind}: ${mutation.what} (${mutation.id})`, () => { - const { files: mutated, changed } = mutate(files, mutation); + const { files: mutated, changed, sites } = mutate(files, mutation); expect(changed).toBeGreaterThanOrEqual(MINIMUM_FILES_TOUCHED); + expect(sites).toBeGreaterThanOrEqual(MINIMUM_SITES_TOUCHED); const after = measure(mutated); - // A mutation that stops the tree parsing, or that hides most of the routes from the scanner, - // has not tested the property: whatever the score does afterwards is measuring a different - // tree. Both guards fail loudly rather than letting such a mutation report a pass. + // A mutation that stops the tree parsing, or that hides a route from the scanner, has not + // tested the property: whatever the score does afterwards is measuring a different tree. The + // route guard is exact rather than tolerant, because a route the mutated scan cannot see is + // one `risesIn` and `commonMean` both skip, and a rewrite that makes a route unscannable is + // itself a finding. expect(after.parseFailures).toBe(baseline!.parseFailures); - expect(after.entryPoints).toBeGreaterThanOrEqual(baseline!.entryPoints - 5); + expect(after.entryPoints).toBe(baseline!.entryPoints); + expect([...baseline!.perEntry.keys()].filter((f) => !after.perEntry.has(f))).toEqual([]); const rises = risesIn(baseline!, after); const common = commonMean(baseline!, after); @@ -224,7 +261,7 @@ describeCorpus("mutation corpus over the real route tree", () => { `[corpus] ${mutation.id}: global ${baseline!.global} -> ${after.global} ` + `(mean ${baseline!.exactMean.toFixed(3)} -> ${after.exactMean.toFixed(3)}, ` + `common mean ${common.before.toFixed(3)} -> ${common.after.toFixed(3)}, ` + - `measured ${baseline!.measured} -> ${after.measured}, files ${changed}, ` + + `measured ${baseline!.measured} -> ${after.measured}, files ${changed}, sites ${sites}, ` + `routes raised ${rises.length})` + rises .slice(0, 3) diff --git a/internal-packages/observability-map/test/mutations.ts b/internal-packages/observability-map/test/mutations.ts index 4b8f6df4671..894dd8da9d4 100644 --- a/internal-packages/observability-map/test/mutations.ts +++ b/internal-packages/observability-map/test/mutations.ts @@ -16,6 +16,14 @@ import ts from "typescript"; * - `deleting`: the rewrite removes error handling or logging. The route is worse afterwards, so * the score must not rise either, for a different and simpler reason. * + * Within `preserving` there are two directions, and for a long time the corpus only had one of + * them. A subtractive rewrite takes real signal away or moves it about: delete the catches, wrap + * the body, merge the statements. An ADDITIVE rewrite puts fake signal in: a classifying catch over + * a try that cannot throw, a test whose two arms are the same, a rethrow that can never run. The + * additive direction is the one someone reaches for when a CI comment nags them, and it is where + * the two largest holes ever found here lived. `ADDITIVE_IDS` lists the entries that cover it, and + * `mutationCorpus.test.ts` asserts the class is not empty so it cannot quietly go away again. + * * Neither kind is ever executed. "Semantics-preserving" here means preserving the observable * behaviour of the route as written, which is what the scanner claims to measure; it is not a * claim that the mutated tree compiles against its real types. @@ -23,13 +31,21 @@ import ts from "typescript"; export type MutationKind = "preserving" | "deleting"; +/** + * The result of rewriting one file: the new source, and how many places in it the rewrite actually + * landed. `sites` is what the corpus's anti-vacuity guard reads. A file count says nothing about + * whether the rewrite reached anything, and a mutation that quietly matched two constructs in a + * file would otherwise look identical to one that matched forty. + */ +export type MutationResult = { source: string; sites: number }; + export type Mutation = { id: string; kind: MutationKind; /** What the rewrite does, in one line, for the corpus table in the report. */ what: string; - /** The mutated source, or null when this file has nothing for the mutation to touch. */ - apply(fileName: string, source: string): string | null; + /** The mutated file, or null when this file has nothing for the mutation to touch. */ + apply(fileName: string, source: string): MutationResult | null; }; type Edit = { start: number; end: number; text: string }; @@ -51,7 +67,7 @@ function parse(fileName: string, source: string): ts.SourceFile { * deletes a catch clause and one that rewrites a statement inside that clause would otherwise * produce overlapping splices. Dropping the inner one is what "the outer rewrite won" means. */ -function applyEdits(source: string, edits: Edit[]): string | null { +function applyEdits(source: string, edits: Edit[]): MutationResult | null { if (edits.length === 0) return null; const sorted = [...edits].sort((a, b) => a.start - b.start || a.end - b.end); const kept: Edit[] = []; @@ -65,7 +81,7 @@ function applyEdits(source: string, edits: Edit[]): string | null { const edit = kept[i]!; out = out.slice(0, edit.start) + edit.text + out.slice(edit.end); } - return out === source ? null : out; + return out === source ? null : { source: out, sites: kept.length }; } function insert(at: number, text: string): Edit { @@ -214,11 +230,17 @@ function bindingNameOf(clause: ts.CatchClause): string | null { } /** - * Append a statement at the end of every catch clause that names its binding. `snippet` receives - * the binding name. Appending is the position that matters: a shape spliced in after a `return` is - * already unreachable and proves nothing. + * Splice a statement in at the HEAD of every catch clause that names its binding. `snippet` + * receives the binding name. + * + * The head, not the tail, and that is the whole point of the helper. Appending put the shape after + * whatever the clause already did, and 234 of the tree's 260 clauses end in a `return` or a + * `throw`, so in those the spliced shape was dead by ordering before the rule under test ever + * looked at it: eleven corpus entries reported touching 172 files while exercising 26 clauses. At + * the head every clause is reachable, so every clause exercises the rule. The shapes spliced this + * way are dead wherever they sit, so moving them does not make the rewrite any less preserving. */ -function appendToEveryCatch( +function prependToEveryCatch( id: string, kind: MutationKind, what: string, @@ -234,7 +256,73 @@ function appendToEveryCatch( for (const clause of catchClauses(sf)) { const binding = bindingNameOf(clause); if (binding === null) continue; - edits.push(insert(clause.block.end - 1, `\n${snippet(binding)}\n`)); + edits.push(insert(clause.block.getStart() + 1, `\n${snippet(binding)}\n`)); + } + return applyEdits(source, edits); + }, + }; +} + +/** Whether a statement list ends in a way that makes anything spliced in after it dead. Used only + * to keep the dead-throw mutations honest: appending `throw e;` after statements that might fall + * through would change what the route does, and this corpus is not allowed to do that. */ +function endsInAnExit(statements: readonly ts.Statement[]): boolean { + const last = statements[statements.length - 1]; + return last !== undefined && (ts.isReturnStatement(last) || ts.isThrowStatement(last)); +} + +/** Whether the tree rooted at `node` contains a `break` or `continue` outside a nested function or + * a loop of its own. Wrapping such statements in a `do` or a `switch` would rebind them. */ +function containsLooseJump(node: ts.Node): boolean { + let found = false; + const visit = (n: ts.Node) => { + if (found) return; + if (ts.isFunctionLike(n)) return; + if ( + ts.isForStatement(n) || + ts.isForOfStatement(n) || + ts.isForInStatement(n) || + ts.isWhileStatement(n) || + ts.isDoStatement(n) || + ts.isSwitchStatement(n) + ) { + return; + } + if (ts.isBreakStatement(n) || ts.isContinueStatement(n)) found = true; + ts.forEachChild(n, visit); + }; + ts.forEachChild(node, visit); + return found; +} + +/** + * Wrap every catch clause's body in a construct that definitely exits, then write `throw e;` after + * it. The throw can never run, and before `definitelyExits` learned to see through the wrapper each + * of these read as the clause rethrowing, which is `not-applicable` instead of `fail` and worth 50 + * points a route. + * + * Only applied to a clause whose statements already end in a `return` or a `throw`, so the appended + * throw really is unreachable, and never to one holding a loose `break` or `continue`, which a `do` + * or a `switch` would capture. + */ +function deadThrowAfter(id: string, what: string, wrap: (body: string) => string): Mutation { + return { + id, + kind: "preserving", + what, + apply(fileName, source) { + const sf = parse(fileName, source); + const edits: Edit[] = []; + for (const clause of catchClauses(sf)) { + const binding = bindingNameOf(clause); + if (binding === null) continue; + const statements = clause.block.statements; + if (statements.length === 0 || !endsInAnExit(statements)) continue; + if (containsLooseJump(clause.block)) continue; + const first = statements[0]!.getStart(); + const last = statements[statements.length - 1]!.end; + const body = source.slice(first, last); + edits.push({ start: first, end: last, text: `${wrap(body)}\nthrow ${binding};` }); } return applyEdits(source, edits); }, @@ -266,7 +354,7 @@ function prependToEveryFile(id: string, what: string, text: string): Mutation { kind: "preserving", what, apply(_fileName, source) { - return `${text}\n${source}`; + return { source: `${text}\n${source}`, sites: 1 }; }, }; } @@ -347,7 +435,10 @@ export const MUTATIONS: Mutation[] = [ what: "add a component whose JSX text begins with a // directive", apply(fileName, source) { if (!fileName.endsWith(".tsx")) return null; - return `${source}\nexport function ObsMapMutationA() {\n return

// obs-map-disable error-classification -- mutation corpus

;\n}\n`; + return { + source: `${source}\nexport function ObsMapMutationA() {\n return

// obs-map-disable error-classification -- mutation corpus

;\n}\n`, + sites: 1, + }; }, }, { @@ -356,7 +447,10 @@ export const MUTATIONS: Mutation[] = [ what: "add a component whose JSX text starts a // directive right after an expression container", apply(fileName, source) { if (!fileName.endsWith(".tsx")) return null; - return `${source}\nexport function ObsMapMutationB({ name }: { name: string }) {\n return

{name}// obs-map-disable request-context -- mutation corpus

;\n}\n`; + return { + source: `${source}\nexport function ObsMapMutationB({ name }: { name: string }) {\n return

{name}// obs-map-disable request-context -- mutation corpus

;\n}\n`, + sites: 1, + }; }, }, { @@ -365,7 +459,10 @@ export const MUTATIONS: Mutation[] = [ what: "add a component whose JSX text is a /* */ directive", apply(fileName, source) { if (!fileName.endsWith(".tsx")) return null; - return `${source}\nexport function ObsMapMutationC() {\n return

/* obs-map-disable audit-trail -- mutation corpus */

;\n}\n`; + return { + source: `${source}\nexport function ObsMapMutationC() {\n return

/* obs-map-disable audit-trail -- mutation corpus */

;\n}\n`, + sites: 1, + }; }, }, @@ -435,7 +532,7 @@ export const MUTATIONS: Mutation[] = [ { id: "throw-after-return-in-catch", kind: "preserving", - what: "append throw e; after the first return in every catch", + what: "splice throw e; after the first return in every catch", apply(fileName, source) { const sf = parse(fileName, source); const edits: Edit[] = []; @@ -450,85 +547,188 @@ export const MUTATIONS: Mutation[] = [ }, }, - appendToEveryCatch( + prependToEveryCatch( "dead-if-false", "preserving", - "append if (false) { throw e; } to every catch", + "splice if (false) { throw e; } into every catch", (e) => `if (false) { throw ${e}; }` ), - appendToEveryCatch( + prependToEveryCatch( "dead-while-false", "preserving", - "append while (false) { throw e; } to every catch", + "splice while (false) { throw e; } into every catch", (e) => `while (false) { throw ${e}; }` ), - appendToEveryCatch( + prependToEveryCatch( "dead-for-false", "preserving", - "append for (;false;) { throw e; } to every catch", + "splice for (;false;) { throw e; } into every catch", (e) => `for (;false;) { throw ${e}; }` ), - appendToEveryCatch( + prependToEveryCatch( "dead-if-true-else", "preserving", - "append if (true) { 0; } else { throw e; } to every catch", + "splice if (true) { 0; } else { throw e; } into every catch", (e) => `if (true) { 0; } else { throw ${e}; }` ), - appendToEveryCatch( + prependToEveryCatch( "dead-switch-no-case", "preserving", - "append switch (1) { case 2: throw e; } to every catch", + "splice switch (1) { case 2: throw e; } into every catch", (e) => `switch (1) { case 2: throw ${e}; }` ), - appendToEveryCatch( + prependToEveryCatch( "dead-inner-try", "preserving", - "append try { 0; } catch { throw e; } to every catch", + "splice try { 0; } catch { throw e; } into every catch", (e) => `try { 0; } catch { throw ${e}; }` ), - appendToEveryCatch( + prependToEveryCatch( "dead-for-of-empty", "preserving", - "append for (const x of []) { throw e; } to every catch", + "splice for (const x of []) { throw e; } into every catch", (e) => `for (const obsMapItem of []) { throw ${e}; }` ), - appendToEveryCatch( + prependToEveryCatch( "dead-for-in-empty", "preserving", - "append for (const k in {}) { throw e; } to every catch", + "splice for (const k in {}) { throw e; } into every catch", (e) => `for (const obsMapKey in {}) { throw ${e}; }` ), - appendToEveryCatch( + prependToEveryCatch( "dead-if-empty-string", "preserving", 'append if ("") { throw e; } to every catch', (e) => `if ("") { throw ${e}; }` ), - appendToEveryCatch( + prependToEveryCatch( "dead-if-not-true", "preserving", - "append if (!true) { throw e; } to every catch", + "splice if (!true) { throw e; } into every catch", (e) => `if (!true) { throw ${e}; }` ), - appendToEveryCatch( + prependToEveryCatch( "dead-if-const-compare", "preserving", - "append if (1 === 2) { throw e; } to every catch", + "splice if (1 === 2) { throw e; } into every catch", (e) => `if (1 === 2) { throw ${e}; }` ), - appendToEveryCatch( + prependToEveryCatch( "registered-throw", "preserving", - "append [].push(() => { throw e; }) to every catch", + "splice [].push(() => { throw e; }) into every catch", (e) => `[].push(() => { throw ${e}; });` ), - appendToEveryCatch( + prependToEveryCatch( "empty-instanceof-if", "preserving", - "append if (e instanceof Error) { } to every catch", + "splice if (e instanceof Error) { } into every catch", (e) => `if (${e} instanceof Error) { }` ), + // The additive class. Everything above either takes signal away or moves it about; these put in + // signal that is not real, which is the direction the corpus was blind to. + { + id: "dead-classifying-try", + kind: "preserving", + what: "prepend a classifying try/catch over a try block that cannot throw", + apply(fileName, source) { + const sf = parse(fileName, source); + const edits: Edit[] = []; + for (const body of entryBodies(sf)) { + edits.push( + insert( + body.getStart() + 1, + "\ntry { 0; } catch (obsMapDead) {" + + " if (obsMapDead instanceof Error) { return new Response(null, { status: 400 }); }" + + " throw obsMapDead; }\n" + ) + ); + } + return applyEdits(source, edits); + }, + }, + { + id: "same-arms-ternary", + kind: "preserving", + what: "rewrite a catch's return value as a ternary on the error with identical arms", + apply(fileName, source) { + const sf = parse(fileName, source); + const edits: Edit[] = []; + for (const clause of catchClauses(sf)) { + const binding = bindingNameOf(clause); + if (binding === null) continue; + for (const statement of clause.block.statements) { + if (!ts.isReturnStatement(statement) || !statement.expression) continue; + const value = source.slice(statement.expression.getStart(), statement.expression.end); + edits.push({ + start: statement.expression.getStart(), + end: statement.expression.end, + text: `${binding} instanceof Error ? (${value}) : (${value})`, + }); + } + } + return applyEdits(source, edits); + }, + }, + deadThrowAfter( + "dead-throw-after-block", + "wrap every catch body in a bare block and write throw e; after it", + (body) => `{\n${body}\n}` + ), + deadThrowAfter( + "dead-throw-after-do", + "wrap every catch body in do { ... } while (false) and write throw e; after it", + (body) => `do {\n${body}\n} while (false);` + ), + deadThrowAfter( + "dead-throw-after-if-true", + "wrap every catch body in if (true) { ... } and write throw e; after it", + (body) => `if (true) {\n${body}\n}` + ), + deadThrowAfter( + "dead-throw-after-if-else", + "wrap every catch body in both arms of an if/else and write throw e; after it", + (body) => `if (obsMapPick()) {\n${body}\n} else {\n${body}\n}` + ), + deadThrowAfter( + "dead-throw-after-switch", + "wrap every catch body in a switch default and write throw e; after it", + (body) => `switch (1) { default: {\n${body}\n} }` + ), + deadThrowAfter( + "dead-throw-after-try-finally", + "wrap every catch body in try { ... } finally { } and write throw e; after it", + (body) => `try {\n${body}\n} finally { }` + ), + { + id: "dead-branch-after-if-true", + kind: "preserving", + what: "wrap every catch body in if (true) { ... } and write a dead error test after it", + apply(fileName, source) { + const sf = parse(fileName, source); + const edits: Edit[] = []; + for (const clause of catchClauses(sf)) { + const binding = bindingNameOf(clause); + if (binding === null) continue; + const statements = clause.block.statements; + if (statements.length === 0 || !endsInAnExit(statements)) continue; + if (containsLooseJump(clause.block)) continue; + const first = statements[0]!.getStart(); + const last = statements[statements.length - 1]!.end; + const body = source.slice(first, last); + edits.push({ + start: first, + end: last, + text: + `if (true) {\n${body}\n}\n` + + `if (${binding} instanceof Error) { return new Response(null, { status: 400 }); }`, + }); + } + return applyEdits(source, edits); + }, + }, + { id: "merge-declarations", kind: "preserving", @@ -577,7 +777,7 @@ export const MUTATIONS: Mutation[] = [ { id: "inert-statements-after-try", kind: "preserving", - what: "append five unused const declarations after every try statement in a route body", + what: "splice five unused const declarations after every try statement in a route body", apply(fileName, source) { const sf = parse(fileName, source); const edits: Edit[] = []; @@ -595,6 +795,25 @@ export const MUTATIONS: Mutation[] = [ }, ]; +/** + * The entries that add fake signal rather than removing or restructuring real signal. Named so + * `mutationCorpus.test.ts` can assert the class exists: the corpus went three rounds with this half + * of the property untested, and an empty list here is exactly that state coming back. + */ +export const ADDITIVE_IDS = [ + "dead-classifying-try", + "same-arms-ternary", + "dead-throw-after-block", + "dead-throw-after-do", + "dead-throw-after-if-true", + "dead-throw-after-if-else", + "dead-throw-after-switch", + "dead-throw-after-try-finally", + "wrap-body-in-rethrow", + "empty-instanceof-if", + "registered-throw", +]; + function isSingleConst(statement: ts.Statement): statement is ts.VariableStatement { return ( ts.isVariableStatement(statement) && From a11b6616b667a39ac7987b18f815f131033ab16b Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Sat, 1 Aug 2026 21:40:08 +0100 Subject: [PATCH 059/106] fix(observability-map): cut a dead error test at the right moment The exit cut for branch credit was rejected last round on a measurement of the wrong thing. The variant measured raised the exit flag before checking the current statement, so every deciding statement refused itself: `if (e instanceof X) return y` contains an exit by definition. That variant does take the tree from 15 to 6 and accuse 78 routes, and it is not the rule. Raising the flag after the statement's own branch check leaves the real-tree report and all 240 clauses' evidence byte-identical, verified by comparing the full report and every CatchEvidence field before and after. It closes dead-branch-after-if-true, which went from 15 to 23 with 68 routes raised and now raises none, and it closes the labelled-block, for-of and while members of the same family. None of them needs a condition evaluated, so that claim is gone from the corpus and from the docstrings. The precision given up is a conditional exit before the error test, which also stops the credit. No clause in the tree is that shape, which is why the report does not move, and the case is pinned as a test so it stays a decision. Also two false statements the check could make. A clause that throws for some errors was told it takes one way out regardless of what was thrown. It does not. The strengthened rethrows makes such a clause a swallow by this check's definition, since it decides nothing about the error, and 16 clauses flipped rethrows this round. CatchEvidence now carries the raw throw alongside the strengthened one and the wording follows it. Whether fail is the right verdict for those clauses is a separate question, parked. A route that owns a real classifying catch could be told it owns none. canRaise does not list destructuring and `const { a } = undefined` throws, so the owned catch dropped out of the reachable set and the callback branch, ordered off that set, accused the route of keeping all its error handling in a callback. It reads ep.catches now. --- .../src/checks/errorClassification.ts | 39 +++++++--- .../observability-map/src/scan.ts | 74 ++++++++++++------- .../observability-map/src/types.ts | 11 ++- .../observability-map/test/checks.test.ts | 60 +++++++++++++++ .../observability-map/test/scan.test.ts | 48 +++++++++--- 5 files changed, 186 insertions(+), 46 deletions(-) diff --git a/internal-packages/observability-map/src/checks/errorClassification.ts b/internal-packages/observability-map/src/checks/errorClassification.ts index 22a431ab764..c380b80a946 100644 --- a/internal-packages/observability-map/src/checks/errorClassification.ts +++ b/internal-packages/observability-map/src/checks/errorClassification.ts @@ -152,13 +152,23 @@ export function usesBuilder(ep: EntryPoint): boolean { * `Promise.all([0].map(async () => { ... }))` did. It fails instead. The precision cost is a route * that genuinely only handles errors per item, which now fails rather than sitting out. * - * A clause whose try block cannot raise is read as no clause at all, `guardCanRaise` on the - * evidence. It is not error handling, and crediting one was the largest hole ever found here: - * prepending `try { 0; } catch (e) { if (e instanceof Error) { return json(x, { status: 400 }); } - * throw e; }` to every body took the tree from 15 to 42 and raised 224 routes, because the 261 - * routes that catch nothing were sitting at not-applicable and a dead clause moved each of them to - * pass. Dropping it here rather than in the scan keeps the evidence honest about what is written - * and puts the judgement where the other judgements are. + * A clause whose try block holds nothing that could raise is read as no clause at all, + * `guardCanRaise` on the evidence. Prepending `try { 0; } catch (e) { if (e instanceof Error) { + * return json(x, { status: 400 }); } throw e; }` to every body took the tree from 15 to 42 and + * raised 224 routes, because the 261 routes that catch nothing were sitting at not-applicable and a + * dead clause moved each of them to pass. + * + * What that refuses is `try { 0; }`, and it is defeated by one inert call: `try { String(0); }` + * reads as classification and pays the same 224 routes, because `canRaise` accepts any call at all. + * The rule closes the shape that was found, not the family, and telling an inert call from a + * throwing one needs types the scanner does not have. `dead-classifying-try-with-call` in the + * mutation corpus is the open shape, running as an expected failure. + * + * The filter also has to run BEFORE nothing. Ordering the callback branch off `reachable` rather + * than `ep.catches` accused a route that owns a real classifying catch of owning none, whenever + * `canRaise` missed what that catch guarded: a destructuring declaration is not on its list, so + * `try { const { a } = undefined; } catch (e) { ... }` beside a per-item `.map` catch failed with + * "its only error handling sits in a callback the route does not own", which was simply untrue. */ export const errorClassification = { id: ID, @@ -171,13 +181,24 @@ export const errorClassification = { if (swallowed.length > 0) { const which = reachable.length > 1 ? ` (${swallowed.length} of ${reachable.length} catches)` : ""; + // "One way out" is only true of a clause that never throws. A clause holding a `throw` that + // is not its only exit is a swallow by this check's definition (it decides nothing about the + // error) and it is NOT one way out, so saying so was a false accusation. 16 clauses in the + // tree changed `rethrows` from true to false this round and every one of them would have + // been eligible for it. + const everyWayOut = swallowed.every((c) => !c.throws); return { id: ID, status: "fail", - detail: `catches its errors and takes one way out regardless of what was thrown${which}`, + detail: everyWayOut + ? `catches its errors and takes one way out regardless of what was thrown${which}` + : `catches its errors and chooses what to do without looking at what was thrown${which}`, }; } - if (reachable.length === 0 && ep.callbackCatches > 0) { + // Read off `ep.catches`, not `reachable`: a route that owns a catch owns one, whether or not + // `canRaise` could see what it guarded. Ordering this off `reachable` turned every `canRaise` + // miss on a route that also has a per-item catch into an accusation that was flatly false. + if (ep.catches.length === 0 && ep.callbackCatches > 0) { return { id: ID, status: "fail", diff --git a/internal-packages/observability-map/src/scan.ts b/internal-packages/observability-map/src/scan.ts index eaf14cd4aa6..5521833a382 100644 --- a/internal-packages/observability-map/src/scan.ts +++ b/internal-packages/observability-map/src/scan.ts @@ -196,13 +196,20 @@ function canRaise(node: ts.Node): boolean { * to 42 and raised 224 routes, which is more than every other shape found on this branch put * together. `dead-classifying-try` in the mutation corpus is the tree-scale version. * - * The alternative considered was proving unreachability the other way round, by ruling out every - * expression that could throw. It is the same predicate read backwards and it fails the same way, - * so the cheaper direction won: ask what the block DOES, and require it to do something. The - * residual is what `canRaise` does not list. A temporal-dead-zone read (`try { const x = later; }`), - * a coercion that raises (`try { const x = 1 + someSymbol; }`) and a `delete` on a frozen object are - * all treated as unable to raise. None is expressible as a preserving rewrite of a real route, and - * all of them would need types the scanner does not have. + * What this refuses is `try { 0; }` and nothing cleverer. `canRaise` accepts ANY call, member + * access or `in`, and none of those has to be able to throw, so one inert call defeats the rule: + * `try { String(0); } catch (e) { if (e instanceof Error) { return json(x, { status: 400 }); } throw + * e; }` reads as classification and takes the tree back to 15 to 42, exactly as `try { 0; }` did. + * `dead-classifying-try-with-call` in the mutation corpus is that shape, running as an expected + * failure. Telling a call that can throw from one that cannot needs types the scanner does not have, + * so the rule closes the shape found rather than the family it belongs to. Read the docstrings that + * point here as "refuses `try { 0; }`", never as "an unreachable catch cannot be credited". + * + * The list also misses things that CAN raise, which is the safe direction, and the misses matter + * because a real clause can be dropped by one: a destructuring declaration (`const { a } = undefined` + * throws), a temporal-dead-zone read (`try { const x = later; }`), a coercion that raises + * (`try { const x = 1 + someSymbol; }`) and a `delete` on a frozen object all read as unable to + * raise. * * Nested function bodies are skipped throughout: a callback written inside the try is not work the * try is guarding on this pass through. A `throw` inside one is not either, which is deliberate. @@ -479,10 +486,22 @@ function selectsADistinctPath(statement: ts.IfStatement | ts.SwitchStatement): b * `catch (e) { if (e instanceof Response) return e; throw e; }`, which passes on its branch instead. * That is the direction to be wrong in, since the reverse hands out points. */ -function catchClauseEvidence(clause: ts.CatchClause): { rethrows: boolean; branches: boolean } { +function catchClauseEvidence(clause: ts.CatchClause): { + rethrows: boolean; + throws: boolean; + branches: boolean; +} { let rethrows = false; let returns = false; let branches = false; + // Set once a statement the walk has already passed could have left the clause. An error test + // after one of those is dead code, so it decides nothing. Raised at the END of each statement, + // after that statement's own branch check: a deciding statement contains an exit by definition, + // so raising it first makes every such statement refuse itself, which was measured at 78 routes + // losing their pass and the tree dropping from 15 to 6. This ordering leaves the real-tree report + // and all 240 clauses' evidence byte-identical. The tests are the cases in `dead throw written + // after something that already exited`. + let exited = false; const bindingName = catchBindingName(clause); const walk = (statements: readonly ts.Statement[]) => { @@ -494,10 +513,12 @@ function catchClauseEvidence(clause: ts.CatchClause): { rethrows: boolean; branc for (const statement of reachableStatements(statements)) { if (ts.isThrowStatement(statement)) { rethrows = true; + exited = true; continue; } if (ts.isBlock(statement)) { walk(statement.statements); + if (containsExit(statement)) exited = true; continue; } // A `do` body runs before its condition is ever read, so it is on the straight-line path @@ -505,36 +526,38 @@ function catchClauseEvidence(clause: ts.CatchClause): { rethrows: boolean; branc if (ts.isDoStatement(statement)) { const body = statement.statement; walk(ts.isBlock(body) ? body.statements : [body]); + if (containsExit(statement)) exited = true; continue; } // Any other reachable statement that could return means throwing is not the only way out. // Read here rather than over the whole clause so a `return` the walk has already cut as dead // does not count, which is what a `do { throw e; } while (false); return null;` produces. if (containsReturn(statement)) returns = true; - if (bindingName === null || shadowed) continue; - - if ( - (ts.isIfStatement(statement) || ts.isSwitchStatement(statement)) && - referencesBinding(statement.expression, bindingName) && - selectsADistinctPath(statement) - ) { - branches = true; - continue; - } - if ( - (ts.isReturnStatement(statement) || ts.isThrowStatement(statement)) && - statement.expression !== undefined - ) { - const value = unwrap(statement.expression); - if (ts.isConditionalExpression(value) && selectsAnErrorPath(value, bindingName)) { + + if (bindingName !== null && !shadowed && !exited) { + if ( + (ts.isIfStatement(statement) || ts.isSwitchStatement(statement)) && + referencesBinding(statement.expression, bindingName) && + selectsADistinctPath(statement) + ) { branches = true; + } else if ( + (ts.isReturnStatement(statement) || ts.isThrowStatement(statement)) && + statement.expression !== undefined + ) { + const value = unwrap(statement.expression); + if (ts.isConditionalExpression(value) && selectsAnErrorPath(value, bindingName)) { + branches = true; + } } } + + if (containsExit(statement)) exited = true; } }; walk(clause.block.statements); - return { rethrows: rethrows && !returns, branches }; + return { rethrows: rethrows && !returns, throws: rethrows, branches }; } /** @@ -1008,6 +1031,7 @@ export function scanFile(fileName: string, source: string): EntryPoint | null { const clause = catchClauseEvidence(node.catchClause); catches.push({ rethrows: clause.rethrows, + throws: clause.throws, branches: clause.branches, ...guardedWork(node.tryBlock), tryStatementCount, diff --git a/internal-packages/observability-map/src/types.ts b/internal-packages/observability-map/src/types.ts index e05d5e7595d..3ce969494f4 100644 --- a/internal-packages/observability-map/src/types.ts +++ b/internal-packages/observability-map/src/types.ts @@ -21,6 +21,10 @@ export type CatchEvidence = { * has already returned. */ rethrows: boolean; + /** A `throw` is reached on that same straight-line path, whether or not it is the only way out. + * `rethrows` is this AND no reachable `return`. Kept separately so a verdict can say what is true + * of a clause that both throws and returns. */ + throws: boolean; /** * The clause picks what to do from what it caught, on that same straight-line path: an `if` or * `switch` whose condition references the caught error binding AND at least one of whose arms @@ -41,9 +45,10 @@ export type CatchEvidence = { guardsParse: boolean; /** * The guarded region does something that could raise at all: a call, a construction, an `await`, - * a member access, a `throw`, an iteration, an `instanceof`. A clause whose try block cannot - * raise is unreachable, so it is not error handling and `error-classification` reads no evidence - * off it. See `canRaise` in `scan.ts` for what is not on that list. + * a member access, a `throw`, an iteration, an `instanceof`. False means `try { 0; }` and little + * else: any call counts, including one that cannot throw, so `try { String(0); }` reads as true. + * See `canRaise` in `scan.ts` for both directions of that, including the destructuring + * declaration it misses. */ guardCanRaise: boolean; /** diff --git a/internal-packages/observability-map/test/checks.test.ts b/internal-packages/observability-map/test/checks.test.ts index bca5e4f6f4c..b10d6c44e85 100644 --- a/internal-packages/observability-map/test/checks.test.ts +++ b/internal-packages/observability-map/test/checks.test.ts @@ -603,6 +603,66 @@ describe("error-classification", () => { expect(r.status).toBe("pass"); }); + // I3. "Takes one way out regardless of what was thrown" is false of a clause that throws for + // some errors, and the strengthened `rethrows` makes such a clause a swallow by this check's + // definition: it decides nothing about the error, but it does not send everything the same way + // either. 16 clauses in the tree flipped `rethrows` this round and every one was eligible for the + // false wording. Whether `fail` is the right verdict for them is a separate question, parked. + it("does not accuse a clause that throws of taking one way out", () => { + const r = run( + "error-classification", + "mixed.ts", + `import { prisma } from "~/db.server"; + export async function loader() { + try { return json(await prisma.thing.findMany()); } + catch (e) { if (rare) { return null; } throw e; } + }` + ); + expect(r.status).toBe("fail"); + expect(r.detail).toContain("without looking at what was thrown"); + expect(r.detail).not.toContain("one way out"); + }); + + it("still says one way out for a clause that never throws", () => { + const r = run( + "error-classification", + "swallow.ts", + `import { prisma } from "~/db.server"; + export async function loader() { + try { return json(await prisma.thing.findMany()); } + catch (e) { logger.error(e); return null; } + }` + ); + expect(r.status).toBe("fail"); + expect(r.detail).toContain("one way out"); + }); + + // I4. A route that owns a real classifying catch must never be told it owns none. `canRaise` does + // not list destructuring, and `const { a } = undefined` throws, so the owned catch dropped out of + // `reachable`; with the callback branch ordered off `reachable` the route was then accused of + // having all its error handling in a callback, which was simply false. + it("does not accuse a route that owns a catch of owning none", () => { + const r = run( + "error-classification", + "owned.ts", + `import { prisma } from "~/db.server"; + export async function action({ request }) { + const items = await prisma.item.findMany(); + await Promise.all( + items.map(async (item) => { + try { await processItem(item); } catch { return null; } + }) + ); + try { const { a } = undefined; } catch (e) { + if (e instanceof TypeError) { return new Response(null, { status: 400 }); } + throw e; + } + return json({ ok: true }); + }` + ); + expect(r.detail).not.toContain("callback the route does not own"); + }); + // C2. A single-element array cannot iterate, so `[0].map(async () => { whole body })` is not a // per-item boundary and the route's own catch is found where it always was. Before this, the // wrapper deleted the route's catches and took a swallow from fail to not-applicable. diff --git a/internal-packages/observability-map/test/scan.test.ts b/internal-packages/observability-map/test/scan.test.ts index 5e08f0f3a5a..c10f6d2af66 100644 --- a/internal-packages/observability-map/test/scan.test.ts +++ b/internal-packages/observability-map/test/scan.test.ts @@ -754,16 +754,26 @@ describe("scanFile: catch clause evidence", () => { }); } - // The same six wrappers on the branches side, which is what pins `definitelyExits` itself: the - // no-return rule above says nothing about branch credit, so only the cut sees these. An error - // test written after a construct that already returned is dead code and must not read as the - // clause deciding anything. + // The same wrappers on the branches side, plus three the rethrow list has no use for. An error + // test written after a statement that could already have left the clause is dead code and must + // not read as the clause deciding anything. // - // `an if (true)` is absent from this list on purpose. Nothing here evaluates a condition, so - // the cut cannot see that the wrapper always exits, and the error test after it is still - // credited. That residual is `dead-branch-after-if-true` in the mutation corpus, which runs as - // an expected failure with the two rejected alternatives written out beside it. - const BRANCH_EXITED = EXITED.filter(([label]) => label !== "an if (true)"); + // This asks a weaker question than `definitelyExits` does, and on purpose: "could this have + // exited", not "must it have". That is why `if (true)`, a labelled block, a `for...of` and a + // `while` are all on the list even though none of them is guaranteed to run its body. Nothing + // here evaluates a condition, and none of these needs one evaluated. + // + // The ordering is the whole trick and it is easy to get backwards. The flag is raised at the + // END of each statement, after that statement's own branch check. Raising it first makes every + // deciding statement refuse itself, because `if (e instanceof X) return y` contains an exit by + // definition; that variant was measured and it takes the tree from 15 to 6 and accuses 78 + // routes. This one leaves the real-tree report and all 240 clauses' evidence byte-identical. + const BRANCH_EXITED: Array<[string, string]> = [ + ...EXITED, + ["a labelled block", "outer: { return null; }"], + ["a for...of that returns", "for (const q of items) { return q; }"], + ["a while that returns", "while (go) { return null; }"], + ]; for (const [label, wrapped] of BRANCH_EXITED) { it(`does not credit an error test written after ${label}`, () => { @@ -781,6 +791,24 @@ describe("scanFile: catch clause evidence", () => { }); } + // The precision this gives up, pinned so it is a decision and not a surprise: a conditional + // exit before the error test also stops the credit, because the walk cannot tell a guard that + // usually falls through from one that always leaves. No clause in the route tree is this shape, + // which is why the report is byte-identical, but one could be written tomorrow. + it("does not credit an error test written after a conditional return", () => { + const ep = scanFile( + "x.ts", + `export async function loader() { + try { return await prisma.thing.findMany(); } + catch (e) { + if (rare) { return null; } + if (e instanceof Error) { return json({ a: 1 }); } + } + }` + ); + expect(ep!.catches[0]!.branches).toBe(false); + }); + it("still credits an error test with nothing exiting before it", () => { const ep = scanFile( "x.ts", @@ -1250,6 +1278,7 @@ describe("scanFile: per-catch evidence", () => { expect(ep!.catches[0]).toEqual({ rethrows: false, branches: false, + throws: false, guardsParse: true, awaitsOnlyParse: true, guardCanRaise: true, @@ -1353,6 +1382,7 @@ describe("scanFile: per-catch evidence", () => { expect(ep!.catches[0]).toEqual({ rethrows: false, branches: false, + throws: false, guardsParse: false, awaitsOnlyParse: false, guardCanRaise: true, From 3e1ee436ccab6fc9bda8b122524b19e984ecc292 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Sat, 1 Aug 2026 21:40:30 +0100 Subject: [PATCH 060/106] docs(observability-map): say what the unreachable-catch rule actually refuses It refuses `try { 0; }`. Five places said or implied it refuses an unreachable catch, which is a family, and canRaise accepts any call, member access or `in`, none of which has to be able to throw. `try { String(0); }` therefore reads as real classification and pays the same 224 routes and the same 15 to 42 that `try { 0; }` did before it was refused. Telling an inert call from one that can throw needs types the scanner does not have, so the shape is parked rather than chased: dead-classifying-try-with-call is in the corpus as an expected failure, and scan.ts, types.ts, errorClassification.ts, the README and the corpus entry all now say refuses `try { 0; }` and is defeated by one call. canRaise also misses things that can raise, which drops a real clause rather than crediting a fake one. Destructuring is now named in that residual, since it is the miss that produced the false accusation fixed alongside this. Two more claims that ran ahead of the code. The corpus said a defended mutation moves no verdict anywhere, which several defended entries plainly disprove; the true reason movement cannot be required is narrower, that the ideal defended shape is one the scanner is blind to and those move nothing. And the docstring checker collects leading comment ranges only, so a comment with no node after it is never scanned; that is now in its list of what it does not check, alongside the other three. --- internal-packages/observability-map/README.md | 14 +++++--- .../test/docstringReferences.test.ts | 16 ++++++--- .../test/mutationCorpus.test.ts | 33 +++++++++++-------- .../observability-map/test/mutations.ts | 21 ++++++++++++ 4 files changed, 62 insertions(+), 22 deletions(-) diff --git a/internal-packages/observability-map/README.md b/internal-packages/observability-map/README.md index c62c4ad4360..6f864877490 100644 --- a/internal-packages/observability-map/README.md +++ b/internal-packages/observability-map/README.md @@ -40,7 +40,7 @@ every catch clause in the tree used to score it 100, which meant the metric paid error handling. The property behind that is now a test corpus rather than a claim. `test/mutationCorpus.test.ts` -applies 39 semantics-preserving or handling-deleting rewrites to the whole route tree in a temp copy +applies 40 semantics-preserving or handling-deleting rewrites to the whole route tree in a temp copy and asserts three things for each: the published global does not rise, the mean over the routes measured in both runs does not rise, and for a semantics-preserving rewrite no individual route's score rises or drops out of the measured set. Every laundering shape a reviewer has found on this @@ -53,13 +53,17 @@ not pay you for adding error handling that does nothing either. The rewrites come in two directions and both matter. A subtractive one takes real signal away or moves it about: delete the catches, wrap the body, merge the statements. An additive one puts fake -signal in: a classifying catch over a try that cannot throw, a test whose two arms are the same, a +signal in: a classifying catch over a try that does nothing, a test whose two arms are the same, a rethrow that can never run. The corpus had only the subtractive half for a while, and the two largest holes ever found here were both additive. -The honest statement is "these 38 rewrites are defended, here they are, and here is the one that is -not", not "unpaddable". One entry, `dead-branch-after-if-true`, runs as an expected failure with the -residual written out beside it. The corpus takes about four minutes, so it is gated behind +One of those is still open and the corpus says so. A catch over `try { 0; }` is refused, but +`canRaise` accepts any call, so `try { String(0); }` reads as real error handling and pays the same +224 routes. Telling an inert call from one that can throw needs types the scanner does not have. + +The honest statement is "these 39 rewrites are defended, here they are, and here is the one that is +not", not "unpaddable". One entry, `dead-classifying-try-with-call`, runs as an expected failure +with the residual written out beside it. The corpus takes about four minutes, so it is gated behind `OBS_MAP_MUTATION_CORPUS=1` and run as its own CI job rather than in `pnpm test`. If you change this package, run it: diff --git a/internal-packages/observability-map/test/docstringReferences.test.ts b/internal-packages/observability-map/test/docstringReferences.test.ts index 3800066b2c4..51bfefa5d16 100644 --- a/internal-packages/observability-map/test/docstringReferences.test.ts +++ b/internal-packages/observability-map/test/docstringReferences.test.ts @@ -21,10 +21,18 @@ import { MUTATIONS } from "./mutations.js"; * punctuation, e.g. `jsx text is content, not a comment`. That is what a test title looks like * and what a code sample does not. * - * What is NOT checked: a reference written without backticks, a test title of fewer than - * `MINIMUM_TITLE_WORDS` words (`throw e` and `new URL` are code, and telling a short title from - * short code needs more than punctuation), and anything outside `src/`. A docstring can still name - * a nonexistent short test. The kebab half is the half that has actually failed. + * What is NOT checked, and each of these is a place a bad reference can still hide: + * + * - a reference written without backticks. + * - a test title of fewer than `MINIMUM_TITLE_WORDS` words. `throw e` and `new URL` are code, and + * telling a short title from short code needs more than punctuation. + * - a comment with no node after it. `commentText` collects leading ranges only, so a comment on + * the last line of a block or at the end of a file is never scanned at all. Every docstring in + * this package precedes a declaration, which is why the collector was written that way, and it + * is a coverage hole rather than a design choice. + * - anything outside `src/`, including the docstrings in this file and in `mutations.ts`. + * + * The kebab half is the half that has actually failed. */ const SRC = resolve(__dirname, "../src"); diff --git a/internal-packages/observability-map/test/mutationCorpus.test.ts b/internal-packages/observability-map/test/mutationCorpus.test.ts index 8dd03ee6ded..02685541f2d 100644 --- a/internal-packages/observability-map/test/mutationCorpus.test.ts +++ b/internal-packages/observability-map/test/mutationCorpus.test.ts @@ -39,17 +39,20 @@ const ENABLED = process.env.OBS_MAP_MUTATION_CORPUS === "1"; * Where a corpus entry goes when the tool does not defend it. `it.fails` keeps the entry running, * so closing the hole later turns this file red until the entry is moved back out deliberately. * - * `dead-branch-after-if-true` wraps a catch clause's body in `if (true) { ... }` and writes an - * error test after it. The test can never run, and it takes the tree from 15 to 23 and raises 68 - * routes. Nothing in the scanner evaluates a condition, so `definitelyExits` cannot see that the - * wrapper always exits, and the two ways to make it see are both worse than the hole. Folding the - * literal `true` starts the same list an earlier round lost with `false`, where every spelling - * nobody thought of pays. Cutting the branch credit at the first statement that MIGHT exit is - * sound and folding-free, and it was measured: it takes the tree from 15 to 6 and accuses 78 real - * routes, because `catch (e) { if (rare) return null; if (e instanceof X) return y; }` is an - * ordinary shape. See the round A fix 3 report. + * `dead-classifying-try-with-call` is the shape `dead-classifying-try` only looked like it closed. + * `canRaise` accepts any call at all, so `try { String(0); }` reads as a clause guarding real work + * and takes the tree from 15 to 42, raising 224 routes, exactly as `try { 0; }` did before it was + * refused. Telling an inert call from one that can throw needs types the scanner does not have. + * The docstrings in `scan.ts`, `types.ts` and `errorClassification.ts` say the rule refuses + * `try { 0; }` and is defeated by one call, rather than claiming the family is closed. + * + * `dead-branch-after-if-true` used to be listed here on a measurement that was wrong. See the round + * A fix 3 report; the short version is that the rejected alternative was implemented with the exit + * flag raised before each statement's own branch check, which makes every deciding statement refuse + * itself. Raising it after is byte-identical on the real tree and closes the shape, so the entry is + * defended now and the `if (true)` family needed no condition folding after all. */ -const KNOWN_GAPS = new Set(["dead-branch-after-if-true"]); +const KNOWN_GAPS = new Set(["dead-classifying-try-with-call"]); type SourceFile = { relativeName: string; source: string }; @@ -229,9 +232,13 @@ describeCorpus("mutation corpus over the real route tree", () => { * at the head of the clause, so all 260 count, and this threshold is what would notice if a later * change quietly took that back. * - * The guard the design asked for, verdict movement, cannot be used: a defended mutation moves no - * verdict anywhere, and that is precisely what "defended" means, so requiring movement would fail - * every entry that works. Site count is the reachable version of the same intent. + * The guard the design asked for, verdict movement, cannot be used, though not for the reason an + * earlier version of this comment gave. Plenty of defended entries move verdicts hard: + * `delete-every-catch` takes the tree from 15 to 2 and `dead-throw-after-switch` to 6. The + * narrower true reason is that the IDEAL defended shape is one the scanner is blind to, and those + * move nothing at all: `dead-if-false` and the ten entries beside it are defended precisely + * because the tree comes out identical. Requiring movement would fail exactly the entries that + * work best. Site count is the reachable version of the same intent. */ const MINIMUM_FILES_TOUCHED = 20; const MINIMUM_SITES_TOUCHED = 40; diff --git a/internal-packages/observability-map/test/mutations.ts b/internal-packages/observability-map/test/mutations.ts index 894dd8da9d4..7d94064efa1 100644 --- a/internal-packages/observability-map/test/mutations.ts +++ b/internal-packages/observability-map/test/mutations.ts @@ -648,6 +648,26 @@ export const MUTATIONS: Mutation[] = [ return applyEdits(source, edits); }, }, + { + id: "dead-classifying-try-with-call", + kind: "preserving", + what: "prepend a classifying try/catch over a try block whose only work is an inert call", + apply(fileName, source) { + const sf = parse(fileName, source); + const edits: Edit[] = []; + for (const body of entryBodies(sf)) { + edits.push( + insert( + body.getStart() + 1, + "\ntry { String(0); } catch (obsMapDead) {" + + " if (obsMapDead instanceof Error) { return new Response(null, { status: 400 }); }" + + " throw obsMapDead; }\n" + ) + ); + } + return applyEdits(source, edits); + }, + }, { id: "same-arms-ternary", kind: "preserving", @@ -802,6 +822,7 @@ export const MUTATIONS: Mutation[] = [ */ export const ADDITIVE_IDS = [ "dead-classifying-try", + "dead-classifying-try-with-call", "same-arms-ternary", "dead-throw-after-block", "dead-throw-after-do", From b2e030b718fa81a04f5e72e37bedb114e5dcdc76 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Sat, 1 Aug 2026 22:22:40 +0100 Subject: [PATCH 061/106] fix(observability-map): read parse diagnostics through public typescript api The parse-failure guard read sf.parseDiagnostics, an internal property the compiler is free to rename. If it went away the detection would stop working silently and the report would keep saying zero parse failures, while an undetected failure shrinks the denominator and inflates the score. It now asks a ts.Program for the syntactic diagnostics, with a host that hands the program the source file already parsed, so nothing is parsed twice. The cost is real and measured: a full scan of the route tree goes from about 850ms to about 1450ms over five runs of each. A slower scan is the cheaper of the two prices. The change is invisible to every existing test, both spellings find the same malformed files, so the new guard is a source-level one. Three more malformed shapes are covered as well: an unclosed jsx element, an unterminated template literal, and a stray closing brace. --- .../observability-map/src/scan.ts | 40 ++++++++++++++-- .../observability-map/test/scan.test.ts | 47 ++++++++++++++++++- 2 files changed, 81 insertions(+), 6 deletions(-) diff --git a/internal-packages/observability-map/src/scan.ts b/internal-packages/observability-map/src/scan.ts index 5521833a382..f2ddbd77156 100644 --- a/internal-packages/observability-map/src/scan.ts +++ b/internal-packages/observability-map/src/scan.ts @@ -891,15 +891,47 @@ function collectLocalFunctions(sf: ts.SourceFile): Map { return functions; } +/** + * Compiler options for the throwaway program below. `noLib` and `noResolve` keep it from going to + * disk: nothing here needs a type, only the syntax the parser already produced. + */ +const SYNTAX_ONLY_OPTIONS: ts.CompilerOptions = { noLib: true, noResolve: true, allowJs: true }; + +/** + * Syntactic diagnostics for an already-parsed source file, through `ts.Program` rather than off + * the diagnostics array the parser hangs on the source file, which is internal and which the + * compiler is free to rename. The whole parse-failure discipline rests on this, and an undetected + * parse failure shrinks the denominator and inflates the score, so it must not be the kind of + * thing a compiler upgrade can switch off silently. + * + * The host hands the program the `sf` we already have, so this does not parse the source a second + * time. The cost is the program machinery around it, and it is not free: a full scan of the real + * route tree went from about 850ms to about 1450ms, measured over five runs of each. A slower + * scan of a tool that runs once a pull request is the cheaper of the two prices. + */ +function syntacticDiagnostics(sf: ts.SourceFile): readonly ts.Diagnostic[] { + const host: ts.CompilerHost = { + getSourceFile: (name) => (name === sf.fileName ? sf : undefined), + getDefaultLibFileName: () => "lib.d.ts", + writeFile: () => {}, + getCurrentDirectory: () => "", + getCanonicalFileName: (name) => name, + useCaseSensitiveFileNames: () => true, + getNewLine: () => "\n", + fileExists: (name) => name === sf.fileName, + readFile: () => undefined, + }; + return ts.createProgram([sf.fileName], SYNTAX_ONLY_OPTIONS, host).getSyntacticDiagnostics(sf); +} + export function scanFile(fileName: string, source: string): EntryPoint | null { const sf = ts.createSourceFile(fileName, source, ts.ScriptTarget.Latest, true); // `createSourceFile` recovers from malformed input instead of throwing, so the diagnostics are // the only signal that a file did not parse. - const parseDiagnostics = (sf as ts.SourceFile & { parseDiagnostics?: ts.Diagnostic[] }) - .parseDiagnostics; - if (parseDiagnostics && parseDiagnostics.length > 0) { - const first = parseDiagnostics[0]!; + const diagnostics = syntacticDiagnostics(sf); + if (diagnostics.length > 0) { + const first = diagnostics[0]!; throw new ParseFailureError(fileName, ts.flattenDiagnosticMessageText(first.messageText, " ")); } diff --git a/internal-packages/observability-map/test/scan.test.ts b/internal-packages/observability-map/test/scan.test.ts index c10f6d2af66..7e571f33720 100644 --- a/internal-packages/observability-map/test/scan.test.ts +++ b/internal-packages/observability-map/test/scan.test.ts @@ -1,6 +1,6 @@ -import { chmodSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { chmodSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; -import { join } from "node:path"; +import { join, resolve } from "node:path"; import { ParseFailureError, scanDirectory, scanFile } from "../src/scan.js"; const LOADER = ` @@ -494,6 +494,49 @@ describe("scanFile: parse failures", () => { ).toThrow(ParseFailureError); }); + // B8. The detection used to read `sf.parseDiagnostics`, an internal property. These are the + // shapes that prove the public route through `ts.Program` still sees a malformed file. + it("throws on an unclosed jsx element in a tsx route", () => { + expect(() => + scanFile( + "broken.route.tsx", + `export async function loader() { return json({}); } + export default function Page() { return
hi; }` + ) + ).toThrow(ParseFailureError); + }); + + it("throws on an unterminated template literal", () => { + expect(() => + scanFile("broken.ts", `export async function loader() { return \`unterminated; }`) + ).toThrow(ParseFailureError); + }); + + it("throws on a stray closing brace after a complete function", () => { + expect(() => + scanFile("broken.ts", `export async function loader() { return json({}); } }`) + ).toThrow(ParseFailureError); + }); + + it("names the diagnostic rather than reporting a bare failure", () => { + try { + scanFile("broken.ts", `export async function loader() { const a = ; }`); + expect.unreachable("scanFile should have thrown"); + } catch (error) { + expect(error).toBeInstanceOf(ParseFailureError); + expect((error as ParseFailureError).diagnostic.length).toBeGreaterThan(0); + } + }); + + // The change from `sf.parseDiagnostics` to a program-backed lookup is invisible to every test + // above: both spellings find the same malformed files today. What a compiler upgrade can break + // is the private one, and only a source-level guard can fail for that. + it("reads its diagnostics through public typescript api rather than a private field", () => { + const source = readFileSync(resolve(__dirname, "../src/scan.ts"), "utf8"); + expect(source).not.toContain("parseDiagnostics"); + expect(source).toContain("getSyntacticDiagnostics"); + }); + it("does not throw on a well-formed tsx route", () => { expect(() => scanFile( From d0d50f859e345d2a157166f9c30dc2a83a754465 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Sat, 1 Aug 2026 22:23:02 +0100 Subject: [PATCH 062/106] fix(observability-map): report a suppression that names no check instead of dropping it `// obs-map-disable eror-classification -- typo` parsed, landed in the suppression map, matched nothing and appeared nowhere. The author reads the finding as acknowledged and the tool goes on reporting it with no hint why. The parser now splits directives by whether the id names a check in CHECKS. The unknown ones are carried on the scored entry and on the report, and the terminal report prints one UNKNOWN SUPPRESSION line per file naming the file, the bad ids, and the ids that would have worked. It is not thrown and it is not dropped. --- .../observability-map/src/report/terminal.ts | 26 +++++++- .../observability-map/src/score.ts | 14 ++++- .../observability-map/src/suppression.ts | 33 +++++++++-- .../test/docstringReferences.test.ts | 3 + .../test/suppression.test.ts | 59 ++++++++++++++++++- 5 files changed, 126 insertions(+), 9 deletions(-) diff --git a/internal-packages/observability-map/src/report/terminal.ts b/internal-packages/observability-map/src/report/terminal.ts index e85c629bb2d..a0f5e779b52 100644 --- a/internal-packages/observability-map/src/report/terminal.ts +++ b/internal-packages/observability-map/src/report/terminal.ts @@ -1,5 +1,5 @@ import type { MapReport, ScoredEntry } from "../score.js"; -import { SCORED_CHECK_IDS } from "../checks/index.js"; +import { CHECKS, SCORED_CHECK_IDS } from "../checks/index.js"; const NOT_MEASURED = "not measured".padEnd(15); @@ -32,6 +32,24 @@ export const contextOnly = (e: ScoredEntry) => { return failures.length === 1 && failures[0]!.id === "request-context"; }; +/** + * The UNKNOWN SUPPRESSION lines, one per file, shared with `prComment.ts`. Empty when every + * directive named a real check. A typo suppresses nothing, so without this the author reads the + * finding as acknowledged and the tool goes on reporting it with no hint why. + */ +export function unknownSuppressionLine(fileName: string, ids: string[]): string { + return ( + `UNKNOWN SUPPRESSION ${fileName}: ${ids.join(", ")} ` + + `(no such check, nothing suppressed). Known: ${CHECKS.map((c) => c.id).join(", ")}.` + ); +} + +export function unknownSuppressionLines(report: MapReport): string[] { + return report.unknownSuppressions.map(({ fileName, ids }) => + unknownSuppressionLine(fileName, ids) + ); +} + /** The AUDIT figure, shared with `prComment.ts` so both renderers say the same thing. Null when * there is nothing to report, i.e. no sensitive mutation exists. */ export function auditLine(report: MapReport): string | null { @@ -93,6 +111,12 @@ export function renderTerminal(report: MapReport): string { lines.push(context); } + const unknown = unknownSuppressionLines(report); + if (unknown.length > 0) { + lines.push(""); + lines.push(...unknown); + } + if (report.suppressions.checks > 0) { const { entries, checks } = report.suppressions; lines.push( diff --git a/internal-packages/observability-map/src/score.ts b/internal-packages/observability-map/src/score.ts index 420a07b19f4..7f56c5783aa 100644 --- a/internal-packages/observability-map/src/score.ts +++ b/internal-packages/observability-map/src/score.ts @@ -1,6 +1,6 @@ import type { CheckResult, EntryPoint } from "./types.js"; import { CHECKS, SCORED_CHECK_IDS } from "./checks/index.js"; -import { suppressedChecks } from "./suppression.js"; +import { parseSuppressions } from "./suppression.js"; import { familyOf, routePathOf, type Family } from "./adapters/remix.js"; import { classifySensitivity } from "./sensitivity.js"; @@ -30,6 +30,10 @@ export type ScoredEntry = { /** Every check a comment in the source suppressed, scored or not, in `CHECKS` order. Includes * `audit-trail`: a suppression is real regardless of whether its check feeds the score. */ suppressed: string[]; + /** Ids in a suppression directive that name no check, so they suppress nothing. Carried here so + * the renderers can say so: dropping them silently is what made a typo look like an + * acknowledgement. */ + unknownSuppressions: string[]; /** Passed over applicable, across scored checks only. 100 when nothing applies. */ score: number; }; @@ -43,6 +47,8 @@ export type MapReport = { unmeasured: number; /** Suppressions in force: how many entry points carry one, and how many scored checks in total. */ suppressions: { entries: number; checks: number }; + /** Suppression directives naming no check, per file, so a typo is reported rather than dropped. */ + unknownSuppressions: { fileName: string; ids: string[] }[]; byFamily: Record; sensitiveCohort: { n: number; measured: number; mean: number | null }; auditGap: { sensitiveMutations: number; withAudit: number }; @@ -57,7 +63,7 @@ export type MapReport = { }; export function scoreEntry(ep: EntryPoint): ScoredEntry { - const suppressed = suppressedChecks(ep.source, ep.fileName); + const { byId: suppressed, unknown } = parseSuppressions(ep.source, ep.fileName); const raw = CHECKS.map((c) => c.run(ep)); const checks = raw.map((result) => { const reason = suppressed.get(result.id); @@ -86,6 +92,7 @@ export function scoreEntry(ep: EntryPoint): ScoredEntry { checks, rawChecks: raw, suppressed: raw.filter((c) => suppressed.has(c.id)).map((c) => c.id), + unknownSuppressions: unknown, measured: scoredApplicable.length > 0, // Capped by the pre-suppression ratio: removing a failing check from both the numerator and // the denominator otherwise raises the ratio, which is how 33 became 50 became 100 before this @@ -157,6 +164,9 @@ export function buildReport(eps: EntryPoint[], parseFailures: string[]): MapRepo entries: suppressing.length, checks: suppressing.reduce((n, e) => n + e.suppressed.length, 0), }, + unknownSuppressions: entries + .filter((e) => e.unknownSuppressions.length > 0) + .map((e) => ({ fileName: e.fileName, ids: e.unknownSuppressions })), byFamily, sensitiveCohort: groupStats(sensitive), auditGap: { diff --git a/internal-packages/observability-map/src/suppression.ts b/internal-packages/observability-map/src/suppression.ts index 476c2a5a2ee..7499cf74179 100644 --- a/internal-packages/observability-map/src/suppression.ts +++ b/internal-packages/observability-map/src/suppression.ts @@ -1,4 +1,7 @@ import ts from "typescript"; +import { CHECKS } from "./checks/index.js"; + +const KNOWN_CHECK_IDS = new Set(CHECKS.map((c) => c.id)); /** * The directive, and the reason that must follow it. The reason runs to the end of the line: `.` @@ -120,14 +123,26 @@ function commentLines(source: string, sf: ts.SourceFile): string[] { return lines; } +export type Suppressions = { + /** Check id to reason, for ids that name a check in `CHECKS`. */ + byId: Map; + /** + * Ids that parsed as a directive but name no check, in source order and deduplicated. A typo + * (`eror-classification`) used to land in the map, match nothing and appear nowhere, so the + * author read the finding as acknowledged while the tool kept reporting it. + */ + unknown: string[]; +}; + /** - * Check id to reason. A suppression without a reason, or outside a comment, is ignored. + * Every suppression directive in the source, split by whether its id names a real check. A + * directive without a reason, or outside a comment, is ignored either way. * * `fileName` picks the parser's script kind: JSX syntax is only legal, and only correctly * distinguished from a generic type argument list (`(x) => x`), when the file is really a * `.tsx`. Defaults to a plain `.ts` for callers that only have source text. */ -export function suppressedChecks(source: string, fileName = "check.ts"): Map { +export function parseSuppressions(source: string, fileName = "check.ts"): Suppressions { const scriptKind = fileName.endsWith(".tsx") ? ts.ScriptKind.TSX : ts.ScriptKind.TS; const sf = ts.createSourceFile( fileName, @@ -137,13 +152,21 @@ export function suppressedChecks(source: string, fileName = "check.ts"): Map(); + const byId = new Map(); + const unknown = new Set(); for (const line of commentLines(source, sf)) { const match = PATTERN.exec(line); if (!match) continue; const [, id, reason] = match; const trimmedReason = reason?.trim(); - if (id && trimmedReason && trimmedReason.length > 0) out.set(id, trimmedReason); + if (!id || !trimmedReason || trimmedReason.length === 0) continue; + if (KNOWN_CHECK_IDS.has(id)) byId.set(id, trimmedReason); + else unknown.add(id); } - return out; + return { byId, unknown: [...unknown] }; +} + +/** The known half of `parseSuppressions`, for callers that only apply suppressions. */ +export function suppressedChecks(source: string, fileName = "check.ts"): Map { + return parseSuppressions(source, fileName).byId; } diff --git a/internal-packages/observability-map/test/docstringReferences.test.ts b/internal-packages/observability-map/test/docstringReferences.test.ts index 51bfefa5d16..20e922bae92 100644 --- a/internal-packages/observability-map/test/docstringReferences.test.ts +++ b/internal-packages/observability-map/test/docstringReferences.test.ts @@ -45,6 +45,9 @@ const NOT_A_TEST_NAME = new Set([ "not-applicable", // The directive spelling that was retired, named in `suppression.ts` to say it is not honoured. "obs-map-disable-next-line", + // The worked example of a mistyped check id in `suppression.ts`. A misspelling of a check id is + // the thing being described, so it names no test by construction. + "eror-classification", ]); /** A backticked phrase this long or longer, with no code punctuation, is read as a test title. */ diff --git a/internal-packages/observability-map/test/suppression.test.ts b/internal-packages/observability-map/test/suppression.test.ts index c40d1332894..10679133550 100644 --- a/internal-packages/observability-map/test/suppression.test.ts +++ b/internal-packages/observability-map/test/suppression.test.ts @@ -1,4 +1,4 @@ -import { suppressedChecks } from "../src/suppression.js"; +import { parseSuppressions, suppressedChecks } from "../src/suppression.js"; describe("suppressedChecks", () => { it("reads a suppression with its reason", () => { @@ -243,3 +243,60 @@ describe("suppressedChecks", () => { expect(m.get("auth-boundary")).toBe("generic helper"); }); }); + +// B6. `// obs-map-disable eror-classification -- typo` used to parse, land in the map, match no +// check and appear nowhere, so the author read the finding as acknowledged. +describe("a suppression naming a check that does not exist", () => { + it("suppresses nothing and is reported as unknown", () => { + const r = parseSuppressions( + `// obs-map-disable eror-classification -- typo + export async function loader() { return 1; }` + ); + expect(r.byId.size).toBe(0); + expect(r.unknown).toEqual(["eror-classification"]); + }); + + it("does not swallow the real suppressions beside it", () => { + const r = parseSuppressions( + `// obs-map-disable auth-boundry -- typo + // obs-map-disable auth-boundary -- public by design + export async function loader() { return 1; }` + ); + expect(r.byId.get("auth-boundary")).toBe("public by design"); + expect(r.unknown).toEqual(["auth-boundry"]); + }); + + it("reports each unknown id once however many times it appears", () => { + const r = parseSuppressions( + `// obs-map-disable request-contex -- typo + /* obs-map-disable request-contex -- typo again */ + // obs-map-disable audit-trial -- another typo + export async function loader() { return 1; }` + ); + expect(r.unknown).toEqual(["request-contex", "audit-trial"]); + }); + + it("is not reported when the directive had no reason, since it was never a suppression", () => { + const r = parseSuppressions( + `// obs-map-disable eror-classification + export async function loader() { return 1; }` + ); + expect(r.unknown).toEqual([]); + }); + + it("is not read out of a string literal any more than a real one is", () => { + const r = parseSuppressions( + `const help = "// obs-map-disable eror-classification -- typo"; + export async function loader() { return help; }` + ); + expect(r.unknown).toEqual([]); + }); + + it("keeps suppressedChecks returning only the ids that name a check", () => { + const m = suppressedChecks( + `// obs-map-disable eror-classification -- typo + export async function loader() { return 1; }` + ); + expect(m.size).toBe(0); + }); +}); From e0860dd2492b072d6cf34d9b3c1a074fde181fe2 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Sat, 1 Aug 2026 22:23:19 +0100 Subject: [PATCH 063/106] test(observability-map): cover the unknown-suppression line, and guard the fix-list slices Two slices took `out.indexOf("FIX FIRST")` without checking it. When the section is absent that is -1, `slice(-1)` is the last character of the report, and the `not.toContain` below it passes for the wrong reason. It is the trap this file's own comment already warned about. Both now go through one helper that asserts both ends, and the helper has a test of its own so it can fail. Adds the terminal coverage for the unknown-suppression line: the file and the bad id are named, the known ids are listed, the finding is not counted as suppressed, and one line is printed per file rather than one for the run. --- .../observability-map/test/report.test.ts | 66 +++++++++++++++++-- 1 file changed, 61 insertions(+), 5 deletions(-) diff --git a/internal-packages/observability-map/test/report.test.ts b/internal-packages/observability-map/test/report.test.ts index 7beed431da1..7979800a02a 100644 --- a/internal-packages/observability-map/test/report.test.ts +++ b/internal-packages/observability-map/test/report.test.ts @@ -20,7 +20,24 @@ const report = () => ["broken.ts"] ); +/** The FIX FIRST section, with the index asserted rather than assumed. `indexOf` returns -1 for a + * section that is not there, and `slice(-1)` is the last character of the report, which every + * `not.toContain` below would pass against. */ +function sliceFixFirst(out: string): string { + const start = out.indexOf("FIX FIRST"); + expect(start).toBeGreaterThan(-1); + const end = out.indexOf("no findings:"); + expect(end).toBeGreaterThan(start); + return out.slice(start, end); +} + describe("renderTerminal", () => { + // The guard has to be able to fail, or it is decoration in a file whose own comment warns about + // exactly this trap. + it("refuses to slice a report with no fix list rather than returning its last character", () => { + expect(() => sliceFixFirst("a report with neither section in it")).toThrow(); + }); + it("shows the global score, the audit gap and the fix list", () => { const out = renderTerminal(report()); expect(out).toContain("COVERAGE"); @@ -119,9 +136,7 @@ describe("renderTerminal", () => { // Slice to the end of the list, not to a string the I5 fix deleted: `indexOf` returned -1 for // "already solid" and the assertions were quietly running against the whole tail. - const listEnd = out.indexOf("no findings:"); - expect(listEnd).toBeGreaterThan(-1); - const fixFirst = out.slice(out.indexOf("FIX FIRST"), listEnd); + const fixFirst = sliceFixFirst(out); const idxZero = fixFirst.indexOf("api.v1.envvars.ts"); const idxThirtyThree = fixFirst.indexOf("api.v1.auth.tokens.ts"); const idxNotSensitive = fixFirst.indexOf("resources.busy.ts"); @@ -232,7 +247,9 @@ describe("collapsing the house-style finding", () => { // single finding repeated. Same reasoning that keeps audit-trail out of the list. it("keeps an entry whose only finding is request-context out of the fix list", () => { const out = renderTerminal(buildReport([namesNobody()], [])); - const fixFirst = out.slice(out.indexOf("FIX FIRST")); + // Guarded for the same reason as the slice above: an absent section makes `indexOf` return -1, + // `slice(-1)` yields the last character, and the negative assertion passes for the wrong reason. + const fixFirst = sliceFixFirst(out); expect(fixFirst).not.toContain("api.v1.silent.ts"); }); @@ -264,9 +281,48 @@ describe("collapsing the house-style finding", () => { it("still lists request-context when the entry fails something else too", () => { const out = renderTerminal(buildReport([namesNobodyAndSwallows()], [])); - const fixFirst = out.slice(out.indexOf("FIX FIRST")); + const fixFirst = sliceFixFirst(out); expect(fixFirst).toContain("api.v1.auth.tokens.ts"); expect(fixFirst).toContain("request-context"); expect(fixFirst).toContain("error-classification"); }); }); + +// B6. A stderr warning is the minimum; the terminal report is where someone would notice. +describe("reporting a suppression that names no check", () => { + const typo = (fileName: string) => + scanFile( + fileName, + `// obs-map-disable eror-classification -- typo + import { prisma } from "~/db.server"; + export async function loader() { + try { return await prisma.thing.findMany(); } catch (e) { return null; } + }` + )!; + + it("names the file and the bad id", () => { + const out = renderTerminal(buildReport([typo("api.v1.a.ts")], [])); + expect(out).toContain("UNKNOWN SUPPRESSION"); + expect(out).toContain("api.v1.a.ts"); + expect(out).toContain("eror-classification"); + }); + + it("lists the ids that would have worked", () => { + const out = renderTerminal(buildReport([typo("api.v1.a.ts")], [])); + expect(out).toContain("error-classification, auth-boundary, request-context, audit-trail"); + }); + + it("does not report the finding as suppressed", () => { + const out = renderTerminal(buildReport([typo("api.v1.a.ts")], [])); + expect(out).not.toMatch(/SUPPRESSED\s+\d/); + }); + + it("reports one line per file rather than one for the run", () => { + const out = renderTerminal(buildReport([typo("api.v1.a.ts"), typo("api.v1.b.ts")], [])); + expect(out.split("\n").filter((l) => l.startsWith("UNKNOWN SUPPRESSION"))).toHaveLength(2); + }); + + it("says nothing when every directive named a real check", () => { + expect(renderTerminal(report())).not.toContain("UNKNOWN SUPPRESSION"); + }); +}); From 3241352fc5542602cfcb8fd17ce9c80732b369a4 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Sat, 1 Aug 2026 22:23:39 +0100 Subject: [PATCH 064/106] fix(observability-map): render measured state in the pr comment, and post only when it moves The changed-entries table compared raw scores. An entry no scored check applies to carries 100 as a placeholder, which the score itself excludes from every mean, so the table rendered a route refactored down to a trivial body as a 67-point improvement and a trivial route gaining real work as the pull request's worst regression. Both columns now say "not measured" for an unmeasured entry, the way the terminal gauge already does for a null mean, and the early-out compares measured state as well as score so a measured 100 becoming an unmeasured 100 still produces a row. A new entry that passes every check it was measured against no longer gets a row: its drop is 0, which by the drop doc comment's own logic means there is nothing to fix. A new entry nothing applied to is a different statement and still gets one. The job also stops commenting on a pull request that moves nothing. hasDelta is a pure function of the two reports, tested rather than written as shell logic, and it is true when there is no base, when the global moved, when an entry was added, removed, rescored or newly unmeasured, when a check fails at head that did not at base, and when the parse-failure or unknown-suppression warnings changed. With no delta and a comment already on the pull request, prCommentCli renders a short resolved state instead of going quiet and leaving findings that no longer exist standing. --- .../observability-map/src/report/prComment.ts | 121 +++++++++++-- .../src/report/prCommentCli.ts | 44 ++++- .../observability-map/test/prComment.test.ts | 171 +++++++++++++++++- .../test/prCommentCli.test.ts | 62 ++++++- 4 files changed, 376 insertions(+), 22 deletions(-) diff --git a/internal-packages/observability-map/src/report/prComment.ts b/internal-packages/observability-map/src/report/prComment.ts index 8e857b8c6d9..80c0b4b70fe 100644 --- a/internal-packages/observability-map/src/report/prComment.ts +++ b/internal-packages/observability-map/src/report/prComment.ts @@ -1,6 +1,12 @@ import type { MapReport, ScoredEntry } from "../score.js"; import { SCORED_CHECK_IDS } from "../checks/index.js"; -import { auditLine, contextLine, contextOnly, scoredFailures } from "./terminal.js"; +import { + auditLine, + contextLine, + contextOnly, + scoredFailures, + unknownSuppressionLines, +} from "./terminal.js"; /** First line of every comment this job posts, so the upsert step can find its own comment again. */ export const MARKER = ""; @@ -32,17 +38,30 @@ function scoreLine(head: MapReport, base: MapReport | null): string { return `${headline} ${comparison}`; } +/** What goes in a score column: a figure, an absence, or no prior entry to compare against. */ +const NOT_MEASURED = "not measured"; + +/** + * `score` is 100 for an entry no scored check applied to, a placeholder the score itself excludes + * from every mean. Rendering that 100 as a figure turned a route refactored down to a trivial body + * into a 67-point improvement, and a trivial route gaining real work into the PR's worst + * regression. So the cell says what the terminal gauge says for a null mean instead. + */ +const scoreCell = (e: ScoredEntry): number | string => (e.measured ? e.score : NOT_MEASURED); + type ChangedRow = { routePath: string; sensitive: boolean; - baseScore: number | "new"; - headScore: number; + baseScore: number | string; + headScore: number | string; nowFailing: string[]; /** * How much the entry got worse, used to sort the table. A new entry has no base score to * subtract from, so it is scored against a perfect 100: a new entry landing at 60 sorts the * same as an existing one that dropped 40 points, which is the ordering "what needs fixing - * first" implies. + * first" implies. Zero whenever either side is unmeasured, because there is no arithmetic to do + * between a figure and an absence; such a row is in the table to disclose the transition, not to + * claim a size for it. */ drop: number; }; @@ -55,25 +74,31 @@ function changedRows(head: MapReport, base: MapReport): { rows: ChangedRow[]; re for (const h of head.entries) { const b = baseByFile.get(h.fileName); if (!b) { + // A new entry that passes every check it was measured against has nothing to fix, which is + // what `drop: 0` means everywhere else in this table. A new entry nothing applied to is a + // different statement and still gets a row, since its 100 is a placeholder rather than a pass. + if (h.measured && h.score === 100) continue; rows.push({ routePath: h.routePath, sensitive: h.sensitive, baseScore: "new", - headScore: h.score, + headScore: scoreCell(h), nowFailing: failingIds(h), - drop: 100 - h.score, + drop: h.measured ? 100 - h.score : 0, }); continue; } - if (b.score === h.score) continue; + // Measured state is part of what changed: a measured-to-unmeasured transition can leave the + // score untouched at its placeholder value, and skipping on the number alone hid it. + if (b.measured === h.measured && b.score === h.score) continue; const baseFailing = new Set(failingIds(b)); rows.push({ routePath: h.routePath, sensitive: h.sensitive, - baseScore: b.score, - headScore: h.score, + baseScore: scoreCell(b), + headScore: scoreCell(h), nowFailing: failingIds(h).filter((id) => !baseFailing.has(id)), - drop: b.score - h.score, + drop: b.measured && h.measured ? b.score - h.score : 0, }); } @@ -148,6 +173,78 @@ function fixFirstSection(head: MapReport): string[] { return lines; } +/** + * Whether this pull request moves the report at all, so the job can stay quiet when it does not. + * + * True on: no base to compare against, a global score change, an entry added or removed, an + * entry's score or measured state changed, a check that fails at head and did not at base, a + * change in the parse failure count, or a change in the unknown suppression warnings. The last two + * are here because both render a line in the comment, so a run that gains one has moved the report + * even though no score did. + * + * The residual: adding a suppression whose check was already failing moves nothing this asks + * about. The score is capped at the pre-suppression ratio so it cannot move, and the finding + * leaves the fix list quietly. `SUPPRESSED` in the terminal report is where that shows up. + */ +export function hasDelta(head: MapReport, base: MapReport | null): boolean { + if (!base) return true; + if (head.global !== base.global) return true; + if (head.parseFailures.length !== base.parseFailures.length) return true; + if (JSON.stringify(head.unknownSuppressions) !== JSON.stringify(base.unknownSuppressions)) { + return true; + } + + const baseByFile = new Map(base.entries.map((e) => [e.fileName, e])); + const headFiles = new Set(head.entries.map((e) => e.fileName)); + if (base.entries.some((e) => !headFiles.has(e.fileName))) return true; + + for (const h of head.entries) { + const b = baseByFile.get(h.fileName); + if (!b) return true; + if (b.measured !== h.measured || b.score !== h.score) return true; + const baseFailing = new Set(b.checks.filter((c) => c.status === "fail").map((c) => c.id)); + if (h.checks.some((c) => c.status === "fail" && !baseFailing.has(c.id))) return true; + } + return false; +} + +/** + * What replaces a comment whose findings a later push fixed. Going silent would leave the earlier + * comment standing with findings that no longer exist, which is worse than a redundant comment. + */ +export function renderResolvedComment(): string { + return [ + MARKER, + "", + "## Observability map", + "", + "Nothing in this pull request moves the report any more. The findings an earlier push " + + "reported are gone.", + "", + "Report only, nothing here gates the merge. The rules and their reasons: " + + "internal-packages/observability-map/README.md.", + ].join("\n"); +} + +/** + * What the job posts when the head scan did not produce a report. The alternative was a red x on + * a job that must never block a pull request, and the alternative to that was swallowing the + * failure so the only signal was a comment that never appeared. + */ +export function renderScanFailedComment(): string { + return [ + MARKER, + "", + "## Observability map", + "", + "The scan failed for this run, so there is no report. Anything above is from an earlier push " + + "and is stale. The workflow log has the error.", + "", + "Report only, nothing here gates the merge. The rules and their reasons: " + + "internal-packages/observability-map/README.md.", + ].join("\n"); +} + /** * Pure function, no I/O: `head` and `base` are already-built reports. Matches entries across the * two by `fileName`, the same identifier `renderJson` carries. @@ -162,7 +259,9 @@ export function renderPrComment(head: MapReport, base: MapReport | null): string if (audit) lines.push(audit); const context = contextLine(head); if (context) lines.push(context); - if (audit || context) lines.push(""); + const unknown = unknownSuppressionLines(head); + lines.push(...unknown); + if (audit || context || unknown.length > 0) lines.push(""); lines.push( "Report only, nothing here gates the merge. The rules and their reasons: " + diff --git a/internal-packages/observability-map/src/report/prCommentCli.ts b/internal-packages/observability-map/src/report/prCommentCli.ts index 0283eb96591..a2aef9a3477 100644 --- a/internal-packages/observability-map/src/report/prCommentCli.ts +++ b/internal-packages/observability-map/src/report/prCommentCli.ts @@ -2,7 +2,12 @@ import { readFileSync } from "node:fs"; import { resolve } from "node:path"; import { fileURLToPath } from "node:url"; import type { MapReport } from "../score.js"; -import { renderPrComment } from "./prComment.js"; +import { + hasDelta, + renderPrComment, + renderResolvedComment, + renderScanFailedComment, +} from "./prComment.js"; /** Where output goes. Injectable so tests can read it without spawning a process. */ export type Io = { out: (s: string) => void; err: (s: string) => void }; @@ -28,15 +33,33 @@ function readReport(path: string, label: string): MapReport { } } -/** `-` or a missing second arg means no base: the CI job falls back to this when the base scan - * itself failed, so the comment still renders rather than the job going red. */ +/** + * `-` or a missing second arg means no base: the CI job falls back to this when the base scan + * itself failed, so the comment still renders rather than the job going red. + * + * Empty output means "post nothing". The job only comments when the pull request moves the report, + * and `--existing-comment` is how the workflow says a comment from an earlier push is already on + * the pull request: with the delta gone, that comment is replaced with a resolved state rather + * than left standing with findings that no longer exist. + * + * `--scan-failed` takes no report and prints the stale-report comment, for the case where the head + * scan produced nothing to read. + */ export function main(argv: string[], io: Io = processIo): number { const args = argv.slice(2); - const headPath = args[0]; - const basePath = args[1]; + const scanFailed = args.includes("--scan-failed"); + const existingComment = args.includes("--existing-comment"); + const positional = args.filter((a) => !a.startsWith("--")); + const headPath = positional[0]; + const basePath = positional[1]; + + if (scanFailed) { + io.out(`${renderScanFailedComment()}\n`); + return 0; + } if (!headPath) { - io.err("usage: prCommentCli.ts [base.json|-]\n"); + io.err("usage: prCommentCli.ts [base.json|-] [--existing-comment]\n"); return 1; } @@ -50,8 +73,13 @@ export function main(argv: string[], io: Io = processIo): number { return 1; } - io.out(renderPrComment(head, base)); - io.out("\n"); + if (hasDelta(head, base)) { + io.out(`${renderPrComment(head, base)}\n`); + return 0; + } + if (existingComment) { + io.out(`${renderResolvedComment()}\n`); + } return 0; } diff --git a/internal-packages/observability-map/test/prComment.test.ts b/internal-packages/observability-map/test/prComment.test.ts index 7f651d57dc2..4299b1ae3a7 100644 --- a/internal-packages/observability-map/test/prComment.test.ts +++ b/internal-packages/observability-map/test/prComment.test.ts @@ -1,4 +1,4 @@ -import { renderPrComment } from "../src/report/prComment.js"; +import { hasDelta, renderPrComment } from "../src/report/prComment.js"; import { buildReport } from "../src/score.js"; import { scanFile } from "../src/scan.js"; @@ -96,6 +96,94 @@ describe("renderPrComment", () => { expect(row).toMatch(/error-classification|auth-boundary|request-context/); }); + it("skips a new entry that passes every check it was measured against", () => { + const head = buildReport( + [scanFile("api.v1.auth.tokens.ts", cleanSource)!, scanFile("api.v1.new.ts", cleanSource)!], + [] + ); + const base = buildReport([scanFile("api.v1.auth.tokens.ts", cleanSource)!], []); + const out = renderPrComment(head, base); + + expect(out).not.toContain("/api/v1/new"); + expect(out).toContain("No entry point this PR touches changed its score."); + }); + + // B3. `score` is 100 for an entry no scored check applied to, and the table read that placeholder + // as a figure: a route refactored down to a trivial body rendered as a 67-point improvement. + describe("an unmeasured entry", () => { + const trivial = `export const loader = () => new Response("ok");`; + + it("renders as not measured rather than as 100 when the head stopped being measurable", () => { + const head = buildReport([scanFile("api.v1.auth.tokens.ts", trivial)!], []); + const base = buildReport([scanFile("api.v1.auth.tokens.ts", brokenSource)!], []); + const out = renderPrComment(head, base); + + const row = out.split("\n").find((l) => l.startsWith("| /api/v1/auth/tokens |"))!; + expect(row).toBeDefined(); + expect(row).toContain("not measured"); + expect(row).not.toMatch(/\|\s*100\s*\|/); + }); + + it("renders as not measured in the base column when the head gained real work", () => { + const head = buildReport([scanFile("api.v1.auth.tokens.ts", brokenSource)!], []); + const base = buildReport([scanFile("api.v1.auth.tokens.ts", trivial)!], []); + const out = renderPrComment(head, base); + + const row = out.split("\n").find((l) => l.startsWith("| /api/v1/auth/tokens |"))!; + expect(row).toBeDefined(); + expect(row).toMatch(/\| not measured \| \d+ \|/); + }); + + // The early-out compared scores only, so a measured 100 turning into an unmeasured placeholder + // 100 produced no row at all: the table said nothing happened. + it("still produces a row when a measured 100 becomes an unmeasured placeholder 100", () => { + const head = buildReport([scanFile("api.v1.auth.tokens.ts", trivial)!], []); + const base = buildReport([scanFile("api.v1.auth.tokens.ts", cleanSource)!], []); + expect(base.entries[0]!.score).toBe(100); + expect(head.entries[0]!.score).toBe(100); + + const out = renderPrComment(head, base); + const row = out.split("\n").find((l) => l.startsWith("| /api/v1/auth/tokens |"))!; + expect(row).toBeDefined(); + expect(row).toMatch(/\| 100 \| not measured \|/); + }); + + // Not the same statement as a new entry that passes everything, which is skipped above. + it("still gets a row when it is new, since its 100 is a placeholder and not a pass", () => { + const head = buildReport( + [ + scanFile("api.v1.auth.tokens.ts", cleanSource)!, + scanFile("resources.health.ts", trivial)!, + ], + [] + ); + const base = buildReport([scanFile("api.v1.auth.tokens.ts", cleanSource)!], []); + const out = renderPrComment(head, base); + + const row = out.split("\n").find((l) => l.startsWith("| /resources/health |"))!; + expect(row).toBeDefined(); + expect(row).toMatch(/\| new \| not measured \|/); + }); + + it("does not sort an unmeasured transition above a real regression", () => { + const head = buildReport( + [scanFile("resources.gone.ts", trivial)!, scanFile("resources.busy.ts", brokenSource)!], + [] + ); + const base = buildReport( + [scanFile("resources.gone.ts", brokenSource)!, scanFile("resources.busy.ts", cleanSource)!], + [] + ); + const out = renderPrComment(head, base); + + const busy = out.indexOf("/resources/busy"); + const gone = out.indexOf("/resources/gone"); + expect(busy).toBeGreaterThan(-1); + expect(gone).toBeGreaterThan(-1); + expect(busy).toBeLessThan(gone); + }); + }); + it("sorts a sensitive entry with a small drop above a non-sensitive entry with a large drop", () => { const sensitiveSmallDropBase = scanFile("api.v1.auth.tokens.ts", cleanSource)!; const sensitiveSmallDropHead = scanFile( @@ -185,3 +273,84 @@ describe("renderPrComment", () => { expect(out).toContain("internal-packages/observability-map/README.md"); }); }); + +// B4. The job posts only when the pull request moves the report, so the decision has to be a +// tested function of the two reports rather than shell logic in the workflow. +describe("hasDelta", () => { + const trivial = `export const loader = () => new Response("ok");`; + const one = (name: string, source: string) => buildReport([scanFile(name, source)!], []); + + it("is true when there is no base to compare against", () => { + expect(hasDelta(one("api.v1.a.ts", cleanSource), null)).toBe(true); + }); + + it("is false for two identical reports", () => { + expect(hasDelta(one("api.v1.a.ts", cleanSource), one("api.v1.a.ts", cleanSource))).toBe(false); + }); + + it("is true when the global score moved", () => { + expect(hasDelta(one("api.v1.a.ts", brokenSource), one("api.v1.a.ts", cleanSource))).toBe(true); + }); + + it("is true when an entry was added", () => { + const head = buildReport( + [scanFile("api.v1.a.ts", cleanSource)!, scanFile("api.v1.b.ts", cleanSource)!], + [] + ); + expect(hasDelta(head, one("api.v1.a.ts", cleanSource))).toBe(true); + }); + + it("is true when an entry was removed", () => { + const base = buildReport( + [scanFile("api.v1.a.ts", cleanSource)!, scanFile("api.v1.b.ts", cleanSource)!], + [] + ); + expect(hasDelta(one("api.v1.a.ts", cleanSource), base)).toBe(true); + }); + + // The global is a mean over measured entries, so two entries moving in opposite directions can + // leave it where it was. The per-entry comparison is what catches that. + it("is true when an entry's score moved but the global mean did not", () => { + const head = buildReport( + [scanFile("api.v1.a.ts", brokenSource)!, scanFile("api.v1.b.ts", cleanSource)!], + [] + ); + const base = buildReport( + [scanFile("api.v1.a.ts", cleanSource)!, scanFile("api.v1.b.ts", brokenSource)!], + [] + ); + expect(head.global).toBe(base.global); + expect(hasDelta(head, base)).toBe(true); + }); + + // audit-trail does not feed the score, so it can start failing without moving a single figure. + it("is true when an unscored check started failing and no score moved", () => { + const head = one("api.v1.envvars.ts", cleanSource); + const base = one( + "api.v1.envvars.ts", + `// obs-map-disable audit-trail -- no helper exists yet\n${cleanSource}` + ); + expect(head.global).toBe(base.global); + expect(hasDelta(head, base)).toBe(true); + }); + + it("is true when an entry stopped being measured at the same placeholder score", () => { + const head = one("api.v1.a.ts", trivial); + const base = one("api.v1.a.ts", cleanSource); + expect(head.entries[0]!.score).toBe(base.entries[0]!.score); + expect(hasDelta(head, base)).toBe(true); + }); + + it("is true when a parse failure appeared, since the comment warns about it", () => { + const head = buildReport([scanFile("api.v1.a.ts", cleanSource)!], ["broken.ts"]); + expect(hasDelta(head, one("api.v1.a.ts", cleanSource))).toBe(true); + }); + + it("is true when a suppression names a check that does not exist", () => { + const head = one( + "api.v1.a.ts", + `// obs-map-disable eror-classification -- typo\n${cleanSource}` + ); + expect(hasDelta(head, one("api.v1.a.ts", cleanSource))).toBe(true); + }); +}); diff --git a/internal-packages/observability-map/test/prCommentCli.test.ts b/internal-packages/observability-map/test/prCommentCli.test.ts index b464725ea77..2b171f676db 100644 --- a/internal-packages/observability-map/test/prCommentCli.test.ts +++ b/internal-packages/observability-map/test/prCommentCli.test.ts @@ -19,15 +19,31 @@ const run = (...args: string[]) => { return { code, out: c.out(), err: c.err() }; }; +const swallows = `import { prisma } from "~/db.server"; + export async function action() { + try { return await prisma.token.create({ data: {} }); } catch (e) { return null; } + }`; + +const handles = `import { requireUserId } from "~/services/session.server"; + import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function action({ request }) { + const userId = await requireUserId(request); + try { return await prisma.token.create({ data: { userId } }); } + catch (error) { logger.error("token create failed", { userId, error }); throw error; } + }`; + describe("prCommentCli", () => { const dir = mkdtempSync(join(tmpdir(), "obs-map-cli-")); const headPath = join(dir, "head.json"); const basePath = join(dir, "base.json"); + const unchangedPath = join(dir, "unchanged.json"); const source = `export const loader = () => new Response("ok");`; const report = buildReport([scanFile("resources.a.ts", source)!], []); - writeFileSync(headPath, renderJson(report)); - writeFileSync(basePath, renderJson(report)); + writeFileSync(headPath, renderJson(buildReport([scanFile("api.v1.t.ts", swallows)!], []))); + writeFileSync(basePath, renderJson(buildReport([scanFile("api.v1.t.ts", handles)!], []))); + writeFileSync(unchangedPath, renderJson(report)); afterAll(() => rmSync(dir, { recursive: true })); @@ -37,6 +53,40 @@ describe("prCommentCli", () => { expect(r.out.split("\n")[0]).toBe(""); }); + // B4. The job used to comment on every push, including one that moved nothing. + it("posts nothing when the report did not move and no comment exists yet", () => { + const r = run(unchangedPath, unchangedPath); + expect(r.code).toBe(0); + expect(r.out).toBe(""); + }); + + it("replaces an existing comment with a resolved state when the delta has gone", () => { + const r = run(unchangedPath, unchangedPath, "--existing-comment"); + expect(r.code).toBe(0); + expect(r.out.split("\n")[0]).toBe(""); + expect(r.out).toContain("Nothing in this pull request moves the report any more."); + expect(r.out).not.toContain("FIX FIRST"); + }); + + it("posts the full comment when there is a delta, existing comment or not", () => { + for (const args of [ + [headPath, basePath], + [headPath, basePath, "--existing-comment"], + ]) { + const r = run(...args); + expect(r.code).toBe(0); + expect(r.out).toContain("FIX FIRST"); + expect(r.out).not.toContain("moves the report any more"); + } + }); + + it("prints the stale-report comment for --scan-failed without reading any file", () => { + const r = run("--scan-failed"); + expect(r.code).toBe(0); + expect(r.out.split("\n")[0]).toBe(""); + expect(r.out).toContain("The scan failed for this run"); + }); + it("treats '-' as no base", () => { const r = run(headPath, "-"); expect(r.code).toBe(0); @@ -49,6 +99,14 @@ describe("prCommentCli", () => { expect(r.out).toContain("Base comparison unavailable."); }); + // Without a base there is no delta to compute, so silence would be a guess. Posting is the + // honest answer even for a report identical to one nobody can see. + it("posts even for an unmoved report when the base is unavailable", () => { + const r = run(unchangedPath, "-"); + expect(r.code).toBe(0); + expect(r.out).toContain("Base comparison unavailable."); + }); + it("exits 1 with a usage message when head.json is missing", () => { const r = run(); expect(r.code).toBe(1); From 5ed954495d40df7eb309bd51b47d19930e5359ee Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Sat, 1 Aug 2026 22:24:04 +0100 Subject: [PATCH 065/106] ci(observability-map): stop the head scan reddening a pull request, and say when it failed The scan head step had no guard while render and upsert did, under a comment saying the job must never block a pull request. Swallowing it silently is not the answer either, because then nobody learns the report is stale. It is now guarded so it cannot fail the job, and a missing head report makes the render step post a comment saying the scan failed and the report is stale for this run. The head report is written to a temp path and moved on success, so a failed scan leaves no stale or partial file behind. The render step clears comment.md first and writes through a temp path too, so a render failure cannot leave the upsert posting a file that was never written, and it falls back to the same stale-report comment when the render itself fails on a malformed report. The upsert step posts nothing when comment.md is empty, which is how prCommentCli says the pull request does not move the report. The comment lookup moves ahead of the render step, because the render decision needs to know whether a comment already exists, and the upsert reuses the id rather than asking twice. A failed lookup is treated as "a comment exists": that can post a redundant resolved state, where the other guess leaves a stale comment standing. Also moves pull-requests: write from the workflow onto the one job that comments, which is what zizmor asks for. --- .github/workflows/observability-map.yml | 80 +++++++++++++++++++++---- 1 file changed, 70 insertions(+), 10 deletions(-) diff --git a/.github/workflows/observability-map.yml b/.github/workflows/observability-map.yml index 0406f1c3730..4d30b29cc5b 100644 --- a/.github/workflows/observability-map.yml +++ b/.github/workflows/observability-map.yml @@ -13,7 +13,6 @@ concurrency: permissions: contents: read - pull-requests: write jobs: # The tree-scale mutation corpus: every known laundering shape applied to the whole route tree, @@ -53,6 +52,10 @@ jobs: report: runs-on: warp-ubuntu-latest-x64-4x + # Only this job comments, so only this job gets the write. + permissions: + contents: read + pull-requests: write # Fork PRs get a read-only token, so the comment cannot post. Skipping the job beats a red x. if: github.event.pull_request.head.repo.full_name == github.repository steps: @@ -76,9 +79,18 @@ jobs: - name: 📥 Download deps run: pnpm install --frozen-lockfile + # Guarded rather than allowed to fail: this job must never block a pull request. The failure + # is not swallowed either, the render step below turns a missing head report into a comment + # saying so, because a swallowed failure with no comment is the outcome nobody wants. - name: 🔎 Scan head run: | - pnpm --filter @internal/observability-map exec tsx src/cli.ts --json --no-write > /tmp/head.json + if pnpm --filter @internal/observability-map exec tsx src/cli.ts --json --no-write \ + > /tmp/head.json.partial; then + mv /tmp/head.json.partial /tmp/head.json + else + rm -f /tmp/head.json /tmp/head.json.partial + echo "head scan failed; the comment will say the report is stale for this run" >&2 + fi - name: 🔎 Scan base with the head's scanner run: | @@ -91,16 +103,58 @@ jobs: echo "base scan failed or the worktree could not be added; falling back to no base" >&2 fi - # continue-on-error: this job must never block a PR. A malformed head.json or a rendering - # bug would otherwise turn the job red the same way a base-scan failure would not, since that - # step already falls back to "-" instead of failing. + # Looked up before the render step because the render step needs it: with no delta to report, + # a pull request that already has a comment gets a resolved state rather than being left with + # findings that no longer exist, and one that does not gets nothing at all. The upsert step + # reuses the id rather than asking again. + - name: 🔍 Look for a comment from an earlier push + continue-on-error: true + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} + run: | + rm -f /tmp/existing-comment-id + if gh api "repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/comments" --paginate \ + --jq '[.[] | select(.body | startswith(""))][0].id // empty' \ + > /tmp/existing-comment-id.partial; then + mv /tmp/existing-comment-id.partial /tmp/existing-comment-id + else + rm -f /tmp/existing-comment-id.partial + echo "comment lookup failed; assuming a comment exists so a stale one still gets replaced" >&2 + fi + + # continue-on-error for the same reason as the scan: a rendering bug must not turn the job + # red. An empty /tmp/comment.md means there is nothing to post, which is a decision + # prCommentCli makes, not this shell. - name: 📝 Render comment continue-on-error: true run: | - if [ "$(cat /tmp/base.json)" = "-" ]; then - pnpm --filter @internal/observability-map exec tsx src/report/prCommentCli.ts /tmp/head.json - > /tmp/comment.md + rm -f /tmp/comment.md + render() { pnpm --filter @internal/observability-map exec tsx src/report/prCommentCli.ts "$@"; } + + if [ ! -s /tmp/head.json ]; then + render --scan-failed > /tmp/comment.md + exit 0 + fi + + base=/tmp/base.json + if [ ! -s /tmp/base.json ] || [ "$(cat /tmp/base.json)" = "-" ]; then + base="-" + fi + + # An absent file means the lookup itself failed, and assuming there is a comment is the + # safer half of that guess: it can post a redundant resolved state, not leave a stale one. + flags=() + if [ ! -f /tmp/existing-comment-id ] || [ -s /tmp/existing-comment-id ]; then + flags=(--existing-comment) + fi + + if render /tmp/head.json "$base" "${flags[@]}" > /tmp/comment.md.partial; then + mv /tmp/comment.md.partial /tmp/comment.md else - pnpm --filter @internal/observability-map exec tsx src/report/prCommentCli.ts /tmp/head.json /tmp/base.json > /tmp/comment.md + rm -f /tmp/comment.md.partial + echo "render failed; falling back to the stale-report comment" >&2 + render --scan-failed > /tmp/comment.md fi # continue-on-error for the same reason: a transient gh api failure (rate limit, network) @@ -111,8 +165,14 @@ jobs: GH_TOKEN: ${{ github.token }} PR_NUMBER: ${{ github.event.pull_request.number }} run: | - existing=$(gh api "repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/comments" --paginate \ - --jq '[.[] | select(.body | startswith(""))][0].id // empty') + if [ ! -s /tmp/comment.md ]; then + echo "nothing to post: this pull request does not move the report" + exit 0 + fi + existing="" + if [ -f /tmp/existing-comment-id ]; then + existing=$(cat /tmp/existing-comment-id) + fi if [ -n "$existing" ]; then gh api -X PATCH "repos/${GITHUB_REPOSITORY}/issues/comments/${existing}" -F body=@/tmp/comment.md else From 4d39fb4c735912b48fb02a2dcc358b36a81fb97b Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Sat, 1 Aug 2026 22:24:20 +0100 Subject: [PATCH 066/106] feat(observability-map): add --out, honour --json for one route, and declare the module type `map /api/v1/token --json` printed the text format and dropped the flag. It now prints the scored entry as json, with the unknown-suppression warning on stderr in both formats so it cannot land inside json a caller parses. `--out` names where the whole-tree report is written, defaulting to the repo root as before. It exists so a test can point the write at a temp directory: the only way to exercise the write path before was to let the tests delete and recreate observability-map.json in the monorepo root, which destroyed a developer's generated artifact on every run. package.json gains "type": "module", matching the sibling internal packages. The build emits ESM and cli.ts uses import.meta, while main and types advertised that to a manifest with no module type: node 24 loads it with a MODULE_TYPELESS_PACKAGE_JSON warning and a reparse rather than the throw an older node gives. The alternative, dropping main, types and build to be honest that this is a tool, also passes the suite; the convention the other internal packages already follow won. --- .../observability-map/package.json | 1 + .../observability-map/src/cli.ts | 37 ++++++++++++++++--- 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/internal-packages/observability-map/package.json b/internal-packages/observability-map/package.json index 776ec458963..cd8c3fc14f4 100644 --- a/internal-packages/observability-map/package.json +++ b/internal-packages/observability-map/package.json @@ -2,6 +2,7 @@ "name": "@internal/observability-map", "private": true, "version": "0.0.1", + "type": "module", "main": "./dist/src/index.js", "types": "./dist/src/index.d.ts", "dependencies": { diff --git a/internal-packages/observability-map/src/cli.ts b/internal-packages/observability-map/src/cli.ts index cb9c9b6017a..6ef722d9afd 100644 --- a/internal-packages/observability-map/src/cli.ts +++ b/internal-packages/observability-map/src/cli.ts @@ -5,7 +5,11 @@ import type { EntryPoint } from "./types.js"; import { scanDirectory } from "./scan.js"; import { buildReport, scoreEntry } from "./score.js"; import { routePathOf } from "./adapters/remix.js"; -import { renderTerminal } from "./report/terminal.js"; +import { + renderTerminal, + unknownSuppressionLine, + unknownSuppressionLines, +} from "./report/terminal.js"; import { renderJson } from "./report/json.js"; const DEFAULT_ROUTES = "apps/webapp/app/routes"; @@ -56,6 +60,13 @@ function findMatches(entryPoints: EntryPoint[], target: string): EntryPoint[] { ); } +/** Value of a `--flag=value` argument, or null when the flag is absent. */ +function flagValue(args: string[], flag: string): string | null { + const prefix = `${flag}=`; + const found = args.find((a) => a.startsWith(prefix)); + return found === undefined ? null : found.slice(prefix.length); +} + export function main(argv: string[], io: Io = processIo): number { const args = argv.slice(2); const asJson = args.includes("--json"); @@ -63,11 +74,11 @@ export function main(argv: string[], io: Io = processIo): number { const target = args.find((a) => !a.startsWith("--")); const repoRoot = findRepoRoot(dirname(fileURLToPath(import.meta.url))); - const routesFlag = args.find((a) => a.startsWith("--routes=")); + const routesFlag = flagValue(args, "--routes"); let routesDir: string; - if (routesFlag) { - routesDir = resolve(process.cwd(), routesFlag.slice("--routes=".length)); + if (routesFlag !== null) { + routesDir = resolve(process.cwd(), routesFlag); let isDir = false; try { isDir = statSync(routesDir).isDirectory(); @@ -99,6 +110,14 @@ export function main(argv: string[], io: Io = processIo): number { ); } const scored = scoreEntry(matches[0]!); + // On stderr in both formats: a warning on stdout would be inside the JSON a caller parses. + if (scored.unknownSuppressions.length > 0) { + io.err(`${unknownSuppressionLine(scored.fileName, scored.unknownSuppressions)}\n`); + } + if (asJson) { + io.out(`${JSON.stringify(scored, null, 2)}\n`); + return 0; + } const measuredNote = scored.measured ? "" : " (not measured: no applicable checks)"; io.out( `${scored.routePath} ${scored.score}/100${measuredNote}\n${scored.fileName}\n\nCHECKS\n` @@ -111,10 +130,18 @@ export function main(argv: string[], io: Io = processIo): number { } const report = buildReport(entryPoints, parseFailures); + for (const line of unknownSuppressionLines(report)) io.err(`${line}\n`); io.out(asJson ? renderJson(report) : renderTerminal(report)); io.out("\n"); if (!noWrite) { - writeFileSync(resolve(repoRoot, "observability-map.json"), renderJson(report)); + // `--out` exists so a test can point the write somewhere disposable. Without it the only way + // to exercise the write path was to let the tests create and delete a file in the repo root. + const outFlag = flagValue(args, "--out"); + const outPath = + outFlag === null + ? resolve(repoRoot, "observability-map.json") + : resolve(process.cwd(), outFlag); + writeFileSync(outPath, renderJson(report)); } return 0; } From 6c54ec7465a65c782c8d41adbd4d92456f5ac439 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Sat, 1 Aug 2026 22:24:38 +0100 Subject: [PATCH 067/106] test(observability-map): give the cli tests their own routes tree, and watch the live one in ci cli.test.ts asserted that /api/v1/token exists, that api.v1.runs.ts is line 2 of the output, and that /admin/api/v1/runs-replication matches several entries. Those are assertions about the webapp's contents, not about this package. Meanwhile the internal filter in pr_checks.yml did not cover apps/webapp, so a webapp-only pull request that renamed a route never ran this suite, merged green, and broke main or the next unrelated internal-packages pull request. Verified both halves: with api.v1.token.ts renamed away the old tests fail two cases and the new ones pass all fifteen. cli.test.ts now builds its own routes directory and points the cli at it with --routes. The one deliberate real-tree test stays in integration.test.ts and asserts only what survives churn: the scan does not crash, the count sits in a wide band, parse failures are zero. pr_checks.yml's internal filter gains apps/webapp/app/routes so that test cannot go stale unnoticed. countCandidates was a verbatim copy of the scanner's own walk, so the bound it guarded could not fail for any route shape both of them missed. It is a plain recursive readdir now, with a test proving the two disagree. Both real-tree tests opened with `if (!existsSync(ROUTES)) return;`, which turned them into green no-ops if this package ever moved relative to apps/webapp; a missing routes directory is a hard failure now. Adds the packaging invariant the module-type fix needs to stay fixed. --- .github/workflows/pr_checks.yml | 5 + .../observability-map/test/cli.test.ts | 149 +++++++++++++++--- .../test/integration.test.ts | 97 +++++++++--- 3 files changed, 211 insertions(+), 40 deletions(-) diff --git a/.github/workflows/pr_checks.yml b/.github/workflows/pr_checks.yml index b0fbd6ac040..89b5dc8c29b 100644 --- a/.github/workflows/pr_checks.yml +++ b/.github/workflows/pr_checks.yml @@ -74,6 +74,11 @@ jobs: internal: - 'internal-packages/**' - 'packages/**' + # @internal/observability-map has one deliberate test against the live route tree + # (internal-packages/observability-map/test/integration.test.ts). Without this the + # internal suite never ran for a webapp-only PR, so a renamed route merged green and + # broke main, or the next unrelated internal-packages PR. + - 'apps/webapp/app/routes/**' - '.github/workflows/pr_checks.yml' - '.github/workflows/unit-tests-internal.yml' - '.configs/**' diff --git a/internal-packages/observability-map/test/cli.test.ts b/internal-packages/observability-map/test/cli.test.ts index 2fd28955d85..3feffc08eb3 100644 --- a/internal-packages/observability-map/test/cli.test.ts +++ b/internal-packages/observability-map/test/cli.test.ts @@ -1,9 +1,52 @@ -import { existsSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; -import { join, resolve } from "node:path"; +import { join } from "node:path"; import { main, type Io } from "../src/cli.js"; -const REPORT_FILE = resolve(__dirname, "../../../observability-map.json"); +/** + * A routes tree of this package's own making. These tests used to run against + * `apps/webapp/app/routes` and assert that `/api/v1/token` exists and that `api.v1.runs.ts` is line + * 2 of the output, which is an assertion about the webapp's contents rather than about this CLI. + * A webapp-only pull request renaming a route broke them, and `pr_checks.yml` did not run this + * suite for such a pull request, so the break landed on whoever pushed next. The one deliberate + * real-tree test lives in `integration.test.ts` and asserts only invariants that survive churn. + */ +const ROUTES = mkdtempSync(join(tmpdir(), "obs-map-fixture-routes-")); + +const withWork = (name: string) => `import { prisma } from "~/db.server"; + export async function loader() { + try { return await prisma.${name}.findMany(); } catch (e) { return null; } + }`; + +const FIXTURES: Record = { + // Exact target, and also the prefix of the two below it. + "api.v1.runs.ts": withWork("run"), + "api.v1.runs.$runId.ts": withWork("run"), + "api.v1.runs.$runId.cancel.ts": withWork("run"), + "api.v1.token.ts": withWork("token"), + // Five routes sharing a prefix that is not itself a route, for the ambiguity warning and for the + // "and N more" tail it grows past four matches. + "admin.api.v1.runs-replication.start.ts": withWork("replication"), + "admin.api.v1.runs-replication.stop.ts": withWork("replication"), + "admin.api.v1.runs-replication.status.ts": withWork("replication"), + "admin.api.v1.runs-replication.retry.ts": withWork("replication"), + "admin.api.v1.runs-replication.purge.ts": withWork("replication"), + // Nothing applicable: exercises the not-measured note. + "resources.health.ts": `export const loader = () => new Response("ok");`, + // A directive whose id names no check, for the warning it has to produce. + "api.v1.typo.ts": `// obs-map-disable eror-classification -- typo\n${withWork("thing")}`, +}; + +beforeAll(() => { + for (const [name, source] of Object.entries(FIXTURES)) { + writeFileSync(join(ROUTES, name), source); + } + // A directory route, so the fixture covers both shapes `scanDirectory` walks. + mkdirSync(join(ROUTES, "_app.orgs.$slug")); + writeFileSync(join(ROUTES, "_app.orgs.$slug", "route.tsx"), withWork("organization")); +}); + +afterAll(() => rmSync(ROUTES, { recursive: true, force: true })); const capture = () => { const out: string[] = []; @@ -14,7 +57,7 @@ const capture = () => { const run = (...args: string[]) => { const c = capture(); - const code = main(["node", "cli.js", ...args], c.io); + const code = main(["node", "cli.js", `--routes=${ROUTES}`, ...args], c.io); return { code, out: c.out(), err: c.err() }; }; @@ -34,6 +77,12 @@ describe("map ", () => { expect(r.out).toContain("api.v1.token.ts"); }); + it("accepts a directory route by the path its directory segment spells", () => { + const r = run("/_app/orgs/:slug"); + expect(r.code).toBe(0); + expect(r.out).toContain("_app.orgs.$slug/route.tsx"); + }); + it("exits 1 with a message when nothing matches", () => { const r = run("/api/v1/does-not-exist"); expect(r.code).toBe(1); @@ -43,31 +92,89 @@ describe("map ", () => { it("warns when a prefix matches more than one route rather than silently taking the first", () => { const r = run("/admin/api/v1/runs-replication"); expect(r.code).toBe(0); - expect(r.err).toMatch(/matches \d+ entry points, showing the first/); + expect(r.err).toMatch(/matches 5 entry points, showing the first/); expect(r.err).toContain("Others:"); + expect(r.err).toContain("and 1 more"); }); - // `/api/v1/runs` is a prefix of a dozen others, and also a route in its own right. + // `/api/v1/runs` is a prefix of two others in the fixture, and also a route in its own right. it("prefers an exact match over the routes it is a prefix of", () => { const r = run("/api/v1/runs"); expect(r.err).toBe(""); expect(r.out.split("\n")[1]).toBe("api.v1.runs.ts"); }); + + it("says so rather than printing a bare 100 when nothing applied", () => { + const r = run("/resources/health"); + expect(r.code).toBe(0); + expect(r.out).toContain("not measured"); + }); + + // B7. `map /api/v1/token --json` printed the text format and dropped the flag on the floor. + it("honours --json for a single route instead of printing the text format", () => { + const r = run("/api/v1/token", "--json"); + expect(r.code).toBe(0); + expect(r.out).not.toContain("CHECKS"); + const parsed = JSON.parse(r.out); + expect(parsed.fileName).toBe("api.v1.token.ts"); + expect(parsed.routePath).toBe("/api/v1/token"); + expect(Array.isArray(parsed.checks)).toBe(true); + }); }); describe("map", () => { - // The flag is the only thing standing between a test run and a file written into the repo root, - // so the test has to check the file, not just the exit code. + // The flag is the only thing standing between a run and a written report, so the test has to + // check the file, not just the exit code. `--out` keeps that file in a temp directory: this test + // used to delete `observability-map.json` from the repo root and never put it back. it("renders the whole report without writing when asked not to", () => { - const existedBefore = existsSync(REPORT_FILE); - if (existedBefore) rmSync(REPORT_FILE); + const dir = mkdtempSync(join(tmpdir(), "obs-map-out-")); + const out = join(dir, "report.json"); - const r = run("--no-write"); + const r = run("--out=" + out, "--no-write"); expect(r.code).toBe(0); expect(r.out).toContain("COVERAGE"); expect(r.out).toContain("FIX FIRST"); - expect(existsSync(REPORT_FILE)).toBe(false); + expect(existsSync(out)).toBe(false); + + rmSync(dir, { recursive: true, force: true }); + }); + + it("writes the report where --out names it when not asked to skip the write", () => { + const dir = mkdtempSync(join(tmpdir(), "obs-map-out-")); + const out = join(dir, "report.json"); + + const r = run("--out=" + out); + + expect(r.code).toBe(0); + expect(existsSync(out)).toBe(true); + const parsed = JSON.parse(readFileSync(out, "utf8")); + expect(parsed.entries.length).toBe(Object.keys(FIXTURES).length + 1); + + rmSync(dir, { recursive: true, force: true }); + }); +}); + +// B6. Stdout can be JSON a caller parses, so the warning goes to stderr in every mode, and the +// terminal report carries it too (see report.test.ts). +describe("warning about a suppression that names no check", () => { + it("names the file and the bad id on stderr for the whole report", () => { + const r = run("--no-write"); + expect(r.code).toBe(0); + expect(r.err).toContain("api.v1.typo.ts"); + expect(r.err).toContain("eror-classification"); + }); + + it("warns for a single route without putting the warning in the json", () => { + const r = run("/api/v1/typo", "--json"); + expect(r.code).toBe(0); + expect(r.err).toContain("eror-classification"); + expect(JSON.parse(r.out).fileName).toBe("api.v1.typo.ts"); + }); + + it("says nothing on stderr for a route whose directives all name a check", () => { + const r = run("/api/v1/token"); + expect(r.err).toBe(""); }); }); @@ -79,22 +186,24 @@ describe("map --routes=", () => { `export const loader = () => new Response("ok");` ); - const r = run("--routes=" + dir, "--json", "--no-write"); + const c = capture(); + const code = main(["node", "cli.js", "--routes=" + dir, "--json", "--no-write"], c.io); - expect(r.code).toBe(0); - const parsed = JSON.parse(r.out); + expect(code).toBe(0); + const parsed = JSON.parse(c.out()); expect(parsed.entries).toHaveLength(1); expect(parsed.entries[0].fileName).toBe("resources.only.ts"); - rmSync(dir, { recursive: true }); + rmSync(dir, { recursive: true, force: true }); }); it("exits 1 with a message when the directory does not exist", () => { const dir = join(tmpdir(), "obs-map-routes-does-not-exist"); - const r = run("--routes=" + dir); + const c = capture(); + const code = main(["node", "cli.js", "--routes=" + dir], c.io); - expect(r.code).toBe(1); - expect(r.err).toContain("not a readable directory"); - expect(r.out).toBe(""); + expect(code).toBe(1); + expect(c.err()).toContain("not a readable directory"); + expect(c.out()).toBe(""); }); }); diff --git a/internal-packages/observability-map/test/integration.test.ts b/internal-packages/observability-map/test/integration.test.ts index 4db7dfc223a..ca4f22892c0 100644 --- a/internal-packages/observability-map/test/integration.test.ts +++ b/internal-packages/observability-map/test/integration.test.ts @@ -1,25 +1,42 @@ -import { existsSync, readdirSync } from "node:fs"; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; import { join, resolve } from "node:path"; import { scanDirectory, scanFile } from "../src/scan.js"; import { buildReport } from "../src/score.js"; import { SCORED_CHECK_IDS } from "../src/checks/index.js"; +/** + * The one deliberate coupling to `apps/webapp/app/routes` in the suite. Everything else, including + * the CLI tests, runs against a fixture tree of this package's own making. + * + * The coupling is acceptable because nothing here names a route or a count: the scan must not + * crash, the entry point count must sit inside a wide band, and parse failures must be zero. Those + * survive routes being added, renamed and deleted, and they are the only things a fixture tree + * cannot tell us, since a fixture only contains shapes somebody thought to write down. `pr_checks. + * yml`'s `internal` filter watches `apps/webapp/app/routes/**` so a webapp pull request runs this + * rather than leaving the break for whoever pushes next. + */ const ROUTES = resolve(__dirname, "../../../apps/webapp/app/routes"); /** - * Counts route module candidates the same way `scanDirectory` walks the tree, without scanning - * their contents: a flat `.ts`/`.tsx` file, or one `route.ts`/`route.tsx` per directory. This is a - * structural upper bound, not a golden number - not every candidate exports a loader or action, so - * `entryPoints.length` must stay below it, and the route set is free to grow or shrink over time - * without breaking this test. + * Every `.ts`/`.tsx` file under the tree, at any depth. Deliberately not the scanner's walk, which + * looks at flat files and at one `route.ts`/`route.tsx` per directory: this used to be a verbatim + * copy of that walk, which made `entryPoints.length < countCandidates()` a tautology that could + * not fail for any route shape both of them missed. */ -function countCandidates(dir: string): number { +function countRouteModuleFiles(dir: string): number { let count = 0; for (const entry of readdirSync(dir, { withFileTypes: true })) { if (entry.isDirectory()) { - for (const child of readdirSync(join(dir, entry.name), { withFileTypes: true })) { - if (child.isFile() && (child.name === "route.ts" || child.name === "route.tsx")) count++; - } + count += countRouteModuleFiles(join(dir, entry.name)); continue; } if (entry.isFile() && /\.tsx?$/.test(entry.name) && !entry.name.endsWith(".d.ts")) count++; @@ -27,19 +44,60 @@ function countCandidates(dir: string): number { return count; } +beforeAll(() => { + // A hard failure rather than the `if (!existsSync(ROUTES)) return;` these tests opened with: if + // this package moves relative to apps/webapp, the real-tree coverage must disappear loudly. + if (!existsSync(ROUTES)) { + throw new Error(`the webapp routes directory is missing: ${ROUTES}`); + } +}); + +// B7. The build emits ESM (`module: ESNext`, and `cli.ts` uses `import.meta`) while `main` and +// `types` advertised it to a package.json with no module type. On node 24 that loads with a +// MODULE_TYPELESS_PACKAGE_JSON warning and a reparse rather than the throw older nodes give, which +// is a warning about an artifact this package tells other packages to import. +describe("the package it advertises", () => { + const manifest = JSON.parse( + readFileSync(resolve(__dirname, "../package.json"), "utf8") + ) as Record; + + // Asserted flat rather than behind an `if (!advertised) return`, which is the silent skip this + // round removed from the tests below: the decision was to keep the entry point and declare the + // module type, so dropping the entry point later should have to edit this, not slip past it. + it("declares the module type its build emits alongside the entry point it advertises", () => { + expect(manifest.main).toBe("./dist/src/index.js"); + expect(manifest.types).toBe("./dist/src/index.d.ts"); + expect(manifest.type).toBe("module"); + }); +}); + +describe("counting candidates independently of the scanner", () => { + // The counter is only worth having if it disagrees with the scanner somewhere. It does: the + // scanner attributes nothing to a nested file that is not `route.ts`/`route.tsx`, and the + // counter counts every module file at every depth. + it("counts a nested non-route file the scanner does not attribute to any route", () => { + const dir = mkdtempSync(join(tmpdir(), "obs-map-count-")); + mkdirSync(join(dir, "components")); + writeFileSync( + join(dir, "components", "helper.ts"), + `export const loader = () => new Response("ok");` + ); + + expect(countRouteModuleFiles(dir)).toBe(1); + expect(scanDirectory(dir).entryPoints).toHaveLength(0); + + rmSync(dir, { recursive: true, force: true }); + }); +}); + describe("scanning the real webapp routes", () => { - it("parses every route file without crashing", () => { - if (!existsSync(ROUTES)) return; + it("parses every route file and produces a report inside a wide band", () => { const { entryPoints, parseFailures } = scanDirectory(ROUTES); - // Invariants, not exact numbers: the route set changes constantly. - expect(entryPoints.length).toBeGreaterThan(200); - expect(entryPoints.length).toBeLessThan(countCandidates(ROUTES)); + expect(parseFailures).toEqual([]); - }); + expect(entryPoints.length).toBeGreaterThan(100); + expect(entryPoints.length).toBeLessThan(countRouteModuleFiles(ROUTES)); - it("produces a report with a score in range", () => { - if (!existsSync(ROUTES)) return; - const { entryPoints, parseFailures } = scanDirectory(ROUTES); const report = buildReport(entryPoints, parseFailures); expect(report.global).toBeGreaterThanOrEqual(0); expect(report.global).toBeLessThanOrEqual(100); @@ -51,7 +109,6 @@ describe("scanning the real webapp routes", () => { // 176, because every entry whose only applicable checks were suppressed dropped out of the // mean. Measured must not move: every entry point that had something applicable still does. it("suppressing every scored check on every real route does not raise the global", () => { - if (!existsSync(ROUTES)) return; const { entryPoints, parseFailures } = scanDirectory(ROUTES); const before = buildReport(entryPoints, parseFailures); From 7273d128c1e2912bc47f3219040ec41034bb72dc Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Sat, 1 Aug 2026 23:17:04 +0100 Subject: [PATCH 068/106] fix(observability-map): mark a suppression-caused row, and post when only a suppression moved Two halves of the same hole. hasDelta compared the score, the measured flag and newly failing checks, so a pull request that only suppressed findings moved none of them and posted nothing, while a MISTYPED directive did post because the unknown warnings were compared. It now also compares the per-entry suppression set, the audit gap and the context gap. All three are reachable with nothing else moving: an already-failing check being silenced, audit-trail going fail to pass (the webapp's first audit record), and the CONTEXT figure moving behind a suppression, which reads pre-suppression data. MapReport.suppressions is deliberately not compared, because its totals are summed from the same per-entry arrays the loop already walks. The changed-entries table now includes a row when the suppression set moved even if no score did, and marks any row a new suppression caused. Suppressing a check that was PASSING drops the score by round A's cap and rendered a row with an empty "now failing" column, sorted among the real regressions: _app.@.orgs.$organizationSlug.$.tsx renders 67 to 50 exactly that way on the real tree. The score movement is honest; the unmarked row was not. An earlier report claimed this class could not be constructed, which was wrong. Caps the unknown-suppression section of the comment at ten files. It was the one unbounded section, and one mistyped directive applied tree wide renders 87,938 characters against GitHub's 65,536 limit, for a 422 the job then swallows. --- .../observability-map/src/report/prComment.ts | 83 ++++++-- .../observability-map/test/prComment.test.ts | 183 ++++++++++++++++++ 2 files changed, 249 insertions(+), 17 deletions(-) diff --git a/internal-packages/observability-map/src/report/prComment.ts b/internal-packages/observability-map/src/report/prComment.ts index 80c0b4b70fe..58010e97efd 100644 --- a/internal-packages/observability-map/src/report/prComment.ts +++ b/internal-packages/observability-map/src/report/prComment.ts @@ -13,6 +13,14 @@ export const MARKER = ""; const MAX_CHANGED_ROWS = 15; +/** + * Every other section of this comment is bounded by construction. This one was not, and a single + * mistyped directive applied tree wide renders one line per file: 87,938 characters against + * GitHub's 65,536 limit, a 422, and the workflow's error tolerance swallowing it. The cap is what + * stops the whole comment being lost to the section warning about a typo. + */ +const MAX_UNKNOWN_SUPPRESSION_LINES = 10; + // Scored checks only, same exclusion terminal.ts's scoredFailures makes: audit-trail fails almost // every sensitive mutation today, so listing it per route would nag with something unfixable // instead of surfacing the route-specific gaps this column exists for. @@ -49,12 +57,25 @@ const NOT_MEASURED = "not measured"; */ const scoreCell = (e: ScoredEntry): number | string => (e.measured ? e.score : NOT_MEASURED); +/** Ids suppressed at head that were not suppressed at base. `[]` covers both "no change" and a + * suppression being removed, which shows up as the score going back up. */ +const newlySuppressed = (head: ScoredEntry, base: ScoredEntry | undefined): string[] => + head.suppressed.filter((id) => !(base?.suppressed ?? []).includes(id)); + type ChangedRow = { routePath: string; sensitive: boolean; baseScore: number | string; headScore: number | string; nowFailing: string[]; + /** + * Ids this pull request newly suppressed on the entry. Rendered on the route cell, because a + * suppression added to a check that was passing drops the score by round A's cap and produces a + * row with an empty "now failing" column, which is indistinguishable from a real regression: + * `_app.@.orgs.$organizationSlug.$.tsx` renders 67 to 50 that way. The score movement is honest, + * the row without this note was not. + */ + suppressed: string[]; /** * How much the entry got worse, used to sort the table. A new entry has no base score to * subtract from, so it is scored against a perfect 100: a new entry landing at 60 sorts the @@ -77,20 +98,25 @@ function changedRows(head: MapReport, base: MapReport): { rows: ChangedRow[]; re // A new entry that passes every check it was measured against has nothing to fix, which is // what `drop: 0` means everywhere else in this table. A new entry nothing applied to is a // different statement and still gets a row, since its 100 is a placeholder rather than a pass. - if (h.measured && h.score === 100) continue; + if (h.measured && h.score === 100 && h.suppressed.length === 0) continue; rows.push({ routePath: h.routePath, sensitive: h.sensitive, baseScore: "new", headScore: scoreCell(h), nowFailing: failingIds(h), + suppressed: newlySuppressed(h, b), drop: h.measured ? 100 - h.score : 0, }); continue; } - // Measured state is part of what changed: a measured-to-unmeasured transition can leave the - // score untouched at its placeholder value, and skipping on the number alone hid it. - if (b.measured === h.measured && b.score === h.score) continue; + // Measured state and the suppression set are both part of what changed. A measured-to- + // unmeasured transition can leave the score at its placeholder value, and suppressing a check + // that was already failing moves no score at all, so skipping on the number alone hid both. A + // pull request whose whole purpose is to silence findings has to produce a row. + const suppressed = newlySuppressed(h, b); + const suppressionChanged = suppressed.length > 0 || h.suppressed.length !== b.suppressed.length; + if (b.measured === h.measured && b.score === h.score && !suppressionChanged) continue; const baseFailing = new Set(failingIds(b)); rows.push({ routePath: h.routePath, @@ -98,6 +124,7 @@ function changedRows(head: MapReport, base: MapReport): { rows: ChangedRow[]; re baseScore: scoreCell(b), headScore: scoreCell(h), nowFailing: failingIds(h).filter((id) => !baseFailing.has(id)), + suppressed, drop: b.measured && h.measured ? b.score - h.score : 0, }); } @@ -133,8 +160,9 @@ function whatChangedSection(head: MapReport, base: MapReport | null): string[] { lines.push("| route | base | head | now failing |"); lines.push("| --- | --- | --- | --- |"); for (const row of rows.slice(0, MAX_CHANGED_ROWS)) { + const note = row.suppressed.length > 0 ? ` (suppressed: ${row.suppressed.join(", ")})` : ""; lines.push( - `| ${row.routePath} | ${row.baseScore} | ${row.headScore} | ${row.nowFailing.join(", ")} |` + `| ${row.routePath}${note} | ${row.baseScore} | ${row.headScore} | ${row.nowFailing.join(", ")} |` ); } if (rows.length > MAX_CHANGED_ROWS) { @@ -173,26 +201,43 @@ function fixFirstSection(head: MapReport): string[] { return lines; } +const same = (a: unknown, b: unknown) => JSON.stringify(a) === JSON.stringify(b); + /** * Whether this pull request moves the report at all, so the job can stay quiet when it does not. * - * True on: no base to compare against, a global score change, an entry added or removed, an - * entry's score or measured state changed, a check that fails at head and did not at base, a - * change in the parse failure count, or a change in the unknown suppression warnings. The last two - * are here because both render a line in the comment, so a run that gains one has moved the report - * even though no score did. + * The rule this has to satisfy is that it must be true whenever `renderPrComment` would say + * something different, because anything it misses is a change the pull request silently does not + * report. So it covers every figure the comment renders, not only the score: the global, the + * per-entry score, measured state and suppression set, an entry added or removed, a check failing + * at head that did not at base, the parse failure count, the unknown suppression warnings, and the + * audit and context gaps. + * + * The per-entry suppression set and the two gaps are the half that was missing, and it ran the + * dangerous way. Suppressing an already-failing check moves no score, no measured flag and no new + * failure, so a pull request whose entire purpose was to silence findings posted nothing, while a + * mistyped directive did post because the unknown warnings were compared. `audit-trail` going from + * fail to pass, the first audit record in the webapp, was in the same hole, and so was the CONTEXT + * figure moving behind a suppression, since that figure reads pre-suppression data. + * + * The terms overlap on purpose, and what is defended is that their union is complete rather than + * that each one is load bearing. Four are individually reachable, each with a test that fails when + * only that term is removed: the parse failure count, the unknown suppression warnings, the audit + * gap and the context gap. The global, the removed-entry check and the per-entry score are each + * shadowed by another term today, and are kept because which term shadows which depends on the + * shape of the change rather than on anything stable. * - * The residual: adding a suppression whose check was already failing moves nothing this asks - * about. The score is capped at the pre-suppression ratio so it cannot move, and the finding - * leaves the fix list quietly. `SUPPRESSED` in the terminal report is where that shows up. + * `MapReport.suppressions` is the one term deliberately left out. Its two totals are summed from + * the very per-entry `suppressed` arrays the loop below compares one by one, so it cannot move + * without the loop moving. That is arithmetic rather than a happy overlap. */ export function hasDelta(head: MapReport, base: MapReport | null): boolean { if (!base) return true; if (head.global !== base.global) return true; if (head.parseFailures.length !== base.parseFailures.length) return true; - if (JSON.stringify(head.unknownSuppressions) !== JSON.stringify(base.unknownSuppressions)) { - return true; - } + if (!same(head.unknownSuppressions, base.unknownSuppressions)) return true; + if (!same(head.auditGap, base.auditGap)) return true; + if (!same(head.contextGap, base.contextGap)) return true; const baseByFile = new Map(base.entries.map((e) => [e.fileName, e])); const headFiles = new Set(head.entries.map((e) => e.fileName)); @@ -202,6 +247,7 @@ export function hasDelta(head: MapReport, base: MapReport | null): boolean { const b = baseByFile.get(h.fileName); if (!b) return true; if (b.measured !== h.measured || b.score !== h.score) return true; + if (!same(h.suppressed, b.suppressed)) return true; const baseFailing = new Set(b.checks.filter((c) => c.status === "fail").map((c) => c.id)); if (h.checks.some((c) => c.status === "fail" && !baseFailing.has(c.id))) return true; } @@ -260,7 +306,10 @@ export function renderPrComment(head: MapReport, base: MapReport | null): string const context = contextLine(head); if (context) lines.push(context); const unknown = unknownSuppressionLines(head); - lines.push(...unknown); + lines.push(...unknown.slice(0, MAX_UNKNOWN_SUPPRESSION_LINES)); + if (unknown.length > MAX_UNKNOWN_SUPPRESSION_LINES) { + lines.push(`and ${unknown.length - MAX_UNKNOWN_SUPPRESSION_LINES} more files with unknown ids`); + } if (audit || context || unknown.length > 0) lines.push(""); lines.push( diff --git a/internal-packages/observability-map/test/prComment.test.ts b/internal-packages/observability-map/test/prComment.test.ts index 4299b1ae3a7..8bea1e2edf3 100644 --- a/internal-packages/observability-map/test/prComment.test.ts +++ b/internal-packages/observability-map/test/prComment.test.ts @@ -184,6 +184,109 @@ describe("renderPrComment", () => { }); }); + // I4. A suppression added to a check that was PASSING drops the score by round A's cap and + // produces a row with an empty "now failing" column, sorted among the real regressions. On the + // real tree `_app.@.orgs.$organizationSlug.$.tsx` renders 67 to 50 exactly that way. + describe("a row a suppression caused", () => { + // Two of three applicable scored checks pass, so suppressing one of the passes takes the + // visible ratio from 2/3 to 1/2, which is the 67 to 50 the real tree renders on + // `_app.@.orgs.$organizationSlug.$.tsx`. The catch has to decide something for + // error-classification to apply at all, and nothing may name a tenant, or the ratio is 3/3. + const twoOfThree = `import { requireUserId } from "~/services/session.server"; + import { prisma } from "~/db.server"; + export async function action({ request }) { + const userId = await requireUserId(request); + try { return await prisma.token.create({ data: { userId } }); } + catch (error) { + if (error instanceof BadRequest) return json({ error: "bad" }, { status: 400 }); + throw error; + } + }`; + const silence = (id: string, source: string) => + `// obs-map-disable ${id} -- silenced\n${source}`; + + it("says so on the route, so it is not read as a regression", () => { + const base = buildReport([scanFile("api.v1.auth.tokens.ts", twoOfThree)!], []); + const head = buildReport( + [scanFile("api.v1.auth.tokens.ts", silence("error-classification", twoOfThree))!], + [] + ); + expect(base.entries[0]!.score).toBe(67); + expect(head.entries[0]!.score).toBe(50); + + const row = renderPrComment(head, base) + .split("\n") + .find((l) => l.startsWith("| /api/v1/auth/tokens"))!; + expect(row).toBeDefined(); + expect(row).toContain("(suppressed: error-classification)"); + // The column that would otherwise explain the drop is empty, which is the whole problem. + expect(row.split("|")[4]!.trim()).toBe(""); + }); + + // I3. This one moves no score at all, so before the suppression set was compared it produced + // no row and no comment: a pull request whose whole purpose is to silence findings was silent. + it("appears even when suppressing an already-failing check moved no score", () => { + const base = buildReport([scanFile("api.v1.t.ts", brokenSource)!], []); + const head = buildReport( + [scanFile("api.v1.t.ts", silence("error-classification", brokenSource))!], + [] + ); + expect(head.entries[0]!.score).toBe(base.entries[0]!.score); + expect(head.global).toBe(base.global); + + const out = renderPrComment(head, base); + expect(out).not.toContain("No entry point this PR touches changed its score."); + const row = out.split("\n").find((l) => l.startsWith("| /api/v1/t "))!; + expect(row).toBeDefined(); + expect(row).toContain("(suppressed: error-classification)"); + }); + + it("says nothing about suppression on a row that has none", () => { + const head = buildReport([scanFile("api.v1.auth.tokens.ts", brokenSource)!], []); + const base = buildReport([scanFile("api.v1.auth.tokens.ts", cleanSource)!], []); + const row = renderPrComment(head, base) + .split("\n") + .find((l) => l.startsWith("| /api/v1/auth/tokens"))!; + expect(row).not.toContain("suppressed:"); + }); + + it("gives a new entry that lands at 100 only because of a suppression a row", () => { + const base = buildReport([scanFile("api.v1.a.ts", cleanSource)!], []); + const head = buildReport( + [ + scanFile("api.v1.a.ts", cleanSource)!, + scanFile("api.v1.new.ts", silence("request-context", cleanSource))!, + ], + [] + ); + const row = renderPrComment(head, base) + .split("\n") + .find((l) => l.startsWith("| /api/v1/new"))!; + expect(row).toBeDefined(); + expect(row).toContain("(suppressed: request-context)"); + }); + }); + + // M5. Every other section is bounded by construction; this one rendered one line per file, and a + // tree-wide typo took the comment past GitHub's 65,536 character limit for a 422 nobody sees. + it("caps the unknown-suppression lines instead of running past the comment size limit", () => { + const entries = []; + for (let i = 0; i < 40; i++) { + entries.push( + scanFile( + `api.v1.route${i}.ts`, + `// obs-map-disable eror-classification -- typo\n${brokenSource}` + )! + ); + } + const head = buildReport(entries, []); + const out = renderPrComment(head, null); + + expect(out.split("\n").filter((l) => l.startsWith("UNKNOWN SUPPRESSION"))).toHaveLength(10); + expect(out).toContain("and 30 more files with unknown ids"); + expect(out.length).toBeLessThan(65536); + }); + it("sorts a sensitive entry with a small drop above a non-sensitive entry with a large drop", () => { const sensitiveSmallDropBase = scanFile("api.v1.auth.tokens.ts", cleanSource)!; const sensitiveSmallDropHead = scanFile( @@ -346,6 +449,86 @@ describe("hasDelta", () => { expect(hasDelta(head, one("api.v1.a.ts", cleanSource))).toBe(true); }); + // I3. These three are the half that was missing, and it ran the dangerous way: a pull request + // that only silences findings posted nothing, while a mistyped directive did post. + it("is true when a suppression was added to a check that was already failing", () => { + const base = one("api.v1.t.ts", brokenSource); + const head = one( + "api.v1.t.ts", + `// obs-map-disable error-classification -- silenced\n${brokenSource}` + ); + expect(head.global).toBe(base.global); + expect(head.entries[0]!.score).toBe(base.entries[0]!.score); + expect(head.measured).toBe(base.measured); + expect(hasDelta(head, base)).toBe(true); + }); + + it("is true when the audit gap closed, which no score reports", () => { + const audited = `import { auditLog } from "~/services/audit.server"; + import { prisma } from "~/db.server"; + export async function action() { + const token = await prisma.token.create({ data: {} }); + await auditLog("token.created", { tokenId: token.id }); + return json(token); + }`; + const unaudited = `import { prisma } from "~/db.server"; + export async function action() { + const token = await prisma.token.create({ data: {} }); + return json(token); + }`; + const head = one("api.v1.auth.tokens.ts", audited); + const base = one("api.v1.auth.tokens.ts", unaudited); + expect(head.auditGap).not.toEqual(base.auditGap); + expect(hasDelta(head, base)).toBe(true); + }); + + // The CONTEXT line reads pre-suppression data, so with request-context suppressed its figure can + // move while the post-suppression checks, the score and the global all stay put. The comment + // says "0 of 1" and then "1 of 1"; nothing else in the report moves at all. + it("is true when the context figure moved behind a suppression", () => { + const silence = "// obs-map-disable request-context -- reported as a figure\n"; + const namesNobody = `${silence}import { prisma } from "~/db.server"; + export async function action() { + try { return await prisma.envVar.update({ where: {}, data: {} }); } catch (e) { return null; } + }`; + const namesTenant = `${silence}import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function action({ params }) { + try { return await prisma.envVar.update({ where: {}, data: {} }); } + catch (error) { logger.error("failed", { environmentId: params.envId, error }); return null; } + }`; + const head = one("api.v1.envvars.ts", namesTenant); + const base = one("api.v1.envvars.ts", namesNobody); + + expect(head.global).toBe(base.global); + expect(head.entries[0]!.score).toBe(base.entries[0]!.score); + expect(head.entries[0]!.suppressed).toEqual(base.entries[0]!.suppressed); + expect(head.contextGap).not.toEqual(base.contextGap); + expect(hasDelta(head, base)).toBe(true); + }); + + // Suppressing a check that was not applicable anyway changes no status and no score, and moving + // that suppression between two entries leaves the report-level totals identical too. The + // per-entry suppression comparison is the only thing left that can see it. + it("is true when a suppression of an inapplicable check moved between entries", () => { + const silenced = `// obs-map-disable auth-boundary -- not a user-facing route\n${brokenSource}`; + const head = buildReport( + [scanFile("api.v1.a.ts", silenced)!, scanFile("api.v1.b.ts", brokenSource)!], + [] + ); + const base = buildReport( + [scanFile("api.v1.a.ts", brokenSource)!, scanFile("api.v1.b.ts", silenced)!], + [] + ); + expect(head.suppressions).toEqual(base.suppressions); + expect(head.global).toBe(base.global); + expect(head.entries.map((e) => e.score)).toEqual(base.entries.map((e) => e.score)); + const statuses = (r: typeof head) => + r.entries.map((e) => e.checks.map((c) => `${c.id}=${c.status}`).join(" ")); + expect(statuses(head)).toEqual(statuses(base)); + expect(hasDelta(head, base)).toBe(true); + }); + it("is true when a suppression names a check that does not exist", () => { const head = one( "api.v1.a.ts", From b0d6dc0cc528b5c9d88387bc7f587de495663196 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Sat, 1 Aug 2026 23:17:24 +0100 Subject: [PATCH 069/106] ci(observability-map): make a failed comment lookup mean one thing, and run the tests here The render step read a missing /tmp/existing-comment-id as "a comment exists" and emitted the resolved state; the upsert step read it as "no id" and POSTed. So a transient lookup failure either added a second marker comment beside the stale one, or announced that an earlier push's findings were gone on a pull request that never had findings, which is the thing the post-only-on-delta change exists to prevent. The comment above it claimed the two agreed. The guess bought nothing in either direction: with no id, the render step's --existing-comment can only ever produce a body the upsert then POSTs. The lookup now retries three times, and on a failure that outlasts the retries it drops a sentinel both steps read, so the run posts nothing. Worst case is no comment this run, which the next push fixes. gh api --paginate runs the jq once per page, so a marker comment on two different pages yielded two ids: a newline in the PATCH url and a step that dies silently, or a second comment whose stale findings stand forever. The oldest id is taken, and a count above one is logged. The stale-report comment now goes through the same temp file dance as every other write, so a renderer that exits non-zero cannot leave a 0-byte comment.md for the upsert to skip in silence. Adds a blocking unit-tests job running this package alone. That is what protects the one test against the live route tree, in place of widening pr_checks.yml's internal filter, which ran all eighteen internal packages with postgres, clickhouse, redis and electric for the same protection. This workflow already triggers on exactly the paths that can break that test, and unlike the report job this one needs no token, so it runs for fork pull requests and is meant to fail. Verified by extracting the three comment step bodies and running them against a stubbed gh across seven cases: hard lookup failure, flaky lookup, no delta with and without an existing comment, two marker comments across pages, malformed head report, and a head scan that produced nothing. --- .github/workflows/observability-map.yml | 117 +++++++++++++++++++----- 1 file changed, 96 insertions(+), 21 deletions(-) diff --git a/.github/workflows/observability-map.yml b/.github/workflows/observability-map.yml index 4d30b29cc5b..3aa8f5018b5 100644 --- a/.github/workflows/observability-map.yml +++ b/.github/workflows/observability-map.yml @@ -50,6 +50,39 @@ jobs: pnpm --filter @internal/observability-map exec vitest run \ test/mutationCorpus.test.ts --testTimeout=120000 --disable-console-intercept + # The package has one deliberate test against the live route tree + # (internal-packages/observability-map/test/integration.test.ts), and this workflow already + # triggers on exactly the paths that can break it. Running just this package here is what keeps + # a webapp-only pull request from merging a renamed route green and leaving the failure for main + # or for the next unrelated internal-packages pull request. Unlike the report job below, this one + # is meant to be able to fail, and it runs for fork pull requests because it needs no token. + unit-tests: + name: 🧪 Package tests + runs-on: warp-ubuntu-latest-x64-4x + steps: + - name: ⬇️ Checkout repo + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + fetch-depth: 1 + persist-credentials: false + + - name: ⎔ Setup pnpm + uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0 + with: + version: 10.33.2 + + - name: ⎔ Setup node + uses: WarpBuilds/setup-node@bc639b444d583175926b588962199c247d23e8d3 # v6 + with: + node-version: 24.18.0 + cache: "pnpm" + + - name: 📥 Download deps + run: pnpm install --frozen-lockfile + + - name: 🧪 Run the package tests + run: pnpm --filter @internal/observability-map run test + report: runs-on: warp-ubuntu-latest-x64-4x # Only this job comments, so only this job gets the write. @@ -103,25 +136,50 @@ jobs: echo "base scan failed or the worktree could not be added; falling back to no base" >&2 fi - # Looked up before the render step because the render step needs it: with no delta to report, - # a pull request that already has a comment gets a resolved state rather than being left with - # findings that no longer exist, and one that does not gets nothing at all. The upsert step - # reuses the id rather than asking again. + # Looked up before the render step because the render decision needs it: with no delta to + # report, a pull request that already has a comment gets a resolved state rather than being + # left with findings that no longer exist, and one that does not gets nothing at all. The + # upsert step reuses the id rather than asking twice. + # + # On a failure that outlasts the retries, both steps below do nothing. Guessing is worse than + # silence here: this step is the only thing that knows which comment to PATCH, so a guess of + # "a comment exists" still reaches an upsert with no id to patch, which POSTs. That either + # adds a second marker comment beside the stale one, or says "the findings an earlier push + # reported are gone" on a pull request that never had findings. Worst case now is no comment + # this run, which the next push fixes. - name: 🔍 Look for a comment from an earlier push continue-on-error: true env: GH_TOKEN: ${{ github.token }} PR_NUMBER: ${{ github.event.pull_request.number }} run: | - rm -f /tmp/existing-comment-id - if gh api "repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/comments" --paginate \ - --jq '[.[] | select(.body | startswith(""))][0].id // empty' \ - > /tmp/existing-comment-id.partial; then - mv /tmp/existing-comment-id.partial /tmp/existing-comment-id - else - rm -f /tmp/existing-comment-id.partial - echo "comment lookup failed; assuming a comment exists so a stale one still gets replaced" >&2 + rm -f /tmp/existing-comment-id /tmp/comment-lookup-failed + found="" + ok="" + for attempt in 1 2 3; do + if found=$(gh api "repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/comments" --paginate \ + --jq '[.[] | select(.body | startswith(""))][0].id // empty'); then + ok=1 + break + fi + echo "comment lookup attempt ${attempt} failed" >&2 + sleep $((attempt * 5)) + done + + if [ -z "$ok" ]; then + touch /tmp/comment-lookup-failed + echo "comment lookup failed after 3 attempts; this run posts nothing" >&2 + exit 0 + fi + + # --paginate runs the jq once per page, so a marker comment on more than one page yields + # one id per page. Unhandled, that puts a newline in the PATCH url and the step dies under + # continue-on-error. The oldest wins: it is the one the upsert has been updating. + count=$(printf '%s\n' "$found" | grep -c '[0-9]' || true) + if [ "$count" -gt 1 ]; then + echo "warning: ${count} marker comments on this pull request; updating the oldest" >&2 fi + printf '%s\n' "$found" | awk 'NF { print $1; exit }' > /tmp/existing-comment-id # continue-on-error for the same reason as the scan: a rendering bug must not turn the job # red. An empty /tmp/comment.md means there is nothing to post, which is a decision @@ -132,8 +190,24 @@ jobs: rm -f /tmp/comment.md render() { pnpm --filter @internal/observability-map exec tsx src/report/prCommentCli.ts "$@"; } + # Every write goes through this, so a renderer that exits non-zero never leaves a 0-byte + # comment.md for the upsert to skip in silence. + emit() { + if render "$@" > /tmp/comment.md.partial; then + mv /tmp/comment.md.partial /tmp/comment.md + return 0 + fi + rm -f /tmp/comment.md.partial + return 1 + } + + if [ -f /tmp/comment-lookup-failed ]; then + echo "the comment lookup failed, so this run posts nothing" >&2 + exit 0 + fi + if [ ! -s /tmp/head.json ]; then - render --scan-failed > /tmp/comment.md + emit --scan-failed || echo "could not render the stale-report comment either" >&2 exit 0 fi @@ -142,19 +216,14 @@ jobs: base="-" fi - # An absent file means the lookup itself failed, and assuming there is a comment is the - # safer half of that guess: it can post a redundant resolved state, not leave a stale one. flags=() - if [ ! -f /tmp/existing-comment-id ] || [ -s /tmp/existing-comment-id ]; then + if [ -s /tmp/existing-comment-id ]; then flags=(--existing-comment) fi - if render /tmp/head.json "$base" "${flags[@]}" > /tmp/comment.md.partial; then - mv /tmp/comment.md.partial /tmp/comment.md - else - rm -f /tmp/comment.md.partial + if ! emit /tmp/head.json "$base" "${flags[@]}"; then echo "render failed; falling back to the stale-report comment" >&2 - render --scan-failed > /tmp/comment.md + emit --scan-failed || echo "could not render the stale-report comment either" >&2 fi # continue-on-error for the same reason: a transient gh api failure (rate limit, network) @@ -165,6 +234,12 @@ jobs: GH_TOKEN: ${{ github.token }} PR_NUMBER: ${{ github.event.pull_request.number }} run: | + # The same sentinel the render step reads, so the two cannot disagree about what a failed + # lookup means. Without it this step reads a missing id as "no comment exists" and POSTs. + if [ -f /tmp/comment-lookup-failed ]; then + echo "the comment lookup failed, so this run posts nothing" + exit 0 + fi if [ ! -s /tmp/comment.md ]; then echo "nothing to post: this pull request does not move the report" exit 0 From bccaf9f457ce5357094bc4c6463ab0b3af9bf2a1 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Sat, 1 Aug 2026 23:18:21 +0100 Subject: [PATCH 070/106] ci: revert the observability-map widening of the internal filter Widening pr_checks.yml's internal filter to apps/webapp/app/routes/** ran the whole eighteen package internal suite, twelve shards with postgres, clickhouse, redis and electric, for every pull request touching a route file, all to protect one test. The unit-tests job in observability-map.yml runs that package alone on the same paths and is the proportionate version. Adds the guard the docstring checker cannot give: it walks src/ only, so workflow prose goes unpoliced, and the lookup defect fixed alongside this was two steps disagreeing under a comment claiming they agreed. The new test is a text check over the workflow, and says so rather than selling itself as coverage of the file. --- .github/workflows/pr_checks.yml | 5 --- .../test/integration.test.ts | 44 +++++++++++++++++-- 2 files changed, 41 insertions(+), 8 deletions(-) diff --git a/.github/workflows/pr_checks.yml b/.github/workflows/pr_checks.yml index 89b5dc8c29b..b0fbd6ac040 100644 --- a/.github/workflows/pr_checks.yml +++ b/.github/workflows/pr_checks.yml @@ -74,11 +74,6 @@ jobs: internal: - 'internal-packages/**' - 'packages/**' - # @internal/observability-map has one deliberate test against the live route tree - # (internal-packages/observability-map/test/integration.test.ts). Without this the - # internal suite never ran for a webapp-only PR, so a renamed route merged green and - # broke main, or the next unrelated internal-packages PR. - - 'apps/webapp/app/routes/**' - '.github/workflows/pr_checks.yml' - '.github/workflows/unit-tests-internal.yml' - '.configs/**' diff --git a/internal-packages/observability-map/test/integration.test.ts b/internal-packages/observability-map/test/integration.test.ts index ca4f22892c0..f16c7b28b50 100644 --- a/internal-packages/observability-map/test/integration.test.ts +++ b/internal-packages/observability-map/test/integration.test.ts @@ -20,9 +20,13 @@ import { SCORED_CHECK_IDS } from "../src/checks/index.js"; * The coupling is acceptable because nothing here names a route or a count: the scan must not * crash, the entry point count must sit inside a wide band, and parse failures must be zero. Those * survive routes being added, renamed and deleted, and they are the only things a fixture tree - * cannot tell us, since a fixture only contains shapes somebody thought to write down. `pr_checks. - * yml`'s `internal` filter watches `apps/webapp/app/routes/**` so a webapp pull request runs this - * rather than leaving the break for whoever pushes next. + * cannot tell us, since a fixture only contains shapes somebody thought to write down. + * + * What runs this for a webapp pull request is the `unit-tests` job in + * `.github/workflows/observability-map.yml`, which already triggers on `apps/webapp/app/routes/**` + * and runs this package alone. Widening `pr_checks.yml`'s `internal` filter to those paths was + * tried and reverted: it ran all eighteen internal packages, twelve shards with postgres, + * clickhouse, redis and electric, to protect this one test. */ const ROUTES = resolve(__dirname, "../../../apps/webapp/app/routes"); @@ -71,6 +75,40 @@ describe("the package it advertises", () => { }); }); +/** + * The one thing the docstring checker cannot reach. It walks `src/` only, so workflow prose is + * unpoliced, and the C1 defect was exactly that: two steps disagreeing about what a missing + * `/tmp/existing-comment-id` meant, under a comment claiming they agreed. The render step read it + * as "a comment exists" and emitted the resolved state, the upsert step read it as "no id" and + * POSTed, so a transient lookup failure either added a second marker comment beside the stale one + * or announced that findings were gone on a pull request that never had any. + * + * This is a text check over the workflow, not a parse of its semantics, so it catches one shape of + * that class and no other. Named as such rather than sold as coverage of the file. + */ +describe("the report workflow's two readers of the comment lookup", () => { + const WORKFLOW = resolve(__dirname, "../../../.github/workflows/observability-map.yml"); + + /** Step bodies, split on the `- name:` lines, which is all the structure this needs. */ + function steps(): string[] { + if (!existsSync(WORKFLOW)) throw new Error(`the report workflow is missing: ${WORKFLOW}`); + const text = readFileSync(WORKFLOW, "utf8"); + return text.split(/^ {6}- name: /m).slice(1); + } + + it("both honour the same sentinel, so a failed lookup cannot mean two things", () => { + const readers = steps().filter((step) => step.includes("/tmp/existing-comment-id")); + expect(readers.length).toBeGreaterThanOrEqual(2); + expect(readers.filter((step) => !step.includes("/tmp/comment-lookup-failed"))).toEqual([]); + }); + + it("takes one id from a lookup that paginates rather than passing every line on", () => { + const lookup = steps().find((step) => step.includes('startswith(""; const MAX_CHANGED_ROWS = 15; /** - * Every other section of this comment is bounded by construction. This one was not, and a single - * mistyped directive applied tree wide renders one line per file: 87,938 characters against + * A mistyped directive applied tree wide renders one line per file: 87,938 characters against * GitHub's 65,536 limit, a 422, and the workflow's error tolerance swallowing it. The cap is what * stops the whole comment being lost to the section warning about a typo. */ const MAX_UNKNOWN_SUPPRESSION_LINES = 10; +/** + * The same failure in the other section that grows with the size of the tree. `delegating` holds + * one file name per route whose body lives elsewhere, joined into a single line, and a codemod that + * moves route bodies into `.server.ts` modules is both the refactor this feature exists to notice + * and the one that makes the list tree-sized. The cap was claimed here before it was written: the + * note above used to open "every other section of this comment is bounded by construction", which + * was not true of this one. + * + * Fifteen matches the changed-entries table rather than the ten above, because a delegating file + * name is one comma-separated item rather than a line naming every known check. The bound that + * matters is the section's worst case: the longest route file name in the tree is 130 characters, + * so fifteen of those plus separators is under 2kB against GitHub's 65,536. + */ +const MAX_DELEGATED_ROUTES = 15; + // Scored checks only, same exclusion terminal.ts's scoredFailures makes: audit-trail fails almost // every sensitive mutation today, so listing it per route would nag with something unfixable // instead of surfacing the route-specific gaps this column exists for. @@ -314,7 +328,7 @@ export function renderPrComment(head: MapReport, base: MapReport | null): string if (audit) lines.push(audit); const context = contextLine(head); if (context) lines.push(context); - const delegated = delegatedLines(head); + const delegated = delegatedLines(head, MAX_DELEGATED_ROUTES); lines.push(...delegated); const unknown = unknownSuppressionLines(head); lines.push(...unknown.slice(0, MAX_UNKNOWN_SUPPRESSION_LINES)); diff --git a/internal-packages/observability-map/src/report/terminal.test.ts b/internal-packages/observability-map/src/report/terminal.test.ts index 74221ddc7a5..61b7aa9c462 100644 --- a/internal-packages/observability-map/src/report/terminal.test.ts +++ b/internal-packages/observability-map/src/report/terminal.test.ts @@ -194,6 +194,24 @@ describe("rendering honestly when there is nothing to say", () => { expect(out).not.toContain("No audit helper exists"); }); + // Round E item 2. The other half of the branch, which the test above did not pin. A zero used to + // print "No audit helper exists in the webapp", which is false: `models/admin.server.ts` writes + // `prisma.impersonationAuditLog.create(...)` and `AUDIT_SYMBOLS` names the helpers that reach it. + // The full tree reads 3 of 49 so nobody sees the sentence today, and a scan of any subset with no + // impersonation route in it brings the sentence straight back. + it("does not claim the webapp has no audit helper when nothing reached one", () => { + const unaudited = scanFile( + "api.v1.auth.tokens.ts", + `import { prisma } from "~/db.server"; + export async function action() { + return json(await prisma.token.create({ data: {} })); + }` + )!; + const out = renderTerminal(buildReport([unaudited], [])); + expect(out).toContain("AUDIT 0 of 1 sensitive mutations record an actor. 1 without one."); + expect(out).not.toContain("No audit helper exists"); + }); + it("says nothing about audit when no sensitive mutation was found", () => { const plain = scanFile( "resources.things.ts", diff --git a/internal-packages/observability-map/src/report/terminal.ts b/internal-packages/observability-map/src/report/terminal.ts index 843501b68c4..dc59e205342 100644 --- a/internal-packages/observability-map/src/report/terminal.ts +++ b/internal-packages/observability-map/src/report/terminal.ts @@ -50,18 +50,29 @@ export function unknownSuppressionLines(report: MapReport): string[] { ); } -/** The AUDIT figure, shared with `prComment.ts` so both renderers say the same thing. Null when - * there is nothing to report, i.e. no sensitive mutation exists. */ +/** + * The AUDIT figure, shared with `prComment.ts` so both renderers say the same thing. Null when + * there is nothing to report, i.e. no sensitive mutation exists. + * + * One shape for every count, and no branch on the count, because the branch is what carried the + * bug. A zero used to print "No audit helper exists in the webapp", which is false: the helper + * exists and `AUDIT_SYMBOLS` names it, `apps/webapp/app/models/admin.server.ts` writes + * `prisma.impersonationAuditLog.create(...)`, and `webappSymbols.test.ts` proves those symbols + * resolve. The count was already correct, so the sentence was the only wrong thing and it is gone + * rather than reworded. A zero here means nothing reached the helper, which is what + * "0 of N record an actor" already says. + * + * The full-tree scan reads 3 of 49 today, so the zero branch is not taken and nobody sees it. It is + * one `--routes=` away from being taken, and one reshaped impersonation route away on the full + * tree, which is why removing it beats leaving it unreachable. + */ export function auditLine(report: MapReport): string | null { const { sensitiveMutations, withAudit } = report.auditGap; if (sensitiveMutations === 0) return null; - // The closing sentence is a claim about the codebase, so it is only made when the figure in - // front of it supports it. It was printed unconditionally, including next to a non-zero count. - const gap = - withAudit === 0 - ? " No audit helper exists in the webapp." - : ` ${sensitiveMutations - withAudit} without one.`; - return `AUDIT ${withAudit} of ${sensitiveMutations} sensitive mutations record an actor.${gap}`; + return ( + `AUDIT ${withAudit} of ${sensitiveMutations} sensitive mutations record an actor. ` + + `${sensitiveMutations - withAudit} without one.` + ); } /** The CONTEXT figure, shared with `prComment.ts`. Null when nothing is applicable. */ @@ -83,14 +94,21 @@ export function contextLine(report: MapReport): string | null { * The DELEGATED lines, shared with `prComment.ts`. Empty when every route's body is in its own * file. Worded as a shortfall rather than a note: these routes left the denominator and no check * looked at any of them. + * + * `limit` caps how many file names are named, for the caller that has a size limit to respect. The + * count in front of the list is always the full one, so a capped line still reports the real + * shortfall and only shortens the evidence. The terminal passes no limit and prints them all. */ -export function delegatedLines(report: MapReport): string[] { +export function delegatedLines(report: MapReport, limit = Infinity): string[] { if (report.delegating.length === 0) return []; const n = report.delegating.length; + const shown = report.delegating.slice(0, limit); + const tail = n > shown.length ? `, and ${n - shown.length} more` : ""; return [ `DELEGATED ${n} route${n === 1 ? "" : "s"} keep${n === 1 ? "s" : ""} the body in another ` + `module, so nothing here was checked and ${n === 1 ? "it is" : "they are"} out of the score: ` + - report.delegating.join(", "), + shown.join(", ") + + tail, ]; } diff --git a/internal-packages/observability-map/src/scan.test.ts b/internal-packages/observability-map/src/scan.test.ts index 458c6328e81..70efae6eb33 100644 --- a/internal-packages/observability-map/src/scan.test.ts +++ b/internal-packages/observability-map/src/scan.test.ts @@ -2711,6 +2711,77 @@ describe("scanFile: the signals auth-scope reads", () => { expect(ep!.loaderScopesByCaller).toBe(false); }); + // Round E item 3. Neither of the two conditions above constrains the CALLEE, so a log line + // carrying the caller's id satisfied a tenant-scoping security check. Cheaper to write than the + // dead object, and unlike the dead object it survives review, because a log line is real code + // somebody wants. + it("does not read a caller id handed to a log call as a scope", () => { + const logger = scanFile( + "api.v1.orgs.ts", + `${PAT} + export const loader = createActionPATApiRoute({}, async ({ user }) => { + logger.error("create failed", { userId: user.id }); + return json(await prisma.org.findMany({ where: { slug: "x" } })); + });` + ); + expect(logger!.loaderScopesByCaller).toBe(false); + + // A different logger family: `LOGGER_CALLEE` does not match `console.warn`, so this is the + // second sink rather than a restatement of the first. + const console_ = scanFile( + "api.v1.orgs.ts", + `${PAT} + export const loader = createActionPATApiRoute({}, async ({ user }) => { + console.warn("create failed", { userId: user.id }); + return json(await prisma.org.findMany({ where: { slug: "x" } })); + });` + ); + expect(console_!.loaderScopesByCaller).toBe(false); + }); + + // The refusal is made at the call, not at the property, so burying the id under the depth of + // nesting a real filter has does not get it past. + it("does not read a caller id nested inside a log call's payload as a scope", () => { + const ep = scanFile( + "api.v1.orgs.ts", + `${PAT} + export const loader = createActionPATApiRoute({}, async ({ user }) => { + logger.info("looking up", { where: { OR: [{ userId: user.id }] } }); + return json(await prisma.org.findMany({ where: { slug: "x" } })); + });` + ); + expect(ep!.loaderScopesByCaller).toBe(false); + }); + + // The response body is the other sink that takes the same object and cannot narrow a read. + it("does not read a caller id handed to a response serializer as a scope", () => { + const ep = scanFile( + "api.v1.orgs.ts", + `${PAT} + export const loader = createActionPATApiRoute({}, async ({ user }) => { + return typedjson({ userId: user.id }); + });` + ); + expect(ep!.loaderScopesByCaller).toBe(false); + }); + + // The direction that matters more than any of the above: refusing the log line must not accuse a + // handler that also runs the query. This is the shape on the real tree today, + // `engine.v1.dev.runs.$runFriendlyId.snapshots.$snapshotFriendlyId.attempts.start.ts`, which logs + // the environment id and reads with it, and which must keep its pass. + it("still reads a real query filter written beside a log call", () => { + const ep = scanFile( + "api.v1.orgs.ts", + `${PAT} + export const loader = createActionPATApiRoute({}, async ({ authentication }) => { + const run = await runStore.findRun({ runtimeEnvironmentId: authentication.environment.id }); + if (!run) logger.error("no run", { environmentId: authentication.environment.id }); + return json(run); + });` + ); + expect(ep!.loaderScopesByCaller).toBe(true); + }); + // A resource's owner is not the caller. it("does not read another object's userId as a scope", () => { const ep = scanFile( diff --git a/internal-packages/observability-map/src/scan.ts b/internal-packages/observability-map/src/scan.ts index 7cd8e5caaa5..c868fc142d1 100644 --- a/internal-packages/observability-map/src/scan.ts +++ b/internal-packages/observability-map/src/scan.ts @@ -96,21 +96,61 @@ const CALLER_ID_FIELD = /^(id|userId|user|memberId|orgMemberId|createdBy|createdByUserId|environmentId|runtimeEnvironmentId|organizationId|orgId|projectId)$/; /** - * Whether the object literal holding this property is handed to a call, through any depth of - * nesting: `findMany({ where: { members: { some: { userId } } } })` is, and + * Callees that are handed the caller's id and cannot narrow a read with it: the log line and the + * response body. Both take the very `{ userId: user.id }` object a query filter takes, so crediting + * them let one log statement clear `auth-scope` for a whole export. That is cheaper than the + * actor-argument residual `checks/authScope.ts` discloses, and it lands on the one check whose + * purpose is catching cross-org exposure. + * + * The shape is already in the tree rather than hypothetical: `engine.v1.dev.runs...attempts.start` + * writes `logger.error("...", { environmentId: authentication.environment.id })` beside the + * `runStore.findRun` that earns that export its credit honestly. Loggers account for 13 of the + * caller-id sites under `apps/webapp/app/routes` and the two response serializers for 2 more. + * + * A denylist of sinks rather than an allowlist of query callees, and that is a measurement rather + * than a preference. 72 distinct callees are handed a caller id across the route tree, running from + * `prisma.project.findFirst` through `presenter.call` and `new DeleteProjectService().call` to bare + * `regenerateApiKey` and `resolveOrganizationForApiUser`. No name pattern separates those from + * `sendToPlain`, so an allowlist would accuse whichever route named its helper next, and a wrong + * accusation is the failure this check cannot afford. Refusing the sinks that are known not to + * scope shrinks the residual without pretending to close it: `someHelper({ userId: user.id })` that + * ignores its argument still credits, which needs types the scanner does not have. + */ +const NON_SCOPING_CALLEE = /(^|\.)console\.[A-Za-z_$][\w$]*$|^(json|typedjson|defer)$/; + +/** + * Whether a callee could plausibly narrow a read with the object it is handed. A callee with no + * readable name of its own is credited: refusing it would ACCUSE the route, and under-crediting the + * constraint beats accusing a route that is fine. + */ +function couldScopeAQuery(callee: ts.Expression): boolean { + const text = calleeText(callee); + if (text === null) return true; + return !LOGGER_CALLEE.test(text) && !NON_SCOPING_CALLEE.test(text); +} + +/** + * Whether the object literal holding this property is handed to a call that could scope a query, + * through any depth of nesting: `findMany({ where: { members: { some: { userId } } } })` is, and * `const unused = { userId };` is not. Arrays count, so `{ OR: [{ userId }] }` still reaches its * call. * - * What this refuses is a filter built and dropped. What it does NOT refuse is a filter built and - * handed to a call that ignores it: `String({ userId: user.id });` reads as scoping, the same way - * `try { String(0); }` reads as error handling, and for the same reason. Knowing whether the callee - * uses the argument needs types the scanner does not have. + * Two things are refused. A filter built and dropped, which is the dead-object shape + * `dead-caller-scope-object` and `dead-caller-scope-userid` cover. And a filter handed to a callee + * that provably cannot read with it, which is `NON_SCOPING_CALLEE` and which the corpus entry + * `log-caller-scope-userid` covers at tree scale. + * + * What this does NOT refuse is a filter handed to a named call that ignores it: + * `String({ userId: user.id });` reads as scoping, the same way `try { String(0); }` reads as error + * handling, and for the same reason. Knowing whether an arbitrary callee uses the argument needs + * types the scanner does not have. */ -function isHandedToACall(property: ts.PropertyAssignment): boolean { +function isHandedToAScopingCall(property: ts.PropertyAssignment): boolean { let node: ts.Node = property; for (let parent = node.parent; parent; node = parent, parent = node.parent) { if (ts.isCallExpression(parent) || ts.isNewExpression(parent)) { - return parent.arguments?.some((a) => a === node) === true; + if (parent.arguments?.some((a) => a === node) !== true) return false; + return couldScopeAQuery(parent.expression); } if ( !ts.isObjectLiteralExpression(parent) && @@ -147,7 +187,9 @@ function isHandedToACall(property: ts.PropertyAssignment): boolean { * halves of that shape. * * So the property NAME has to be an identity field, and the object it sits in has to be handed to a - * call. See `isHandedToACall` for what that does and does not refuse. + * call that could scope a query. The third condition is what stops `logger.error("create failed", + * { userId: user.id })`, written anywhere in a builder-wrapped handler, clearing the check for that + * export. See `isHandedToAScopingCall` for what that does and does not refuse. */ function scopesByCallerIn(fns: Iterable): boolean { let found = false; @@ -159,7 +201,7 @@ function scopesByCallerIn(fns: Iterable): boolean { CALLER_ID_FIELD.test(node.name.text) ) { const path = propertyPath(node.initializer); - if (path !== null && CALLER_ID_PATH.test(path) && isHandedToACall(node)) { + if (path !== null && CALLER_ID_PATH.test(path) && isHandedToAScopingCall(node)) { found = true; return; } From 32c301db3a3bd76ff6afc89d5f4c628e04fc847b Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Mon, 3 Aug 2026 02:22:11 +0100 Subject: [PATCH 097/106] ci(observability-map): stop the nightly corpus depending on the paths filter `needs: changes` carries an implicit success() that outranks the event test in the `if`, so a failed or skipped filter job silently skipped the nightly mutation corpus and the tree-drift scan stopped without saying so. Adding a status-check function to the `if` drops the implicit success() and lets the event test decide alone. The filter job now runs only for pull requests, the sole path that reads its output. Pull request gating is unchanged. --- .github/workflows/observability-map.yml | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/.github/workflows/observability-map.yml b/.github/workflows/observability-map.yml index 48508374231..cb4a82a4ab9 100644 --- a/.github/workflows/observability-map.yml +++ b/.github/workflows/observability-map.yml @@ -26,6 +26,10 @@ jobs: # evaluates it once per workflow. This narrows it again for the corpus job alone. changes: name: 🔍 Which paths moved + # Only the pull request path reads this job's output. On a schedule the action has no base to + # diff, warns that `before` is missing and reports the files in the last commit on main, which + # nothing then consults. Skipping it there keeps the nightly off a job it does not need. + if: github.event_name == 'pull_request' runs-on: warp-ubuntu-latest-x64-2x outputs: package: ${{ steps.filter.outputs.package }} @@ -63,7 +67,20 @@ jobs: mutation-corpus: name: 🧬 Mutation corpus needs: changes - if: github.event_name != 'pull_request' || needs.changes.outputs.package == 'true' + # `!cancelled()` is here for the nightly, not for tidiness. `needs` carries an implicit + # success() on the job it names, and that implicit test outranks the `||` below: with a plain + # condition, a `changes` job that failed or was skipped skips this one, so the nightly would + # stop scanning for tree drift and report nothing about having stopped. A status-check function + # in the `if` is what drops the implicit success(), so the event test below decides alone. + # `!cancelled()` rather than `always()` because `cancel-in-progress` above is a real path and a + # superseded run should not finish this job. + # + # Pull request behaviour is deliberately unchanged: on a PR a failed `changes` leaves + # `needs.changes.outputs.package` empty, so the corpus still skips. The nightly is the backstop + # for that, which is the same trade the paths gate already makes for routes-only pull requests. + if: >- + !cancelled() && + (github.event_name != 'pull_request' || needs.changes.outputs.package == 'true') runs-on: warp-ubuntu-latest-x64-4x steps: - name: ⬇️ Checkout repo From f06e723eab9fd4b6ae45482aee2e8e877853659d Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Mon, 3 Aug 2026 02:22:33 +0100 Subject: [PATCH 098/106] chore(observability-map): declare the ES2020 lib the tests already need The tests call String.prototype.matchAll, an ES2020 library feature, while `lib` said ES2019. Typecheck passed anyway because @types/node v24 declares `/// `, so the program already contained es2020. No behaviour changes; the declaration now states the requirement instead of resting on a transitive reference. --- internal-packages/observability-map/tsconfig.json | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/internal-packages/observability-map/tsconfig.json b/internal-packages/observability-map/tsconfig.json index 61669556f76..ea5663ae806 100644 --- a/internal-packages/observability-map/tsconfig.json +++ b/internal-packages/observability-map/tsconfig.json @@ -1,7 +1,11 @@ { "compilerOptions": { "target": "ES2019", - "lib": ["ES2019"], + // ES2020 rather than the ES2019 most sibling packages use, because the tests call + // String.prototype.matchAll. It already resolved: @types/node carries a + // `/// `, so the program had es2020 whatever this line said. Stating + // it here stops the requirement resting on a transitive reference from a types package. + "lib": ["ES2020"], "module": "ESNext", "moduleResolution": "Bundler", "esModuleInterop": true, From 5ad60264f39e15fa19bd3dfa120c32774701c398 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Mon, 3 Aug 2026 02:56:57 +0100 Subject: [PATCH 099/106] fix(observability-map): keep the test task out of the turbo cache The suite scans apps/webapp/app, packages/plugins/src, internal-packages/rbac/src and four files under .github/workflows, none of which turbo hashes for this package, so turbo run test replayed a pass recorded before those trees changed. Measured rather than argued: a route file with a syntax error fails the suite under vitest, and the same tree came back FULL TURBO in 301ms with the failure cached away as a success. inputs was tried and rejected rather than assumed unworkable. Turbo 1.x does accept .. in an input glob, and ../../apps/webapp/app/** did bust the cache on a route change, but it replaces the default file set instead of adding to it, so the same config silently dropped this package's own vitest.config.ts from the hash. The $TURBO_DEFAULT$ token that would add rather than replace is turbo 2.x only and matches nothing on 1.10.3. Costs about 23s per run and no CI job pays it: the dedicated workflow calls vitest without turbo, and unit-tests-internal.yml runs cold. Reported by Devin on #4455. --- .../observability-map/src/integration.test.ts | 31 +++++++++++++++++++ .../observability-map/turbo.json | 31 +++++++++++++++++++ 2 files changed, 62 insertions(+) create mode 100644 internal-packages/observability-map/turbo.json diff --git a/internal-packages/observability-map/src/integration.test.ts b/internal-packages/observability-map/src/integration.test.ts index 03e8ca78b2c..5c673a6632b 100644 --- a/internal-packages/observability-map/src/integration.test.ts +++ b/internal-packages/observability-map/src/integration.test.ts @@ -213,6 +213,37 @@ describe("the package's tests are wired into the gate", () => { }); }); +/** + * The third road into this suite, after the two workflows above: `turbo run test`, which is what + * `pnpm run test` and `pnpm run test:internal` reach it by. + * + * Turbo keys a task's cache on the package's own files. This suite's real inputs are mostly not + * its own files, they are `apps/webapp/app`, `packages/plugins/src`, `internal-packages/rbac/src` + * and the workflow files read above, so turbo happily replayed a pass recorded before a route + * changed. Measured rather than argued: a route file with a syntax error in it makes + * `parses every route file and produces a report inside a wide band` fail under vitest, and the + * same tree came back FULL TURBO in 301ms with the failure cached away as a success. + * + * So the task is uncacheable, and this asserts that, because the config is one line and reads like + * a performance oversight to anyone who does not know what the suite reads. + * + * What this does not assert is the rejected alternative. `inputs` can name `../../apps/webapp/...` + * and does bust the cache, but it replaces turbo 1.x's default file set instead of adding to it, + * so it drops the package's own files from the hash unless every one of them is listed too; that + * was measured the same way, by editing `vitest.config.ts` and getting FULL TURBO back. The + * reasoning lives in `turbo.json` next to the config it explains. + */ +describe("the third road in, turbo", () => { + it("keeps its test task out of the turbo cache", () => { + const config = readFileSync(resolve(__dirname, "../turbo.json"), "utf8"); + // Comments are legal in turbo.json and this one carries the reasoning, so strip them to parse. + const pipeline = JSON.parse(config.replace(/^\s*\/\/.*$/gm, "")) as { + pipeline?: { test?: { cache?: boolean } }; + }; + expect(pipeline.pipeline?.test?.cache).toBe(false); + }); +}); + describe("counting candidates independently of the scanner", () => { // The counter is only worth having if it disagrees with the scanner somewhere. It does: the // scanner attributes nothing to a nested file that is not `route.ts`/`route.tsx`, and the diff --git a/internal-packages/observability-map/turbo.json b/internal-packages/observability-map/turbo.json new file mode 100644 index 00000000000..42fa065651d --- /dev/null +++ b/internal-packages/observability-map/turbo.json @@ -0,0 +1,31 @@ +{ + "$schema": "https://turborepo.org/schema.json", + "extends": ["//"], + "pipeline": { + // Uncacheable on purpose. This suite's real inputs live outside the package: it scans + // `apps/webapp/app`, `packages/plugins/src`, `internal-packages/rbac/src` and four files under + // `.github/workflows`. The root `test` task keys the cache on this package's own files, so a + // cached pass replayed after a route change broke the scan, which is a guard that stops + // guarding while still reading green. + // + // `inputs` was tried and rejected rather than assumed unworkable. Turbo 1.x does accept `..` + // in an input glob, and `../../apps/webapp/app/**` did bust the cache on a route change. What + // it also does is replace the default file set rather than add to it, so the same config + // silently dropped this package's own `vitest.config.ts` from the hash: editing it replayed a + // cached pass. The `$TURBO_DEFAULT$` token that adds rather than replaces is turbo 2.x only + // and matches nothing on the 1.10.3 here. Trading a stale-on-routes hole for a + // stale-on-own-config hole is not a fix, and an inputs list mirroring what the tests read is + // one more thing that drifts out of sync without saying so. + // + // Cost is about 23s per run, and no CI job pays it: the dedicated workflow calls vitest + // without turbo, and `unit-tests-internal.yml` runs cold. + // + // `it("keeps its test task out of the turbo cache")` in `src/integration.test.ts` fails if + // this is removed. + "test": { + "dependsOn": ["^build"], + "outputs": [], + "cache": false + } + } +} From 00b78cb63761d8c40440f6fc17817ca9eb437ba3 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Mon, 3 Aug 2026 02:57:29 +0100 Subject: [PATCH 100/106] ci(observability-map): write scan reports with --out instead of capturing stdout Both scan steps redirected pnpm --filter ... exec stdout into files the renderer JSON.parses. pnpm takes its recursive path under --filter and some versions announce 'Scope: N of M workspace projects' on it; one such line in head.json fails the parse and degrades every run to the stale-report comment, which is a permanent quiet failure rather than a loud one. It does not reproduce on the 10.33.2 the workflow pins, which was checked, so this closes the class rather than a reproduction: the scanner writes its own file and stdout is left to be log output. The -s guard keeps the partial dance honest now the redirect no longer creates the file, so a scanner that exits 0 without writing takes the stale-report branch instead of failing the mv and turning the job red. The render step still captures stdout, since prCommentCli has no --out and a banner there puts a stray line in a markdown comment rather than breaking a parse. Reported by Devin on #4455. --- .github/workflows/observability-map.yml | 23 ++++++++++--- .../observability-map/src/integration.test.ts | 34 +++++++++++++++++++ 2 files changed, 53 insertions(+), 4 deletions(-) diff --git a/.github/workflows/observability-map.yml b/.github/workflows/observability-map.yml index cb4a82a4ab9..6043ce15976 100644 --- a/.github/workflows/observability-map.yml +++ b/.github/workflows/observability-map.yml @@ -149,10 +149,24 @@ jobs: # Guarded rather than allowed to fail: this job must never block a pull request. The failure # is not swallowed either, the render step below turns a missing head report into a comment # saying so, because a swallowed failure with no comment is the outcome nobody wants. + # + # `--out` rather than a stdout redirect, so nothing a tool decides to print can end up inside + # the document `prCommentCli` parses. `pnpm --filter` takes its recursive path and some + # versions announce `Scope: N of M workspace projects` on the way; that line landing in + # head.json would fail the parse and degrade every run to the stale-report comment, which is + # a permanent quiet failure rather than a loud one. It does not reproduce on the 10.33.2 + # pinned above, so this closes the class rather than a reproduction: the file is written by + # the process that owns it and stdout is left to be log output. Held by + # `it("let the scanner write its own report rather than capturing stdout")` in + # `internal-packages/observability-map/src/integration.test.ts`. + # + # `-s` keeps the partial dance honest now the redirect no longer creates the file: a scanner + # that exits 0 without writing takes the else branch and the stale-report comment, instead of + # failing the `mv` and turning the job red. - name: 🔎 Scan head run: | - if pnpm --filter @internal/observability-map exec tsx src/cli.ts --json --no-write \ - > /tmp/head.json.partial; then + if pnpm --filter @internal/observability-map exec tsx src/cli.ts \ + --out=/tmp/head.json.partial && [ -s /tmp/head.json.partial ]; then mv /tmp/head.json.partial /tmp/head.json else rm -f /tmp/head.json /tmp/head.json.partial @@ -168,8 +182,9 @@ jobs: - name: 🔎 Scan base with the head's scanner run: | if git worktree add /tmp/base-tree ${{ github.event.pull_request.base.sha }} \ - && pnpm --filter @internal/observability-map exec tsx src/cli.ts --json --no-write \ - --routes=/tmp/base-tree/apps/webapp/app/routes > /tmp/base.json; then + && pnpm --filter @internal/observability-map exec tsx src/cli.ts \ + --routes=/tmp/base-tree/apps/webapp/app/routes --out=/tmp/base.json \ + && [ -s /tmp/base.json ]; then : else echo "-" > /tmp/base.json || true diff --git a/internal-packages/observability-map/src/integration.test.ts b/internal-packages/observability-map/src/integration.test.ts index 5c673a6632b..4be9d22ecc7 100644 --- a/internal-packages/observability-map/src/integration.test.ts +++ b/internal-packages/observability-map/src/integration.test.ts @@ -117,6 +117,40 @@ describe("the report workflow's two readers of the comment lookup", () => { }); }); +/** + * Both scan steps used to capture the scanner's stdout with a shell redirect, into files the + * renderer then `JSON.parse`s. Anything else reaching stdout therefore corrupted the report: + * `pnpm --filter` takes its recursive path, and some versions of pnpm announce + * `Scope: N of M workspace projects` on it. A single line of that in head.json fails the parse, + * and the workflow degrades to the stale-report comment on every run, quietly and permanently. + * + * It does not reproduce on the 10.33.2 the workflow pins, which was checked. What is asserted here + * is the shape that cannot have the bug at all rather than the version that happens not to: the + * scanner writes its own file through `--out`, so stdout carries log output and nothing else. + * The same is not yet true of the render step, which has no `--out` to reach for; a banner there + * puts a stray line in a markdown comment instead of breaking a parse, so it is left alone. + */ +describe("the report workflow's two scan steps", () => { + const WORKFLOW = resolve(__dirname, "../../../.github/workflows/observability-map.yml"); + + it("let the scanner write its own report rather than capturing stdout", () => { + const scans = readFileSync(WORKFLOW, "utf8") + .split(/^ {6}- name: /m) + .slice(1) + .filter((step) => step.startsWith("🔎 Scan")); + expect(scans).toHaveLength(2); + + for (const step of scans) { + // The `if ...; then` condition only, which is where the scanner runs. The else branch writes + // `echo "-" > /tmp/base.json`, a redirect of the workflow's own making that has nothing to + // do with capturing the scanner, and an earlier version of this test failed on it. + const command = step.split("; then")[0]!; + expect(command).toMatch(/--out=\S+\.json/); + expect(command).not.toMatch(/>\s*\S*\.json/); + } + }); +}); + /** * The gating half of the same problem. A test job that nothing waits for is decoration, and the * first attempt at this was exactly that: a job inside `observability-map.yml`, which reads well From 26c405967e01dd234530281ef8ba4e10041affee Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Mon, 3 Aug 2026 03:31:34 +0100 Subject: [PATCH 101/106] ci(observability-map): widen the obsmap filter to what the suite reads The filter watched apps/webapp/app/routes only, which was narrower than the suite's actual coupling. webappSymbols.test.ts walks all of apps/webapp/app and fails when a guard, sensitive or audit symbol stops resolving, so renaming e.g. requireUserId in app/services/session.server.ts matched the webapp filter and nothing else: no job ran this suite and the break landed on main, or on the next unrelated internal-packages PR. integration.test.ts also asserts on the text of observability-map.yml, which no filter watched at all, so editing the report workflow alone ran nothing. Derived the coupling set from the code rather than from the comment. Outside its own directory the suite reads apps/webapp/app (whole tree for symbols, the route subtree for the scan), packages/plugins/src, internal-packages/rbac/src, and four workflow files. packages/plugins/src and internal-packages/rbac/src stay out: internal already matches packages/** and internal-packages/**, and unit-tests-internal.yml runs the same suite, so listing them here would run it twice. A new test pins that reasoning. Cost, over the last 400 commits on main: 31% touch routes, 52% touch apps/webapp/app, so the job fires on roughly half of PRs instead of roughly a third. It is the cheap one, a single 4x runner with no containers and no database. Reported by Devin on #4455. --- .github/workflows/pr_checks.yml | 33 +++++++++---- .../unit-tests-observability-map.yml | 5 +- .../observability-map/src/integration.test.ts | 49 +++++++++++++++---- 3 files changed, 65 insertions(+), 22 deletions(-) diff --git a/.github/workflows/pr_checks.yml b/.github/workflows/pr_checks.yml index ddedf3559f2..e3df9b416f2 100644 --- a/.github/workflows/pr_checks.yml +++ b/.github/workflows/pr_checks.yml @@ -82,20 +82,33 @@ jobs: - 'pnpm-lock.yaml' - 'pnpm-workspace.yaml' - 'turbo.json' - # Routes only, and that is the whole reason this filter exists: one test in - # @internal/observability-map scans the live route tree, and without this a webapp-only - # PR that renames a route merges green and the break lands on main, or on the next - # unrelated internal-packages PR. + # The whole webapp app tree, not just its routes, and that is the whole reason this + # filter exists. Two tests in @internal/observability-map read it: integration.test.ts + # scans the live route tree, and webappSymbols.test.ts walks all of apps/webapp/app and + # fails when a guard, sensitive or audit symbol stops resolving. Routes-only was this + # filter's own bug: renaming e.g. requireUserId in app/services/session.server.ts + # matched `webapp` and nothing else, so no job ran the suite and the break landed on + # main, or on the next unrelated internal-packages PR. # - # The package's own paths are deliberately NOT here. `internal` above already matches - # `internal-packages/**`, and `unit-tests-internal.yml` runs `turbo run test --filter - # "@internal/*"`, which picks up @internal/observability-map and runs the same vitest - # suite including that route-tree test. Listing the package here as well ran the suite - # twice on every PR touching it, which was this filter's own doing. + # The cost of the wider set, measured over the last 400 commits on main: 31% touch + # routes, 52% touch apps/webapp/app, so the job goes from firing on roughly a third of + # PRs to roughly a half. It is the cheap one -- a single 4x runner, no containers, no + # database, no prisma generate -- which is what makes that affordable. + # + # observability-map.yml is here because integration.test.ts asserts on its text and no + # other filter watches it, so editing the report workflow alone ran nothing at all. + # + # Deliberately NOT here: this package's own paths, and packages/plugins/src and + # internal-packages/rbac/src, the other two trees webappSymbols.test.ts reads. + # `internal` above already matches `internal-packages/**` and `packages/**`, and + # `unit-tests-internal.yml` runs `turbo run test --filter "@internal/*"`, which picks up + # @internal/observability-map and runs the same vitest suite. Listing them here as well + # ran the suite twice on every PR touching them, which was this filter's own doing. obsmap: - - 'apps/webapp/app/routes/**' + - 'apps/webapp/app/**' - '.github/workflows/pr_checks.yml' - '.github/workflows/unit-tests-observability-map.yml' + - '.github/workflows/observability-map.yml' - 'package.json' - 'pnpm-lock.yaml' - 'pnpm-workspace.yaml' diff --git a/.github/workflows/unit-tests-observability-map.yml b/.github/workflows/unit-tests-observability-map.yml index d71aecf985e..eebf1ef971c 100644 --- a/.github/workflows/unit-tests-observability-map.yml +++ b/.github/workflows/unit-tests-observability-map.yml @@ -36,7 +36,8 @@ jobs: - name: 📥 Download deps run: pnpm install --frozen-lockfile - # One test in this suite scans apps/webapp/app/routes, which is why the filter that gates - # this workflow watches those paths as well as the package's own. + # This suite reads apps/webapp/app (the route tree for the scan, the whole app tree for the + # symbol check) and the report workflow's text, which is why the filter that gates this + # workflow watches all of those and not only the routes folder. - name: 🧪 Run tests run: pnpm --filter @internal/observability-map run test diff --git a/internal-packages/observability-map/src/integration.test.ts b/internal-packages/observability-map/src/integration.test.ts index 4be9d22ecc7..8351dc333dc 100644 --- a/internal-packages/observability-map/src/integration.test.ts +++ b/internal-packages/observability-map/src/integration.test.ts @@ -14,8 +14,12 @@ import { buildReport } from "./score.js"; import { SCORED_CHECK_IDS } from "./checks/index.js"; /** - * The one deliberate coupling to `apps/webapp/app/routes` in the suite. Everything else, including - * the CLI tests, runs against a fixture tree of this package's own making. + * This file's deliberate coupling to `apps/webapp/app/routes`. It is not the suite's only one, and + * saying it was is what let the paths filter be written for this file alone: + * `webappSymbols.test.ts` walks all of `apps/webapp/app`, `packages/plugins/src` and + * `internal-packages/rbac/src`, and `mutationCorpus.test.ts` scans the route tree behind an env + * gate. Everything else, including the CLI tests, runs against a fixture tree of this package's own + * making. * * The coupling is acceptable because nothing here names a route or a count: the scan must not * crash, the entry point count must sit inside a wide band, and parse failures must be zero. Those @@ -23,12 +27,16 @@ import { SCORED_CHECK_IDS } from "./checks/index.js"; * cannot tell us, since a fixture only contains shapes somebody thought to write down. * * What runs this for a webapp pull request is `.github/workflows/unit-tests-observability-map.yml`, - * called from `pr_checks.yml` behind an `obsmap` paths filter covering `apps/webapp/app/routes/**`, - * and listed in the `all-checks` aggregate so it actually gates. A pull request touching this - * PACKAGE reaches the same test by the other road: `internal` already matches - * `internal-packages/**`, and `unit-tests-internal.yml` runs `turbo run test --filter "@internal/*"` - * over this package too. So both directions are gated, and neither is gated twice; the `obsmap` - * filter used to name the package as well, which ran this suite twice on every PR touching it. + * called from `pr_checks.yml` behind an `obsmap` paths filter covering the whole of + * `apps/webapp/app` plus the report workflow, and listed in the `all-checks` aggregate so it + * actually gates. The filter is wider than this file's own coupling because the suite's is: + * `webappSymbols.test.ts` walks all of `apps/webapp/app`, and the describes below read + * `observability-map.yml`. A pull request touching this PACKAGE, or `packages/plugins/src` or + * `internal-packages/rbac/src`, reaches the same test by the other road: `internal` already matches + * `internal-packages/**` and `packages/**`, and `unit-tests-internal.yml` runs `turbo run test + * --filter "@internal/*"` over this package too. So every direction is gated and none is gated + * twice; the `obsmap` filter used to name the package as well, which ran this suite twice on every + * PR touching it. * * Two shapes were tried and rejected on the way here. Widening `pr_checks.yml`'s `internal` filter * to the route paths ran all eighteen internal packages, twelve shards with postgres, clickhouse, @@ -179,9 +187,30 @@ describe("the package's tests are wired into the gate", () => { expect(read(REUSABLE)).toContain("workflow_call"); }); - it("watches the live route tree, which is the only thing the internal filter misses", () => { + // Round 5. The filter watched `apps/webapp/app/routes/**` while the suite reads more than that, + // so a rename outside the routes folder matched only `webapp`, ran no job that runs this suite, + // and broke the build for whoever pushed next. Asserted as the whole set the suite reads and no + // other filter covers, rather than as the one path that prompted the filter, because the routes + // entry looked complete right up until it wasn't. + it("watches every webapp path the internal filter misses, not just the routes folder", () => { const filter = read(PR_CHECKS).split(" obsmap:")[1]!.split(" cli:")[0]!; - expect(filter).toContain("'apps/webapp/app/routes/**'"); + // webappSymbols.test.ts walks all of apps/webapp/app, not just routes. + expect(filter).toContain("'apps/webapp/app/**'"); + // The report workflow, whose text the two describes above assert on. No other filter names it. + expect(filter).toContain("'.github/workflows/observability-map.yml'"); + }); + + // The other two trees webappSymbols.test.ts reads. They belong to `internal`, not here, and this + // pins the reason so the obvious-looking addition has to argue with a test first. + it("leaves the two non-webapp roots it reads to the internal filter", () => { + const text = read(PR_CHECKS); + const obsmap = text.split(" obsmap:")[1]!.split(" cli:")[0]!; + expect(obsmap).not.toContain("packages/plugins"); + expect(obsmap).not.toContain("internal-packages/rbac"); + + const internal = text.split(" internal:")[1]!.split(" # ")[0]!; + expect(internal).toContain("'packages/**'"); + expect(internal).toContain("'internal-packages/**'"); }); // Round E item 6. `internal` matches `internal-packages/**` and `unit-tests-internal.yml` runs From 4ea20b03f1f20362eb5347b75b33ad1a6b5eb842 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Mon, 3 Aug 2026 03:32:28 +0100 Subject: [PATCH 102/106] test(observability-map): size the real-tree timeouts for shard contention The 30s and 60s per-test timeouts on the two real-tree tests were chosen on an idle machine, and the suite also runs inside unit-tests-internal.yml, which executes turbo run test --filter "@internal/*" as twelve concurrent shard processes on one runner. The 30s one does flake under that. Measured on an 8-core box. This file alone at load average 0.9: 6.3-6.4s for the scan, 10.8-11.2s for the sweep, both well above the 1.6-2.6s the old comment claimed. Two batches of twelve concurrent copies on those same 8 cores: 24.2-34.0s for the scan and 27.6-39.7s for the sweep, with one of the first twelve dying on "Test timed out in 30000ms". Twelve processes over 8 cores is 1.5 per core where the 32-vCPU runner is 0.375, so the reproduction is harsher than CI, which is why it is the thing to size against. Both now use one 120s constant, which is 3x the worst contended run measured. 60s was the other candidate and is not enough: the sweep already reached 39.7s. Neither test asserts anything about elapsed time, so the number is a hang detector rather than a performance budget, and the docstring says so. Reported by Devin on #4455. --- .../observability-map/src/integration.test.ts | 91 ++++++++++++------- 1 file changed, 60 insertions(+), 31 deletions(-) diff --git a/internal-packages/observability-map/src/integration.test.ts b/internal-packages/observability-map/src/integration.test.ts index 8351dc333dc..593ff311d69 100644 --- a/internal-packages/observability-map/src/integration.test.ts +++ b/internal-packages/observability-map/src/integration.test.ts @@ -326,41 +326,70 @@ describe("counting candidates independently of the scanner", () => { }); }); +/** + * Timeout for the two real-tree tests, which do not fit the suite's 10s default: the first runs a + * ts.Program per route file for the parse diagnostics and walks the tree a second time to count + * candidates, the second scans the tree twice and re-scans every source with the suppression + * directive prepended. + * + * It is a hang detector and nothing else. Neither test asserts anything about how long the scan + * takes, so a number tight enough to be a performance budget would only be a way to fail on a busy + * runner, and a performance budget that flakes gets the whole suite marked unreliable. + * + * The old 30s and 60s were chosen on an idle machine and the 30s one does flake. Measured on an + * 8-core box, this file alone at load average 0.9: 6.3-6.4s for the scan, 10.8-11.2s for the sweep. + * Twenty-four runs of it as two batches of twelve concurrent copies on those same 8 cores: + * 24.2-34.0s for the scan and 27.6-39.7s for the sweep, with one of the first twelve dying on + * "Test timed out in 30000ms". That contention is not hypothetical: + * `.github/workflows/unit-tests-internal.yml` runs `turbo run test --filter "@internal/*"` as + * twelve concurrent shard processes on one runner, and this file executes inside one of them. + * + * The local reproduction is deliberately harsher than CI, which is why it is the thing to size + * against: twelve processes over 8 cores is 1.5 per core where the 32-vCPU runner is 0.375. 120s is + * 3x the worst contended run measured and about 11x the idle sweep. 60s was the other candidate and + * is not enough on those numbers: the sweep already reached 39.7s, which is 1.5x, and a margin that + * thin on a machine nobody controls is how the 30s got here. + */ +const TREE_SCAN_TIMEOUT = 120_000; + describe("scanning the real webapp routes", () => { - it("parses every route file and produces a report inside a wide band", () => { - const { entryPoints, parseFailures } = scanDirectory(ROUTES); - - expect(parseFailures).toEqual([]); - expect(entryPoints.length).toBeGreaterThan(100); - expect(entryPoints.length).toBeLessThan(countRouteModuleFiles(ROUTES)); - - const report = buildReport(entryPoints, parseFailures); - expect(report.global).toBeGreaterThanOrEqual(0); - expect(report.global).toBeLessThanOrEqual(100); - expect(Object.keys(report.byFamily).length).toBeGreaterThan(1); - // A full scan of the real tree runs a `ts.Program` per file for the parse diagnostics, and - // `countRouteModuleFiles` walks the tree a second time. That is 1.6 to 2.6 seconds on an idle - // machine and it flaked past the suite's 10s default under parallel load. Budgeted rather than - // left marginal, the same way the exhaustive sweep below is. - }, 30_000); + it( + "parses every route file and produces a report inside a wide band", + () => { + const { entryPoints, parseFailures } = scanDirectory(ROUTES); + + expect(parseFailures).toEqual([]); + expect(entryPoints.length).toBeGreaterThan(100); + expect(entryPoints.length).toBeLessThan(countRouteModuleFiles(ROUTES)); + + const report = buildReport(entryPoints, parseFailures); + expect(report.global).toBeGreaterThanOrEqual(0); + expect(report.global).toBeLessThanOrEqual(100); + expect(Object.keys(report.byFamily).length).toBeGreaterThan(1); + }, + TREE_SCAN_TIMEOUT + ); // A1, exhaustive: every scored check suppressed on every real route, zero behavioural change. // The old measured-from-visible logic took this global from 17 to 33 and measured from 412 to // 176, because every entry whose only applicable checks were suppressed dropped out of the // mean. Measured must not move: every entry point that had something applicable still does. - it("suppressing every scored check on every real route does not raise the global", () => { - const { entryPoints, parseFailures } = scanDirectory(ROUTES); - const before = buildReport(entryPoints, parseFailures); - - const directive = SCORED_CHECK_IDS.map( - (id) => `// obs-map-disable ${id} -- exhaustive sweep\n` - ).join(""); - const suppressed = entryPoints.map((ep) => scanFile(ep.fileName, directive + ep.source)!); - const after = buildReport(suppressed, parseFailures); - - expect(after.measured).toBe(before.measured); - expect(after.unmeasured).toBe(before.unmeasured); - expect(after.global).not.toBeGreaterThan(before.global!); - // Two full tree scans plus a re-scan of every source, which does not fit the suite default. - }, 60_000); + it( + "suppressing every scored check on every real route does not raise the global", + () => { + const { entryPoints, parseFailures } = scanDirectory(ROUTES); + const before = buildReport(entryPoints, parseFailures); + + const directive = SCORED_CHECK_IDS.map( + (id) => `// obs-map-disable ${id} -- exhaustive sweep\n` + ).join(""); + const suppressed = entryPoints.map((ep) => scanFile(ep.fileName, directive + ep.source)!); + const after = buildReport(suppressed, parseFailures); + + expect(after.measured).toBe(before.measured); + expect(after.unmeasured).toBe(before.unmeasured); + expect(after.global).not.toBeGreaterThan(before.global!); + }, + TREE_SCAN_TIMEOUT + ); }); From 30fdb02384f38f2550f29a9ece028395b41d0e37 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Mon, 3 Aug 2026 04:45:46 +0100 Subject: [PATCH 103/106] fix(observability-map): attribute auth-boundary guards per export Every input auth-boundary read was entry-point-wide, so one guarded export spoke for the whole file: calleeNames is the union of both bodies, checkedCallees was too, and usesBuilder was an OR over both initializer callees. A file whose loader called requireUser and whose action called nothing read as guarded in the body, and a createLoaderApiRoute loader authenticated a hand-written action beside it. This is the same defect auth-scope was fixed for a round earlier, in its sibling check. scanFile now splits calleeNames, calleeTexts, checkedCallees, statementCount and hasTryCatch per export, filled from one push site each so the union and the split cannot drift apart. usesBuilder had no other caller and is gone. routeExports is the single enumeration of a file's exports, shared with auth-scope, which had grown its own [loader, action] literal. Triviality had to follow, or the fix trades a false pass for a false accusation: naive per-export attribution moved auth.github.ts and auth.google.ts to fail, both being a one-line redirect-stub loader beside a guarded action that the entry-point-wide rule called non-trivial. isTrivial is now one rule over two views. The per-export view matches the side-effect hints against that export's own callee paths: the whole file is defeatable (the corpus's log-caller-scope-userid puts the word logger in the file and un-excuses the untouched loader) and nothing at all guts the check (five fixtures go from fail to not-applicable, because calleeNames keeps only a call's last segment and prisma.x.findMany reads as findMany). login.mfa's action verifies a TOTP or recovery code, which is a login-surface proof of possession like the verify* guards already listed, so it joins them rather than being accused once its loader stops speaking for it. Real tree unmoved: global 19, 62 auth-boundary applicable, 59 passing, no route changing any check. scan.ts also picks up routeModuleFiles here, shared with the corpus harness, because it sits in the same hunk as the per-export return shape. --- .../src/checks/authBoundary.ts | 91 ++++++-- .../observability-map/src/checks/authScope.ts | 37 ++-- .../src/checks/errorClassification.ts | 7 - .../src/checks/index.test.ts | 193 ++++++++++++++++ .../observability-map/src/integration.test.ts | 35 ++- .../observability-map/src/routeExports.ts | 71 ++++++ .../observability-map/src/scan.test.ts | 104 ++++++++- .../observability-map/src/scan.ts | 207 +++++++++++++----- .../observability-map/src/triviality.ts | 98 +++++++-- .../observability-map/src/types.ts | 55 ++++- 10 files changed, 766 insertions(+), 132 deletions(-) create mode 100644 internal-packages/observability-map/src/routeExports.ts diff --git a/internal-packages/observability-map/src/checks/authBoundary.ts b/internal-packages/observability-map/src/checks/authBoundary.ts index 13327f582fb..4abbeb7dc73 100644 --- a/internal-packages/observability-map/src/checks/authBoundary.ts +++ b/internal-packages/observability-map/src/checks/authBoundary.ts @@ -1,14 +1,16 @@ import type { CheckResult, EntryPoint } from "../types.js"; import { classifySensitivity } from "../sensitivity.js"; -import { isTrivial } from "../triviality.js"; -import { usesBuilder } from "./errorClassification.js"; +import { routeExports, type ExportName, type RouteExport } from "../routeExports.js"; +import { isTrivialExport } from "../triviality.js"; +import { BUILDERS } from "./errorClassification.js"; const ID = "auth-boundary"; /** - * The guard helpers this webapp actually has, matched against `calleeNames`, which is scoped to the - * loader/action bodies and follows one hop into a same-file helper. A guard the route only imports - * and never calls does not count. + * The guard helpers this webapp actually has, matched against the calling export's own + * `loaderCalleeNames`/`actionCalleeNames`, each scoped to that export's handlers and following one + * hop into a same-file helper. A guard the route only imports and never calls does not count, and + * neither does one the OTHER export calls. * * A name list rather than the three patterns it replaces, because all three over-matched and this * is the one check where a false pass hides a security gap: @@ -66,16 +68,22 @@ export const GUARDS = new Set([ // Local helpers, each declared inside the one route that uses it. "authenticateAdmin", "authenticatePlainRequest", - // Proof of possession: a callback URL carrying an HMAC is authenticated by checking that HMAC. + // Proof of possession: a callback URL carrying an HMAC is authenticated by checking that HMAC, + // and a login-surface second factor is authenticated by checking the code presented. + // `login.mfa`'s action is the second half of a login, so like `authenticate` above it establishes + // identity from the credential rather than requiring an already authenticated caller. It reached + // this list when per-export attribution stopped its loader's `isAuthenticated` speaking for it. "verifyHttpCallbackHash", "verifyWebhook", "verifyUserActorToken", + "verifyTotpForLogin", + "verifyRecoveryCodeForLogin", ]); /** * Guards that answer with null instead of throwing. Calling one is not evidence of a boundary, - * because the route is free to ignore the answer, so these are only credited when the body - * demonstrably reads what they returned (`EntryPoint.checkedCallees`). + * because the route is free to ignore the answer, so these are only credited when THAT EXPORT's + * handlers demonstrably read what they returned (`EntryPoint.loaderCheckedCallees`). * * The distinction is the whole reason this set is separate from `GUARDS`. `requireUserId` redirects * on its own, so calling it IS the boundary; `getUserId` hands back `string | null` and a route @@ -85,10 +93,43 @@ export const GUARDS = new Set([ * rather than something a hand-read established once. * * What it still cannot see is whether the test that reads the answer guards anything. See - * `EntryPoint.checkedCallees` for the exact shape of that residual. + * `EntryPoint.loaderCheckedCallees` for the exact shape of that residual. */ export const SOFT_GUARDS = new Set(["getUser", "getUserId"]); +type GuardedExport = { name: ExportName; guarded: boolean; how: string; export: RouteExport }; + +/** + * The exports this file declares, each with its own verdict. + * + * Per export, because the exposure is per export, and this is the same defect `auth-scope` was + * fixed for one round earlier. Every input here was entry-point-wide: `calleeNames` is the union of + * both bodies, `checkedCallees` was too, and `usesBuilder` was an OR over the two initializer + * callees. So a file whose loader called `requireUser` and whose action called nothing read as + * "guarded in the body", and a file whose loader was `createLoaderApiRoute(...)` credited its + * hand-written action with the builder's authentication. Three inputs, one bug, and it is a false + * PASS on the one check where that hides a security gap. + * + * `routeExports` lists only the exports the file actually declares, so an export that calls nothing + * at all is judged rather than skipped: an empty body is exactly the unguarded case. It is shared + * with `auth-scope`, which grew its own copy of the same `[loader, action]` literal. + */ +function guardedExports(ep: EntryPoint): GuardedExport[] { + return routeExports(ep).map((e) => { + const verdict = (guarded: boolean, how: string) => ({ name: e.name, guarded, how, export: e }); + if (e.initializerCallee !== null && BUILDERS.has(e.initializerCallee)) { + return verdict(true, "authenticated by the builder"); + } + if (e.calleeNames.some((n) => GUARDS.has(n))) { + return verdict(true, "guarded in the body"); + } + if (e.checkedCallees.some((n) => SOFT_GUARDS.has(n))) { + return verdict(true, "resolves the caller and reads the answer"); + } + return verdict(false, ""); + }); +} + /** * Whether a route that handles credentials, tokens or money checks who is asking. * @@ -123,26 +164,30 @@ export const authBoundary = { if (!sensitivity.sensitive) { return { id: ID, status: "not-applicable", detail: "not sensitive" }; } - if (usesBuilder(ep)) { - return { id: ID, status: "pass", detail: "authenticated by the builder" }; - } - if (ep.calleeNames.some((n) => GUARDS.has(n))) { - return { id: ID, status: "pass", detail: "guarded in the body" }; - } - if (ep.checkedCallees.some((n) => SOFT_GUARDS.has(n))) { - return { id: ID, status: "pass", detail: "resolves the caller and reads the answer" }; - } - if (isTrivial(ep)) { + // Never empty: `scanFile` returns null unless the file declares a loader or an action. + const exports = guardedExports(ep); + const guarded = exports.filter((e) => e.guarded); + // Triviality excuses per export, matching the attribution: the reasoning below is about one + // body being the place a guard would have to be, and reading it entry-point-wide let a busy + // action make a redirect-stub loader answerable for a guard it has nothing to guard. + const accused = exports.filter((e) => !e.guarded && !isTrivialExport(e.export)); + if (accused.length > 0) { return { id: ID, - status: "not-applicable", - detail: "cannot verify: no privileged work in the body, any guard is behind an import", + status: "fail", + detail: `sensitive (${sensitivity.reasons.join(", ")}) with no auth guard in the body: ${accused + .map((e) => e.name) + .join(", ")}`, }; } + if (guarded.length > 0) { + const how = [...new Set(guarded.map((e) => e.how))].join(" and "); + return { id: ID, status: "pass", detail: how }; + } return { id: ID, - status: "fail", - detail: `sensitive (${sensitivity.reasons.join(", ")}) with no auth guard in the body`, + status: "not-applicable", + detail: "cannot verify: no privileged work in the body, any guard is behind an import", }; }, }; diff --git a/internal-packages/observability-map/src/checks/authScope.ts b/internal-packages/observability-map/src/checks/authScope.ts index 2aa4a74dc6d..832928fa620 100644 --- a/internal-packages/observability-map/src/checks/authScope.ts +++ b/internal-packages/observability-map/src/checks/authScope.ts @@ -1,5 +1,6 @@ import type { CheckResult, EntryPoint } from "../types.js"; import { classifySensitivity } from "../sensitivity.js"; +import { routeExports } from "../routeExports.js"; import { BUILDERS } from "./errorClassification.js"; const ID = "auth-scope"; @@ -12,30 +13,22 @@ type BuilderExport = { name: string; callee: string; scoped: boolean; why: strin * Per export, because the exposure is per export. `authorization` is declared on the builder call * one export made, and a caller filter is written in the handler one export runs, so neither says * anything about the other half of the file. + * + * The enumeration itself is `routeExports`, shared with `auth-boundary`, which had to be given the + * same per-export treatment a round later and wrote a second copy of this literal to get it. */ function builderExports(ep: EntryPoint): BuilderExport[] { - const all = [ - { - name: "loader", - callee: ep.loaderInitializerCallee, - authorization: ep.loaderBuilderOptions.includes("authorization"), - filters: ep.loaderScopesByCaller, - }, - { - name: "action", - callee: ep.actionInitializerCallee, - authorization: ep.actionBuilderOptions.includes("authorization"), - filters: ep.actionScopesByCaller, - }, - ]; - return all - .filter((e) => e.callee !== null && BUILDERS.has(e.callee)) - .map((e) => ({ - name: e.name, - callee: e.callee!, - scoped: e.authorization || e.filters, - why: e.authorization ? "an authorization gate" : "a filter on the caller's identity", - })); + return routeExports(ep) + .filter((e) => e.initializerCallee !== null && BUILDERS.has(e.initializerCallee)) + .map((e) => { + const authorization = e.builderOptions.includes("authorization"); + return { + name: e.name, + callee: e.initializerCallee!, + scoped: authorization || e.scopesByCaller, + why: authorization ? "an authorization gate" : "a filter on the caller's identity", + }; + }); } /** diff --git a/internal-packages/observability-map/src/checks/errorClassification.ts b/internal-packages/observability-map/src/checks/errorClassification.ts index 91d86c15088..34cfce33d34 100644 --- a/internal-packages/observability-map/src/checks/errorClassification.ts +++ b/internal-packages/observability-map/src/checks/errorClassification.ts @@ -118,13 +118,6 @@ function swallows(clause: CatchEvidence): boolean { return !decides(clause) && !inert(clause); } -export function usesBuilder(ep: EntryPoint): boolean { - return ( - (ep.loaderInitializerCallee !== null && BUILDERS.has(ep.loaderInitializerCallee)) || - (ep.actionInitializerCallee !== null && BUILDERS.has(ep.actionInitializerCallee)) - ); -} - /** * Who decides what a failure means, and on what evidence. * diff --git a/internal-packages/observability-map/src/checks/index.test.ts b/internal-packages/observability-map/src/checks/index.test.ts index 84b330b180f..f0e80af528d 100644 --- a/internal-packages/observability-map/src/checks/index.test.ts +++ b/internal-packages/observability-map/src/checks/index.test.ts @@ -1833,6 +1833,199 @@ describe("auth-boundary: the guard accept-list", () => { }); }); +/** + * Per-export attribution. Every input `auth-boundary` reads used to be entry-point-wide, so one + * guarded export spoke for the whole file. Each `it` here goes green on the entry-point-wide + * version of exactly one of those inputs, which is why they are separate cases rather than one. + */ +describe("auth-boundary: a guard credits only the export that calls it", () => { + const TOKENS = `import { requireUserId } from "~/services/session.server"; + import { prisma } from "~/db.server";`; + + it("fails an unguarded action beside a loader that calls a guard", () => { + const r = run( + "auth-boundary", + "api.v1.tokens.ts", + `${TOKENS} + export async function loader({ request }) { + const userId = await requireUserId(request); + return json(await prisma.token.findMany({ where: { userId } })); + } + export async function action({ request }) { + const body = await request.json(); + await prisma.token.deleteMany({ where: { id: body.id } }); + return json({ ok: true }); + }` + ); + expect(r.status).toBe("fail"); + expect(r.detail).toContain("action"); + }); + + it("fails an unguarded loader beside an action that calls a guard", () => { + const r = run( + "auth-boundary", + "api.v1.tokens.ts", + `${TOKENS} + export async function loader({ params }) { + const tokens = await prisma.token.findMany({ where: { orgId: params.orgId } }); + return json({ tokens }); + } + export async function action({ request }) { + const userId = await requireUserId(request); + await prisma.token.deleteMany({ where: { userId } }); + return json({ ok: true }); + }` + ); + expect(r.status).toBe("fail"); + expect(r.detail).toContain("loader"); + }); + + // The soft-guard arm reads its own export's checked-callee list for the same reason. + it("fails an unguarded action beside a loader that reads what getUserId returned", () => { + const r = run( + "auth-boundary", + "api.v1.tokens.ts", + `import { getUserId } from "~/services/session.server"; + import { prisma } from "~/db.server"; + export async function loader({ request }) { + const userId = await getUserId(request); + if (!userId) return redirect("/login"); + return json(await prisma.token.findMany({ where: { userId } })); + } + export async function action({ request }) { + const body = await request.json(); + await prisma.token.deleteMany({ where: { id: body.id } }); + return json({ ok: true }); + }` + ); + expect(r.status).toBe("fail"); + expect(r.detail).toContain("action"); + }); + + // The builder arm. `usesBuilder` was an OR over both initializer callees, so a builder on one + // export authenticated a hand-written handler on the other. + it("fails a hand-written action beside a builder-wrapped loader", () => { + const r = run( + "auth-boundary", + "api.v1.tokens.ts", + `import { createLoaderApiRoute } from "~/services/routeBuilders/apiBuilder.server"; + import { prisma } from "~/db.server"; + export const loader = createLoaderApiRoute({}, async ({ authentication }) => { + return json(await prisma.token.findMany({ where: { userId: authentication.userId } })); + }); + export async function action({ request }) { + const body = await request.json(); + await prisma.token.deleteMany({ where: { id: body.id } }); + return json({ ok: true }); + }` + ); + expect(r.status).toBe("fail"); + expect(r.detail).toContain("action"); + }); + + it("passes when both exports call a guard of their own", () => { + const r = run( + "auth-boundary", + "api.v1.tokens.ts", + `${TOKENS} + export async function loader({ request }) { + const userId = await requireUserId(request); + return json(await prisma.token.findMany({ where: { userId } })); + } + export async function action({ request }) { + const userId = await requireUserId(request); + await prisma.token.deleteMany({ where: { userId } }); + return json({ ok: true }); + }` + ); + expect(r.status).toBe("pass"); + }); + + // One handler serving both exports guards both, which is the dominant API-route shape. + it("passes a shared builder handler that both exports resolve to", () => { + const r = run( + "auth-boundary", + "api.v1.tokens.ts", + `import { createActionApiRoute } from "~/services/routeBuilders/apiBuilder.server"; + export const { action, loader } = createActionApiRoute({}, async ({ authentication }) => { + return json({ userId: authentication.userId }); + });` + ); + expect(r.status).toBe("pass"); + }); + + /** + * The damper on the attribution, and the reason it is not simply "accuse every unguarded export". + * `auth.github.ts` and `auth.google.ts` are this shape: per export the loader is unguarded, and + * an entry-point-wide triviality rule calls the file non-trivial because the ACTION is not. Both + * routes went pass to fail on the real tree until `isTrivialExport` existed. + */ + it("reports not-applicable for a redirect-stub loader beside a guarded action", () => { + const r = run( + "auth-boundary", + "auth.github.ts", + `import { authenticator } from "~/services/auth.server"; + export let loader = () => redirect("/login"); + export let action = async ({ request }) => { + const url = new URL(request.url); + const safeRedirect = sanitizeRedirectPath(url.searchParams.get("redirectTo"), "/"); + return await authenticator.authenticate("github", request, { + successRedirect: safeRedirect, + failureRedirect: "/login", + }); + };` + ); + expect(r.status).toBe("pass"); + expect(r.detail).toBe("guarded in the body"); + }); + + /** + * The per-export excuse must read the export's own body and not the file's text. It read + * `ep.source` first, and `log-caller-scope-userid` in the mutation corpus, which prepends + * `logger.error(...)` to every body, put the word `logger` in this file and turned the untouched + * loader from excused into accused. A five-minute corpus run is the wrong place to catch that. + */ + it("does not un-excuse a redirect-stub loader because the file mentions a logger", () => { + const r = run( + "auth-boundary", + "auth.github.ts", + `import { authenticator } from "~/services/auth.server"; + import { logger } from "~/services/logger.server"; + export let loader = () => redirect("/login"); + export let action = async ({ request }) => { + logger.error("obs-map", { userId: request.userId }); + return await authenticator.authenticate("github", request, { + successRedirect: "/", + failureRedirect: "/login", + }); + };` + ); + expect(r.status).toBe("pass"); + }); + + it("fails an export whose own body does real work unguarded", () => { + const r = run( + "auth-boundary", + "auth.github.ts", + `import { authenticator } from "~/services/auth.server"; + import { prisma } from "~/db.server"; + export let loader = async ({ params }) => { + const org = await prisma.organization.findFirst({ where: { slug: params.slug } }); + const members = await prisma.orgMember.findMany({ where: { orgId: org.id } }); + return json({ org, members }); + }; + export let action = async ({ request }) => { + return await authenticator.authenticate("github", request, { + successRedirect: "/", + failureRedirect: "/login", + }); + };` + ); + expect(r.status).toBe("fail"); + expect(r.detail).toContain("loader"); + }); +}); + // C1b. A builder authenticates the request; it does not necessarily scope it. `authorization` is // optional on every one of them and the RBAC gate only runs when it is declared. describe("auth-scope", () => { diff --git a/internal-packages/observability-map/src/integration.test.ts b/internal-packages/observability-map/src/integration.test.ts index 593ff311d69..99b6f5ed851 100644 --- a/internal-packages/observability-map/src/integration.test.ts +++ b/internal-packages/observability-map/src/integration.test.ts @@ -9,7 +9,7 @@ import { } from "node:fs"; import { tmpdir } from "node:os"; import { join, resolve } from "node:path"; -import { scanDirectory, scanFile } from "./scan.js"; +import { isScannableFile, scanDirectory, scanFile } from "./scan.js"; import { buildReport } from "./score.js"; import { SCORED_CHECK_IDS } from "./checks/index.js"; @@ -59,7 +59,7 @@ function countRouteModuleFiles(dir: string): number { count += countRouteModuleFiles(join(dir, entry.name)); continue; } - if (entry.isFile() && /\.tsx?$/.test(entry.name) && !entry.name.endsWith(".d.ts")) count++; + if (entry.isFile() && isScannableFile(entry.name)) count++; } return count; } @@ -370,6 +370,37 @@ describe("scanning the real webapp routes", () => { TREE_SCAN_TIMEOUT ); + /** + * The per-export split against the entry-point-wide union it came from, on every real route. + * `scan.test.ts` pins the same property on fixtures; this is the version that sees the shapes + * nobody thought to write down, and the two representations only stay honest while both hold. + */ + it( + "every callee name is attributed to an export that exists", + () => { + const { entryPoints } = scanDirectory(ROUTES); + const orphaned: string[] = []; + const unattributed: string[] = []; + + for (const ep of entryPoints) { + const attributed = new Set([...ep.loaderCalleeNames, ...ep.actionCalleeNames]); + for (const name of new Set(ep.calleeNames)) { + if (!attributed.has(name)) unattributed.push(`${ep.fileName}: ${name}`); + } + const union = new Set(ep.calleeNames); + for (const name of attributed) { + if (!union.has(name)) orphaned.push(`${ep.fileName}: ${name}`); + } + if (!ep.hasLoader) expect(ep.loaderCalleeNames).toEqual([]); + if (!ep.hasAction) expect(ep.actionCalleeNames).toEqual([]); + } + + expect(unattributed).toEqual([]); + expect(orphaned).toEqual([]); + }, + TREE_SCAN_TIMEOUT + ); + // A1, exhaustive: every scored check suppressed on every real route, zero behavioural change. // The old measured-from-visible logic took this global from 17 to 33 and measured from 412 to // 176, because every entry whose only applicable checks were suppressed dropped out of the diff --git a/internal-packages/observability-map/src/routeExports.ts b/internal-packages/observability-map/src/routeExports.ts new file mode 100644 index 00000000000..a53b2ccf979 --- /dev/null +++ b/internal-packages/observability-map/src/routeExports.ts @@ -0,0 +1,71 @@ +import type { EntryPoint } from "./types.js"; + +export type ExportName = "loader" | "action"; + +/** + * One export of a route file, carrying that export's own evidence and nothing from the other one. + * + * Every field here has an entry-point-wide twin on `EntryPoint`, and reaching for the twin is the + * mistake this type exists to make hard. `calleeNames` is the union of both bodies, `hasTryCatch` + * is true if either has one, and `statementCount` counts both. + */ +export type RouteExport = { + name: ExportName; + /** Callee of the initializer call this export is assigned from, if any. */ + initializerCallee: string | null; + /** Top-level keys of the object literal passed to that call. */ + builderOptions: string[]; + /** Callees inside this export's handlers, and inside same-file helpers they call. */ + calleeNames: string[]; + /** The same calls as whole dotted paths, so `prisma.thing.findFirst` keeps its receiver. */ + calleeTexts: string[]; + /** Callees whose answer those handlers demonstrably read. */ + checkedCallees: string[]; + /** Whether those handlers narrow a query by the caller's own id. */ + scopesByCaller: boolean; + statementCount: number; + hasTryCatch: boolean; +}; + +/** + * The exports this route file declares, in `loader`, `action` order. + * + * One enumeration for the whole package, because two checks asking the same per-export question + * each grew their own. `auth-scope`'s `builderExports` and `auth-boundary`'s `guardedExports` were + * hand-maintained `[loader, action]` literals in adjacent files, reading the same six + * `loaderX`/`actionX` field pairs, with different tests for whether an export was there at all: + * one used `hasLoader`/`hasAction` and the other inferred it from a non-null initializer callee. + * Adding a seventh per-export fact meant editing both, and this whole branch is a record of what + * happens when a rule lives in two places and only one gets the fix. + * + * Absent exports are not returned, so a caller never has to remember to filter them: the shape of + * the bug in `auth-boundary` was crediting an export for something the other one did, and a list + * that only contains real exports is one fewer way to write it. + */ +export function routeExports(ep: EntryPoint): RouteExport[] { + const all: RouteExport[] = [ + { + name: "loader", + initializerCallee: ep.loaderInitializerCallee, + builderOptions: ep.loaderBuilderOptions, + calleeNames: ep.loaderCalleeNames, + calleeTexts: ep.loaderCalleeTexts, + checkedCallees: ep.loaderCheckedCallees, + scopesByCaller: ep.loaderScopesByCaller, + statementCount: ep.loaderStatementCount, + hasTryCatch: ep.loaderHasTryCatch, + }, + { + name: "action", + initializerCallee: ep.actionInitializerCallee, + builderOptions: ep.actionBuilderOptions, + calleeNames: ep.actionCalleeNames, + calleeTexts: ep.actionCalleeTexts, + checkedCallees: ep.actionCheckedCallees, + scopesByCaller: ep.actionScopesByCaller, + statementCount: ep.actionStatementCount, + hasTryCatch: ep.actionHasTryCatch, + }, + ]; + return all.filter((e) => (e.name === "loader" ? ep.hasLoader : ep.hasAction)); +} diff --git a/internal-packages/observability-map/src/scan.test.ts b/internal-packages/observability-map/src/scan.test.ts index 70efae6eb33..32f9d224d7e 100644 --- a/internal-packages/observability-map/src/scan.test.ts +++ b/internal-packages/observability-map/src/scan.test.ts @@ -2795,6 +2795,96 @@ describe("scanFile: the signals auth-scope reads", () => { }); }); +/** + * The per-export split of `calleeNames`. `auth-boundary` reads it, so a name landing in the wrong + * half is a wrong verdict on the one check where a false pass hides a security gap. + */ +describe("scanFile: callee names attributed per export", () => { + it("keeps each export's callees out of the other's list", () => { + const ep = scanFile( + "api.v1.tokens.ts", + `export async function loader({ request }) { + const userId = await requireUserId(request); + return json(await prisma.token.findMany({ where: { userId } })); + } + export async function action({ request }) { + await deleteEverything(request); + return json({ ok: true }); + }` + ); + expect(ep!.loaderCalleeNames).toContain("requireUserId"); + expect(ep!.loaderCalleeNames).not.toContain("deleteEverything"); + expect(ep!.actionCalleeNames).toContain("deleteEverything"); + expect(ep!.actionCalleeNames).not.toContain("requireUserId"); + }); + + it("attributes a same-file helper to the export that calls it", () => { + const ep = scanFile( + "api.v1.tokens.ts", + `async function loadTokens(request) { + const userId = await requireUserId(request); + return prisma.token.findMany({ where: { userId } }); + } + export async function loader({ request }) { return json(await loadTokens(request)); } + export async function action({ request }) { return json(await request.json()); }` + ); + expect(ep!.loaderCalleeNames).toContain("requireUserId"); + expect(ep!.actionCalleeNames).not.toContain("requireUserId"); + }); + + it("attributes a helper both exports call to both of them", () => { + const ep = scanFile( + "api.v1.tokens.ts", + `async function guarded(request) { return requireUserId(request); } + export async function loader({ request }) { return json(await guarded(request)); } + export async function action({ request }) { return json(await guarded(request)); }` + ); + expect(ep!.loaderCalleeNames).toContain("requireUserId"); + expect(ep!.actionCalleeNames).toContain("requireUserId"); + }); + + // One handler, both exports: `const { loader, action } = createActionApiRoute({ handler })`. + it("attributes a handler serving both exports to both of them", () => { + const ep = scanFile( + "api.v1.tokens.ts", + `export const { action, loader } = createActionApiRoute({}, async ({ authentication }) => { + return json(await authenticateApiRequest(authentication)); + });` + ); + expect(ep!.loaderCalleeNames).toContain("authenticateApiRequest"); + expect(ep!.actionCalleeNames).toContain("authenticateApiRequest"); + }); + + // The union the split came from. `calleeNames` stays entry-point-wide for `sensitivity.ts`, + // `triviality.ts` and `audit-trail`, so the two representations have to agree. + it("every callee name is attributed to an export that exists", () => { + const ep = scanFile( + "api.v1.tokens.ts", + `async function shared(request) { return audit(request); } + export async function loader({ request }) { return json(await shared(request)); } + export async function action({ request }) { return json(await mutate(request)); }` + ); + const attributed = new Set([...ep!.loaderCalleeNames, ...ep!.actionCalleeNames]); + expect([...new Set(ep!.calleeNames)].filter((n) => !attributed.has(n))).toEqual([]); + for (const name of attributed) expect(ep!.calleeNames).toContain(name); + }); + + it("counts statements and a try per export as well as entry-point-wide", () => { + const ep = scanFile( + "api.v1.tokens.ts", + `export const loader = () => redirect("/login"); + export async function action({ request }) { + try { return json(await request.json()); } catch (e) { throw e; } + }` + ); + expect(ep!.loaderStatementCount).toBe(1); + expect(ep!.loaderHasTryCatch).toBe(false); + expect(ep!.actionHasTryCatch).toBe(true); + expect(ep!.hasTryCatch).toBe(true); + expect(ep!.statementCount).toBe(ep!.loaderStatementCount + ep!.actionStatementCount); + }); +}); + // Round C ruling 2. A guard that answers with null instead of throwing is only a boundary if the // route reads the answer, so the scan records which callees' results a condition looked at. describe("scanFile: callees whose answer the body read", () => { @@ -2808,7 +2898,7 @@ describe("scanFile: callees whose answer the body read", () => { return json({ email: user.email }); }` ); - expect(ep!.checkedCallees).toContain("getUser"); + expect(ep!.loaderCheckedCallees).toContain("getUser"); }); it("records one whose result a plain if tests", () => { @@ -2821,7 +2911,7 @@ describe("scanFile: callees whose answer the body read", () => { return typedjson({}); }` ); - expect(ep!.checkedCallees).toContain("getUserId"); + expect(ep!.loaderCheckedCallees).toContain("getUserId"); }); it("records one whose result a ternary tests", () => { @@ -2832,7 +2922,7 @@ describe("scanFile: callees whose answer the body read", () => { return user ? json({ ok: true }) : redirect("/login"); }` ); - expect(ep!.checkedCallees).toContain("getUser"); + expect(ep!.loaderCheckedCallees).toContain("getUser"); }); it("does not record a result that is bound and never tested", () => { @@ -2843,7 +2933,7 @@ describe("scanFile: callees whose answer the body read", () => { return json(await prisma.invite.findMany({ where: { email: user.email } })); }` ); - expect(ep!.checkedCallees).not.toContain("getUser"); + expect(ep!.loaderCheckedCallees).not.toContain("getUser"); }); it("does not record a result that is dropped entirely", () => { @@ -2854,7 +2944,7 @@ describe("scanFile: callees whose answer the body read", () => { return json(await prisma.invite.findMany()); }` ); - expect(ep!.checkedCallees).not.toContain("getUser"); + expect(ep!.loaderCheckedCallees).not.toContain("getUser"); }); it("does not record a callee just because a same-named local is tested elsewhere", () => { @@ -2866,7 +2956,7 @@ describe("scanFile: callees whose answer the body read", () => { return json(rows); }` ); - expect(ep!.checkedCallees).toContain("findMany"); - expect(ep!.checkedCallees).not.toContain("getUser"); + expect(ep!.loaderCheckedCallees).toContain("findMany"); + expect(ep!.loaderCheckedCallees).not.toContain("getUser"); }); }); diff --git a/internal-packages/observability-map/src/scan.ts b/internal-packages/observability-map/src/scan.ts index c868fc142d1..2067e60203a 100644 --- a/internal-packages/observability-map/src/scan.ts +++ b/internal-packages/observability-map/src/scan.ts @@ -668,13 +668,11 @@ function containsExit(node: ts.Node): boolean { */ function literalTruth(expr: ts.Expression): boolean | null { const target = unwrap(expr); - if (target.kind === ts.SyntaxKind.TrueKeyword) return true; - if (target.kind === ts.SyntaxKind.FalseKeyword) return false; - if (target.kind === ts.SyntaxKind.NullKeyword) return false; - if (ts.isStringLiteral(target) || ts.isNoSubstitutionTemplateLiteral(target)) { - return target.text !== ""; - } - if (ts.isNumericLiteral(target)) return Number(target.text) !== 0; + // The five bare-literal kinds are `literalValue`'s list, not a second copy of it: this was the + // same five node-kind tests written out three lines above the function that already had them, + // differing only in returning the truthiness rather than the value. + const literal = literalValue(target); + if (literal !== undefined) return Boolean(literal); if (ts.isPrefixUnaryExpression(target) && target.operator === ts.SyntaxKind.ExclamationToken) { const inner = literalTruth(target.operand); return inner === null ? null : !inner; @@ -1395,6 +1393,51 @@ function countFunctionStatements(fn: EntryFunction): number { return countStatements(fn.body.statements); } +type ExportName = "loader" | "action"; + +/** + * The call-site facts a body walk accumulates, kept in one shape so the entry-point-wide totals and + * each export's own totals are filled by the same code rather than by two similar loops. + */ +type BodyFacts = { + calleeNames: string[]; + /** + * The same calls as `calleeNames`, each as its whole dotted path. `calleeName` keeps only the + * last segment, so `prisma.organization.findFirst` arrives in `calleeNames` as `findFirst` and + * the receiver that says WHAT is being called is gone. The per-export triviality rule needs it + * back: `prisma` in the path is how a short body is known to touch the datastore. + */ + calleeTexts: string[]; + /** Locals initialised from a call, by local name. */ + declaredFrom: Map; + /** Every identifier read by an `if`, `while`, `switch` or conditional condition. */ + testedNames: Set; + statementCount: number; + hasTryCatch: boolean; +}; + +function newBodyFacts(): BodyFacts { + return { + calleeNames: [], + calleeTexts: [], + declaredFrom: new Map(), + testedNames: new Set(), + statementCount: 0, + hasTryCatch: false, + }; +} + +/** Callees whose answer these bodies looked at: declared from a call AND read by a condition. */ +function checkedCalleesOf(facts: BodyFacts): string[] { + return [ + ...new Set( + [...facts.declaredFrom] + .filter(([local]) => facts.testedNames.has(local)) + .flatMap(([, callees]) => callees) + ), + ]; +} + type EntryTarget = { hasLoader: boolean; hasAction: boolean; @@ -1616,31 +1659,42 @@ export function scanFile(fileName: string, source: string): EntryPoint | null { if (!target.hasLoader && !target.hasAction) return null; - let statementCount = 0; - let hasTryCatch = false; const callbackCatches: CatchEvidence[] = []; const catches: CatchEvidence[] = []; - const calleeNames: string[] = []; const logCalls: LogCall[] = []; - // Locals initialised from a call, and the names any condition in the body reads. Their - // intersection is `checkedCallees`: callees whose answer the route demonstrably looked at. - const declaredFrom = new Map(); - const testedNames = new Set(); - const collectTested = (node: ts.Node) => { - if (ts.isIdentifier(node)) testedNames.add(node.text); - ts.forEachChild(node, collectTested); + const wholeEntry = newBodyFacts(); + const byExport: Record = { + loader: newBodyFacts(), + action: newBodyFacts(), }; const localFunctions = collectLocalFunctions(sf); // A body that delegates to a same-file helper does the work in that helper, so the helper's // statements, try/catch and callees belong to the entry point. One hop only: a helper's own // helpers are not followed, and the visited set stops a cycle and any double counting. + // + // The helper's callees belong to whichever EXPORTS reach it, too, which is what `helperOwners` + // carries. A helper called from both halves of the file is owned by both; the union is taken on + // the second discovery rather than dropped, because `visited` has already queued it by then. const visited = new Set(target.functions); const helpers: EntryFunction[] = []; + const helperOwners = new Map>(); + + const walkBody = (fn: EntryFunction, followHelpers: boolean, owners: ReadonlySet) => { + // One push site feeds the entry-point-wide list and each owning export's list, so + // `calleeNames` and the per-export lists cannot drift apart. See `EntryPoint.loaderCalleeNames`. + const sinks: BodyFacts[] = [wholeEntry]; + for (const owner of owners) sinks.push(byExport[owner]); + const collectTested = (node: ts.Node) => { + if (ts.isIdentifier(node)) for (const s of sinks) s.testedNames.add(node.text); + ts.forEachChild(node, collectTested); + }; - const walkBody = (fn: EntryFunction, followHelpers: boolean) => { - statementCount += countFunctionStatements(fn); + const addStatements = (n: number) => { + for (const sink of sinks) sink.statementCount += n; + }; + addStatements(countFunctionStatements(fn)); if (!fn.body) return; // `inCallback` is true once the walk has entered a per-item iteration callback @@ -1665,13 +1719,13 @@ export function scanFile(fileName: string, source: string): EntryPoint | null { // the mutation corpus is that shape. const visit = (node: ts.Node, inCatch: boolean, inCallback: boolean) => { if (ts.isFunctionLike(node)) { - if (isEntryFunction(node)) statementCount += countFunctionStatements(node); + if (isEntryFunction(node)) addStatements(countFunctionStatements(node)); const entersIterationCallback = inCallback || isIterationCallback(node); ts.forEachChild(node, (child) => visit(child, inCatch, entersIterationCallback)); return; } if (ts.isTryStatement(node)) { - hasTryCatch = true; + for (const sink of sinks) sink.hasTryCatch = true; if (node.catchClause) { // Built the same way for a refused catch as for an own one, so the dead-code defence // and the walk's guaranteed-execution rules apply to both. Which list it lands in is @@ -1699,9 +1753,11 @@ export function scanFile(fileName: string, source: string): EntryPoint | null { if (ts.isCallExpression(initializer)) { const cn = calleeName(initializer.expression); if (cn) { - const existing = declaredFrom.get(node.name.text); - if (existing) existing.push(cn); - else declaredFrom.set(node.name.text, [cn]); + for (const sink of sinks) { + const existing = sink.declaredFrom.get(node.name.text); + if (existing) existing.push(cn); + else sink.declaredFrom.set(node.name.text, [cn]); + } } } } @@ -1715,7 +1771,10 @@ export function scanFile(fileName: string, source: string): EntryPoint | null { const cn = calleeName(node.expression); if (cn) { const text = calleeText(node.expression) ?? cn; - calleeNames.push(cn); + for (const sink of sinks) { + sink.calleeNames.push(cn); + sink.calleeTexts.push(text); + } if (LOGGER_CALLEE.test(text)) { logCalls.push({ @@ -1733,6 +1792,10 @@ export function scanFile(fileName: string, source: string): EntryPoint | null { if (helper && !visited.has(helper)) { visited.add(helper); helpers.push(helper); + helperOwners.set(helper, new Set(owners)); + } else if (helper) { + const already = helperOwners.get(helper); + if (already) for (const owner of owners) already.add(owner); } } } @@ -1742,8 +1805,16 @@ export function scanFile(fileName: string, source: string): EntryPoint | null { visit(fn.body, false, false); }; - for (const fn of target.functions) walkBody(fn, true); - for (const helper of helpers) walkBody(helper, false); + // Every handler is walked exactly once, whichever exports own it, so the entry-point-wide + // statement count and catch list stay single while the per-export lists see it from both sides. + const ownersOf = (fn: EntryFunction): Set => { + const owners = new Set(); + if (target.loaderFunctions.has(fn)) owners.add("loader"); + if (target.actionFunctions.has(fn)) owners.add("action"); + return owners; + }; + for (const fn of target.functions) walkBody(fn, true, ownersOf(fn)); + for (const helper of helpers) walkBody(helper, false, helperOwners.get(helper) ?? new Set()); return { fileName, @@ -1762,31 +1833,76 @@ export function scanFile(fileName: string, source: string): EntryPoint | null { target.actionInitializerCallee === null, loaderScopesByCaller: scopesByCallerIn(target.loaderFunctions), actionScopesByCaller: scopesByCallerIn(target.actionFunctions), - checkedCallees: [ - ...new Set( - [...declaredFrom] - .filter(([local]) => testedNames.has(local)) - .flatMap(([, callees]) => callees) - ), - ], + loaderCheckedCallees: checkedCalleesOf(byExport.loader), + actionCheckedCallees: checkedCalleesOf(byExport.action), importedNames, - calleeNames, - hasTryCatch, + calleeNames: wholeEntry.calleeNames, + loaderCalleeNames: byExport.loader.calleeNames, + actionCalleeNames: byExport.action.calleeNames, + loaderCalleeTexts: byExport.loader.calleeTexts, + actionCalleeTexts: byExport.action.calleeTexts, + hasTryCatch: wholeEntry.hasTryCatch, + loaderHasTryCatch: byExport.loader.hasTryCatch, + actionHasTryCatch: byExport.action.hasTryCatch, catches, callbackCatches, logCalls, - statementCount, + statementCount: wholeEntry.statementCount, + loaderStatementCount: byExport.loader.statementCount, + actionStatementCount: byExport.action.statementCount, }; } const SOURCE_FILE = /\.tsx?$/; -/** Exported so `mutationCorpus.test.ts` materializes exactly the files `scanDirectory` reads. Its - * anti-vacuity thresholds count files and sites the scanner never saw if the two predicates drift. */ +/** + * Whether a file name is one the scanner reads at all. + * + * Exported because three other places ask the same question and each had written its own copy: + * `mutationCorpus.test.ts` materializes exactly the files `scanDirectory` reads, and + * `integration.test.ts` and `webappSymbols.test.ts` walk trees of their own. The corpus's + * anti-vacuity thresholds count files and sites the scanner never saw if those predicates drift, + * and `integration.test.ts`'s `entryPoints.length < countRouteModuleFiles(ROUTES)` stops meaning + * anything if its denominator counts files the scanner skips. + */ export function isScannableFile(fileName: string): boolean { return SOURCE_FILE.test(fileName) && !fileName.endsWith(".d.ts"); } +/** One route module: where to read it, and the name the report and the scan record it under. */ +export type RouteModuleFile = { absolutePath: string; relativeName: string }; + +/** + * The route modules under `dir`: every scannable flat file, plus the `route.ts`/`route.tsx` of each + * immediate subdirectory. + * + * Exported so `mutationCorpus.test.ts` can materialize exactly this set rather than re-deriving it. + * Its `readTree` was a verbatim copy of the walk below; the FILE half of that copy was later + * replaced by a call to `isScannableFile` while the DIRECTORY half stayed duplicated, which is the + * usual way this package's duplicates half-die. A corpus that enumerates a different tree from the + * scanner reports file and site counts for files the scan never reads, and those counts are the + * only thing standing between a mutation that reaches nothing and a green test. + */ +export function routeModuleFiles(dir: string): RouteModuleFile[] { + const files: RouteModuleFile[] = []; + for (const entry of readdirSync(dir, { withFileTypes: true })) { + if (entry.isDirectory()) { + // Flat-route directories hold the route module in `route.ts`/`route.tsx`. + for (const child of readdirSync(join(dir, entry.name), { withFileTypes: true })) { + if (!child.isFile() || (child.name !== "route.ts" && child.name !== "route.tsx")) continue; + files.push({ + absolutePath: join(dir, entry.name, child.name), + relativeName: `${entry.name}/${child.name}`, + }); + } + continue; + } + if (!entry.isFile() || !isScannableFile(entry.name)) continue; + files.push({ absolutePath: join(dir, entry.name), relativeName: entry.name }); + } + return files; +} + export function scanDirectory(dir: string): { entryPoints: EntryPoint[]; parseFailures: string[]; @@ -1810,18 +1926,7 @@ export function scanDirectory(dir: string): { if (ep) entryPoints.push(ep); }; - for (const entry of readdirSync(dir, { withFileTypes: true })) { - if (entry.isDirectory()) { - // Flat-route directories hold the route module in `route.ts`/`route.tsx`. - for (const child of readdirSync(join(dir, entry.name), { withFileTypes: true })) { - if (!child.isFile() || (child.name !== "route.ts" && child.name !== "route.tsx")) continue; - scan(join(dir, entry.name, child.name), `${entry.name}/${child.name}`); - } - continue; - } - if (!entry.isFile() || !isScannableFile(entry.name)) continue; - scan(join(dir, entry.name), entry.name); - } + for (const file of routeModuleFiles(dir)) scan(file.absolutePath, file.relativeName); return { entryPoints, parseFailures }; } diff --git a/internal-packages/observability-map/src/triviality.ts b/internal-packages/observability-map/src/triviality.ts index e36b80dc9b0..15515d1ec36 100644 --- a/internal-packages/observability-map/src/triviality.ts +++ b/internal-packages/observability-map/src/triviality.ts @@ -1,9 +1,10 @@ +import type { RouteExport } from "./routeExports.js"; import type { EntryPoint } from "./types.js"; /** - * Substrings that say the route touches a service, a datastore or the network. Matched against the - * callee names and the whole file, so an import of `prisma` disqualifies the file even when the - * query itself sits somewhere the scanner does not walk. + * Substrings that say the route touches a service, a datastore or the network. Always matched + * against the callee names, and additionally against `TrivialityView.hintText`, which is the whole + * file for the entry-point-wide view and empty for a per-export one. */ const SIDE_EFFECT_HINTS = ["prisma", "logger", "fetch", "$transaction", "redis", "engine"]; @@ -23,6 +24,42 @@ const MAX_CALLS = 3; */ const MAX_STATEMENTS = 3; +/** + * What the rule reads, so the entry-point-wide answer and a single export's answer are the same + * rule over different bodies rather than two rules that can drift. + */ +type TrivialityView = { + statementCount: number; + calleeNames: string[]; + hasTryCatch: boolean; + /** Every builder call in scope of this view. A view with one is never trivial. */ + initializerCallees: (string | null)[]; + /** + * Text to match the side-effect hints against besides the callee names. + * + * The whole file for the entry-point-wide view, so an import of `prisma` disqualifies it even + * when the query sits somewhere the scanner does not walk. For a per-export view it is that + * export's own callee PATHS instead, and the difference is not a convenience: + * + * - The file's text is a fact about the file, so reading it into one export's verdict is the + * per-file-for-per-export substitution this rule exists to damp. It is also defeatable. + * `log-caller-scope-userid` in the mutation corpus prepends `logger.error(...)` to every body; + * with this term file-wide that put the word `logger` in `auth.github.ts` and turned its + * untouched one-line redirect loader from excused into accused, on a rewrite that changed + * nothing the loader does. + * - Emptying it instead is not the answer either, and that was measured: `calleeNames` keeps only + * a call's last segment, so `prisma.orgMember.findMany` reads as `findMany` and a + * three-statement body that queries the datastore matches no hint at all. Five existing + * `auth-boundary` fixtures went from `fail` to `not-applicable`, which is the check being + * switched off rather than fixed. + * + * The callee paths are body-scoped like the first option wants and name the receiver like the + * second needs. Comments and imports are not in them, which is deliberate: everything in this + * view is something the export actually does. + */ + hintText: string; +}; + /** * Nothing to instrument: a body of a statement or two that only redirects, returns a fixed * response, or hands off in a single call. Checks report not-applicable for these rather than @@ -31,22 +68,55 @@ const MAX_STATEMENTS = 3; * Deliberately reluctant. A route wrongly called trivial is exempted and never shows up in the * report again, so every signal that the body might be doing real work rules triviality out: * - * - `statementCount` does not descend into inline callbacks, so a two-statement body can still hold - * a pile of work. `calleeNames` does descend, so the call count catches what the statement count - * misses. + * - `statementCount` counts a nested function's statements but `calleeNames` descends further, into + * the callee of every call at any depth, so the call count still catches bodies the statement + * count reads as short. * - An initializer callee means the route is wrapped in a builder, and the config passed to that * builder (`findResource`, `authorization`) is work the scanner never walks. The visible body is * not the whole route, so we cannot claim it is trivial. * - A try/catch is exactly what the error-classification check reads, so a body with one has an * error path worth reporting on however short it is. */ +function isTrivialView(view: TrivialityView): boolean { + if (view.statementCount > MAX_STATEMENTS) return false; + if (view.calleeNames.length > MAX_CALLS) return false; + if (view.hasTryCatch) return false; + if (view.initializerCallees.some((c) => c !== null)) return false; + + const callees = view.calleeNames.join(" ").toLowerCase(); + const hints = view.hintText.toLowerCase(); + return !SIDE_EFFECT_HINTS.some((h) => callees.includes(h) || hints.includes(h)); +} + +/** The rule over everything the entry point does, both exports and their same-file helpers. */ export function isTrivial(ep: EntryPoint): boolean { - if (ep.statementCount > MAX_STATEMENTS) return false; - if (ep.calleeNames.length > MAX_CALLS) return false; - if (ep.hasTryCatch) return false; - if (ep.loaderInitializerCallee !== null || ep.actionInitializerCallee !== null) return false; - - const callees = ep.calleeNames.join(" ").toLowerCase(); - const source = ep.source.toLowerCase(); - return !SIDE_EFFECT_HINTS.some((h) => callees.includes(h) || source.includes(h)); + return isTrivialView({ + statementCount: ep.statementCount, + calleeNames: ep.calleeNames, + hasTryCatch: ep.hasTryCatch, + initializerCallees: [ep.loaderInitializerCallee, ep.actionInitializerCallee], + hintText: ep.source, + }); +} + +/** + * The same rule over ONE export's handlers. + * + * Needed because a per-export verdict judged against an entry-point-wide triviality rule accuses + * the wrong half of a file. `auth.github.ts` and `auth.google.ts` are + * `export let loader = () => redirect("/login")` beside an action that calls + * `authenticator.authenticate`: per export the loader is unguarded, and the entry-point-wide rule + * calls the file non-trivial because the ACTION is not, so `auth-boundary` accused a one-line + * redirect stub of missing an auth guard. `checks/index.test.ts` pins both directions of that + * ("reports not-applicable for a redirect-stub loader beside a guarded action" and "fails an export + * whose own body does real work unguarded"). + */ +export function isTrivialExport(e: RouteExport): boolean { + return isTrivialView({ + statementCount: e.statementCount, + calleeNames: e.calleeNames, + hasTryCatch: e.hasTryCatch, + initializerCallees: [e.initializerCallee], + hintText: e.calleeTexts.join(" "), + }); } diff --git a/internal-packages/observability-map/src/types.ts b/internal-packages/observability-map/src/types.ts index b75e4714476..f11fb4a4a1f 100644 --- a/internal-packages/observability-map/src/types.ts +++ b/internal-packages/observability-map/src/types.ts @@ -129,31 +129,67 @@ export type EntryPoint = { loaderScopesByCaller: boolean; actionScopesByCaller: boolean; /** - * Callees whose answer the body demonstrably looked at: the call's result was bound to a local - * and some `if`, `while`, `switch` or conditional in the same bodies reads that local. - * `const user = await getUser(request); if (!user) return redirect("/login");` puts `getUser` - * here; a call whose result is dropped, or bound and never tested, does not appear. + * Callees whose answer THIS export's handlers demonstrably looked at: the call's result was bound + * to a local and some `if`, `while`, `switch` or conditional in the same handlers reads that + * local. `const user = await getUser(request); if (!user) return redirect("/login");` puts + * `getUser` here; a call whose result is dropped, or bound and never tested, does not appear. * * Read by `auth-boundary` for the guards that answer with null instead of throwing, where being * called is not evidence that the route acted on the answer. * + * Split per export for the same reason `loaderScopesByCaller` is, and there is no entry-point-wide + * version on purpose: a loader that reads what `getUser` returned says nothing about the action + * beside it, so the union is not a fact any check should be able to reach for. + * * Deliberately coarse. It does not check that the test guards anything, that the local is the one * tested rather than a same-named one in another scope, or that the branch exits: a route * that writes `if (!user) { logger.warn("anonymous"); }` and carries on is credited. It separates * "looked at the answer" from "ignored it", which is the distinction the check needs, and not * "acted correctly on the answer", which it cannot see. */ - checkedCallees: string[]; + loaderCheckedCallees: string[]; + actionCheckedCallees: string[]; /** Named and default imports, file-wide. */ importedNames: string[]; - /** Names of functions called inside the loader/action bodies, or in a same-file helper they call. */ + /** + * Names of functions called inside the loader/action bodies, or in a same-file helper they call. + * + * Entry-point-wide, and read only by the questions that are themselves entry-point-wide: + * `sensitivity.ts` asks what the file touches, `triviality.ts` counts how much the file does, + * `audit-trail` asks whether the file records anything. A question about ONE export's exposure + * must read `loaderCalleeNames`/`actionCalleeNames` instead. `auth-boundary` read this and + * credited a file whose loader called a guard for an action that called none. + */ calleeNames: string[]; + /** + * The same callee names attributed to the export whose handlers made the call. A handler serving + * both exports (`const { loader, action } = createActionApiRoute({ handler })`) contributes to + * both, and a same-file helper contributes to whichever exports reach it. + * + * Every name here appears in `calleeNames` and every name in `calleeNames` appears in at least one + * of these, because all three are filled from one push in `scanFile`. `scan.test.ts` pins that + * ("every callee name is attributed to an export that exists") and `integration.test.ts` pins it + * again across the real route tree, so the split cannot drift away from the union it came from. + */ + loaderCalleeNames: string[]; + actionCalleeNames: string[]; + /** + * The same calls as `loaderCalleeNames`/`actionCalleeNames`, each as its whole dotted path + * (`prisma.organization.findFirst` rather than `findFirst`). Read by the per-export triviality + * rule, which has to know that a three-statement body reaches the datastore; the bare name that + * `auth-boundary` matches guards against throws that receiver away. + */ + loaderCalleeTexts: string[]; + actionCalleeTexts: string[]; /** * Whether a `try` appears in the loader/action bodies, or in a same-file helper they call. Note * that this says a `try`, not a catch: a `try`/`finally` sets it while `catches` stays empty and * every catch-shaped field stays false. Read `catches.length` to ask whether anything is caught. */ hasTryCatch: boolean; + /** The same fact for one export's handlers alone. Read by the per-export triviality rule. */ + loaderHasTryCatch: boolean; + actionHasTryCatch: boolean; /** One entry per catch clause in those bodies, in source order. */ catches: CatchEvidence[]; /** @@ -176,4 +212,11 @@ export type EntryPoint = { * one hop only: work in a helper's own helpers, or in an imported module, is not counted. */ statementCount: number; + /** + * The same count for one export's handlers alone, counted by the same walk. A handler serving both + * exports is counted once in `statementCount` and once in each of these, so the two do not sum to + * the entry point's total and must not be used as though they did. + */ + loaderStatementCount: number; + actionStatementCount: number; }; From 4f80d58267e100d55a4d17e6aa53b4b59fb7f684 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Mon, 3 Aug 2026 04:46:11 +0100 Subject: [PATCH 104/106] fix(observability-map): widen the corpus helper to every export form mutations.ts entryBodies collected exported function declarations and exported const identifiers only, so it missed the object binding pattern (export const { action, loader } = createActionApiRoute(...)), the export clause (const { action } = builder(...); export { action }), and export const action = route.action. That is 36 of the tree's 427 entry points, all of them API routes: every whole-body corpus entry skipped them while the file count suggested otherwise. The scanner has read all four forms since early on, so this was the harness lagging it. No assertion could have noticed. A mutation that reaches fewer routes lowers the score rather than raising it, which is exactly how the suppress-every-check omission hid, so the answer is the same: assert the population. admin.tsx is the one exclusion, named rather than counted, because its handler is a concise arrow with no block for a block wrapper to wrap. wrap-body-in-rethrow goes from 391 files with 36 entry points missed to 426 files, 1020 sites, 1 missed. Widening changes no entry's verdict: the full corpus is 55 passed and 1 expected fail either way, and with the narrow population the only failure is the new population assertion itself. readTree now calls routeModuleFiles rather than keeping its own copy of the directory walk. isScannableFile had already replaced the file half of that copy; the directory half survived. --- .../src/mutationCorpus.test.ts | 74 ++++++--- .../observability-map/src/mutations.ts | 140 ++++++++++++++++-- 2 files changed, 178 insertions(+), 36 deletions(-) diff --git a/internal-packages/observability-map/src/mutationCorpus.test.ts b/internal-packages/observability-map/src/mutationCorpus.test.ts index cc8128c3750..658ce32eca9 100644 --- a/internal-packages/observability-map/src/mutationCorpus.test.ts +++ b/internal-packages/observability-map/src/mutationCorpus.test.ts @@ -1,7 +1,7 @@ -import { existsSync, mkdirSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs"; +import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { dirname, join, resolve } from "node:path"; -import { isScannableFile, scanDirectory } from "./scan.js"; +import { routeModuleFiles, scanDirectory } from "./scan.js"; import { buildReport } from "./score.js"; import { ADDITIVE_IDS, MUTATIONS, type Mutation } from "./mutations.js"; import { CHECKS } from "./checks/index.js"; @@ -62,27 +62,20 @@ const KNOWN_GAPS = new Set(["dead-classifying-try-with-call"]); type SourceFile = { relativeName: string; source: string }; -/** Route modules exactly as `scanDirectory` enumerates them: flat files, plus one `route.ts(x)` per - * directory. Read once; every mutation rewrites this list rather than the tree on disk. */ +/** + * Route modules exactly as `scanDirectory` enumerates them, because it is the same enumeration and + * no longer a copy of it. `isScannableFile` had already replaced the file half of the copy; the + * directory half survived, so "one `route.ts(x)` per immediate subdirectory" was still written + * twice. A harness that reads a different tree from the scanner reports files and sites the scan + * never saw, and those counts are what the thresholds below rest on. + * + * Read once; every mutation rewrites this list rather than the tree on disk. + */ function readTree(dir: string): SourceFile[] { - const files: SourceFile[] = []; - const take = (absolutePath: string, relativeName: string) => { - files.push({ relativeName, source: readFileSync(absolutePath, "utf8") }); - }; - for (const entry of readdirSync(dir, { withFileTypes: true })) { - if (entry.isDirectory()) { - for (const child of readdirSync(join(dir, entry.name), { withFileTypes: true })) { - if (!child.isFile() || (child.name !== "route.ts" && child.name !== "route.tsx")) continue; - take(join(dir, entry.name, child.name), `${entry.name}/${child.name}`); - } - continue; - } - // `scanDirectory`'s own predicate rather than a copy of it. A copy that drifts lets a mutation - // report files and sites the scanner never read, which is what the thresholds below are for. - if (!entry.isFile() || !isScannableFile(entry.name)) continue; - take(join(dir, entry.name), entry.name); - } - return files; + return routeModuleFiles(dir).map((file) => ({ + relativeName: file.relativeName, + source: readFileSync(file.absolutePath, "utf8"), + })); } function materialize(files: SourceFile[]): string { @@ -318,6 +311,43 @@ describeCorpus("mutation corpus over the real route tree", { timeout: ENTRY_TIME baseline = measure(files); }, ENTRY_TIMEOUT_MS); + /** + * The corpus's own population, against the scanner's. + * + * The whole-body entries wrap what `entryBodies` finds, and that helper read two of the four + * export forms `scan.ts` reads. It missed `export const { action, loader } = builder(...)`, + * `const { action } = builder(...); export { action };` and `export const action = route.action`, + * which is 36 of the tree's entry points: the corpus was testing less than its entry count + * implied, and no assertion could notice, because a mutation that reaches fewer routes lowers the + * score rather than raising it. Same failure mode as the `suppress-every-check` omission above, + * so the answer is the same: assert the population rather than wait for a verdict to move. + * + * `admin.tsx` is the one documented exclusion. Its handler is a concise arrow + * (`async ({ user }) => typedjson({ user })`) with no block for a block wrapper to wrap, which is + * a limit of the rewrite rather than a gap in the enumeration. It is named rather than counted so + * a second one cannot appear silently. + */ + const CONCISE_ARROW_BODIES = new Set(["admin.tsx"]); + + it("wraps a body in every non-delegating entry point the scanner finds", () => { + const wrap = MUTATIONS.find((m) => m.id === "wrap-body-in-rethrow")!; + const root = materialize(files); + let entryPoints; + try { + ({ entryPoints } = scanDirectory(root)); + } finally { + rmSync(root, { recursive: true, force: true }); + } + + const byName = new Map(files.map((f) => [f.relativeName, f.source])); + const untouched = entryPoints + .filter((ep) => !ep.delegating && !CONCISE_ARROW_BODIES.has(ep.fileName)) + .filter((ep) => wrap.apply(ep.fileName, byName.get(ep.fileName)!) === null) + .map((ep) => ep.fileName); + + expect(untouched).toEqual([]); + }); + it("has a baseline worth mutating", () => { expect(baseline).not.toBeNull(); expect(baseline!.entryPoints).toBeGreaterThan(300); diff --git a/internal-packages/observability-map/src/mutations.ts b/internal-packages/observability-map/src/mutations.ts index d273728ec03..64d6a07f9ad 100644 --- a/internal-packages/observability-map/src/mutations.ts +++ b/internal-packages/observability-map/src/mutations.ts @@ -179,18 +179,44 @@ function rootCall(call: ts.CallExpression): ts.CallExpression { } } -function fromInitializer(expr: ts.Expression, out: EntryFunction[]): void { +/** + * The handler functions an export's initializer resolves to. + * + * `locals` is consulted for the two indirect spellings, both of which `scan.ts` resolves and + * neither of which this reached: `export const action = route.action` beside + * `const route = createActionApiRoute(...)`, which is 7 of the tree's API routes, and + * `export const action = handleThing` naming a local. `seen` stops `const a = b; const b = a`. + */ +function fromInitializer( + expr: ts.Expression, + out: EntryFunction[], + locals: LocalDeclarations, + seen: Set = new Set() +): void { const target = unwrap(expr); if (isEntryFunction(target)) { out.push(target); return; } - if (!ts.isCallExpression(target)) return; - for (const arg of rootCall(target).arguments) { - const unwrapped = unwrap(arg); - if (isEntryFunction(unwrapped)) out.push(unwrapped); - else if (ts.isObjectLiteralExpression(unwrapped)) collectNamedHandlers(unwrapped, out); + if (ts.isCallExpression(target)) { + for (const arg of rootCall(target).arguments) { + const unwrapped = unwrap(arg); + if (isEntryFunction(unwrapped)) out.push(unwrapped); + else if (ts.isObjectLiteralExpression(unwrapped)) collectNamedHandlers(unwrapped, out); + } + return; } + // `route.action`, and any longer chain, is resolved from whatever declared its root identifier. + let root: ts.Expression = target; + while (ts.isPropertyAccessExpression(root) || ts.isElementAccessExpression(root)) { + root = unwrap(root.expression); + } + if (!ts.isIdentifier(root) || seen.has(root.text)) return; + const declaration = locals.get(root.text); + if (declaration === undefined) return; + seen.add(root.text); + if (ts.isFunctionDeclaration(declaration)) out.push(declaration); + else fromInitializer(declaration, out, locals, seen); } const ENTRY_NAMES = new Set(["loader", "action"]); @@ -202,23 +228,109 @@ function isExported(node: ts.Node): boolean { ); } -/** Block bodies of the exported `loader`/`action` handlers, the region a whole-body wrapper wraps. */ -function entryBodies(sf: ts.SourceFile): ts.Block[] { - const functions: EntryFunction[] = []; +/** + * Top-level declarations by binding name, so a named export clause (`export { action }`) resolves + * back to the initializer it came from. Object binding patterns are read element by element, which + * is what makes `const { action, loader } = createActionApiRoute(...)` resolvable. + */ +type LocalDeclarations = Map; + +function localDeclarations(sf: ts.SourceFile): LocalDeclarations { + const locals: LocalDeclarations = new Map(); for (const statement of sf.statements) { - if (!isExported(statement)) continue; if (ts.isFunctionDeclaration(statement) && statement.name) { - if (ENTRY_NAMES.has(statement.name.text)) functions.push(statement); + locals.set(statement.name.text, statement); continue; } if (!ts.isVariableStatement(statement)) continue; for (const decl of statement.declarationList.declarations) { - if (!decl.initializer || !ts.isIdentifier(decl.name)) continue; - if (ENTRY_NAMES.has(decl.name.text)) fromInitializer(decl.initializer, functions); + if (!decl.initializer) continue; + if (ts.isIdentifier(decl.name)) { + locals.set(decl.name.text, decl.initializer); + continue; + } + if (ts.isObjectBindingPattern(decl.name)) { + for (const element of decl.name.elements) { + if (ts.isIdentifier(element.name)) locals.set(element.name.text, decl.initializer); + } + } } } + return locals; +} + +/** + * Block bodies of the exported `loader`/`action` handlers, the region a whole-body wrapper wraps. + * + * Reads the same four export forms `scan.ts` reads: an exported function declaration, an exported + * `const`, an exported object binding pattern, and a named export clause resolved back through a + * local. It read only the first two, which is the shape of every API route in the tree + * (`const { action, loader } = createActionApiRoute(...); export { action, loader };` and the + * direct `export const { action } = ...`), so `wrapEveryBody` and the other whole-body entries + * silently skipped 36 of the 427 entry points while reporting a file count that suggested + * otherwise. `mutationCorpus.test.ts` pins the population now ("wraps a body in every + * non-delegating entry point the scanner finds"), so the harness cannot lag the scanner here again + * without going red. + * + * This is NOT a retreat from the deliberate independence `collectNamedHandlers` documents. That + * independence is about disagreeing over where a HANDLER sits inside a builder's argument, which is + * a judgement the corpus has to be able to make for itself. Which exports exist is not a judgement, + * and the harness was simply behind. + */ +function entryBodies(sf: ts.SourceFile): ts.Block[] { + const functions: EntryFunction[] = []; + const locals = localDeclarations(sf); + const fromLocal = (name: string) => { + const decl = locals.get(name); + if (decl === undefined) return; + if (ts.isFunctionDeclaration(decl)) functions.push(decl); + else fromInitializer(decl, functions, locals); + }; + + for (const statement of sf.statements) { + if (ts.isFunctionDeclaration(statement) && statement.name && isExported(statement)) { + if (ENTRY_NAMES.has(statement.name.text)) functions.push(statement); + continue; + } + + if (ts.isVariableStatement(statement) && isExported(statement)) { + for (const decl of statement.declarationList.declarations) { + if (!decl.initializer) continue; + if (ts.isIdentifier(decl.name)) { + if (ENTRY_NAMES.has(decl.name.text)) fromInitializer(decl.initializer, functions, locals); + continue; + } + if (!ts.isObjectBindingPattern(decl.name)) continue; + for (const element of decl.name.elements) { + if (ts.isIdentifier(element.name) && ENTRY_NAMES.has(element.name.text)) { + fromInitializer(decl.initializer, functions, locals); + } + } + } + continue; + } + + // A re-export (`export { loader } from "./x"`) has no local binding to resolve, and a namespace + // clause cannot name a loader or an action. + if ( + ts.isExportDeclaration(statement) && + statement.exportClause && + !statement.moduleSpecifier && + ts.isNamedExports(statement.exportClause) + ) { + for (const element of statement.exportClause.elements) { + if (!ENTRY_NAMES.has(element.name.text)) continue; + fromLocal(element.propertyName?.text ?? element.name.text); + } + } + } + + // One handler can serve both exports, and both reach it by their own road: the loader and the + // action of `const { loader, action } = createActionApiRoute({ handler })` resolve to the same + // node. Wrapping it twice would splice the same text in twice at the same offset, because + // `applyEdits` treats two zero-width inserts at one position as non-overlapping. const bodies: ts.Block[] = []; - for (const fn of functions) { + for (const fn of new Set(functions)) { if (fn.body && ts.isBlock(fn.body)) bodies.push(fn.body); } return bodies; From 2de22e4e031b0cd8c3e07aa8b80223fdce2fb754 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Mon, 3 Aug 2026 04:46:35 +0100 Subject: [PATCH 105/106] refactor(observability-map): share or pin the rest of the duplicated rules Sweeping the package for the defect behind the two review threads: one question answered by two pieces of code, where only one copy gets fixed. Shared: - the scannable-file predicate, copied into integration.test.ts and webappSymbols.test.ts after it was exported to stop mutationCorpus.test.ts copying it - the FIX FIRST filter and sort, byte-identical in terminal.ts and prComment.ts, which already imports five helpers from it; failingIds is now scoredFailures plus a map - normalizeSegment, in the test that validates SENSITIVE_SEGMENTS against the real tree. It was splitting segments with /_+$/, the regex that function's own comment says not to use - the five bare-literal node kinds, written out in literalTruth three lines above the literalValue that already had them Pinned: - contextGap and auditGap, which redo by hand what checkContributions computes generically, on two headline figures with nothing saying they had to agree. Reverting either to a different denominator or numerator now goes red Left alone, with reasons recorded in the sweep report: canRaise vs tryBlockMayThrow, the two exact true-keyword folds, the two comment extractors, the three means, ratio vs globalWithout, and the eight AST helpers mutations.ts keeps its own copies of so the corpus can disagree with the scanner. --- .../observability-map/src/report/prComment.ts | 15 +---- .../observability-map/src/report/terminal.ts | 28 +++++--- .../observability-map/src/score.test.ts | 66 +++++++++++++++++++ .../observability-map/src/sensitivity.ts | 2 +- .../src/webappSymbols.test.ts | 15 ++++- 5 files changed, 102 insertions(+), 24 deletions(-) diff --git a/internal-packages/observability-map/src/report/prComment.ts b/internal-packages/observability-map/src/report/prComment.ts index 03872cb7c80..3a5d5842aee 100644 --- a/internal-packages/observability-map/src/report/prComment.ts +++ b/internal-packages/observability-map/src/report/prComment.ts @@ -1,10 +1,9 @@ import type { MapReport, ScoredEntry } from "../score.js"; -import { SCORED_CHECK_IDS } from "../checks/index.js"; import { auditLine, checkContributionLines, contextLine, - contextOnly, + fixFirst, delegatedLines, scoredFailures, unknownSuppressionLines, @@ -40,8 +39,7 @@ const MAX_DELEGATED_ROUTES = 15; // Scored checks only, same exclusion terminal.ts's scoredFailures makes: audit-trail fails almost // every sensitive mutation today, so listing it per route would nag with something unfixable // instead of surfacing the route-specific gaps this column exists for. -const failingIds = (e: ScoredEntry) => - e.checks.filter((c) => SCORED_CHECK_IDS.includes(c.id) && c.status === "fail").map((c) => c.id); +const failingIds = (e: ScoredEntry) => scoredFailures(e).map((c) => c.id); function scoreLine(head: MapReport, base: MapReport | null): string { const headline = @@ -197,14 +195,7 @@ function whatChangedSection(head: MapReport, base: MapReport | null): string[] { function fixFirstSection(head: MapReport): string[] { const lines = ["FIX FIRST"]; - const worst = head.entries - .filter((e) => scoredFailures(e).length > 0 && !contextOnly(e)) - .sort( - (a, b) => - Number(b.sensitive) - Number(a.sensitive) || - a.score - b.score || - a.fileName.localeCompare(b.fileName) - ); + const worst = fixFirst(head.entries); for (const e of worst.slice(0, 3)) { const marks = e.sensitive ? " (sensitive)" : ""; diff --git a/internal-packages/observability-map/src/report/terminal.ts b/internal-packages/observability-map/src/report/terminal.ts index dc59e205342..0bb904bae50 100644 --- a/internal-packages/observability-map/src/report/terminal.ts +++ b/internal-packages/observability-map/src/report/terminal.ts @@ -27,6 +27,25 @@ export const scoredFailures = (e: ScoredEntry) => * An entry that fails something else as well stays in the list with all of its findings, so a * route like `/account/tokens` still shows the request-context gap alongside the rest. */ +/** + * The routes the FIX FIRST list is drawn from, worst first: sensitive before not, then by score, + * then by name. Exported because `prComment.ts` renders the same list with different bullets and + * had a byte-identical copy of this filter and sort, in a file that already imports + * `scoredFailures` and `contextOnly` from here. + * + * `contextOnly` routes are left out because `request-context` fails almost everything, so a list + * headed by three of them tells a reader nothing they cannot read off the gap figure. + */ +export const fixFirst = (entries: ScoredEntry[]): ScoredEntry[] => + entries + .filter((e) => scoredFailures(e).length > 0 && !contextOnly(e)) + .sort( + (a, b) => + Number(b.sensitive) - Number(a.sensitive) || + a.score - b.score || + a.fileName.localeCompare(b.fileName) + ); + export const contextOnly = (e: ScoredEntry) => { const failures = scoredFailures(e); return failures.length === 1 && failures[0]!.id === "request-context"; @@ -197,14 +216,7 @@ export function renderTerminal(report: MapReport): string { ); } - const worst = report.entries - .filter((e) => scoredFailures(e).length > 0 && !contextOnly(e)) - .sort( - (a, b) => - Number(b.sensitive) - Number(a.sensitive) || - a.score - b.score || - a.fileName.localeCompare(b.fileName) - ); + const worst = fixFirst(report.entries); lines.push(""); lines.push("FIX FIRST"); diff --git a/internal-packages/observability-map/src/score.test.ts b/internal-packages/observability-map/src/score.test.ts index a8f38a7f4ce..a8d6943d0e0 100644 --- a/internal-packages/observability-map/src/score.test.ts +++ b/internal-packages/observability-map/src/score.test.ts @@ -561,3 +561,69 @@ export async function action() { expect(after.unmeasured).toBe(0); }); }); + +/** + * `contextGap` and `auditGap` are the same arithmetic `checkContributions` already does for every + * check, written out again by hand for two named ids: `map(find).filter(status)` for the context + * figure, `filter(some)` for the audit one, and a third spelling of "passed" for each. Three + * implementations of "applicable, and how many of those passed", and nothing said they had to + * agree, on the two figures the report puts in front of a reader as headline numbers. + * + * Pinned rather than shared. Collapsing them would mean the gap figures reading their check's row + * out of `checkContributions`, which is a fine refactor and a wider blast radius than the property + * is worth: what matters is that they cannot disagree, and an assertion says that without moving + * any code the renderers read. + */ +describe("the hand-rolled gap figures agree with the per-check contributions", () => { + const SOURCE = `import { prisma } from "~/db.server"; +import { logger } from "~/services/logger.server"; +export async function action({ params }) { + try { + return await prisma.apiKey.create({ data: { orgId: params.orgId } }); + } catch (e) { + logger.error("failed", { orgId: params.orgId }); + return null; + } +}`; + + // A sensitive mutation that DOES record an audit event, so `withAudit` is not simply + // `sensitiveMutations`. Without it the audit assertion held whatever the numerator counted. + const AUDITED = `import { prisma } from "~/db.server"; +import { startImpersonation } from "~/models/admin.server"; +export async function action({ request, params }) { + const session = await startImpersonation(request, params.userId); + await prisma.apiKey.create({ data: { orgId: params.orgId } }); + return redirect("/", { headers: session }); +}`; + + const report = buildReport( + [ + scanFile("api.v1.orgs.$orgId.apikeys.ts", SOURCE)!, + scanFile("api.v1.tokens.ts", SOURCE)!, + scanFile("resources.impersonation.ts", AUDITED)!, + scanFile("healthcheck.ts", `export const loader = () => new Response("ok");`)!, + ], + [] + ); + + const contribution = (id: string) => report.checkContributions.find((c) => c.id === id)!; + + it("reports the same request-context denominator and numerator", () => { + expect(report.contextGap.applicable).toBe(contribution("request-context").applicable); + expect(report.contextGap.naming).toBe(contribution("request-context").passed); + }); + + it("reports the same audit-trail denominator and numerator", () => { + expect(report.auditGap.sensitiveMutations).toBe(contribution("audit-trail").applicable); + expect(report.auditGap.withAudit).toBe(contribution("audit-trail").passed); + }); + + // A denominator of zero would make both assertions above hold vacuously. + // Both assertions above hold vacuously on a zero denominator, and the audit one holds vacuously + // whenever every applicable route fails, since the two counts coincide. + it("measured something for both of them, with the audit numerator strictly between", () => { + expect(report.contextGap.applicable).toBeGreaterThan(0); + expect(report.auditGap.withAudit).toBeGreaterThan(0); + expect(report.auditGap.withAudit).toBeLessThan(report.auditGap.sensitiveMutations); + }); +}); diff --git a/internal-packages/observability-map/src/sensitivity.ts b/internal-packages/observability-map/src/sensitivity.ts index 760d7cd189e..ba13454fe17 100644 --- a/internal-packages/observability-map/src/sensitivity.ts +++ b/internal-packages/observability-map/src/sensitivity.ts @@ -122,7 +122,7 @@ export const SENSITIVE_SEGMENTS = [ * written the way a reader would say it. A trailing underscore opts a route out of its parent * layout (`resources.impersonation_.view-as.ts`) and changes nothing about what the route does. */ -function normalizeSegment(segment: string): string { +export function normalizeSegment(segment: string): string { // Trimmed by hand rather than with /_+$/, which backtracks polynomially on a run of underscores // and trips CodeQL. Nothing here is attacker-controlled (the input is a filename read off disk), // so this is about not spending a reviewer's attention on the alert. diff --git a/internal-packages/observability-map/src/webappSymbols.test.ts b/internal-packages/observability-map/src/webappSymbols.test.ts index f9542a0261e..3a709d2d913 100644 --- a/internal-packages/observability-map/src/webappSymbols.test.ts +++ b/internal-packages/observability-map/src/webappSymbols.test.ts @@ -3,7 +3,13 @@ import { readdirSync, readFileSync } from "node:fs"; import { join, resolve } from "node:path"; import { AUDIT_SYMBOLS } from "./checks/auditTrail.js"; import { GUARDS, SOFT_GUARDS } from "./checks/authBoundary.js"; -import { ANTICIPATED_SEGMENTS, SENSITIVE_SEGMENTS, SENSITIVE_SYMBOLS } from "./sensitivity.js"; +import { isScannableFile } from "./scan.js"; +import { + ANTICIPATED_SEGMENTS, + normalizeSegment, + SENSITIVE_SEGMENTS, + SENSITIVE_SYMBOLS, +} from "./sensitivity.js"; /** * Every name and every path segment the tool matches on must exist in the codebase it is pointed @@ -82,7 +88,7 @@ function walkFiles(dir: string, out: string[] = []): string[] { for (const entry of readdirSync(dir, { withFileTypes: true })) { const path = join(dir, entry.name); if (entry.isDirectory()) walkFiles(path, out); - else if (/\.tsx?$/.test(entry.name) && !entry.name.endsWith(".d.ts")) out.push(path); + else if (isScannableFile(entry.name)) out.push(path); } return out; } @@ -137,7 +143,10 @@ function routeSegments(): Set { const segments = new Set(); for (const entry of readdirSync(ROUTES, { withFileTypes: true })) { for (const part of entry.name.replace(/\.tsx?$/, "").split(".")) { - segments.add(part.replace(/_+$/, "")); + // `sensitivity.ts`'s own normalizer. This validates the vocabulary that file matches on, so + // a segment has to be trimmed here exactly as it is trimmed there; the local `/_+$/` was + // also the regex `normalizeSegment`'s own comment says not to use. + segments.add(normalizeSegment(part)); } } return segments; From 4b0b24364a2868b27ae00411bd33234fd12e630b Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Mon, 3 Aug 2026 05:38:26 +0100 Subject: [PATCH 106/106] fix(observability-map): read a branch arm's exit through the liveness fold selectsADistinctPath decided whether an if or a switch in a catch clause made a real classification decision by asking containsExit, a plain containment walk. Containment is true of an exit that can never run, so catch (e) { if (e instanceof Error) { if (false) { return null; } } return json(x, { status: 500 }); } read as a decision while the same clause without the if read as a swallow: 50 points a route for a behaviour-preserving mechanical edit. Measured over apps/webapp/app/routes, that shape took the tree from 19 to 27 and raised 80 of 412 routes. An earlier wave had already moved catchClauseEvidence's exited flag onto containsLiveExit for the same eleven dead spellings. The branch predicate 120 lines below it kept the containment read, so this is one rule fixed in one place and left in its sibling. The three exit reads now go through containsLiveExit and containsExit is deleted, so there is one exit read in the file. The property that makes one helper safe for two callers reading it for opposite purposes is now written down on containsLiveWhere: it is strictly subtractive against containment, so it only ever un-blinds the exited flag and only ever withholds a branch grant. Conservatism is a property of the helper plus what the caller does with a true, and auditing it at the definition is how this was missed. Adds dead-armed-instanceof-if to the mutation corpus, additive class, and dead-conjunction-instanceof-if under KNOWN_GAPS. The second is the sibling this fix does not close: folding the arm does not fold a dead condition, and if (e instanceof Error && false) reaches the same grant for the same 80 routes. literalTruth treats && and || as always null on purpose, so closing it means widening that fold, a different rule with its own measurement. Recorded and running rather than left to be rediscovered. Requiring the arm to definitelyExits was measured and rejected: it accuses admin.api.v1.orgs.$organizationId.environments.staging.ts, which classifies Prisma's P2002 and rethrows everything else, of taking one way out regardless of what was thrown. Pinned by 'still credits an arm guarded by a condition that does not fold'. The real tree does not move: every route's score and every check's status are byte-identical before and after, global 19 either way. --- .../src/mutationCorpus.test.ts | 13 ++- .../observability-map/src/mutations.ts | 30 +++++++ .../observability-map/src/scan.test.ts | 84 ++++++++++++++++++- .../observability-map/src/scan.ts | 59 ++++++++----- 4 files changed, 164 insertions(+), 22 deletions(-) diff --git a/internal-packages/observability-map/src/mutationCorpus.test.ts b/internal-packages/observability-map/src/mutationCorpus.test.ts index 658ce32eca9..ef2094e0aee 100644 --- a/internal-packages/observability-map/src/mutationCorpus.test.ts +++ b/internal-packages/observability-map/src/mutationCorpus.test.ts @@ -57,8 +57,19 @@ const ENABLED = process.env.OBS_MAP_MUTATION_CORPUS === "1"; * flag raised before each statement's own branch check, which makes every deciding statement refuse * itself. Raising it after is byte-identical on the real tree and closes the shape, so the entry is * defended now and the `if (true)` family needed no condition folding after all. + * + * `dead-conjunction-instanceof-if` is the sibling of `dead-armed-instanceof-if` that the arm-liveness + * fix does not close. `selectsADistinctPath` now folds a dead ARM; a dead CONDITION still reaches + * the grant, and `e instanceof Error && false` is exactly that, a guard that references the caught + * binding and can never be true. No fold in `scan.ts` can see it, because `literalTruth` treats + * `&&` and `||` as always null on purpose so that a live guard can never be read as dead. Widening + * that fold is a different rule from the one this round fixed and needs its own measurement, so the + * shape is recorded and running rather than left for the next person to rediscover. */ -const KNOWN_GAPS = new Set(["dead-classifying-try-with-call"]); +const KNOWN_GAPS = new Set([ + "dead-classifying-try-with-call", + "dead-conjunction-instanceof-if", +]); type SourceFile = { relativeName: string; source: string }; diff --git a/internal-packages/observability-map/src/mutations.ts b/internal-packages/observability-map/src/mutations.ts index 64d6a07f9ad..4273223f881 100644 --- a/internal-packages/observability-map/src/mutations.ts +++ b/internal-packages/observability-map/src/mutations.ts @@ -843,6 +843,34 @@ export const MUTATIONS: Mutation[] = [ (e) => `if (false) { if (${e} instanceof Error) { return new Response(null, { status: 400 }); } } else { 0; }` ), + // The sibling of `empty-instanceof-if`. That entry's arm is empty; this one's arm holds an exit + // that can never run, which is the same no-op written so that a containment read cannot tell the + // difference. `selectsADistinctPath` asked a plain containment question, true of + // `if (false) { return null; }`, so the test read as a real classification decision and turned a + // swallowing catch into a passing one: 80 routes and the tree from 19 to 27 when measured. + // Additive: it plants fake signal. `catchClauseEvidence`'s own `exited` flag had already been + // moved onto `containsLiveExit` for exactly this reason and the branch predicate beside it was + // left behind, which is why the shape is spelled with the corpus's own `if (false)` and not + // something exotic. + prependToEveryCatch( + "dead-armed-instanceof-if", + "preserving", + "splice if (e instanceof Error) { if (false) { return null; } } into every catch", + (e) => `if (${e} instanceof Error) { if (false) { return null; } }` + ), + // The sibling the entry above does NOT close, found while closing it and filed here rather than + // fixed. Moving the ARM's exit read onto `containsLiveExit` folds a dead arm; it does not fold a + // dead CONDITION, and a condition that both references the caught binding and is provably false + // reaches the same grant. `literalTruth` cannot see it: `&&` and `||` are documented there as + // always null, deliberately, so `e instanceof Error && false` is an undecidable guard to every + // fold in the file. Closing it means widening that fold, which is a different rule with its own + // measurement, so this runs as a `KNOWN_GAPS` expected failure instead of sitting unrecorded. + prependToEveryCatch( + "dead-conjunction-instanceof-if", + "preserving", + "splice if (e instanceof Error && false) { return null; } into every catch", + (e) => `if (${e} instanceof Error && false) { return null; }` + ), // A finally that leaves itself by `break` cancels the try's completion, so nothing hosted in // that tryBlock ever escapes the clause: the whole statement is a no-op. The walk's // catchless-try entry credited it anyway, minting a branch from the hosted classifier on 80 @@ -1087,6 +1115,8 @@ export const ADDITIVE_IDS = [ "wrap-body-in-rethrow", "wrap-body-in-same-arms-throw-ternary", "empty-instanceof-if", + "dead-armed-instanceof-if", + "dead-conjunction-instanceof-if", "dead-classifier-one-arm", "dead-throw-in-cancelled-try", "dead-deciding-map", diff --git a/internal-packages/observability-map/src/scan.test.ts b/internal-packages/observability-map/src/scan.test.ts index 32f9d224d7e..458fab79ac4 100644 --- a/internal-packages/observability-map/src/scan.test.ts +++ b/internal-packages/observability-map/src/scan.test.ts @@ -759,7 +759,7 @@ describe("scanFile: catch clause evidence", () => { }); // The mirror of the family above. Each dead spelling earns nothing, and it must also COST - // nothing: `containsExit` was true of the dead statement itself, so prepending one raised the + // nothing: a plain containment read was true of the dead statement itself, so prepending one raised the // `exited` flag and blinded the walk to the real classification below it, turning a pass into a // swallow verdict on 78 real routes. `containsLiveExit` folds the literal guard and sees no live // exit, so the deciding statements keep their credit. The spellings are the CORPUS spellings @@ -2470,6 +2470,88 @@ describe("a ternary on the error has to send its arms somewhere different", () = }); }); +// The liveness gap in the branch predicate. `selectsADistinctPath` asked a plain containment +// question, so an arm holding an exit that can never run read as an arm that takes the error +// somewhere. `catch (e) { if (e instanceof Error) { if (false) { return null; } } return json(x, +// { status: 500 }); }` is the same swallow as the clause without the `if`, and it was worth 50 +// points a route. The same eleven dead spellings had already been folded out of +// `catchClauseEvidence`'s `exited` flag by `containsLiveExit`, and this predicate beside it kept +// the containment read. `dead-armed-instanceof-if` in the mutation corpus is the tree-scale +// version: global 19 -> 27 and 80 routes raised, before the fix. +describe("an arm whose only exit is dead decides nothing", () => { + const swallow = (mutation: string) => ` + export async function loader() { + try { + return await prisma.thing.findMany(); + } catch (error) { + ${mutation} + return json({ error: "generic" }, { status: 500 }); + } + } + `; + + it("reads the unmutated clause as a swallow, as a baseline", () => { + const ep = scanFile("x.ts", swallow("")); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + // The reported shape, plus three further spellings of the same no-op reaching the same + // predicate by different routes: a dead loop body, a dead else arm, and a switch clause. + const DEAD_ARMS: Array<[string, string]> = [ + ["an if (false) arm", "if (error instanceof Error) { if (false) { return null; } }"], + ["a while (false) body", "if (error instanceof Error) { while (false) { throw error; } }"], + [ + "a dead else arm beside an arm that goes nowhere", + "if (error instanceof Error) { doThing(); } else { for (const k in {}) { return null; } }", + ], + [ + "a switch clause whose exit is dead", + "switch (error.code) { case 'x': if (1 === 2) { throw error; } }", + ], + ]; + + for (const [label, mutation] of DEAD_ARMS) { + it(`does not credit ${label}`, () => { + const ep = scanFile("x.ts", swallow(mutation)); + expect(ep!.catches[0]!.branches).toBe(false); + }); + } + + // The positive controls. The fold is subtractive against containment, so anything it cannot + // prove dead reads exactly as it did, including a guard whose truth is not decidable from the + // token alone. Without these the fix could be "always false" and the four cases above would + // still pass. + // + // `an arm guarded by a condition that does not fold` is also the pin on the alternative that was + // measured and rejected: asking the arm to `definitelyExits` rather than to hold a live exit. + // That is the "guaranteed" reading, it refuses all four shapes above, and it accuses + // `admin.api.v1.orgs.$organizationId.environments.staging.ts` on the real tree, taking the global + // from 19 to 18. That clause recognises Prisma's P2002, re-reads the conflicting row and returns + // `{ status: "updated" }`, rethrowing everything else: a textbook classification whose arm + // happens to fall through to the rethrow when the re-read finds nothing. Accusing it of taking + // one way out regardless of what was thrown is simply false, and a new false accusation is the + // direction that gets the tool switched off. + const LIVE_ARMS: Array<[string, string]> = [ + ["a plain returning arm", "if (error instanceof Error) { return badRequest(); }"], + [ + "an arm guarded by a condition that does not fold", + "if (error instanceof Error) { if (error.code === 'P2002') { return conflict(); } }", + ], + [ + "a live exit in the else arm only", + "if (error instanceof Error) { doThing(); } else { return badRequest(); }", + ], + ["a switch clause that returns", "switch (error.code) { case 'P2002': return conflict(); }"], + ]; + + for (const [label, mutation] of LIVE_ARMS) { + it(`still credits ${label}`, () => { + const ep = scanFile("x.ts", swallow(mutation)); + expect(ep!.catches[0]!.branches).toBe(true); + }); + } +}); + // C4a. `export const { action, loader } = createActionApiRoute(...)` produced no entry point at // all: `scanFile` skipped a non-identifier binding name at the export site, so the route was // absent from the denominator rather than parsed, failed or unmeasured. The two-step spelling diff --git a/internal-packages/observability-map/src/scan.ts b/internal-packages/observability-map/src/scan.ts index 2067e60203a..78b48d54e08 100644 --- a/internal-packages/observability-map/src/scan.ts +++ b/internal-packages/observability-map/src/scan.ts @@ -650,15 +650,6 @@ function containsEscapingJump(node: ts.Node, jumps: BareJumps = ESCAPES): boolea return ts.forEachChild(node, (child) => containsEscapingJump(child, jumps)) === true; } -/** Whether the tree rooted at `node` contains a `return` or a `throw` of its own, not counting one - * inside a nested function. What separates an arm that takes the error somewhere from an arm that - * runs and falls back into the clause's single common exit. */ -function containsExit(node: ts.Node): boolean { - if (ts.isFunctionLike(node)) return false; - if (ts.isReturnStatement(node) || ts.isThrowStatement(node)) return true; - return ts.forEachChild(node, containsExit) === true; -} - /** * Literal truthiness of a guard expression: true, false, or null when not decidable from the * token alone. Only literal tokens fold; an identifier, call, bigint, `&&`, `||` or a template @@ -713,13 +704,25 @@ function tryBlockMayThrow(block: ts.Block): boolean { } /** - * `containsExit`, minus exits that sit in a provably-untaken branch. `if (false) { throw e; }` - * contains an exit and can never run one; treating it as an exit is what let a dead statement - * blind the walk to the real classification below it, prepending one to a deciding clause turned - * its pass into a swallow verdict on 78 real routes. Folds literal guards only, so an unknown - * condition keeps the containsExit answer, which is the direction that refuses credit rather than - * inventing it. The mirror twins under `dead and deferred code prepended to a deciding catch does - * not blind it` hold the recovered half; the `BRANCH_EXITED` family holds the refusing half. + * Whether the tree rooted at `root` contains a node `hit` accepts that a provably-untaken branch + * does not already rule out. A plain containment walk, minus the hits it can prove never run: + * `if (false) { throw e; }` contains a throw and can never run one. + * + * Folds literal guards only, so wherever `literalTruth` cannot decide, every hit the plain walk + * would have found is still found. That makes this strictly subtractive against containment, which + * is what lets both of its callers read it for opposite purposes: + * + * - `catchClauseEvidence`'s `exited` flag, where a hit BLINDS the walk to whatever follows. + * Containment blinded it on a dead statement, so prepending one to a deciding clause turned its + * pass into a swallow verdict on 78 real routes. Subtracting dead hits only ever un-blinds. + * - `selectsADistinctPath`, where a hit GRANTS a branch. Containment granted one for an arm whose + * only exit was dead, which is `dead-armed-instanceof-if` in the mutation corpus, measured at 80 + * routes and the tree from 19 to 27. Subtracting dead hits only ever withholds. + * + * The `exited` half is pinned by the mirror twins under `dead and deferred code prepended to a + * deciding catch does not blind it` (recovered) and the `BRANCH_EXITED` family (refusing). The + * `selectsADistinctPath` half is pinned by `an arm whose only exit is dead decides nothing` and its + * siblings, plus the corpus entry. */ function containsLiveWhere(root: ts.Node, hit: (n: ts.Node) => boolean): boolean { const walk = (node: ts.Node): boolean => { @@ -809,6 +812,16 @@ function containsLiveReturn(node: ts.Node): boolean { * An `if`/`else` whose two arms are textually identical does not count, the same comparison * `selectsAnErrorPath` makes of a ternary's arms. * + * The exit an arm is credited for has to be a LIVE one, `containsLiveExit` and never a plain + * containment read. `if (e instanceof Error) { if (false) { return null; } }` contains an exit that + * can never run, so under containment it read as a real decision and took a swallowing catch to a + * pass for the price of a mechanical edit: 80 routes and the tree from 19 to 27 when measured. The + * same liveness rule had already been put on `catchClauseEvidence`'s `exited` flag, for the same + * eleven dead spellings, and this predicate beside it kept the containment read. `an arm whose only + * exit is dead decides nothing` and its siblings are the unit pins; `dead-armed-instanceof-if` in + * the mutation corpus is the tree-scale version. Being subtractive against containment, the fold + * can only ever withhold a branch, never invent one, so a live arm reads exactly as it did. + * * The residual both branch tests share, stated here once for both: two arms that produce the same * outcome by different spellings still read as a real decision. * `if (e instanceof Error) { return json(x); } return Response.json(x);` counts and decides @@ -822,11 +835,17 @@ function selectsADistinctPath(statement: ts.IfStatement | ts.SwitchStatement): b const otherwise = statement.elseStatement; if (otherwise !== undefined) { if (normalizedText(statement.thenStatement) === normalizedText(otherwise)) return false; - return containsExit(statement.thenStatement) || containsExit(otherwise); + return containsLiveExit(statement.thenStatement) || containsLiveExit(otherwise); } - return containsExit(statement.thenStatement); + return containsLiveExit(statement.thenStatement); } - return statement.caseBlock.clauses.some((clause) => clause.statements.some(containsExit)); + // Per clause statement rather than over the whole switch, so a live exit in any clause counts + // whatever the discriminant is. Reading the switch as one node would hand `containsLiveWhere`'s + // discriminant fold a `switch (e.code)` it cannot decide, which changes nothing, and a + // `switch (1)` it can, which is not this predicate's business: an unreachable CLAUSE is caught + // by the same fold one level down, and the statement is only reached at all when its condition + // references the caught binding. + return statement.caseBlock.clauses.some((clause) => clause.statements.some(containsLiveExit)); } /** @@ -893,7 +912,7 @@ function catchClauseEvidence(clause: ts.CatchClause): { // and all 240 clauses' evidence byte-identical. The tests are the cases in `dead throw written // after something that already exited`. // - // Raised off `containsLiveExit`, never `containsExit`. The containment read is true of + // Raised off `containsLiveExit`, never a plain containment read. Containment is true of // `if (false) { throw e; }` itself, so a provably dead statement raised the flag and blinded the // walk to the real classification below it: prepending one to a deciding clause turned its pass // into a swallow verdict on 78 real routes, the same false accusation for all eleven dead