diff --git a/.github/workflows/observability-map.yml b/.github/workflows/observability-map.yml new file mode 100644 index 00000000000..6043ce15976 --- /dev/null +++ b/.github/workflows/observability-map.yml @@ -0,0 +1,310 @@ +name: πŸ—ΊοΈ Observability Map + +on: + pull_request: + types: [opened, synchronize, reopened] + paths: + - "apps/webapp/app/routes/**" + - "internal-packages/observability-map/**" + # The corpus job below is gated to this package's own paths, so a scheduled run is what still + # scans the tree as it drifts. Nightly rather than per route pull request: a new route can make a + # known laundering shape start paying, but that is a property of the tree accumulating, not of any + # one pull request, and it does not need catching within five minutes of the merge. + schedule: + - cron: "0 3 * * *" + workflow_dispatch: + +concurrency: + group: observability-map-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + # The workflow's paths filter is the union of what the two jobs below want, because GitHub + # evaluates it once per workflow. This narrows it again for the corpus job alone. + changes: + name: πŸ” Which paths moved + # Only the pull request path reads this job's output. On a schedule the action has no base to + # diff, warns that `before` is missing and reports the files in the last commit on main, which + # nothing then consults. Skipping it there keeps the nightly off a job it does not need. + if: github.event_name == 'pull_request' + runs-on: warp-ubuntu-latest-x64-2x + outputs: + package: ${{ steps.filter.outputs.package }} + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + fetch-depth: 1 + persist-credentials: false + - uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1 + id: filter + with: + filters: | + package: + - 'internal-packages/observability-map/**' + - '.github/workflows/observability-map.yml' + + # The tree-scale mutation corpus: every known laundering shape applied to the whole route tree, + # asserting the score does not rise. Roughly four and a half minutes for 45 entries, which is why + # it is gated out of the package's default `pnpm test` and run here instead. Unlike the report + # job below it has no token to lose, so it runs for fork PRs too, and unlike the report job it is + # allowed to fail the build. + # + # Gated to this package's own paths rather than running on every route pull request. What the + # corpus measures is the TOOL's resistance to laundering, and only an edit to the tool can weaken + # that, so a routes-only change was paying four and a half minutes of a 4x runner for a result + # that could not differ from the last one. It was also the worst kind of job to spend that on: a + # red x that fires on a large share of webapp pull requests, is allowed to fail, and gates + # nothing, which is the shape people learn to scroll past. + # + # What this gives up is real and small. A route landing a shape no corpus entry has seen can make + # a known laundering mutation start paying, and that is now caught by the nightly rather than by + # the pull request that caused it. Tree drift accrues over months, so a day is the right + # granularity for it; the tool's own regressions, which are the ones a single commit can cause, + # still gate per pull request. + mutation-corpus: + name: 🧬 Mutation corpus + needs: changes + # `!cancelled()` is here for the nightly, not for tidiness. `needs` carries an implicit + # success() on the job it names, and that implicit test outranks the `||` below: with a plain + # condition, a `changes` job that failed or was skipped skips this one, so the nightly would + # stop scanning for tree drift and report nothing about having stopped. A status-check function + # in the `if` is what drops the implicit success(), so the event test below decides alone. + # `!cancelled()` rather than `always()` because `cancel-in-progress` above is a real path and a + # superseded run should not finish this job. + # + # Pull request behaviour is deliberately unchanged: on a PR a failed `changes` leaves + # `needs.changes.outputs.package` empty, so the corpus still skips. The nightly is the backstop + # for that, which is the same trade the paths gate already makes for routes-only pull requests. + if: >- + !cancelled() && + (github.event_name != 'pull_request' || needs.changes.outputs.package == 'true') + runs-on: warp-ubuntu-latest-x64-4x + steps: + - name: ⬇️ Checkout repo + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + fetch-depth: 1 + persist-credentials: false + + - name: βŽ” Setup pnpm + uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0 + with: + version: 10.33.2 + + - name: βŽ” Setup node + uses: WarpBuilds/setup-node@bc639b444d583175926b588962199c247d23e8d3 # v6 + with: + node-version: 24.18.0 + cache: "pnpm" + + - name: πŸ“₯ Download deps + run: pnpm install --frozen-lockfile + + - name: 🧬 Run the corpus + env: + OBS_MAP_MUTATION_CORPUS: "1" + run: | + pnpm --filter @internal/observability-map exec vitest run \ + src/mutationCorpus.test.ts --disable-console-intercept + + # The package's own tests are NOT run here. They gate through pr_checks.yml, which is the only + # workflow the all-checks aggregate can see, so a job in this file would report a result nobody + # is required to wait for. See unit-tests-observability-map.yml and the obsmap filter. + report: + runs-on: warp-ubuntu-latest-x64-4x + # Only this job comments, so only this job gets the write. + permissions: + contents: read + pull-requests: write + # Fork PRs get a read-only token, so the comment cannot post. Skipping the job beats a red x. + # The event test is what keeps this job off the nightly, which has no pull request to comment on + # and only exists for the corpus job above. + if: >- + github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name == github.repository + steps: + - name: ⬇️ Checkout repo + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + fetch-depth: 0 + persist-credentials: false + + - name: βŽ” Setup pnpm + uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0 + with: + version: 10.33.2 + + - name: βŽ” Setup node + uses: WarpBuilds/setup-node@bc639b444d583175926b588962199c247d23e8d3 # v6 + with: + node-version: 24.18.0 + cache: "pnpm" + + - name: πŸ“₯ Download deps + run: pnpm install --frozen-lockfile + + # Guarded rather than allowed to fail: this job must never block a pull request. The failure + # is not swallowed either, the render step below turns a missing head report into a comment + # saying so, because a swallowed failure with no comment is the outcome nobody wants. + # + # `--out` rather than a stdout redirect, so nothing a tool decides to print can end up inside + # the document `prCommentCli` parses. `pnpm --filter` takes its recursive path and some + # versions announce `Scope: N of M workspace projects` on the way; that line landing in + # head.json would fail the parse and degrade every run to the stale-report comment, which is + # a permanent quiet failure rather than a loud one. It does not reproduce on the 10.33.2 + # pinned above, so this closes the class rather than a reproduction: the file is written by + # the process that owns it and stdout is left to be log output. Held by + # `it("let the scanner write its own report rather than capturing stdout")` in + # `internal-packages/observability-map/src/integration.test.ts`. + # + # `-s` keeps the partial dance honest now the redirect no longer creates the file: a scanner + # that exits 0 without writing takes the else branch and the stale-report comment, instead of + # failing the `mv` and turning the job red. + - name: πŸ”Ž Scan head + run: | + if pnpm --filter @internal/observability-map exec tsx src/cli.ts \ + --out=/tmp/head.json.partial && [ -s /tmp/head.json.partial ]; then + mv /tmp/head.json.partial /tmp/head.json + else + rm -f /tmp/head.json /tmp/head.json.partial + echo "head scan failed; the comment will say the report is stale for this run" >&2 + fi + + # base.sha, not a merge base, and two reviewers have now read that as a bug. The checkout + # above is the default for a pull_request event, so the working tree is GitHub's test merge + # commit, whose parents are base.sha and the PR head. The head tree therefore already contains + # the base branch up to base.sha, and diffing it against base.sha is what isolates this pull + # request's own work. A merge base would leave the intervening base-branch commits in the head + # tree and out of the base tree, and blame the pull request for all of them. + - name: πŸ”Ž Scan base with the head's scanner + run: | + if git worktree add /tmp/base-tree ${{ github.event.pull_request.base.sha }} \ + && pnpm --filter @internal/observability-map exec tsx src/cli.ts \ + --routes=/tmp/base-tree/apps/webapp/app/routes --out=/tmp/base.json \ + && [ -s /tmp/base.json ]; then + : + else + echo "-" > /tmp/base.json || true + echo "base scan failed or the worktree could not be added; falling back to no base" >&2 + fi + + # Looked up before the render step because the render decision needs it: with no delta to + # report, a pull request that already has a comment gets a resolved state rather than being + # left with findings that no longer exist, and one that does not gets nothing at all. The + # upsert step reuses the id rather than asking twice. + # + # On a failure that outlasts the retries, both steps below do nothing. Guessing is worse than + # silence here: this step is the only thing that knows which comment to PATCH, so a guess of + # "a comment exists" still reaches an upsert with no id to patch, which POSTs. That either + # adds a second marker comment beside the stale one, or says "the findings an earlier push + # reported are gone" on a pull request that never had findings. Worst case now is no comment + # this run, which the next push fixes. + - name: πŸ” Look for a comment from an earlier push + continue-on-error: true + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} + run: | + rm -f /tmp/existing-comment-id /tmp/comment-lookup-failed + found="" + ok="" + for attempt in 1 2 3; do + if found=$(gh api "repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/comments" --paginate \ + --jq '[.[] | select(.body | startswith(""))][0].id // empty'); then + ok=1 + break + fi + echo "comment lookup attempt ${attempt} failed" >&2 + sleep $((attempt * 5)) + done + + if [ -z "$ok" ]; then + touch /tmp/comment-lookup-failed + echo "comment lookup failed after 3 attempts; this run posts nothing" >&2 + exit 0 + fi + + # --paginate runs the jq once per page, so a marker comment on more than one page yields + # one id per page. Unhandled, that puts a newline in the PATCH url and the step dies under + # continue-on-error. The oldest wins: it is the one the upsert has been updating. + count=$(printf '%s\n' "$found" | grep -c '[0-9]' || true) + if [ "$count" -gt 1 ]; then + echo "warning: ${count} marker comments on this pull request; updating the oldest" >&2 + fi + printf '%s\n' "$found" | awk 'NF { print $1; exit }' > /tmp/existing-comment-id + + # continue-on-error for the same reason as the scan: a rendering bug must not turn the job + # red. An empty /tmp/comment.md means there is nothing to post, which is a decision + # prCommentCli makes, not this shell. + - name: πŸ“ Render comment + continue-on-error: true + run: | + rm -f /tmp/comment.md + render() { pnpm --filter @internal/observability-map exec tsx src/report/prCommentCli.ts "$@"; } + + # Every write goes through this, so a renderer that exits non-zero never leaves a 0-byte + # comment.md for the upsert to skip in silence. + emit() { + if render "$@" > /tmp/comment.md.partial; then + mv /tmp/comment.md.partial /tmp/comment.md + return 0 + fi + rm -f /tmp/comment.md.partial + return 1 + } + + if [ -f /tmp/comment-lookup-failed ]; then + echo "the comment lookup failed, so this run posts nothing" >&2 + exit 0 + fi + + if [ ! -s /tmp/head.json ]; then + emit --scan-failed || echo "could not render the stale-report comment either" >&2 + exit 0 + fi + + base=/tmp/base.json + if [ ! -s /tmp/base.json ] || [ "$(cat /tmp/base.json)" = "-" ]; then + base="-" + fi + + flags=() + if [ -s /tmp/existing-comment-id ]; then + flags=(--existing-comment) + fi + + if ! emit /tmp/head.json "$base" "${flags[@]}"; then + echo "render failed; falling back to the stale-report comment" >&2 + emit --scan-failed || echo "could not render the stale-report comment either" >&2 + fi + + # continue-on-error for the same reason: a transient gh api failure (rate limit, network) + # must not fail the job either. Worst case, the PR gets no comment this run. + - name: πŸ’¬ Upsert PR comment + continue-on-error: true + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} + run: | + # The same sentinel the render step reads, so the two cannot disagree about what a failed + # lookup means. Without it this step reads a missing id as "no comment exists" and POSTs. + if [ -f /tmp/comment-lookup-failed ]; then + echo "the comment lookup failed, so this run posts nothing" + exit 0 + fi + if [ ! -s /tmp/comment.md ]; then + echo "nothing to post: this pull request does not move the report" + exit 0 + fi + existing="" + if [ -f /tmp/existing-comment-id ]; then + existing=$(cat /tmp/existing-comment-id) + fi + if [ -n "$existing" ]; then + gh api -X PATCH "repos/${GITHUB_REPOSITORY}/issues/comments/${existing}" -F body=@/tmp/comment.md + else + gh api "repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/comments" -F body=@/tmp/comment.md + fi diff --git a/.github/workflows/pr_checks.yml b/.github/workflows/pr_checks.yml index b0fbd6ac040..e3df9b416f2 100644 --- a/.github/workflows/pr_checks.yml +++ b/.github/workflows/pr_checks.yml @@ -22,6 +22,7 @@ jobs: webapp: ${{ steps.filter.outputs.webapp }} packages: ${{ steps.filter.outputs.packages }} internal: ${{ steps.filter.outputs.internal }} + obsmap: ${{ steps.filter.outputs.obsmap }} cli: ${{ steps.filter.outputs.cli }} sdk: ${{ steps.filter.outputs.sdk }} steps: @@ -81,6 +82,36 @@ jobs: - 'pnpm-lock.yaml' - 'pnpm-workspace.yaml' - 'turbo.json' + # The whole webapp app tree, not just its routes, and that is the whole reason this + # filter exists. Two tests in @internal/observability-map read it: integration.test.ts + # scans the live route tree, and webappSymbols.test.ts walks all of apps/webapp/app and + # fails when a guard, sensitive or audit symbol stops resolving. Routes-only was this + # filter's own bug: renaming e.g. requireUserId in app/services/session.server.ts + # matched `webapp` and nothing else, so no job ran the suite and the break landed on + # main, or on the next unrelated internal-packages PR. + # + # The cost of the wider set, measured over the last 400 commits on main: 31% touch + # routes, 52% touch apps/webapp/app, so the job goes from firing on roughly a third of + # PRs to roughly a half. It is the cheap one -- a single 4x runner, no containers, no + # database, no prisma generate -- which is what makes that affordable. + # + # observability-map.yml is here because integration.test.ts asserts on its text and no + # other filter watches it, so editing the report workflow alone ran nothing at all. + # + # Deliberately NOT here: this package's own paths, and packages/plugins/src and + # internal-packages/rbac/src, the other two trees webappSymbols.test.ts reads. + # `internal` above already matches `internal-packages/**` and `packages/**`, and + # `unit-tests-internal.yml` runs `turbo run test --filter "@internal/*"`, which picks up + # @internal/observability-map and runs the same vitest suite. Listing them here as well + # ran the suite twice on every PR touching them, which was this filter's own doing. + obsmap: + - 'apps/webapp/app/**' + - '.github/workflows/pr_checks.yml' + - '.github/workflows/unit-tests-observability-map.yml' + - '.github/workflows/observability-map.yml' + - 'package.json' + - 'pnpm-lock.yaml' + - 'pnpm-workspace.yaml' cli: - 'packages/cli-v3/**' - 'packages/build/**' @@ -149,6 +180,11 @@ jobs: DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + obsmap: + needs: changes + if: needs.changes.outputs.obsmap == 'true' + uses: ./.github/workflows/unit-tests-observability-map.yml + e2e: needs: changes if: needs.changes.outputs.cli == 'true' @@ -172,6 +208,7 @@ jobs: - e2e-webapp - packages - internal + - obsmap - e2e - sdk-compat if: always() diff --git a/.github/workflows/unit-tests-observability-map.yml b/.github/workflows/unit-tests-observability-map.yml new file mode 100644 index 00000000000..eebf1ef971c --- /dev/null +++ b/.github/workflows/unit-tests-observability-map.yml @@ -0,0 +1,43 @@ +name: "πŸ§ͺ Unit Tests: Observability Map" + +permissions: + contents: read + +# Its own workflow rather than a job inside observability-map.yml, because that workflow is not +# reachable from pr_checks.yml's all-checks aggregate and so gates nothing. Called from there +# instead, behind a paths filter, which is how every other test suite in this repo is gated. +on: + workflow_call: + +jobs: + unitTests: + name: "πŸ§ͺ Unit Tests: Observability Map" + # No containers and no database: the package is a static analyser over source text, so the + # suite is CPU bound on parsing the route tree and needs nothing the runner does not have. + runs-on: warp-ubuntu-latest-x64-4x + steps: + - name: ⬇️ Checkout repo + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + fetch-depth: 1 + persist-credentials: false + + - name: βŽ” Setup pnpm + uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0 + with: + version: 10.33.2 + + - name: βŽ” Setup node + uses: WarpBuilds/setup-node@bc639b444d583175926b588962199c247d23e8d3 # v6 + with: + node-version: 24.18.0 + cache: "pnpm" + + - name: πŸ“₯ Download deps + run: pnpm install --frozen-lockfile + + # This suite reads apps/webapp/app (the route tree for the scan, the whole app tree for the + # symbol check) and the report workflow's text, which is why the filter that gates this + # workflow watches all of those and not only the routes folder. + - name: πŸ§ͺ Run tests + run: pnpm --filter @internal/observability-map run test diff --git a/.gitignore b/.gitignore index 7d9dc169042..f540927e32b 100644 --- a/.gitignore +++ b/.gitignore @@ -82,3 +82,6 @@ ailogger-output.log # local planning/design docs, not committed **/docs/superpowers/ + +# observability-map CLI output artifact, not committed +observability-map.json diff --git a/internal-packages/observability-map/README.md b/internal-packages/observability-map/README.md new file mode 100644 index 00000000000..c428c2741a9 --- /dev/null +++ b/internal-packages/observability-map/README.md @@ -0,0 +1,424 @@ +# @internal/observability-map + +Scores every webapp entry point on whether it could explain itself during an incident, and prints +the ones worth fixing. An entry point is a Remix `loader` or `action` under +`apps/webapp/app/routes`, 427 of them at the time of writing. + +The number it prints today is 19 out of 100. That is not a bug, and the rest of this file is mostly +about why you should believe it. + +Every figure below was re-derived from a run of the tool on the tree as it stands. Every invariant +below names the test that holds it, because on this branch a claim written down without one has +turned out to be false more often than not. + +## Running it + +```bash +pnpm --filter @internal/observability-map run map # the whole tree +pnpm --filter @internal/observability-map run map --json # same, as JSON on stdout +pnpm --filter @internal/observability-map run map /api/v1/token # one route, with its check results +``` + +The whole-tree run also writes `observability-map.json` at the repo root, which `--no-write` +suppresses. The single-route mode takes either the route path the report prints (`/api/v1/token`) or +the file name (`api.v1.token.ts`). An exact match wins over the routes it is a prefix of, and an +ambiguous prefix warns and names the alternatives rather than silently picking one +(`src/cli.test.ts`: `prefers an exact match over the routes it is a prefix of`, `warns when a +prefix matches more than one route rather than silently taking the first`). + +## CI + +A PR that touches `apps/webapp/app/routes` or this package gets a sticky comment scanning head +against the tip of the base branch, with the score, what changed, and the current fix list. It is +report-only: nothing here fails the build or blocks a merge, and the gate stays deferred until a +later phase decides to add one. See `.github/workflows/observability-map.yml`. + +This paragraph used to say "merge base", and two reviewers read that against +`github.event.pull_request.base.sha` and reported the workflow as the thing that was wrong. It is +the other way round. `actions/checkout` on a `pull_request` event checks out GitHub's test merge +commit, whose two parents are `base.sha` and the PR head, so the head tree being scanned already +contains everything on the base branch up to `base.sha`. Diffing that against `base.sha` isolates +the PR's own work, which is what the comment is for. A real merge base would be the wrong base +here: it would leave the intervening base-branch commits in the head tree and out of the base tree, +and attribute all of them to the pull request. `git merge-base HEAD ` would not even do +that, since `base.sha` is a parent of `HEAD` and the command returns `base.sha` unchanged. + +## What 19 means + +It is the mean score of the 412 entry points that had at least one applicable check, where an +entry's score is the share of its applicable checks that passed. + +Read that definition carefully before reading the number, because it has a property that will +mislead you otherwise. **Changing which routes a check applies to moves the score without anything +in the webapp changing.** It happened in the round that added this paragraph: widening the sensitive +cohort from 26 routes to 67 gave `auth-boundary` 39 more routes to look at, 36 of which already +passed it, and the global went from 15 to 19. Not one line of `apps/webapp` changed. The same thing +runs in reverse: narrowing a check, or a refactor that takes routes out of the denominator, lowers +or raises it for reasons that are about the tool. So a movement is only evidence about the codebase +once you have checked the CHECKS block below for an applicability change. Compare fix lists, not +scores. It is low because the webapp does +not attach tenant identity to its failures: **11 of 412 entry points name an environment, project, +organization or user on a failure path.** Everything else, when it breaks at 3am, tells you the +route and the request id and nothing about whose request it was. + +The score was 76 until we stopped crediting routes for the error handling they do not do. Emptying +every catch clause in the tree used to score it 100, which meant the metric paid you for deleting +error handling. + +The property behind that is now a test corpus rather than a claim. `src/mutationCorpus.test.ts` +applies 44 semantics-preserving or handling-deleting rewrites to the whole route tree in a temp copy +and asserts three things for each: the published global does not rise, the mean over the routes +measured in both runs does not rise, and for a semantics-preserving rewrite no individual route's +score rises or drops out of the measured set. Every laundering shape a reviewer has found on this +branch is an entry in it, `src/mutations.ts` holds them, and each entry says which it is. + +Two of them are worth naming because they are the ones the design turns on. Deleting every catch +clause in the tree drops the score from 19 to 8, so the metric does not pay you for removing error +handling. Wrapping every body in `try { ... } catch (e) { throw e }` leaves the global at 19 and +raises no route, so it does not pay you for adding error handling that does nothing either. + +The rewrites come in two directions and both matter. A subtractive one takes real signal away or +moves it about: delete the catches, wrap the body, merge the statements. An additive one puts fake +signal in: a classifying catch over a try that does nothing, a test whose two arms are the same, a +rethrow that can never run, a call whose name starts with `require`. The corpus had only the +subtractive half for a while, and the two largest holes ever found here were both additive. + +One of those is still open and the corpus says so. A catch over `try { 0; }` is refused, but +`canRaise` accepts any call, so `try { String(0); }` reads as real error handling: it takes the tree +from 19 to 44 and raises 224 routes. Telling an inert call from one that can throw needs types the +scanner does not have. + +The honest statement is "these 43 rewrites are defended, here they are, and here is the one that is +not", not "unpaddable". One entry, `dead-classifying-try-with-call`, runs as an expected failure +with the residual written out beside it. The corpus takes about four and a half minutes, so it is +gated behind `OBS_MAP_MUTATION_CORPUS=1` and run as its own CI job rather than in `pnpm test`. If +you change this package, run it: + +```bash +OBS_MAP_MUTATION_CORPUS=1 pnpm --filter @internal/observability-map exec vitest run \ + src/mutationCorpus.test.ts --disable-console-intercept +``` + +So the number is deliberately unflattering, and one platform change would move most of it. Nothing +central attaches a tenant: `logger` pushes `{ requestId, path, host, method }` onto every line +through AsyncLocalStorage and forwards errors to Sentry, and the route builders log +`logBoundaryError(message, error, url)`. If the auth path ever pushed `environmentId` through +`trace(...)`, several hundred entry points would flip at once, and this check would want rethinking +rather than celebrating. + +## The five checks + +- **error-classification**: does every catch clause decide what it caught, by branching on the + error or by guarding a parse it can answer for. A clause that only rethrows decides nothing and + is read as though there were no catch, so it neither passes nor fails. +- **auth-boundary**: does a route handling credentials, access control, sessions, billing or + impersonation check who is asking. +- **auth-scope**: a route builder authenticates the request, and its `authorization` option is + optional, so a route can be authenticated and scoped to nobody. This asks whether a sensitive + builder-wrapped route also narrows itself to the caller, in every export, by declaring + `authorization` or by filtering on the caller's own id. +- **request-context**: when this entry point's failure is reported, is the tenant named. +- **audit-trail**: does a sensitive mutation leave a record of who did it. Three routes do, all of + them impersonation paths reaching `prisma.impersonationAuditLog.create` in + `models/admin.server.ts`; the other 46 do not. + +`audit-trail` is excluded from the score. The other four are in it. + +## What the score is made of + +The check list describes a composite the number mostly is not, so the report discloses the +shape instead of hiding it behind a weight. Today: + +```text +CHECKS + error-classification 166 applicable, 94 pass, 0 sole, global without it 10 + auth-boundary 62 applicable, 59 pass, 0 sole, global without it 15 + auth-scope 19 applicable, 17 pass, 0 sole, global without it 18 + request-context 412 applicable, 11 pass, 223 sole, global without it 65 + audit-trail 49 applicable, 3 pass, 0 sole, not in the score +``` + +`sole` is the figure that says the most: 223 of the 412 measured entry points have exactly one +applicable scored check, so their score is 0 or 100 on a single boolean. Read the family bars with +that in mind. They do not compare families on observability in general; they mostly compare them on +whether someone wrote a tenant field into a catch log. + +Weighting was considered and rejected in the design, and that reasoning has not changed: a +coefficient nobody can explain invites argument about the number instead of about the finding. The +block above is in the terminal report and in the JSON as `checkContributions` +(`src/score.test.ts`: `per-check contribution`; `src/report/terminal.test.ts`: `reporting what the score +is made of`). + +## Two findings are headlines, not list entries + +`audit-trail` fails 46 of 49, and `request-context` fails 401 of 412. Printing either one per route +would bury the route-specific findings under the same sentence repeated hundreds of times, so both +are reported as a figure: the `AUDIT` and `CONTEXT` lines. 328 entry points fail nothing except +`request-context` and appear only in that figure, which leaves 76 in the fix list. An entry that +fails `request-context` *and* another scored check keeps both findings and stays in the list, so +`/account/tokens` still shows the whole picture. `audit-trail` does not count as "another" for this +purpose: it is already a headline, so a route failing only `request-context` and `audit-trail` +collapses too (28 do today, all of them sensitive). + +42 of those 328 are sensitive, so the `CONTEXT` line says how many. Read them out of +`observability-map.json`, where every entry keeps its full check results, rather than assuming the +list is the whole story. + +`request-context` is still scored, unlike `audit-trail`. The gap it measures is real and the score +is meant to show it. Only the presentation collapses. + +## Not applicable is not a pass + +An entry with no applicable scored check is `measured: false`, and it is left out of every mean the +report computes (`src/score.test.ts`: `excludes an unmeasured entry point from the global mean`, +`excludes an unmeasured entry point from its family mean too`). Its `score` field reads 100, which +is a placeholder for "nothing was measured here", not a verdict, and nothing averages it. This +matters because the alternative, letting unmeasured entries into the mean at 100, would let the tool +look better the less it understood. The header prints every count (`412 measured, 15 unmeasured`) so +the denominator is never hidden, and a family with nothing measured renders as `not measured` rather +than as a full green bar (`src/report/terminal.test.ts`: `renders a family with nothing measured as not +measured, not as 100`). + +15 of those 427 routes are unmeasured because `isTrivial` (`src/triviality.ts`) rules them out before +any check runs. Trivial means a body of three statements or fewer, three or fewer calls, no +try/catch, no builder wrapping it, and nothing in the calls or the source naming a datastore or a +service (`prisma`, `logger`, `fetch`, `redis`, and the like). Parse the params, build a path, +redirect: nothing there for a check to find evidence in either way. Exclusion is a denominator exit, +not a credit: a trivial route's `score` is the same placeholder 100 that an unmeasured entry always +carries, and it is left out of every mean for the same reason. + +## A route whose body is somewhere else + +`export { action } from "./handler.server"` and `export const action = handleWebhook` are not +trivial routes. A redirect stub genuinely has nothing to instrument; a delegating route has work the +scanner cannot see. Both used to produce the same verdict, so moving a body into a `.server.ts` +file, an ordinary refactor, deleted the route from the metric while the report said nothing. + +Delegating routes are now counted apart from the unmeasured ones, listed on a `DELEGATED` line and +carried in the JSON as `delegating`, the same treatment a parse failure gets and for the same +reason: the denominator is smaller than the entry point count and nothing about these routes has +been checked (`src/score.test.ts`: `refactoring a body out of the route file`, +`a route that delegates its body to another module`; `src/report/terminal.test.ts`: `reporting a route whose +body is in another module`). There are none in the tree today, which is exactly why it would have +gone unnoticed when someone wrote one. + +## When a check declines to judge + +The rule every applicability decision follows: **would this evidence necessarily be visible in the +body if it existed?** + +A log call inside a catch would be, because the catch is right there in the body being read. So its +absence is evidence of absence and `request-context` fails the route. A guard on work that happens +inside an imported helper would not be, because neither the work nor the guard is in the body. So +`auth-boundary` reports not-applicable with a detail saying it could not verify, rather than +accusing the route of being unguarded. `resources.impersonation.ts` is the worked example: it calls +`clearImpersonation`, which authenticates and writes an audit row in `app/models/admin.server.ts`, +a file this tool never opens. Five of the 67 sensitive routes sit out for this reason today. + +The failure mode this rule exists to prevent is a fix list whose top three entries are all wrong. +That happened, twice, and both times the cause was a check asserting something the evidence did not +support. + +## Sensitivity, and the names the tool matches on + +`auth-boundary`, `auth-scope` and `audit-trail` only look at routes `src/sensitivity.ts` calls +sensitive, and that cohort is the fix list's primary sort key, so what goes in it decides what a +reader sees first. 67 routes are in it today: credentials and tokens, envvars, billing and the two +billing settings the bare `billing` segment does not match, impersonation, membership and invites +and roles and the team page, the login surface, API keys, and org or project deletion. + +Two rules hold the vocabulary honest. + +Calling a guard can never be what makes a route sensitive. `requireAdminApiRequest` was on the +symbol list once and made 34 of the then 67 sensitive routes sensitive purely for being guarded, +which `auth-boundary` then passed every one of them for (`src/sensitivity.test.ts`: `does not treat +calling the admin guard as what makes a route sensitive`). + +Every name and every segment has to exist. Half the symbol list once named nothing at all: +`Set.has` is exact, and `setImpersonation`, `createJWT`, `signJWT` and `updateEnvVars` are exported +nowhere in the webapp, so the symbol half of the classifier was quietly doing almost nothing. +`src/webappSymbols.test.ts` resolves every sensitive symbol, every path segment and every entry in +`auth-boundary`'s guard list against `apps/webapp/app` and the two packages the webapp +authenticates through, and fails if one stops resolving. The one exception is +`ANTICIPATED_SEGMENTS`, three words that name no route yet and are held to naming none. + +The same test is what stops `auth-boundary`'s guard list rotting. That check used to match +`/^(require|authenticate)/`, so any callee at all beginning `require` cleared a sensitive route: +`requireSsoEntitlement`, a plan check, cleared the org SSO settings page, and a local +`requireValidParams(request)` would clear whatever route was written next. It also matched +`/Authenticated/`, which passed `resolveAuthenticatedEnv` on ten routes, a `findFirst` by +environment id that authenticates nothing. Both shapes are corpus entries now +(`fake-require-guard`, `fake-authenticated-lookup`): under the patterns they took the tree from 18 +to 19 and raised five routes, and under the accept-list they raise nothing. + +## Authenticated is not the same as scoped + +All nine route builders authenticate the request, which is why `auth-boundary` passes a +builder-wrapped route. Their `authorization` option is optional and `apiBuilder.server.ts` runs the +RBAC gate inside `if (authorization)`, so a route can be authenticated and scoped to nobody. That is +the cross-org IDOR class `apps/webapp/CLAUDE.md` names: "A PAT route must resolve its target +org/project scoped to the caller's membership. Skipping it opens cross-org access." + +`auth-scope` is the check that can say "authenticated but not scoped" as a finding in its own +right. It applies to 19 routes and 17 pass. It reads two things as scoping, and requires EVERY +builder-wrapped export of a file to have one of them: an `authorization` option with a real value, +or a query in that export's own handler filtered on the caller's own id +(`userId: authentication.userId`, `userId: user.id`). + +An `ability.can(...)` call in the handler is deliberately not a third way. The same CLAUDE.md +passage says why: the OSS fallback ability is permissive +(`internal-packages/rbac/src/fallback.ts` returns `permissiveAbility` for a PAT and +`buildFallbackAbility(user.admin)` for a session, neither of which reads org membership), so an +ability check enforces the role while the membership-scoped query is the tenant floor. + +The two routes that fail both resolve their target organization from the URL slug with no +membership filter, and put nothing but an ability check in front of it: +`_app.orgs.$organizationSlug.settings.sso/route.tsx` in its loader, whose `resolveOrg` is +`findFirst({ where: { slug } })`, and `_app.orgs.$organizationSlug.settings.team/route.tsx` in its +action, whose org id comes from `resolveOrgIdFromSlug`. Both were hand-read. The fix in each is to +put `members: { some: { userId } }` on the lookup. + +That per-export rule is the load-bearing half. Both of those files scope themselves in their OTHER +export, so an entry-point-wide reading passed them, and the exposure is per export. + +One thing to know before reading a score on any of these 19 routes. `auth-scope` is only applicable +when the route uses a builder, and `auth-boundary` passes any route that uses a builder, so +**`auth-scope` applicable structurally implies `auth-boundary` pass**: all 19 carry the same +`auth-boundary` detail, "authenticated by the builder". That free point is a third or a quarter of +each of their scores. The 19 average 59.7 as scored and 44.6 with `auth-boundary` taken out, and +`settings.team`, a confirmed cross-org exposure, scores 25 rather than 0 because of it. The score is +not wrong, since the builder does authenticate. It is just less informative here than it looks, and +the finding is the thing to read. + +## Suppression + +```ts +// obs-map-disable auth-boundary -- public by design, see ADR 12 +``` + +The reason is mandatory: a suppression without one is ignored (`src/suppression.test.ts`: `ignores +a suppression with no reason`). The directive is read from comments only, so a string literal +quoting it does not switch a check off (`does not suppress from a directive quoted inside a string +literal`, and six more for template literals and JSX text). + +It applies to the whole entry point, not to the line under it. It was called +`obs-map-disable-next-line`, which was untrue in a way that mattered: a directive on the last line +of a file switched a check off for everything above it. Genuine line scoping is not available, +because a finding is attached to an entry point and carries no line number to match against, so the +name was corrected instead. The old spelling is not honoured (`does not honour the old -next-line +spelling`). + +A suppression cannot raise a score. The suppressed check leaves the numerator and the denominator, +and the result is capped by what the entry would have scored unsuppressed, so suppressing a failing +check holds the number still rather than improving it (`src/score.test.ts`: `does not raise the +score when a failing check is suppressed`; `scoring 100 and 0, suppressing every check on the +failing entry leaves the global at 50`), and `suppress-every-check` is the tree-scale version in +the corpus. What you buy is removal from the worklist with a reason on the record. The report prints how many suppressions are +in force so the practice stays visible. + +## The gaming boundary + +`request-context` checks that a failure-path log names a tenant field. It does not check that the +value is real. A codemod that added `environmentId` to every in-catch `logger.error` call, wiring it +up to the wrong variable or a constant, would move the score exactly as far as one that wired it up +correctly. Measured on the real tree: adding a synthetic `environmentId: "obs-map"` field to the +first object argument of all 139 in-catch log calls, with no other change, takes the global from 19 +to 29 and the CONTEXT figure from 11 to 98. That measurement is a one-off script rather than a +corpus entry, because the corpus asserts that the score must not rise and this rewrite is supposed +to. + +That is the tool verifying presence, not meaning, and it is not a bug to fix. Every check here reads +syntax: a field name, a call, a binding reference. None of them can tell a genuine tenant id from a +hardcoded string with the right key. What a reviewer owns is whether the value behind the field is +real, the same way a Lighthouse accessibility score checks that an `alt` attribute exists and not +that its text describes the image. The number tells you where to look. It does not tell you what +you will find there. + +## Known limits + +Read these before trusting a specific verdict. + +- **One hop, same file only.** If a loader delegates to a helper in the same file, that helper's + statements, catches and calls count as the route's. A helper's own helpers do not, and nothing + imported from another module is ever opened. `auth-boundary` applies to 62 of the 67 sensitive + entry points; the other 5 hand their work to an imported helper and are reported as unverified + rather than unguarded. +- **A guard is matched by name, not by what it does.** The accept-list is 29 names read off the + webapp, plus two `SOFT_GUARDS`. `src/webappSymbols.test.ts` proves each one is declared + somewhere; nothing proves the declaration it found is the guard we meant. `authenticateAdmin` and + `authenticatePlainRequest` are local helpers inside one route file each, so a second route + declaring its own no-op function of either name would be credited. +- **Two guard names are only checked as far as being read.** `getUser` and `getUserId` answer with + null instead of throwing, so calling one is not a boundary. They are credited only when the body + binds the result and some condition reads it (`EntryPoint.checkedCallees`). What that cannot see + is whether the test guards anything: `if (!user) { logger.warn("anonymous"); }` followed by the + work reads the same as returning. +- **`authenticate` and `isAuthenticated` are unresolved on purpose.** They are remix-auth's, and + resolving them meant reading a path inside `apps/webapp/node_modules`, which fails confusingly on + an install-layout change. They are listed in `EXTERNAL_GUARDS` instead, so the resolution test + still rejects a name that is neither first-party nor listed. +- **`auth-scope` cannot tell a caller-id filter from a caller-id actor argument.** + `presenter.call({ userId: user.id })` narrows the query; `generatePortalLink({ organizationId, + userId: user.id })` just records who asked. Both read as scoping. Separating them means following + the argument into the callee, so the four helpers credited this way + (`ApiKeysPresenter`, `TeamPresenter`, `regenerateApiKey`, `DeleteOrganizationService`, all of + which do `members: { some: { userId } }` and throw) were hand-read instead. No route in the tree + passes on an actor argument alone. +- **`auth-scope` reads property assignments in that export's own handler.** A handler that pulls the + id into a local first, `const userId = user.id; ... { userId }`, or that builds its filter in a + same-file helper, scopes itself and is not seen, so it would be reported as unscoped. +- **`auth-scope` reads the builder-wrapped exports and says nothing about the rest of the file.** A + route whose action is builder-wrapped and whose loader is a plain `export async function loader` + is judged on the action alone, and the pass detail, "every builder-wrapped export has an + authorization gate", is true of what it read while reading as a claim about the whole route. Ten + routes in the tree mix the two, and one of them is sensitive, so it is the only one the check runs + on: `_app.orgs.$organizationSlug.settings._index/route.tsx`, whose builder-wrapped action carries + the pass and whose plain loader filters on `members: { some: { userId } }` and is scoped. That was + hand-read; nothing in the check saw it. Widening the check to a hand-written export means deciding + first whether that export is authenticated at all, which is `auth-boundary`'s question rather than + this one. +- **Three login-flow routes fail `auth-boundary` correctly and unhelpfully.** `/auth/sso`, + `/api/v1/authorization-code` and `/api/v1/token` are unauthenticated by design: the caller is + anonymous at that point, which is the whole purpose. The check's statement about them is true and + there is nothing to fix, so they are candidates for a suppression comment with the reason on the + record. +- **Loggers are matched by spelling.** A call counts as logging when the callee reads `logger.*` or + `log.*`. An aliased logger, one wrapped in a helper, or `console.error` is invisible, so a route + can be reported as recording nothing while it records plenty. +- **A catch that logs and rethrows reads as though it only rethrows.** The clause evidence cannot + say whether a clause does anything besides rethrow, so `error-classification` withholds credit + rather than granting it. Crediting it would reopen the free-points path a single `logger.error` + line wide. +- **Only the first object-literal argument is read** for identifier fields, and only its property + names. `logger.error("failed", ctx)` where `ctx` is a variable contributes nothing, and neither + does a second object. +- **A catch inside a per-item callback is not the route's.** `items.map((item) => { try {...} })` + is a fresh boundary per element, so its clause is not read as the route's own error handling. The + test is the method name, which cannot tell `users.map` from `Result.map`. Being wrong there costs + precision rather than points: a refused catch fails the route rather than excusing it, so no + wrapper can turn a swallow into a not-applicable by getting the boundary rule to refuse it. +- **A route that delegates only one of its two exports is judged on the other.** + `export { action } from "./x"` beside a loader written in the file is not counted as delegating, + so half the route is scored and half is invisible. +- **`try { String(0); }` still buys a pass.** The open corpus entry, above. It is the largest single + hole known in the tool: measured live, it takes the tree from 19 to 44 and raises 224 routes. +- **A forged tenant field buys a pass too.** The gaming boundary above, restated here because it + belongs on this list: `request-context` reads the field name, never the value, so a codemod + writing `environmentId: "obs-map"` into every in-catch log call takes the global from 19 to 29. + Unlike the entry above this one is not a bug to fix, since no syntactic check can tell a real + tenant id from a constant, but it bounds what the number can mean either way. +- **The score is a mean of means over a heuristic.** Read the fix list, the two headline figures and + the CHECKS block. Watching the single number for small movements will mislead you. + +## Layout + +`scan.ts` walks the routes directory and produces an `EntryPoint` per module, carrying only +body-scoped evidence. `checks/` holds the five checks, each a pure function of an `EntryPoint`. +`score.ts` turns checks into an entry score and a report, `report/` renders it, `cli.ts` is the +entry point. `sensitivity.ts`, `triviality.ts` and `suppression.ts` are the three inputs the checks +share. + +Tests sit next to their subject in `src/`. Every check has a false-positive fixture, something it +must not flag, alongside the positive one. Keep that: most of the bugs this package has had were +checks that fired on the wrong thing, and a test that only proves the heuristic fires would have +caught none of them. diff --git a/internal-packages/observability-map/package.json b/internal-packages/observability-map/package.json new file mode 100644 index 00000000000..cd8c3fc14f4 --- /dev/null +++ b/internal-packages/observability-map/package.json @@ -0,0 +1,25 @@ +{ + "name": "@internal/observability-map", + "private": true, + "version": "0.0.1", + "type": "module", + "main": "./dist/src/index.js", + "types": "./dist/src/index.d.ts", + "dependencies": { + "typescript": "catalog:" + }, + "devDependencies": { + "@types/node": "^24.13.3", + "rimraf": "6.0.1", + "tsx": "^4.19.2", + "vitest": "4.1.7" + }, + "scripts": { + "clean": "rimraf dist", + "typecheck": "tsc --noEmit", + "build": "pnpm run clean && tsc -p tsconfig.build.json", + "test": "vitest run", + "test:watch": "vitest", + "map": "tsx src/cli.ts" + } +} diff --git a/internal-packages/observability-map/src/adapters/remix.test.ts b/internal-packages/observability-map/src/adapters/remix.test.ts new file mode 100644 index 00000000000..be45ce6c62c --- /dev/null +++ b/internal-packages/observability-map/src/adapters/remix.test.ts @@ -0,0 +1,57 @@ +import { familyOf, routePathOf } from "./remix.js"; + +describe("familyOf", () => { + it("classifies each family from the flat-route filename", () => { + expect(familyOf("api.v1.runs.$runId.ts")).toBe("api.v1"); + expect(familyOf("api.something.ts")).toBe("api.other"); + expect(familyOf("admin.api.v1.gc.ts")).toBe("admin"); + expect(familyOf("resources.queues.ts")).toBe("resources"); + expect(familyOf("_app.orgs.$slug.ts")).toBe("dashboard"); + expect(familyOf("otel.v1.logs.ts")).toBe("ingest"); + expect(familyOf("@.ts")).toBe("other"); + }); + + it("prefers admin over api.v1 for admin-prefixed api routes", () => { + expect(familyOf("admin.api.v1.environments.$id.ts")).toBe("admin"); + }); +}); + +describe("routePathOf", () => { + it("turns a flat-route filename into a path", () => { + expect(routePathOf("api.v1.runs.$runId.ts")).toBe("/api/v1/runs/:runId"); + }); + + it("strips a trailing method suffix", () => { + expect(routePathOf("api.v1.runs.ts")).toBe("/api/v1/runs"); + }); +}); + +// Directory routes: the scanner recurses into Remix directory routes, so `fileName` can be a +// relative path like `_app.orgs.$organizationSlug.projects.$projectParam/route.tsx` rather than a +// flat dot-separated name. The directory name (not `route.tsx`) carries the meaning. +describe("familyOf: directory routes", () => { + it("classifies a directory route by its directory name, not the flat rules", () => { + expect(familyOf("_app.orgs.$organizationSlug.projects.$projectParam/route.tsx")).toBe( + "dashboard" + ); + }); + + it("classifies each family the same whether the route is flat or a directory", () => { + expect(familyOf("api.v1.runs.$runId/route.tsx")).toBe("api.v1"); + expect(familyOf("admin.api.v1.environments.$id/route.tsx")).toBe("admin"); + expect(familyOf("resources.queues/route.tsx")).toBe("resources"); + expect(familyOf("storybook.callout/route.tsx")).toBe("other"); + }); +}); + +describe("routePathOf: directory routes", () => { + it("does not emit a literal 'route' segment for a directory route", () => { + const path = routePathOf("_app.orgs.$organizationSlug.projects.$projectParam/route.tsx"); + expect(path).not.toContain("route"); + expect(path).toBe("/_app/orgs/:organizationSlug/projects/:projectParam"); + }); + + it("produces the same path shape for a directory route as its flat equivalent", () => { + expect(routePathOf("api.v1.runs.$runId/route.tsx")).toBe(routePathOf("api.v1.runs.$runId.ts")); + }); +}); diff --git a/internal-packages/observability-map/src/adapters/remix.ts b/internal-packages/observability-map/src/adapters/remix.ts new file mode 100644 index 00000000000..7815fa3e6a4 --- /dev/null +++ b/internal-packages/observability-map/src/adapters/remix.ts @@ -0,0 +1,44 @@ +export type Family = + | "api.v1" + | "api.other" + | "webhooks" + | "admin" + | "resources" + | "dashboard" + | "ingest" + | "other"; + +/** + * The name that carries routing meaning for a given `fileName`. + * + * Flat routes (`api.v1.runs.$runId.ts`) are already that name. Directory routes + * (`_app.orgs.$slug/route.tsx`) hold their module in a fixed `route.ts`/`route.tsx` file, so the + * directory segment before the slash is the meaningful name and `route.tsx` itself is not a path + * segment. + */ +function routeName(fileName: string): string { + const slashIndex = fileName.indexOf("/"); + return slashIndex === -1 ? fileName : fileName.slice(0, slashIndex); +} + +export function familyOf(fileName: string): Family { + const name = routeName(fileName); + // admin is checked first: admin.api.v1.* is an admin route, not an api.v1 one. + if (name.startsWith("admin.")) return "admin"; + if (name.startsWith("api.v1.")) return "api.v1"; + if (name.startsWith("api.")) return "api.other"; + if (name.startsWith("webhooks.")) return "webhooks"; + if (name.startsWith("resources.")) return "resources"; + if (name.startsWith("_app.")) return "dashboard"; + if (name.startsWith("otel.") || name.startsWith("engine.")) return "ingest"; + return "other"; +} + +export function routePathOf(fileName: string): string { + const withoutExt = routeName(fileName).replace(/\.(ts|tsx)$/, ""); + const segments = withoutExt + .split(".") + .filter((s) => s.length > 0) + .map((s) => (s.startsWith("$") ? `:${s.slice(1)}` : s)); + return `/${segments.join("/")}`; +} diff --git a/internal-packages/observability-map/src/checks/auditTrail.ts b/internal-packages/observability-map/src/checks/auditTrail.ts new file mode 100644 index 00000000000..001e5f6e8dc --- /dev/null +++ b/internal-packages/observability-map/src/checks/auditTrail.ts @@ -0,0 +1,74 @@ +import type { CheckResult, EntryPoint } from "../types.js"; +import { classifySensitivity } from "../sensitivity.js"; +import { isTrivial } from "../triviality.js"; + +const ID = "audit-trail"; + +/** + * Calls that write a record of who did something. + * + * This list named nothing at all until it was checked. `auditLog`, `recordAudit` and + * `writeAuditEvent` are exported nowhere in apps/webapp, packages/core or internal-packages, + * so the pass branch below could never fire, every applicable route failed, and both renderers + * printed "No audit helper exists in the webapp" while `models/admin.server.ts` was writing + * `prisma.impersonationAuditLog.create({ action, adminId, targetId, ipAddress })` on two paths. + * The rot went unnoticed because `webappSymbols.test.ts` covered every other name list in the + * package and not this one. It covers this one now. + * + * All three names below reach that write: `redirectWithImpersonation` writes the START row, + * `clearImpersonation` writes the STOP row, and `startImpersonation` returns one or the other. + * + * Two of them are also in `SENSITIVE_SYMBOLS`, which is worth saying out loud because it looks like + * the circularity the sensitivity list was cleaned up to remove. It is not quite the same shape: + * `requireAdminApiRequest` was a pure mitigation counted as a hazard, whereas impersonation + * genuinely is the hazard AND genuinely writes the record. The consequence is real all the same, so + * here it is: a route made sensitive only by one of these calls cannot fail this check, because the + * call that put it in the cohort is the call that satisfies it. + * + * Matched against `importedNames` and `calleeNames`, so an import of one counts. Nothing matches + * the underlying `prisma.impersonationAuditLog.create` path directly: `calleeNames` records + * `create` for a member call, and no route in the tree writes the row itself. + */ +export const AUDIT_SYMBOLS = [ + "redirectWithImpersonation", + "clearImpersonation", + "startImpersonation", +]; + +/** + * Whether a sensitive mutation leaves a record of who did it. + * + * Applicability follows the same rule as every other check, which it did not before: would this + * evidence necessarily be visible in the body if it existed? It gated on sensitivity and + * `hasAction` alone, so on `resources.impersonation.ts`, a four-statement body, `auth-boundary` + * declined to judge because any guard would be behind the import while this check accused the route + * over an audit write behind that same import. Two checks, opposite verdicts, one fact. + * + * So a trivial body is not-applicable here too. A delegating one is handled centrally by + * `scoreEntry`, which answers for every check before any of them runs, so there is no test for it + * here. The order matters and mirrors + * `auth-boundary`: a known audit call is read BEFORE the triviality exemption, because presence is + * evidence even where absence is not. + */ +export const auditTrail = { + id: ID, + run(ep: EntryPoint): CheckResult { + // Mutations only: a sensitive read does not need an actor record. + if (!classifySensitivity(ep).sensitive || !ep.hasAction) { + return { id: ID, status: "not-applicable", detail: "not a sensitive mutation" }; + } + const symbols = new Set([...ep.importedNames, ...ep.calleeNames]); + if (AUDIT_SYMBOLS.some((s) => symbols.has(s))) { + return { id: ID, status: "pass", detail: "records an audit event" }; + } + if (isTrivial(ep)) { + return { + id: ID, + status: "not-applicable", + detail: + "cannot verify: no privileged work in the body, any audit write is behind an import", + }; + } + return { id: ID, status: "fail", detail: "sensitive mutation with no audit record" }; + }, +}; diff --git a/internal-packages/observability-map/src/checks/authBoundary.ts b/internal-packages/observability-map/src/checks/authBoundary.ts new file mode 100644 index 00000000000..4abbeb7dc73 --- /dev/null +++ b/internal-packages/observability-map/src/checks/authBoundary.ts @@ -0,0 +1,193 @@ +import type { CheckResult, EntryPoint } from "../types.js"; +import { classifySensitivity } from "../sensitivity.js"; +import { routeExports, type ExportName, type RouteExport } from "../routeExports.js"; +import { isTrivialExport } from "../triviality.js"; +import { BUILDERS } from "./errorClassification.js"; + +const ID = "auth-boundary"; + +/** + * The guard helpers this webapp actually has, matched against the calling export's own + * `loaderCalleeNames`/`actionCalleeNames`, each scoped to that export's handlers and following one + * hop into a same-file helper. A guard the route only imports and never calls does not count, and + * neither does one the OTHER export calls. + * + * A name list rather than the three patterns it replaces, because all three over-matched and this + * is the one check where a false pass hides a security gap: + * + * - `/^(require|authenticate)/` passed any callee at all beginning `require`. Live in the tree: + * `requireSsoEntitlement`, a plan check, cleared `_app.orgs.$organizationSlug.settings.sso`. A + * local `requireValidParams(request)` would do the same for any route someone writes next. + * - `/Authenticated/` passed `resolveAuthenticatedEnv`, used by ten routes, which is + * `findFirst({ where: { id: environmentId } })` in + * `internal-packages/run-engine/src/engine/controlPlaneResolver.ts`: it hydrates an environment + * record, it authenticates nothing. The docstring that put it here asserted the opposite. It also + * passed `commitAuthenticatedSession`, a cookie write, on six routes. + * - `/^verify.*(Hash|Hmac|Signature|Webhook|Callback|Token)/` was sound on the tree, and is kept as + * three names for the same reason as the rest. + * + * Every name resolves to a declaration in the webapp or in the packages it authenticates through; + * `webappSymbols.test.ts` fails if one stops doing so. What that test cannot check is that a + * declaration with the right name is the guard we meant: `authenticateAdmin` and + * `authenticatePlainRequest` are local helpers inside a single route file, so a second route + * declaring its own no-op `authenticateAdmin` would be credited. That is a narrower hole than a + * five-character prefix and it is the reason the list is names rather than patterns. + */ +export const GUARDS = new Set([ + // Session and PAT identity, `apps/webapp/app/services/session.server.ts` and friends. + "requireUser", + "requireUserId", + "requireOrganization", + "requireAdminApiRequest", + "authenticateApiRequest", + "authenticateApiRequestWithFailure", + "authenticateApiRequestWithPersonalAccessToken", + "authenticateApiRequestWithOrganizationAccessToken", + "authenticateApiKey", + "authenticateAuthorizationHeader", + "authenticateOrganizationAccessToken", + "authenticatePersonalAccessToken", + "authenticateRequest", + "authenticateAdminRequest", + "authenticatedEnvironmentForAuthentication", + "authenticateAndAuthorize", + // The RBAC controller, `packages/plugins/src/rbac.ts`, reached as `rbac.authenticateSession(...)`. + // `calleeName` records the property for a member call, so these arrive here unqualified. + "authenticateSession", + "authenticatePat", + "authenticateBearer", + "authenticateUserActor", + "authenticateAuthorizeSession", + "authenticateAuthorizeBearer", + // remix-auth, reached as `authenticator.authenticate(...)` / `authenticator.isAuthenticated(...)`. + // The login surface is sensitive under `sensitivity.ts` and cannot require an already + // authenticated caller, so establishing identity from the credential presented is what a guard + // means there. + "authenticate", + "isAuthenticated", + // Local helpers, each declared inside the one route that uses it. + "authenticateAdmin", + "authenticatePlainRequest", + // Proof of possession: a callback URL carrying an HMAC is authenticated by checking that HMAC, + // and a login-surface second factor is authenticated by checking the code presented. + // `login.mfa`'s action is the second half of a login, so like `authenticate` above it establishes + // identity from the credential rather than requiring an already authenticated caller. It reached + // this list when per-export attribution stopped its loader's `isAuthenticated` speaking for it. + "verifyHttpCallbackHash", + "verifyWebhook", + "verifyUserActorToken", + "verifyTotpForLogin", + "verifyRecoveryCodeForLogin", +]); + +/** + * Guards that answer with null instead of throwing. Calling one is not evidence of a boundary, + * because the route is free to ignore the answer, so these are only credited when THAT EXPORT's + * handlers demonstrably read what they returned (`EntryPoint.loaderCheckedCallees`). + * + * The distinction is the whole reason this set is separate from `GUARDS`. `requireUserId` redirects + * on its own, so calling it IS the boundary; `getUserId` hands back `string | null` and a route + * that drops it has no boundary at all. Both routes in the sensitive cohort that use one read the + * answer, `invite-accept.tsx` refusing an invite addressed to another email and `login._index` + * sending an already-authenticated caller away, and this rule is what makes that a measured fact + * rather than something a hand-read established once. + * + * What it still cannot see is whether the test that reads the answer guards anything. See + * `EntryPoint.loaderCheckedCallees` for the exact shape of that residual. + */ +export const SOFT_GUARDS = new Set(["getUser", "getUserId"]); + +type GuardedExport = { name: ExportName; guarded: boolean; how: string; export: RouteExport }; + +/** + * The exports this file declares, each with its own verdict. + * + * Per export, because the exposure is per export, and this is the same defect `auth-scope` was + * fixed for one round earlier. Every input here was entry-point-wide: `calleeNames` is the union of + * both bodies, `checkedCallees` was too, and `usesBuilder` was an OR over the two initializer + * callees. So a file whose loader called `requireUser` and whose action called nothing read as + * "guarded in the body", and a file whose loader was `createLoaderApiRoute(...)` credited its + * hand-written action with the builder's authentication. Three inputs, one bug, and it is a false + * PASS on the one check where that hides a security gap. + * + * `routeExports` lists only the exports the file actually declares, so an export that calls nothing + * at all is judged rather than skipped: an empty body is exactly the unguarded case. It is shared + * with `auth-scope`, which grew its own copy of the same `[loader, action]` literal. + */ +function guardedExports(ep: EntryPoint): GuardedExport[] { + return routeExports(ep).map((e) => { + const verdict = (guarded: boolean, how: string) => ({ name: e.name, guarded, how, export: e }); + if (e.initializerCallee !== null && BUILDERS.has(e.initializerCallee)) { + return verdict(true, "authenticated by the builder"); + } + if (e.calleeNames.some((n) => GUARDS.has(n))) { + return verdict(true, "guarded in the body"); + } + if (e.checkedCallees.some((n) => SOFT_GUARDS.has(n))) { + return verdict(true, "resolves the caller and reads the answer"); + } + return verdict(false, ""); + }); +} + +/** + * Whether a route that handles credentials, tokens or money checks who is asking. + * + * A fail here is an accusation, and it is only supportable when the body is the place a guard + * would have to be. That holds when the route does its privileged work in the open: reads the + * request, queries the datastore, mints the token. It does not hold for a trivial body, so those + * are reported not-applicable rather than failed. + * + * The reasoning is the triviality rule's own definition rather than a convenience. A trivial body + * has three statements or fewer, three calls or fewer, no try/catch, no builder, and no mention of + * prisma, redis, fetch or the engine anywhere in its source. It therefore cannot contain a visible + * privileged operation. Either it does nothing privileged at all, like the `/orgs/:slug/billing` + * redirect stub, or the privileged work sits behind an import, like `clearImpersonation`, which + * authenticates and writes an audit row in `app/models/admin.server.ts`. In the second case the + * guard is in the same unopened file as the work. Absence of evidence, and reporting it as a + * finding puts a wrong answer at the top of the fix list. + * + * This is not the rule `request-context` uses, deliberately. There the thing being looked for, a + * field on a log call inside a catch, would be in the body if it existed at all, because the catch + * is in the body. Absence of a log is evidence. Here the thing being looked for guards work that + * is not in the body either, so its absence proves nothing. The test that separates them: would + * this evidence necessarily be visible in the body if it existed? + * + * The design also matched `importedNames`. Across the 67 sensitive entry points that widening + * changes nothing, every route with a `require*` import calls it from the body too, so the + * file-wide half only ever stood to hand out a pass for a dead import. It is gone. + */ +export const authBoundary = { + id: ID, + run(ep: EntryPoint): CheckResult { + const sensitivity = classifySensitivity(ep); + if (!sensitivity.sensitive) { + return { id: ID, status: "not-applicable", detail: "not sensitive" }; + } + // Never empty: `scanFile` returns null unless the file declares a loader or an action. + const exports = guardedExports(ep); + const guarded = exports.filter((e) => e.guarded); + // Triviality excuses per export, matching the attribution: the reasoning below is about one + // body being the place a guard would have to be, and reading it entry-point-wide let a busy + // action make a redirect-stub loader answerable for a guard it has nothing to guard. + const accused = exports.filter((e) => !e.guarded && !isTrivialExport(e.export)); + if (accused.length > 0) { + return { + id: ID, + status: "fail", + detail: `sensitive (${sensitivity.reasons.join(", ")}) with no auth guard in the body: ${accused + .map((e) => e.name) + .join(", ")}`, + }; + } + if (guarded.length > 0) { + const how = [...new Set(guarded.map((e) => e.how))].join(" and "); + return { id: ID, status: "pass", detail: how }; + } + return { + id: ID, + status: "not-applicable", + detail: "cannot verify: no privileged work in the body, any guard is behind an import", + }; + }, +}; diff --git a/internal-packages/observability-map/src/checks/authScope.ts b/internal-packages/observability-map/src/checks/authScope.ts new file mode 100644 index 00000000000..832928fa620 --- /dev/null +++ b/internal-packages/observability-map/src/checks/authScope.ts @@ -0,0 +1,109 @@ +import type { CheckResult, EntryPoint } from "../types.js"; +import { classifySensitivity } from "../sensitivity.js"; +import { routeExports } from "../routeExports.js"; +import { BUILDERS } from "./errorClassification.js"; + +const ID = "auth-scope"; + +type BuilderExport = { name: string; callee: string; scoped: boolean; why: string }; + +/** + * The builder-wrapped exports of an entry point, each with its own verdict. + * + * Per export, because the exposure is per export. `authorization` is declared on the builder call + * one export made, and a caller filter is written in the handler one export runs, so neither says + * anything about the other half of the file. + * + * The enumeration itself is `routeExports`, shared with `auth-boundary`, which had to be given the + * same per-export treatment a round later and wrote a second copy of this literal to get it. + */ +function builderExports(ep: EntryPoint): BuilderExport[] { + return routeExports(ep) + .filter((e) => e.initializerCallee !== null && BUILDERS.has(e.initializerCallee)) + .map((e) => { + const authorization = e.builderOptions.includes("authorization"); + return { + name: e.name, + callee: e.initializerCallee!, + scoped: authorization || e.scopesByCaller, + why: authorization ? "an authorization gate" : "a filter on the caller's identity", + }; + }); +} + +/** + * Whether a route the builder authenticated is also narrowed to the caller. + * + * `auth-boundary` passes every builder-wrapped route, and that is correct as far as it goes: the + * nine builders in `BUILDERS` all authenticate. What they do not all do is authorize. + * `authorization` is an optional option and `apiBuilder.server.ts` runs the RBAC gate inside + * `if (authorization)`, so a PAT route can be authenticated and completely unscoped. A PAT names + * its target org or project by id or slug, and with no plugin installed the OSS fallback ability is + * permissive, so nothing on that path stops a member of one org naming another org's project. + * `apps/webapp/CLAUDE.md` states the rule this measures: "A PAT route must resolve its target + * org/project scoped to the caller's membership. Skipping it opens cross-org access." + * + * Two ways for an export to be scoped, and EVERY builder-wrapped export has to be one of them: + * + * - its builder options declare `authorization:` with a real value, which is the RBAC gate, or + * - its own handler filters by the caller's own id, the + * `members: { some: { userId: authentication.userId } }` shape in `api.v1.projects.ts` and the + * `presenter.call({ userId: user.id })` shape the dashboard routes use. + * + * `ability.can(...)` in the handler is deliberately NOT a third way, and it was one for part of + * round C. `apps/webapp/CLAUDE.md` is explicit that it cannot be: the OSS fallback ability is + * permissive (`internal-packages/rbac/src/fallback.ts` returns `permissiveAbility` for a PAT and + * `buildFallbackAbility(user.admin)` for a session, neither of which reads org membership), so an + * ability check enforces the ROLE and the membership-scoped query is the tenant floor. Crediting it + * made this check agree with a route that resolves its target org from a URL slug and puts nothing + * else in front of it. + * + * Applicable only where it is answerable: sensitive and builder-wrapped. Outside + * that it would be a second near-universal fail, which is the shape the `request-context` figure + * already has and which the report has to collapse rather than list. There is no triviality test + * here because there is nothing left for one to refuse: `isTrivial` answers false for any route + * with an initializer callee, so a builder-wrapped route is never trivial. Nor is there a + * delegating test: `scoreEntry` answers not-applicable for a delegating entry before any check + * runs, so one here would be unreachable, and one WAS here saying otherwise. + * + * Three residuals, running in both directions. + * + * Accusing: `scopesByCallerIn` reads property assignments in that export's own handlers only. A + * handler that pulls the id into a local first, `const userId = user.id; ... { userId }`, or that + * builds its filter in a same-file helper, scopes itself and is not seen. + * + * Crediting: a caller id passed as an ACTOR argument rather than as a filter still counts. + * `ssoController.generatePortalLink({ organizationId: orgId, userId: user.id })` records who asked; + * it does not constrain which org is read. Telling the two apart means knowing what the callee does + * with the argument, which for the dashboard means following it into a presenter. No route in the + * tree is credited by this alone today. + * + * Crediting: a helper that runs the membership query for you is credited through the caller id + * handed to it. `ApiKeysPresenter`, `TeamPresenter`, `regenerateApiKey` and + * `DeleteOrganizationService` all do `members: { some: { userId } }` internally and throw when it + * misses, which makes those four correct, and it is the same syntax as the actor-argument case + * above. All were hand-read in round C. + */ +export const authScope = { + id: ID, + run(ep: EntryPoint): CheckResult { + if (!classifySensitivity(ep).sensitive) { + return { id: ID, status: "not-applicable", detail: "not sensitive" }; + } + const builders = builderExports(ep); + if (builders.length === 0) { + return { id: ID, status: "not-applicable", detail: "no route builder to read options from" }; + } + const unscoped = builders.filter((b) => !b.scoped); + if (unscoped.length === 0) { + const how = [...new Set(builders.map((b) => b.why))].join(" and "); + return { id: ID, status: "pass", detail: `every builder-wrapped export has ${how}` }; + } + const which = unscoped.map((b) => `${b.name} (${b.callee})`).join(", "); + return { + id: ID, + status: "fail", + detail: `authenticated but not scoped to the caller: ${which} declares no authorization gate and does not filter by the caller's identity`, + }; + }, +}; diff --git a/internal-packages/observability-map/src/checks/errorClassification.ts b/internal-packages/observability-map/src/checks/errorClassification.ts new file mode 100644 index 00000000000..34cfce33d34 --- /dev/null +++ b/internal-packages/observability-map/src/checks/errorClassification.ts @@ -0,0 +1,253 @@ +import type { CatchEvidence, CheckResult, EntryPoint } from "../types.js"; +import { isTrivial } from "../triviality.js"; + +const ID = "error-classification"; + +/** + * The route builders that authenticate the request, which is what `auth-boundary` reads them for. + * They also catch and classify, passing a thrown `Response` through untouched and reporting + * anything else through `logBoundaryError`, but `error-classification` no longer credits that: a + * route with no catch of its own is judged on nothing, wrapper or not. + * + * `createSSELoader` is deliberately absent. It turns a non-Response error into a 500 but does not + * authenticate, so counting it here would hand two routes a free pass on `auth-boundary`. + * `createHybridActionApiRoute`, which the design named, exists nowhere in the tree. + */ +export const BUILDERS = new Set([ + "createLoaderApiRoute", + "createActionApiRoute", + "createLoaderPATApiRoute", + "createActionPATApiRoute", + "createMultiMethodApiRoute", + "createLoaderWorkerApiRoute", + "createActionWorkerApiRoute", + "dashboardLoader", + "dashboardAction", +]); + +/** + * How much a try block may guard and still count as narrow. Two, so the guarded operation can bind + * its result (`const stripped = ...; new RegExp(stripped);`), but a third statement means the try + * has started to cover the handler rather than one operation. The idiom this was chosen for and + * hand-read against originally: 55 of 427 entry points, 11 of the failures at the time, all eleven + * the deliberate `try { body = await request.json() } catch { 400 }` shape. + * + * An absolute count, not a ratio against the enclosing body. A ratio is diluted by anything else in + * the same body: padding the action with unrelated statements after the try relabelled the same + * broad swallow as a narrow guard, moving the denominator without touching the clause at all. + * `inert-statements-after-try` in the mutation corpus is that shape, and it holds. + * + * What the count is NOT is unpaddable, which an earlier docstring and commit subject both claimed. + * `countStatement` now counts declarators and comma operands rather than semicolons, so the two + * known ways to pack a try into fewer statements move the number the same as writing it out; that + * is what `merge-declarations` and `merge-comma-expressions` in the corpus check. A third + * way nobody has written down would work, which is why the count is no longer the only condition + * and no longer the load-bearing one. + */ +const NARROW_TRY_STATEMENTS = 2; + +/** + * Whether a catch clause is a guard rather than the route's error handling: the try block parses, + * waits for nothing except that parse, and is short. + * + * `awaitsOnlyParse` is the condition the previous wave was missing, and it is the one a statement + * count cannot express. `try { const body = await request.json(); return await handleEverything(body); } + * catch { return 500; }` is two statements, one of them a parse, and the whole handler inside it: + * the count reads it as narrow and it is the `otel.v1.logs.ts` swallow written compactly. Asking + * what the block waits for separates them, and unlike the count it does not care how the statements + * are punctuated or how deeply the work is nested inside one of them. + * + * The design's own suggestion, requiring the clause to answer with a 4xx, was measured first and is + * not used. On its own it credits 11 clauses guarding four to thirty statements, the widest swallows + * in the tree, including `admin.api.v1.workers.ts`, whose 28-statement try answers every failure + * with a 400 carrying the internal error message. Added on top it costs three routes their pass, + * all three narrow parse guards that compute a fallback value rather than answering a request + * (`try { return new URL(referer).origin; } catch { return undefined; }`), and it buys only the case + * of a narrow parse guard answering 500. Requiring every CALL to be a parse, rather than every + * await, was measured too and is worse still: it refuses the four `matchPattern.slice(4); new + * RegExp(...)` guards, because preparing a parse's input is ordinary synchronous string work. + * + * Two residuals, since awaiting is the signal. A try block that does its non-parse work + * synchronously still reads as a guard. And `guardedWork` looks for a `ts.AwaitExpression`, which + * `for await (const chunk of work(await request.json()))` and `await using` are not, so a block + * whose only non-parse work is one of those reads as a guard too. Neither occurs in the tree and + * neither is reachable by rewriting a real route, since both need work that is not there to begin + * with. Both are in the round A fix 3 report. + */ +function isParseGuard(clause: CatchEvidence): boolean { + return ( + clause.guardsParse && + clause.awaitsOnlyParse && + clause.tryStatementCount <= NARROW_TRY_STATEMENTS + ); +} + +/** + * Whether a clause decides anything about the error it caught. Two ways to qualify: it branches, on + * an `if`, a `switch` or an `instanceof`, or it guards a parse it can answer for. + * + * Rethrowing is not a third way, which is the correction from the last wave. A clause whose only + * effect is `throw e` leaves the error propagating exactly as it would with no catch at all, so + * treating that as a pass while no catch is not-applicable paid 50 points a route for wrapping a + * body in `try { ... } catch (e) { throw e }`, and 27 across the tree. The two are observationally + * identical and are now scored identically. + * + * The cost is real and worth stating: `catch (e) { logger.error(...); throw e }` also reads as + * inert, because `CatchEvidence` cannot say whether a clause does anything besides rethrow. That + * withholds credit from a route that reports before propagating, which is the safe direction to be + * wrong in, since crediting it would reopen the hole a bare `logger.error` line wide. + * `request-context` still reads that log and asks whether it names a tenant, so the reporting is + * unrewarded here rather than unmeasured. + * + * A narrow guard is not a way to qualify either. A one-statement try around `await + * service.call(run)` is narrow and is still a swallow: reading all eleven entry points that limb + * would clear said six were real, including a silent run cancellation and two credential paths + * that report a database failure to the browser as a 400 with an internal message in it. + */ +function decides(clause: CatchEvidence): boolean { + return clause.branches || isParseGuard(clause); +} + +/** Passes the error through unchanged, which is the same outcome as not catching it. */ +function inert(clause: CatchEvidence): boolean { + return clause.rethrows && !decides(clause); +} + +/** The error stops here and nothing chose what it meant. */ +function swallows(clause: CatchEvidence): boolean { + return !decides(clause) && !inert(clause); +} + +/** + * Who decides what a failure means, and on what evidence. + * + * Judged per catch clause, so an entry point is only as good as its worst one. That is the point of + * the per-clause evidence: 39 routes have more than one catch and 17 mix a narrow guard with a + * broad handler, and under the old aggregate booleans a single well-behaved catch spoke for the + * swallow next to it. + * + * A route with no catch is not-applicable, not a pass. It makes no classification decision, so + * there is nothing here to judge and nothing to credit. Crediting it was worse than merely + * generous: with `request-context` also passing the same routes, emptying every catch clause in the + * tree scored it 100, so the metric paid you for deleting error handling. Out of the denominator + * is the honest place for it, and it takes the builder credit with it: a builder-wrapped route with + * no catch of its own now sits out too, rather than collecting a point for the wrapper. + * + * "Does this route catch anything" is `catches.length`, never `hasTryCatch`. A try/finally with no + * catch leaves `hasTryCatch` true and `catches` empty: nothing is swallowed there, the error + * propagates once the cleanup has run, and reading the old flag as a catch put + * `admin.api.v1.runs-replication.status.ts` at the top of the first rendered fix list. + * + * `callbackCatches` is the third case, and it is what stops "no catch is not-applicable" from being + * a payout. A refused catch is judged on its evidence, never on its placement: the same + * `catchClauseEvidence` an own catch gets, with two arms reading it. A refused swallow fails the + * route whenever nothing the route owns decides, and that arm is deliberately not conditioned on + * the route owning no catches, so an own inert rethrow catch cannot lift a refused swallow out of + * the verdict (`fails a per-item swallow even when the route owns an inert rethrow catch`). A + * route whose only catches are refused and none of them swallows sits out, and never passes: the + * not-applicable ceiling is what keeps a prepended dead deciding `.map` from minting a pass on the + * 261 catchless routes, which `dead-deciding-map` in the mutation corpus holds at tree scale and + * `sits out a catchless route with a prepended dead deciding map` pins on a fixture. What the old + * blanket placement rule blocked, relocating a swallow behind the boundary, still fails + * (`still fails a swallow wrapped in a non-array receiver's .map(...)`); what it wrongly accused, + * a route whose only error handling genuinely is per item, now sits out instead of failing + * (`sits out a route whose only catch is a deciding per-item boundary`). + * + * A clause whose try block holds nothing that could raise is read as no clause at all, + * `guardCanRaise` on the evidence. Prepending `try { 0; } catch (e) { if (e instanceof Error) { + * return json(x, { status: 400 }); } throw e; }` to every body takes the tree from 19 to 44 and + * raised 224 routes, because the 261 routes that catch nothing were sitting at not-applicable and a + * dead clause moved each of them to pass. + * + * What that refuses is `try { 0; }`, and it is defeated by one inert call: `try { String(0); }` + * reads as classification and pays the same 224 routes, because `canRaise` accepts any call at all. + * The rule closes the shape that was found, not the family, and telling an inert call from a + * throwing one needs types the scanner does not have. `dead-classifying-try-with-call` in the + * mutation corpus is the open shape, running as an expected failure. + * + * The refused-swallow arm reads the route's own deciding catches through `guardMayRaise`, never + * through `guardCanRaise`. `canRaise` is a whitelist and misses real raising code (a destructuring + * declaration is not on its list, and `const { a } = undefined` throws), so ordering the arm off + * `reachable` accused a route that owns a real classifying catch of owning none, which was simply + * untrue; `does not accuse a route that owns a catch of owning none` pins the verdict. The + * containment read `guardMayRaise` is false only for the provably-inert `try { 0; }`, so the one + * clause that must not block the accusation, the prepended dead classifier `dead-classifying-try` + * refuses, still does not block it (`still fails a per-item swallow beside a deciding catch over a + * dead guard`). The already-open residual is unchanged: `try { String(0); }` reads as may-raise + * AND can-raise, which is `dead-classifying-try-with-call`, the corpus's expected failure. + */ +export const errorClassification = { + id: ID, + run(ep: EntryPoint): CheckResult { + if (isTrivial(ep)) { + return { id: ID, status: "not-applicable", detail: "trivial route" }; + } + const reachable = ep.catches.filter((c) => c.guardCanRaise); + const swallowed = reachable.filter(swallows); + if (swallowed.length > 0) { + const which = + reachable.length > 1 ? ` (${swallowed.length} of ${reachable.length} catches)` : ""; + // "One way out" is only true of a clause that never throws. A clause holding a `throw` that + // is not its only exit is a swallow by this check's definition (it decides nothing about the + // error) and it is NOT one way out, so saying so was a false accusation. 16 clauses in the + // tree changed `rethrows` from true to false this round and every one of them would have + // been eligible for it. + const everyWayOut = swallowed.every((c) => !c.throws); + return { + id: ID, + status: "fail", + detail: everyWayOut + ? `catches its errors and takes one way out regardless of what was thrown${which}` + : `catches its errors and chooses what to do without looking at what was thrown${which}`, + }; + } + // A refused (iteration-callback) catch is judged on its evidence, never on its placement. + // The fail arm first: a refused swallow fails whenever nothing the route owns decides. + // Deliberately NOT conditioned on `ep.catches.length === 0`: an own inert catch, which + // `wrap-body-in-rethrow` adds to every route, must not lift a refused swallow out of the + // verdict, or wrapping a per-item-swallow route in try/rethrow reads "every catch rethrows". + // `fails a per-item swallow even when the route owns an inert rethrow catch` pins that. + // "Nothing the route owns decides" is read off `ep.catches` under `guardMayRaise`, not off + // `reachable`: a deciding catch `canRaise` cannot see still decides, and only the + // provably-inert `try { 0; }` guard is excluded. See the `guardMayRaise` paragraph above. + const reachableCb = ep.callbackCatches.filter((c) => c.guardCanRaise); + const ownDecides = ep.catches.some((c) => decides(c) && c.guardMayRaise); + if (!ownDecides && reachableCb.some(swallows)) { + return { + id: ID, + status: "fail", + detail: + "a catch inside an iteration callback swallows what it caught, and nothing the route owns decides", + }; + } + // The ceiling: refused catches never reach the pass arm, so a route whose only catches are + // refused and none of them swallows sits out of the denominator rather than collecting + // anything. Read off `ep.catches`, not `reachable`: a route that owns a catch owns one, + // whether or not `canRaise` could see what it guarded; ordering this off `reachable` turned + // every `canRaise` miss on a route that also has a per-item catch into an accusation that was + // flatly false. The detail asserts nothing about ownership or per-item-ness the scanner + // cannot know: a once-invoked Result-style wrapper with a deciding inner catch reads the same + // as its inline equivalent would. + if (ep.catches.length === 0 && ep.callbackCatches.length > 0) { + return { + id: ID, + status: "not-applicable", + detail: + "its only catches sit in iteration callbacks and none swallows, so the route itself classifies nothing", + }; + } + if (!reachable.some(decides)) { + return { + id: ID, + status: "not-applicable", + detail: + reachable.length === 0 + ? ep.catches.length === 0 + ? "catches nothing, so it classifies nothing" + : "guards nothing that can throw, so it classifies nothing" + : "every catch rethrows and nothing else, so it classifies nothing", + }; + } + return { id: ID, status: "pass", detail: "every catch decides what it caught" }; + }, +}; diff --git a/internal-packages/observability-map/src/checks/index.test.ts b/internal-packages/observability-map/src/checks/index.test.ts new file mode 100644 index 00000000000..f0e80af528d --- /dev/null +++ b/internal-packages/observability-map/src/checks/index.test.ts @@ -0,0 +1,2234 @@ +import { CHECKS, SCORED_CHECK_IDS } from "./index.js"; +import { scanFile } from "../scan.js"; + +const run = (id: string, fileName: string, source: string) => { + const ep = scanFile(fileName, source)!; + return CHECKS.find((c) => c.id === id)!.run(ep); +}; + +/** + * The React component that lives alongside the loader in a `.tsx` route. Every check reads + * body-scoped evidence, so nothing in here may change a verdict: it try/catches, it logs, it names + * every request identifier the checks look for, and it calls an auth helper. + */ +const COMPONENT = ` + export default function Page() { + const { environmentId, organizationId, projectId, runId } = useTypedLoaderData(); + useEffect(() => { + try { + requireUserId(environmentId); + logger.error("render failed", { environmentId, organizationId, projectId, runId }); + } catch (e) { + if (e instanceof Error) return; + throw e; + } + }, [environmentId]); + return
{runId}
; + } +`; + +describe("registry", () => { + it("holds the five checks, with audit-trail left out of the score", () => { + expect(CHECKS.map((c) => c.id)).toEqual([ + "error-classification", + "auth-boundary", + "auth-scope", + "request-context", + "audit-trail", + ]); + expect(SCORED_CHECK_IDS).toEqual([ + "error-classification", + "auth-boundary", + "auth-scope", + "request-context", + ]); + }); +}); + +describe("error-classification", () => { + // C1. A route with no catch makes no classification decision, so there is nothing here to judge + // and nothing to credit. Crediting it made deleting error handling raise the score. + it("is not applicable to a builder-wrapped route with no local try/catch", () => { + const r = run( + "error-classification", + "api.v1.x.ts", + `import { createLoaderApiRoute } from "~/services/routeBuilders/apiBuilder.server"; + export const loader = createLoaderApiRoute({}, async () => new Response("ok"));` + ); + expect(r.status).toBe("not-applicable"); + }); + + it("fails a raw route whose catch swallows every error identically", () => { + const r = run( + "error-classification", + "api.v1.y.ts", + `import { prisma } from "~/db.server"; + export async function loader() { + try { return await prisma.thing.findMany(); } catch (e) { return null; } + }` + ); + expect(r.status).toBe("fail"); + }); + + // Restored from the brief: the clause branches, on the `instanceof` and the `if`. + it("passes a raw route whose catch branches on the error", () => { + const r = run( + "error-classification", + "api.v1.z.ts", + `import { prisma } from "~/db.server"; + export async function loader() { + try { return await prisma.thing.findMany(); } + catch (e) { if (e instanceof NotFound) return null; throw e; } + }` + ); + expect(r.status).toBe("pass"); + }); + + // A clause that only rethrows makes no classification decision: the error propagates exactly as + // it would with no catch at all, so it is read as no catch at all. Scoring the two differently + // paid 50 points a route for wrapping a body in `try { ... } catch (e) { throw e }`. + it("is not applicable to a raw route whose catch only rethrows", () => { + const r = run( + "error-classification", + "api.v1.v.ts", + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function loader() { + try { return await prisma.thing.findMany(); } + catch (e) { logger.error("thing lookup failed", { error: e }); throw e; } + }` + ); + expect(r.status).toBe("not-applicable"); + }); + + // The builder only classifies what reaches it. A swallow inside the handler never does, so the + // swallow is read before the builder is credited. + it("fails a builder-wrapped route whose handler swallows", () => { + const r = run( + "error-classification", + "api.v2.runs.$runParam.cancel.ts", + `import { createActionApiRoute } from "~/services/routeBuilders/apiBuilder.server"; + import { CancelTaskRunService } from "~/services/cancelTaskRun.server"; + const { action } = createActionApiRoute({}, async ({ params }) => { + const service = new CancelTaskRunService(); + try { await service.call(params.runParam); } + catch { return json({ error: "Internal Server Error" }, { status: 500 }); } + return json({ ok: true }); + }); + export { action };` + ); + expect(r.status).toBe("fail"); + }); + + it("is not applicable to a raw route that lets its errors propagate", () => { + const r = run( + "error-classification", + "api.v1.w.ts", + `import { prisma } from "~/db.server"; + export async function loader() { + const rows = await prisma.thing.findMany(); + return json({ rows }); + }` + ); + expect(r.status).toBe("not-applicable"); + }); + + it("is not applicable to a trivial redirect", () => { + const r = run( + "error-classification", + "@.ts", + `import { redirect } from "@remix-run/server-runtime"; + export async function loader() { return redirect("/admin"); }` + ); + expect(r.status).toBe("not-applicable"); + }); + + // A narrow guard around one operation classifies an expected failure without needing to branch + // or rethrow. Guarding a parse over a small part of the body is what tells it apart from a + // handler-wide catch. + it("passes a narrow guard around a single parse", () => { + const r = run( + "error-classification", + "resources.timezone.ts", + `import { prisma } from "~/db.server"; + export async function action({ request }) { + let data; + try { data = await request.json(); } + catch { return json({ error: "Invalid JSON" }, { status: 400 }); } + const saved = await prisma.preference.create({ data }); + return json({ saved }); + }` + ); + expect(r.status).toBe("pass"); + }); + + // I6. NARROW_TRY_STATEMENTS is an absolute count over the try block alone (guardsParse still + // required), not a ratio against the enclosing body, so it holds the exact boundary regardless of + // how big or small the rest of the function is: two statements binds the parsed result and still + // passes, a third means the try has started to cover the handler and fails, even though both + // guard the same parse. + it("passes a parse guard that binds its result in exactly two statements", () => { + const r = run( + "error-classification", + "resources.pattern.ts", + `export async function action({ request }) { + let parsed; + try { + const raw = await request.text(); + parsed = new RegExp(raw); + } catch { + return json({ error: "Invalid pattern" }, { status: 400 }); + } + return json({ parsed: parsed.source }); + }` + ); + expect(r.status).toBe("pass"); + }); + + // C5. The count is not the only condition any more, and this is the shape that showed why: two + // statements, one of them a parse, and the whole handler inside the try. Before `awaitsOnlyParse` + // the count read it as a narrow guard and passed it, which is the `otel.v1.logs.ts` swallow + // written compactly. Three spellings of the same thing, all of which the count reads as narrow. + const COMPACT_SWALLOWS: Array<[string, string]> = [ + [ + "two statements", + `try { const body = await request.json(); return await handleEverything(body); } + catch (error) { return new Response("Internal Server Error", { status: 500 }); }`, + ], + [ + "one statement, the parse nested inside the call", + `try { return await handleEverything(await request.json()); } + catch (error) { return new Response("Internal Server Error", { status: 500 }); }`, + ], + [ + "one statement, merged into a declaration list", + `try { const body = await request.json(), out = await handleEverything(body); return out; } + catch (error) { return new Response("Internal Server Error", { status: 500 }); }`, + ], + ]; + + for (const [label, body] of COMPACT_SWALLOWS) { + it(`fails a whole handler wrapped in a parse-guard-shaped try (${label})`, () => { + const r = run( + "error-classification", + "otel.v1.logs.ts", + `export async function action({ request }) {\n${body}\n}` + ); + expect(r.status).toBe("fail"); + }); + } + + // The counterpart: the same route with the handler moved out of the try is a real guard and + // still passes, so the rule above is not just "any try containing an await fails". + it("passes the same route once the handler moves out of the try", () => { + const r = run( + "error-classification", + "otel.v1.logs.ts", + `export async function action({ request }) { + let body; + try { body = await request.json(); } + catch { return json({ error: "bad json" }, { status: 400 }); } + return await handleEverything(body); + }` + ); + expect(r.status).toBe("pass"); + }); + + // Synchronous string work preparing a parse's input is not what `awaitsOnlyParse` refuses. Four + // real routes are this shape, `admin.llm-models.new.tsx` among them. + it("passes a guard that prepares its input synchronously before parsing", () => { + const r = run( + "error-classification", + "admin.llm-models.new.tsx", + `export async function action({ request }) { + const matchPattern = String(await request.text()); + try { + const testPattern = matchPattern.startsWith("(?i)") ? matchPattern.slice(4) : matchPattern; + new RegExp(testPattern); + } catch { + return json({ error: "Invalid regex" }, { status: 400 }); + } + return await save(matchPattern); + }` + ); + expect(r.status).toBe("pass"); + }); + + it("fails a parse guard that takes a third statement beyond binding the result", () => { + const r = run( + "error-classification", + "resources.pattern.ts", + `export async function action({ request }) { + let parsed; + try { + const raw = await request.text(); + const trimmed = raw.trim(); + parsed = new RegExp(trimmed); + } catch { + return json({ error: "Invalid pattern" }, { status: 400 }); + } + return json({ parsed: parsed.source }); + }` + ); + expect(r.status).toBe("fail"); + }); + + // False positive fixture for the narrow rule: a narrow parse guard must not launder the broad + // handler catch sitting next to it. Clauses are judged one at a time, so the broad one still + // counts against the entry point. + it("still fails when a narrow guard sits beside a handler-wide swallow", () => { + const r = run( + "error-classification", + "api.v1.thing.ts", + `import { prisma } from "~/db.server"; + export async function action({ request }) { + let data; + try { data = await request.json(); } + catch { return json({ error: "Invalid JSON" }, { status: 400 }); } + try { + const thing = await prisma.thing.create({ data }); + const audit = await prisma.audit.create({ data: { thing: thing.id } }); + return json({ thing, audit }); + } catch (error) { + return json({ error: "Something went wrong" }, { status: 500 }); + } + }` + ); + expect(r.status).toBe("fail"); + }); + + // try/finally with no catch clause. `hasTryCatch` is true here and `catches` is empty, and it is + // `catches` that answers "does this route catch anything". Nothing is swallowed: the error + // propagates once the connection is closed. + it("is not applicable to a try/finally that catches nothing", () => { + const r = run( + "error-classification", + "admin.api.v1.runs-replication.status.ts", + `import Redis from "ioredis"; + export async function loader() { + const redis = new Redis({ host: "localhost" }); + try { + const exists = await redis.exists("some-key"); + const other = await redis.exists("other-key"); + return json({ exists, other }); + } finally { + await redis.quit(); + } + }` + ); + expect(r.status).toBe("not-applicable"); + }); + + // Multi-catch, the case the aggregate booleans could not describe. Judged per clause: the parse + // guard is a guard, the handler catch rethrows, so both are accounted for. + it("passes a parse guard sitting beside a handler catch that rethrows", () => { + const r = run( + "error-classification", + "api.v1.thing.ts", + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function action({ request }) { + let data; + try { data = await request.json(); } + catch { return json({ error: "Invalid JSON" }, { status: 400 }); } + try { + const thing = await prisma.thing.create({ data }); + const audit = await prisma.audit.create({ data: { thing: thing.id } }); + const count = await prisma.thing.count(); + return json({ thing, audit, count }); + } catch (error) { + logger.error("create failed", { error }); + throw error; + } + }` + ); + expect(r.status).toBe("pass"); + }); + + // A parse guard that has grown to cover the handler is not a guard any more. `otel.v1.logs.ts` + // catches 15 of its 18 statements around a `request.json()` and answers 500 for all of them. + it("fails a parse guard that covers most of the body", () => { + const r = run( + "error-classification", + "otel.v1.logs.ts", + `import { otlpExporter } from "~/v3/otlpExporter.server"; + export async function action({ request }) { + try { + const exporter = await otlpExporter; + const contentType = request.headers.get("content-type"); + const body = await request.json(); + const result = await exporter.exportLogs(body); + const encoded = encodeResponse(result); + const headers = buildHeaders(contentType); + return new Response(encoded, { status: 200, headers }); + } catch (error) { + console.error(error); + return new Response("Internal Server Error", { status: 500 }); + } + }` + ); + expect(r.status).toBe("fail"); + }); + + // A6. isParseGuard compared the clause against ep.statementCount, the loader and the action and + // every one-hop helper summed together, rather than the statements of the body the clause is + // actually in. So an unrelated sibling handler or a fat helper in the same file diluted the + // denominator and relabelled the same broad swallow as a narrow parse guard. Byte-identical + // action, verdict must not move. + it("gives the same verdict to a byte-identical swallow whether or not an unrelated sibling and helper share the file", () => { + const action = `import { otlpExporter } from "~/v3/otlpExporter.server"; + export async function action({ request }) { + try { + const exporter = await otlpExporter; + const contentType = request.headers.get("content-type"); + const body = await request.json(); + const result = await exporter.exportLogs(body); + const encoded = encodeResponse(result); + const headers = buildHeaders(contentType); + return new Response(encoded, { status: 200, headers }); + } catch (error) { + console.error(error); + return new Response("Internal Server Error", { status: 500 }); + } + }`; + + const withUnrelatedSiblingAndHelper = `${action} + function unrelatedHelper() { + let total = 0; + total += 1; + total += 2; + total += 3; + total += 4; + total += 5; + total += 6; + total += 7; + total += 8; + total += 9; + total += 10; + total += 11; + return total; + } + export async function loader() { + const helperTotal = unrelatedHelper(); + return new Response(String(helperTotal)); + }`; + + const alone = run("error-classification", "otel.v1.logs.ts", action); + const withSiblingAndHelper = run( + "error-classification", + "otel.v1.logs.ts", + withUnrelatedSiblingAndHelper + ); + + expect(alone.status).toBe("fail"); + expect(withSiblingAndHelper.status).toBe("fail"); + }); + + // I6. Moving the denominator from the entry point to the enclosing body (A6) closed + // cross-body dilution but not same-body dilution: the rule was still a ratio, "unrelated + // statements dilute", wherever the unrelated statements live. Padding the SAME action with 11 + // inert statements after the try relabelled the identical broad swallow from fail to pass. + // isParseGuard is now an absolute count over the try block alone (NARROW_TRY_STATEMENTS), + // which nothing outside the try can dilute, in the same body or another. + it("gives the same verdict to a byte-identical swallow whether or not it is padded with inert statements in the same body", () => { + const action = `import { otlpExporter } from "~/v3/otlpExporter.server"; + export async function action({ request }) { + try { + const exporter = await otlpExporter; + const contentType = request.headers.get("content-type"); + const body = await request.json(); + const result = await exporter.exportLogs(body); + const encoded = encodeResponse(result); + const headers = buildHeaders(contentType); + return new Response(encoded, { status: 200, headers }); + } catch (error) { + console.error(error); + return new Response("Internal Server Error", { status: 500 }); + } + }`; + + const padding = Array.from({ length: 11 }, (_, i) => `const pad${i} = ${i};`).join("\n"); + const paddedInSameBody = `import { otlpExporter } from "~/v3/otlpExporter.server"; + export async function action({ request }) { + try { + const exporter = await otlpExporter; + const contentType = request.headers.get("content-type"); + const body = await request.json(); + const result = await exporter.exportLogs(body); + const encoded = encodeResponse(result); + const headers = buildHeaders(contentType); + return new Response(encoded, { status: 200, headers }); + } catch (error) { + console.error(error); + return new Response("Internal Server Error", { status: 500 }); + } + ${padding} + return new Response("unreachable", { status: 200 }); + }`; + + const alone = run("error-classification", "otel.v1.logs.ts", action); + const padded = run("error-classification", "otel.v1.logs.ts", paddedInSameBody); + + expect(alone.status).toBe("fail"); + expect(padded.status).toBe("fail"); + }); + + // A3. `referencesBinding` used to match any identifier with the binding's text, including a + // property name in a member expression. A catch whose only `if` tests `fallback.error`, never the + // caught binding itself, was credited with classifying an error it never inspected. + it("fails a catch whose only if tests a same-named property, not the caught error", () => { + const r = run( + "error-classification", + "api.v1.y.ts", + `import { prisma } from "~/db.server"; + export async function loader() { + try { return await prisma.thing.findMany(); } + catch (error) { + if (fallback.error) return json({}, { status: 500 }); + return json({}, { status: 500 }); + } + }` + ); + expect(r.status).toBe("fail"); + }); + + // False positive fixture: the only try/catch in the file belongs to the component. + it("does not judge a route whose try/catch is in the React component", () => { + const r = run( + "error-classification", + "_app.orgs.$organizationSlug.things/route.tsx", + `import { prisma } from "~/db.server"; + export async function loader() { + const rows = await prisma.thing.findMany(); + return typedjson({ rows }); + } + ${COMPONENT}` + ); + expect(r.status).toBe("not-applicable"); + }); + + // A7 as revised twice. A per-item error boundary inside a `.map()` callback is still not judged + // as the route's own catch, so it never sets `catches` and never speaks for the route's + // `tryStatementCount`. This catch SWALLOWS what it caught, and nothing the route owns decides, + // so the route still fails: judging refused catches on their evidence must not stop failing the + // relocated swallow, which is the anti-laundering half of the rule. + it("fails a route whose only catch is inside a Promise.all(items.map(...)) callback", () => { + const source = `import { prisma } from "~/db.server"; + export async function action({ request }) { + const items = await prisma.item.findMany(); + await Promise.all( + items.map(async (item) => { + try { + await processItem(item); + } catch { + return null; + } + }) + ); + return json({ ok: true }); + }`; + const ep = scanFile("batch.process.ts", source)!; + expect(ep.catches).toEqual([]); + expect(ep.callbackCatches).toHaveLength(1); + const r = run("error-classification", "batch.process.ts", source); + expect(r.status).toBe("fail"); + expect(r.detail).toContain("a catch inside an iteration callback swallows"); + }); + + // The evidence half of the mechanism-C rule: a refused catch that DECIDES caps at not-applicable + // rather than failing (the old placement rule) or passing (the crediting rule `dead-deciding-map` + // exists to refuse). The route's error handling is real and per item; the route itself decides + // nothing, so out of the denominator is the honest place for it. + it("sits out a route whose only catch is a deciding per-item boundary", () => { + const r = run( + "error-classification", + "batch.decide.ts", + `import { prisma } from "~/db.server"; + export async function action({ request }) { + const results = await stream.map(async (item) => { + try { + await service.call(item); + } catch (e) { + if (e instanceof KnownError) { return new Response(e.code, { status: 400 }); } + throw e; + } + }); + return json({ results }); + }` + ); + expect(r.status).toBe("not-applicable"); + expect(r.detail).toContain("its only catches sit in iteration callbacks"); + }); + + // Same shape with an inert per-item rethrow: not a swallow, so it sits out too. + it("sits out a route whose only catch is an inert per-item rethrow", () => { + const r = run( + "error-classification", + "batch.rethrow.ts", + `import { prisma } from "~/db.server"; + export async function action({ request }) { + const results = await stream.map(async (item) => { + try { + await service.call(item); + } catch (e) { + throw e; + } + }); + return json({ results }); + }` + ); + expect(r.status).toBe("not-applicable"); + expect(r.detail).toContain("its only catches sit in iteration callbacks"); + }); + + // The refused-swallow arm is deliberately not conditioned on the route owning no catches. An + // own inert catch is what `wrap-body-in-rethrow`, a preserving corpus entry, adds to every + // route: were the arm gated on `catches.length === 0`, wrapping a per-item-swallow route in + // try/rethrow would read "every catch rethrows" and lift the fail to not-applicable, a rise + // that existed in the pre-evidence code and was masked only by the affected routes scoring 0 on + // every other check. + it("fails a per-item swallow even when the route owns an inert rethrow catch", () => { + const r = run( + "error-classification", + "batch.wrapped.ts", + `import { prisma } from "~/db.server"; + export async function action({ request }) { + try { + const items = await prisma.item.findMany(); + await Promise.all( + items.map(async (item) => { + try { + await processItem(item); + } catch { + return null; + } + }) + ); + return json({ ok: true }); + } catch (e) { + throw e; + } + }` + ); + expect(r.status).toBe("fail"); + expect(r.detail).toContain("a catch inside an iteration callback swallows"); + }); + + // The no-pass ceiling at the fixture scale: a catchless route with a prepended dead deciding + // map sits out. A crediting rule would read pass here, which is 50 free points on the tree's + // 261 catchless routes; the old placement rule read fail, a false accusation on a preserving + // prepend. `dead-deciding-map` in the mutation corpus is the tree-scale version. + it("sits out a catchless route with a prepended dead deciding map", () => { + const r = run( + "error-classification", + "prepended-map.ts", + `import { prisma } from "~/db.server"; + export async function action({ request }) { + [0, 1].map((v) => { try { JSON.parse("0"); } catch (e) { if (e instanceof SyntaxError) { return null; } throw e; } return v; }); + const rows = await prisma.thing.findMany(); + return json({ rows }); + }` + ); + expect(r.status).toBe("not-applicable"); + expect(r.detail).toContain("its only catches sit in iteration callbacks"); + }); + + // The same route with nothing caught anywhere stays not-applicable, so the fail above is + // attributable to the refused catch and not to the check having stopped excusing anything. + it("is not applicable to a route that catches nothing at all", () => { + const r = run( + "error-classification", + "batch.process.ts", + `import { prisma } from "~/db.server"; + export async function action({ request }) { + const items = await prisma.item.findMany(); + await Promise.all(items.map(async (item) => processItem(item))); + return json({ ok: true }); + }` + ); + expect(r.status).toBe("not-applicable"); + expect(r.detail).toContain("catches nothing"); + }); + + // S2 at the check level. The evidence tests in `scan.test.ts` pin `guardCanRaise` itself; these + // pin the check reading it, which is where the 50 points were. Prepending this to a route that + // catches nothing took it from not-applicable to pass, and 224 routes were in exactly that state. + it("is not applicable to a route whose only catch guards a try that cannot throw", () => { + const r = run( + "error-classification", + "prepended.ts", + `import { prisma } from "~/db.server"; + export async function action({ request }) { + try { 0; } catch (e) { + if (e instanceof Error) { return new Response(null, { status: 400 }); } + throw e; + } + const rows = await prisma.thing.findMany(); + return json({ rows }); + }` + ); + expect(r.status).toBe("not-applicable"); + expect(r.detail).toContain("guards nothing that can throw"); + }); + + it("still fails a swallow that a dead classifying catch was prepended to", () => { + const r = run( + "error-classification", + "prepended-swallow.ts", + `import { prisma } from "~/db.server"; + export async function action({ request }) { + try { 0; } catch (e) { + if (e instanceof Error) { return new Response(null, { status: 400 }); } + throw e; + } + try { + return json(await prisma.thing.findMany()); + } catch (error) { + return new Response(null, { status: 500 }); + } + }` + ); + expect(r.status).toBe("fail"); + }); + + it("still passes the same classifying catch once its try does real work", () => { + const r = run( + "error-classification", + "live.ts", + `import { prisma } from "~/db.server"; + export async function action({ request }) { + try { await prisma.thing.findMany(); } catch (e) { + if (e instanceof Error) { return new Response(null, { status: 400 }); } + throw e; + } + return json({ ok: true }); + }` + ); + expect(r.status).toBe("pass"); + }); + + // I3. "Takes one way out regardless of what was thrown" is false of a clause that throws for + // some errors, and the strengthened `rethrows` makes such a clause a swallow by this check's + // definition: it decides nothing about the error, but it does not send everything the same way + // either. 16 clauses in the tree flipped `rethrows` this round and every one was eligible for the + // false wording. Whether `fail` is the right verdict for them is a separate question, parked. + it("does not accuse a clause that throws of taking one way out", () => { + const r = run( + "error-classification", + "mixed.ts", + `import { prisma } from "~/db.server"; + export async function loader() { + try { return json(await prisma.thing.findMany()); } + catch (e) { if (rare) { return null; } throw e; } + }` + ); + expect(r.status).toBe("fail"); + expect(r.detail).toContain("without looking at what was thrown"); + expect(r.detail).not.toContain("one way out"); + }); + + it("still says one way out for a clause that never throws", () => { + const r = run( + "error-classification", + "swallow.ts", + `import { prisma } from "~/db.server"; + export async function loader() { + try { return json(await prisma.thing.findMany()); } + catch (e) { logger.error(e); return null; } + }` + ); + expect(r.status).toBe("fail"); + expect(r.detail).toContain("one way out"); + }); + + // I4. A route that owns a real classifying catch must never be told it owns none. `canRaise` does + // not list destructuring, and `const { a } = undefined` throws, so the owned catch dropped out of + // `reachable`; with the refused-swallow arm ordered off `reachable` the route was then accused of + // owning nothing that decides, which was simply false. The arm now reads own deciding catches + // through `guardMayRaise`, so the accusation is withheld and the route sits out exactly as it did + // before the arm existed. Asserted on `status`: an earlier version of this test asserted the + // absence of a detail string no arm ever emits, which could not fail. + it("does not accuse a route that owns a catch of owning none", () => { + const r = run( + "error-classification", + "owned.ts", + `import { prisma } from "~/db.server"; + export async function action({ request }) { + const items = await prisma.item.findMany(); + await Promise.all( + items.map(async (item) => { + try { await processItem(item); } catch { return null; } + }) + ); + try { const { a } = undefined; } catch (e) { + if (e instanceof TypeError) { return new Response(null, { status: 400 }); } + throw e; + } + return json({ ok: true }); + }` + ); + expect(r.status).toBe("not-applicable"); + expect(r.detail).toContain("guards nothing that can throw"); + }); + + // I4 sibling: the same shape through `.filter` and a different `canRaise` miss (a plain + // declaration is not on the whitelist either), so the fix is the rule and not the fixture. + it("does not accuse a route whose deciding catch guards a declaration beside a filter swallow", () => { + const r = run( + "error-classification", + "owned-filter.ts", + `import { prisma } from "~/db.server"; + export async function action({ request }) { + const items = await prisma.item.findMany(); + const kept = items.filter((item) => { + try { return check(item); } catch { return false; } + }); + try { const parsed = { ...raw }; } catch (e) { + if (e instanceof TypeError) { return new Response(null, { status: 400 }); } + throw e; + } + return json({ kept }); + }` + ); + expect(r.status).toBe("not-applicable"); + }); + + // The blocking catch has to DECIDE: an own inert rethrow catch over the same invisible guard + // still leaves the refused swallow in the verdict, or `wrap-body-in-rethrow` spelled with a + // destructuring guard would lift every per-item swallow out of it. + it("still fails a per-item swallow beside an inert catch over an invisible guard", () => { + const r = run( + "error-classification", + "owned-inert.ts", + `import { prisma } from "~/db.server"; + export async function action({ request }) { + const items = await prisma.item.findMany(); + await Promise.all( + items.map(async (item) => { + try { await processItem(item); } catch { return null; } + }) + ); + try { const { a } = undefined; } catch (e) { throw e; } + return json({ ok: true }); + }` + ); + expect(r.status).toBe("fail"); + expect(r.detail).toContain("nothing the route owns decides"); + }); + + // And the blocking catch has to guard something that MAY raise: the provably-inert `try { 0; }` + // clause `dead-classifying-try` prepends decides and must still block nothing, or the prepend + // would lift a refused-swallow fail to not-applicable at tree scale. + it("still fails a per-item swallow beside a deciding catch over a dead guard", () => { + const r = run( + "error-classification", + "owned-dead.ts", + `import { prisma } from "~/db.server"; + export async function action({ request }) { + const items = await prisma.item.findMany(); + await Promise.all( + items.map(async (item) => { + try { await processItem(item); } catch { return null; } + }) + ); + try { 0; } catch (e) { + if (e instanceof Error) { return new Response(null, { status: 400 }); } + throw e; + } + return json({ ok: true }); + }` + ); + expect(r.status).toBe("fail"); + expect(r.detail).toContain("nothing the route owns decides"); + }); + + // C2. A single-element array cannot iterate, so `[0].map(async () => { whole body })` is not a + // per-item boundary and the route's own catch is found where it always was. Before this, the + // wrapper deleted the route's catches and took a swallow from fail to not-applicable. + it("still fails a swallow wrapped in Promise.all([0].map(...))", () => { + const r = run( + "error-classification", + "wrapped.ts", + `import { prisma } from "~/db.server"; + export async function action({ request }) { + const [result] = await Promise.all([0].map(async () => { + try { + const body = await request.json(); + const a = await stepOne(body); + const b = await stepTwo(a); + return json({ b }); + } catch (e) { + return new Response("nope", { status: 500 }); + } + })); + return result; + }` + ); + expect(r.status).toBe("fail"); + }); + + // A second receiver exercising the same mechanism: an empty array literal, which no name list + // would treat differently from a populated one. + it("still fails a swallow wrapped in [].flatMap(...)", () => { + const r = run( + "error-classification", + "wrapped-empty.ts", + `import { prisma } from "~/db.server"; + export async function action({ request }) { + return [].flatMap(async () => { + try { + const body = await request.json(); + const a = await stepOne(body); + const b = await stepTwo(a); + return json({ b }); + } catch (e) { + return new Response("nope", { status: 500 }); + } + }); + }` + ); + expect(r.status).toBe("fail"); + }); + + // A third, where the name list cannot help at all: a non-array receiver whose method is called + // `map`. The boundary rule still refuses the callback, so the route has no catch of its own, and + // the refusal now fails rather than excusing. This is the shape the name list cannot tell from + // `users.map(...)`, and it is why the refusal had to stop paying. + it("still fails a swallow wrapped in a non-array receiver's .map(...)", () => { + const r = run( + "error-classification", + "wrapped-result.ts", + `import { prisma } from "~/db.server"; + export async function action({ request }) { + return await Result.map(async () => { + try { + const body = await request.json(); + const a = await stepOne(body); + const b = await stepTwo(a); + return json({ b }); + } catch (e) { + return new Response("nope", { status: 500 }); + } + }); + }` + ); + expect(r.status).toBe("fail"); + }); + + // The verdict end of the `break and continue inside the construct they target` finding. A clause + // that sorts the error by code and then rethrows was failed, with a detail line asserting it + // takes one way out regardless of what was thrown, which is the opposite of what it does. Both + // spellings are here because the pair is the evidence: the switch must not change the verdict. + const SORTED_RETHROW = (sorter: string) => `import { prisma } from "~/db.server"; + export async function action({ request, params }) { + try { + return json(await prisma.thing.update({ where: { id: params.id }, data: {} })); + } catch (e) { + ${sorter} + throw e; + } + }`; + + it("does not accuse a clause that sorts the error by code and rethrows", () => { + const r = run( + "error-classification", + "api.v1.sorted.ts", + SORTED_RETHROW( + 'switch (e.code) { case "P2025": handleNotFound(e); break; default: handleOther(e); break; }' + ) + ); + expect(r.status).toBe("not-applicable"); + expect(r.detail).not.toContain("one way out"); + }); + + it("reads the same clause written without the switch identically", () => { + const withSwitch = run( + "error-classification", + "api.v1.sorted.ts", + SORTED_RETHROW( + 'switch (e.code) { case "P2025": handleNotFound(e); break; default: handleOther(e); break; }' + ) + ); + const without = run( + "error-classification", + "api.v1.sorted.ts", + SORTED_RETHROW("handleOther(e);") + ); + expect(withSwitch).toEqual(without); + }); + + // The other direction, so the rule above is not just "a switch is ignored": the same sorter with + // a clause that answers the request is a decision, and still passes. + it("still passes a clause whose switch on the error code answers the request", () => { + const r = run( + "error-classification", + "api.v1.sorted.ts", + SORTED_RETHROW( + 'switch (e.code) { case "P2025": return new Response(null, { status: 404 }); default: break; }' + ) + ); + expect(r.status).toBe("pass"); + }); + + // The verdict end of the walk's guaranteed-execution entries, one pair per entered construct. + // The evidence end is `the walk enters exactly the positions guaranteed to execute` in + // scan.test.ts; these hold the wrapped and unwrapped spellings to the same verdict, modeled on + // the switch pair above. Before the entries existed, every wrapper here turned a passing + // deciding clause into a fail with a detail line accusing it of ignoring the error. + const CLAUSE_WRAPPED = (clauseBody: string) => `import { prisma } from "~/db.server"; + export async function loader() { + try { + return json(await prisma.thing.findMany()); + } catch (e) { + ${clauseBody} + } + }`; + + const DECIDING_CLAUSE = + "if (e instanceof Error) { return new Response(null, { status: 400 }); }\n" + + "return new Response(null, { status: 500 });"; + + const GUARANTEED_WRAPPERS: Array<[string, (body: string) => string]> = [ + ["a catchless try/finally", (body) => `try {\n${body}\n} finally { }`], + ["a single-default switch", (body) => `switch (pick()) { default: {\n${body}\n} }`], + ["an if (true)", (body) => `if (true) {\n${body}\n}`], + [ + "an if/else with the body in both arms", + (body) => `if (pick()) {\n${body}\n} else {\n${body}\n}`, + ], + ]; + + for (const [label, wrap] of GUARANTEED_WRAPPERS) { + it(`reads a deciding clause relocated into ${label} with the same verdict`, () => { + const wrapped = run( + "error-classification", + "api.v1.wrapped.ts", + CLAUSE_WRAPPED(wrap(DECIDING_CLAUSE)) + ); + const bare = run( + "error-classification", + "api.v1.wrapped.ts", + CLAUSE_WRAPPED(DECIDING_CLAUSE) + ); + expect(bare.status).toBe("pass"); + expect(wrapped).toEqual(bare); + }); + } +}); + +describe("auth-boundary", () => { + it("passes a sensitive route guarded by a require helper", () => { + // Sensitive on the impersonation call, not on the guard: calling a guard is not what makes a + // route sensitive, see sensitivity.test.ts. + const r = run( + "auth-boundary", + "admin.api.v1.impersonate.ts", + `import { requireAdminApiRequest } from "~/services/personalAccessToken.server"; + import { setImpersonation } from "~/models/admin.server"; + export async function action({ request }) { + await requireAdminApiRequest(request); + return setImpersonation(request, "user_1"); + }` + ); + expect(r.status).toBe("pass"); + }); + + it("passes a sensitive route guarded by an authenticate helper", () => { + const r = run( + "auth-boundary", + "api.v1.tokens.ts", + `import { authenticateApiRequest } from "~/services/apiAuth.server"; + import { prisma } from "~/db.server"; + export async function loader({ request }) { + const auth = await authenticateApiRequest(request); + if (!auth) throw new Response(null, { status: 401 }); + return prisma.token.findMany(); + }` + ); + expect(r.status).toBe("pass"); + }); + + it("fails a sensitive route with no guard", () => { + const r = run( + "auth-boundary", + "api.v1.tokens.ts", + `import { prisma } from "~/db.server"; + export async function loader() { + const tokens = await prisma.token.findMany(); + return json({ tokens }); + }` + ); + expect(r.status).toBe("fail"); + }); + + it("is not applicable to a non-sensitive route", () => { + const r = run( + "auth-boundary", + "api.v1.timezones.ts", + `import { prisma } from "~/db.server"; + export async function loader() { return prisma.tz.findMany(); }` + ); + expect(r.status).toBe("not-applicable"); + }); + + // False positive fixture for the delegated guard. `clearImpersonation` authenticates and writes + // an audit row, in `app/models/admin.server.ts`, which the scanner cannot open. The body shows no + // privileged work either, so there is nothing here to accuse: absence of evidence, not evidence + // of absence. + it("does not flag a sensitive route that hands its work to an imported helper", () => { + const r = run( + "auth-boundary", + "resources.impersonation.ts", + `import { clearImpersonation } from "~/models/admin.server"; + export async function action({ request }) { + return clearImpersonation(request, "/admin"); + }` + ); + expect(r.status).toBe("not-applicable"); + expect(r.detail).toMatch(/verif/i); + }); + + it("does not flag a sensitive redirect stub", () => { + const r = run( + "auth-boundary", + "orgs.$organizationSlug.billing.ts", + `import { redirect } from "@remix-run/server-runtime"; + import { OrganizationParamsSchema, v3BillingPath } from "~/utils/pathBuilder"; + export const loader = async ({ params }) => { + const { organizationSlug } = OrganizationParamsSchema.parse(params); + return redirect(v3BillingPath({ slug: organizationSlug })); + };` + ); + expect(r.status).toBe("not-applicable"); + }); + + // The gate must not swallow the real thing: a body doing its own privileged work, unguarded. + it("still fails a sensitive route whose visible body does the work unguarded", () => { + const r = run( + "auth-boundary", + "api.v1.token.ts", + `import { prisma } from "~/db.server"; + import { createPersonalAccessToken } from "~/services/personalAccessToken.server"; + export async function action({ request }) { + const body = await request.json(); + const code = await prisma.authorizationCode.findFirst({ where: { code: body.code } }); + if (!code) return json({ error: "Not found" }, { status: 404 }); + const token = await createPersonalAccessToken(code.userId); + return json({ token }); + }` + ); + expect(r.status).toBe("fail"); + }); + + // Possession of a valid signature is the auth boundary for a callback URL. + const signatureRoute = (guard: string) => + run( + "auth-boundary", + "webhooks.v1.billing.$hash.ts", + `import { ${guard} } from "~/services/webhooks.server"; + import { prisma } from "~/db.server"; + export async function action({ request, params }) { + const invoice = await prisma.invoice.findFirst({ where: { id: params.id } }); + if (!${guard}(params.hash, invoice)) { + return json({ error: "Invalid" }, { status: 401 }); + } + return json({ ok: true }); + }` + ); + + it("passes a sensitive callback guarded by a signature check", () => { + expect(signatureRoute("verifyWebhook").status).toBe("pass"); + }); + + // C1a. The accept-list is derived from the helpers this webapp has. `verifyWebhookSignature` is + // a plausible name that exists nowhere in it, and the pattern this list replaced passed it. + it("does not pass a signature guard the webapp does not have", () => { + expect(signatureRoute("verifyWebhookSignature").status).toBe("fail"); + }); + + // False positive fixture: the guard sits one hop away, in a same-file helper. + it("does not flag a sensitive route whose guard is in a same-file helper", () => { + const r = run( + "auth-boundary", + "api.v1.tokens.ts", + `import { requireUserId } from "~/services/session.server"; + import { prisma } from "~/db.server"; + async function loadTokens(request) { + const userId = await requireUserId(request); + return prisma.token.findMany({ where: { userId } }); + } + export async function loader({ request }) { return json(await loadTokens(request)); }` + ); + expect(r.status).toBe("pass"); + }); + + // The guard has to be called, not merely imported: importedNames is file-wide. + it("fails a sensitive route that imports a guard it never calls", () => { + const r = run( + "auth-boundary", + "api.v1.tokens.ts", + `import { requireUserId } from "~/services/session.server"; + import { prisma } from "~/db.server"; + export async function loader() { + const tokens = await prisma.token.findMany(); + return json({ tokens }); + } + export function meta() { return requireUserId; }` + ); + expect(r.status).toBe("fail"); + }); +}); + +describe("request-context", () => { + it("passes a route whose failure log names an identifier", () => { + const r = run( + "request-context", + "engine.v1.dev.config.ts", + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function loader({ params }) { + try { return await prisma.thing.findMany(); } + catch (error) { + logger.error("dev config failed", { environmentId: params.envId, error }); + throw error; + } + }` + ); + expect(r.status).toBe("pass"); + }); + + it("passes on a route param, which names the tenant just as well", () => { + const r = run( + "request-context", + "resources.things.ts", + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function loader({ params }) { + try { return await prisma.thing.findMany(); } + catch (error) { + logger.error("lookup failed", { organizationSlug: params.organizationSlug, error }); + throw error; + } + }` + ); + expect(r.status).toBe("pass"); + }); + + it("fails a failure log that carries only the error", () => { + const r = run( + "request-context", + "api.v1.r.ts", + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function loader() { + try { return await prisma.thing.findMany(); } + catch (error) { logger.error("failed", { error }); throw error; } + }` + ); + expect(r.status).toBe("fail"); + }); + + it("fails a bare failure log with no object argument at all", () => { + const r = run( + "request-context", + "api.v1.r.ts", + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function loader() { + try { return await prisma.thing.findMany(); } catch (e) { logger.error("failed"); throw e; } + }` + ); + expect(r.status).toBe("fail"); + }); + + // The builder logs `{ error, url }` at its boundary and nothing that names a tenant, so being + // wrapped in one earns no pass here. This is what stops the check echoing `auth-boundary`. + it("fails a builder-wrapped route whose own failure log names nobody", () => { + const r = run( + "request-context", + "api.v1.q.ts", + `import { createLoaderApiRoute } from "~/services/routeBuilders/apiBuilder.server"; + import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export const loader = createLoaderApiRoute({}, async () => { + try { return json(await prisma.thing.findMany()); } + catch (error) { logger.error("failed", { error }); throw error; } + });` + ); + expect(r.status).toBe("fail"); + }); + + // C1. The global handler carries requestId, path, host and method, and no tenant. A route that + // never catches cannot name one, so it fails rather than being credited or excused. + it("fails a route that leaves everything to the central handler", () => { + const r = run( + "request-context", + "api.v1.q.ts", + `import { authenticateApiRequest } from "~/services/apiAuth.server"; + import { prisma } from "~/db.server"; + export async function loader({ request }) { + const auth = await authenticateApiRequest(request); + return json(await prisma.thing.findMany({ where: { environmentId: auth.environment.id } })); + }` + ); + expect(r.status).toBe("fail"); + }); + + it("fails a route that catches but only names an identifier outside the catch", () => { + const r = run( + "request-context", + "api.v1.q.ts", + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function loader() { + logger.info("starting", { environmentId: "env_1" }); + try { return await prisma.thing.findMany(); } catch (e) { throw e; } + }` + ); + expect(r.status).toBe("fail"); + }); + + // A route that catches and reports nothing at all is the case the log-based applicability gate + // used to excuse. It is a finding, not an exemption. + it("fails a route that catches and reports nothing", () => { + const r = run( + "request-context", + "api.v1.q.ts", + `import { prisma } from "~/db.server"; + export async function loader() { + try { return await prisma.thing.findMany(); } + catch (error) { return json({ error: "Internal Server Error" }, { status: 500 }); } + }` + ); + expect(r.status).toBe("fail"); + }); + + // A guard around a parse is not the route taking over its failure path: whatever its real work + // throws still reaches the central handler. Same reading error-classification gives the field. + it("fails a route whose only catch guards a parse", () => { + const r = run( + "request-context", + "api.v1.q.ts", + `import { prisma } from "~/db.server"; + export async function loader({ request }) { + let body; + try { body = await request.json(); } + catch { return json({ error: "Invalid JSON" }, { status: 400 }); } + return json(await prisma.thing.findMany({ where: body })); + }` + ); + expect(r.status).toBe("fail"); + }); + + // Was a known false positive: `new URL()` is a constructor, so the parse was invisible to the + // call-callee scan the evidence used to come from. `CatchEvidence.guardsParse` covers + // constructors, so the guard is legible now and the route is no longer judged as though it kept + // its failures. + it("fails a route whose only catch guards a constructor parse", () => { + const r = run( + "request-context", + "_app.@.orgs.$organizationSlug.$.tsx", + `import { prisma } from "~/db.server"; + function refererOrigin(request) { + const referer = request.headers.get("referer"); + try { return new URL(referer).origin; } + catch { return undefined; } + } + export async function loader({ request }) { + const origin = refererOrigin(request); + return typedjson({ origin, things: await prisma.thing.findMany() }); + }` + ); + expect(r.status).toBe("fail"); + }); + + it("fails a try/finally that catches nothing", () => { + const r = run( + "request-context", + "admin.api.v1.runs-replication.status.ts", + `import Redis from "ioredis"; + export async function loader() { + const redis = new Redis({ host: "localhost" }); + try { + const exists = await redis.exists("some-key"); + return json({ exists }); + } finally { + await redis.quit(); + } + }` + ); + expect(r.status).toBe("fail"); + }); + + it("still judges a route with a handler-wide catch beside a narrow guard", () => { + const r = run( + "request-context", + "api.v1.q.ts", + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function loader({ request }) { + let body; + try { body = await request.json(); } + catch { return json({ error: "Invalid JSON" }, { status: 400 }); } + try { + const rows = await prisma.thing.findMany({ where: body }); + const count = await prisma.thing.count(); + return json({ rows, count }); + } catch (error) { + logger.error("failed", { error }); + return json({ error: "Internal Server Error" }, { status: 500 }); + } + }` + ); + expect(r.status).toBe("fail"); + }); + + // The incentive fixture pair. The two routes differ by one line, the log call, and nothing else. + // Deleting that line must never improve the verdict or drop the route out of the report. + it("never improves a verdict when the log call is deleted", () => { + const body = (log: string) => + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function loader({ params }) { + try { return await prisma.thing.findMany(); } + catch (error) { ${log} throw error; } + }`; + + const withLog = run( + "request-context", + "api.v1.q.ts", + body(`logger.error("failed", { environmentId: params.envId, error });`) + ); + const withoutLog = run("request-context", "api.v1.q.ts", body("")); + + expect(withLog.status).toBe("pass"); + expect(withoutLog.status).toBe("fail"); + expect(withoutLog.status).not.toBe("not-applicable"); + }); + + // False positive fixture: the component logs every identifier there is, inside its own catch. + // Only the loader's own failure log may decide this. + it("does not read the React component's log calls", () => { + const bare = run( + "request-context", + "_app.orgs.$organizationSlug.things/route.tsx", + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function loader() { + try { return typedjson(await prisma.thing.findMany()); } + catch (error) { logger.error("failed", { error }); throw error; } + } + ${COMPONENT}` + ); + expect(bare.status).toBe("fail"); + + const attributed = run( + "request-context", + "_app.orgs.$organizationSlug.things/route.tsx", + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function loader({ params }) { + try { return typedjson(await prisma.thing.findMany()); } + catch (error) { logger.error("failed", { projectParam: params.projectParam, error }); throw error; } + } + ${COMPONENT}` + ); + expect(attributed.status).toBe("pass"); + }); + + it("is not applicable to a trivial redirect", () => { + const r = run( + "request-context", + "@.ts", + `import { redirect } from "@remix-run/server-runtime"; + export async function loader() { return redirect("/admin"); }` + ); + expect(r.status).toBe("not-applicable"); + }); + + // A5. IDENTIFIER_FIELD matched any suffix, so a resource id (a run, a batch, a notification, a + // chat, a span) that shares the same `Id`/`Param` shape as a tenant field passed, and a bare `id` + // passed too. TENANT_FIELD requires the root word itself to be environment, organization, project + // or user. + it("does not pass a failure log that only names a resource, not a tenant", () => { + const r = run( + "request-context", + "api.v3.batches.$batchId.ts", + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function loader({ params }) { + try { return await prisma.thing.findMany(); } + catch (error) { + logger.error("batch lookup failed", { batchId: params.batchId, error }); + throw error; + } + }` + ); + expect(r.status).toBe("fail"); + }); + + it("does not pass a failure log that only names a bare id", () => { + const r = run( + "request-context", + "api.v1.q.ts", + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function loader() { + try { return await prisma.thing.findMany(); } + catch (error) { + logger.debug("cache miss", { id: 1 }); + logger.error("lookup failed", { id: 1, error }); + throw error; + } + }` + ); + expect(r.status).toBe("fail"); + }); + + it("passes on the abbreviated envId/orgId forms the webapp also writes", () => { + const r = run( + "request-context", + "api.v1.q.ts", + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function loader({ params }) { + try { return await prisma.thing.findMany(); } + catch (error) { + logger.error("lookup failed", { envId: params.envId, orgId: params.orgId, error }); + throw error; + } + }` + ); + expect(r.status).toBe("pass"); + }); + + // A5. request-context filtered failure-path logs on inCatch only, not on level, so a debug log + // naming a tenant field passed the check even though debug lines are routinely dropped or + // sampled out before an incident is read. + it("does not pass a debug-level failure log, even one that names a tenant", () => { + const r = run( + "request-context", + "api.v1.q.ts", + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function loader({ params }) { + try { return await prisma.thing.findMany(); } + catch (error) { + logger.debug("lookup failed", { environmentId: params.envId, error }); + throw error; + } + }` + ); + expect(r.status).toBe("fail"); + }); + + it("does not pass an info-level failure log either", () => { + const r = run( + "request-context", + "api.v1.q.ts", + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function loader({ params }) { + try { return await prisma.thing.findMany(); } + catch (error) { + logger.info("lookup failed", { environmentId: params.envId, error }); + throw error; + } + }` + ); + expect(r.status).toBe("fail"); + }); + + it("still passes a warn-level failure log that names a tenant", () => { + const r = run( + "request-context", + "api.v1.q.ts", + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function loader({ params }) { + try { return await prisma.thing.findMany(); } + catch (error) { + logger.warn("lookup failed", { environmentId: params.envId, error }); + throw error; + } + }` + ); + expect(r.status).toBe("pass"); + }); + + // M7. The real logger (packages/core/src/logger.ts) has log/error/warn/info/debug/verbose, no + // fatal and no trace, and log is level 0, never filtered by TRIGGER_LOG_LEVEL, so it must + // qualify. verbose is the actual noisiest level this codebase has, not trace. + it("passes a log-level failure log, which the real logger never filters", () => { + const r = run( + "request-context", + "api.v1.q.ts", + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function loader({ params }) { + try { return await prisma.thing.findMany(); } + catch (error) { + logger.log("lookup failed", { environmentId: params.envId, error }); + throw error; + } + }` + ); + expect(r.status).toBe("pass"); + }); + + it("does not pass a verbose-level failure log", () => { + const r = run( + "request-context", + "api.v1.q.ts", + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function loader({ params }) { + try { return await prisma.thing.findMany(); } + catch (error) { + logger.verbose("lookup failed", { environmentId: params.envId, error }); + throw error; + } + }` + ); + expect(r.status).toBe("fail"); + }); + + // M8. A bare `env` field is ambiguous with a deployment environment name + // (`{ env: process.env.NODE_ENV }`), not a tenant, so it must not qualify on its own; the + // abbreviated root still works with a real suffix. + it("does not pass a failure log that only names a bare env field", () => { + const r = run( + "request-context", + "api.v1.q.ts", + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function loader() { + try { return await prisma.thing.findMany(); } + catch (error) { + logger.error("lookup failed", { env: process.env.NODE_ENV, error }); + throw error; + } + }` + ); + expect(r.status).toBe("fail"); + }); + + it("still passes envId, the abbreviated root with a real suffix", () => { + const r = run( + "request-context", + "api.v1.q.ts", + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function loader({ params }) { + try { return await prisma.thing.findMany(); } + catch (error) { + logger.error("lookup failed", { envId: params.envId, error }); + throw error; + } + }` + ); + expect(r.status).toBe("pass"); + }); +}); + +describe("audit-trail", () => { + it("is applicable only to sensitive mutations", () => { + const readOnly = run( + "audit-trail", + "api.v1.auth.jwt.ts", + `export async function loader() { return 1; }` + ); + expect(readOnly.status).toBe("not-applicable"); + + const mutation = run( + "audit-trail", + "api.v1.auth.jwt.ts", + `import { prisma } from "~/db.server"; + export async function action() { return prisma.token.create({ data: {} }); }` + ); + expect(mutation.status).toBe("fail"); + }); + + // False positive fixture: an ordinary mutation is not an audit target. + it("is not applicable to a non-sensitive mutation", () => { + const r = run( + "audit-trail", + "resources.things.ts", + `import { prisma } from "~/db.server"; + export async function action() { return prisma.thing.create({ data: {} }); }` + ); + expect(r.status).toBe("not-applicable"); + }); + + // The pass branch had never been exercised against a real name: the fixture imported `auditLog` + // from `~/services/audit.server`, a helper and a module that both exist nowhere. All three names + // on the list now reach `prisma.impersonationAuditLog.create` in `models/admin.server.ts`. + it.each(["redirectWithImpersonation", "clearImpersonation", "startImpersonation"])( + "passes a sensitive mutation that records an audit event through %s", + (writer) => { + const r = run( + "audit-trail", + "admin.impersonate.tsx", + `import { ${writer} } from "~/models/admin.server"; + import { prisma } from "~/db.server"; + export async function action({ request }) { + const target = await prisma.user.findFirst({ where: { admin: false } }); + const session = await ${writer}(request, target.id, "/"); + return session; + }` + ); + expect(r.status).toBe("pass"); + expect(r.detail).toBe("records an audit event"); + } + ); + + it("still fails a sensitive mutation that writes no record", () => { + const r = run( + "audit-trail", + "api.v1.auth.jwt.ts", + `import { prisma } from "~/db.server"; + export async function action({ request }) { + const token = await prisma.token.create({ data: { name: request.url } }); + return json(token); + }` + ); + expect(r.status).toBe("fail"); + }); + + // The coherence fix. `auth-boundary` declines to judge a trivial body because a guard would be + // behind the import; this check accused the same body over an audit write behind the same + // import. Same rule now, and presence is still read before the exemption, so a trivial body that + // does call a writer passes rather than sitting out. + it("declines to judge a trivial sensitive mutation, as auth-boundary does", () => { + const source = `import { doTheThing } from "~/models/admin.server"; + export async function action({ request }) { return doTheThing(request, "/admin"); }`; + expect(run("audit-trail", "resources.impersonation.ts", source).status).toBe("not-applicable"); + expect(run("auth-boundary", "resources.impersonation.ts", source).status).toBe( + "not-applicable" + ); + }); + + it("still passes a trivial sensitive mutation that calls a writer", () => { + const r = run( + "audit-trail", + "resources.impersonation.ts", + `import { clearImpersonation } from "~/models/admin.server"; + export async function action({ request }) { return clearImpersonation(request, "/admin"); }` + ); + expect(r.status).toBe("pass"); + }); +}); + +// C1a. The guard list is names now, not a five-character prefix. Both directions matter: a real +// helper must still clear a sensitive route, and a callee that merely starts the right way must +// not. +describe("auth-boundary: the guard accept-list", () => { + const sensitiveRoute = (guard: string) => + run( + "auth-boundary", + "api.v1.orgs.$orgParam.members.ts", + `import { prisma } from "~/db.server"; + export async function loader({ request, params }) { + const caller = await ${guard}(request); + const members = await prisma.orgMember.findMany({ where: { orgId: params.orgParam } }); + return json({ members, caller }); + }` + ); + + it.each(["requireUserId", "requireUser", "authenticateApiRequest", "authenticateSession"])( + "passes a sensitive route guarded by %s", + (guard) => { + expect(sensitiveRoute(guard).status).toBe("pass"); + } + ); + + // The live case. `requireSsoEntitlement` is a plan check inside one route file, and the prefix + // pattern cleared the org SSO settings route on it. + it("does not pass a plan check that happens to start with require", () => { + const r = sensitiveRoute("requireSsoEntitlement"); + expect(r.status).toBe("fail"); + expect(r.detail).toContain("no auth guard in the body"); + }); + + // The two live shapes that made the non-throwing variants worth crediting. Both act on the + // answer, which is why they are on the list; a route that calls one and ignores it is the + // residual, stated on `GUARDS`. + // Round C ruling 2. getUser and getUserId answer with null instead of throwing, so being called + // is not evidence of a boundary. They are credited only when the body reads the answer. + it("does not pass a route that resolves the caller and ignores the answer", () => { + const r = run( + "auth-boundary", + "invite-accept.tsx", + `import { getUser } from "~/services/session.server"; + import { getInviteFromToken } from "~/models/member.server"; + export async function loader({ request }) { + const user = await getUser(request); + const token = new URL(request.url).searchParams.get("token"); + const invite = await getInviteFromToken({ token }); + return json({ invite, email: user.email }); + }` + ); + expect(r.status).toBe("fail"); + }); + + it("does not pass a route that drops the caller entirely", () => { + const r = run( + "auth-boundary", + "invite-accept.tsx", + `import { getUserId } from "~/services/session.server"; + import { getInviteFromToken } from "~/models/member.server"; + export async function loader({ request }) { + await getUserId(request); + const token = new URL(request.url).searchParams.get("token"); + return json(await getInviteFromToken({ token })); + }` + ); + expect(r.status).toBe("fail"); + }); + + it("passes an invite acceptance that resolves the caller and refuses a mismatch", () => { + const r = run( + "auth-boundary", + "invite-accept.tsx", + `import { getUser } from "~/services/session.server"; + import { getInviteFromToken } from "~/models/member.server"; + export async function loader({ request }) { + const user = await getUser(request); + const token = new URL(request.url).searchParams.get("token"); + if (!user) return redirect("/login"); + const invite = await getInviteFromToken({ token }); + if (invite.email !== user.email) return redirect("/"); + return redirect("/"); + }` + ); + expect(r.status).toBe("pass"); + }); + + it("passes a login page that sends an already authenticated caller away", () => { + const r = run( + "auth-boundary", + "login._index/route.tsx", + `import { getUserId } from "~/services/session.server"; + import { prisma } from "~/db.server"; + export async function loader({ request }) { + const userId = await getUserId(request); + if (userId) return redirect("/"); + const flags = await prisma.featureFlag.findMany(); + return typedjson({ flags }); + }` + ); + expect(r.status).toBe("pass"); + }); + + it("does not pass an invented require helper", () => { + expect(sensitiveRoute("requireValidParams").status).toBe("fail"); + expect(sensitiveRoute("requireQueryParam").status).toBe("fail"); + }); + + // `resolveAuthenticatedEnv` hydrates an environment record from its id. Ten routes call it and + // the /Authenticated/ pattern read every one of them as guarded. + it("does not pass a lookup whose name merely contains Authenticated", () => { + expect(sensitiveRoute("resolveAuthenticatedEnv").status).toBe("fail"); + expect(sensitiveRoute("commitAuthenticatedSession").status).toBe("fail"); + }); +}); + +/** + * Per-export attribution. Every input `auth-boundary` reads used to be entry-point-wide, so one + * guarded export spoke for the whole file. Each `it` here goes green on the entry-point-wide + * version of exactly one of those inputs, which is why they are separate cases rather than one. + */ +describe("auth-boundary: a guard credits only the export that calls it", () => { + const TOKENS = `import { requireUserId } from "~/services/session.server"; + import { prisma } from "~/db.server";`; + + it("fails an unguarded action beside a loader that calls a guard", () => { + const r = run( + "auth-boundary", + "api.v1.tokens.ts", + `${TOKENS} + export async function loader({ request }) { + const userId = await requireUserId(request); + return json(await prisma.token.findMany({ where: { userId } })); + } + export async function action({ request }) { + const body = await request.json(); + await prisma.token.deleteMany({ where: { id: body.id } }); + return json({ ok: true }); + }` + ); + expect(r.status).toBe("fail"); + expect(r.detail).toContain("action"); + }); + + it("fails an unguarded loader beside an action that calls a guard", () => { + const r = run( + "auth-boundary", + "api.v1.tokens.ts", + `${TOKENS} + export async function loader({ params }) { + const tokens = await prisma.token.findMany({ where: { orgId: params.orgId } }); + return json({ tokens }); + } + export async function action({ request }) { + const userId = await requireUserId(request); + await prisma.token.deleteMany({ where: { userId } }); + return json({ ok: true }); + }` + ); + expect(r.status).toBe("fail"); + expect(r.detail).toContain("loader"); + }); + + // The soft-guard arm reads its own export's checked-callee list for the same reason. + it("fails an unguarded action beside a loader that reads what getUserId returned", () => { + const r = run( + "auth-boundary", + "api.v1.tokens.ts", + `import { getUserId } from "~/services/session.server"; + import { prisma } from "~/db.server"; + export async function loader({ request }) { + const userId = await getUserId(request); + if (!userId) return redirect("/login"); + return json(await prisma.token.findMany({ where: { userId } })); + } + export async function action({ request }) { + const body = await request.json(); + await prisma.token.deleteMany({ where: { id: body.id } }); + return json({ ok: true }); + }` + ); + expect(r.status).toBe("fail"); + expect(r.detail).toContain("action"); + }); + + // The builder arm. `usesBuilder` was an OR over both initializer callees, so a builder on one + // export authenticated a hand-written handler on the other. + it("fails a hand-written action beside a builder-wrapped loader", () => { + const r = run( + "auth-boundary", + "api.v1.tokens.ts", + `import { createLoaderApiRoute } from "~/services/routeBuilders/apiBuilder.server"; + import { prisma } from "~/db.server"; + export const loader = createLoaderApiRoute({}, async ({ authentication }) => { + return json(await prisma.token.findMany({ where: { userId: authentication.userId } })); + }); + export async function action({ request }) { + const body = await request.json(); + await prisma.token.deleteMany({ where: { id: body.id } }); + return json({ ok: true }); + }` + ); + expect(r.status).toBe("fail"); + expect(r.detail).toContain("action"); + }); + + it("passes when both exports call a guard of their own", () => { + const r = run( + "auth-boundary", + "api.v1.tokens.ts", + `${TOKENS} + export async function loader({ request }) { + const userId = await requireUserId(request); + return json(await prisma.token.findMany({ where: { userId } })); + } + export async function action({ request }) { + const userId = await requireUserId(request); + await prisma.token.deleteMany({ where: { userId } }); + return json({ ok: true }); + }` + ); + expect(r.status).toBe("pass"); + }); + + // One handler serving both exports guards both, which is the dominant API-route shape. + it("passes a shared builder handler that both exports resolve to", () => { + const r = run( + "auth-boundary", + "api.v1.tokens.ts", + `import { createActionApiRoute } from "~/services/routeBuilders/apiBuilder.server"; + export const { action, loader } = createActionApiRoute({}, async ({ authentication }) => { + return json({ userId: authentication.userId }); + });` + ); + expect(r.status).toBe("pass"); + }); + + /** + * The damper on the attribution, and the reason it is not simply "accuse every unguarded export". + * `auth.github.ts` and `auth.google.ts` are this shape: per export the loader is unguarded, and + * an entry-point-wide triviality rule calls the file non-trivial because the ACTION is not. Both + * routes went pass to fail on the real tree until `isTrivialExport` existed. + */ + it("reports not-applicable for a redirect-stub loader beside a guarded action", () => { + const r = run( + "auth-boundary", + "auth.github.ts", + `import { authenticator } from "~/services/auth.server"; + export let loader = () => redirect("/login"); + export let action = async ({ request }) => { + const url = new URL(request.url); + const safeRedirect = sanitizeRedirectPath(url.searchParams.get("redirectTo"), "/"); + return await authenticator.authenticate("github", request, { + successRedirect: safeRedirect, + failureRedirect: "/login", + }); + };` + ); + expect(r.status).toBe("pass"); + expect(r.detail).toBe("guarded in the body"); + }); + + /** + * The per-export excuse must read the export's own body and not the file's text. It read + * `ep.source` first, and `log-caller-scope-userid` in the mutation corpus, which prepends + * `logger.error(...)` to every body, put the word `logger` in this file and turned the untouched + * loader from excused into accused. A five-minute corpus run is the wrong place to catch that. + */ + it("does not un-excuse a redirect-stub loader because the file mentions a logger", () => { + const r = run( + "auth-boundary", + "auth.github.ts", + `import { authenticator } from "~/services/auth.server"; + import { logger } from "~/services/logger.server"; + export let loader = () => redirect("/login"); + export let action = async ({ request }) => { + logger.error("obs-map", { userId: request.userId }); + return await authenticator.authenticate("github", request, { + successRedirect: "/", + failureRedirect: "/login", + }); + };` + ); + expect(r.status).toBe("pass"); + }); + + it("fails an export whose own body does real work unguarded", () => { + const r = run( + "auth-boundary", + "auth.github.ts", + `import { authenticator } from "~/services/auth.server"; + import { prisma } from "~/db.server"; + export let loader = async ({ params }) => { + const org = await prisma.organization.findFirst({ where: { slug: params.slug } }); + const members = await prisma.orgMember.findMany({ where: { orgId: org.id } }); + return json({ org, members }); + }; + export let action = async ({ request }) => { + return await authenticator.authenticate("github", request, { + successRedirect: "/", + failureRedirect: "/login", + }); + };` + ); + expect(r.status).toBe("fail"); + expect(r.detail).toContain("loader"); + }); +}); + +// C1b. A builder authenticates the request; it does not necessarily scope it. `authorization` is +// optional on every one of them and the RBAC gate only runs when it is declared. +describe("auth-scope", () => { + const patRoute = (options: string, body: string) => + run( + "auth-scope", + "api.v1.orgs.$orgParam.members.ts", + `import { createActionPATApiRoute } from "~/services/routeBuilders/apiBuilder.server"; + import { prisma } from "~/db.server"; + export const action = createActionPATApiRoute( + { method: "POST"${options}}, + async ({ authentication, params }) => { + ${body} + return json({ members }); + } + );` + ); + + const UNSCOPED = `const members = await prisma.orgMember.findMany({ + where: { organization: { slug: params.orgParam } }, + });`; + + it("fails a sensitive PAT route that authenticates and scopes nothing", () => { + const r = patRoute("", UNSCOPED); + expect(r.status).toBe("fail"); + expect(r.detail).toContain("authenticated but not scoped to the caller"); + expect(r.detail).toContain("createActionPATApiRoute"); + }); + + it("passes when the builder declares the authorization gate", () => { + const r = patRoute( + `, authorization: { action: "read", resource: { type: "members" } }`, + UNSCOPED + ); + expect(r.status).toBe("pass"); + expect(r.detail).toContain("authorization gate"); + }); + + it("passes when the handler filters by the caller's membership instead", () => { + const r = patRoute( + "", + `const members = await prisma.orgMember.findMany({ + where: { organization: { slug: params.orgParam, members: { some: { userId: authentication.userId } } } }, + });` + ); + expect(r.status).toBe("pass"); + expect(r.detail).toContain("caller's identity"); + }); + + // Round C ruling 1. apps/webapp/CLAUDE.md: the OSS fallback ability is permissive, so + // `ability.can(...)` enforces the role and the membership-scoped query is the tenant floor. + // Crediting it made this check agree with `_app.orgs.$organizationSlug.settings.sso/route.tsx`, + // which resolves its target org from the URL slug and puts nothing else in front of it. + it("does not accept an ability gate in the handler as scoping", () => { + const r = run( + "auth-scope", + "_app.orgs.$slug.settings.team/route.tsx", + `import { dashboardAction } from "~/services/routeBuilders/dashboardBuilder"; + import { prisma } from "~/db.server"; + export const action = dashboardAction({ params: Params }, async ({ ability, params }) => { + if (!ability.can("manage", { type: "members" })) throw new Response(null, { status: 403 }); + const members = await prisma.orgMember.findMany({ where: { slug: params.slug } }); + return json({ members }); + });` + ); + expect(r.status).toBe("fail"); + }); + + // The two shapes on the real tree, both hand-read for round C. + it("fails when the loader is unscoped and only the action filters by the caller", () => { + const r = run( + "auth-scope", + "_app.orgs.$slug.settings.sso/route.tsx", + `import { dashboardAction, dashboardLoader } from "~/services/routeBuilders/dashboardBuilder"; + import { prisma } from "~/db.server"; + export const loader = dashboardLoader({ params: Params }, async ({ context, ability }) => { + if (!ability.can("manage", { type: "sso" })) throwPermissionDenied(); + return json(await prisma.ssoConnection.findMany({ where: { organizationId: context.organizationId } })); + }); + export const action = dashboardAction({ params: Params }, async ({ context, user }) => + json(await ssoController.generatePortalLink({ organizationId: context.organizationId, userId: user.id })) + );` + ); + expect(r.status).toBe("fail"); + expect(r.detail).toContain("loader (dashboardLoader)"); + expect(r.detail).not.toContain("action"); + }); + + it("fails when the action is unscoped and only the loader filters by the caller", () => { + const r = run( + "auth-scope", + "_app.orgs.$slug.settings.team/route.tsx", + `import { dashboardAction, dashboardLoader } from "~/services/routeBuilders/dashboardBuilder"; + import { prisma } from "~/db.server"; + export const loader = dashboardLoader({ params: Params }, async ({ user }) => + json(await new TeamPresenter().call({ userId: user.id })) + ); + export const action = dashboardAction({ params: Params }, async ({ context, ability }) => { + if (!ability.can("manage", { type: "members" })) throwPermissionDenied(); + return json(await prisma.orgMember.deleteMany({ where: { organizationId: context.organizationId } })); + });` + ); + expect(r.status).toBe("fail"); + expect(r.detail).toContain("action (dashboardAction)"); + }); + + // A file whose loader is gated and whose action is not is not a gated route. + it("fails when only one of two builder exports declares the gate", () => { + const r = run( + "auth-scope", + "api.v1.orgs.$orgParam.members.ts", + `import { createActionPATApiRoute, createLoaderPATApiRoute } from "~/services/routeBuilders/apiBuilder.server"; + import { prisma } from "~/db.server"; + export const loader = createLoaderPATApiRoute( + { authorization: { action: "read", resource: { type: "members" } } }, + async ({ params }) => json(await prisma.orgMember.findMany({ where: { slug: params.orgParam } })) + ); + export const action = createActionPATApiRoute({ method: "POST" }, async ({ params }) => + json(await prisma.orgMember.deleteMany({ where: { slug: params.orgParam } })) + );` + ); + expect(r.status).toBe("fail"); + expect(r.detail).toContain("action (createActionPATApiRoute)"); + }); + + // Round D item 3, at the check level: the shape that cleared both real findings. + it("is not cleared by a dead object holding the caller id", () => { + const r = run( + "auth-scope", + "api.v1.orgs.$orgParam.members.ts", + `import { createActionPATApiRoute } from "~/services/routeBuilders/apiBuilder.server"; + import { prisma } from "~/db.server"; + export const action = createActionPATApiRoute({ method: "POST" }, async ({ user, params }) => { + const unused = { userId: user.id }; + return json(await prisma.orgMember.deleteMany({ where: { slug: params.orgParam } })); + });` + ); + expect(r.status).toBe("fail"); + }); + + it("is not applicable to a route that is not sensitive", () => { + const r = run( + "auth-scope", + "api.v1.runs.ts", + `import { createActionApiRoute } from "~/services/routeBuilders/apiBuilder.server"; + import { prisma } from "~/db.server"; + export const action = createActionApiRoute({ method: "POST" }, async ({ params }) => + json(await prisma.taskRun.findMany({ where: { id: params.id } })) + );` + ); + expect(r.status).toBe("not-applicable"); + expect(r.detail).toBe("not sensitive"); + }); + + it("is not applicable to a sensitive route with no builder to read options from", () => { + const r = run( + "auth-scope", + "api.v1.orgs.$orgParam.members.ts", + `import { requireUserId } from "~/services/session.server"; + import { prisma } from "~/db.server"; + export async function action({ request, params }) { + await requireUserId(request); + return json(await prisma.orgMember.deleteMany({ where: { slug: params.orgParam } })); + }` + ); + expect(r.status).toBe("not-applicable"); + expect(r.detail).toContain("no route builder"); + }); + + // A builder-wrapped route is never trivial, so the sensitive routes that sit out for having + // nothing in the body sit out here for having no builder instead. + it("is not applicable to a trivial sensitive route with no builder", () => { + const r = run( + "auth-scope", + "orgs.$organizationSlug.team.ts", + `import { redirect } from "@remix-run/server-runtime"; + export async function loader({ params }) { return redirect(teamPath(params.slug)); }` + ); + expect(r.status).toBe("not-applicable"); + expect(r.detail).toContain("no route builder"); + }); +}); + +// The cheapest way to launder auth-scope would be to write the option and give it nothing, which +// the builder's own `if (authorization)` treats as absent. +describe("auth-scope: an option declared as nothing is not declared", () => { + const withValue = (value: string) => + run( + "auth-scope", + "api.v1.orgs.$orgParam.members.ts", + `import { createActionPATApiRoute } from "~/services/routeBuilders/apiBuilder.server"; + import { prisma } from "~/db.server"; + export const action = createActionPATApiRoute( + { method: "POST", authorization: ${value} }, + async ({ params }) => json(await prisma.orgMember.findMany({ where: { slug: params.orgParam } })) + );` + ); + + it.each(["undefined", "null", "false"])("fails on authorization: %s", (value) => { + expect(withValue(value).status).toBe("fail"); + }); + + it("passes on a real one, so the rule is about the value and not the key", () => { + expect(withValue(`{ action: "read", resource: { type: "members" } }`).status).toBe("pass"); + }); +}); diff --git a/internal-packages/observability-map/src/checks/index.ts b/internal-packages/observability-map/src/checks/index.ts new file mode 100644 index 00000000000..6dd1fbc4886 --- /dev/null +++ b/internal-packages/observability-map/src/checks/index.ts @@ -0,0 +1,23 @@ +import type { CheckResult, EntryPoint } from "../types.js"; +import { errorClassification } from "./errorClassification.js"; +import { authBoundary } from "./authBoundary.js"; +import { authScope } from "./authScope.js"; +import { requestContext } from "./requestContext.js"; +import { auditTrail } from "./auditTrail.js"; + +export type Check = { id: string; run: (ep: EntryPoint) => CheckResult }; + +/** audit-trail is scored separately, see score.ts. */ +export const CHECKS: Check[] = [ + errorClassification, + authBoundary, + authScope, + requestContext, + auditTrail, +]; +export const SCORED_CHECK_IDS = [ + "error-classification", + "auth-boundary", + "auth-scope", + "request-context", +]; diff --git a/internal-packages/observability-map/src/checks/requestContext.ts b/internal-packages/observability-map/src/checks/requestContext.ts new file mode 100644 index 00000000000..f54dc8d5f9e --- /dev/null +++ b/internal-packages/observability-map/src/checks/requestContext.ts @@ -0,0 +1,92 @@ +import type { CheckResult, EntryPoint } from "../types.js"; +import { isTrivial } from "../triviality.js"; + +const ID = "request-context"; + +/** + * A field name that plausibly names a TENANT: environment, organization, project or user, the four + * things every entry point ultimately belongs to. Anchored on the root word, not just the suffix, + * in the full and abbreviated camelCase the webapp actually writes for each: `environmentId`/ + * `envId`, `organizationId`/`organizationSlug`/`orgId`, `projectId`/`projectParam`, `userId`. A bare + * `id`, and a resource id that happens to share the same `Id`/`Param` suffix, `batchId`, + * `notificationId`, `chatId`, `spanParam`, `runFriendlyId`, `taskIdentifier`, does not qualify: + * those name a resource the failure touched, not who it happened to. + * + * The abbreviated roots, `env` and `org`, require a suffix; the full words do not. A bare `env` is + * ambiguous with a deployment environment name (`{ env: process.env.NODE_ENV }`), which is not a + * tenant, and nothing in the tree relies on it being bare, so the field alone cannot qualify. + */ +const TENANT_FIELD = + /^(environment|organization|project|user)(Id|Ids|Slug|Ref|Param|Identifier)?$|^(env|org)(Id|Ids|Slug|Ref|Param|Identifier)$/; + +/** + * Levels against the real logger (`packages/core/src/logger.ts`): `log`, `error`, `warn`, `info`, + * `debug`, `verbose`, in that order, no `fatal` and no `trace` (the `trace` in + * `apps/webapp/app/services/logger.server.ts` is the AsyncLocalStorage field helper, unrelated to + * log level). `log` is level 0, the level `TRIGGER_LOG_LEVEL` never filters out, so it qualifies + * alongside `error` and `warn`. `info`, `debug` and `verbose` do not: `info` is not reserved for + * failure reporting, so a route can log an info line inside a catch that says nothing about the + * catch actually handling anything, and `debug`/`verbose` are routinely dropped or sampled out + * before anyone reads an incident. + */ +const QUALIFYING_LEVELS = new Set(["log", "error", "warn"]); + +/** The level a `LogCall`'s callee was made at, e.g. `"error"` from `logger.error`. */ +function logLevel(callee: string): string { + return callee.slice(callee.lastIndexOf(".") + 1); +} + +/** + * Whether a failure here can be traced to whoever it happened to. + * + * Everything the platform attaches centrally is accounted for, which is what makes this worth + * asking. `logger` pushes the http context, `{ requestId, path, host, method }`, onto every line + * through AsyncLocalStorage, and `Logger.onError` forwards the error to Sentry. Neither carries a + * tenant: no route calls `trace({ environmentId }, ...)`, and the builders' own boundary log is + * `logBoundaryError(message, error, url)`, a url and an error. So an incident tells you which route + * and which request failed, and never whose environment it was, unless the route passed the field + * itself. 11 of 427 entry points do, naming an environment, organization, project or user; the + * other 10 that used to be counted here only named a resource the failure touched, not a tenant. + * + * Every non-trivial entry point is judged, and a route that never catches fails like any other. + * That is the whole point rather than an oversight: its failures go to the global handler, which + * names no tenant, so it genuinely cannot say whose request broke. Passing those routes, as this + * check used to, meant deleting every catch clause in the tree scored it 100. Excusing them as + * not-applicable would be the same mistake in quieter clothes, since it would once again reward + * having no failure handling to inspect. + * + * The consequence is a check that fails 90% of what it looks at, which is an honest reading of a + * codebase where the fix is one platform change, tenant fields through `trace(...)` in the auth + * path, rather than 300 route edits. Weight it accordingly, but do not read the count as noise. + */ +export const requestContext = { + id: ID, + run(ep: EntryPoint): CheckResult { + if (isTrivial(ep)) { + return { id: ID, status: "not-applicable", detail: "trivial route" }; + } + const failurePathLogs = ep.logCalls.filter( + (l) => l.inCatch && QUALIFYING_LEVELS.has(logLevel(l.callee)) + ); + const named = failurePathLogs.find((l) => l.fields.some((f) => TENANT_FIELD.test(f))); + if (named) { + const fields = named.fields.filter((f) => TENANT_FIELD.test(f)); + return { id: ID, status: "pass", detail: `failure log names ${fields.join(", ")}` }; + } + if (failurePathLogs.length > 0) { + return { + id: ID, + status: "fail", + detail: "logs its failure without naming an environment, organization, project or user", + }; + } + return { + id: ID, + status: "fail", + detail: + ep.catches.length > 0 + ? "keeps its failures and records nothing about whose they were" + : "leaves its failures to the central handler, which names no tenant", + }; + }, +}; diff --git a/internal-packages/observability-map/src/cli.test.ts b/internal-packages/observability-map/src/cli.test.ts new file mode 100644 index 00000000000..9444d603b09 --- /dev/null +++ b/internal-packages/observability-map/src/cli.test.ts @@ -0,0 +1,224 @@ +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { main, type Io } from "./cli.js"; + +/** + * A routes tree of this package's own making. These tests used to run against + * `apps/webapp/app/routes` and assert that `/api/v1/token` exists and that `api.v1.runs.ts` is line + * 2 of the output, which is an assertion about the webapp's contents rather than about this CLI. + * A webapp-only pull request renaming a route broke them, and `pr_checks.yml` did not run this + * suite for such a pull request, so the break landed on whoever pushed next. The one deliberate + * real-tree test lives in `integration.test.ts` and asserts only invariants that survive churn. + */ +const ROUTES = mkdtempSync(join(tmpdir(), "obs-map-fixture-routes-")); + +const withWork = (name: string) => `import { prisma } from "~/db.server"; + export async function loader() { + try { return await prisma.${name}.findMany(); } catch (e) { return null; } + }`; + +const FIXTURES: Record = { + // Exact target, and also the prefix of the two below it. + "api.v1.runs.ts": withWork("run"), + "api.v1.runs.$runId.ts": withWork("run"), + "api.v1.runs.$runId.cancel.ts": withWork("run"), + "api.v1.token.ts": withWork("token"), + // Five routes sharing a prefix that is not itself a route, for the ambiguity warning and for the + // "and N more" tail it grows past four matches. + "admin.api.v1.runs-replication.start.ts": withWork("replication"), + "admin.api.v1.runs-replication.stop.ts": withWork("replication"), + "admin.api.v1.runs-replication.status.ts": withWork("replication"), + "admin.api.v1.runs-replication.retry.ts": withWork("replication"), + "admin.api.v1.runs-replication.purge.ts": withWork("replication"), + // Nothing applicable: exercises the not-measured note. + "resources.health.ts": `export const loader = () => new Response("ok");`, + // A directive whose id names no check, for the warning it has to produce. + "api.v1.typo.ts": `// obs-map-disable eror-classification -- typo\n${withWork("thing")}`, +}; + +beforeAll(() => { + for (const [name, source] of Object.entries(FIXTURES)) { + writeFileSync(join(ROUTES, name), source); + } + // A directory route, so the fixture covers both shapes `scanDirectory` walks. + mkdirSync(join(ROUTES, "_app.orgs.$slug")); + writeFileSync(join(ROUTES, "_app.orgs.$slug", "route.tsx"), withWork("organization")); +}); + +afterAll(() => rmSync(ROUTES, { recursive: true, force: true })); + +const capture = () => { + const out: string[] = []; + const err: string[] = []; + const io: Io = { out: (s) => out.push(s), err: (s) => err.push(s) }; + return { io, out: () => out.join(""), err: () => err.join("") }; +}; + +const run = (...args: string[]) => { + const c = capture(); + const code = main(["node", "cli.js", `--routes=${ROUTES}`, ...args], c.io); + return { code, out: c.out(), err: c.err() }; +}; + +describe("map ", () => { + // I8. The report prints route paths, so the identifier on screen has to be one you can paste + // back in. Matching file names only meant `map /api/v1/token` exited 1. + it("accepts the route path the report prints", () => { + const r = run("/api/v1/token"); + expect(r.code).toBe(0); + expect(r.out).toContain("/api/v1/token"); + expect(r.out).toContain("CHECKS"); + }); + + it("accepts a file name too", () => { + const r = run("api.v1.token.ts"); + expect(r.code).toBe(0); + expect(r.out).toContain("api.v1.token.ts"); + }); + + it("accepts a directory route by the path its directory segment spells", () => { + const r = run("/_app/orgs/:slug"); + expect(r.code).toBe(0); + expect(r.out).toContain("_app.orgs.$slug/route.tsx"); + }); + + it("exits 1 with a message when nothing matches", () => { + const r = run("/api/v1/does-not-exist"); + expect(r.code).toBe(1); + expect(r.err).toContain("no entry point matching"); + }); + + it("warns when a prefix matches more than one route rather than silently taking the first", () => { + const r = run("/admin/api/v1/runs-replication"); + expect(r.code).toBe(0); + expect(r.err).toMatch(/matches 5 entry points, showing the first/); + expect(r.err).toContain("Others:"); + expect(r.err).toContain("and 1 more"); + }); + + // `/api/v1/runs` is a prefix of two others in the fixture, and also a route in its own right. + it("prefers an exact match over the routes it is a prefix of", () => { + const r = run("/api/v1/runs"); + expect(r.err).toBe(""); + expect(r.out.split("\n")[1]).toBe("api.v1.runs.ts"); + }); + + it("says so rather than printing a bare 100 when nothing applied", () => { + const r = run("/resources/health"); + expect(r.code).toBe(0); + expect(r.out).toContain("not measured"); + }); + + // B7. `map /api/v1/token --json` printed the text format and dropped the flag on the floor. + it("honours --json for a single route instead of printing the text format", () => { + const r = run("/api/v1/token", "--json"); + expect(r.code).toBe(0); + expect(r.out).not.toContain("CHECKS"); + const parsed = JSON.parse(r.out); + expect(parsed.fileName).toBe("api.v1.token.ts"); + expect(parsed.routePath).toBe("/api/v1/token"); + expect(Array.isArray(parsed.checks)).toBe(true); + }); +}); + +describe("map", () => { + // The flag is the only thing standing between a run and a written report, so the test has to + // check the file, not just the exit code. `--out` keeps that file in a temp directory: this test + // used to delete `observability-map.json` from the repo root and never put it back. + it("renders the whole report without writing when asked not to", () => { + const dir = mkdtempSync(join(tmpdir(), "obs-map-out-")); + const out = join(dir, "report.json"); + + const r = run("--out=" + out, "--no-write"); + + expect(r.code).toBe(0); + expect(r.out).toContain("COVERAGE"); + expect(r.out).toContain("FIX FIRST"); + expect(existsSync(out)).toBe(false); + + rmSync(dir, { recursive: true, force: true }); + }); + + it("writes the report where --out names it when not asked to skip the write", () => { + const dir = mkdtempSync(join(tmpdir(), "obs-map-out-")); + const out = join(dir, "report.json"); + + const r = run("--out=" + out); + + expect(r.code).toBe(0); + expect(existsSync(out)).toBe(true); + const parsed = JSON.parse(readFileSync(out, "utf8")); + expect(parsed.entries.length).toBe(Object.keys(FIXTURES).length + 1); + + rmSync(dir, { recursive: true, force: true }); + }); +}); + +// B6. Stdout can be JSON a caller parses, so the warning goes to stderr whenever stdout is JSON. +// The terminal report carries it in its body instead (`src/report/terminal.test.ts`), and printing +// it on both streams meant a plain run showed every warning twice. +describe("warning about a suppression that names no check", () => { + it("names the file and the bad id in the whole report", () => { + const r = run("--no-write"); + expect(r.code).toBe(0); + expect(r.out).toContain("api.v1.typo.ts"); + expect(r.out).toContain("eror-classification"); + }); + + it("prints the warning once for a terminal run of the whole report", () => { + const r = run("--no-write"); + const lines = `${r.out}${r.err}`.split("\n").filter((l) => l.startsWith("UNKNOWN SUPPRESSION")); + expect(lines).toHaveLength(1); + }); + + it("names the file and the bad id on stderr when the whole report is json", () => { + const r = run("--no-write", "--json"); + expect(r.code).toBe(0); + expect(r.err).toContain("api.v1.typo.ts"); + expect(r.err).toContain("eror-classification"); + expect(r.out).not.toContain("UNKNOWN SUPPRESSION"); + }); + + it("warns for a single route without putting the warning in the json", () => { + const r = run("/api/v1/typo", "--json"); + expect(r.code).toBe(0); + expect(r.err).toContain("eror-classification"); + expect(JSON.parse(r.out).fileName).toBe("api.v1.typo.ts"); + }); + + it("says nothing on stderr for a route whose directives all name a check", () => { + const r = run("/api/v1/token"); + expect(r.err).toBe(""); + }); +}); + +describe("map --routes=", () => { + it("scans the directory it names instead of the repo's routes tree", () => { + const dir = mkdtempSync(join(tmpdir(), "obs-map-routes-")); + writeFileSync( + join(dir, "resources.only.ts"), + `export const loader = () => new Response("ok");` + ); + + const c = capture(); + const code = main(["node", "cli.js", "--routes=" + dir, "--json", "--no-write"], c.io); + + expect(code).toBe(0); + const parsed = JSON.parse(c.out()); + expect(parsed.entries).toHaveLength(1); + expect(parsed.entries[0].fileName).toBe("resources.only.ts"); + + rmSync(dir, { recursive: true, force: true }); + }); + + it("exits 1 with a message when the directory does not exist", () => { + const dir = join(tmpdir(), "obs-map-routes-does-not-exist"); + const c = capture(); + const code = main(["node", "cli.js", "--routes=" + dir], c.io); + + expect(code).toBe(1); + expect(c.err()).toContain("not a readable directory"); + expect(c.out()).toBe(""); + }); +}); diff --git a/internal-packages/observability-map/src/cli.ts b/internal-packages/observability-map/src/cli.ts new file mode 100644 index 00000000000..4ec6be1ba8e --- /dev/null +++ b/internal-packages/observability-map/src/cli.ts @@ -0,0 +1,156 @@ +import { existsSync, statSync, writeFileSync } from "node:fs"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import type { EntryPoint } from "./types.js"; +import { scanDirectory } from "./scan.js"; +import { buildReport, scoreEntry } from "./score.js"; +import { routePathOf } from "./adapters/remix.js"; +import { + renderTerminal, + unknownSuppressionLine, + unknownSuppressionLines, +} from "./report/terminal.js"; +import { renderJson } from "./report/json.js"; + +const DEFAULT_ROUTES = "apps/webapp/app/routes"; + +/** + * Walks up from this file looking for `pnpm-workspace.yaml`, so the routes directory resolves + * correctly whether `map` is run from the repo root or from the package directory (where + * `pnpm --filter` puts you). Resolving `DEFAULT_ROUTES` against `process.cwd()` instead would only + * work from the repo root. + */ +function findRepoRoot(startDir: string): string { + let dir = startDir; + for (let i = 0; i < 10; i++) { + if (existsSync(resolve(dir, "pnpm-workspace.yaml"))) return dir; + const parent = dirname(dir); + if (parent === dir) break; + dir = parent; + } + throw new Error("could not find repo root (no pnpm-workspace.yaml in any parent directory)"); +} + +/** Where output goes. Injectable so the tests can read it without spawning a process. */ +export type Io = { out: (s: string) => void; err: (s: string) => void }; + +const processIo: Io = { + out: (s) => process.stdout.write(s), + err: (s) => process.stderr.write(s), +}; + +/** + * Entry points matching what the user typed, by file name or by route path, exact first. + * + * Route paths matter because they are what the report prints: `map /api/v1/token` used to exit 1 + * because only the file name was matched, so the identifier on screen was not one you could paste + * back in. + */ +function findMatches(entryPoints: EntryPoint[], target: string): EntryPoint[] { + const asPath = target.startsWith("/") ? target : `/${target}`; + const asFile = target.replace(/^\//, ""); + + const exact = entryPoints.filter( + (e) => e.fileName === target || routePathOf(e.fileName) === asPath + ); + if (exact.length > 0) return exact; + + return entryPoints.filter( + (e) => e.fileName.startsWith(asFile) || routePathOf(e.fileName).startsWith(asPath) + ); +} + +/** Value of a `--flag=value` argument, or null when the flag is absent. */ +function flagValue(args: string[], flag: string): string | null { + const prefix = `${flag}=`; + const found = args.find((a) => a.startsWith(prefix)); + return found === undefined ? null : found.slice(prefix.length); +} + +export function main(argv: string[], io: Io = processIo): number { + const args = argv.slice(2); + const asJson = args.includes("--json"); + const noWrite = args.includes("--no-write"); + const target = args.find((a) => !a.startsWith("--")); + + const repoRoot = findRepoRoot(dirname(fileURLToPath(import.meta.url))); + const routesFlag = flagValue(args, "--routes"); + + let routesDir: string; + if (routesFlag !== null) { + routesDir = resolve(process.cwd(), routesFlag); + let isDir = false; + try { + isDir = statSync(routesDir).isDirectory(); + } catch { + isDir = false; + } + if (!isDir) { + io.err(`--routes: not a readable directory: ${routesDir}\n`); + return 1; + } + } else { + routesDir = resolve(repoRoot, DEFAULT_ROUTES); + } + + const { entryPoints, parseFailures } = scanDirectory(routesDir); + + if (target) { + const matches = findMatches(entryPoints, target); + if (matches.length === 0) { + io.err(`no entry point matching "${target}"\n`); + return 1; + } + if (matches.length > 1) { + const others = matches.slice(1, 4).map((m) => routePathOf(m.fileName)); + const rest = matches.length - 1 - others.length; + io.err( + `"${target}" matches ${matches.length} entry points, showing the first. ` + + `Others: ${others.join(", ")}${rest > 0 ? `, and ${rest} more` : ""}\n` + ); + } + const scored = scoreEntry(matches[0]!); + // On stderr in both formats: a warning on stdout would be inside the JSON a caller parses. + if (scored.unknownSuppressions.length > 0) { + io.err(`${unknownSuppressionLine(scored.fileName, scored.unknownSuppressions)}\n`); + } + if (asJson) { + io.out(`${JSON.stringify(scored, null, 2)}\n`); + return 0; + } + const measuredNote = scored.measured ? "" : " (not measured: no applicable checks)"; + io.out( + `${scored.routePath} ${scored.score}/100${measuredNote}\n${scored.fileName}\n\nCHECKS\n` + ); + for (const c of scored.checks) { + const mark = c.status === "pass" ? "PASS" : c.status === "fail" ? "FAIL" : "n/a "; + io.out(` ${mark} ${c.id}${c.detail ? ` (${c.detail})` : ""}\n`); + } + return 0; + } + + const report = buildReport(entryPoints, parseFailures); + // JSON only. `renderTerminal` puts these lines in the report body, so warning here as well + // printed each one twice in a terminal run. Stderr is what the JSON path has instead, since a + // warning on stdout would be inside the document a caller parses. + if (asJson) for (const line of unknownSuppressionLines(report)) io.err(`${line}\n`); + io.out(asJson ? renderJson(report) : renderTerminal(report)); + io.out("\n"); + if (!noWrite) { + // `--out` exists so a test can point the write somewhere disposable. Without it the only way + // to exercise the write path was to let the tests create and delete a file in the repo root. + const outFlag = flagValue(args, "--out"); + const outPath = + outFlag === null + ? resolve(repoRoot, "observability-map.json") + : resolve(process.cwd(), outFlag); + writeFileSync(outPath, renderJson(report)); + } + return 0; +} + +// Only when run as a program. Importing the module, which the tests do, must not scan the tree or +// write a report. +const invokedDirectly = + process.argv[1] !== undefined && resolve(process.argv[1]) === fileURLToPath(import.meta.url); +if (invokedDirectly) process.exitCode = main(process.argv); diff --git a/internal-packages/observability-map/src/docstringReferences.test.ts b/internal-packages/observability-map/src/docstringReferences.test.ts new file mode 100644 index 00000000000..64666ac5b95 --- /dev/null +++ b/internal-packages/observability-map/src/docstringReferences.test.ts @@ -0,0 +1,206 @@ +import ts from "typescript"; +import { readFileSync, readdirSync } from "node:fs"; +import { join, resolve } from "node:path"; +import { CHECKS } from "./checks/index.js"; +import { MUTATIONS } from "./mutations.js"; + +/** + * Every test name a docstring in `src/` claims to be covered by must exist. + * + * The rule this enforces has been asked for six times in prose and broken six times, most recently + * by a docstring naming `content-is-not-a-comment`, a test that was never written. Prose cannot + * enforce itself, so this does. + * + * What is checked, precisely, because a checker that overstates its reach is the same defect again: + * + * - every backticked kebab-case token in a `src/` comment, e.g. `empty-instanceof-if`. Those are + * never valid JavaScript identifiers, so in this package they are always a check id, a mutation + * corpus id, a test name, or one of the handful of domain words in `NOT_A_TEST_NAME` below. + * - a backticked glob, `dead-*`, which must match at least one corpus id by prefix. + * - every backticked prose phrase of `MINIMUM_TITLE_WORDS` words or more that contains no code + * punctuation, e.g. `jsx text is content, not a comment`. That is what a test title looks like + * and what a code sample does not. + * + * What is NOT checked, and each of these is a place a bad reference can still hide: + * + * - a reference written without backticks. + * - a test title of fewer than `MINIMUM_TITLE_WORDS` words. `throw e` and `new URL` are code, and + * telling a short title from short code needs more than punctuation. + * - a comment with no node after it. `commentText` collects leading ranges only, so a comment on + * the last line of a block or at the end of a file is never scanned at all. Every docstring in + * this package precedes a declaration, which is why the collector was written that way, and it + * is a coverage hole rather than a design choice. + * - a `.test.ts` file, or `mutations.ts`. Tests live in `src/` for colocation, but a docstring in a + * test or in the mutation corpus helper is exempted from this scan by name, the same as it was + * when both lived outside `src/` in a separate `test/` directory. + * + * The kebab half is the half that has actually failed. + */ + +const SRC = resolve(__dirname); +const TESTS = resolve(__dirname); +/** Excluded from the `files` scan below: every `.test.ts` is a test rather than a source, and + * `mutations.ts` is the mutation-corpus helper, not production source. Both live in `src/` now for + * colocation, so the exclusion has to be by name rather than by directory. */ +const MUTATIONS_HELPER = resolve(SRC, "mutations.ts"); + +/** Kebab-case tokens that are domain vocabulary rather than a test or corpus name. Anything added + * here is a deliberate statement that the token names no test, and shows up in review as such. */ +const NOT_A_TEST_NAME = new Set([ + // A `CheckStatus` value. + "not-applicable", + // The directive spelling that was retired, named in `suppression.ts` to say it is not honoured. + "obs-map-disable-next-line", + // The worked example of a mistyped check id in `suppression.ts`. A misspelling of a check id is + // the thing being described, so it names no test by construction. + "eror-classification", + // A route path segment quoted in `sensitivity.ts`, part of the vocabulary that file is about. + "session-duration", +]); + +/** A backticked phrase this long or longer, with no code punctuation, is read as a test title. */ +const MINIMUM_TITLE_WORDS = 5; + +/** Characters that mean a backticked phrase is a code sample rather than a test title. */ +const CODE_PUNCTUATION = /[{}()[\];=<>"'`|&$/\\]|\.\.\.|\.tsx?\b/; + +function walkFiles(dir: string, suffix: string, out: string[] = []): string[] { + for (const entry of readdirSync(dir, { withFileTypes: true })) { + const path = join(dir, entry.name); + if (entry.isDirectory()) walkFiles(path, suffix, out); + else if (entry.name.endsWith(suffix)) out.push(path); + } + return out; +} + +/** Comment text with jsdoc line prefixes removed, so a backticked phrase that wrapped across two + * lines reads as one phrase rather than one with a stray asterisk in it. */ +function commentText(file: string): string { + const source = readFileSync(file, "utf8"); + const sf = ts.createSourceFile(file, source, ts.ScriptTarget.Latest, true); + const seen = new Set(); + const parts: string[] = []; + const visit = (node: ts.Node) => { + for (const range of ts.getLeadingCommentRanges(source, node.getFullStart()) ?? []) { + if (seen.has(range.pos)) continue; + seen.add(range.pos); + parts.push(source.slice(range.pos, range.end)); + } + ts.forEachChild(node, visit); + }; + visit(sf); + return parts.join("\n").replace(/\n\s*\*\s?/g, " "); +} + +/** Static titles from every `it`/`test`/`describe` call, including the literal chunks of a + * template-literal title, so a reference to part of a generated name still resolves. */ +function testTitles(): Set { + const titles = new Set(); + const add = (value: string) => { + const trimmed = value.replace(/\s+/g, " ").trim(); + if (trimmed.length > 0) titles.add(trimmed); + }; + for (const file of walkFiles(TESTS, ".test.ts")) { + const source = readFileSync(file, "utf8"); + const sf = ts.createSourceFile(file, source, ts.ScriptTarget.Latest, true); + const visit = (node: ts.Node) => { + if (ts.isCallExpression(node)) { + const callee = node.expression; + const root = ts.isPropertyAccessExpression(callee) ? callee.expression : callee; + if (ts.isIdentifier(root) && ["it", "test", "describe"].includes(root.text)) { + const first = node.arguments[0]; + if (first) { + if (ts.isStringLiteralLike(first)) add(first.text); + if (ts.isTemplateExpression(first)) { + add(first.head.text); + for (const span of first.templateSpans) add(span.literal.text); + } + } + } + } + ts.forEachChild(node, visit); + }; + visit(sf); + } + return titles; +} + +describe("docstrings in src name things that exist", () => { + const known = new Set([ + ...CHECKS.map((c) => c.id), + ...MUTATIONS.map((m) => m.id), + ...NOT_A_TEST_NAME, + ]); + const titles = testTitles(); + const corpusIds = MUTATIONS.map((m) => m.id); + const files = walkFiles(SRC, ".ts").filter( + (f) => !f.endsWith(".test.ts") && f !== MUTATIONS_HELPER + ); + + it("finds source files and test titles to check against", () => { + expect(files.length).toBeGreaterThan(5); + expect(titles.size).toBeGreaterThan(50); + expect(corpusIds.length).toBeGreaterThan(20); + }); + + it("every backticked kebab-case token names a check, a corpus entry or a test", () => { + const unknown: string[] = []; + for (const file of files) { + for (const match of commentText(file).matchAll(/`([a-z][a-z0-9]*(?:-[a-z0-9]+)+)`/g)) { + const token = match[1]!; + if (known.has(token)) continue; + if ([...titles].some((t) => t.includes(token))) continue; + unknown.push(`${file}: ${token}`); + } + } + expect(unknown).toEqual([]); + }); + + it("every backticked glob matches at least one corpus entry", () => { + const unmatched: string[] = []; + for (const file of files) { + for (const match of commentText(file).matchAll(/`([a-z][a-z0-9-]*)-\*`/g)) { + const prefix = `${match[1]!}-`; + if (corpusIds.some((id) => id.startsWith(prefix))) continue; + unmatched.push(`${file}: ${prefix}*`); + } + } + expect(unmatched).toEqual([]); + }); + + it("every backticked prose phrase long enough to be a test title is one", () => { + const unknown: string[] = []; + for (const file of files) { + for (const match of commentText(file).matchAll(/`([a-z][^`\n]*)`/g)) { + const phrase = match[1]!.replace(/\s+/g, " ").trim(); + if (phrase.split(" ").length < MINIMUM_TITLE_WORDS) continue; + if (CODE_PUNCTUATION.test(phrase)) continue; + if (titles.has(phrase)) continue; + unknown.push(`${file}: ${phrase}`); + } + } + expect(unknown).toEqual([]); + }); + + // The checker has to be able to fail, or it is decoration. These run the same predicates over an + // invented docstring rather than over `src/`, so the guarantee does not rest on `src/` currently + // happening to contain a bad reference. + it("would reject a docstring naming a test that does not exist", () => { + const invented = "see `content-is-not-a-comment` for the proof"; + const token = /`([a-z][a-z0-9]*(?:-[a-z0-9]+)+)`/.exec(invented)![1]!; + expect(known.has(token)).toBe(false); + expect([...titles].some((t) => t.includes(token))).toBe(false); + }); + + it("would reject a docstring naming a corpus glob that matches nothing", () => { + const prefix = /`([a-z][a-z0-9-]*)-\*`/.exec("covered by `no-such-family-*` above")![1]!; + expect(corpusIds.some((id) => id.startsWith(`${prefix}-`))).toBe(false); + }); + + it("would reject a docstring naming a prose test title that does not exist", () => { + const phrase = "reads a directive that nobody ever wrote down anywhere"; + expect(phrase.split(" ").length).toBeGreaterThanOrEqual(MINIMUM_TITLE_WORDS); + expect(CODE_PUNCTUATION.test(phrase)).toBe(false); + expect(titles.has(phrase)).toBe(false); + }); +}); diff --git a/internal-packages/observability-map/src/index.ts b/internal-packages/observability-map/src/index.ts new file mode 100644 index 00000000000..1a8e9904931 --- /dev/null +++ b/internal-packages/observability-map/src/index.ts @@ -0,0 +1,6 @@ +export { scanDirectory, scanFile } from "./scan.js"; +export { buildReport, scoreEntry } from "./score.js"; +export { renderTerminal } from "./report/terminal.js"; +export { renderJson } from "./report/json.js"; +export type { MapReport, ScoredEntry } from "./score.js"; +export type { EntryPoint, CheckResult, CheckStatus } from "./types.js"; diff --git a/internal-packages/observability-map/src/integration.test.ts b/internal-packages/observability-map/src/integration.test.ts new file mode 100644 index 00000000000..99b6f5ed851 --- /dev/null +++ b/internal-packages/observability-map/src/integration.test.ts @@ -0,0 +1,426 @@ +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { isScannableFile, scanDirectory, scanFile } from "./scan.js"; +import { buildReport } from "./score.js"; +import { SCORED_CHECK_IDS } from "./checks/index.js"; + +/** + * This file's deliberate coupling to `apps/webapp/app/routes`. It is not the suite's only one, and + * saying it was is what let the paths filter be written for this file alone: + * `webappSymbols.test.ts` walks all of `apps/webapp/app`, `packages/plugins/src` and + * `internal-packages/rbac/src`, and `mutationCorpus.test.ts` scans the route tree behind an env + * gate. Everything else, including the CLI tests, runs against a fixture tree of this package's own + * making. + * + * The coupling is acceptable because nothing here names a route or a count: the scan must not + * crash, the entry point count must sit inside a wide band, and parse failures must be zero. Those + * survive routes being added, renamed and deleted, and they are the only things a fixture tree + * cannot tell us, since a fixture only contains shapes somebody thought to write down. + * + * What runs this for a webapp pull request is `.github/workflows/unit-tests-observability-map.yml`, + * called from `pr_checks.yml` behind an `obsmap` paths filter covering the whole of + * `apps/webapp/app` plus the report workflow, and listed in the `all-checks` aggregate so it + * actually gates. The filter is wider than this file's own coupling because the suite's is: + * `webappSymbols.test.ts` walks all of `apps/webapp/app`, and the describes below read + * `observability-map.yml`. A pull request touching this PACKAGE, or `packages/plugins/src` or + * `internal-packages/rbac/src`, reaches the same test by the other road: `internal` already matches + * `internal-packages/**` and `packages/**`, and `unit-tests-internal.yml` runs `turbo run test + * --filter "@internal/*"` over this package too. So every direction is gated and none is gated + * twice; the `obsmap` filter used to name the package as well, which ran this suite twice on every + * PR touching it. + * + * Two shapes were tried and rejected on the way here. Widening `pr_checks.yml`'s `internal` filter + * to the route paths ran all eighteen internal packages, twelve shards with postgres, clickhouse, + * redis and electric, to protect this one test. Putting the job in `observability-map.yml` + * instead was targeted but gated nothing, because `all-checks` needs an explicit list of jobs and + * cannot see another workflow. + */ +const ROUTES = resolve(__dirname, "../../../apps/webapp/app/routes"); + +/** + * Every `.ts`/`.tsx` file under the tree, at any depth. Deliberately not the scanner's walk, which + * looks at flat files and at one `route.ts`/`route.tsx` per directory: this used to be a verbatim + * copy of that walk, which made `entryPoints.length < countCandidates()` a tautology that could + * not fail for any route shape both of them missed. + */ +function countRouteModuleFiles(dir: string): number { + let count = 0; + for (const entry of readdirSync(dir, { withFileTypes: true })) { + if (entry.isDirectory()) { + count += countRouteModuleFiles(join(dir, entry.name)); + continue; + } + if (entry.isFile() && isScannableFile(entry.name)) count++; + } + return count; +} + +beforeAll(() => { + // A hard failure rather than the `if (!existsSync(ROUTES)) return;` these tests opened with: if + // this package moves relative to apps/webapp, the real-tree coverage must disappear loudly. + if (!existsSync(ROUTES)) { + throw new Error(`the webapp routes directory is missing: ${ROUTES}`); + } +}); + +// B7. The build emits ESM (`module: ESNext`, and `cli.ts` uses `import.meta`) while `main` and +// `types` advertised it to a package.json with no module type. On node 24 that loads with a +// MODULE_TYPELESS_PACKAGE_JSON warning and a reparse rather than the throw older nodes give, which +// is a warning about an artifact this package tells other packages to import. +describe("the package it advertises", () => { + const manifest = JSON.parse( + readFileSync(resolve(__dirname, "../package.json"), "utf8") + ) as Record; + + // Asserted flat rather than behind an `if (!advertised) return`, which is the silent skip this + // round removed from the tests below: the decision was to keep the entry point and declare the + // module type, so dropping the entry point later should have to edit this, not slip past it. + it("declares the module type its build emits alongside the entry point it advertises", () => { + expect(manifest.main).toBe("./dist/src/index.js"); + expect(manifest.types).toBe("./dist/src/index.d.ts"); + expect(manifest.type).toBe("module"); + }); +}); + +/** + * The one thing the docstring checker cannot reach. It walks `src/` only, so workflow prose is + * unpoliced, and the C1 defect was exactly that: two steps disagreeing about what a missing + * `/tmp/existing-comment-id` meant, under a comment claiming they agreed. The render step read it + * as "a comment exists" and emitted the resolved state, the upsert step read it as "no id" and + * POSTed, so a transient lookup failure either added a second marker comment beside the stale one + * or announced that findings were gone on a pull request that never had any. + * + * This is a text check over the workflow, not a parse of its semantics, so it catches one shape of + * that class and no other. Named as such rather than sold as coverage of the file. + */ +describe("the report workflow's two readers of the comment lookup", () => { + const WORKFLOW = resolve(__dirname, "../../../.github/workflows/observability-map.yml"); + + /** Step bodies, split on the `- name:` lines, which is all the structure this needs. */ + function steps(): string[] { + if (!existsSync(WORKFLOW)) throw new Error(`the report workflow is missing: ${WORKFLOW}`); + const text = readFileSync(WORKFLOW, "utf8"); + return text.split(/^ {6}- name: /m).slice(1); + } + + it("both honour the same sentinel, so a failed lookup cannot mean two things", () => { + const readers = steps().filter((step) => step.includes("/tmp/existing-comment-id")); + expect(readers.length).toBeGreaterThanOrEqual(2); + expect(readers.filter((step) => !step.includes("/tmp/comment-lookup-failed"))).toEqual([]); + }); + + it("takes one id from a lookup that paginates rather than passing every line on", () => { + const lookup = steps().find((step) => step.includes('startswith(""); + expect(renderPrComment(head, head).split("\n")[0]).toBe(""); + }); + + it("says the comparison is unavailable when base is null", () => { + const head = buildReport([scanFile("api.v1.a.ts", brokenSource)!], []); + const out = renderPrComment(head, null); + expect(out).toContain("Base comparison unavailable."); + expect(out).not.toContain("no change"); + }); + + it("reports a score drop and the newly failing checks", () => { + const head = buildReport([scanFile("api.v1.auth.tokens.ts", brokenSource)!], []); + const base = buildReport([scanFile("api.v1.auth.tokens.ts", cleanSource)!], []); + const out = renderPrComment(head, base); + + expect(out).toMatch(/\(base \d+, down \d+\)/); + expect(out).toContain("| /api/v1/auth/tokens |"); + // request-context and error-classification regress; auth-boundary is not applicable here + // (no sensitivity signal on this route), so it must not show up as newly failing. + expect(out).toMatch(/\| \/api\/v1\/auth\/tokens \| \d+ \| \d+ \|[^|]*error-classification/); + }); + + it("reports a score improvement the other way round", () => { + const head = buildReport([scanFile("api.v1.auth.tokens.ts", cleanSource)!], []); + const base = buildReport([scanFile("api.v1.auth.tokens.ts", brokenSource)!], []); + const out = renderPrComment(head, base); + expect(out).toMatch(/\(base \d+, up \d+\)/); + }); + + it("says nothing changed when every score matches", () => { + const head = buildReport([scanFile("api.v1.auth.tokens.ts", cleanSource)!], []); + const base = buildReport([scanFile("api.v1.auth.tokens.ts", cleanSource)!], []); + const out = renderPrComment(head, base); + expect(out).toContain("No entry point this PR touches changed its score."); + expect(out).toContain("(base 100, no change)"); + }); + + it("shows a new entry with its base column as 'new' and lists its failing checks", () => { + const head = buildReport( + [scanFile("api.v1.auth.tokens.ts", cleanSource)!, scanFile("api.v1.new.ts", brokenSource)!], + [] + ); + const base = buildReport([scanFile("api.v1.auth.tokens.ts", cleanSource)!], []); + const out = renderPrComment(head, base); + + expect(out).toMatch(/\| \/api\/v1\/new \| new \| \d+ \|/); + }); + + // Mirrors the guard report.test.ts has for the terminal renderer: audit-trail fails almost + // every sensitive mutation today (no audit helper exists), so it is a headline figure, not a + // per-route nag. A regression here previously let it leak into the "now failing" column. + it("does not list audit-trail among a new sensitive entry's failing checks", () => { + const sensitiveMutation = scanFile( + "api.v1.envvars.ts", + `import { prisma } from "~/db.server"; + export async function action() { + try { + return await prisma.envVar.update({ where: {}, data: {} }); + } catch (e) { + return null; + } + }` + )!; + const head = buildReport([sensitiveMutation], []); + const base = buildReport([], []); + const out = renderPrComment(head, base); + + const row = out.split("\n").find((l) => l.startsWith("| /api/v1/envvars |"))!; + expect(row).toBeDefined(); + expect(row).toContain("new"); + expect(row).not.toContain("audit-trail"); + expect(row).toMatch(/error-classification|auth-boundary|request-context/); + }); + + it("skips a new entry that passes every check it was measured against", () => { + const head = buildReport( + [scanFile("api.v1.auth.tokens.ts", cleanSource)!, scanFile("api.v1.new.ts", cleanSource)!], + [] + ); + const base = buildReport([scanFile("api.v1.auth.tokens.ts", cleanSource)!], []); + const out = renderPrComment(head, base); + + expect(out).not.toContain("/api/v1/new"); + expect(out).toContain("No entry point this PR touches changed its score."); + }); + + // B3. `score` is 100 for an entry no scored check applied to, and the table read that placeholder + // as a figure: a route refactored down to a trivial body rendered as a 67-point improvement. + describe("an unmeasured entry", () => { + const trivial = `export const loader = () => new Response("ok");`; + + it("renders as not measured rather than as 100 when the head stopped being measurable", () => { + const head = buildReport([scanFile("api.v1.auth.tokens.ts", trivial)!], []); + const base = buildReport([scanFile("api.v1.auth.tokens.ts", brokenSource)!], []); + const out = renderPrComment(head, base); + + const row = out.split("\n").find((l) => l.startsWith("| /api/v1/auth/tokens |"))!; + expect(row).toBeDefined(); + expect(row).toContain("not measured"); + expect(row).not.toMatch(/\|\s*100\s*\|/); + }); + + it("renders as not measured in the base column when the head gained real work", () => { + const head = buildReport([scanFile("api.v1.auth.tokens.ts", brokenSource)!], []); + const base = buildReport([scanFile("api.v1.auth.tokens.ts", trivial)!], []); + const out = renderPrComment(head, base); + + const row = out.split("\n").find((l) => l.startsWith("| /api/v1/auth/tokens |"))!; + expect(row).toBeDefined(); + expect(row).toMatch(/\| not measured \| \d+ \|/); + }); + + // The early-out compared scores only, so a measured 100 turning into an unmeasured placeholder + // 100 produced no row at all: the table said nothing happened. + it("still produces a row when a measured 100 becomes an unmeasured placeholder 100", () => { + const head = buildReport([scanFile("api.v1.auth.tokens.ts", trivial)!], []); + const base = buildReport([scanFile("api.v1.auth.tokens.ts", cleanSource)!], []); + expect(base.entries[0]!.score).toBe(100); + expect(head.entries[0]!.score).toBe(100); + + const out = renderPrComment(head, base); + const row = out.split("\n").find((l) => l.startsWith("| /api/v1/auth/tokens |"))!; + expect(row).toBeDefined(); + expect(row).toMatch(/\| 100 \| not measured \|/); + }); + + // Not the same statement as a new entry that passes everything, which is skipped above. + it("still gets a row when it is new, since its 100 is a placeholder and not a pass", () => { + const head = buildReport( + [ + scanFile("api.v1.auth.tokens.ts", cleanSource)!, + scanFile("resources.health.ts", trivial)!, + ], + [] + ); + const base = buildReport([scanFile("api.v1.auth.tokens.ts", cleanSource)!], []); + const out = renderPrComment(head, base); + + const row = out.split("\n").find((l) => l.startsWith("| /resources/health |"))!; + expect(row).toBeDefined(); + expect(row).toMatch(/\| new \| not measured \|/); + }); + + it("does not sort an unmeasured transition above a real regression", () => { + const head = buildReport( + [scanFile("resources.gone.ts", trivial)!, scanFile("resources.busy.ts", brokenSource)!], + [] + ); + const base = buildReport( + [scanFile("resources.gone.ts", brokenSource)!, scanFile("resources.busy.ts", cleanSource)!], + [] + ); + const out = renderPrComment(head, base); + + const busy = out.indexOf("/resources/busy"); + const gone = out.indexOf("/resources/gone"); + expect(busy).toBeGreaterThan(-1); + expect(gone).toBeGreaterThan(-1); + expect(busy).toBeLessThan(gone); + }); + }); + + // I4. A suppression added to a check that was PASSING drops the score by round A's cap and + // produces a row with an empty "now failing" column, sorted among the real regressions. On the + // real tree `_app.@.orgs.$organizationSlug.$.tsx` renders 67 to 50 exactly that way. + describe("a row a suppression caused", () => { + // Two of three applicable scored checks pass, so suppressing one of the passes takes the + // visible ratio from 2/3 to 1/2, which is the 67 to 50 the real tree renders on + // `_app.@.orgs.$organizationSlug.$.tsx`. The catch has to decide something for + // error-classification to apply at all, and nothing may name a tenant, or the ratio is 3/3. + const twoOfThree = `import { requireUserId } from "~/services/session.server"; + import { prisma } from "~/db.server"; + export async function action({ request }) { + const userId = await requireUserId(request); + try { return await prisma.token.create({ data: { userId } }); } + catch (error) { + if (error instanceof BadRequest) return json({ error: "bad" }, { status: 400 }); + throw error; + } + }`; + const silence = (id: string, source: string) => + `// obs-map-disable ${id} -- silenced\n${source}`; + + it("says so on the route, so it is not read as a regression", () => { + const base = buildReport([scanFile("api.v1.auth.tokens.ts", twoOfThree)!], []); + const head = buildReport( + [scanFile("api.v1.auth.tokens.ts", silence("error-classification", twoOfThree))!], + [] + ); + expect(base.entries[0]!.score).toBe(67); + expect(head.entries[0]!.score).toBe(50); + + const row = renderPrComment(head, base) + .split("\n") + .find((l) => l.startsWith("| /api/v1/auth/tokens"))!; + expect(row).toBeDefined(); + expect(row).toContain("(suppressed: error-classification)"); + // The column that would otherwise explain the drop is empty, which is the whole problem. + expect(row.split("|")[4]!.trim()).toBe(""); + }); + + // I3. This one moves no score at all, so before the suppression set was compared it produced + // no row and no comment: a pull request whose whole purpose is to silence findings was silent. + it("appears even when suppressing an already-failing check moved no score", () => { + const base = buildReport([scanFile("api.v1.t.ts", brokenSource)!], []); + const head = buildReport( + [scanFile("api.v1.t.ts", silence("error-classification", brokenSource))!], + [] + ); + expect(head.entries[0]!.score).toBe(base.entries[0]!.score); + expect(head.global).toBe(base.global); + + const out = renderPrComment(head, base); + expect(out).not.toContain("No entry point this PR touches changed its score."); + const row = out.split("\n").find((l) => l.startsWith("| /api/v1/t "))!; + expect(row).toBeDefined(); + expect(row).toContain("(suppressed: error-classification)"); + }); + + it("says nothing about suppression on a row that has none", () => { + const head = buildReport([scanFile("api.v1.auth.tokens.ts", brokenSource)!], []); + const base = buildReport([scanFile("api.v1.auth.tokens.ts", cleanSource)!], []); + const row = renderPrComment(head, base) + .split("\n") + .find((l) => l.startsWith("| /api/v1/auth/tokens"))!; + expect(row).not.toContain("suppressed:"); + }); + + it("gives a new entry that lands at 100 only because of a suppression a row", () => { + const base = buildReport([scanFile("api.v1.a.ts", cleanSource)!], []); + const head = buildReport( + [ + scanFile("api.v1.a.ts", cleanSource)!, + scanFile("api.v1.new.ts", silence("request-context", cleanSource))!, + ], + [] + ); + const row = renderPrComment(head, base) + .split("\n") + .find((l) => l.startsWith("| /api/v1/new"))!; + expect(row).toBeDefined(); + expect(row).toContain("(suppressed: request-context)"); + }); + }); + + // M5. This section rendered one line per file, and a tree-wide typo took the comment past + // GitHub's 65,536 character limit for a 422 nobody sees. + it("caps the unknown-suppression lines instead of running past the comment size limit", () => { + const entries = []; + for (let i = 0; i < 40; i++) { + entries.push( + scanFile( + `api.v1.route${i}.ts`, + `// obs-map-disable eror-classification -- typo\n${brokenSource}` + )! + ); + } + const head = buildReport(entries, []); + const out = renderPrComment(head, null); + + expect(out.split("\n").filter((l) => l.startsWith("UNKNOWN SUPPRESSION"))).toHaveLength(10); + expect(out).toContain("and 30 more files with unknown ids"); + expect(out.length).toBeLessThan(65536); + }); + + // Round E item 1. The same unbounded-section failure, in the one other section that grows with + // the tree. A codemod moving route bodies into `.server.ts` modules is the refactor `delegating` + // exists to notice, and it is what makes this list tree-sized. + it("caps the delegated route list instead of running past the comment size limit", () => { + const entries = []; + for (let i = 0; i < 400; i++) { + const padding = `route-with-a-realistically-long-name-${String(i).padStart(4, "0")}`; + entries.push( + scanFile( + `_app.orgs.$organizationSlug.projects.$projectParam.${padding}/route.tsx`, + `export { action } from "./handler.server";` + )! + ); + } + const head = buildReport(entries, []); + const line = renderPrComment(head, null) + .split("\n") + .find((l) => l.startsWith("DELEGATED"))!; + + expect(line).toContain("400 routes"); + expect(line).toContain(", and 385 more"); + expect(line.match(/\/route\.tsx/g)).toHaveLength(15); + expect(renderPrComment(head, null).length).toBeLessThan(65536); + }); + + // The terminal has no size limit to respect, so the cap must not reach it. + it("leaves the terminal report naming every delegating route", () => { + const entries = []; + for (let i = 0; i < 40; i++) { + entries.push(scanFile(`webhooks.v1.hook${i}.ts`, `export { action } from "./h.server";`)!); + } + const line = renderTerminal(buildReport(entries, [])) + .split("\n") + .find((l) => l.startsWith("DELEGATED"))!; + expect(line).toContain("webhooks.v1.hook39.ts"); + expect(line).not.toContain("more"); + }); + + it("sorts a sensitive entry with a small drop above a non-sensitive entry with a large drop", () => { + const sensitiveSmallDropBase = scanFile("api.v1.auth.tokens.ts", cleanSource)!; + const sensitiveSmallDropHead = scanFile( + "api.v1.auth.tokens.ts", + `import { requireUserId } from "~/services/session.server"; + import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function action({ request }) { + const userId = await requireUserId(request); + try { return await prisma.token.create({ data: { userId } }); } + catch (error) { logger.error("token create failed", { error }); throw error; } + }` + )!; + + const notSensitiveLargeDropBase = scanFile( + "resources.busy.ts", + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function loader({ params }) { + try { return await prisma.thing.findMany(); } + catch (error) { logger.error("failed", { environmentId: params.envId, error }); throw error; } + }` + )!; + const notSensitiveLargeDropHead = scanFile( + "resources.busy.ts", + `import { prisma } from "~/db.server"; + export async function loader() { + try { return await prisma.thing.findMany(); } catch (e) { return null; } + }` + )!; + + const head = buildReport([sensitiveSmallDropHead, notSensitiveLargeDropHead], []); + const base = buildReport([sensitiveSmallDropBase, notSensitiveLargeDropBase], []); + const out = renderPrComment(head, base); + + const sensitiveIndex = out.indexOf("/api/v1/auth/tokens"); + const notSensitiveIndex = out.indexOf("/resources/busy"); + expect(sensitiveIndex).toBeGreaterThan(-1); + expect(notSensitiveIndex).toBeGreaterThan(-1); + expect(sensitiveIndex).toBeLessThan(notSensitiveIndex); + }); + + it("reports a removed entry as a count line, not a row", () => { + const head = buildReport([scanFile("api.v1.auth.tokens.ts", cleanSource)!], []); + const base = buildReport( + [scanFile("api.v1.auth.tokens.ts", cleanSource)!, scanFile("api.v1.gone.ts", brokenSource)!], + [] + ); + const out = renderPrComment(head, base); + + expect(out).toContain("1 entries removed"); + expect(out).not.toContain("/api/v1/gone"); + }); + + it("caps the changed-entries table at 15 rows and says how many more", () => { + const headEntries = []; + const baseEntries = []; + for (let i = 0; i < 20; i++) { + headEntries.push(scanFile(`api.v1.route${i}.ts`, brokenSource)!); + baseEntries.push(scanFile(`api.v1.route${i}.ts`, cleanSource)!); + } + const head = buildReport(headEntries, []); + const base = buildReport(baseEntries, []); + const out = renderPrComment(head, base); + + const rows = out.split("\n").filter((l) => l.startsWith("| /api/v1/route")); + expect(rows).toHaveLength(15); + expect(out).toContain("and 5 more"); + }); + + it("warns about parse failures in either report, since they shrink the denominator", () => { + const head = buildReport([scanFile("api.v1.a.ts", cleanSource)!], ["broken-head.ts"]); + const base = buildReport([scanFile("api.v1.a.ts", cleanSource)!], ["broken-base.ts"]); + const out = renderPrComment(head, base); + expect(out).toMatch(/Warning: parse failures \(1 at head, 1 at base\)/); + }); + + it("does not warn about parse failures when there are none", () => { + const head = buildReport([scanFile("api.v1.a.ts", cleanSource)!], []); + expect(renderPrComment(head, null)).not.toContain("Warning: parse failures"); + }); + + it("footer names the report-only rule and the readme", () => { + const head = buildReport([scanFile("api.v1.a.ts", cleanSource)!], []); + const out = renderPrComment(head, null); + expect(out).toContain("Report only, nothing here gates the merge."); + expect(out).toContain("internal-packages/observability-map/README.md"); + }); +}); + +// B4. The job posts only when the pull request moves the report, so the decision has to be a +// tested function of the two reports rather than shell logic in the workflow. +describe("hasDelta", () => { + const trivial = `export const loader = () => new Response("ok");`; + const one = (name: string, source: string) => buildReport([scanFile(name, source)!], []); + + it("is true when there is no base to compare against", () => { + expect(hasDelta(one("api.v1.a.ts", cleanSource), null)).toBe(true); + }); + + it("is false for two identical reports", () => { + expect(hasDelta(one("api.v1.a.ts", cleanSource), one("api.v1.a.ts", cleanSource))).toBe(false); + }); + + it("is true when the global score moved", () => { + expect(hasDelta(one("api.v1.a.ts", brokenSource), one("api.v1.a.ts", cleanSource))).toBe(true); + }); + + it("is true when an entry was added", () => { + const head = buildReport( + [scanFile("api.v1.a.ts", cleanSource)!, scanFile("api.v1.b.ts", cleanSource)!], + [] + ); + expect(hasDelta(head, one("api.v1.a.ts", cleanSource))).toBe(true); + }); + + it("is true when an entry was removed", () => { + const base = buildReport( + [scanFile("api.v1.a.ts", cleanSource)!, scanFile("api.v1.b.ts", cleanSource)!], + [] + ); + expect(hasDelta(one("api.v1.a.ts", cleanSource), base)).toBe(true); + }); + + // The global is a mean over measured entries, so two entries moving in opposite directions can + // leave it where it was. The per-entry comparison is what catches that. + it("is true when an entry's score moved but the global mean did not", () => { + const head = buildReport( + [scanFile("api.v1.a.ts", brokenSource)!, scanFile("api.v1.b.ts", cleanSource)!], + [] + ); + const base = buildReport( + [scanFile("api.v1.a.ts", cleanSource)!, scanFile("api.v1.b.ts", brokenSource)!], + [] + ); + expect(head.global).toBe(base.global); + expect(hasDelta(head, base)).toBe(true); + }); + + // audit-trail does not feed the score, so it can start failing without moving a single figure. + it("is true when an unscored check started failing and no score moved", () => { + const head = one("api.v1.envvars.ts", cleanSource); + const base = one( + "api.v1.envvars.ts", + `// obs-map-disable audit-trail -- no helper exists yet\n${cleanSource}` + ); + expect(head.global).toBe(base.global); + expect(hasDelta(head, base)).toBe(true); + }); + + it("is true when an entry stopped being measured at the same placeholder score", () => { + const head = one("api.v1.a.ts", trivial); + const base = one("api.v1.a.ts", cleanSource); + expect(head.entries[0]!.score).toBe(base.entries[0]!.score); + expect(hasDelta(head, base)).toBe(true); + }); + + it("is true when a parse failure appeared, since the comment warns about it", () => { + const head = buildReport([scanFile("api.v1.a.ts", cleanSource)!], ["broken.ts"]); + expect(hasDelta(head, one("api.v1.a.ts", cleanSource))).toBe(true); + }); + + // I3. These three are the half that was missing, and it ran the dangerous way: a pull request + // that only silences findings posted nothing, while a mistyped directive did post. + it("is true when a suppression was added to a check that was already failing", () => { + const base = one("api.v1.t.ts", brokenSource); + const head = one( + "api.v1.t.ts", + `// obs-map-disable error-classification -- silenced\n${brokenSource}` + ); + expect(head.global).toBe(base.global); + expect(head.entries[0]!.score).toBe(base.entries[0]!.score); + expect(head.measured).toBe(base.measured); + expect(hasDelta(head, base)).toBe(true); + }); + + it("is true when the audit gap closed, which no score reports", () => { + const audited = `import { clearImpersonation } from "~/models/admin.server"; + import { prisma } from "~/db.server"; + export async function action() { + const token = await prisma.token.create({ data: {} }); + await clearImpersonation(request, "/admin"); + return json(token); + }`; + const unaudited = `import { prisma } from "~/db.server"; + export async function action() { + const token = await prisma.token.create({ data: {} }); + return json(token); + }`; + const head = one("api.v1.auth.tokens.ts", audited); + const base = one("api.v1.auth.tokens.ts", unaudited); + expect(head.auditGap).not.toEqual(base.auditGap); + expect(hasDelta(head, base)).toBe(true); + }); + + // The CONTEXT line reads pre-suppression data, so with request-context suppressed its figure can + // move while the post-suppression checks, the score and the global all stay put. The comment + // says "0 of 1" and then "1 of 1"; nothing else in the report moves at all. + it("is true when the context figure moved behind a suppression", () => { + const silence = "// obs-map-disable request-context -- reported as a figure\n"; + const namesNobody = `${silence}import { prisma } from "~/db.server"; + export async function action() { + try { return await prisma.envVar.update({ where: {}, data: {} }); } catch (e) { return null; } + }`; + const namesTenant = `${silence}import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function action({ params }) { + try { return await prisma.envVar.update({ where: {}, data: {} }); } + catch (error) { logger.error("failed", { environmentId: params.envId, error }); return null; } + }`; + const head = one("api.v1.envvars.ts", namesTenant); + const base = one("api.v1.envvars.ts", namesNobody); + + expect(head.global).toBe(base.global); + expect(head.entries[0]!.score).toBe(base.entries[0]!.score); + expect(head.entries[0]!.suppressed).toEqual(base.entries[0]!.suppressed); + expect(head.contextGap).not.toEqual(base.contextGap); + expect(hasDelta(head, base)).toBe(true); + }); + + // Suppressing a check that was not applicable anyway changes no status and no score, and moving + // that suppression between two entries leaves the report-level totals identical too. The + // per-entry suppression comparison is the only thing left that can see it. + it("is true when a suppression of an inapplicable check moved between entries", () => { + const silenced = `// obs-map-disable auth-boundary -- not a user-facing route\n${brokenSource}`; + const head = buildReport( + [scanFile("api.v1.a.ts", silenced)!, scanFile("api.v1.b.ts", brokenSource)!], + [] + ); + const base = buildReport( + [scanFile("api.v1.a.ts", brokenSource)!, scanFile("api.v1.b.ts", silenced)!], + [] + ); + expect(head.suppressions).toEqual(base.suppressions); + expect(head.global).toBe(base.global); + expect(head.entries.map((e) => e.score)).toEqual(base.entries.map((e) => e.score)); + const statuses = (r: typeof head) => + r.entries.map((e) => e.checks.map((c) => `${c.id}=${c.status}`).join(" ")); + expect(statuses(head)).toEqual(statuses(base)); + expect(hasDelta(head, base)).toBe(true); + }); + + it("is true when a suppression names a check that does not exist", () => { + const head = one( + "api.v1.a.ts", + `// obs-map-disable eror-classification -- typo\n${cleanSource}` + ); + expect(hasDelta(head, one("api.v1.a.ts", cleanSource))).toBe(true); + }); +}); + +// C4b and C5. Both are rendered, so both have to move `hasDelta`, and neither is reachable through +// the per-entry score comparison the loop makes. +describe("hasDelta: the figures round C added", () => { + const one = (name: string, source: string) => buildReport([scanFile(name, source)!], []); + const delegated = `export { action } from "./handler.server";`; + + it("is true when a route started delegating its body", () => { + const head = one("webhooks.v1.stripe.ts", delegated); + const base = one("webhooks.v1.stripe.ts", brokenSource); + expect(head.delegating).toEqual(["webhooks.v1.stripe.ts"]); + expect(hasDelta(head, base)).toBe(true); + }); + + // The shape the per-entry loop cannot see: the score stays where it was and what the score is + // made of changed underneath it. + it("is true when a check stopped applying without moving any entry's score", () => { + const head = one("api.v1.auth.tokens.ts", cleanSource); + const base = one("api.v1.auth.tokens.ts", cleanSource); + const contribution = base.checkContributions.find((c) => c.id === "auth-boundary")!; + contribution.applicable = contribution.applicable + 1; + expect(head.entries[0]!.score).toBe(base.entries[0]!.score); + expect(hasDelta(head, base)).toBe(true); + }); + + it("says the comment renders the delegated line it is comparing", () => { + const head = one("webhooks.v1.stripe.ts", delegated); + expect(renderPrComment(head, null)).toContain("DELEGATED"); + }); + + it("says the comment renders the check contributions it is comparing", () => { + const head = one("api.v1.auth.tokens.ts", cleanSource); + expect(renderPrComment(head, null)).toContain("What the score is made of"); + }); +}); diff --git a/internal-packages/observability-map/src/report/prComment.ts b/internal-packages/observability-map/src/report/prComment.ts new file mode 100644 index 00000000000..3a5d5842aee --- /dev/null +++ b/internal-packages/observability-map/src/report/prComment.ts @@ -0,0 +1,355 @@ +import type { MapReport, ScoredEntry } from "../score.js"; +import { + auditLine, + checkContributionLines, + contextLine, + fixFirst, + delegatedLines, + scoredFailures, + unknownSuppressionLines, +} from "./terminal.js"; + +/** First line of every comment this job posts, so the upsert step can find its own comment again. */ +export const MARKER = ""; + +const MAX_CHANGED_ROWS = 15; + +/** + * A mistyped directive applied tree wide renders one line per file: 87,938 characters against + * GitHub's 65,536 limit, a 422, and the workflow's error tolerance swallowing it. The cap is what + * stops the whole comment being lost to the section warning about a typo. + */ +const MAX_UNKNOWN_SUPPRESSION_LINES = 10; + +/** + * The same failure in the other section that grows with the size of the tree. `delegating` holds + * one file name per route whose body lives elsewhere, joined into a single line, and a codemod that + * moves route bodies into `.server.ts` modules is both the refactor this feature exists to notice + * and the one that makes the list tree-sized. The cap was claimed here before it was written: the + * note above used to open "every other section of this comment is bounded by construction", which + * was not true of this one. + * + * Fifteen matches the changed-entries table rather than the ten above, because a delegating file + * name is one comma-separated item rather than a line naming every known check. The bound that + * matters is the section's worst case: the longest route file name in the tree is 130 characters, + * so fifteen of those plus separators is under 2kB against GitHub's 65,536. + */ +const MAX_DELEGATED_ROUTES = 15; + +// Scored checks only, same exclusion terminal.ts's scoredFailures makes: audit-trail fails almost +// every sensitive mutation today, so listing it per route would nag with something unfixable +// instead of surfacing the route-specific gaps this column exists for. +const failingIds = (e: ScoredEntry) => scoredFailures(e).map((c) => c.id); + +function scoreLine(head: MapReport, base: MapReport | null): string { + const headline = + head.global === null + ? `not measured over ${head.measured} measured of ${head.entries.length} entry points` + : `**${head.global}/100** over ${head.measured} measured of ${head.entries.length} entry points`; + + if (!base) return headline; + if (base.global === null || head.global === null) return `${headline} (base not measured)`; + + const diff = head.global - base.global; + const comparison = + diff === 0 + ? `(base ${base.global}, no change)` + : diff > 0 + ? `(base ${base.global}, up ${diff})` + : `(base ${base.global}, down ${-diff})`; + return `${headline} ${comparison}`; +} + +/** What goes in a score column: a figure, an absence, or no prior entry to compare against. */ +const NOT_MEASURED = "not measured"; + +/** + * `score` is 100 for an entry no scored check applied to, a placeholder the score itself excludes + * from every mean. Rendering that 100 as a figure turned a route refactored down to a trivial body + * into a 67-point improvement, and a trivial route gaining real work into the PR's worst + * regression. So the cell says what the terminal gauge says for a null mean instead. + */ +const scoreCell = (e: ScoredEntry): number | string => (e.measured ? e.score : NOT_MEASURED); + +/** Ids suppressed at head that were not suppressed at base. `[]` covers both "no change" and a + * suppression being removed, which shows up as the score going back up. */ +const newlySuppressed = (head: ScoredEntry, base: ScoredEntry | undefined): string[] => + head.suppressed.filter((id) => !(base?.suppressed ?? []).includes(id)); + +type ChangedRow = { + routePath: string; + sensitive: boolean; + baseScore: number | string; + headScore: number | string; + nowFailing: string[]; + /** + * Ids this pull request newly suppressed on the entry. Rendered on the route cell, because a + * suppression added to a check that was passing drops the score by round A's cap and produces a + * row with an empty "now failing" column, which is indistinguishable from a real regression: + * `_app.@.orgs.$organizationSlug.$.tsx` renders 67 to 50 that way. The score movement is honest, + * the row without this note was not. + */ + suppressed: string[]; + /** + * How much the entry got worse, used to sort the table. A new entry has no base score to + * subtract from, so it is scored against a perfect 100: a new entry landing at 60 sorts the + * same as an existing one that dropped 40 points, which is the ordering "what needs fixing + * first" implies. Zero whenever either side is unmeasured, because there is no arithmetic to do + * between a figure and an absence; such a row is in the table to disclose the transition, not to + * claim a size for it. + */ + drop: number; +}; + +function changedRows(head: MapReport, base: MapReport): { rows: ChangedRow[]; removed: number } { + const baseByFile = new Map(base.entries.map((e) => [e.fileName, e])); + const headFiles = new Set(head.entries.map((e) => e.fileName)); + + const rows: ChangedRow[] = []; + for (const h of head.entries) { + const b = baseByFile.get(h.fileName); + if (!b) { + // A new entry that passes every check it was measured against has nothing to fix, which is + // what `drop: 0` means everywhere else in this table. A new entry nothing applied to is a + // different statement and still gets a row, since its 100 is a placeholder rather than a pass. + if (h.measured && h.score === 100 && h.suppressed.length === 0) continue; + rows.push({ + routePath: h.routePath, + sensitive: h.sensitive, + baseScore: "new", + headScore: scoreCell(h), + nowFailing: failingIds(h), + suppressed: newlySuppressed(h, b), + drop: h.measured ? 100 - h.score : 0, + }); + continue; + } + // Measured state and the suppression set are both part of what changed. A measured-to- + // unmeasured transition can leave the score at its placeholder value, and suppressing a check + // that was already failing moves no score at all, so skipping on the number alone hid both. A + // pull request whose whole purpose is to silence findings has to produce a row. + const suppressed = newlySuppressed(h, b); + const suppressionChanged = suppressed.length > 0 || h.suppressed.length !== b.suppressed.length; + if (b.measured === h.measured && b.score === h.score && !suppressionChanged) continue; + const baseFailing = new Set(failingIds(b)); + rows.push({ + routePath: h.routePath, + sensitive: h.sensitive, + baseScore: scoreCell(b), + headScore: scoreCell(h), + nowFailing: failingIds(h).filter((id) => !baseFailing.has(id)), + suppressed, + drop: b.measured && h.measured ? b.score - h.score : 0, + }); + } + + rows.sort( + (a, b) => + Number(b.sensitive) - Number(a.sensitive) || + b.drop - a.drop || + a.routePath.localeCompare(b.routePath) + ); + + const removed = base.entries.filter((e) => !headFiles.has(e.fileName)).length; + return { rows, removed }; +} + +function whatChangedSection(head: MapReport, base: MapReport | null): string[] { + const lines = ["**What this PR changed**"]; + + if (!base) { + lines.push("Base comparison unavailable."); + return lines; + } + + const { rows, removed } = changedRows(head, base); + + if (rows.length === 0 && removed === 0) { + lines.push("No entry point this PR touches changed its score."); + return lines; + } + + if (rows.length > 0) { + lines.push(""); + lines.push("| route | base | head | now failing |"); + lines.push("| --- | --- | --- | --- |"); + for (const row of rows.slice(0, MAX_CHANGED_ROWS)) { + const note = row.suppressed.length > 0 ? ` (suppressed: ${row.suppressed.join(", ")})` : ""; + lines.push( + `| ${row.routePath}${note} | ${row.baseScore} | ${row.headScore} | ${row.nowFailing.join(", ")} |` + ); + } + if (rows.length > MAX_CHANGED_ROWS) { + lines.push(""); + lines.push(`and ${rows.length - MAX_CHANGED_ROWS} more`); + } + } + + if (removed > 0) { + lines.push(""); + lines.push(`${removed} entries removed`); + } + + return lines; +} + +function fixFirstSection(head: MapReport): string[] { + const lines = ["FIX FIRST"]; + const worst = fixFirst(head.entries); + + for (const e of worst.slice(0, 3)) { + const marks = e.sensitive ? " (sensitive)" : ""; + lines.push( + `- ${e.routePath}${marks} - ${scoredFailures(e) + .map((c) => c.id) + .join(", ")}` + ); + } + return lines; +} + +const same = (a: unknown, b: unknown) => JSON.stringify(a) === JSON.stringify(b); + +/** + * Whether this pull request moves the report at all, so the job can stay quiet when it does not. + * + * The rule this has to satisfy is that it must be true whenever `renderPrComment` would say + * something different, because anything it misses is a change the pull request silently does not + * report. So it covers every figure the comment renders, not only the score: the global, the + * per-entry score, measured state and suppression set, an entry added or removed, a check failing + * at head that did not at base, the parse failure count, the unknown suppression warnings, and the + * audit and context gaps. + * + * `delegating` and `checkContributions` are compared outright rather than through the per-entry + * loop. Both are rendered, and both can move while every entry keeps its score: a check going from + * applicable-and-passing to not-applicable leaves an entry at 100 and changes what the CHECKS block + * says about it. + * + * The per-entry suppression set and the two gaps are the half that was missing, and it ran the + * dangerous way. Suppressing an already-failing check moves no score, no measured flag and no new + * failure, so a pull request whose entire purpose was to silence findings posted nothing, while a + * mistyped directive did post because the unknown warnings were compared. `audit-trail` going from + * fail to pass, the first audit record in the webapp, was in the same hole, and so was the CONTEXT + * figure moving behind a suppression, since that figure reads pre-suppression data. + * + * The terms overlap on purpose, and what is defended is that their union is complete rather than + * that each one is load bearing. Four are individually reachable, each with a test that fails when + * only that term is removed: the parse failure count, the unknown suppression warnings, the audit + * gap and the context gap. The global, the removed-entry check and the per-entry score are each + * shadowed by another term today, and are kept because which term shadows which depends on the + * shape of the change rather than on anything stable. + * + * `MapReport.suppressions` is the one term deliberately left out. Its two totals are summed from + * the very per-entry `suppressed` arrays the loop below compares one by one, so it cannot move + * without the loop moving. That is arithmetic rather than a happy overlap. + */ +export function hasDelta(head: MapReport, base: MapReport | null): boolean { + if (!base) return true; + if (head.global !== base.global) return true; + if (head.parseFailures.length !== base.parseFailures.length) return true; + if (!same(head.unknownSuppressions, base.unknownSuppressions)) return true; + if (!same(head.auditGap, base.auditGap)) return true; + if (!same(head.contextGap, base.contextGap)) return true; + if (!same(head.delegating, base.delegating)) return true; + if (!same(head.checkContributions, base.checkContributions)) return true; + + const baseByFile = new Map(base.entries.map((e) => [e.fileName, e])); + const headFiles = new Set(head.entries.map((e) => e.fileName)); + if (base.entries.some((e) => !headFiles.has(e.fileName))) return true; + + for (const h of head.entries) { + const b = baseByFile.get(h.fileName); + if (!b) return true; + if (b.measured !== h.measured || b.score !== h.score) return true; + if (!same(h.suppressed, b.suppressed)) return true; + const baseFailing = new Set(b.checks.filter((c) => c.status === "fail").map((c) => c.id)); + if (h.checks.some((c) => c.status === "fail" && !baseFailing.has(c.id))) return true; + } + return false; +} + +/** + * What replaces a comment whose findings a later push fixed. Going silent would leave the earlier + * comment standing with findings that no longer exist, which is worse than a redundant comment. + */ +export function renderResolvedComment(): string { + return [ + MARKER, + "", + "## Observability map", + "", + "Nothing in this pull request moves the report any more. The findings an earlier push " + + "reported are gone.", + "", + "Report only, nothing here gates the merge. The rules and their reasons: " + + "internal-packages/observability-map/README.md.", + ].join("\n"); +} + +/** + * What the job posts when the head scan did not produce a report. The alternative was a red x on + * a job that must never block a pull request, and the alternative to that was swallowing the + * failure so the only signal was a comment that never appeared. + */ +export function renderScanFailedComment(): string { + return [ + MARKER, + "", + "## Observability map", + "", + "The scan failed for this run, so there is no report. Anything above is from an earlier push " + + "and is stale. The workflow log has the error.", + "", + "Report only, nothing here gates the merge. The rules and their reasons: " + + "internal-packages/observability-map/README.md.", + ].join("\n"); +} + +/** + * Pure function, no I/O: `head` and `base` are already-built reports. Matches entries across the + * two by `fileName`, the same identifier `renderJson` carries. + */ +export function renderPrComment(head: MapReport, base: MapReport | null): string { + const lines = [MARKER, "", "## Observability map", "", scoreLine(head, base), ""]; + + lines.push(...whatChangedSection(head, base), ""); + lines.push(...fixFirstSection(head), ""); + + const audit = auditLine(head); + if (audit) lines.push(audit); + const context = contextLine(head); + if (context) lines.push(context); + const delegated = delegatedLines(head, MAX_DELEGATED_ROUTES); + lines.push(...delegated); + const unknown = unknownSuppressionLines(head); + lines.push(...unknown.slice(0, MAX_UNKNOWN_SUPPRESSION_LINES)); + if (unknown.length > MAX_UNKNOWN_SUPPRESSION_LINES) { + lines.push(`and ${unknown.length - MAX_UNKNOWN_SUPPRESSION_LINES} more files with unknown ids`); + } + if (audit || context || delegated.length > 0 || unknown.length > 0) lines.push(""); + + const contributions = checkContributionLines(head); + if (contributions.length > 0) { + lines.push("
What the score is made of", "", "```"); + lines.push(...contributions); + lines.push("```", "", "
", ""); + } + + lines.push( + "Report only, nothing here gates the merge. The rules and their reasons: " + + "internal-packages/observability-map/README.md." + ); + + const headFailures = head.parseFailures.length; + const baseFailures = base?.parseFailures.length ?? 0; + if (headFailures > 0 || baseFailures > 0) { + const parts: string[] = []; + if (headFailures > 0) parts.push(`${headFailures} at head`); + if (baseFailures > 0) parts.push(`${baseFailures} at base`); + lines.push( + `Warning: parse failures (${parts.join(", ")}) are excluded from the score, shrinking the denominator.` + ); + } + + return lines.join("\n"); +} diff --git a/internal-packages/observability-map/src/report/prCommentCli.test.ts b/internal-packages/observability-map/src/report/prCommentCli.test.ts new file mode 100644 index 00000000000..31ddfeeecb3 --- /dev/null +++ b/internal-packages/observability-map/src/report/prCommentCli.test.ts @@ -0,0 +1,141 @@ +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { main, type Io } from "./prCommentCli.js"; +import { buildReport } from "../score.js"; +import { renderJson } from "./json.js"; +import { scanFile } from "../scan.js"; + +const capture = () => { + const out: string[] = []; + const err: string[] = []; + const io: Io = { out: (s) => out.push(s), err: (s) => err.push(s) }; + return { io, out: () => out.join(""), err: () => err.join("") }; +}; + +const run = (...args: string[]) => { + const c = capture(); + const code = main(["node", "prCommentCli.js", ...args], c.io); + return { code, out: c.out(), err: c.err() }; +}; + +const swallows = `import { prisma } from "~/db.server"; + export async function action() { + try { return await prisma.token.create({ data: {} }); } catch (e) { return null; } + }`; + +const handles = `import { requireUserId } from "~/services/session.server"; + import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function action({ request }) { + const userId = await requireUserId(request); + try { return await prisma.token.create({ data: { userId } }); } + catch (error) { logger.error("token create failed", { userId, error }); throw error; } + }`; + +describe("prCommentCli", () => { + const dir = mkdtempSync(join(tmpdir(), "obs-map-cli-")); + const headPath = join(dir, "head.json"); + const basePath = join(dir, "base.json"); + const unchangedPath = join(dir, "unchanged.json"); + + const source = `export const loader = () => new Response("ok");`; + const report = buildReport([scanFile("resources.a.ts", source)!], []); + writeFileSync(headPath, renderJson(buildReport([scanFile("api.v1.t.ts", swallows)!], []))); + writeFileSync(basePath, renderJson(buildReport([scanFile("api.v1.t.ts", handles)!], []))); + writeFileSync(unchangedPath, renderJson(report)); + + afterAll(() => rmSync(dir, { recursive: true })); + + it("renders the markdown comment for head and base file arguments", () => { + const r = run(headPath, basePath); + expect(r.code).toBe(0); + expect(r.out.split("\n")[0]).toBe(""); + }); + + // B4. The job used to comment on every push, including one that moved nothing. + it("posts nothing when the report did not move and no comment exists yet", () => { + const r = run(unchangedPath, unchangedPath); + expect(r.code).toBe(0); + expect(r.out).toBe(""); + }); + + it("replaces an existing comment with a resolved state when the delta has gone", () => { + const r = run(unchangedPath, unchangedPath, "--existing-comment"); + expect(r.code).toBe(0); + expect(r.out.split("\n")[0]).toBe(""); + expect(r.out).toContain("Nothing in this pull request moves the report any more."); + expect(r.out).not.toContain("FIX FIRST"); + }); + + it("posts the full comment when there is a delta, existing comment or not", () => { + for (const args of [ + [headPath, basePath], + [headPath, basePath, "--existing-comment"], + ]) { + const r = run(...args); + expect(r.code).toBe(0); + expect(r.out).toContain("FIX FIRST"); + expect(r.out).not.toContain("moves the report any more"); + } + }); + + it("prints the stale-report comment for --scan-failed without reading any file", () => { + const r = run("--scan-failed"); + expect(r.code).toBe(0); + expect(r.out.split("\n")[0]).toBe(""); + expect(r.out).toContain("The scan failed for this run"); + }); + + it("treats '-' as no base", () => { + const r = run(headPath, "-"); + expect(r.code).toBe(0); + expect(r.out).toContain("Base comparison unavailable."); + }); + + it("treats a missing second argument as no base", () => { + const r = run(headPath); + expect(r.code).toBe(0); + expect(r.out).toContain("Base comparison unavailable."); + }); + + // Without a base there is no delta to compute, so silence would be a guess. Posting is the + // honest answer even for a report identical to one nobody can see. + it("posts even for an unmoved report when the base is unavailable", () => { + const r = run(unchangedPath, "-"); + expect(r.code).toBe(0); + expect(r.out).toContain("Base comparison unavailable."); + }); + + it("exits 1 with a usage message when head.json is missing", () => { + const r = run(); + expect(r.code).toBe(1); + expect(r.err).toContain("usage:"); + }); + + it("exits 1 with a one-line message, not a stack trace, when head.json does not exist", () => { + const r = run(join(dir, "does-not-exist.json")); + expect(r.code).toBe(1); + expect(r.err.split("\n").filter(Boolean)).toHaveLength(1); + expect(r.err).toContain("cannot read head report"); + expect(r.err).not.toContain(" at "); + }); + + it("exits 1 with a one-line message, not a stack trace, when head.json is malformed", () => { + const malformedPath = join(dir, "malformed.json"); + writeFileSync(malformedPath, "{ not json"); + const r = run(malformedPath); + expect(r.code).toBe(1); + expect(r.err.split("\n").filter(Boolean)).toHaveLength(1); + expect(r.err).toContain("head report is not valid JSON"); + expect(r.err).not.toContain(" at "); + }); + + it("exits 1 with a one-line message when base.json is malformed", () => { + const malformedBasePath = join(dir, "malformed-base.json"); + writeFileSync(malformedBasePath, "not json at all"); + const r = run(headPath, malformedBasePath); + expect(r.code).toBe(1); + expect(r.err).toContain("base report is not valid JSON"); + }); +}); diff --git a/internal-packages/observability-map/src/report/prCommentCli.ts b/internal-packages/observability-map/src/report/prCommentCli.ts new file mode 100644 index 00000000000..a2aef9a3477 --- /dev/null +++ b/internal-packages/observability-map/src/report/prCommentCli.ts @@ -0,0 +1,89 @@ +import { readFileSync } from "node:fs"; +import { resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import type { MapReport } from "../score.js"; +import { + hasDelta, + renderPrComment, + renderResolvedComment, + renderScanFailedComment, +} from "./prComment.js"; + +/** Where output goes. Injectable so tests can read it without spawning a process. */ +export type Io = { out: (s: string) => void; err: (s: string) => void }; + +const processIo: Io = { + out: (s) => process.stdout.write(s), + err: (s) => process.stderr.write(s), +}; + +/** Reads and parses one report file, raising a message naming the file rather than letting an + * unreadable path or malformed JSON surface as a stack trace. */ +function readReport(path: string, label: string): MapReport { + let raw: string; + try { + raw = readFileSync(path, "utf8"); + } catch { + throw new Error(`cannot read ${label}: ${path}`); + } + try { + return JSON.parse(raw) as MapReport; + } catch { + throw new Error(`${label} is not valid JSON: ${path}`); + } +} + +/** + * `-` or a missing second arg means no base: the CI job falls back to this when the base scan + * itself failed, so the comment still renders rather than the job going red. + * + * Empty output means "post nothing". The job only comments when the pull request moves the report, + * and `--existing-comment` is how the workflow says a comment from an earlier push is already on + * the pull request: with the delta gone, that comment is replaced with a resolved state rather + * than left standing with findings that no longer exist. + * + * `--scan-failed` takes no report and prints the stale-report comment, for the case where the head + * scan produced nothing to read. + */ +export function main(argv: string[], io: Io = processIo): number { + const args = argv.slice(2); + const scanFailed = args.includes("--scan-failed"); + const existingComment = args.includes("--existing-comment"); + const positional = args.filter((a) => !a.startsWith("--")); + const headPath = positional[0]; + const basePath = positional[1]; + + if (scanFailed) { + io.out(`${renderScanFailedComment()}\n`); + return 0; + } + + if (!headPath) { + io.err("usage: prCommentCli.ts [base.json|-] [--existing-comment]\n"); + return 1; + } + + let head: MapReport; + let base: MapReport | null; + try { + head = readReport(headPath, "head report"); + base = !basePath || basePath === "-" ? null : readReport(basePath, "base report"); + } catch (error) { + io.err(`${error instanceof Error ? error.message : String(error)}\n`); + return 1; + } + + if (hasDelta(head, base)) { + io.out(`${renderPrComment(head, base)}\n`); + return 0; + } + if (existingComment) { + io.out(`${renderResolvedComment()}\n`); + } + return 0; +} + +// Only when run as a program. Importing the module, which the tests do, must not read a file. +const invokedDirectly = + process.argv[1] !== undefined && resolve(process.argv[1]) === fileURLToPath(import.meta.url); +if (invokedDirectly) process.exitCode = main(process.argv); diff --git a/internal-packages/observability-map/src/report/terminal.test.ts b/internal-packages/observability-map/src/report/terminal.test.ts new file mode 100644 index 00000000000..61b7aa9c462 --- /dev/null +++ b/internal-packages/observability-map/src/report/terminal.test.ts @@ -0,0 +1,404 @@ +import { renderTerminal } from "./terminal.js"; +import { renderJson } from "./json.js"; +import { buildReport } from "../score.js"; +import { scanFile } from "../scan.js"; + +const report = () => + buildReport( + [ + scanFile( + "api.v1.a.ts", + `import { createLoaderApiRoute } from "~/services/routeBuilders/apiBuilder.server"; + export const loader = createLoaderApiRoute({}, async () => new Response("ok"));` + )!, + scanFile( + "api.v1.auth.tokens.ts", + `import { prisma } from "~/db.server"; + export async function action() { return prisma.token.create({ data: {} }); }` + )!, + ], + ["broken.ts"] + ); + +/** The FIX FIRST section, with the index asserted rather than assumed. `indexOf` returns -1 for a + * section that is not there, and `slice(-1)` is the last character of the report, which every + * `not.toContain` below would pass against. */ +function sliceFixFirst(out: string): string { + const start = out.indexOf("FIX FIRST"); + expect(start).toBeGreaterThan(-1); + const end = out.indexOf("no findings:"); + expect(end).toBeGreaterThan(start); + return out.slice(start, end); +} + +describe("renderTerminal", () => { + // The guard has to be able to fail, or it is decoration in a file whose own comment warns about + // exactly this trap. + it("refuses to slice a report with no fix list rather than returning its last character", () => { + expect(() => sliceFixFirst("a report with neither section in it")).toThrow(); + }); + + it("shows the global score, the audit gap and the fix list", () => { + const out = renderTerminal(report()); + expect(out).toContain("COVERAGE"); + expect(out).toContain("FIX FIRST"); + expect(out).toContain("audit"); + expect(out).toContain("/api/v1/auth/tokens"); + }); + + it("surfaces suppressions so laundering is visible rather than silent", () => { + const suppressed = scanFile( + "api.v1.d.ts", + `// obs-map-disable error-classification -- deliberate, see ticket + import { prisma } from "~/db.server"; + export async function loader() { + try { return await prisma.thing.findMany(); } catch (e) { return null; } + }` + )!; + const out = renderTerminal(buildReport([suppressed], [])); + expect(out).toMatch(/SUPPRESSED\s+1 check across 1 entry point/); + }); + + it("does not mention suppressions when there are none", () => { + expect(renderTerminal(report())).not.toMatch(/suppress/i); + }); + + it("surfaces parse failures so the denominator is not silently wrong", () => { + expect(renderTerminal(report())).toContain("broken.ts"); + }); + + it("shows the unmeasured count so 415 vs 427 is not silently confusing", () => { + const trivial = scanFile( + "resources.health.ts", + `export const loader = () => new Response("ok");` + )!; + const out = renderTerminal(buildReport([trivial], [])); + expect(out).toContain("1 unmeasured"); + }); + + it("orders FIX FIRST by sensitivity first, then ascending score, and excludes audit-only gaps", () => { + // Sensitive, score 0: both applicable scored checks fail (auth-boundary, error-classification, + // request-context all fail because the catch swallows without naming who it happened to). + const sensitiveZero = scanFile( + "api.v1.envvars.ts", + `import { prisma } from "~/db.server"; + export async function action() { + try { + return await prisma.envVar.update({ where: {}, data: {} }); + } catch (e) { + return null; + } + }` + )!; + + // Sensitive, score 33: its catch decides something, telling a bad request apart from the rest, + // but it has no guard and names nobody. Fails more than request-context, so it stays in the + // list rather than collapsing into the figure. + const sensitiveThirtyThree = scanFile( + "api.v1.auth.tokens.ts", + `import { prisma } from "~/db.server"; + export async function action() { + try { return await prisma.token.create({ data: {} }); } + catch (error) { + if (error instanceof BadRequest) return json({ error: "bad" }, { status: 400 }); + throw error; + } + }` + )!; + + // Not sensitive, score 0: worse score than sensitiveThirtyThree, but must still sort after both + // sensitive entries because sensitivity outranks raw score. + const notSensitiveZero = scanFile( + "resources.busy.ts", + `import { prisma } from "~/db.server"; + export async function loader() { + try { return await prisma.thing.findMany(); } catch (e) { return null; } + }` + )!; + + // Sensitive mutation, scored checks all pass (guarded, no try/catch): the only gap is + // audit-trail, which is a headline figure, not a per-route fix-list item. Must not appear. + const sensitiveAuditOnly = scanFile( + "api.v1.billing.ts", + `import { prisma } from "~/db.server"; + import { logger } from "~/services/logger.server"; + import { requireUserId } from "~/services/session.server"; + export async function action({ request }) { + const userId = await requireUserId(request); + try { return await prisma.billing.update({ where: { userId }, data: {} }); } + catch (error) { logger.error("billing update failed", { userId, error }); throw error; } + }` + )!; + + const out = renderTerminal( + buildReport([sensitiveThirtyThree, sensitiveZero, notSensitiveZero, sensitiveAuditOnly], []) + ); + + // Slice to the end of the list, not to a string the I5 fix deleted: `indexOf` returned -1 for + // "already solid" and the assertions were quietly running against the whole tail. + const fixFirst = sliceFixFirst(out); + const idxZero = fixFirst.indexOf("api.v1.envvars.ts"); + const idxThirtyThree = fixFirst.indexOf("api.v1.auth.tokens.ts"); + const idxNotSensitive = fixFirst.indexOf("resources.busy.ts"); + + expect(idxZero).toBeGreaterThan(-1); + expect(idxThirtyThree).toBeGreaterThan(idxZero); + expect(idxNotSensitive).toBeGreaterThan(idxThirtyThree); + expect(fixFirst).not.toContain("api.v1.billing.ts"); + }); +}); + +describe("renderJson", () => { + it("round-trips to an object carrying the score and entries", () => { + const parsed = JSON.parse(renderJson(report())); + expect(typeof parsed.global).toBe("number"); + expect(Array.isArray(parsed.entries)).toBe(true); + }); +}); + +describe("rendering honestly when there is nothing to say", () => { + // I4. mean([]) returned 100, so a family with nothing measured rendered a full green bar. + it("renders a family with nothing measured as not measured, not as 100", () => { + const trivial = scanFile( + "resources.health.ts", + `export const loader = () => new Response("ok");` + )!; + const out = renderTerminal(buildReport([trivial], [])); + const line = out.split("\n").find((l) => l.includes("resources"))!; + expect(line).not.toMatch(/100/); + expect(line).toMatch(/not measured/i); + }); + + it("renders the global score as not measured when nothing was measured", () => { + const trivial = scanFile( + "resources.health.ts", + `export const loader = () => new Response("ok");` + )!; + expect(renderTerminal(buildReport([trivial], []))).toMatch(/score not measured/i); + }); + + // I11. The audit sentence was printed unconditionally, including when the figure said otherwise. + it("does not claim no audit helper exists when one is in use", () => { + const audited = scanFile( + "api.v1.auth.tokens.ts", + `import { clearImpersonation } from "~/models/admin.server"; + import { prisma } from "~/db.server"; + export async function action() { + const token = await prisma.token.create({ data: {} }); + await clearImpersonation(request, "/admin"); + return json(token); + }` + )!; + const out = renderTerminal(buildReport([audited], [])); + expect(out).toContain("1 of 1"); + expect(out).not.toContain("No audit helper exists"); + }); + + // Round E item 2. The other half of the branch, which the test above did not pin. A zero used to + // print "No audit helper exists in the webapp", which is false: `models/admin.server.ts` writes + // `prisma.impersonationAuditLog.create(...)` and `AUDIT_SYMBOLS` names the helpers that reach it. + // The full tree reads 3 of 49 so nobody sees the sentence today, and a scan of any subset with no + // impersonation route in it brings the sentence straight back. + it("does not claim the webapp has no audit helper when nothing reached one", () => { + const unaudited = scanFile( + "api.v1.auth.tokens.ts", + `import { prisma } from "~/db.server"; + export async function action() { + return json(await prisma.token.create({ data: {} })); + }` + )!; + const out = renderTerminal(buildReport([unaudited], [])); + expect(out).toContain("AUDIT 0 of 1 sensitive mutations record an actor. 1 without one."); + expect(out).not.toContain("No audit helper exists"); + }); + + it("says nothing about audit when no sensitive mutation was found", () => { + const plain = scanFile( + "resources.things.ts", + `import { prisma } from "~/db.server"; + export async function loader() { return prisma.thing.findMany(); }` + )!; + expect(renderTerminal(buildReport([plain], []))).not.toMatch(/AUDIT/); + }); + + // I5. "already solid" counted entries that are clean because they do nothing, alongside entries + // nothing applied to, in one flattering number. + it("separates entries that passed from entries nothing applied to", () => { + const clean = scanFile( + "api.v1.clean.ts", + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function loader({ params }) { + try { return await prisma.thing.findMany(); } + catch (error) { logger.error("failed", { environmentId: params.envId, error }); throw error; } + }` + )!; + const trivial = scanFile( + "resources.health.ts", + `export const loader = () => new Response("ok");` + )!; + const out = renderTerminal(buildReport([clean, trivial], [])); + expect(out).not.toMatch(/already solid/i); + expect(out).toMatch(/1 passed every applicable check/i); + expect(out).toMatch(/1 had none to apply/i); + }); +}); + +describe("collapsing the house-style finding", () => { + const namesNobody = () => + scanFile( + "api.v1.silent.ts", + `import { prisma } from "~/db.server"; + export async function loader() { return prisma.thing.findMany(); }` + )!; + + const namesNobodyAndSwallows = () => + scanFile( + "api.v1.auth.tokens.ts", + `import { prisma } from "~/db.server"; + export async function action() { + try { return await prisma.token.create({ data: {} }); } catch (e) { return null; } + }` + )!; + + // request-context fails 401 of 412 entry points, so listing each one turns the fix list into a + // single finding repeated. Same reasoning that keeps audit-trail out of the list. + it("keeps an entry whose only finding is request-context out of the fix list", () => { + const out = renderTerminal(buildReport([namesNobody()], [])); + // Guarded for the same reason as the slice above: an absent section makes `indexOf` return -1, + // `slice(-1)` yields the last character, and the negative assertion passes for the wrong reason. + const fixFirst = sliceFixFirst(out); + expect(fixFirst).not.toContain("api.v1.silent.ts"); + }); + + it("reports the gap as a headline figure instead", () => { + const out = renderTerminal(buildReport([namesNobody()], [])); + expect(out).toMatch(/CONTEXT\s+0 of 1 entry points name a tenant on a failure path/); + }); + + // NEW-3. 18 of the collapsed entries are sensitive, including /admin/impersonate and the envvars + // routes, so the line has to say a reader should go and look at them. + it("says how many of the collapsed entries are sensitive", () => { + const sensitiveAndSilent = scanFile( + "api.v1.auth.jwt.ts", + `import { requireUserId } from "~/services/session.server"; + import { prisma } from "~/db.server"; + export async function loader({ request }) { + const userId = await requireUserId(request); + return prisma.token.findMany({ where: { userId } }); + }` + )!; + const out = renderTerminal(buildReport([sensitiveAndSilent], [])); + expect(out).toMatch(/1 appears? only here[^\n]*1 of them sensitive/i); + }); + + it("says how many entries the collapse took out of the list", () => { + const out = renderTerminal(buildReport([namesNobody(), namesNobodyAndSwallows()], [])); + expect(out).toMatch(/1 appears? only here/i); + }); + + it("still lists request-context when the entry fails something else too", () => { + const out = renderTerminal(buildReport([namesNobodyAndSwallows()], [])); + const fixFirst = sliceFixFirst(out); + expect(fixFirst).toContain("api.v1.auth.tokens.ts"); + expect(fixFirst).toContain("request-context"); + expect(fixFirst).toContain("error-classification"); + }); +}); + +// B6. A stderr warning is the minimum; the terminal report is where someone would notice. +describe("reporting a suppression that names no check", () => { + const typo = (fileName: string) => + scanFile( + fileName, + `// obs-map-disable eror-classification -- typo + import { prisma } from "~/db.server"; + export async function loader() { + try { return await prisma.thing.findMany(); } catch (e) { return null; } + }` + )!; + + it("names the file and the bad id", () => { + const out = renderTerminal(buildReport([typo("api.v1.a.ts")], [])); + expect(out).toContain("UNKNOWN SUPPRESSION"); + expect(out).toContain("api.v1.a.ts"); + expect(out).toContain("eror-classification"); + }); + + it("lists the ids that would have worked", () => { + const out = renderTerminal(buildReport([typo("api.v1.a.ts")], [])); + expect(out).toContain( + "error-classification, auth-boundary, auth-scope, request-context, audit-trail" + ); + }); + + it("does not report the finding as suppressed", () => { + const out = renderTerminal(buildReport([typo("api.v1.a.ts")], [])); + expect(out).not.toMatch(/SUPPRESSED\s+\d/); + }); + + it("reports one line per file rather than one for the run", () => { + const out = renderTerminal(buildReport([typo("api.v1.a.ts"), typo("api.v1.b.ts")], [])); + expect(out.split("\n").filter((l) => l.startsWith("UNKNOWN SUPPRESSION"))).toHaveLength(2); + }); + + it("says nothing when every directive named a real check", () => { + expect(renderTerminal(report())).not.toContain("UNKNOWN SUPPRESSION"); + }); +}); + +// C4b. A delegating route must not read as a clean bill of health. It is surfaced the way a parse +// failure is, because it shrinks the denominator the same way. +describe("reporting a route whose body is in another module", () => { + const delegated = () => + buildReport( + [ + scanFile("webhooks.v1.stripe.ts", `export { action } from "./handler.server";`)!, + scanFile( + "api.v1.a.ts", + `import { prisma } from "~/db.server"; + export async function loader() { return prisma.thing.findMany(); }` + )!, + ], + [] + ); + + it("names the route and says nothing was checked", () => { + const out = renderTerminal(delegated()); + expect(out).toContain("DELEGATED"); + expect(out).toContain("webhooks.v1.stripe.ts"); + expect(out).toContain("nothing here was checked"); + }); + + it("separates it from the entries nothing applied to, in the headline", () => { + expect(renderTerminal(delegated())).toContain("1 measured, 0 unmeasured, 1 delegated of 2"); + }); + + it("carries the file names into the JSON", () => { + expect(JSON.parse(renderJson(delegated())).delegating).toEqual(["webhooks.v1.stripe.ts"]); + }); + + it("says nothing when every route keeps its body", () => { + expect(renderTerminal(report())).not.toContain("DELEGATED"); + }); +}); + +// C5. What the composite is made of, disclosed on screen rather than folded into a weight. +describe("reporting what the score is made of", () => { + it("gives a line per check with applicability, passes and worth", () => { + const out = renderTerminal(report()); + expect(out).toContain("CHECKS"); + expect(out).toMatch( + /request-context\s+\d+ applicable,\s+\d+ pass,\s+\d+ sole, global without it/ + ); + }); + + it("marks the check that does not feed the score", () => { + expect(renderTerminal(report())).toMatch(/audit-trail\s+.*not in the score/); + }); + + it("carries the same figures into the JSON", () => { + const parsed = JSON.parse(renderJson(report())); + expect(parsed.checkContributions.map((c: { id: string }) => c.id)).toContain("auth-scope"); + }); +}); diff --git a/internal-packages/observability-map/src/report/terminal.ts b/internal-packages/observability-map/src/report/terminal.ts new file mode 100644 index 00000000000..0bb904bae50 --- /dev/null +++ b/internal-packages/observability-map/src/report/terminal.ts @@ -0,0 +1,248 @@ +import type { MapReport, ScoredEntry } from "../score.js"; +import { CHECKS, SCORED_CHECK_IDS } from "../checks/index.js"; + +const NOT_MEASURED = "not measured".padEnd(15); + +/** A bar and a figure, or a plain "not measured" where there is no figure to draw. */ +const gauge = (score: number | null) => { + if (score === null) return NOT_MEASURED; + const filled = Math.round(score / 10); + return `${"β–°".repeat(filled)}${"β–±".repeat(10 - filled)} ${String(score).padStart(3)}`; +}; + +/** + * Failing checks that actually feed `score`. `audit-trail` is deliberately excluded here: it is + * excluded from the score for the same reason (see `score.ts`), and every sensitive mutation fails + * it today, so folding it in would flood this list with the same finding repeated 46 times instead + * of the fixable, route-specific gaps the list exists to surface. That gap is reported once, as + * `AUDIT`, below. + */ +export const scoredFailures = (e: ScoredEntry) => + e.checks.filter((c) => SCORED_CHECK_IDS.includes(c.id) && c.status === "fail"); + +/** + * An entry whose only finding is `request-context`. 401 of 412 entry points fail that check, so + * listing each one turns the fix list into a single house-style finding repeated, which is the + * reason `audit-trail` is kept out of the list too. Collapsed into the `CONTEXT` figure instead. + * An entry that fails something else as well stays in the list with all of its findings, so a + * route like `/account/tokens` still shows the request-context gap alongside the rest. + */ +/** + * The routes the FIX FIRST list is drawn from, worst first: sensitive before not, then by score, + * then by name. Exported because `prComment.ts` renders the same list with different bullets and + * had a byte-identical copy of this filter and sort, in a file that already imports + * `scoredFailures` and `contextOnly` from here. + * + * `contextOnly` routes are left out because `request-context` fails almost everything, so a list + * headed by three of them tells a reader nothing they cannot read off the gap figure. + */ +export const fixFirst = (entries: ScoredEntry[]): ScoredEntry[] => + entries + .filter((e) => scoredFailures(e).length > 0 && !contextOnly(e)) + .sort( + (a, b) => + Number(b.sensitive) - Number(a.sensitive) || + a.score - b.score || + a.fileName.localeCompare(b.fileName) + ); + +export const contextOnly = (e: ScoredEntry) => { + const failures = scoredFailures(e); + return failures.length === 1 && failures[0]!.id === "request-context"; +}; + +/** + * The UNKNOWN SUPPRESSION lines, one per file, shared with `prComment.ts`. Empty when every + * directive named a real check. A typo suppresses nothing, so without this the author reads the + * finding as acknowledged and the tool goes on reporting it with no hint why. + */ +export function unknownSuppressionLine(fileName: string, ids: string[]): string { + return ( + `UNKNOWN SUPPRESSION ${fileName}: ${ids.join(", ")} ` + + `(no such check, nothing suppressed). Known: ${CHECKS.map((c) => c.id).join(", ")}.` + ); +} + +export function unknownSuppressionLines(report: MapReport): string[] { + return report.unknownSuppressions.map(({ fileName, ids }) => + unknownSuppressionLine(fileName, ids) + ); +} + +/** + * The AUDIT figure, shared with `prComment.ts` so both renderers say the same thing. Null when + * there is nothing to report, i.e. no sensitive mutation exists. + * + * One shape for every count, and no branch on the count, because the branch is what carried the + * bug. A zero used to print "No audit helper exists in the webapp", which is false: the helper + * exists and `AUDIT_SYMBOLS` names it, `apps/webapp/app/models/admin.server.ts` writes + * `prisma.impersonationAuditLog.create(...)`, and `webappSymbols.test.ts` proves those symbols + * resolve. The count was already correct, so the sentence was the only wrong thing and it is gone + * rather than reworded. A zero here means nothing reached the helper, which is what + * "0 of N record an actor" already says. + * + * The full-tree scan reads 3 of 49 today, so the zero branch is not taken and nobody sees it. It is + * one `--routes=` away from being taken, and one reshaped impersonation route away on the full + * tree, which is why removing it beats leaving it unreachable. + */ +export function auditLine(report: MapReport): string | null { + const { sensitiveMutations, withAudit } = report.auditGap; + if (sensitiveMutations === 0) return null; + return ( + `AUDIT ${withAudit} of ${sensitiveMutations} sensitive mutations record an actor. ` + + `${sensitiveMutations - withAudit} without one.` + ); +} + +/** The CONTEXT figure, shared with `prComment.ts`. Null when nothing is applicable. */ +export function contextLine(report: MapReport): string | null { + const { applicable, naming } = report.contextGap; + if (applicable === 0) return null; + const collapsed = report.entries.filter(contextOnly); + const sensitive = collapsed.filter((e) => e.sensitive).length; + return ( + `CONTEXT ${naming} of ${applicable} entry points name a tenant on a failure path.` + + (collapsed.length > 0 + ? ` ${collapsed.length} appear${collapsed.length === 1 ? "s" : ""} only here, ` + + `${sensitive} of them sensitive, in the JSON rather than the fix list.` + : "") + ); +} + +/** + * The DELEGATED lines, shared with `prComment.ts`. Empty when every route's body is in its own + * file. Worded as a shortfall rather than a note: these routes left the denominator and no check + * looked at any of them. + * + * `limit` caps how many file names are named, for the caller that has a size limit to respect. The + * count in front of the list is always the full one, so a capped line still reports the real + * shortfall and only shortens the evidence. The terminal passes no limit and prints them all. + */ +export function delegatedLines(report: MapReport, limit = Infinity): string[] { + if (report.delegating.length === 0) return []; + const n = report.delegating.length; + const shown = report.delegating.slice(0, limit); + const tail = n > shown.length ? `, and ${n - shown.length} more` : ""; + return [ + `DELEGATED ${n} route${n === 1 ? "" : "s"} keep${n === 1 ? "s" : ""} the body in another ` + + `module, so nothing here was checked and ${n === 1 ? "it is" : "they are"} out of the score: ` + + shown.join(", ") + + tail, + ]; +} + +/** + * The CHECKS block: what the composite is made of. `sole` is the figure that says most, since an + * entry only one scored check applies to scores 0 or 100 on that one boolean. + */ +export function checkContributionLines(report: MapReport): string[] { + if (report.checkContributions.every((c) => c.applicable === 0)) return []; + const width = Math.max(...report.checkContributions.map((c) => c.id.length)); + return [ + "CHECKS", + ...report.checkContributions.map((c) => { + const worth = !c.scored + ? "not in the score" + : c.globalWithout === null + ? "nothing left measured without it" + : `global without it ${c.globalWithout}`; + return ( + ` ${c.id.padEnd(width)} ${String(c.applicable).padStart(3)} applicable, ` + + `${String(c.passed).padStart(3)} pass, ${String(c.sole).padStart(3)} sole, ${worth}` + ); + }), + ]; +} + +export function renderTerminal(report: MapReport): string { + const lines: string[] = []; + + const headline = report.global === null ? "score not measured" : `score ${report.global}/100`; + const delegatedCount = + report.delegating.length > 0 ? `, ${report.delegating.length} delegated` : ""; + lines.push( + `${headline} ${report.measured} measured, ${report.unmeasured} unmeasured${delegatedCount} of ${report.entries.length} entry points` + ); + lines.push(""); + lines.push("COVERAGE"); + for (const [family, stats] of Object.entries(report.byFamily).sort((a, b) => b[1].n - a[1].n)) { + lines.push( + ` ${family.padEnd(12)} ${gauge(stats.mean)} ${stats.measured}/${stats.n} entry points` + ); + } + lines.push( + ` ${"sensitive".padEnd(12)} ${gauge(report.sensitiveCohort.mean)} ${ + report.sensitiveCohort.measured + }/${report.sensitiveCohort.n} entry points` + ); + + const contributions = checkContributionLines(report); + if (contributions.length > 0) { + lines.push(""); + lines.push(...contributions); + } + + const audit = auditLine(report); + if (audit) { + lines.push(""); + lines.push(audit); + } + + const context = contextLine(report); + if (context) { + lines.push(""); + lines.push(context); + } + + const delegated = delegatedLines(report); + if (delegated.length > 0) { + lines.push(""); + lines.push(...delegated); + } + + const unknown = unknownSuppressionLines(report); + if (unknown.length > 0) { + lines.push(""); + lines.push(...unknown); + } + + if (report.suppressions.checks > 0) { + const { entries, checks } = report.suppressions; + lines.push( + `SUPPRESSED ${checks} check${checks === 1 ? "" : "s"} across ${entries} entry point${ + entries === 1 ? "" : "s" + }, each with a reason on the record. A suppression removes a finding from this list, ` + + `it does not raise a score.` + ); + } + + const worst = fixFirst(report.entries); + + lines.push(""); + lines.push("FIX FIRST"); + for (const e of worst.slice(0, 3)) { + const marks = e.sensitive ? " (sensitive)" : ""; + lines.push( + ` ${e.routePath}${marks} - ${scoredFailures(e) + .map((c) => c.id) + .join(", ")}` + ); + lines.push(` ${e.fileName}`); + } + if (worst.length > 3) { + lines.push(""); + lines.push(`THEN ${worst.length - 3} more with gaps`); + } + + lines.push(""); + // Not one flattering number: an entry with nothing applicable is not the same as an entry that + // passed, and lumping them together counted routes as solid for doing nothing. + const clean = report.entries.filter((e) => e.measured && scoredFailures(e).length === 0).length; + lines.push( + `no findings: ${clean} passed every applicable check, ${report.unmeasured} had none to apply` + ); + if (report.parseFailures.length > 0) { + lines.push(`parse failures (excluded from the score): ${report.parseFailures.join(", ")}`); + } + return lines.join("\n"); +} diff --git a/internal-packages/observability-map/src/routeExports.ts b/internal-packages/observability-map/src/routeExports.ts new file mode 100644 index 00000000000..a53b2ccf979 --- /dev/null +++ b/internal-packages/observability-map/src/routeExports.ts @@ -0,0 +1,71 @@ +import type { EntryPoint } from "./types.js"; + +export type ExportName = "loader" | "action"; + +/** + * One export of a route file, carrying that export's own evidence and nothing from the other one. + * + * Every field here has an entry-point-wide twin on `EntryPoint`, and reaching for the twin is the + * mistake this type exists to make hard. `calleeNames` is the union of both bodies, `hasTryCatch` + * is true if either has one, and `statementCount` counts both. + */ +export type RouteExport = { + name: ExportName; + /** Callee of the initializer call this export is assigned from, if any. */ + initializerCallee: string | null; + /** Top-level keys of the object literal passed to that call. */ + builderOptions: string[]; + /** Callees inside this export's handlers, and inside same-file helpers they call. */ + calleeNames: string[]; + /** The same calls as whole dotted paths, so `prisma.thing.findFirst` keeps its receiver. */ + calleeTexts: string[]; + /** Callees whose answer those handlers demonstrably read. */ + checkedCallees: string[]; + /** Whether those handlers narrow a query by the caller's own id. */ + scopesByCaller: boolean; + statementCount: number; + hasTryCatch: boolean; +}; + +/** + * The exports this route file declares, in `loader`, `action` order. + * + * One enumeration for the whole package, because two checks asking the same per-export question + * each grew their own. `auth-scope`'s `builderExports` and `auth-boundary`'s `guardedExports` were + * hand-maintained `[loader, action]` literals in adjacent files, reading the same six + * `loaderX`/`actionX` field pairs, with different tests for whether an export was there at all: + * one used `hasLoader`/`hasAction` and the other inferred it from a non-null initializer callee. + * Adding a seventh per-export fact meant editing both, and this whole branch is a record of what + * happens when a rule lives in two places and only one gets the fix. + * + * Absent exports are not returned, so a caller never has to remember to filter them: the shape of + * the bug in `auth-boundary` was crediting an export for something the other one did, and a list + * that only contains real exports is one fewer way to write it. + */ +export function routeExports(ep: EntryPoint): RouteExport[] { + const all: RouteExport[] = [ + { + name: "loader", + initializerCallee: ep.loaderInitializerCallee, + builderOptions: ep.loaderBuilderOptions, + calleeNames: ep.loaderCalleeNames, + calleeTexts: ep.loaderCalleeTexts, + checkedCallees: ep.loaderCheckedCallees, + scopesByCaller: ep.loaderScopesByCaller, + statementCount: ep.loaderStatementCount, + hasTryCatch: ep.loaderHasTryCatch, + }, + { + name: "action", + initializerCallee: ep.actionInitializerCallee, + builderOptions: ep.actionBuilderOptions, + calleeNames: ep.actionCalleeNames, + calleeTexts: ep.actionCalleeTexts, + checkedCallees: ep.actionCheckedCallees, + scopesByCaller: ep.actionScopesByCaller, + statementCount: ep.actionStatementCount, + hasTryCatch: ep.actionHasTryCatch, + }, + ]; + return all.filter((e) => (e.name === "loader" ? ep.hasLoader : ep.hasAction)); +} diff --git a/internal-packages/observability-map/src/scan.test.ts b/internal-packages/observability-map/src/scan.test.ts new file mode 100644 index 00000000000..458fab79ac4 --- /dev/null +++ b/internal-packages/observability-map/src/scan.test.ts @@ -0,0 +1,3044 @@ +import { chmodSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { ParseFailureError, scanDirectory, scanFile } from "./scan.js"; + +const LOADER = ` +import { json } from "@remix-run/server-runtime"; +export async function loader() { return json({}); } +`; + +const COMPONENT_ONLY = ` +export default function Page() { return null; } +`; + +describe("scanFile", () => { + it("detects an exported loader as a server entry point", () => { + const ep = scanFile("api.v1.things.ts", LOADER); + expect(ep).not.toBeNull(); + expect(ep!.hasLoader).toBe(true); + expect(ep!.hasAction).toBe(false); + }); + + it("ignores a route that only exports a component", () => { + expect(scanFile("_app.things.tsx", COMPONENT_ONLY)).toBeNull(); + }); + + it("detects a loader assigned from a call expression", () => { + const ep = scanFile("api.v1.x.ts", `export const loader = createLoaderApiRoute({});`); + expect(ep!.hasLoader).toBe(true); + expect(ep!.loaderInitializerCallee).toBe("createLoaderApiRoute"); + }); +}); + +describe("scanFile: named export clauses", () => { + it("detects `export { loader }` and resolves the builder callee from the local declaration", () => { + const ep = scanFile( + "api.v1.query.ts", + ` + const { loader } = createLoaderApiRoute({ findResource: async () => 1 }, async () => { + const a = 1; + return json({ a }); + }); + export { loader }; + ` + ); + expect(ep).not.toBeNull(); + expect(ep!.hasLoader).toBe(true); + expect(ep!.loaderInitializerCallee).toBe("createLoaderApiRoute"); + expect(ep!.statementCount).toBe(2); + }); + + it("detects an aliased named export `export { h as loader }`", () => { + const ep = scanFile( + "api.v1.aliased.ts", + ` + async function h() { return json({}); } + export { h as loader }; + ` + ); + expect(ep).not.toBeNull(); + expect(ep!.hasLoader).toBe(true); + expect(ep!.statementCount).toBe(1); + }); + + it("reports both `export const loader` and a separate `export { action }`", () => { + const ep = scanFile( + "api.v1.both.ts", + ` + export const loader = createLoaderApiRoute({}, async () => json({})); + const { action } = createActionApiRoute({}, async ({ body }) => { + const x = body.x; + return json({ x }); + }); + export { action }; + ` + ); + expect(ep).not.toBeNull(); + expect(ep!.hasLoader).toBe(true); + expect(ep!.hasAction).toBe(true); + expect(ep!.actionInitializerCallee).toBe("createActionApiRoute"); + }); + + it("resolves an export assigned from a property of a local builder result", () => { + const ep = scanFile( + "api.v1.errors.$errorId.ignore.ts", + ` + const route = createActionApiRoute({ method: "POST" }, async ({ body }) => { + const a = 1; + const b = 2; + return json({ a, b }); + }); + export const action = route.action; + export const loader = route.loader; + ` + ); + expect(ep!.hasAction).toBe(true); + expect(ep!.actionInitializerCallee).toBe("createActionApiRoute"); + // Both exports share one handler, so its statements are counted once. + expect(ep!.statementCount).toBe(3); + }); + + it("counts a `handler` property body but not the surrounding builder config lambdas", () => { + const ep = scanFile( + "engine.v1.dev.presence.ts", + ` + export const loader = createSSELoader({ + timeout: 1000, + findResource: async (params) => { + const a = 1; + const b = 2; + const c = 3; + return lookup(a, b, c); + }, + handler: async ({ request }) => { + const auth = await authenticate(request); + return stream(auth); + }, + }); + ` + ); + expect(ep!.statementCount).toBe(2); + expect(ep!.calleeNames).not.toContain("lookup"); + }); + + it("counts the per-method `methods.POST.handler` bodies", () => { + const ep = scanFile( + "api.v1.prompts.$slug.override.ts", + ` + const { action, loader } = createMultiMethodApiRoute({ + params: ParamsSchema, + methods: { + POST: { + body: CreateBody, + handler: async ({ body }) => { + const created = await create(body); + return json(created); + }, + }, + DELETE: { + handler: async ({ params }) => { + return json({ ok: true }); + }, + }, + }, + }); + export { action, loader }; + ` + ); + expect(ep!.statementCount).toBe(3); + expect(ep!.calleeNames).toContain("create"); + }); + + it("ignores a nested config callback that happens to be named `handler`", () => { + const ep = scanFile( + "api.v1.named-collision.ts", + ` + export const loader = build({ + onError: { + handler: async () => { + const a = 1; + const b = 2; + const c = 3; + return null; + }, + }, + handler: async () => json({}), + }); + ` + ); + expect(ep!.statementCount).toBe(1); + }); + + it("ignores a callback passed to a decorator further along the builder chain", () => { + const ep = scanFile( + "api.v1.chained.ts", + ` + export const loader = createLoaderApiRoute({}, async () => json({})).withCors(async () => { + const a = 1; + const b = 2; + return a + b; + }); + ` + ); + expect(ep!.loaderInitializerCallee).toBe("createLoaderApiRoute"); + expect(ep!.statementCount).toBe(1); + }); + + it("detects an action-only route", () => { + const ep = scanFile( + "api.v1.action-only.ts", + `export async function action() { return json({}); }` + ); + expect(ep).not.toBeNull(); + expect(ep!.hasAction).toBe(true); + expect(ep!.hasLoader).toBe(false); + }); + + it("does not crash on a re-export or a star export", () => { + expect(() => scanFile("re-export.ts", `export { loader } from "./other";`)).not.toThrow(); + expect(() => scanFile("star.ts", `export * from "./other";`)).not.toThrow(); + const ep = scanFile("re-export.ts", `export { loader } from "./other";`); + expect(ep!.hasLoader).toBe(true); + expect(ep!.loaderInitializerCallee).toBeNull(); + }); +}); + +describe("scanFile: statement counting", () => { + it("counts only the loader's statements, not a fat exported component's", () => { + const ep = scanFile( + "route.tsx", + ` + export async function loader() { + return json({}); + } + export default function Page() { + const a = 1; + const b = 2; + const c = 3; + const d = 4; + return null; + } + export function ErrorBoundary() { + const e = 1; + return null; + } + ` + ); + expect(ep!.statementCount).toBe(1); + }); + + it("counts through a try/catch wrapper rather than reporting 1", () => { + const ep = scanFile( + "otel.v1.traces.ts", + ` + export async function action({ request }) { + try { + const body = await request.arrayBuffer(); + const result = await process(body); + return json(result); + } catch (e) { + logger.error(e); + return json({ error: true }, { status: 500 }); + } + } + ` + ); + // try (1) + 3 in the try block + 2 in the catch block + expect(ep!.statementCount).toBe(6); + }); + + it("counts a same-file helper the body delegates to", () => { + const ep = scanFile( + "ph.$.ts", + ` + async function proxyToPostHog(request) { + const url = new URL(request.url); + try { + const upstream = await fetch(url); + return new Response(upstream.body); + } catch (e) { + logger.error(e); + return new Response(null, { status: 502 }); + } + } + export async function loader({ request }) { + return proxyToPostHog(request); + } + export async function action({ request }) { + return proxyToPostHog(request); + } + ` + ); + // loader (1) + action (1) + helper: const url, try (1) + 2 + 2 + expect(ep!.statementCount).toBe(8); + expect(ep!.hasTryCatch).toBe(true); + expect(ep!.calleeNames).toContain("proxyToPostHog"); + expect(ep!.calleeNames).toContain("fetch"); + }); + + it("counts a shared helper once when both the loader and the action delegate to it", () => { + const ep = scanFile( + "shared.ts", + ` + function work() { + const a = 1; + const b = 2; + return a + b; + } + export async function loader() { return work(); } + export async function action() { return work(); } + ` + ); + // loader (1) + action (1) + helper (3), the helper counted once + expect(ep!.statementCount).toBe(5); + }); + + it("follows a delegating helper one hop only", () => { + const ep = scanFile( + "two-hop.ts", + ` + function deep() { + const a = 1; + const b = 2; + const c = 3; + return a + b + c; + } + function shallow() { + return deep(); + } + export async function loader() { return shallow(); } + ` + ); + // loader (1) + shallow (1). `deep` is a second hop and is not counted. + expect(ep!.statementCount).toBe(2); + }); + + it("terminates on a recursive helper", () => { + const ep = scanFile( + "recursive.ts", + ` + function recurse(n) { + if (n <= 0) return 0; + return recurse(n - 1); + } + export async function loader() { return recurse(3); } + ` + ); + // loader (1) + recurse: if (1) + return (1) + return (1) + expect(ep!.statementCount).toBe(4); + }); + + it("does not count an imported helper it cannot resolve", () => { + const ep = scanFile( + "imported.ts", + ` + import { proxy } from "./proxy.server"; + export async function loader({ request }) { + return proxy(request); + } + ` + ); + expect(ep!.statementCount).toBe(1); + expect(ep!.hasTryCatch).toBe(false); + }); + + it("does not count a same-file function the body never calls", () => { + const ep = scanFile( + "unused-helper.ts", + ` + function unrelated() { + try { + const a = 1; + const b = 2; + return a + b; + } catch (e) { + return null; + } + } + export async function loader() { + return json({}); + } + ` + ); + expect(ep!.statementCount).toBe(1); + expect(ep!.hasTryCatch).toBe(false); + }); + + it("counts statements nested in if/for/while/switch blocks", () => { + const ep = scanFile( + "nested.ts", + ` + export async function loader() { + if (a) { + const x = 1; + doThing(x); + } + for (const i of list) { + use(i); + } + return json({}); + } + ` + ); + // if (1) + 2 nested + for (1) + 1 nested + return (1) + expect(ep!.statementCount).toBe(6); + }); +}); + +describe("scanFile: entry-point scoping", () => { + it("reports hasTryCatch false when the only try is in a non-entry-point export", () => { + const ep = scanFile( + "route.tsx", + ` + export async function loader() { + return json({}); + } + export default function Page() { + try { + render(); + } catch (e) { + report(e); + } + return null; + } + ` + ); + expect(ep!.hasLoader).toBe(true); + expect(ep!.hasTryCatch).toBe(false); + }); + + it("reports hasTryCatch true when the try is inside the loader", () => { + const ep = scanFile( + "route.tsx", + ` + export async function loader() { + try { + return json(await load()); + } catch (e) { + return json({}, { status: 500 }); + } + } + ` + ); + expect(ep!.hasTryCatch).toBe(true); + }); + + it("excludes callees invoked outside the loader/action body", () => { + const ep = scanFile( + "route.tsx", + ` + const schema = z.object({}); + export async function loader() { + const data = await fetchThings(); + return json(data); + } + export default function Page() { + useFancyHook(); + return null; + } + ` + ); + expect(ep!.calleeNames).toContain("fetchThings"); + expect(ep!.calleeNames).toContain("json"); + expect(ep!.calleeNames).not.toContain("useFancyHook"); + expect(ep!.calleeNames).not.toContain("object"); + }); +}); + +describe("scanFile: callee resolution", () => { + it("records the root callee of a chained builder call", () => { + const ep = scanFile( + "api.v1.cors.ts", + `export const loader = createLoaderApiRoute({}).withCors();` + ); + expect(ep!.loaderInitializerCallee).toBe("createLoaderApiRoute"); + }); + + it("leaves the callee null for a shape that cannot be named", () => { + const ep = scanFile("api.v1.anon.ts", `export const loader = async () => json({});`); + expect(ep!.hasLoader).toBe(true); + expect(ep!.loaderInitializerCallee).toBeNull(); + }); + + it("resolves a multi-level call and falls back to the bare name past an unnameable one", () => { + const ep = scanFile( + "api.v1.things.ts", + ` + export async function loader({ request }) { + try { + const org = await prisma.organization.findFirst({ where: { id: 1 } }); + return json(await new PromptService().createOverride(org)); + } catch (e) { + logger.error("nope", { error: e }); + return json({}, { status: 500 }); + } + } + ` + ); + // A three-level property chain still lands on its bare method name. + expect(ep!.calleeNames).toContain("findFirst"); + // A chain through a `new` expression has no name of its own, so this also falls back to the + // bare name rather than losing the call. + expect(ep!.calleeNames).toContain("createOverride"); + // The full path still builds where nothing unnameable sits in it, which is what `LogCall.callee` + // depends on. + expect(ep!.logCalls[0]!.callee).toBe("logger.error"); + }); +}); + +describe("scanFile: parse failures", () => { + it("throws on a malformed source rather than returning a clean entry point", () => { + expect(() => + scanFile("broken.ts", `export async function loader() { const a = ; return json(`) + ).toThrow(ParseFailureError); + }); + + // B8. The detection used to read `sf.parseDiagnostics`, an internal property. These are the + // shapes that prove the public route through `ts.Program` still sees a malformed file. + it("throws on an unclosed jsx element in a tsx route", () => { + expect(() => + scanFile( + "broken.route.tsx", + `export async function loader() { return json({}); } + export default function Page() { return
hi; }` + ) + ).toThrow(ParseFailureError); + }); + + it("throws on an unterminated template literal", () => { + expect(() => + scanFile("broken.ts", `export async function loader() { return \`unterminated; }`) + ).toThrow(ParseFailureError); + }); + + it("throws on a stray closing brace after a complete function", () => { + expect(() => + scanFile("broken.ts", `export async function loader() { return json({}); } }`) + ).toThrow(ParseFailureError); + }); + + it("names the diagnostic rather than reporting a bare failure", () => { + try { + scanFile("broken.ts", `export async function loader() { const a = ; }`); + expect.unreachable("scanFile should have thrown"); + } catch (error) { + expect(error).toBeInstanceOf(ParseFailureError); + expect((error as ParseFailureError).diagnostic.length).toBeGreaterThan(0); + } + }); + + // The change from `sf.parseDiagnostics` to a program-backed lookup is invisible to every test + // above: both spellings find the same malformed files today. What a compiler upgrade can break + // is the private one, and only a source-level guard can fail for that. + it("reads its diagnostics through public typescript api rather than a private field", () => { + const source = readFileSync(resolve(__dirname, "./scan.ts"), "utf8"); + expect(source).not.toContain("parseDiagnostics"); + expect(source).toContain("getSyntacticDiagnostics"); + }); + + it("does not throw on a well-formed tsx route", () => { + expect(() => + scanFile( + "route.tsx", + `export async function loader() { return json({}); } + export default function Page() { return
hi
; }` + ) + ).not.toThrow(); + }); +}); + +describe("scanDirectory", () => { + let dir: string; + + beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), "obs-map-scan-")); + }); + + afterEach(() => { + rmSync(dir, { recursive: true, force: true }); + }); + + it("recurses into route directories and keeps file names distinct", () => { + writeFileSync(join(dir, "a.ts"), `export async function loader() { return json({}); }`); + mkdirSync(join(dir, "nested.route")); + writeFileSync( + join(dir, "nested.route", "route.tsx"), + `export async function loader() { return json({}); }` + ); + mkdirSync(join(dir, "other.route")); + writeFileSync( + join(dir, "other.route", "route.tsx"), + `export async function action() { return json({}); }` + ); + + const { entryPoints, parseFailures } = scanDirectory(dir); + const names = entryPoints.map((ep) => ep.fileName).sort(); + + expect(parseFailures).toEqual([]); + expect(names).toEqual(["a.ts", "nested.route/route.tsx", "other.route/route.tsx"]); + }); + + it("records a malformed file as a parse failure instead of an entry point", () => { + writeFileSync( + join(dir, "broken.ts"), + `export async function loader() { const a = ; return json(` + ); + + const { entryPoints, parseFailures } = scanDirectory(dir); + + expect(entryPoints).toEqual([]); + expect(parseFailures).toHaveLength(1); + // The file name, then the diagnostic that made it a failure. + expect(parseFailures[0]).toMatch(/^broken\.ts: \S/); + }); + + // root ignores the mode bits, so the unreadable file would read fine. + it.skipIf(process.getuid?.() === 0)( + "rethrows an error that is not a parse failure instead of counting it as one", + () => { + const unreadable = join(dir, "unreadable.ts"); + writeFileSync(unreadable, `export async function loader() { return json({}); }`); + chmodSync(unreadable, 0o000); + + try { + expect(() => scanDirectory(dir)).toThrow(/EACCES|EPERM/); + } finally { + chmodSync(unreadable, 0o600); + } + } + ); + + it("skips a non-route file inside a route directory", () => { + mkdirSync(join(dir, "nested.route")); + writeFileSync( + join(dir, "nested.route", "route.tsx"), + `export async function loader() { return json({}); }` + ); + writeFileSync( + join(dir, "nested.route", "loaders.server.ts"), + `export async function loader() { return json({}); }` + ); + + const { entryPoints } = scanDirectory(dir); + + expect(entryPoints.map((ep) => ep.fileName)).toEqual(["nested.route/route.tsx"]); + }); + + it("scans a route file whose name ends in .test.ts but skips .d.ts", () => { + writeFileSync( + join(dir, "projects.v3.$projectRef.test.ts"), + `export async function loader() { return json({}); }` + ); + writeFileSync(join(dir, "types.d.ts"), `export declare const x: number;`); + + const { entryPoints } = scanDirectory(dir); + + expect(entryPoints.map((ep) => ep.fileName)).toEqual(["projects.v3.$projectRef.test.ts"]); + }); +}); + +describe("scanFile: catch clause evidence", () => { + it("sets rethrows on the clause when a catch rethrows", () => { + const ep = scanFile( + "rethrow.ts", + ` + export async function loader({ request }) { + try { + return json(await load(request)); + } catch (e) { + logger.error(e); + throw e; + } + } + ` + ); + expect(ep!.hasTryCatch).toBe(true); + expect(ep!.catches[0]!.rethrows).toBe(true); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + // A4. `rethrows` used to be set by any `ThrowStatement` in the clause, reachable or not, so a + // `throw e;` appended after a `return` flipped a swallowing catch from rethrows: false to true + // with no behavioural change, which read as inert instead of a swallow. + it("does not set rethrows for a throw that is dead code after a return", () => { + const ep = scanFile( + "dead-throw.ts", + ` + export async function loader({ request }) { + try { + return json(await load(request)); + } catch (e) { + return null; + throw e; + } + } + ` + ); + expect(ep!.catches[0]!.rethrows).toBe(false); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + // C2. `reachableStatements` only handled dead code from statement ordering (A4), not a + // statically-false condition, and `catchClauseEvidence`'s walk descended into every function + // body unconditionally, so a throw or an error test merely REGISTERED in a callback the clause + // constructs (never executed as part of the clause's own synchronous handling) was credited to + // it. All four shapes below must leave a plain swallow (`catch (e) { return null; }`) inert. + describe("dead and deferred code inside a catch does not count as evidence", () => { + const swallow = (mutation: string) => ` + export async function loader() { + try { + return await prisma.thing.findMany(); + } catch (e) { + ${mutation} + return null; + } + } + `; + + it("is inert as a baseline with no mutation", () => { + const ep = scanFile("x.ts", swallow("")); + expect(ep!.catches[0]).toMatchObject({ rethrows: false, branches: false }); + }); + + // Eleven shapes that put a `throw` somewhere it can never run, all of them found by review + // rather than by this suite. An earlier round recognised the first two by folding the literal + // `false` and lost to the other nine. None of them is named in the rule now: a throw counts + // when it is unconditional, and every one of these is guarded by something. There is no claim + // that the list is complete, and a twelfth family arrived the round after it was written, see + // `dead throw written after something that already exited`. `dead-*` in the mutation corpus + // runs the same list over the whole route tree. + const DEAD_SHAPES: Array<[string, string]> = [ + ["if (false)", "if (false) { throw e; }"], + ["while (false)", "while (false) { throw e; }"], + ["for (;false;)", "for (;false;) { throw e; }"], + ["if (true) else", "if (true) { doThing(); } else { throw e; }"], + ["switch with no matching case", "switch (1) { case 2: throw e; }"], + ["inner try/catch", "try { doThing(); } catch { throw e; }"], + ["for...of an empty array", "for (const item of []) { throw e; }"], + ["for...in an empty object", "for (const key in {}) { throw e; }"], + ["if on an empty string", 'if ("") { throw e; }'], + ["if on a negated literal", "if (!true) { throw e; }"], + ["if on a constant comparison", "if (1 === 2) { throw e; }"], + ]; + + for (const [label, shape] of DEAD_SHAPES) { + it(`does not set rethrows for a throw inside ${label}`, () => { + const ep = scanFile("x.ts", swallow(shape)); + expect(ep!.catches[0]).toMatchObject({ rethrows: false, branches: false }); + }); + } + + it("does not set rethrows for a throw merely registered in a constructed callback", () => { + const ep = scanFile("x.ts", swallow("queue.push(() => { throw e; });")); + expect(ep!.catches[0]).toMatchObject({ rethrows: false, branches: false }); + }); + + it("does not set branches for an error test merely registered in a constructed callback", () => { + const ep = scanFile( + "x.ts", + swallow("queue.push(() => { if (e instanceof Error) { doThing(); } });") + ); + expect(ep!.catches[0]).toMatchObject({ rethrows: false, branches: false }); + }); + + // Extra input beyond the brief's four, exercising the same "merely registered" mechanism with + // a different callback-taking call, to check the fix is not scoped to `.push` specifically. + it("does not set rethrows for a throw registered in a setTimeout callback", () => { + const ep = scanFile("x.ts", swallow("setTimeout(() => { throw e; }, 0);")); + expect(ep!.catches[0]).toMatchObject({ rethrows: false, branches: false }); + }); + + // Positive control: a do/while runs its body at least once regardless of the trailing + // condition, so a throw in one is genuinely unconditional and must still register. This checks + // the while(false) fix was not implemented broadly enough to swallow a real rethrow too. + it("still sets rethrows for a throw in a do/while, which runs its body once regardless", () => { + const ep = scanFile("x.ts", swallow("do { throw e; } while (false);")); + expect(ep!.catches[0]!.rethrows).toBe(true); + }); + }); + + // The mirror of the family above. Each dead spelling earns nothing, and it must also COST + // nothing: a plain containment read was true of the dead statement itself, so prepending one raised the + // `exited` flag and blinded the walk to the real classification below it, turning a pass into a + // swallow verdict on 78 real routes. `containsLiveExit` folds the literal guard and sees no live + // exit, so the deciding statements keep their credit. The spellings are the CORPUS spellings + // from `dead-*` in `mutations.ts`, not the `DEAD_SHAPES` table's: that table's inner-try twin is + // `try { doThing(); } catch { throw e; }`, which is NOT provably dead (`doThing` may throw and + // the rethrow then runs), so conservatively raising the flag after it is correct and it gets no + // twin here. + describe("dead and deferred code prepended to a deciding catch does not blind it", () => { + const deciding = (mutation: string) => ` + export async function loader() { + try { + return await prisma.thing.findMany(); + } catch (e) { + ${mutation} + if (e instanceof Error) { return new Response(null, { status: 400 }); } + return new Response(null, { status: 500 }); + } + } + `; + + it("decides as a baseline with no mutation", () => { + const ep = scanFile("x.ts", deciding("")); + expect(ep!.catches[0]!.branches).toBe(true); + }); + + const DEAD_PREPENDS: Array<[string, string]> = [ + ["if (false)", "if (false) { throw e; }"], + ["while (false)", "while (false) { throw e; }"], + ["for (;false;)", "for (;false;) { throw e; }"], + ["if (true) else", "if (true) { 0; } else { throw e; }"], + ["switch with no matching case", "switch (1) { case 2: throw e; }"], + ["an inner try over a literal", "try { 0; } catch { throw e; }"], + ["for...of an empty array", "for (const obsMapItem of []) { throw e; }"], + ["for...in an empty object", "for (const obsMapKey in {}) { throw e; }"], + ["if on an empty string", 'if ("") { throw e; }'], + ["if on a negated literal", "if (!true) { throw e; }"], + ["if on a constant comparison", "if (1 === 2) { throw e; }"], + ]; + + for (const [label, shape] of DEAD_PREPENDS) { + it(`keeps branches true past a dead throw inside ${label}`, () => { + const ep = scanFile("x.ts", deciding(shape)); + expect(ep!.catches[0]!.branches).toBe(true); + }); + } + + // The composed shape: the dead if wrapped in a bare block, which the walk enters. The block's + // own live-exit read has to fold too, or entering it re-raises the flag the fold lowered. + it("keeps branches true past a dead throw in a block around an if (false)", () => { + const ep = scanFile("x.ts", deciding("{ if (false) { throw e; } }")); + expect(ep!.catches[0]!.branches).toBe(true); + }); + + // The returns half. A dead `return null;` must not veto the rethrow: `containsReturn` saw the + // return token inside `if (false)` and turned a rethrow-only clause into a swallow verdict, + // which regressed 11 real routes from not-applicable to fail. `dead-if-false-return` in the + // mutation corpus is the tree-scale version. + it("still sets rethrows past a dead return in an if (false) arm", () => { + const ep = scanFile( + "x.ts", + `export async function loader() { + try { return await prisma.thing.findMany(); } + catch (e) { + if (false) { return null; } + throw e; + } + }` + ); + expect(ep!.catches[0]!.rethrows).toBe(true); + }); + + // Negative controls: the fold only withholds blindness, it must never withhold refusal. + // An always-true guard really can run its throw, so the error test after it stays dead. + it("still refuses an error test after an always-true spelling that throws", () => { + const ep = scanFile( + "x.ts", + `export async function loader() { + try { return await prisma.thing.findMany(); } + catch (e) { + if (!false) { throw e; } + if (e instanceof Error) { return new Response(null, { status: 400 }); } + } + }` + ); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + // The fall-through slice: `case 1` matches and runs on into `case 2`, so the return is live + // and vetoes the rethrow. Misreading the slice as dead would blind the returns veto, which is + // the direction that hands out credit. + it("reads a switch fall-through onto a live return as live", () => { + const ep = scanFile( + "x.ts", + `export async function loader() { + try { return await prisma.thing.findMany(); } + catch (e) { + switch (1) { case 1: 0; case 2: return null; } + throw e; + } + }` + ); + expect(ep!.catches[0]!.rethrows).toBe(false); + }); + }); + + // The walk may enter a construct exactly where the entered statements are guaranteed to execute + // whenever the clause body runs. Before these entries existed, relocating a clause's own + // statements inside `if (true)`, a switch default, an if/else or a try/finally put the branch + // evidence out of reach while the returns veto still saw the return, so a deciding clause read + // as a swallow: 83 real routes regressed per corpus entry. Each identity pair here holds the + // wrapped and unwrapped spellings to the same evidence; `checks/index.test.ts` holds them to the + // same verdict. + describe("the walk enters exactly the positions guaranteed to execute", () => { + const clauseEvidence = (body: string) => { + const ep = scanFile( + "x.ts", + `export async function loader() { + try { return await prisma.thing.findMany(); } + catch (e) { + ${body} + } + }` + ); + return ep!.catches[0]!; + }; + + const DECIDING = + "if (e instanceof KnownError) { return new Response(e.code, { status: 400 }); }\n" + + "return new Response(null, { status: 500 });"; + const RETHROW = "logger.error(e);\nthrow e;"; + + const WRAPPERS: Array<[string, (body: string) => string]> = [ + ["a catchless try/finally", (body) => `try {\n${body}\n} finally { }`], + ["a single-default switch", (body) => `switch (pick()) { default: {\n${body}\n} }`], + ["an if (true)", (body) => `if (true) {\n${body}\n}`], + [ + "an if/else with the body in both arms", + (body) => `if (pick()) {\n${body}\n} else {\n${body}\n}`, + ], + ]; + + for (const [label, wrap] of WRAPPERS) { + it(`reads a deciding clause wrapped in ${label} identically`, () => { + expect(clauseEvidence(wrap(DECIDING))).toEqual(clauseEvidence(DECIDING)); + }); + + it(`reads a rethrowing clause wrapped in ${label} identically`, () => { + expect(clauseEvidence(wrap(RETHROW))).toEqual(clauseEvidence(RETHROW)); + }); + } + + // The switch entry is exact: one clause and it is a default. Anything else is not entered and + // keeps its top-level treatment, which is what the identity pair below and the `break and + // continue inside the construct they target` family already pin for the multi-clause shapes. + it("reads a clause wrapped in a single-default switch as the bare clause", () => { + expect(clauseEvidence(`switch (1) { default: {\n${DECIDING}\n} }`)).toEqual( + clauseEvidence(DECIDING) + ); + }); + + // Intersection, not union. One arm running is a condition, not a guarantee, so evidence in a + // single arm earns nothing; `dead-classifier-one-arm` in the mutation corpus is the tree-scale + // twin with the arm provably dead. + it("does not credit a classifier that sits in one arm only", () => { + const evidence = clauseEvidence( + "if (pick()) { if (e instanceof Error) { return new Response(null, { status: 400 }); } } else { 0; }\n" + + "return null;" + ); + expect(evidence.branches).toBe(false); + }); + + it("does not credit a classifier in a dead arm beside an inert arm", () => { + const evidence = clauseEvidence( + "if (false) { if (e instanceof Error) { return new Response(null, { status: 400 }); } } else { 0; }\n" + + "return null;" + ); + expect(evidence).toMatchObject({ branches: false, rethrows: false, throws: false }); + }); + + // The else-arm under a literal-true guard can never run, so nothing in it is evidence: no + // rethrow minted, and the deciding statements after the wrapper keep their credit. + it("reads a dead else arm under if true as contributing nothing", () => { + const evidence = clauseEvidence(`if (true) { 0; } else { throw e; }\n${DECIDING}`); + expect(evidence).toMatchObject({ throws: false, branches: true }); + }); + + // A throw in the tryBlock of a CAUGHT try never escapes the clause: the nested catch takes + // it. Crediting it as a rethrow would launder a returnless swallow into not-applicable. + it("does not read the tryBlock of a caught try as this clause's rethrow", () => { + const evidence = clauseEvidence("try { throw e; } catch {}\nlogger.error(e);"); + expect(evidence).toMatchObject({ rethrows: false, throws: false }); + }); + + // The walk does not enter a finally block, so the returns veto must still read it off the + // whole statement: this clause's finally return eats the throw, and the error never leaves. + it("reads a try whose finally returns as swallowing, not rethrowing", () => { + const evidence = clauseEvidence("try { throw e; } finally { return null; }"); + expect(evidence.rethrows).toBe(false); + }); + + // A finally that leaves itself by `break` or `continue` cancels the try's completion the same + // way a finally return does, so a throw in that tryBlock never escapes the clause. Crediting + // it made `do { try { throw e; } finally { break; } } while (false);` a no-op that minted + // rethrows, and its classifier-hosting variant minted branches on 80 real routes; + // `dead-throw-in-cancelled-try` in the mutation corpus is the tree-scale twin. + it("reads a throw a finally break discards as no rethrow", () => { + const evidence = clauseEvidence( + "do { try { throw e; } finally { break; } } while (false);\nlogger.error(e);" + ); + expect(evidence).toMatchObject({ rethrows: false, throws: false, branches: false }); + }); + + it("reads a throw a finally continue discards as no rethrow", () => { + const evidence = clauseEvidence( + 'do { try { throw e; } finally { continue; } } while (false);\nlogger.error("x", { e });' + ); + expect(evidence).toMatchObject({ rethrows: false, throws: false, branches: false }); + }); + + it("reads a throw a switch-hosted finally break discards as no rethrow", () => { + const evidence = clauseEvidence( + "switch (0) { default: try { throw e; } finally { break; } }\nlogger.error(e);" + ); + expect(evidence).toMatchObject({ rethrows: false, throws: false, branches: false }); + }); + + // The refusal is a containment read: a jump that only MAY run still cancels entry, because + // entry grants credit and a wrong grant pays. + it("refuses the tryBlock when the finally only may break", () => { + const evidence = clauseEvidence( + "do { try { throw e; } finally { if (pick()) { break; } } } while (false);\nlogger.error(e);" + ); + expect(evidence).toMatchObject({ rethrows: false, throws: false }); + }); + + // A loop inside the finally captures its own bare jumps, so nothing there leaves the finally + // and the try's completion stands: the rethrow is genuine. + it("does not refuse a finally whose loop captures its own break", () => { + const evidence = clauseEvidence("try { throw e; } finally { while (pick()) { break; } }"); + expect(evidence).toMatchObject({ rethrows: true, throws: true }); + }); + + // The cancelled statement contributes nothing, in either direction: no credit from inside it, + // and no blinding of the real classification after it. Same evidence as the bare clause. + it("keeps the classification after a finally-break no-op", () => { + expect( + clauseEvidence( + `do { try { if (e instanceof Error) { throw e; } } finally { break; } } while (false);\n${DECIDING}` + ) + ).toEqual(clauseEvidence(DECIDING)); + }); + + // The `definitelyExits` fold: `if (true) { X }` definitely exits iff X does, so the trailing + // throw is cut rather than read. Without the fold the throw still walks and mints `throws`. + it("cuts a dead trailing statement after an if true that exits", () => { + const evidence = clauseEvidence("if (true) { return null; }\nthrow e;"); + expect(evidence.throws).toBe(false); + }); + }); + + // S1. The other end of the same problem. `reachableStatements` used to cut the statement list + // only on a BARE `return`/`throw`, while the walk descended into blocks and `do` bodies, so a + // `throw e;` written after a nested construct that had already returned was still read as the + // clause rethrowing. Every one of these takes a swallow from `fail` to `not-applicable`, worth 50 + // points a route, and they are semantics-preserving because the throw cannot run. + // + // Two rules answer them together. `definitelyExits` sees through the block, the `do` and the + // `if`/`else`, the `switch` and the `try`/`finally`; the `if (true)` form needs constant folding + // that this file deliberately does not do, and is answered instead by `rethrows` requiring the + // clause to contain no reachable `return` at all. `dead-throw-after-*` in the mutation corpus + // runs all six over the whole route tree. + describe("dead throw written after something that already exited", () => { + const exiting = (wrapped: string) => ` + export async function loader() { + try { + return await prisma.thing.findMany(); + } catch (e) { + ${wrapped} + throw e; + } + } + `; + + const EXITED: Array<[string, string]> = [ + ["a bare block", "{ logger.error(e); return null; }"], + ["a do body", "do { return null; } while (false);"], + ["an if (true)", "if (true) { return null; }"], + ["an if/else where both arms return", "if (pick()) { return null; } else { return 0; }"], + ["a switch with a returning default", "switch (1) { default: return null; }"], + ["a try/finally that returns", "try { return null; } finally { }"], + ]; + + for (const [label, wrapped] of EXITED) { + it(`does not set rethrows for a throw after ${label}`, () => { + const ep = scanFile("x.ts", exiting(wrapped)); + expect(ep!.catches[0]!.rethrows).toBe(false); + }); + } + + // The same wrappers on the branches side, plus three the rethrow list has no use for. An error + // test written after a statement that could already have left the clause is dead code and must + // not read as the clause deciding anything. + // + // This asks a weaker question than `definitelyExits` does, and on purpose: "could this have + // exited", not "must it have". That is why `if (true)`, a labelled block, a `for...of` and a + // `while` are all on the list even though none of them is guaranteed to run its body. The flag + // is read through `containsLiveExit`, which folds LITERAL guards only, so every wrapper here + // still counts: `if (true)` descends its then-arm and finds the return, and a loop guarded by + // an identifier keeps the plain containment answer. What the fold withholds is the provably + // dead statement raising the flag itself, which is the twin family in `dead and deferred code + // prepended to a deciding catch does not blind it`. + // + // The ordering is the whole trick and it is easy to get backwards. The flag is raised at the + // END of each statement, after that statement's own branch check. Raising it first makes every + // deciding statement refuse itself, because `if (e instanceof X) return y` contains an exit by + // definition; that variant was measured against the pre-round-C tree and it takes it from 15 to 6, accusing 78 + // routes. This one leaves the real-tree report and all 240 clauses' evidence byte-identical. + const BRANCH_EXITED: Array<[string, string]> = [ + ...EXITED, + ["a labelled block", "outer: { return null; }"], + ["a for...of that returns", "for (const q of items) { return q; }"], + ["a while that returns", "while (go) { return null; }"], + ]; + + for (const [label, wrapped] of BRANCH_EXITED) { + it(`does not credit an error test written after ${label}`, () => { + const ep = scanFile( + "x.ts", + `export async function loader() { + try { return await prisma.thing.findMany(); } + catch (e) { + ${wrapped} + if (e instanceof Error) { return json({ a: 1 }); } + } + }` + ); + expect(ep!.catches[0]!.branches).toBe(false); + }); + } + + // The precision this gives up, pinned so it is a decision and not a surprise: a conditional + // exit before the error test also stops the credit, because the walk cannot tell a guard that + // usually falls through from one that always leaves. No clause in the route tree is this shape, + // which is why the report is byte-identical, but one could be written tomorrow. + it("does not credit an error test written after a conditional return", () => { + const ep = scanFile( + "x.ts", + `export async function loader() { + try { return await prisma.thing.findMany(); } + catch (e) { + if (rare) { return null; } + if (e instanceof Error) { return json({ a: 1 }); } + } + }` + ); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + it("still credits an error test with nothing exiting before it", () => { + const ep = scanFile( + "x.ts", + `export async function loader() { + try { return await prisma.thing.findMany(); } + catch (e) { + logger.error(e); + if (e instanceof Error) { return json({ a: 1 }); } + } + }` + ); + expect(ep!.catches[0]!.branches).toBe(true); + }); + + // Positive control: nothing before the throw exits, so the throw is real and the clause has no + // other way out. + it("still sets rethrows when nothing before the throw returns", () => { + const ep = scanFile("x.ts", exiting("logger.error(e);")); + expect(ep!.catches[0]!.rethrows).toBe(true); + }); + + // Second positive control, for the no-return half specifically: a `return` the walk has already + // cut as dead must not count against the rethrow. + it("still sets rethrows when the only return is dead code after the throw", () => { + const ep = scanFile( + "x.ts", + `export async function loader() { + try { return await prisma.thing.findMany(); } + catch (e) { throw e; return null; } + }` + ); + expect(ep!.catches[0]!.rethrows).toBe(true); + }); + }); + + // S3. `definitelyExits` counted a bare `break` and a bare `continue` wherever it found one, and + // both of those target the nearest enclosing construct of their kind rather than the statement + // list the question is about. A `switch` whose clauses all break falls through to the statement + // written after it, so cutting that statement as unreachable accused a route of swallowing an + // error it rethrows, with a detail line saying it "takes one way out regardless of what was + // thrown" about a clause that takes the same way out it arrived by. This is the false-accusation + // direction, so both halves are pinned: what must now stay reachable, and what must still be cut. + describe("break and continue inside the construct they target", () => { + const clause = (body: string) => ` + export async function loader() { + try { + return await prisma.thing.findMany(); + } catch (e) { + ${body} + } + } + `; + + // Reachable, so the throw after them is a real rethrow. Each jump targets the construct it is + // written in, and every one of these constructs falls through to the next statement. + const FALLS_THROUGH: Array<[string, string]> = [ + [ + "a switch whose clauses all break", + 'switch (e.code) { case "P2025": handleNotFound(); break; default: handleOther(); break; }', + ], + ["a switch whose default is a bare break", "switch (e.code) { default: break; }"], + ["a do body that breaks", "do { break; } while (false);"], + ["a do body that continues", "do { continue; } while (false);"], + [ + "a do body whose if/else both break", + "do { if (pick()) { break; } else { break; } } while (false);", + ], + ]; + + for (const [label, wrapped] of FALLS_THROUGH) { + it(`still sets rethrows for a throw written after ${label}`, () => { + const ep = scanFile("x.ts", clause(`${wrapped}\nthrow e;`)); + expect(ep!.catches[0]!.rethrows).toBe(true); + }); + + it(`still credits an error test written after ${label}`, () => { + const ep = scanFile( + "x.ts", + clause(`${wrapped}\nif (e instanceof Error) { return json({ a: 1 }); }\nthrow e;`) + ); + expect(ep!.catches[0]!.branches).toBe(true); + }); + } + + // The clause the whole finding was about, end to end: sorting the error by code and then + // rethrowing is a rethrow, which is `not-applicable`, and never a swallow. + it("reads a switch on the error code followed by a rethrow as a rethrow", () => { + const ep = scanFile( + "x.ts", + clause( + 'switch (e.code) { case "P2025": handleNotFound(); break; default: handleOther(); break; }\nthrow e;' + ) + ); + expect(ep!.catches[0]).toMatchObject({ rethrows: true, throws: true, branches: false }); + }); + + // Cut, so the throw after them is dead and must not be credited. The first two are the + // over-correction control: a clause that returns and also breaks still exits, and reading the + // break as "no exit" would take the whole `dead-throw-after-*` family back. + const EXITS: Array<[string, string]> = [ + [ + "a switch clause that returns before it breaks", + "switch (1) { default: { return null; } break; }", + ], + [ + "a switch whose every clause returns", + "switch (e.code) { case 1: return null; default: return 0; }", + ], + ["a do body that returns before it breaks", "do { return null; break; } while (false);"], + ]; + + for (const [label, wrapped] of EXITS) { + it(`does not set rethrows for a throw written after ${label}`, () => { + const ep = scanFile("x.ts", clause(`${wrapped}\nthrow e;`)); + expect(ep!.catches[0]!.rethrows).toBe(false); + }); + } + + // A `continue` in a switch clause targets the enclosing loop, not the switch, so it is + // inherited through the clause rather than dropped with the `break`. Dropping it would leave + // the throw below reachable, and it is not: the continue goes to the `do`'s condition. + // The labelled jumps beside it leave the `for` entirely, so they are exits wherever they are + // written. The bare `break` is the control that separates the three. + const IN_LOOP: Array<[string, boolean]> = [ + ["break outer", false], + ["continue outer", false], + ["continue", false], + ["break", true], + ]; + + for (const [jump, rethrows] of IN_LOOP) { + it(`reads a switch clause that says ${jump} inside a labelled loop as rethrows=${rethrows}`, () => { + const ep = scanFile( + "x.ts", + `export async function loader() { + outer: for (const x of items) { + try { await service.call(x); } + catch (e) { + switch (e.code) { case 1: ${jump}; default: ${jump}; } + throw e; + } + } + return null; + }` + ); + expect(ep!.catches[0]!.rethrows).toBe(rethrows); + }); + } + }); + + // S2. A clause whose try block cannot throw is unreachable, so it is not error handling and + // nothing should be read off it. Crediting one was the largest hole ever found here: prepending + // this to a body takes the real tree from 19 to 44 and raises 224 routes, because the routes + // that catch nothing sat at `not-applicable` and a dead clause moved every one of them to `pass`. + // `dead-classifying-try` in the mutation corpus is the tree-scale version. + describe("a catch over a try block that cannot throw", () => { + const guarding = (guarded: string) => ` + export async function loader({ request }) { + try { ${guarded} } catch (e) { + if (e instanceof Error) { return new Response(null, { status: 400 }); } + throw e; + } + return await prisma.thing.findMany(); + } + `; + + const INERT: Array<[string, string]> = [ + ["an empty block", ""], + ["a literal expression statement", "0;"], + ["a literal declaration", "const x = 1;"], + ["arithmetic on literals", "const x = 1 + 2 * 3;"], + ["a bare identifier read", "const x = someLocal;"], + ]; + + for (const [label, guarded] of INERT) { + it(`is not read as error handling when the try holds only ${label}`, () => { + const ep = scanFile("x.ts", guarding(guarded)); + expect(ep!.catches[0]!.guardCanRaise).toBe(false); + }); + } + + // Positive controls, one per reason `canRaise` recognises, so the predicate is not passing the + // cases above by being false for everything. + const LIVE: Array<[string, string]> = [ + ["a call", "doThing();"], + ["a construction", "new Thing();"], + ["an await", "await later;"], + ["a member access", "const x = thing.value;"], + ["an element access", "const x = thing[0];"], + ["a throw", "throw new Error('x');"], + ["an iteration", "for (const item of items) { }"], + ["an instanceof", "const x = thing instanceof Error;"], + ]; + + for (const [label, guarded] of LIVE) { + it(`is read as error handling when the try holds ${label}`, () => { + const ep = scanFile("x.ts", guarding(guarded)); + expect(ep!.catches[0]!.guardCanRaise).toBe(true); + }); + } + }); + + it("leaves both flags false when the catch only returns", () => { + const ep = scanFile( + "swallow.ts", + ` + export async function loader({ request }) { + try { + return json(await load(request)); + } catch (e) { + return null; + } + } + ` + ); + expect(ep!.hasTryCatch).toBe(true); + expect(ep!.catches[0]!.rethrows).toBe(false); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + it("sets branches for an `if` on the error", () => { + const ep = scanFile( + "branch-if.ts", + ` + export async function loader({ request }) { + try { + return json(await request.json()); + } catch (e) { + if (e instanceof SyntaxError) { + return json({ error: "bad json" }, { status: 400 }); + } + return json({ error: "failed" }, { status: 500 }); + } + } + ` + ); + expect(ep!.catches[0]!.branches).toBe(true); + expect(ep!.catches[0]!.rethrows).toBe(false); + }); + + it("sets branches for an instanceof conditional that is the whole returned expression", () => { + const ep = scanFile( + "branch-instanceof.ts", + ` + export async function loader({ request }) { + try { + return json(await load(request)); + } catch (e) { + return e instanceof Response ? e : json({}, { status: 500 }); + } + } + ` + ); + expect(ep!.catches[0]!.branches).toBe(true); + }); + + it("sets branches for a switch on the error", () => { + const ep = scanFile( + "branch-switch.ts", + ` + export async function loader({ request }) { + try { + return json(await load(request)); + } catch (e) { + switch (e.code) { + case "P2025": + return json({}, { status: 404 }); + default: + return json({}, { status: 500 }); + } + } + } + ` + ); + expect(ep!.catches[0]!.branches).toBe(true); + }); + + it("ignores a catch that lives in the React component", () => { + const ep = scanFile( + "route.tsx", + ` + export async function loader() { + return json({}); + } + export default function Page() { + try { + render(); + } catch (e) { + if (e instanceof RenderError) throw e; + return null; + } + } + ` + ); + expect(ep!.hasTryCatch).toBe(false); + expect(ep!.catches).toEqual([]); + }); + + it("reads a catch inside a same-file helper the body delegates to", () => { + const ep = scanFile( + "ph.$.ts", + ` + async function proxy(request) { + try { + return await fetch(request.url); + } catch (e) { + if (e.name === "AbortError") throw e; + return new Response(null, { status: 502 }); + } + } + export async function loader({ request }) { + return proxy(request); + } + ` + ); + expect(ep!.hasTryCatch).toBe(true); + // The `throw e` here is guarded by an `if`, so it is not on the clause's straight-line path and + // does not read as a rethrow. The `if` itself does: it reads the binding and one arm throws, so + // the clause decides. The verdict the checks care about is unchanged. + expect(ep!.catches[0]!.rethrows).toBe(false); + expect(ep!.catches[0]!.branches).toBe(true); + }); + + it("leaves catches empty for a try with no catch", () => { + const ep = scanFile( + "finally-only.ts", + ` + export async function loader() { + try { + return json(await load()); + } finally { + release(); + } + } + ` + ); + expect(ep!.hasTryCatch).toBe(true); + expect(ep!.catches).toEqual([]); + }); + + it("flags a try that guards a single request.json()", () => { + const ep = scanFile( + "admin.api.v1.platform-notifications.ts", + ` + export async function action({ request }) { + const user = await requireUser(request); + let body; + try { + body = await request.json(); + } catch { + return json({ error: "Invalid JSON body" }, { status: 400 }); + } + const result = await createPlatformNotification(body); + return json(result); + } + ` + ); + expect(ep!.hasTryCatch).toBe(true); + expect(ep!.catches[0]!.rethrows).toBe(false); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + it("is empty when there is no try at all", () => { + const ep = scanFile("plain.ts", `export async function loader() { return json({}); }`); + expect(ep!.hasTryCatch).toBe(false); + expect(ep!.catches).toEqual([]); + }); + + // A7 / C1. The first fix stopped at ANY function-like node, purely lexical, which correctly + // excludes a per-item `.map()` boundary but also deletes the route's own catch when the whole + // body is wrapped in a single-shot callback: `trace(async () => { ...whole body... })`, + // `mutateWithFallback({ pgMutation: async (t) => {...} })`, `new ReadableStream({ start: async (c) + // => {...} })`. All three invoke their callback exactly once, as the route's own continuation. + // The real distinction is per-item iteration versus everything else, so only a callback passed to + // `map`/`forEach`/`filter`/`reduce`/`reduceRight`/`flatMap`/`some`/`every` is a boundary now. + describe("inline single-shot wrappers are attributed to the route", () => { + it("attributes a catch wrapped in trace(async () => {...})", () => { + const ep = scanFile( + "x.ts", + ` + export async function action({ request }) { + return trace("update", async () => { + try { + await doWork(request); + return json({ ok: true }); + } catch { + return json({ error: "failed" }, { status: 500 }); + } + }); + } + ` + ); + expect(ep!.catches).toHaveLength(1); + }); + + it("attributes a catch inside a pgMutation callback passed as an object property", () => { + const ep = scanFile( + "x.ts", + ` + export async function action({ request }) { + const outcome = await mutateWithFallback({ + pgMutation: async (taskRun) => { + try { + await doWork(taskRun); + } catch { + return json({ error: "Internal Server Error" }, { status: 500 }); + } + }, + }); + return outcome; + } + ` + ); + expect(ep!.catches).toHaveLength(1); + }); + + it("attributes a catch inside new ReadableStream({ start })", () => { + const ep = scanFile( + "x.ts", + ` + export async function action({ request }) { + const stream = new ReadableStream({ + async start(controller) { + try { + await doWork(controller); + } catch { + controller.error("failed"); + } + }, + }); + return new Response(stream); + } + ` + ); + expect(ep!.catches).toHaveLength(1); + }); + }); + + describe("per-item iteration callbacks are not attributed to the route", () => { + it("does not attribute a catch inside items.map(...)", () => { + const ep = scanFile( + "x.ts", + ` + export async function action({ request }) { + const items = await load(request); + return items.map((item) => { + try { + return process(item); + } catch { + return null; + } + }); + } + ` + ); + expect(ep!.catches).toEqual([]); + }); + + it("does not attribute a catch inside Promise.all(items.map(...))", () => { + const ep = scanFile( + "batch.process.ts", + ` + export async function action({ request }) { + const items = await loadItems(request); + await Promise.all( + items.map(async (item) => { + try { + await processItem(item); + } catch { + return null; + } + }) + ); + return json({ ok: true }); + } + ` + ); + expect(ep!.catches).toEqual([]); + // A try/catch appeared somewhere in the body, which is still a real signal for triviality. + expect(ep!.hasTryCatch).toBe(true); + }); + + it("does not attribute a catch inside items.forEach(...)", () => { + const ep = scanFile( + "x.ts", + ` + export async function action({ request }) { + const items = await load(request); + items.forEach((item) => { + try { + process(item); + } catch { + return; + } + }); + return json({ ok: true }); + } + ` + ); + expect(ep!.catches).toEqual([]); + }); + + // A refused catch keeps its evidence, built by the same machinery as an own catch, so + // `error-classification` can judge what it does rather than where it sits. Both flavours are + // pinned: the deciding per-item catch and the inert one. + it("populates evidence for a refused per-item catch that decides", () => { + const ep = scanFile( + "x.ts", + ` + export async function action({ request }) { + const items = await load(request); + return items.map(async (item) => { + try { + return await process(item); + } catch (e) { + if (e instanceof KnownError) { return new Response(e.code, { status: 400 }); } + throw e; + } + }); + } + ` + ); + expect(ep!.catches).toEqual([]); + expect(ep!.callbackCatches).toHaveLength(1); + expect(ep!.callbackCatches[0]).toMatchObject({ branches: true, guardCanRaise: true }); + }); + + it("populates evidence for a refused per-item catch that only rethrows", () => { + const ep = scanFile( + "x.ts", + ` + export async function action({ request }) { + const items = await load(request); + return items.map(async (item) => { + try { + return await process(item); + } catch (e) { + throw e; + } + }); + } + ` + ); + expect(ep!.catches).toEqual([]); + expect(ep!.callbackCatches).toHaveLength(1); + expect(ep!.callbackCatches[0]).toMatchObject({ rethrows: true, branches: false }); + }); + }); +}); + +describe("scanFile: log calls", () => { + it("records the fields of a log call's object argument", () => { + const ep = scanFile( + "logging.ts", + ` + export async function loader({ request }) { + try { + return json(await load(request)); + } catch (e) { + logger.error("load failed", { environmentId: env.id, error: e }); + return json({}, { status: 500 }); + } + } + ` + ); + expect(ep!.logCalls).toHaveLength(1); + expect(ep!.logCalls[0]).toEqual({ + callee: "logger.error", + fields: ["environmentId", "error"], + inCatch: true, + }); + }); + + it("records a log call with no object argument, outside a catch", () => { + const ep = scanFile( + "logging-plain.ts", + ` + export async function loader() { + log.info("starting"); + return json({}); + } + ` + ); + expect(ep!.logCalls).toEqual([{ callee: "log.info", fields: [], inCatch: false }]); + }); + + it("ignores a non-logger call and a log call in the React component", () => { + const ep = scanFile( + "route.tsx", + ` + export async function loader() { + return json(await load()); + } + export default function Page() { + logger.debug("rendered", { runId: 1 }); + return null; + } + ` + ); + expect(ep!.logCalls).toEqual([]); + }); +}); + +describe("scanFile: per-catch evidence", () => { + it("records one entry per catch clause, keeping a parse guard distinct from a broad catch", () => { + const ep = scanFile( + "two-catches.ts", + ` + export async function action({ request }) { + let body; + try { + body = await request.json(); + } catch { + return json({}, { status: 400 }); + } + try { + const run = await find(body.id); + const updated = await update(run); + await notify(updated); + return json(updated); + } catch (e) { + return json({}, { status: 500 }); + } + } + ` + ); + expect(ep!.catches).toHaveLength(2); + expect(ep!.catches[0]).toEqual({ + rethrows: false, + branches: false, + throws: false, + guardsParse: true, + awaitsOnlyParse: true, + guardCanRaise: true, + guardMayRaise: true, + tryStatementCount: 1, + }); + expect(ep!.catches[1]).toMatchObject({ + guardsParse: false, + tryStatementCount: 4, + }); + }); + + it("leaves catches empty for a try/finally with no catch clause", () => { + const ep = scanFile( + "runs-replication.status.ts", + ` + export async function loader() { + const redis = createRedis(); + try { + for (const source of sources) { + const exists = await redis.exists(source.slotName); + leaders.set(source.id, exists === 1); + } + } finally { + await redis.quit(); + } + return json({}); + } + ` + ); + expect(ep!.catches).toEqual([]); + // hasTryCatch keeps its meaning: a `try` appears. Nothing is caught here. + expect(ep!.hasTryCatch).toBe(true); + }); + + it("sees a URL constructor as a guarded parse", () => { + const ep = scanFile( + "_app.@.orgs.$organizationSlug.$.tsx", + ` + function refererOrigin(request) { + const referer = request.headers.get("referer"); + if (!referer) return undefined; + try { + return new URL(referer).origin; + } catch { + return undefined; + } + } + export async function action({ request }) { + const origin = refererOrigin(request); + return json({ origin }); + } + ` + ); + expect(ep!.catches).toHaveLength(1); + expect(ep!.catches[0]!.guardsParse).toBe(true); + }); + + it("sees a parse in a try that outgrows a single statement", () => { + const ep = scanFile( + "admin.api.v1.orgs.$organizationId.stream-basin.ts", + ` + export async function action({ request }) { + let parsed; + try { + const text = await request.text(); + const raw = text.length > 0 ? JSON.parse(text) : {}; + const result = BodySchema.safeParse(raw); + if (!result.success) { + return json({ ok: false }, { status: 400 }); + } + parsed = result.data; + } catch { + return json({ ok: false, error: "Invalid JSON body" }, { status: 400 }); + } + return json(parsed); + } + ` + ); + expect(ep!.catches).toHaveLength(1); + expect(ep!.catches[0]!.guardsParse).toBe(true); + expect(ep!.catches[0]!.tryStatementCount).toBe(6); + }); + + it("does not call a broad catch over database work a parse guard", () => { + const ep = scanFile( + "broad.ts", + ` + export async function loader({ params }) { + try { + const run = await prisma.run.findFirst({ where: { id: params.id } }); + const events = await prisma.event.findMany({ where: { runId: run.id } }); + await touch(run); + return json({ run, events }); + } catch (e) { + return json({}, { status: 500 }); + } + } + ` + ); + expect(ep!.catches).toHaveLength(1); + expect(ep!.catches[0]).toEqual({ + rethrows: false, + branches: false, + throws: false, + guardsParse: false, + awaitsOnlyParse: false, + guardCanRaise: true, + guardMayRaise: true, + tryStatementCount: 4, + }); + }); + + it("keeps rethrow and branch evidence per clause", () => { + const ep = scanFile( + "mixed-clauses.ts", + ` + export async function loader({ request }) { + try { + return json(await load(request)); + } catch (e) { + if (e instanceof Response) throw e; + return json({}, { status: 500 }); + } + } + export async function action({ request }) { + try { + return json(await save(request)); + } catch (e) { + return null; + } + } + ` + ); + expect(ep!.catches).toHaveLength(2); + // `if (e instanceof Response) throw e;` branches (it reads the binding and one arm throws) and + // does not rethrow (the throw is guarded, so it is not on the clause's own path). The action's + // catch does neither. What this test is for is that the two clauses stay separate. + expect(ep!.catches.filter((c) => !c.rethrows && c.branches)).toHaveLength(1); + expect(ep!.catches.filter((c) => !c.rethrows && !c.branches)).toHaveLength(1); + }); + + it("includes a catch from a same-file helper and excludes one from the React component", () => { + const ep = scanFile( + "route.tsx", + ` + function parseTags(payload) { + try { + return JSON.parse(payload); + } catch { + return null; + } + } + export async function loader({ params }) { + return json(parseTags(params.payload)); + } + export default function Page() { + try { + render(); + } catch (e) { + throw e; + } + return null; + } + ` + ); + expect(ep!.catches).toHaveLength(1); + expect(ep!.catches[0]!.guardsParse).toBe(true); + }); +}); + +describe("scanFile: guardsParse is limited to parsing constructors", () => { + it("does not treat a presenter construction as a parse guard", () => { + const ep = scanFile( + "_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.branches/route.tsx", + ` + export async function loader({ request, params }) { + try { + const presenter = new BranchesPresenter(); + const result = await presenter.call({ userId: 1, projectSlug: params.projectParam }); + return typedjson(result); + } catch (error) { + logger.error("Error loading preview branches page", { error }); + throw new Response(undefined, { status: 400 }); + } + } + ` + ); + expect(ep!.catches).toHaveLength(1); + expect(ep!.catches[0]!.guardsParse).toBe(false); + }); + + it("does not treat a collection construction as a parse guard", () => { + const ep = scanFile( + "account.tokens/route.tsx", + ` + export async function action({ request }) { + try { + const roles = await loadRoles(request); + const names = new Set(roles.map((r) => r.name)); + return json({ names: [...names] }); + } catch (error) { + return json({ error: "failed" }, { status: 400 }); + } + } + ` + ); + expect(ep!.catches[0]!.guardsParse).toBe(false); + }); + + it("treats RegExp and URLSearchParams construction as a parse guard", () => { + const regexp = scanFile( + "regexp.ts", + ` + export async function action({ request }) { + const pattern = await patternFrom(request); + try { + new RegExp(pattern); + } catch { + return json({ error: "Invalid regex" }, { status: 400 }); + } + return json({ ok: true }); + } + ` + ); + expect(regexp!.catches[0]!.guardsParse).toBe(true); + + const search = scanFile( + "search-params.ts", + ` + export async function loader({ request }) { + try { + return json(Object.fromEntries(new URLSearchParams(request.url))); + } catch { + return json({}, { status: 400 }); + } + } + ` + ); + expect(search!.catches[0]!.guardsParse).toBe(true); + }); + + it("still sees a parse call in a try that also constructs something ordinary", () => { + const ep = scanFile( + "parse-and-construct.ts", + ` + export async function action({ request }) { + try { + const body = JSON.parse(await request.text()); + const service = new PromptService(); + return json(await service.create(body)); + } catch { + return json({}, { status: 400 }); + } + } + ` + ); + expect(ep!.catches[0]!.guardsParse).toBe(true); + }); +}); + +describe("scanFile: branches ignores the error-stringifying ternary", () => { + it("does not count an instanceof nested inside a returned call argument", () => { + const ep = scanFile( + "admin.api.v1.runs-replication.start.ts", + ` + export async function action({ request }) { + try { + return json(await start(request)); + } catch (error) { + return json({ error: error instanceof Error ? error.message : error }, { status: 400 }); + } + } + ` + ); + expect(ep!.catches).toHaveLength(1); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + it("does not count an instanceof used to build a logged message", () => { + const ep = scanFile( + "admin.api.v1.feature-flags.ts", + ` + export async function action({ request }) { + try { + return json(await setFlag(request)); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + logger.error("flag update failed", { message }); + return json({ error: message }, { status: 400 }); + } + } + ` + ); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + it("still counts an instanceof in an `if`", () => { + const ep = scanFile( + "branch-if-instanceof.ts", + ` + export async function loader({ request }) { + try { + return json(await load(request)); + } catch (e) { + if (e instanceof Response) return e; + return json({}, { status: 500 }); + } + } + ` + ); + expect(ep!.catches[0]!.branches).toBe(true); + }); +}); + +describe("scanFile: branches requires the if/switch condition to examine the error", () => { + it("does not set branches for an `if` on an unrelated variable", () => { + const ep = scanFile( + "retry-count.ts", + ` + export async function action({ request }) { + let attempt = 0; + try { + return json(await load(request)); + } catch (e) { + if (attempt > 3) return json({}, { status: 503 }); + return json({}, { status: 500 }); + } + } + ` + ); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + it("sets branches for an `if` whose condition references the caught error", () => { + const ep = scanFile( + "branch-if-e.ts", + ` + export async function loader({ request }) { + try { + return json(await load(request)); + } catch (e) { + if (e instanceof ApiError) return json({}, { status: e.status }); + return json({}, { status: 500 }); + } + } + ` + ); + expect(ep!.catches[0]!.branches).toBe(true); + }); + + it("sets branches for a `switch` on a property of the caught error", () => { + const ep = scanFile( + "branch-switch-error-code.ts", + ` + export async function loader({ request }) { + try { + return json(await load(request)); + } catch (error) { + switch (error.code) { + case "P2025": + return json({}, { status: 404 }); + default: + return json({}, { status: 500 }); + } + } + } + ` + ); + expect(ep!.catches[0]!.branches).toBe(true); + }); + + it("does not set branches for a `switch` on an unrelated discriminant", () => { + const ep = scanFile( + "branch-switch-unrelated.ts", + ` + export async function action({ request }) { + const mode = "strict"; + try { + return json(await load(request)); + } catch (error) { + switch (mode) { + case "strict": + return json({}, { status: 400 }); + default: + return json({}, { status: 500 }); + } + } + } + ` + ); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + it("cannot set branches for a bindingless catch, even with an `if` inside it", () => { + const ep = scanFile( + "bindingless-if.ts", + ` + export async function loader({ request }) { + let attempt = 0; + try { + return json(await load(request)); + } catch { + if (attempt > 3) return json({}, { status: 503 }); + return json({}, { status: 500 }); + } + } + ` + ); + expect(ep!.catches[0]!.branches).toBe(false); + }); +}); + +// A3. `referencesBinding` matched any identifier with the binding's text, including a property +// name, an object literal key and a name re-declared in a nested scope. So a clause that never +// really inspects the error still counted as deciding on it. +describe("scanFile: branches requires a genuine read of the binding, not a lookalike", () => { + it("does not set branches for an `if` that only reads a same-named property", () => { + const ep = scanFile( + "branch-property-name.ts", + ` + export async function loader({ request }) { + try { + return json(await load(request)); + } catch (error) { + if (fallback.error) return json({}, { status: 500 }); + return json({}, { status: 500 }); + } + } + ` + ); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + it("does not set branches for an `if` that only reads a same-named object literal key", () => { + const ep = scanFile( + "branch-object-key.ts", + ` + export async function loader({ request }) { + try { + return json(await load(request)); + } catch (error) { + if (buildOptions({ error: false }).ok) return json({}, { status: 500 }); + return json({}, { status: 500 }); + } + } + ` + ); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + it("does not set branches for an `if` whose only reference is inside a callback that re-declares the name", () => { + const ep = scanFile( + "branch-shadowed-param.ts", + ` + export async function loader({ request }) { + try { + return json(await load(request)); + } catch (error) { + if (items.some(function (error) { return error.code === 1; })) { + return json({}, { status: 500 }); + } + return json({}, { status: 500 }); + } + } + ` + ); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + it("does not set branches for an `if` whose only reference is inside a block that re-declares the name", () => { + const ep = scanFile( + "branch-shadowed-block.ts", + ` + export async function loader({ request }) { + try { + return json(await load(request)); + } catch (error) { + if ( + (() => { + const error = 1; + return error > 0; + })() + ) { + return json({}, { status: 500 }); + } + return json({}, { status: 500 }); + } + } + ` + ); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + it("still sets branches for an `if` that genuinely reads the binding beside a lookalike", () => { + const ep = scanFile( + "branch-genuine-and-lookalike.ts", + ` + export async function loader({ request }) { + try { + return json(await load(request)); + } catch (error) { + if (fallback.error || error instanceof NotFound) return json({}, { status: 404 }); + return json({}, { status: 500 }); + } + } + ` + ); + expect(ep!.catches[0]!.branches).toBe(true); + }); +}); + +// I5. The shadow check only fired inside `referencesBinding`'s own top-down search from an +// if/switch's condition, so it only caught shadowing NESTED inside that condition, exactly what +// the tests above exercise. A shadowing scope that instead WRAPS the if (a for-of loop, a nested +// catch with the same name) was invisible, because nothing walked up from the if to notice it. +// catchClauseEvidence now tracks shadowing as it descends, the same way `inCallback` tracks a +// callback boundary: once a scope re-declares the binding, everything nested inside stays shadowed. +describe("scanFile: a binding shadowed by an enclosing scope, not just a nested one", () => { + const swallow = (mutation: string) => ` + export async function loader() { + try { + return await prisma.thing.findMany(); + } catch (error) { + ${mutation} + return null; + } + } + `; + + it("does not credit an if inside a for...of loop that re-declares the binding", () => { + const ep = scanFile( + "x.ts", + swallow("for (const error of errors) { if (error.code === 1) { doThing(); } }") + ); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + it("does not credit an if inside a for...in loop that re-declares the binding", () => { + const ep = scanFile( + "x.ts", + swallow("for (const error in errorsByKey) { if (error) { doThing(); } }") + ); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + it("does not credit an if inside a classic for loop that re-declares the binding", () => { + const ep = scanFile( + "x.ts", + swallow("for (let error = 0; error < 10; error++) { if (error) { doThing(); } }") + ); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + it("does not credit an if inside a nested catch clause with the same binding name", () => { + const ep = scanFile( + "x.ts", + swallow("try { doWork(); } catch (error) { if (error) { doThing(); } }") + ); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + it("does not credit an if inside a block whose own destructured const shadows the binding", () => { + const ep = scanFile( + "x.ts", + swallow(` + if (attempt > 0) { + const { error } = computeSomething(); + if (error) { doThing(); } + } + `) + ); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + it("does not credit an if reading a destructured parameter inside the if's own condition", () => { + const ep = scanFile( + "x.ts", + swallow("if (items.some(({ error }) => error > 0)) { doThing(); }") + ); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + it("does not credit an if shadowed by an array-destructured declaration", () => { + const ep = scanFile( + "x.ts", + swallow("const [error] = getErrors();\n if (error) { doThing(); }") + ); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + // Positive control: a genuine reference to the real binding, on the clause's own path, with an + // arm that takes the error somewhere the other arm does not go. + it("still credits an if that genuinely reads the outer binding directly", () => { + const ep = scanFile("x.ts", swallow("if (error instanceof Error) { return badRequest(); }")); + expect(ep!.catches[0]!.branches).toBe(true); + }); + + // Two shapes that read the real binding and are still not credited, for reasons that are not + // shadowing. Both are precision the straight-line rule gives up on purpose, and both are + // recorded here so a later reader can tell a deliberate limit from a bug. + it("does not credit an if whose arm does not take the error anywhere", () => { + const ep = scanFile("x.ts", swallow("if (error instanceof Error) { doThing(); }")); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + it("does not credit an if nested inside a loop, even with a different loop variable", () => { + const ep = scanFile( + "x.ts", + swallow("for (const item of items) { if (error.code === item) { return item; } }") + ); + expect(ep!.catches[0]!.branches).toBe(false); + }); +}); + +// S3. The ternary path checked only that the condition tested the error, never that the two arms +// went anywhere different, while the `if`/`switch` path had checked exactly that since the round +// before. Rewriting `return X;` as `return e instanceof Error ? (X) : (X)` was therefore worth 50 +// points a route for a change that decides nothing, and it is semantics-preserving. +// `same-arms-ternary` in the mutation corpus is the tree-scale version. +describe("a ternary on the error has to send its arms somewhere different", () => { + const returning = (value: string) => ` + export async function loader() { + try { + return await prisma.thing.findMany(); + } catch (error) { + return ${value}; + } + } + `; + + it("does not credit a ternary whose arms are identical", () => { + const ep = scanFile("x.ts", returning("error instanceof Error ? (json({})) : (json({}))")); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + it("does not credit a ternary whose arms differ only in whitespace", () => { + const ep = scanFile("x.ts", returning("error instanceof Error ? (json( {} )) : (json({}))")); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + it("does not credit a ternary whose arms differ only in parentheses", () => { + const ep = scanFile("x.ts", returning("error instanceof Error ? ((json({}))) : (json({}))")); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + it("still credits a ternary whose arms go somewhere different", () => { + const ep = scanFile( + "x.ts", + returning("error instanceof Response ? error : json({}, { status: 500 })") + ); + expect(ep!.catches[0]!.branches).toBe(true); + }); + + // S6. The same three cases on the throw path, which read none of them. The throw arm of the + // shared branch check was unreachable, because the walk sets `rethrows` and cuts the path first, + // so a thrown ternary was never offered to `selectsAnErrorPath` and every one of these clauses + // read as inert. Reading it in the throw arm, before the path is cut, uses the same predicate, + // so the arm test arrives with it. `wrap-body-in-same-arms-throw-ternary` is the tree-scale + // version of the refusal. + const throwing = (value: string) => ` + export async function loader() { + try { + return await prisma.thing.findMany(); + } catch (error) { + throw ${value}; + } + } + `; + + it("credits a thrown ternary whose arms go somewhere different", () => { + const ep = scanFile("x.ts", throwing("error instanceof Response ? error : new Error('x')")); + expect(ep!.catches[0]!.branches).toBe(true); + }); + + it("does not credit a thrown ternary whose arms are identical", () => { + const ep = scanFile("x.ts", throwing("error instanceof Error ? error : error")); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + it("does not credit a thrown ternary whose arms differ only in parentheses", () => { + const ep = scanFile("x.ts", throwing("error instanceof Error ? ((error)) : (error)")); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + it("does not credit a thrown ternary that never reads the caught binding", () => { + const ep = scanFile("x.ts", throwing("other instanceof Error ? error : new Error('x')")); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + // The throw still cuts the path, so a decision written after it is dead and stays uncredited. + it("does not credit a thrown ternary written after the clause already threw", () => { + const ep = scanFile( + "x.ts", + `export async function loader() { + try { + return await prisma.thing.findMany(); + } catch (error) { + throw error; + throw error instanceof Response ? error : new Error('x'); + } + }` + ); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + // The same comparison on the `if` path, which had the exit test but not the arm test. + it("does not credit an if/else whose two arms are identical", () => { + const ep = scanFile( + "x.ts", + `export async function loader() { + try { return await prisma.thing.findMany(); } + catch (error) { + if (error instanceof Error) { return json({}); } else { return json({}); } + } + }` + ); + expect(ep!.catches[0]!.branches).toBe(false); + }); +}); + +// The liveness gap in the branch predicate. `selectsADistinctPath` asked a plain containment +// question, so an arm holding an exit that can never run read as an arm that takes the error +// somewhere. `catch (e) { if (e instanceof Error) { if (false) { return null; } } return json(x, +// { status: 500 }); }` is the same swallow as the clause without the `if`, and it was worth 50 +// points a route. The same eleven dead spellings had already been folded out of +// `catchClauseEvidence`'s `exited` flag by `containsLiveExit`, and this predicate beside it kept +// the containment read. `dead-armed-instanceof-if` in the mutation corpus is the tree-scale +// version: global 19 -> 27 and 80 routes raised, before the fix. +describe("an arm whose only exit is dead decides nothing", () => { + const swallow = (mutation: string) => ` + export async function loader() { + try { + return await prisma.thing.findMany(); + } catch (error) { + ${mutation} + return json({ error: "generic" }, { status: 500 }); + } + } + `; + + it("reads the unmutated clause as a swallow, as a baseline", () => { + const ep = scanFile("x.ts", swallow("")); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + // The reported shape, plus three further spellings of the same no-op reaching the same + // predicate by different routes: a dead loop body, a dead else arm, and a switch clause. + const DEAD_ARMS: Array<[string, string]> = [ + ["an if (false) arm", "if (error instanceof Error) { if (false) { return null; } }"], + ["a while (false) body", "if (error instanceof Error) { while (false) { throw error; } }"], + [ + "a dead else arm beside an arm that goes nowhere", + "if (error instanceof Error) { doThing(); } else { for (const k in {}) { return null; } }", + ], + [ + "a switch clause whose exit is dead", + "switch (error.code) { case 'x': if (1 === 2) { throw error; } }", + ], + ]; + + for (const [label, mutation] of DEAD_ARMS) { + it(`does not credit ${label}`, () => { + const ep = scanFile("x.ts", swallow(mutation)); + expect(ep!.catches[0]!.branches).toBe(false); + }); + } + + // The positive controls. The fold is subtractive against containment, so anything it cannot + // prove dead reads exactly as it did, including a guard whose truth is not decidable from the + // token alone. Without these the fix could be "always false" and the four cases above would + // still pass. + // + // `an arm guarded by a condition that does not fold` is also the pin on the alternative that was + // measured and rejected: asking the arm to `definitelyExits` rather than to hold a live exit. + // That is the "guaranteed" reading, it refuses all four shapes above, and it accuses + // `admin.api.v1.orgs.$organizationId.environments.staging.ts` on the real tree, taking the global + // from 19 to 18. That clause recognises Prisma's P2002, re-reads the conflicting row and returns + // `{ status: "updated" }`, rethrowing everything else: a textbook classification whose arm + // happens to fall through to the rethrow when the re-read finds nothing. Accusing it of taking + // one way out regardless of what was thrown is simply false, and a new false accusation is the + // direction that gets the tool switched off. + const LIVE_ARMS: Array<[string, string]> = [ + ["a plain returning arm", "if (error instanceof Error) { return badRequest(); }"], + [ + "an arm guarded by a condition that does not fold", + "if (error instanceof Error) { if (error.code === 'P2002') { return conflict(); } }", + ], + [ + "a live exit in the else arm only", + "if (error instanceof Error) { doThing(); } else { return badRequest(); }", + ], + ["a switch clause that returns", "switch (error.code) { case 'P2002': return conflict(); }"], + ]; + + for (const [label, mutation] of LIVE_ARMS) { + it(`still credits ${label}`, () => { + const ep = scanFile("x.ts", swallow(mutation)); + expect(ep!.catches[0]!.branches).toBe(true); + }); + } +}); + +// C4a. `export const { action, loader } = createActionApiRoute(...)` produced no entry point at +// all: `scanFile` skipped a non-identifier binding name at the export site, so the route was +// absent from the denominator rather than parsed, failed or unmeasured. The two-step spelling +// already worked, because `collectLocalDeclarations` reads the binding pattern and the export +// clause resolves through it, so exactly half the shape was wired. +describe("scanFile: a destructured export declaration", () => { + const BUILDER = `import { createActionApiRoute } from "~/services/routeBuilders/apiBuilder.server";`; + + it("finds an action destructured straight out of a builder call", () => { + const ep = scanFile( + "api.v1.things.ts", + `${BUILDER} + export const { action } = createActionApiRoute({ method: "POST" }, async () => json({}));` + ); + expect(ep).not.toBeNull(); + expect(ep!.hasAction).toBe(true); + expect(ep!.actionInitializerCallee).toBe("createActionApiRoute"); + }); + + it("finds both halves of a destructured builder export", () => { + const ep = scanFile( + "api.v1.things.ts", + `${BUILDER} + export const { action, loader } = createActionApiRoute({}, async () => json({}));` + ); + expect(ep!.hasAction).toBe(true); + expect(ep!.hasLoader).toBe(true); + }); + + it("finds an action destructured from a local the builder produced", () => { + const ep = scanFile( + "api.v1.things.ts", + `${BUILDER} + const route = createActionApiRoute({}, async () => json({})); + export const { action } = route;` + ); + expect(ep).not.toBeNull(); + expect(ep!.actionInitializerCallee).toBe("createActionApiRoute"); + }); + + // The exported name is the element name, so a rename decides what this file exports. + it("reads the exported name rather than the property it came from", () => { + const renamed = scanFile( + "api.v1.things.ts", + `${BUILDER} + export const { loader: action } = createActionApiRoute({}, async () => json({}));` + ); + expect(renamed!.hasAction).toBe(true); + expect(renamed!.hasLoader).toBe(false); + + const hidden = scanFile( + "api.v1.things.ts", + `${BUILDER} + export const { action: internal } = createActionApiRoute({}, async () => json({}));` + ); + expect(hidden).toBeNull(); + }); + + it("counts the statements of a handler reached through the binding pattern", () => { + const ep = scanFile( + "api.v1.things.ts", + `${BUILDER} + export const { action } = createActionApiRoute({}, async ({ request }) => { + const body = await request.json(); + const saved = await prisma.thing.create({ data: body }); + return json(saved); + });` + ); + expect(ep!.statementCount).toBe(3); + }); +}); + +// C4b. A route whose body is in another module gives zero statements and zero callees, so the +// triviality rule read it as a redirect stub and every check reported not-applicable for it. The +// scan says so directly instead, and `score.ts` counts it apart from the routes nothing applied to. +describe("scanFile: a route that delegates its body to another module", () => { + it("marks a re-export of an action from another module", () => { + const ep = scanFile("webhooks.v1.stripe.ts", `export { action } from "./handler.server";`); + expect(ep!.delegating).toBe(true); + }); + + it("marks an action aliased to an imported function", () => { + const ep = scanFile( + "webhooks.v1.stripe.ts", + `import { handleWebhook } from "./handler.server"; + export const action = handleWebhook;` + ); + expect(ep!.delegating).toBe(true); + }); + + it("marks a renamed re-export", () => { + const ep = scanFile( + "webhooks.v1.stripe.ts", + `export { handleWebhook as action } from "./handler.server";` + ); + expect(ep!.delegating).toBe(true); + }); + + it("does not mark a route whose body is in the file", () => { + const ep = scanFile("api.v1.things.ts", LOADER); + expect(ep!.delegating).toBe(false); + }); + + it("does not mark a route wrapped in a builder, whose options are still readable", () => { + const ep = scanFile( + "api.v1.things.ts", + `import { createActionApiRoute } from "~/services/routeBuilders/apiBuilder.server"; + import { handler } from "./handler.server"; + export const action = createActionApiRoute({ method: "POST" }, handler);` + ); + expect(ep!.delegating).toBe(false); + }); + + // Half a route in the file is still half a route to judge. + it("does not mark a route that delegates one export and writes the other", () => { + const ep = scanFile( + "api.v1.things.ts", + `export { action } from "./handler.server"; + export async function loader() { return json({}); }` + ); + expect(ep!.delegating).toBe(false); + }); +}); + +// C1b. What `auth-scope` reads: the options a builder was given, whether the handler filters by +// the caller's own id, and whether it runs the ability gate itself. +describe("scanFile: the signals auth-scope reads", () => { + const PAT = `import { createActionPATApiRoute } from "~/services/routeBuilders/apiBuilder.server";`; + + it("records the top-level option keys of the builder call", () => { + const ep = scanFile( + "api.v1.orgs.ts", + `${PAT} + export const action = createActionPATApiRoute( + { method: "POST", authorization: { action: "manage", resource: { type: "org" } } }, + async () => json({}) + );` + ); + expect(ep!.actionBuilderOptions).toEqual(["method", "authorization"]); + }); + + it("keeps the loader's options and the action's options apart", () => { + const ep = scanFile( + "api.v1.orgs.ts", + `${PAT} + export const loader = createActionPATApiRoute({ authorization: {} }, async () => json({})); + export const action = createActionPATApiRoute({ method: "POST" }, async () => json({}));` + ); + expect(ep!.loaderBuilderOptions).toEqual(["authorization"]); + expect(ep!.actionBuilderOptions).toEqual(["method"]); + }); + + it("sees a query filtered by the caller's id, in both builders' spellings", () => { + const api = scanFile( + "api.v1.orgs.ts", + `${PAT} + export const loader = createActionPATApiRoute({}, async ({ authentication }) => { + return json(await prisma.org.findMany({ + where: { members: { some: { userId: authentication.userId } } }, + })); + });` + ); + expect(api!.loaderScopesByCaller).toBe(true); + + const dashboard = scanFile( + "_app.orgs.$slug.apikeys/route.tsx", + `import { dashboardLoader } from "~/services/routeBuilders/dashboardBuilder"; + export const loader = dashboardLoader({}, async ({ user }) => { + return json(await presenter.call({ userId: user.id, slug: "x" })); + });` + ); + expect(dashboard!.loaderScopesByCaller).toBe(true); + }); + + // Round C ruling 1. The signal is per export because the exposure is per export. Entry-point-wide + // it passed `_app.orgs.$organizationSlug.settings.team/route.tsx`, whose loader narrows itself to + // the caller and whose action resolves the target org from the URL slug. + it("attributes the caller filter to the export it was written in", () => { + const ep = scanFile( + "_app.orgs.$slug.settings.team/route.tsx", + `import { dashboardAction, dashboardLoader } from "~/services/routeBuilders/dashboardBuilder"; + export const loader = dashboardLoader({}, async ({ user }) => + json(await presenter.call({ userId: user.id })) + ); + export const action = dashboardAction({}, async ({ context, request }) => + json(await prisma.orgMember.deleteMany({ where: { organizationId: context.organizationId } })) + );` + ); + expect(ep!.loaderScopesByCaller).toBe(true); + expect(ep!.actionScopesByCaller).toBe(false); + }); + + // Round D item 3. The predicate fired on any property at all whose value was a caller id, so one + // dead statement cleared the check. Both halves are now required: an identity property name, and + // an object that is handed to a call. + it("does not read a dead object holding the caller id as a scope", () => { + const arbitraryKey = scanFile( + "api.v1.orgs.ts", + `${PAT} + export const loader = createActionPATApiRoute({}, async ({ user }) => { + const unused = { anything: user.id }; + return json(await prisma.org.findMany({ where: { slug: "x" } })); + });` + ); + expect(arbitraryKey!.loaderScopesByCaller).toBe(false); + + const identityKey = scanFile( + "api.v1.orgs.ts", + `${PAT} + export const loader = createActionPATApiRoute({}, async ({ user }) => { + const unused = { userId: user.id }; + return json(await prisma.org.findMany({ where: { slug: "x" } })); + });` + ); + expect(identityKey!.loaderScopesByCaller).toBe(false); + }); + + it("reads the caller id through any depth of nesting inside a call argument", () => { + const ep = scanFile( + "api.v1.orgs.ts", + `${PAT} + export const loader = createActionPATApiRoute({}, async ({ user }) => { + return json(await prisma.org.findMany({ + where: { OR: [{ members: { some: { userId: user.id } } }] }, + })); + });` + ); + expect(ep!.loaderScopesByCaller).toBe(true); + }); + + it("does not read a non-identity field off the caller as a scope", () => { + const ep = scanFile( + "api.v1.orgs.ts", + `${PAT} + export const loader = createActionPATApiRoute({}, async ({ user }) => { + return json(await prisma.org.findMany({ where: { title: user.name } })); + });` + ); + expect(ep!.loaderScopesByCaller).toBe(false); + }); + + // Round E item 3. Neither of the two conditions above constrains the CALLEE, so a log line + // carrying the caller's id satisfied a tenant-scoping security check. Cheaper to write than the + // dead object, and unlike the dead object it survives review, because a log line is real code + // somebody wants. + it("does not read a caller id handed to a log call as a scope", () => { + const logger = scanFile( + "api.v1.orgs.ts", + `${PAT} + export const loader = createActionPATApiRoute({}, async ({ user }) => { + logger.error("create failed", { userId: user.id }); + return json(await prisma.org.findMany({ where: { slug: "x" } })); + });` + ); + expect(logger!.loaderScopesByCaller).toBe(false); + + // A different logger family: `LOGGER_CALLEE` does not match `console.warn`, so this is the + // second sink rather than a restatement of the first. + const console_ = scanFile( + "api.v1.orgs.ts", + `${PAT} + export const loader = createActionPATApiRoute({}, async ({ user }) => { + console.warn("create failed", { userId: user.id }); + return json(await prisma.org.findMany({ where: { slug: "x" } })); + });` + ); + expect(console_!.loaderScopesByCaller).toBe(false); + }); + + // The refusal is made at the call, not at the property, so burying the id under the depth of + // nesting a real filter has does not get it past. + it("does not read a caller id nested inside a log call's payload as a scope", () => { + const ep = scanFile( + "api.v1.orgs.ts", + `${PAT} + export const loader = createActionPATApiRoute({}, async ({ user }) => { + logger.info("looking up", { where: { OR: [{ userId: user.id }] } }); + return json(await prisma.org.findMany({ where: { slug: "x" } })); + });` + ); + expect(ep!.loaderScopesByCaller).toBe(false); + }); + + // The response body is the other sink that takes the same object and cannot narrow a read. + it("does not read a caller id handed to a response serializer as a scope", () => { + const ep = scanFile( + "api.v1.orgs.ts", + `${PAT} + export const loader = createActionPATApiRoute({}, async ({ user }) => { + return typedjson({ userId: user.id }); + });` + ); + expect(ep!.loaderScopesByCaller).toBe(false); + }); + + // The direction that matters more than any of the above: refusing the log line must not accuse a + // handler that also runs the query. This is the shape on the real tree today, + // `engine.v1.dev.runs.$runFriendlyId.snapshots.$snapshotFriendlyId.attempts.start.ts`, which logs + // the environment id and reads with it, and which must keep its pass. + it("still reads a real query filter written beside a log call", () => { + const ep = scanFile( + "api.v1.orgs.ts", + `${PAT} + export const loader = createActionPATApiRoute({}, async ({ authentication }) => { + const run = await runStore.findRun({ runtimeEnvironmentId: authentication.environment.id }); + if (!run) logger.error("no run", { environmentId: authentication.environment.id }); + return json(run); + });` + ); + expect(ep!.loaderScopesByCaller).toBe(true); + }); + + // A resource's owner is not the caller. + it("does not read another object's userId as a scope", () => { + const ep = scanFile( + "api.v1.orgs.ts", + `${PAT} + export const loader = createActionPATApiRoute({}, async () => { + return json(await prisma.org.findMany({ where: { userId: run.userId } })); + });` + ); + expect(ep!.loaderScopesByCaller).toBe(false); + }); +}); + +/** + * The per-export split of `calleeNames`. `auth-boundary` reads it, so a name landing in the wrong + * half is a wrong verdict on the one check where a false pass hides a security gap. + */ +describe("scanFile: callee names attributed per export", () => { + it("keeps each export's callees out of the other's list", () => { + const ep = scanFile( + "api.v1.tokens.ts", + `export async function loader({ request }) { + const userId = await requireUserId(request); + return json(await prisma.token.findMany({ where: { userId } })); + } + export async function action({ request }) { + await deleteEverything(request); + return json({ ok: true }); + }` + ); + expect(ep!.loaderCalleeNames).toContain("requireUserId"); + expect(ep!.loaderCalleeNames).not.toContain("deleteEverything"); + expect(ep!.actionCalleeNames).toContain("deleteEverything"); + expect(ep!.actionCalleeNames).not.toContain("requireUserId"); + }); + + it("attributes a same-file helper to the export that calls it", () => { + const ep = scanFile( + "api.v1.tokens.ts", + `async function loadTokens(request) { + const userId = await requireUserId(request); + return prisma.token.findMany({ where: { userId } }); + } + export async function loader({ request }) { return json(await loadTokens(request)); } + export async function action({ request }) { return json(await request.json()); }` + ); + expect(ep!.loaderCalleeNames).toContain("requireUserId"); + expect(ep!.actionCalleeNames).not.toContain("requireUserId"); + }); + + it("attributes a helper both exports call to both of them", () => { + const ep = scanFile( + "api.v1.tokens.ts", + `async function guarded(request) { return requireUserId(request); } + export async function loader({ request }) { return json(await guarded(request)); } + export async function action({ request }) { return json(await guarded(request)); }` + ); + expect(ep!.loaderCalleeNames).toContain("requireUserId"); + expect(ep!.actionCalleeNames).toContain("requireUserId"); + }); + + // One handler, both exports: `const { loader, action } = createActionApiRoute({ handler })`. + it("attributes a handler serving both exports to both of them", () => { + const ep = scanFile( + "api.v1.tokens.ts", + `export const { action, loader } = createActionApiRoute({}, async ({ authentication }) => { + return json(await authenticateApiRequest(authentication)); + });` + ); + expect(ep!.loaderCalleeNames).toContain("authenticateApiRequest"); + expect(ep!.actionCalleeNames).toContain("authenticateApiRequest"); + }); + + // The union the split came from. `calleeNames` stays entry-point-wide for `sensitivity.ts`, + // `triviality.ts` and `audit-trail`, so the two representations have to agree. + it("every callee name is attributed to an export that exists", () => { + const ep = scanFile( + "api.v1.tokens.ts", + `async function shared(request) { return audit(request); } + export async function loader({ request }) { return json(await shared(request)); } + export async function action({ request }) { return json(await mutate(request)); }` + ); + const attributed = new Set([...ep!.loaderCalleeNames, ...ep!.actionCalleeNames]); + expect([...new Set(ep!.calleeNames)].filter((n) => !attributed.has(n))).toEqual([]); + for (const name of attributed) expect(ep!.calleeNames).toContain(name); + }); + + it("counts statements and a try per export as well as entry-point-wide", () => { + const ep = scanFile( + "api.v1.tokens.ts", + `export const loader = () => redirect("/login"); + export async function action({ request }) { + try { return json(await request.json()); } catch (e) { throw e; } + }` + ); + expect(ep!.loaderStatementCount).toBe(1); + expect(ep!.loaderHasTryCatch).toBe(false); + expect(ep!.actionHasTryCatch).toBe(true); + expect(ep!.hasTryCatch).toBe(true); + expect(ep!.statementCount).toBe(ep!.loaderStatementCount + ep!.actionStatementCount); + }); +}); + +// Round C ruling 2. A guard that answers with null instead of throwing is only a boundary if the +// route reads the answer, so the scan records which callees' results a condition looked at. +describe("scanFile: callees whose answer the body read", () => { + it("records a resolver whose result a negated if tests", () => { + const ep = scanFile( + "invite-accept.tsx", + `import { getUser } from "~/services/session.server"; + export async function loader({ request }) { + const user = await getUser(request); + if (!user) return redirect("/login"); + return json({ email: user.email }); + }` + ); + expect(ep!.loaderCheckedCallees).toContain("getUser"); + }); + + it("records one whose result a plain if tests", () => { + const ep = scanFile( + "login._index/route.tsx", + `import { getUserId } from "~/services/session.server"; + export async function loader({ request }) { + const userId = await getUserId(request); + if (userId) throw redirect("/"); + return typedjson({}); + }` + ); + expect(ep!.loaderCheckedCallees).toContain("getUserId"); + }); + + it("records one whose result a ternary tests", () => { + const ep = scanFile( + "x.ts", + `export async function loader({ request }) { + const user = await getUser(request); + return user ? json({ ok: true }) : redirect("/login"); + }` + ); + expect(ep!.loaderCheckedCallees).toContain("getUser"); + }); + + it("does not record a result that is bound and never tested", () => { + const ep = scanFile( + "x.ts", + `export async function loader({ request }) { + const user = await getUser(request); + return json(await prisma.invite.findMany({ where: { email: user.email } })); + }` + ); + expect(ep!.loaderCheckedCallees).not.toContain("getUser"); + }); + + it("does not record a result that is dropped entirely", () => { + const ep = scanFile( + "x.ts", + `export async function loader({ request }) { + await getUser(request); + return json(await prisma.invite.findMany()); + }` + ); + expect(ep!.loaderCheckedCallees).not.toContain("getUser"); + }); + + it("does not record a callee just because a same-named local is tested elsewhere", () => { + const ep = scanFile( + "x.ts", + `export async function loader({ request }) { + const rows = await prisma.invite.findMany(); + if (rows.length === 0) return json([]); + return json(rows); + }` + ); + expect(ep!.loaderCheckedCallees).toContain("findMany"); + expect(ep!.loaderCheckedCallees).not.toContain("getUser"); + }); +}); diff --git a/internal-packages/observability-map/src/scan.ts b/internal-packages/observability-map/src/scan.ts new file mode 100644 index 00000000000..78b48d54e08 --- /dev/null +++ b/internal-packages/observability-map/src/scan.ts @@ -0,0 +1,1951 @@ +import ts from "typescript"; +import { readdirSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +import type { CatchEvidence, EntryPoint, LogCall } from "./types.js"; + +/** Thrown by `scanFile` when the source does not parse cleanly. */ +export class ParseFailureError extends Error { + constructor( + readonly fileName: string, + readonly diagnostic: string + ) { + super(`${fileName}: ${diagnostic}`); + this.name = "ParseFailureError"; + } +} + +type EntryFunction = ts.FunctionDeclaration | ts.FunctionExpression | ts.ArrowFunction; + +function isEntryFunction(node: ts.Node): node is EntryFunction { + return ( + ts.isFunctionDeclaration(node) || ts.isFunctionExpression(node) || ts.isArrowFunction(node) + ); +} + +/** Strip wrappers that do not change which expression is really being referred to. */ +function unwrap(expr: ts.Expression): ts.Expression { + let current = expr; + for (;;) { + if ( + ts.isParenthesizedExpression(current) || + ts.isAwaitExpression(current) || + ts.isAsExpression(current) || + ts.isSatisfiesExpression(current) || + ts.isNonNullExpression(current) + ) { + current = current.expression; + continue; + } + return current; + } +} + +/** + * Root callee of a call, unwrapping chains: `createLoaderApiRoute({}).withCors()` + * resolves to `createLoaderApiRoute`, not `withCors`. + */ +function rootCalleeName(call: ts.CallExpression): string | null { + let current: ts.Expression = unwrap(call.expression); + for (;;) { + if (ts.isIdentifier(current)) return current.text; + if ( + ts.isCallExpression(current) || + ts.isPropertyAccessExpression(current) || + ts.isElementAccessExpression(current) + ) { + current = unwrap(current.expression); + continue; + } + return null; + } +} + +/** + * A dotted property path, `authentication.userId`, or null when the chain does not start from a + * plain identifier. Property accesses only: a call or an index anywhere in the chain gives null. + */ +function propertyPath(expr: ts.Expression): string | null { + const target = unwrap(expr); + if (ts.isIdentifier(target)) return target.text; + if (!ts.isPropertyAccessExpression(target)) return null; + const base = propertyPath(target.expression); + return base === null ? null : `${base}.${target.name.text}`; +} + +/** + * Expressions that are the caller's own id, in the spellings the route tree uses: + * `userId: authentication.userId` under the API builders, `userId: user.id` under the dashboard + * builders, and the `authenticationResult` and `sessionAuth` variants. Read by `auth-scope` as + * evidence that the handler narrowed its query to whoever is asking. + * + * Anchored at both ends. The root has to be one of the auth bindings a builder hands the handler, + * and the last segment has to be an identity field, so `user.name` is not a scope and neither is + * `run.userId`, which is a resource's owner rather than the caller. + */ +const CALLER_ID_PATH = + /^(authentication|authenticationResult|auth|sessionAuth|user)(\.[A-Za-z0-9_$]+)*\.(userId|id|actor)$/; + +/** + * Property names that mean the value is being used to say WHOSE, rather than merely carrying the + * caller's id around. Read off the tree: of the ten names that take a caller-id value in + * `apps/webapp/app/routes`, these are the tenant and identity fields, and `sub`, `value` and + * `consumerId` are the three that are not. `anything: user.id` is what a mutation writes, and it + * does not match. + */ +const CALLER_ID_FIELD = + /^(id|userId|user|memberId|orgMemberId|createdBy|createdByUserId|environmentId|runtimeEnvironmentId|organizationId|orgId|projectId)$/; + +/** + * Callees that are handed the caller's id and cannot narrow a read with it: the log line and the + * response body. Both take the very `{ userId: user.id }` object a query filter takes, so crediting + * them let one log statement clear `auth-scope` for a whole export. That is cheaper than the + * actor-argument residual `checks/authScope.ts` discloses, and it lands on the one check whose + * purpose is catching cross-org exposure. + * + * The shape is already in the tree rather than hypothetical: `engine.v1.dev.runs...attempts.start` + * writes `logger.error("...", { environmentId: authentication.environment.id })` beside the + * `runStore.findRun` that earns that export its credit honestly. Loggers account for 13 of the + * caller-id sites under `apps/webapp/app/routes` and the two response serializers for 2 more. + * + * A denylist of sinks rather than an allowlist of query callees, and that is a measurement rather + * than a preference. 72 distinct callees are handed a caller id across the route tree, running from + * `prisma.project.findFirst` through `presenter.call` and `new DeleteProjectService().call` to bare + * `regenerateApiKey` and `resolveOrganizationForApiUser`. No name pattern separates those from + * `sendToPlain`, so an allowlist would accuse whichever route named its helper next, and a wrong + * accusation is the failure this check cannot afford. Refusing the sinks that are known not to + * scope shrinks the residual without pretending to close it: `someHelper({ userId: user.id })` that + * ignores its argument still credits, which needs types the scanner does not have. + */ +const NON_SCOPING_CALLEE = /(^|\.)console\.[A-Za-z_$][\w$]*$|^(json|typedjson|defer)$/; + +/** + * Whether a callee could plausibly narrow a read with the object it is handed. A callee with no + * readable name of its own is credited: refusing it would ACCUSE the route, and under-crediting the + * constraint beats accusing a route that is fine. + */ +function couldScopeAQuery(callee: ts.Expression): boolean { + const text = calleeText(callee); + if (text === null) return true; + return !LOGGER_CALLEE.test(text) && !NON_SCOPING_CALLEE.test(text); +} + +/** + * Whether the object literal holding this property is handed to a call that could scope a query, + * through any depth of nesting: `findMany({ where: { members: { some: { userId } } } })` is, and + * `const unused = { userId };` is not. Arrays count, so `{ OR: [{ userId }] }` still reaches its + * call. + * + * Two things are refused. A filter built and dropped, which is the dead-object shape + * `dead-caller-scope-object` and `dead-caller-scope-userid` cover. And a filter handed to a callee + * that provably cannot read with it, which is `NON_SCOPING_CALLEE` and which the corpus entry + * `log-caller-scope-userid` covers at tree scale. + * + * What this does NOT refuse is a filter handed to a named call that ignores it: + * `String({ userId: user.id });` reads as scoping, the same way `try { String(0); }` reads as error + * handling, and for the same reason. Knowing whether an arbitrary callee uses the argument needs + * types the scanner does not have. + */ +function isHandedToAScopingCall(property: ts.PropertyAssignment): boolean { + let node: ts.Node = property; + for (let parent = node.parent; parent; node = parent, parent = node.parent) { + if (ts.isCallExpression(parent) || ts.isNewExpression(parent)) { + if (parent.arguments?.some((a) => a === node) !== true) return false; + return couldScopeAQuery(parent.expression); + } + if ( + !ts.isObjectLiteralExpression(parent) && + !ts.isPropertyAssignment(parent) && + !ts.isArrayLiteralExpression(parent) + ) { + return false; + } + } + return false; +} + +/** + * Whether any handler in `fns` assigns the caller's own id to an object-literal property, the + * `where: { members: { some: { userId: authentication.userId } } }` and + * `presenter.call({ userId: user.id })` shapes. + * + * Per export rather than per entry point, and that is the whole point of computing it here instead + * of in the main body walk. A file whose loader narrows itself to the caller and whose action does + * not is not a scoped route, and the entry-point-wide version said it was: it passed + * `_app.orgs.$organizationSlug.settings.team/route.tsx`, whose loader calls + * `TeamPresenter.call({ userId: user.id })` while its action resolves the target org from the URL + * slug and gates only on `ability.can`. + * + * Nested functions are walked, since a filter built inside a callback still filters. Same-file + * helpers are NOT followed, unlike the main walk: a route that computes its filter in a helper is + * reported as unscoped. Nothing in the tree does. + * + * Three conditions, and the first version of this had only the middle one, which made the whole + * check free to defeat. Prepending `const __unused = { anything: user.id };` to every body raised + * `settings.sso` and `settings.team`, the only two findings `auth-scope` has ever produced and both + * confirmed cross-org exposures, because any property at all taking a caller id counted wherever it + * sat. `dead-caller-scope-object` and `dead-caller-scope-userid` in the mutation corpus are the two + * halves of that shape. + * + * So the property NAME has to be an identity field, and the object it sits in has to be handed to a + * call that could scope a query. The third condition is what stops `logger.error("create failed", + * { userId: user.id })`, written anywhere in a builder-wrapped handler, clearing the check for that + * export. See `isHandedToAScopingCall` for what that does and does not refuse. + */ +function scopesByCallerIn(fns: Iterable): boolean { + let found = false; + const visit = (node: ts.Node) => { + if (found) return; + if ( + ts.isPropertyAssignment(node) && + ts.isIdentifier(node.name) && + CALLER_ID_FIELD.test(node.name.text) + ) { + const path = propertyPath(node.initializer); + if (path !== null && CALLER_ID_PATH.test(path) && isHandedToAScopingCall(node)) { + found = true; + return; + } + } + ts.forEachChild(node, visit); + }; + for (const fn of fns) { + if (fn.body) visit(fn.body); + } + return found; +} + +/** Callee as recorded in `calleeNames`: the identifier, or the property for a member call. */ +function calleeName(expr: ts.Expression): string | null { + const target = unwrap(expr); + if (ts.isIdentifier(target)) return target.text; + if (ts.isPropertyAccessExpression(target)) return target.name.text; + return null; +} + +/** + * The whole callee path of a call, `prisma.organization.findFirst` rather than `findFirst`. Used to + * match a call against `LOGGER_CALLEE` and `PARSE_CALLEE`. Null when the path runs through something + * with no name of its own, e.g. `new PromptService().createOverride`, where the caller falls back to + * the bare name. + */ +function calleeText(expr: ts.Expression): string | null { + const target = unwrap(expr); + if (ts.isIdentifier(target)) return target.text; + if (target.kind === ts.SyntaxKind.ThisKeyword) return "this"; + if (ts.isPropertyAccessExpression(target)) { + const base = calleeText(target.expression); + return base === null ? null : `${base}.${target.name.text}`; + } + if (ts.isCallExpression(target)) { + const base = calleeText(target.expression); + return base === null ? null : `${base}()`; + } + return null; +} + +/** `logger.error`, `log.info`, `this.logger.debug`. */ +const LOGGER_CALLEE = /(^|\.)(logger|log)\.[A-Za-z_$][\w$]*$/; + +/** Property names on the first object-literal argument, e.g. `{ environmentId, error }`. */ +function objectArgumentFields(call: ts.CallExpression): string[] { + for (const arg of call.arguments) { + const target = unwrap(arg); + if (!ts.isObjectLiteralExpression(target)) continue; + const fields: string[] = []; + for (const property of target.properties) { + const name = propertyName(property); + if (name) fields.push(name); + } + return fields; + } + return []; +} + +/** + * Calls that turn input into a value and throw when it is malformed. `parse`/`safeParse` cover + * `JSON.parse` and the zod schemas. `.json` has to be a member call, because a bare `json(...)` is + * the remix response helper, which every route calls and which parses nothing. + */ +const PARSE_CALLEE = /(^|\.)(parse|safeParse|parseAsync|safeParseAsync|decode)$|\.json$/; + +/** + * Constructors that parse untrusted input and throw when it is malformed. Deliberately short: any + * constructor at all would mean `new BranchesPresenter()` or `new Set(...)` excuses a catch that + * guards ordinary work, which was true of 77 try blocks in the route tree. + */ +const PARSE_CONSTRUCTORS = new Set(["URL", "URLSearchParams", "RegExp"]); + +function isParseCall(node: ts.Node): boolean { + if (ts.isNewExpression(node)) { + return ts.isIdentifier(node.expression) && PARSE_CONSTRUCTORS.has(node.expression.text); + } + if (!ts.isCallExpression(node)) return false; + const text = calleeText(node.expression) ?? calleeName(node.expression); + return text !== null && PARSE_CALLEE.test(text); +} + +/** + * Body reads: the thing a parse guard waits for before it parses. `request.json()` is in + * `PARSE_CALLEE` already because it reads and parses in one call, and these are the same operation + * with the parse written separately, `const raw = await request.text(); new RegExp(raw);`. + * + * Only consulted for `awaitsOnlyParse`, never for `guardsParse`, which is what bounds it: a body + * read on its own still does not make a try block a parse guard, so the widest this list can do is + * let a block that already parses also read the thing it parses. + */ +const BODY_READ_METHODS = new Set(["text", "formData", "arrayBuffer", "blob", "bytes"]); + +function isBodyRead(node: ts.Node): boolean { + if (!ts.isCallExpression(node)) return false; + const callee = unwrap(node.expression); + return ts.isPropertyAccessExpression(callee) && BODY_READ_METHODS.has(callee.name.text); +} + +/** + * Syntax that can raise. Everything a try block might do that produces something for a catch clause + * to catch: a call, a construction, a tagged template, an `await` or `yield` (the awaited promise + * rejects), a member access (the base may be null or undefined), a `throw`, an iteration (the + * iterator protocol raises on a non-iterable), and `instanceof`/`in` (a TypeError on a non-object + * right side). + * + * See `guardedWork` for what is NOT on this list and why that is a disclosed residual rather than + * an oversight. + */ +function canRaise(node: ts.Node): boolean { + return ( + ts.isCallExpression(node) || + ts.isNewExpression(node) || + ts.isTaggedTemplateExpression(node) || + ts.isAwaitExpression(node) || + ts.isYieldExpression(node) || + ts.isPropertyAccessExpression(node) || + ts.isElementAccessExpression(node) || + ts.isThrowStatement(node) || + ts.isForOfStatement(node) || + ts.isForInStatement(node) || + (ts.isBinaryExpression(node) && + (node.operatorToken.kind === ts.SyntaxKind.InstanceOfKeyword || + node.operatorToken.kind === ts.SyntaxKind.InKeyword)) + ); +} + +/** + * What the guarded region does, in the three terms `error-classification` needs. + * + * `guardsParse` is whether anything in it parses at all. A `new URL(x)` counts and has to be read + * as a `ts.isNewExpression` here, because the call-callee scan that builds `calleeNames` never sees + * it. + * + * `awaitsOnlyParse` is whether everything the block waits for is a parse or a read of the body it + * parses. Awaiting is the signal, not calling: the calls that prepare a parse's input are ordinary + * synchronous string work (`matchPattern.startsWith("(?i)")`, `.slice(4)` before a `new RegExp`), + * and refusing those refuses four of the tree's clearest guards, while the swallow this has to + * catch reaches a service: `try { const body = await request.json(); return await + * handleEverything(body); }`. + * + * `canRaise` is whether the block does anything at all that could reach the clause. A clause whose + * try block cannot raise is not error handling, and reading one as classification paid 50 points a + * route to anyone willing to prepend `try { 0; } catch (e) { if (e instanceof Error) { return + * json(x, { status: 400 }); } throw e; }` to a body: on the real tree that took the global from 15 + * to 42 and raised 224 routes when it was measured, before round C moved the baseline to 19. More + * than every other shape found on this branch put together, and still true at today's figures, which + * `dead-classifying-try-with-call` shows live at 19 to 44. `dead-classifying-try` in the mutation + * corpus is the refused version. + * + * What this refuses is `try { 0; }` and nothing cleverer. `canRaise` accepts ANY call, member + * access or `in`, and none of those has to be able to throw, so one inert call defeats the rule: + * `try { String(0); } catch (e) { if (e instanceof Error) { return json(x, { status: 400 }); } throw + * e; }` reads as classification and takes the tree from 19 to 44, exactly as `try { 0; }` did. + * `dead-classifying-try-with-call` in the mutation corpus is that shape, running as an expected + * failure. Telling a call that can throw from one that cannot needs types the scanner does not have, + * so the rule closes the shape found rather than the family it belongs to. Read the docstrings that + * point here as "refuses `try { 0; }`", never as "an unreachable catch cannot be credited". + * + * The list also misses things that CAN raise, which is the safe direction, and the misses matter + * because a real clause can be dropped by one: a destructuring declaration (`const { a } = undefined` + * throws), a temporal-dead-zone read (`try { const x = later; }`), a coercion that raises + * (`try { const x = 1 + someSymbol; }`) and a `delete` on a frozen object all read as unable to + * raise. + * + * Nested function bodies are skipped throughout: a callback written inside the try is not work the + * try is guarding on this pass through. A `throw` inside one is not either, which is deliberate. + */ +function guardedWork(tryBlock: ts.Block): { + guardsParse: boolean; + awaitsOnlyParse: boolean; + guardCanRaise: boolean; +} { + let guardsParse = false; + let awaitsOnlyParse = true; + let guardCanRaise = false; + const visit = (node: ts.Node) => { + if (ts.isFunctionLike(node)) return; + if (isParseCall(node)) guardsParse = true; + if (canRaise(node)) guardCanRaise = true; + if (ts.isAwaitExpression(node)) { + const awaited = unwrap(node.expression); + if (!isParseCall(awaited) && !isBodyRead(awaited)) awaitsOnlyParse = false; + } + ts.forEachChild(node, visit); + }; + visit(tryBlock); + return { guardsParse, awaitsOnlyParse, guardCanRaise }; +} + +/** Whether some node in the tree rooted at `node` matches `predicate`. */ +function someNode(node: ts.Node, predicate: (n: ts.Node) => boolean): boolean { + if (predicate(node)) return true; + return ts.forEachChild(node, (child) => someNode(child, predicate)) === true; +} + +function containsInstanceOf(node: ts.Node): boolean { + return someNode( + node, + (n) => ts.isBinaryExpression(n) && n.operatorToken.kind === ts.SyntaxKind.InstanceOfKeyword + ); +} + +/** + * Whether a binding name pattern declares `target`, recursively: a plain `error`, a destructured + * `{ error }` (shorthand) or `{ code: error }` (renamed), an array pattern `[error]`, and any of + * those nested inside another. A destructured parameter or declaration re-declares the name just as + * completely as a plain one does, so a shadow check that only recognised `ts.isIdentifier` missed + * every destructured shape, function parameters and variable declarations alike. + */ +function bindingDeclares(name: ts.BindingName, target: string): boolean { + if (ts.isIdentifier(name)) return name.text === target; + for (const element of name.elements) { + if (!ts.isOmittedExpression(element) && bindingDeclares(element.name, target)) return true; + } + return false; +} + +/** Whether `name` is declared by a var/let/const, function or class statement directly in this + * statement list. Not recursive: a nested block's own declarations are handled when the walk + * reaches that block. */ +function declaresInScope(statements: readonly ts.Statement[], name: string): boolean { + for (const statement of statements) { + if (ts.isFunctionDeclaration(statement) && statement.name?.text === name) return true; + if (ts.isClassDeclaration(statement) && statement.name?.text === name) return true; + if ( + ts.isVariableStatement(statement) && + statement.declarationList.declarations.some((d) => bindingDeclares(d.name, name)) + ) { + return true; + } + } + return false; +} + +/** + * Whether `node` contains a genuine read of the given catch binding, e.g. `e` in `e instanceof X` + * or `error.code`. An identifier only counts when it is a real reference. Two shapes share the + * binding's text without reading it: the property side of a member expression (`fallback.error`) + * and an object literal key (`{ error: true }`), both excluded by checking which side of the + * parent node the identifier sits on. A name re-declared in a nested scope, as a function or catch + * parameter (including a destructured one) or as a var/let/const/function/class in a block, refers + * to that declaration instead, so the walk stops at the boundary that re-declares it rather than + * crediting the outer binding. + */ +function referencesBinding(node: ts.Node, bindingName: string): boolean { + if (ts.isIdentifier(node) && node.text === bindingName) { + const parent = node.parent; + if (ts.isPropertyAccessExpression(parent) && parent.name === node) return false; + if (ts.isPropertyAssignment(parent) && parent.name === node) return false; + return true; + } + + if ( + ts.isFunctionLike(node) && + node.parameters.some((p) => bindingDeclares(p.name, bindingName)) + ) { + return false; + } + + if (ts.isCatchClause(node)) { + const decl = node.variableDeclaration; + if (decl && bindingDeclares(decl.name, bindingName)) return false; + } + + if (ts.isBlock(node) && declaresInScope(node.statements, bindingName)) return false; + + return ts.forEachChild(node, (child) => referencesBinding(child, bindingName)) === true; +} + +/** The catch binding's name, or null for a bindingless `catch { ... }` or a destructured one. */ +function catchBindingName(clause: ts.CatchClause): string | null { + const decl = clause.variableDeclaration; + return decl && ts.isIdentifier(decl.name) ? decl.name.text : null; +} + +/** A node's source text with all whitespace removed, for comparing two branch arms. */ +function normalizedText(node: ts.Node): string { + return node.getText().replace(/\s+/g, ""); +} + +/** + * Whether a conditional expression tests the error to pick what the clause does, rather than to + * word what it says. The caller only offers it the whole value of a `return`/`throw`, so + * `return e instanceof Response ? e : json({}, { status: 500 })` reaches here and + * `return json({ error: e instanceof Error ? e.message : String(e) }, { status: 400 })` does not. + * The second is message formatting: every error leaves by the same path. + * + * Goes through `referencesBinding`, the same predicate the `if`/`switch` check uses, rather than + * accepting any `instanceof` in the condition: an `instanceof` that never reads the caught binding + * is not a decision made on the error, and a bindingless catch has nothing here to reference. + * + * The two arms also have to differ, which is the same requirement `selectsADistinctPath` makes of + * an `if`. `return e instanceof Error ? (X) : (X)` is a test whose outcome is the same either way, + * and it was worth 50 points a route; `same-arms-ternary` in the mutation corpus is the tree-scale + * version, and `scan.test.ts` has the unit case. The throw path is held to the same rule by + * `wrap-body-in-same-arms-throw-ternary`, which would take every route in the tree to a pass if it + * were not. Parentheses and whitespace are stripped + * before the comparison, so the shape has to differ in something a reader would call a difference. + * The residual both branch tests share is stated once, on `selectsADistinctPath`. + */ +function selectsAnErrorPath(node: ts.ConditionalExpression, bindingName: string | null): boolean { + if (bindingName === null) return false; + if (!containsInstanceOf(node.condition)) return false; + if (!referencesBinding(node.condition, bindingName)) return false; + return normalizedText(unwrap(node.whenTrue)) !== normalizedText(unwrap(node.whenFalse)); +} + +/** + * Which bare (unlabelled) jumps, at this point in the recursion, leave the statement list the + * question is being asked about. A bare jump targets the nearest enclosing construct of its kind, + * so descending past one of those targets changes the answer for the jumps it captures. + */ +type BareJumps = { break: boolean; continue: boolean }; + +/** A jump written directly in the list under question always leaves it: whatever it targets + * encloses the list. */ +const ESCAPES: BareJumps = { break: true, continue: true }; + +/** A `do` body, asked about from the list the `do` sits in. `break` ends the loop and `continue` + * goes to the condition, and both of those reach the statement written after the `do`. */ +const IN_DO_BODY: BareJumps = { break: false, continue: false }; + +/** + * A statement that leaves the statement list it sits in on every path through itself, so anything + * after it in the same list never runs. + * + * Recognises a nested construct, not only a bare `return`/`throw`/`break`/`continue`. Recognising + * only the bare form is what let a dead `throw error;` count as a rethrow when the statement before + * it was a block, a `do` body or an `if`/`else` that returned; `dead-throw-after-*` in the mutation + * corpus is that family, and `scan.test.ts` has one case per construct. + * + * A bare `break` or `continue` only counts where it actually leaves the list, which is what `jumps` + * carries. A `break` inside a switch clause targets the switch, so a switch whose clauses all break + * falls through to the statement after it and does NOT exit; reading that break as an exit accused + * `catch (e) { switch (e.code) { ... break; } throw e; }` of swallowing a rethrown error, which is + * both a false verdict and a detail line that says the opposite of what the route does. A `continue` + * inside a switch clause targets an enclosing loop instead, which the switch cannot be, so it is + * inherited rather than dropped: dropping it would stop + * `do { switch (x) { default: continue; } throw e; } while (c)` cutting a throw that really is dead. + * `break and continue inside the construct they target` in `scan.test.ts` holds both halves. + * + * A labelled `break`/`continue` always counts. Its target has to enclose the statement list, since + * nothing between the list and the jump can carry the label: `definitelyExits` answers false for a + * labelled statement, so the recursion never descends through one. + * + * A sound under-approximation. `if` without an `else` (unless its guard is the literal `true` + * keyword, the one condition this function folds), a labelled statement (a `break` to the label + * escapes it) and every other loop form answer false, because none of them is guaranteed to run its + * body. That extends to a `do` that never falls through, `do { continue; } while (true)`, which is + * false for the same reason `while (true) { }` always was: separating it from + * `do { continue; } while (c)` means folding a LOOP condition, which this function still does not + * do. Saying false when the truth is true only leaves a later statement in the list, which + * is the direction that withholds evidence rather than inventing it. + */ +function definitelyExits(statement: ts.Statement, jumps: BareJumps = ESCAPES): boolean { + if (ts.isReturnStatement(statement) || ts.isThrowStatement(statement)) return true; + if (ts.isBreakStatement(statement)) return statement.label !== undefined || jumps.break; + if (ts.isContinueStatement(statement)) return statement.label !== undefined || jumps.continue; + if (ts.isBlock(statement)) { + return statement.statements.some((s) => definitelyExits(s, jumps)); + } + // A `do` body runs before its condition is ever read. + if (ts.isDoStatement(statement)) return definitelyExits(statement.statement, IN_DO_BODY); + if (ts.isIfStatement(statement)) { + // A guard that is exactly the `true` keyword always takes its then-arm, so the statement + // definitely exits iff that arm does, with or without an else. Keyword-exact, same spelling + // rule as the walk's `if (true)` entry and for the same reason: this GRANTS a reachability + // cut, and a wrong grant pays. `cuts a dead trailing statement after an if true that exits` + // is the pin; `dead-throw-after-if-true` and `dead-branch-after-if-true` in the mutation + // corpus are the tree-scale versions. + if (unwrap(statement.expression).kind === ts.SyntaxKind.TrueKeyword) { + return definitelyExits(statement.thenStatement, jumps); + } + return ( + statement.elseStatement !== undefined && + definitelyExits(statement.thenStatement, jumps) && + definitelyExits(statement.elseStatement, jumps) + ); + } + if (ts.isTryStatement(statement)) { + if (statement.finallyBlock && definitelyExits(statement.finallyBlock, jumps)) return true; + if (!definitelyExits(statement.tryBlock, jumps)) return false; + return ( + statement.catchClause === undefined || definitelyExits(statement.catchClause.block, jumps) + ); + } + if (ts.isSwitchStatement(statement)) { + const inClause: BareJumps = { break: false, continue: jumps.continue }; + const clauses = statement.caseBlock.clauses; + const last = clauses[clauses.length - 1]; + if (!clauses.some(ts.isDefaultClause) || last === undefined) return false; + // An empty clause falls through to the next one, so it does not have to exit itself; the last + // clause has nothing to fall through to and does. + return ( + clauses.every( + (c) => c.statements.length === 0 || c.statements.some((s) => definitelyExits(s, inClause)) + ) && last.statements.some((s) => definitelyExits(s, inClause)) + ); + } + return false; +} + +/** `statements` up to and including the first one that definitely exits. */ +function reachableStatements(statements: readonly ts.Statement[]): readonly ts.Statement[] { + // The arrow matters: `findIndex` passes an index as the second argument, which `definitelyExits` + // would read as its `jumps` record. + const index = statements.findIndex((s) => definitelyExits(s)); + return index === -1 ? statements : statements.slice(0, index + 1); +} + +/** + * Whether the tree rooted at `node` contains a `break` or `continue` that would leave it, i.e. a + * jump no construct INSIDE `node` captures. What the catch walk asks of a finally block before + * entering the tryBlock beside it: a finally that completes abruptly cancels the try's completion, + * so a throw in that tryBlock never leaves the clause and crediting it minted evidence + * (`reads a throw a finally break discards as no rethrow` and its continue and switch-hosted + * siblings pin the refusal; `dead-throw-in-cancelled-try` in the mutation corpus is the tree-scale + * shape, 80 routes when measured). + * + * A containment read, not a liveness one, on purpose: the caller is deciding whether to GRANT + * credit and a wrong grant pays, so a jump that only may run still refuses + * (`refuses the tryBlock when the finally only may break`). Two over-approximations in the same + * direction: a labelled jump always counts, even when its label sits inside `node`, and a `return` + * is not looked for here because the returns veto already reads it off the whole statement. + */ +function containsEscapingJump(node: ts.Node, jumps: BareJumps = ESCAPES): boolean { + if (ts.isFunctionLike(node)) return false; + if (ts.isBreakStatement(node)) return node.label !== undefined || jumps.break; + if (ts.isContinueStatement(node)) return node.label !== undefined || jumps.continue; + if (ts.isSwitchStatement(node)) { + const inClause: BareJumps = { break: false, continue: jumps.continue }; + return node.caseBlock.clauses.some((c) => + c.statements.some((s) => containsEscapingJump(s, inClause)) + ); + } + // Any loop captures both bare jumps, so nothing inside one can leave `node` + // (`does not refuse a finally whose loop captures its own break`). + if (ts.isIterationStatement(node, false)) { + return ( + ts.forEachChild(node, (child) => + containsEscapingJump(child, { break: false, continue: false }) + ) === true + ); + } + return ts.forEachChild(node, (child) => containsEscapingJump(child, jumps)) === true; +} + +/** + * Literal truthiness of a guard expression: true, false, or null when not decidable from the + * token alone. Only literal tokens fold; an identifier, call, bigint, `&&`, `||` or a template + * literal with substitutions is always null, so a live guard can never be read as dead. The + * always-true side is pinned by `still refuses an error test after an always-true spelling that + * throws` and the fall-through slice by `reads a switch fall-through onto a live return as live`. + */ +function literalTruth(expr: ts.Expression): boolean | null { + const target = unwrap(expr); + // The five bare-literal kinds are `literalValue`'s list, not a second copy of it: this was the + // same five node-kind tests written out three lines above the function that already had them, + // differing only in returning the truthiness rather than the value. + const literal = literalValue(target); + if (literal !== undefined) return Boolean(literal); + if (ts.isPrefixUnaryExpression(target) && target.operator === ts.SyntaxKind.ExclamationToken) { + const inner = literalTruth(target.operand); + return inner === null ? null : !inner; + } + if (ts.isBinaryExpression(target)) { + const op = target.operatorToken.kind; + if ( + op === ts.SyntaxKind.EqualsEqualsEqualsToken || + op === ts.SyntaxKind.ExclamationEqualsEqualsToken + ) { + const left = literalValue(target.left); + const right = literalValue(target.right); + if (left === undefined || right === undefined) return null; + const equal = left === right; + return op === ts.SyntaxKind.EqualsEqualsEqualsToken ? equal : !equal; + } + } + return null; +} + +/** The value of a literal token, or undefined when the expression is not a bare literal. */ +function literalValue(expr: ts.Expression): string | number | boolean | null | undefined { + const target = unwrap(expr); + if (target.kind === ts.SyntaxKind.TrueKeyword) return true; + if (target.kind === ts.SyntaxKind.FalseKeyword) return false; + if (target.kind === ts.SyntaxKind.NullKeyword) return null; + if (ts.isStringLiteral(target) || ts.isNoSubstitutionTemplateLiteral(target)) return target.text; + if (ts.isNumericLiteral(target)) return Number(target.text); + return undefined; +} + +/** Whether a try block could throw at all: false only when every statement is an expression + * statement over a bare literal, the one shape that provably cannot raise. */ +function tryBlockMayThrow(block: ts.Block): boolean { + return !block.statements.every( + (s) => ts.isExpressionStatement(s) && literalValue(s.expression) !== undefined + ); +} + +/** + * Whether the tree rooted at `root` contains a node `hit` accepts that a provably-untaken branch + * does not already rule out. A plain containment walk, minus the hits it can prove never run: + * `if (false) { throw e; }` contains a throw and can never run one. + * + * Folds literal guards only, so wherever `literalTruth` cannot decide, every hit the plain walk + * would have found is still found. That makes this strictly subtractive against containment, which + * is what lets both of its callers read it for opposite purposes: + * + * - `catchClauseEvidence`'s `exited` flag, where a hit BLINDS the walk to whatever follows. + * Containment blinded it on a dead statement, so prepending one to a deciding clause turned its + * pass into a swallow verdict on 78 real routes. Subtracting dead hits only ever un-blinds. + * - `selectsADistinctPath`, where a hit GRANTS a branch. Containment granted one for an arm whose + * only exit was dead, which is `dead-armed-instanceof-if` in the mutation corpus, measured at 80 + * routes and the tree from 19 to 27. Subtracting dead hits only ever withholds. + * + * The `exited` half is pinned by the mirror twins under `dead and deferred code prepended to a + * deciding catch does not blind it` (recovered) and the `BRANCH_EXITED` family (refusing). The + * `selectsADistinctPath` half is pinned by `an arm whose only exit is dead decides nothing` and its + * siblings, plus the corpus entry. + */ +function containsLiveWhere(root: ts.Node, hit: (n: ts.Node) => boolean): boolean { + const walk = (node: ts.Node): boolean => { + if (ts.isFunctionLike(node)) return false; + if (hit(node)) return true; + if (ts.isIfStatement(node)) { + const truth = literalTruth(node.expression); + if (truth === true) return walk(node.thenStatement); + if (truth === false) { + return node.elseStatement !== undefined && walk(node.elseStatement); + } + return ( + walk(node.thenStatement) || (node.elseStatement !== undefined && walk(node.elseStatement)) + ); + } + if (ts.isWhileStatement(node)) { + if (literalTruth(node.expression) === false) return false; + return walk(node.statement); + } + if (ts.isForStatement(node)) { + if (node.condition !== undefined && literalTruth(node.condition) === false) return false; + return ts.forEachChild(node, walk) === true; + } + if (ts.isForOfStatement(node) || ts.isForInStatement(node)) { + const iterable = unwrap(node.expression); + const emptyArray = ts.isArrayLiteralExpression(iterable) && iterable.elements.length === 0; + const emptyObject = + ts.isForInStatement(node) && + ts.isObjectLiteralExpression(iterable) && + iterable.properties.length === 0; + if (emptyArray || emptyObject) return false; + return ts.forEachChild(node, walk) === true; + } + if (ts.isSwitchStatement(node)) { + const disc = literalValue(node.expression); + const clauses = node.caseBlock.clauses; + const allLiteral = + disc !== undefined && + clauses.every((c) => ts.isDefaultClause(c) || literalValue(c.expression) !== undefined); + if (!allLiteral) return clauses.some((c) => c.statements.some(walk)); + // Fall-through: from the first matching (or default) clause, every later clause is reachable. + let matched = clauses.findIndex( + (c) => !ts.isDefaultClause(c) && literalValue(c.expression) === disc + ); + if (matched === -1) matched = clauses.findIndex(ts.isDefaultClause); + if (matched === -1) return false; + return clauses.slice(matched).some((c) => c.statements.some(walk)); + } + if (ts.isTryStatement(node)) { + // A finally that always completes abruptly (a return, a throw, or a jump out of the block) + // supersedes the try's and the catch's completion: an exit written in either never leaves + // the statement, so only the finally's own statements stay live. Folded only when + // `definitelyExits` can prove it; a conditional jump keeps the containment answer, the + // direction that refuses credit rather than inventing it. Without this fold the + // `dead-throw-in-cancelled-try` prepend blinded the walk to every real classification below + // it (`keeps the classification after a finally-break no-op`). + if (node.finallyBlock !== undefined && definitelyExits(node.finallyBlock)) { + return walk(node.finallyBlock); + } + if (walk(node.tryBlock)) return true; + if (node.finallyBlock !== undefined && walk(node.finallyBlock)) return true; + if (node.catchClause !== undefined && tryBlockMayThrow(node.tryBlock)) { + return walk(node.catchClause.block); + } + return false; + } + return ts.forEachChild(node, walk) === true; + }; + return walk(root); +} + +function containsLiveExit(node: ts.Node): boolean { + return containsLiveWhere(node, (n) => ts.isReturnStatement(n) || ts.isThrowStatement(n)); +} + +function containsLiveReturn(node: ts.Node): boolean { + return containsLiveWhere(node, ts.isReturnStatement); +} + +/** + * Whether an `if`/`switch` sends at least one arm somewhere the others do not go, by returning or + * throwing from inside it. `if (e instanceof Error) { }` and `if (e instanceof Error) { log(e); }` + * both fail this: every error still leaves the clause by the same path afterwards, so the test + * changed the wording and not the outcome. The empty-body form was the cheapest no-op in the tool, + * worth 50 points a route; `empty-instanceof-if` in the mutation corpus is the tree-scale version. + * + * An `if`/`else` whose two arms are textually identical does not count, the same comparison + * `selectsAnErrorPath` makes of a ternary's arms. + * + * The exit an arm is credited for has to be a LIVE one, `containsLiveExit` and never a plain + * containment read. `if (e instanceof Error) { if (false) { return null; } }` contains an exit that + * can never run, so under containment it read as a real decision and took a swallowing catch to a + * pass for the price of a mechanical edit: 80 routes and the tree from 19 to 27 when measured. The + * same liveness rule had already been put on `catchClauseEvidence`'s `exited` flag, for the same + * eleven dead spellings, and this predicate beside it kept the containment read. `an arm whose only + * exit is dead decides nothing` and its siblings are the unit pins; `dead-armed-instanceof-if` in + * the mutation corpus is the tree-scale version. Being subtractive against containment, the fold + * can only ever withhold a branch, never invent one, so a live arm reads exactly as it did. + * + * The residual both branch tests share, stated here once for both: two arms that produce the same + * outcome by different spellings still read as a real decision. + * `if (e instanceof Error) { return json(x); } return Response.json(x);` counts and decides + * nothing, and so does the `if` with no `else` whose arm returns what the statement after it + * returns. Telling those apart needs the produced values compared for meaning rather than for text, + * which is a different kind of analysis from anything else in this file. The textual comparison is + * the cheapest thing that catches the copy-paste form, which is the one a mutation produces. + */ +function selectsADistinctPath(statement: ts.IfStatement | ts.SwitchStatement): boolean { + if (ts.isIfStatement(statement)) { + const otherwise = statement.elseStatement; + if (otherwise !== undefined) { + if (normalizedText(statement.thenStatement) === normalizedText(otherwise)) return false; + return containsLiveExit(statement.thenStatement) || containsLiveExit(otherwise); + } + return containsLiveExit(statement.thenStatement); + } + // Per clause statement rather than over the whole switch, so a live exit in any clause counts + // whatever the discriminant is. Reading the switch as one node would hand `containsLiveWhere`'s + // discriminant fold a `switch (e.code)` it cannot decide, which changes nothing, and a + // `switch (1)` it can, which is not this predicate's business: an unreachable CLAUSE is caught + // by the same fold one level down, and the statement is only reached at all when its condition + // references the caught binding. + return statement.caseBlock.clauses.some((clause) => clause.statements.some(containsLiveExit)); +} + +/** + * What a catch clause does with the error, beyond the fact that it caught one. + * + * Both answers are read off the clause's own guaranteed path. The governing rule: the walk may + * enter a construct exactly where the entered statements are guaranteed to execute whenever the + * clause body runs, so no credit can ever come from code a semantics-preserving edit could have + * added dead. Entered on those terms: a bare nested block, a `do` body, the tryBlock of a `try` + * that has NO catch clause and whose finally (if any) contains no jump out of itself (a finally + * that completes abruptly cancels the try's completion, so nothing in that tryBlock ever escapes + * the clause; `reads a throw a finally break discards as no rethrow` and + * `dead-throw-in-cancelled-try` in the mutation corpus hold it), the sole clause of a + * single-DefaultClause `switch`, the then-arm of an + * `if` whose condition is exactly the literal `true` keyword, and both arms of an `if`/`else` with + * per-arm states merged by intersection (evidence in both arms is unconditional; evidence in one + * is not). Each entry is pinned by `reads a clause wrapped in a single-default switch as the bare + * clause` and its sibling identity pairs. + * + * NOT entered, deliberately: a bare `if` without an else (except the literal-true case), loops + * other than `do` (a body that may run zero times), labelled statements, function-like nodes (the + * iteration-callback boundary is `walkBody`'s attribution rule and this walk never crosses any + * function boundary), nested catch clauses, finally blocks, and the tryBlock of a `try` WITH a + * catch clause, where a throw is intercepted by the nested catch rather than escaping the clause. + * A `throw` or a test in any of those positions does not count. + * + * That is the whole dead-code defence, and it replaces the list of statically-false shapes an + * earlier round kept extending. The list was losing: `if (false)` and `while (false)` were + * recognised, and `for (;false;)`, `if (true) {} else`, `switch (1) { case 2: }`, `try {} catch`, + * `for (const x of [])`, `for (const k in {})`, `if ("")`, `if (!true)` and `if (1 === 2)` were not, + * each worth 50 points a route. Asking for the throw to be unconditional refuses all eleven without + * naming any of them. `dead-*` in the mutation corpus is the tree-scale proof, one entry per shape. + * + * `rethrows` asks for one thing more: that the clause contains no `return` at all. The claim it + * feeds is that the clause passes the error through unchanged, which is only true when throwing is + * the ONLY way out. Without it a `throw error;` written after a statement that already exited read + * as a rethrow, in seven spellings: after a bare block, a `do` body, an `if (true)`, an `if`/`else` + * where both arms return, a `switch` with a returning default, and a `try`/`finally` that returns. + * `definitelyExits` handles every one of those, including the `if (true)` spelling since it folds + * the literal `true` keyword, and the no-return rule holds the rest of the line. `dead-throw-after-*` + * in the mutation corpus covers them. + * + * The cost is real, in both rules. `catch (e) { if (transient) throw e; return null; }` no longer + * reads as a rethrow, so it reads as a swallow and fails rather than sitting out, and neither does + * `catch (e) { if (e instanceof Response) return e; throw e; }`, which passes on its branch instead. + * That is the direction to be wrong in, since the reverse hands out points. + */ +function catchClauseEvidence(clause: ts.CatchClause): { + rethrows: boolean; + throws: boolean; + branches: boolean; +} { + // `rethrows`, `branches` and `exited` travel in a state record so a walk can be run against an + // isolated copy (the if/else arm walks) as well as the shared root. `returns` stays a single + // shared flag: it is a clause-wide veto, never per-arm evidence. + // + // On `exited`: set once a statement the walk has already passed could have left the clause. An + // error test + // after one of those is dead code, so it decides nothing. Raised at the END of each statement, + // after that statement's own branch check: a deciding statement contains an exit by definition, + // so raising it first makes every such statement refuse itself, which was measured at 78 routes + // losing their pass and the tree dropping from 15 to 6, measured before round C moved the baseline + // to 19. This ordering leaves the real-tree report + // and all 240 clauses' evidence byte-identical. The tests are the cases in `dead throw written + // after something that already exited`. + // + // Raised off `containsLiveExit`, never a plain containment read. Containment is true of + // `if (false) { throw e; }` itself, so a provably dead statement raised the flag and blinded the + // walk to the real classification below it: prepending one to a deciding clause turned its pass + // into a swallow verdict on 78 real routes, the same false accusation for all eleven dead + // spellings. The liveness fold only ever withholds this blindness; where `literalTruth` cannot + // decide, the containment answer stands and refusal is intact. The recovered half is `dead and + // deferred code prepended to a deciding catch does not blind it`; the refusing half is the + // `BRANCH_EXITED` list plus `still refuses an error test after an always-true spelling that + // throws`. + // + // `vetoReturns` is whether this walk's statements may feed the `returns` veto. True everywhere + // except the if/else arm walks: `returns` is read at the PARENT level, as a live containment + // read over the whole statement, so an arm walk re-reading its own statements adds nothing for + // a live guard and adds a false veto for a folded-dead arm (the walk enters both arms; the fold + // has already excluded the dead one from the parent read). `dead-classifier-one-arm` in the + // mutation corpus is the tree-scale shape this protects. + type ClauseState = { + rethrows: boolean; + branches: boolean; + exited: boolean; + vetoReturns: boolean; + }; + let returns = false; + const state: ClauseState = { rethrows: false, branches: false, exited: false, vetoReturns: true }; + const bindingName = catchBindingName(clause); + + const walk = (statements: readonly ts.Statement[], state: ClauseState) => { + // A block that re-declares the binding name means an `if` below it referencing that name is + // referencing the shadowing declaration, not this clause's error. Nothing in such a block can + // speak for the clause, so the whole list is skipped for branch purposes. + const shadowed = bindingName !== null && declaresInScope(statements, bindingName); + + for (const statement of reachableStatements(statements)) { + if (ts.isThrowStatement(statement)) { + state.rethrows = true; + // Read the branch check here, before the path is cut. A thrown ternary picks WHICH error + // leaves, which is a classification, and reading it only at the shared check below meant + // the throw arm of that condition was unreachable: this arm always continued first. So + // `throw e instanceof Response ? e : new ServerError(e)` read as inert while the same + // clause written with `return` passed. `selectsAnErrorPath` is the same predicate either + // way, so the same-arms rule applies and `throw e instanceof Error ? e : e;` is refused. + if ( + bindingName !== null && + !shadowed && + !state.exited && + statement.expression !== undefined + ) { + const thrown = unwrap(statement.expression); + if (ts.isConditionalExpression(thrown) && selectsAnErrorPath(thrown, bindingName)) { + state.branches = true; + } + } + state.exited = true; + continue; + } + if (ts.isBlock(statement)) { + walk(statement.statements, state); + if (containsLiveExit(statement)) state.exited = true; + continue; + } + // A `do` body runs before its condition is ever read, so it is on the straight-line path + // whatever the condition says. The only loop form that is; `definitelyExits` agrees. + if (ts.isDoStatement(statement)) { + const body = statement.statement; + walk(ts.isBlock(body) ? body.statements : [body], state); + if (containsLiveExit(statement)) state.exited = true; + continue; + } + // The three handlers below share one template: walk the inner list with the SAME shared + // state, then read `returns` and `exited` off the whole statement and continue. The explicit + // `containsLiveReturn` read is load-bearing: the `continue` skips the shared read below, and + // `try { throw e; } finally { return null; }` genuinely swallows (the finally return eats + // the throw), so the veto must still see the finally block the walk does not enter. `reads a + // try whose finally returns as swallowing, not rethrowing` is the pin. + // + // A `try` WITHOUT a catch clause: its tryBlock always runs when the clause body does, and a + // throw there escapes the clause, so rethrow credit is genuine β€” unless the finally can + // complete abruptly. A `finally` holding a `return` is covered by the explicit + // `containsLiveReturn` read below; a `finally` holding a `break` or `continue` that leaves + // it cancels the try's completion the same way, so the throw never escapes and the tryBlock + // must not be entered (`reads a throw a finally break discards as no rethrow`, its continue + // and switch-hosted siblings, and `refuses the tryBlock when the finally only may break`; + // `dead-throw-in-cancelled-try` in the mutation corpus is the tree-scale shape). The refusal + // is a containment read and entry requires its absence, because entry GRANTS credit; the + // matching liveness fold in `containsLiveWhere` then keeps the refused statement from + // blinding what follows it (`keeps the classification after a finally-break no-op`). The + // finallyBlock itself is NOT walked (classification living only in a finally block is + // under-credited; the tree has no such clause). A `try` WITH a catch clause is not entered + // at all: a throw in that tryBlock is intercepted by the nested catch, so crediting it would + // launder a returnless swallow into not-applicable. `does not read the tryBlock of a caught + // try as this clause's rethrow` is the pin, and the nested clause is judged separately as + // its own `ep.catches` entry. + if ( + ts.isTryStatement(statement) && + statement.catchClause === undefined && + (statement.finallyBlock === undefined || !containsEscapingJump(statement.finallyBlock)) + ) { + walk(statement.tryBlock.statements, state); + if (state.vetoReturns && containsLiveReturn(statement)) returns = true; + if (containsLiveExit(statement)) state.exited = true; + continue; + } + // A `switch` whose caseBlock is exactly one DefaultClause: that clause's statements always + // run, as a bare list. A bare `break` in it neither rethrows, branches nor raises `exited` + // (`containsLiveExit` does not count breaks), and `reachableStatements` cuts anything after + // a top-level `break`, which is correct: after a break, nothing in the clause list runs. + // Any other switch shape is not entered and falls through to the branch gate below exactly + // as before, so a real `switch (e.code) { case ...: }` keeps its top-level credit. `reads a + // clause wrapped in a single-default switch as the bare clause` is the pin. + if (ts.isSwitchStatement(statement)) { + const clauses = statement.caseBlock.clauses; + const only = clauses.length === 1 ? clauses[0] : undefined; + if (only !== undefined && ts.isDefaultClause(only)) { + walk(only.statements, state); + if (state.vetoReturns && containsLiveReturn(statement)) returns = true; + if (containsLiveExit(statement)) state.exited = true; + continue; + } + } + // An `if` whose condition (after `unwrap()`) is exactly the `true` keyword: the then-arm + // always runs; the else-arm never does and is NEVER walked (`reads a dead else arm under if + // true as contributing nothing` is the pin). Keyword-exact on purpose: `!!1`, `1` and + // `!false` are deliberately not entry tickets, because entry GRANTS credit and a wrong grant + // pays, where `literalTruth`'s wider folding only withholds blindness. This asymmetry is + // deliberate; do not unify the two folds. Takes precedence over the if/else arm walk below. + if ( + ts.isIfStatement(statement) && + unwrap(statement.expression).kind === ts.SyntaxKind.TrueKeyword + ) { + const arm = statement.thenStatement; + walk(ts.isBlock(arm) ? arm.statements : [arm], state); + if (state.vetoReturns && containsLiveReturn(statement)) returns = true; + if (containsLiveExit(statement)) state.exited = true; + continue; + } + // Any other reachable statement that could return means throwing is not the only way out. + // Read here rather than over the whole clause so a `return` the walk has already cut as dead + // does not count, which is what a `do { throw e; } while (false); return null;` produces. + // The LIVE read, not the containment one: `if (false) { return null; }` holds a return that + // can never run, and vetoing the rethrow on it regressed a rethrow-only clause from + // not-applicable to fail on 11 real routes. `still sets rethrows past a dead return in an + // if (false) arm` is the pin. + if (state.vetoReturns && containsLiveReturn(statement)) returns = true; + + if (bindingName !== null && !shadowed && !state.exited) { + if ( + (ts.isIfStatement(statement) || ts.isSwitchStatement(statement)) && + referencesBinding(statement.expression, bindingName) && + selectsADistinctPath(statement) + ) { + state.branches = true; + } else if (ts.isReturnStatement(statement) && statement.expression !== undefined) { + const value = unwrap(statement.expression); + if (ts.isConditionalExpression(value) && selectsAnErrorPath(value, bindingName)) { + state.branches = true; + } + } + } + + // An `if` WITH an else (its condition not the literal `true` keyword, which the handler + // above already took): one arm always runs, so evidence present in BOTH arms is + // unconditional and evidence in one arm only is conditional and earns nothing. Each arm is + // walked against an isolated state and the results merge into the parent by INTERSECTION. + // Union is the laundering direction: `if (false) { } else { 0; }` must earn + // nothing, which `dead-classifier-one-arm` in the mutation corpus and `does not credit a + // classifier that sits in one arm only` pin. `returns` is never intersected and never + // per-arm: the shared read above already vetoed off the whole statement, over-approximate + // across the live arms, because narrowing a veto per-arm is the unsafe direction. + if (ts.isIfStatement(statement) && statement.elseStatement !== undefined) { + const armWalk = (arm: ts.Statement): ClauseState => { + const armState: ClauseState = { + rethrows: false, + branches: false, + exited: state.exited, + vetoReturns: false, + }; + walk(ts.isBlock(arm) ? arm.statements : [arm], armState); + return armState; + }; + const thenArm = armWalk(statement.thenStatement); + const elseArm = armWalk(statement.elseStatement); + state.rethrows ||= thenArm.rethrows && elseArm.rethrows; + state.branches ||= thenArm.branches && elseArm.branches; + } + + if (containsLiveExit(statement)) state.exited = true; + } + }; + walk(clause.block.statements, state); + + return { + rethrows: state.rethrows && !returns, + throws: state.rethrows, + branches: state.branches, + }; +} + +/** + * Method names that invoke their callback once per element, never once as a whole. The structural + * signal that separates a per-item boundary (`items.map((item) => { try {...} })`, a fresh catch + * for every element) from a route's own body expressed through one more layer of function nesting + * (`trace(async () => {...})`, `mutateWithFallback({ pgMutation: async (t) => {...} })`, + * `new ReadableStream({ start: async (c) => {...} })`), all of which invoke their callback exactly + * once. + * + * A name list, because nothing in a syntactic scan can tell `users.map` from `Result.map`. The + * consequence is written down where it matters, on `isIterationCallback`. + */ +const ITERATION_METHODS = new Set([ + "map", + "forEach", + "filter", + "reduce", + "reduceRight", + "flatMap", + "some", + "every", +]); + +/** Whether an expression is an array literal of fewer than two elements, the one receiver shape + * that cannot be a per-item iteration however the method is named. */ +function isAtMostSingletonArray(expr: ts.Expression): boolean { + const target = unwrap(expr); + return ts.isArrayLiteralExpression(target) && target.elements.length < 2; +} + +/** + * Whether the function-like `node` is the callback argument of a per-item iteration, e.g. the arrow + * function in `items.map((item) => ...)`. + * + * Being wrong here is asymmetric. Calling a per-item callback the route's own continuation + * mis-attributes a per-element catch to the route, which was the bug the boundary was added for. + * Calling the route's own continuation a per-item callback hides the route's catch, and + * `error-classification` used to read a route with no catch as not-applicable, which is 50 points + * more than the swallow it was hiding. So the second direction paid, and `[0].map(async () => { + * whole body })` collected it. + * + * Two things changed. A receiver that is an array literal of one element or none is refused here, + * because it cannot iterate. And the direction that used to pay no longer pays: `walkBody` keeps + * the catches it refuses, evidence and all, and `error-classification` fails a route with a + * refused swallow when nothing the route owns decides, while a refused catch that decides caps at + * not-applicable and never a pass. That is what makes the name list survivable, and it is why + * `Result.map(...)`, which no name list can tell from `users.map(...)`, is a corpus entry that + * passes rather than a hole: relocating a swallow behind the boundary still fails, and relocating + * a decision earns at most the route's exit from the denominator. + * + * The other direction still costs points and the earlier version of this comment said otherwise. + * A per-item callback under a callee the name list does not know, `pMap(items, cb)` or + * `Array.prototype.map.call(items, cb)`, is attributed to the route, so a per-element catch that + * decides can carry the route to `pass`. No mutation of a real route produces it: the reviewer + * tried `Array.prototype.map.call` over the tree and it moved nothing, because a route has to + * already be iterating for the shape to exist. It is a wrong verdict waiting for a route to be + * written that way, not a laundering path, and it is why this list is worth extending when a new + * iteration helper shows up in the tree. + */ +function isIterationCallback(node: ts.Node): boolean { + const parent = node.parent; + if (!parent || !ts.isCallExpression(parent)) return false; + if (!parent.arguments.includes(node as ts.Expression)) return false; + const callee = unwrap(parent.expression); + if (!ts.isPropertyAccessExpression(callee)) return false; + if (!ITERATION_METHODS.has(callee.name.text)) return false; + return !isAtMostSingletonArray(callee.expression); +} + +const HTTP_METHODS = new Set(["GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS"]); + +function propertyName(property: ts.ObjectLiteralElementLike): string | null { + if (!property.name) return null; + if (ts.isIdentifier(property.name) || ts.isStringLiteral(property.name)) + return property.name.text; + return null; +} + +/** `methods: { POST: { handler } }`, the per-method shape of `createMultiMethodApiRoute`. */ +function collectMethodHandlers(methods: ts.ObjectLiteralExpression, out: EntryFunction[]): void { + for (const method of methods.properties) { + const name = propertyName(method); + if (!name || !HTTP_METHODS.has(name)) continue; + if (!ts.isPropertyAssignment(method)) continue; + const config = unwrap(method.initializer); + if (!ts.isObjectLiteralExpression(config)) continue; + for (const property of config.properties) { + if (!ts.isPropertyAssignment(property) || propertyName(property) !== "handler") continue; + const value = unwrap(property.initializer); + if (isEntryFunction(value)) out.push(value); + } + } +} + +/** + * The handler on an object argument, in the two shapes the route builders use: `handler` at the + * top level of the config (`createSSELoader({ handler })`) and `methods.POST.handler`. Matching by + * name at any depth would pick up an unrelated config callback that happens to be called + * `handler`, as well as the sibling lambdas (`findResource`, `authorization.resource`) that are + * not the entry-point body. + */ +function collectNamedHandlers(object: ts.ObjectLiteralExpression, out: EntryFunction[]): void { + for (const property of object.properties) { + if (!ts.isPropertyAssignment(property)) continue; + const name = propertyName(property); + const value = unwrap(property.initializer); + if (name === "handler" && isEntryFunction(value)) out.push(value); + if (name === "methods" && ts.isObjectLiteralExpression(value)) { + collectMethodHandlers(value, out); + } + } +} + +/** The innermost call of a chain: the `createLoaderApiRoute(...)` in `createLoaderApiRoute(...).withCors(...)`. */ +function rootCall(call: ts.CallExpression): ts.CallExpression { + let current = call; + for (;;) { + let next = unwrap(current.expression); + while (ts.isPropertyAccessExpression(next) || ts.isElementAccessExpression(next)) { + next = unwrap(next.expression); + } + if (ts.isCallExpression(next)) { + current = next; + continue; + } + return current; + } +} + +/** + * The handler functions passed to a builder call. Only the root call of a chain is read: a + * callback given to a decorator further along the chain (`.withCors(cb)`) is not the route body. + */ +function collectHandlerFunctions(call: ts.CallExpression, out: EntryFunction[]): void { + for (const arg of rootCall(call).arguments) { + const unwrapped = unwrap(arg); + if (isEntryFunction(unwrapped)) out.push(unwrapped); + else if (ts.isObjectLiteralExpression(unwrapped)) collectNamedHandlers(unwrapped, out); + } +} + +/** Literals a builder option can be given that mean it was not given: `apiBuilder.server.ts` gates + * every one of these behind `if (option)`. Written out because `authorization: undefined` reads as + * a declared gate to anything counting keys, and declaring one is what `auth-scope` credits. */ +function isDeclaredValue(property: ts.ObjectLiteralElementLike): boolean { + if (!ts.isPropertyAssignment(property)) return true; + const value = unwrap(property.initializer); + if (ts.isIdentifier(value) && value.text === "undefined") return false; + return value.kind !== ts.SyntaxKind.NullKeyword && value.kind !== ts.SyntaxKind.FalseKeyword; +} + +/** + * Top-level property names of every object-literal argument to the root call, e.g. `params`, + * `authorization`, `method`. Only the root call and only the top level: `authorization` on + * `createMultiMethodApiRoute` is declared once beside `methods` rather than per method + * (`apiBuilder.server.ts`), so nothing here needs to descend. + */ +function collectOptionKeys(call: ts.CallExpression): string[] { + const keys: string[] = []; + for (const arg of rootCall(call).arguments) { + const target = unwrap(arg); + if (!ts.isObjectLiteralExpression(target)) continue; + for (const property of target.properties) { + const name = propertyName(property); + if (name && isDeclaredValue(property)) keys.push(name); + } + } + return keys; +} + +type Initializer = { callee: string | null; functions: EntryFunction[]; optionKeys: string[] }; + +const NO_INITIALIZER: Initializer = { callee: null, functions: [], optionKeys: [] }; + +/** Top-level `function x` / `const x = ...` declarations, keyed by binding name. */ +type LocalDeclarations = Map; + +function analyzeInitializer( + expr: ts.Expression | undefined, + locals: LocalDeclarations, + seen: Set +): Initializer { + if (!expr) return NO_INITIALIZER; + const target = unwrap(expr); + + if (isEntryFunction(target)) return { callee: null, functions: [target], optionKeys: [] }; + + if (ts.isCallExpression(target)) { + const functions: EntryFunction[] = []; + collectHandlerFunctions(target, functions); + return { + callee: rootCalleeName(target), + functions, + optionKeys: collectOptionKeys(target), + }; + } + + // `export const action = route.action` where `const route = createActionApiRoute(...)`, and the + // plain alias `export const loader = h`. Resolve back to the declaration the name came from. + if (ts.isIdentifier(target)) return resolveLocal(target.text, locals, seen); + if (ts.isPropertyAccessExpression(target)) { + const root = unwrap(target.expression); + if (ts.isIdentifier(root)) return resolveLocal(root.text, locals, seen); + } + + return NO_INITIALIZER; +} + +function resolveLocal(name: string, locals: LocalDeclarations, seen: Set): Initializer { + if (seen.has(name)) return NO_INITIALIZER; + seen.add(name); + const local = locals.get(name); + if (!local) return NO_INITIALIZER; + if (ts.isFunctionDeclaration(local)) return { callee: null, functions: [local], optionKeys: [] }; + return analyzeInitializer(local, locals, seen); +} + +/** + * How many operands a comma expression has, so `a(), b(), c()` is three and not one. Anything else + * is one. + */ +function commaOperands(expr: ts.Expression): number { + const target = unwrap(expr); + if (ts.isBinaryExpression(target) && target.operatorToken.kind === ts.SyntaxKind.CommaToken) { + return commaOperands(target.left) + commaOperands(target.right); + } + return 1; +} + +/** + * Statements in a statement, counting through block-bearing statements so a body wrapped in a + * single `try` reports its real size. Does not descend into nested function bodies. + * + * Counts bindings and comma operands rather than semicolons, which is what makes the number mean + * something. `const a = f(), b = g(), c = h();` is three initializers however it is punctuated, and + * `a(), b(), c()` is three calls: scoring either as one let a seven-statement try be rewritten into + * a two-statement one with no change to what it runs, which took `error-classification` from fail + * to pass. `merge-declarations` and `merge-comma-expressions` in the mutation corpus are + * the tree-scale versions. + */ +function countStatement(statement: ts.Statement): number { + if (ts.isBlock(statement)) { + return countStatements(statement.statements); + } + + if (ts.isVariableStatement(statement)) { + return statement.declarationList.declarations.length; + } + + if (ts.isExpressionStatement(statement)) { + return commaOperands(statement.expression); + } + + let count = 1; + + if (ts.isTryStatement(statement)) { + count += countStatements(statement.tryBlock.statements); + if (statement.catchClause) count += countStatements(statement.catchClause.block.statements); + if (statement.finallyBlock) count += countStatements(statement.finallyBlock.statements); + return count; + } + + if (ts.isIfStatement(statement)) { + count += countStatement(statement.thenStatement); + if (statement.elseStatement) count += countStatement(statement.elseStatement); + return count; + } + + if ( + ts.isForStatement(statement) || + ts.isForInStatement(statement) || + ts.isForOfStatement(statement) || + ts.isWhileStatement(statement) || + ts.isDoStatement(statement) || + ts.isLabeledStatement(statement) || + ts.isWithStatement(statement) + ) { + count += countStatement(statement.statement); + return count; + } + + if (ts.isSwitchStatement(statement)) { + for (const clause of statement.caseBlock.clauses) { + count += countStatements(clause.statements); + } + return count; + } + + return count; +} + +function countStatements(statements: ts.NodeArray): number { + let count = 0; + for (const statement of statements) count += countStatement(statement); + return count; +} + +function countFunctionStatements(fn: EntryFunction): number { + if (!fn.body) return 0; + // A concise arrow body (`() => json({})`) is one expression, so one statement. + if (!ts.isBlock(fn.body)) return 1; + return countStatements(fn.body.statements); +} + +type ExportName = "loader" | "action"; + +/** + * The call-site facts a body walk accumulates, kept in one shape so the entry-point-wide totals and + * each export's own totals are filled by the same code rather than by two similar loops. + */ +type BodyFacts = { + calleeNames: string[]; + /** + * The same calls as `calleeNames`, each as its whole dotted path. `calleeName` keeps only the + * last segment, so `prisma.organization.findFirst` arrives in `calleeNames` as `findFirst` and + * the receiver that says WHAT is being called is gone. The per-export triviality rule needs it + * back: `prisma` in the path is how a short body is known to touch the datastore. + */ + calleeTexts: string[]; + /** Locals initialised from a call, by local name. */ + declaredFrom: Map; + /** Every identifier read by an `if`, `while`, `switch` or conditional condition. */ + testedNames: Set; + statementCount: number; + hasTryCatch: boolean; +}; + +function newBodyFacts(): BodyFacts { + return { + calleeNames: [], + calleeTexts: [], + declaredFrom: new Map(), + testedNames: new Set(), + statementCount: 0, + hasTryCatch: false, + }; +} + +/** Callees whose answer these bodies looked at: declared from a call AND read by a condition. */ +function checkedCalleesOf(facts: BodyFacts): string[] { + return [ + ...new Set( + [...facts.declaredFrom] + .filter(([local]) => facts.testedNames.has(local)) + .flatMap(([, callees]) => callees) + ), + ]; +} + +type EntryTarget = { + hasLoader: boolean; + hasAction: boolean; + loaderInitializerCallee: string | null; + actionInitializerCallee: string | null; + loaderBuilderOptions: string[]; + actionBuilderOptions: string[]; + /** Handler functions per export, so a scope signal can be attributed to the half of the file it + * was found in. `functions` is their union, which is what every entry-point-wide field reads. */ + loaderFunctions: Set; + actionFunctions: Set; + functions: Set; +}; + +/** + * Top-level `function x` / `const x = ...` declarations by binding name, so a named export clause + * (`export { action }`) can be resolved back to the initializer it came from. + */ +function collectLocalDeclarations(sf: ts.SourceFile): LocalDeclarations { + const locals: LocalDeclarations = new Map(); + + for (const statement of sf.statements) { + if (ts.isFunctionDeclaration(statement) && statement.name) { + locals.set(statement.name.text, statement); + continue; + } + if (!ts.isVariableStatement(statement)) continue; + + for (const decl of statement.declarationList.declarations) { + if (!decl.initializer) continue; + if (ts.isIdentifier(decl.name)) { + locals.set(decl.name.text, decl.initializer); + continue; + } + if (ts.isObjectBindingPattern(decl.name)) { + for (const element of decl.name.elements) { + if (ts.isIdentifier(element.name)) locals.set(element.name.text, decl.initializer); + } + } + } + } + + return locals; +} + +/** + * Top-level functions by name, for resolving a body that delegates its work to a same-file helper + * (`export async function loader({ request }) { return proxyToPostHog(request); }`). + */ +function collectLocalFunctions(sf: ts.SourceFile): Map { + const functions = new Map(); + + for (const statement of sf.statements) { + if (ts.isFunctionDeclaration(statement) && statement.name && statement.body) { + functions.set(statement.name.text, statement); + continue; + } + if (!ts.isVariableStatement(statement)) continue; + + for (const decl of statement.declarationList.declarations) { + if (!decl.initializer || !ts.isIdentifier(decl.name)) continue; + const value = unwrap(decl.initializer); + if (isEntryFunction(value)) functions.set(decl.name.text, value); + } + } + + return functions; +} + +/** + * Compiler options for the throwaway program below. `noLib` and `noResolve` keep it from going to + * disk: nothing here needs a type, only the syntax the parser already produced. + */ +const SYNTAX_ONLY_OPTIONS: ts.CompilerOptions = { noLib: true, noResolve: true, allowJs: true }; + +/** + * Syntactic diagnostics for an already-parsed source file, through `ts.Program` rather than off + * the diagnostics array the parser hangs on the source file, which is internal and which the + * compiler is free to rename. The whole parse-failure discipline rests on this, and an undetected + * parse failure shrinks the denominator and inflates the score, so it must not be the kind of + * thing a compiler upgrade can switch off silently. + * + * The host hands the program the `sf` we already have, so this does not parse the source a second + * time. The cost is the program machinery around it, and it is not free: a full scan of the real + * route tree went from about 850ms to about 1450ms, measured over five runs of each. A slower + * scan of a tool that runs once a pull request is the cheaper of the two prices. + */ +function syntacticDiagnostics(sf: ts.SourceFile): readonly ts.Diagnostic[] { + const host: ts.CompilerHost = { + getSourceFile: (name) => (name === sf.fileName ? sf : undefined), + getDefaultLibFileName: () => "lib.d.ts", + writeFile: () => {}, + getCurrentDirectory: () => "", + getCanonicalFileName: (name) => name, + useCaseSensitiveFileNames: () => true, + getNewLine: () => "\n", + fileExists: (name) => name === sf.fileName, + readFile: () => undefined, + }; + return ts.createProgram([sf.fileName], SYNTAX_ONLY_OPTIONS, host).getSyntacticDiagnostics(sf); +} + +export function scanFile(fileName: string, source: string): EntryPoint | null { + const sf = ts.createSourceFile(fileName, source, ts.ScriptTarget.Latest, true); + + // `createSourceFile` recovers from malformed input instead of throwing, so the diagnostics are + // the only signal that a file did not parse. + const diagnostics = syntacticDiagnostics(sf); + if (diagnostics.length > 0) { + const first = diagnostics[0]!; + throw new ParseFailureError(fileName, ts.flattenDiagnosticMessageText(first.messageText, " ")); + } + + const importedNames: string[] = []; + for (const statement of sf.statements) { + if (!ts.isImportDeclaration(statement) || !statement.importClause) continue; + const bindings = statement.importClause.namedBindings; + if (bindings && ts.isNamedImports(bindings)) { + for (const el of bindings.elements) importedNames.push(el.name.text); + } + if (statement.importClause.name) importedNames.push(statement.importClause.name.text); + } + + const target: EntryTarget = { + hasLoader: false, + hasAction: false, + loaderInitializerCallee: null, + actionInitializerCallee: null, + loaderBuilderOptions: [], + actionBuilderOptions: [], + loaderFunctions: new Set(), + actionFunctions: new Set(), + functions: new Set(), + }; + + // The option keys travel with the callee they came from, so a second declaration cannot lend its + // options to the first one's builder. + const record = (name: string, initializer: Initializer) => { + if (name === "loader") { + target.hasLoader = true; + if (target.loaderInitializerCallee === null) { + target.loaderInitializerCallee = initializer.callee; + target.loaderBuilderOptions = initializer.optionKeys; + } + } else { + target.hasAction = true; + if (target.actionInitializerCallee === null) { + target.actionInitializerCallee = initializer.callee; + target.actionBuilderOptions = initializer.optionKeys; + } + } + const perExport = name === "loader" ? target.loaderFunctions : target.actionFunctions; + for (const fn of initializer.functions) { + target.functions.add(fn); + perExport.add(fn); + } + }; + + const isExported = (n: ts.Node) => + ts.canHaveModifiers(n) && + ts.getModifiers(n)?.some((m) => m.kind === ts.SyntaxKind.ExportKeyword) === true; + + const locals = collectLocalDeclarations(sf); + + for (const statement of sf.statements) { + if (ts.isFunctionDeclaration(statement) && statement.name && isExported(statement)) { + const name = statement.name.text; + if (name === "loader" || name === "action") { + record(name, { callee: null, functions: [statement], optionKeys: [] }); + } + continue; + } + + if (ts.isVariableStatement(statement) && isExported(statement)) { + for (const decl of statement.declarationList.declarations) { + if (ts.isIdentifier(decl.name)) { + const name = decl.name.text; + if (name === "loader" || name === "action") { + record(name, analyzeInitializer(decl.initializer, locals, new Set())); + } + continue; + } + // `export const { action, loader } = createActionApiRoute(...)`. Skipping a non-identifier + // binding name here produced no entry point at all for this shape: not a parse failure and + // not unmeasured, simply absent from the denominator. The two-step spelling + // (`const { action } = builder(...); export { action };`) already resolved, because + // `collectLocalDeclarations` reads the binding pattern and the export clause looks the name + // up there, so only the direct form was missing. The exported name is the ELEMENT name, so + // `{ loader: action }` exports an action and `{ action: internal }` exports neither. + if (!ts.isObjectBindingPattern(decl.name)) continue; + for (const element of decl.name.elements) { + if (!ts.isIdentifier(element.name)) continue; + const name = element.name.text; + if (name !== "loader" && name !== "action") continue; + record(name, analyzeInitializer(decl.initializer, locals, new Set())); + } + } + continue; + } + + if (ts.isExportDeclaration(statement) && statement.exportClause) { + // `export * from "./x"` has no clause and reaches nothing here; `export * as ns from "./x"` + // is a namespace clause, which cannot name a loader or action either. + if (!ts.isNamedExports(statement.exportClause)) continue; + + for (const element of statement.exportClause.elements) { + const exportedName = element.name.text; + if (exportedName !== "loader" && exportedName !== "action") continue; + // A re-export (`export { loader } from "./x"`) has no local binding to resolve. + if (statement.moduleSpecifier) { + record(exportedName, NO_INITIALIZER); + continue; + } + const localName = element.propertyName?.text ?? exportedName; + record(exportedName, resolveLocal(localName, locals, new Set())); + } + } + } + + if (!target.hasLoader && !target.hasAction) return null; + + const callbackCatches: CatchEvidence[] = []; + const catches: CatchEvidence[] = []; + const logCalls: LogCall[] = []; + + const wholeEntry = newBodyFacts(); + const byExport: Record = { + loader: newBodyFacts(), + action: newBodyFacts(), + }; + + const localFunctions = collectLocalFunctions(sf); + // A body that delegates to a same-file helper does the work in that helper, so the helper's + // statements, try/catch and callees belong to the entry point. One hop only: a helper's own + // helpers are not followed, and the visited set stops a cycle and any double counting. + // + // The helper's callees belong to whichever EXPORTS reach it, too, which is what `helperOwners` + // carries. A helper called from both halves of the file is owned by both; the union is taken on + // the second discovery rather than dropped, because `visited` has already queued it by then. + const visited = new Set(target.functions); + const helpers: EntryFunction[] = []; + const helperOwners = new Map>(); + + const walkBody = (fn: EntryFunction, followHelpers: boolean, owners: ReadonlySet) => { + // One push site feeds the entry-point-wide list and each owning export's list, so + // `calleeNames` and the per-export lists cannot drift apart. See `EntryPoint.loaderCalleeNames`. + const sinks: BodyFacts[] = [wholeEntry]; + for (const owner of owners) sinks.push(byExport[owner]); + const collectTested = (node: ts.Node) => { + if (ts.isIdentifier(node)) for (const s of sinks) s.testedNames.add(node.text); + ts.forEachChild(node, collectTested); + }; + + const addStatements = (n: number) => { + for (const sink of sinks) sink.statementCount += n; + }; + addStatements(countFunctionStatements(fn)); + + if (!fn.body) return; + // `inCallback` is true once the walk has entered a per-item iteration callback + // (`items.map((item) => { ... })`), never reset back to false: nesting deeper inside one is + // still inside it. `calleeNames` and `logCalls` keep descending regardless. A try/catch does + // not: a per-item catch is not part of this body's own statement list, and `countStatement` + // already stops at a nested function boundary, so counting it here let `tryStatementCount` + // exceed the entry point's whole `statementCount` and judged a per-item error boundary as + // though it were the route's own. What is refused is kept in `callbackCatches` with its + // evidence instead of dropped, so `error-classification` can fail a refused swallow and sit + // out a refused catch that decides, without ever crediting either as the route's own. + // + // Only an iteration callback is a boundary, not every function-like node: a route's own body + // wrapped in `trace(async () => {...})`, `mutateWithFallback({ pgMutation: async (t) => {...} })` + // or `new ReadableStream({ start: async (c) => {...} })` still runs exactly once, as the route's + // own continuation one layer of nesting away, and its catch is the route's own error handling. + // + // A nested function's statements count towards `statementCount` too, whichever kind it is. + // They are work the route does, and leaving them out let `trace("x", async () => { whole body + // })` collapse a route to one statement, which is inside the triviality rule's limit: the route + // then read as trivial and every check reported not-applicable for it. `wrap-body-in-trace` in + // the mutation corpus is that shape. + const visit = (node: ts.Node, inCatch: boolean, inCallback: boolean) => { + if (ts.isFunctionLike(node)) { + if (isEntryFunction(node)) addStatements(countFunctionStatements(node)); + const entersIterationCallback = inCallback || isIterationCallback(node); + ts.forEachChild(node, (child) => visit(child, inCatch, entersIterationCallback)); + return; + } + if (ts.isTryStatement(node)) { + for (const sink of sinks) sink.hasTryCatch = true; + if (node.catchClause) { + // Built the same way for a refused catch as for an own one, so the dead-code defence + // and the walk's guaranteed-execution rules apply to both. Which list it lands in is + // walkBody's attribution decision alone. + const tryStatementCount = countStatements(node.tryBlock.statements); + const clause = catchClauseEvidence(node.catchClause); + (inCallback ? callbackCatches : catches).push({ + rethrows: clause.rethrows, + throws: clause.throws, + branches: clause.branches, + ...guardedWork(node.tryBlock), + guardMayRaise: tryBlockMayThrow(node.tryBlock), + tryStatementCount, + }); + } + } + + if (ts.isCatchClause(node)) { + ts.forEachChild(node, (child) => visit(child, true, inCallback)); + return; + } + + if (ts.isVariableDeclaration(node) && ts.isIdentifier(node.name) && node.initializer) { + const initializer = unwrap(node.initializer); + if (ts.isCallExpression(initializer)) { + const cn = calleeName(initializer.expression); + if (cn) { + for (const sink of sinks) { + const existing = sink.declaredFrom.get(node.name.text); + if (existing) existing.push(cn); + else sink.declaredFrom.set(node.name.text, [cn]); + } + } + } + } + + if (ts.isIfStatement(node) || ts.isWhileStatement(node) || ts.isSwitchStatement(node)) { + collectTested(node.expression); + } + if (ts.isConditionalExpression(node)) collectTested(node.condition); + + if (ts.isCallExpression(node)) { + const cn = calleeName(node.expression); + if (cn) { + const text = calleeText(node.expression) ?? cn; + for (const sink of sinks) { + sink.calleeNames.push(cn); + sink.calleeTexts.push(text); + } + + if (LOGGER_CALLEE.test(text)) { + logCalls.push({ + callee: text, + fields: objectArgumentFields(node), + inCatch, + }); + } + } + + if (followHelpers) { + const callee = unwrap(node.expression); + if (ts.isIdentifier(callee)) { + const helper = localFunctions.get(callee.text); + if (helper && !visited.has(helper)) { + visited.add(helper); + helpers.push(helper); + helperOwners.set(helper, new Set(owners)); + } else if (helper) { + const already = helperOwners.get(helper); + if (already) for (const owner of owners) already.add(owner); + } + } + } + } + ts.forEachChild(node, (child) => visit(child, inCatch, inCallback)); + }; + visit(fn.body, false, false); + }; + + // Every handler is walked exactly once, whichever exports own it, so the entry-point-wide + // statement count and catch list stay single while the per-export lists see it from both sides. + const ownersOf = (fn: EntryFunction): Set => { + const owners = new Set(); + if (target.loaderFunctions.has(fn)) owners.add("loader"); + if (target.actionFunctions.has(fn)) owners.add("action"); + return owners; + }; + for (const fn of target.functions) walkBody(fn, true, ownersOf(fn)); + for (const helper of helpers) walkBody(helper, false, helperOwners.get(helper) ?? new Set()); + + return { + fileName, + source, + hasLoader: target.hasLoader, + hasAction: target.hasAction, + loaderInitializerCallee: target.loaderInitializerCallee, + actionInitializerCallee: target.actionInitializerCallee, + loaderBuilderOptions: target.loaderBuilderOptions, + actionBuilderOptions: target.actionBuilderOptions, + // No handler function and no builder call: `export { action } from "./handler.server"` and + // `export const action = handleWebhook`. See `EntryPoint.delegating`. + delegating: + target.functions.size === 0 && + target.loaderInitializerCallee === null && + target.actionInitializerCallee === null, + loaderScopesByCaller: scopesByCallerIn(target.loaderFunctions), + actionScopesByCaller: scopesByCallerIn(target.actionFunctions), + loaderCheckedCallees: checkedCalleesOf(byExport.loader), + actionCheckedCallees: checkedCalleesOf(byExport.action), + importedNames, + calleeNames: wholeEntry.calleeNames, + loaderCalleeNames: byExport.loader.calleeNames, + actionCalleeNames: byExport.action.calleeNames, + loaderCalleeTexts: byExport.loader.calleeTexts, + actionCalleeTexts: byExport.action.calleeTexts, + hasTryCatch: wholeEntry.hasTryCatch, + loaderHasTryCatch: byExport.loader.hasTryCatch, + actionHasTryCatch: byExport.action.hasTryCatch, + catches, + callbackCatches, + logCalls, + statementCount: wholeEntry.statementCount, + loaderStatementCount: byExport.loader.statementCount, + actionStatementCount: byExport.action.statementCount, + }; +} + +const SOURCE_FILE = /\.tsx?$/; + +/** + * Whether a file name is one the scanner reads at all. + * + * Exported because three other places ask the same question and each had written its own copy: + * `mutationCorpus.test.ts` materializes exactly the files `scanDirectory` reads, and + * `integration.test.ts` and `webappSymbols.test.ts` walk trees of their own. The corpus's + * anti-vacuity thresholds count files and sites the scanner never saw if those predicates drift, + * and `integration.test.ts`'s `entryPoints.length < countRouteModuleFiles(ROUTES)` stops meaning + * anything if its denominator counts files the scanner skips. + */ +export function isScannableFile(fileName: string): boolean { + return SOURCE_FILE.test(fileName) && !fileName.endsWith(".d.ts"); +} + +/** One route module: where to read it, and the name the report and the scan record it under. */ +export type RouteModuleFile = { absolutePath: string; relativeName: string }; + +/** + * The route modules under `dir`: every scannable flat file, plus the `route.ts`/`route.tsx` of each + * immediate subdirectory. + * + * Exported so `mutationCorpus.test.ts` can materialize exactly this set rather than re-deriving it. + * Its `readTree` was a verbatim copy of the walk below; the FILE half of that copy was later + * replaced by a call to `isScannableFile` while the DIRECTORY half stayed duplicated, which is the + * usual way this package's duplicates half-die. A corpus that enumerates a different tree from the + * scanner reports file and site counts for files the scan never reads, and those counts are the + * only thing standing between a mutation that reaches nothing and a green test. + */ +export function routeModuleFiles(dir: string): RouteModuleFile[] { + const files: RouteModuleFile[] = []; + for (const entry of readdirSync(dir, { withFileTypes: true })) { + if (entry.isDirectory()) { + // Flat-route directories hold the route module in `route.ts`/`route.tsx`. + for (const child of readdirSync(join(dir, entry.name), { withFileTypes: true })) { + if (!child.isFile() || (child.name !== "route.ts" && child.name !== "route.tsx")) continue; + files.push({ + absolutePath: join(dir, entry.name, child.name), + relativeName: `${entry.name}/${child.name}`, + }); + } + continue; + } + if (!entry.isFile() || !isScannableFile(entry.name)) continue; + files.push({ absolutePath: join(dir, entry.name), relativeName: entry.name }); + } + return files; +} + +export function scanDirectory(dir: string): { + entryPoints: EntryPoint[]; + parseFailures: string[]; +} { + const entryPoints: EntryPoint[] = []; + const parseFailures: string[] = []; + + const scan = (absolutePath: string, relativeName: string) => { + let ep: EntryPoint | null; + try { + ep = scanFile(relativeName, readFileSync(absolutePath, "utf8")); + } catch (error) { + // Only a genuinely malformed source is a parse failure. An unreadable file or a bug in the + // scanner must not be laundered into the same bucket, or a non-zero count means nothing. + if (error instanceof ParseFailureError) { + parseFailures.push(`${relativeName}: ${error.diagnostic}`); + return; + } + throw error; + } + if (ep) entryPoints.push(ep); + }; + + for (const file of routeModuleFiles(dir)) scan(file.absolutePath, file.relativeName); + + return { entryPoints, parseFailures }; +} diff --git a/internal-packages/observability-map/src/score.test.ts b/internal-packages/observability-map/src/score.test.ts new file mode 100644 index 00000000000..a8d6943d0e0 --- /dev/null +++ b/internal-packages/observability-map/src/score.test.ts @@ -0,0 +1,629 @@ +import { scoreEntry, buildReport } from "./score.js"; +import { scanFile } from "./scan.js"; + +const BUILDER = `import { createLoaderApiRoute } from "~/services/routeBuilders/apiBuilder.server"; +export const loader = createLoaderApiRoute({}, async () => new Response("ok"));`; + +const RAW = `import { prisma } from "~/db.server"; +export async function loader() { return prisma.thing.findMany(); }`; + +/** Trivial and not sensitive: every scored check reports not-applicable. */ +const TRIVIAL = `export const loader = () => new Response("ok");`; + +/** Not trivial (touches prisma, has a try/catch) and not sensitive: swallows every error and + * records nothing about whose failure it was, so both applicable scored checks fail. */ +const BUSY_AND_FAILING = `import { prisma } from "~/db.server"; +export async function loader() { + try { return await prisma.thing.findMany(); } catch (e) { return null; } +}`; + +/** Guarded, classifies what it catches, and names the tenant on the failure path. */ +const CLEAN = `import { requireUserId } from "~/services/session.server"; +import { logger } from "~/services/logger.server"; +import { prisma } from "~/db.server"; +export async function action({ request, params }) { + const userId = await requireUserId(request); + try { + return await prisma.token.create({ data: { userId } }); + } catch (error) { + logger.error("token create failed", { userId, environmentId: params.envId, error }); + throw error; + } +}`; + +describe("scoreEntry", () => { + it("scores an entry that passes every applicable check 100", () => { + expect(scoreEntry(scanFile("api.v1.auth.tokens.ts", CLEAN)!).score).toBe(100); + }); + + // A builder wrapper classifies errors for the route, but the route itself catches nothing and + // names nobody on its failure path, so there is one applicable check and it fails. + it("does not credit a builder route for the error handling it does not do", () => { + const scored = scoreEntry(scanFile("api.v1.a.ts", BUILDER)!); + expect(scored.checks.find((c) => c.id === "error-classification")!.status).toBe( + "not-applicable" + ); + expect(scored.checks.find((c) => c.id === "request-context")!.status).toBe("fail"); + expect(scored.score).toBe(0); + }); + + it("excludes audit-trail from the per-entry score", () => { + const scored = scoreEntry(scanFile("api.v1.auth.jwt.ts", CLEAN)!); + expect(scored.checks.find((c) => c.id === "audit-trail")!.status).toBe("fail"); + expect(scored.score).toBe(100); + }); + + it("counts a suppressed check as not-applicable", () => { + const suppressed = `// obs-map-disable error-classification -- health probe +${RAW}`; + const scored = scoreEntry(scanFile("api.v1.b.ts", suppressed)!); + const ec = scored.checks.find((c) => c.id === "error-classification")!; + expect(ec.status).toBe("not-applicable"); + }); + + it("a suppressed-to-passing entry point does not read as unmeasured", () => { + // error-classification would fail here; auth-boundary is not-applicable (not sensitive). + // Suppressing the only applicable scored check must not be indistinguishable from an entry + // point nothing applies to: it is still reported, just not scored on that axis. + const suppressed = `// obs-map-disable error-classification -- health probe +${BUSY_AND_FAILING}`; + const scored = scoreEntry(scanFile("api.v1.c.ts", suppressed)!); + expect(scored.checks.find((c) => c.id === "error-classification")!.status).toBe( + "not-applicable" + ); + // The point of the test, which it did not previously assert: request-context still applies, so + // the entry is still measured and still counted in the mean. + expect(scored.checks.find((c) => c.id === "request-context")!.status).toBe("fail"); + expect(scored.measured).toBe(true); + }); + + // I1. `score = passed / applicable` meant removing a failing check from the denominator raised + // the entry's score, so suppression laundered findings into points. A suppression buys removal + // from the worklist, never a better number. + it("does not raise the score when a failing check is suppressed", () => { + const source = `import { requireUserId } from "~/services/session.server"; +import { prisma } from "~/db.server"; +export async function action({ request }) { + const userId = await requireUserId(request); + try { return await prisma.token.create({ data: { userId } }); } + catch (error) { return null; } +}`; + const plain = scoreEntry(scanFile("api.v1.auth.tokens.ts", source)!); + const suppressed = scoreEntry( + scanFile( + "api.v1.auth.tokens.ts", + `// obs-map-disable error-classification -- deliberate, see ticket +${source}` + )! + ); + + expect(plain.checks.find((c) => c.id === "error-classification")!.status).toBe("fail"); + expect(suppressed.checks.find((c) => c.id === "error-classification")!.status).toBe( + "not-applicable" + ); + expect(suppressed.score).toBeLessThanOrEqual(plain.score); + }); + + it("records which scored checks were suppressed", () => { + const suppressed = scoreEntry( + scanFile( + "api.v1.b.ts", + `// obs-map-disable error-classification -- health probe +// obs-map-disable request-context -- nothing to name here +${BUSY_AND_FAILING}` + )! + ); + expect(suppressed.suppressed).toEqual(["error-classification", "request-context"]); + }); + + // A1. `measured` reads pre-suppression applicability. Before the fix it read `visible` + // (post-suppression) applicability, so suppressing an entry's only applicable checks flipped + // `measured` to false and dropped the entry, still failing, out of the global mean, every family + // mean and the sensitive cohort. `unmeasured` stays for entries nothing was ever applicable to. + it("still reads as measured when every applicable scored check is suppressed", () => { + const suppressed = scoreEntry( + scanFile( + "api.v1.b.ts", + `// obs-map-disable error-classification -- health probe +// obs-map-disable request-context -- nothing to name here +${BUSY_AND_FAILING}` + )! + ); + expect(suppressed.measured).toBe(true); + expect(suppressed.score).toBe(0); + }); + + it("marks an entry point with nothing applicable as unmeasured, scored 100", () => { + const scored = scoreEntry(scanFile("resources.health.ts", TRIVIAL)!); + expect(scored.checks.every((c) => c.status === "not-applicable")).toBe(true); + expect(scored.measured).toBe(false); + expect(scored.score).toBe(100); + }); + + it("marks an entry point with at least one applicable scored check as measured", () => { + const scored = scoreEntry(scanFile("api.v1.busy.ts", BUSY_AND_FAILING)!); + expect(scored.measured).toBe(true); + }); +}); + +describe("buildReport", () => { + it("reports the request-context gap as a figure, like the audit gap", () => { + const naming = scanFile( + "api.v1.named.ts", + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function loader({ params }) { + try { return await prisma.thing.findMany(); } + catch (error) { logger.error("failed", { environmentId: params.envId, error }); throw error; } + }` + )!; + const silent = scanFile( + "api.v1.silent.ts", + `import { prisma } from "~/db.server"; + export async function loader() { return prisma.thing.findMany(); }` + )!; + const trivial = scanFile( + "resources.health.ts", + `export const loader = () => new Response("ok");` + )!; + + const report = buildReport([naming, silent, trivial], []); + expect(report.contextGap).toEqual({ applicable: 2, naming: 1 }); + }); + + it("counts suppressions so laundering is visible in the report", () => { + const report = buildReport( + [ + scanFile( + "api.v1.b.ts", + `// obs-map-disable error-classification -- health probe +${BUSY_AND_FAILING}` + )!, + scanFile("api.v1.c.ts", BUSY_AND_FAILING)!, + ], + [] + ); + expect(report.suppressions).toEqual({ entries: 1, checks: 1 }); + }); + + // I4. contextGap and auditGap read `checks` (post-suppression), so suppressing the one failing + // request-context on an entry removed it from the denominator too, moving CONTEXT from 1 of 2 + // (50%) to 1 of 1 (100%) printed on the same screen as "a suppression does not raise a score". + // Both gaps now read `rawChecks`, pre-suppression, exactly like `measured`. + it("does not let a suppressed request-context finding shrink the context gap's denominator", () => { + const failing = `import { prisma } from "~/db.server"; +export async function loader() { + try { return await prisma.thing.findMany(); } catch (e) { return null; } +}`; + const passing = `import { logger } from "~/services/logger.server"; +import { prisma } from "~/db.server"; +export async function loader({ params }) { + try { return await prisma.thing.findMany(); } + catch (error) { logger.error("failed", { environmentId: params.envId, error }); throw error; } +}`; + + const before = buildReport([scanFile("a.ts", failing)!, scanFile("b.ts", passing)!], []); + expect(before.contextGap).toEqual({ applicable: 2, naming: 1 }); + + const after = buildReport( + [ + scanFile( + "a.ts", + `// obs-map-disable request-context -- silence +${failing}` + )!, + scanFile("b.ts", passing)!, + ], + [] + ); + expect(after.contextGap).toEqual({ applicable: 2, naming: 1 }); + }); + + // I4, second half. A suppressed audit-trail directive never showed up in `suppressed` because + // audit-trail is not in SCORED_CHECK_IDS, so the SUPPRESSED line undercounted while the audit + // denominator silently shrank underneath it. Every suppression is now counted, scored or not. + it("counts an audit-trail suppression and does not let it shrink the audit gap's denominator", () => { + const missingAudit = `import { prisma } from "~/db.server"; +export async function action() { return prisma.token.create({ data: {} }); }`; + const withAudit = `import { clearImpersonation } from "~/models/admin.server"; +import { prisma } from "~/db.server"; +export async function action({ request }) { + const token = await prisma.token.create({ data: {} }); + await clearImpersonation(request, "/admin"); + return json(token); +}`; + + const before = buildReport( + [ + scanFile("api.v1.auth.tokens.ts", missingAudit)!, + scanFile("api.v1.auth.jwt.ts", withAudit)!, + ], + [] + ); + expect(before.auditGap).toEqual({ sensitiveMutations: 2, withAudit: 1 }); + + const after = buildReport( + [ + scanFile( + "api.v1.auth.tokens.ts", + `// obs-map-disable audit-trail -- accepted risk +${missingAudit}` + )!, + scanFile("api.v1.auth.jwt.ts", withAudit)!, + ], + [] + ); + expect(after.auditGap).toEqual({ sensitiveMutations: 2, withAudit: 1 }); + expect(after.suppressions).toEqual({ entries: 1, checks: 1 }); + }); + + it("reports the audit gap separately from the score", () => { + // A sensitive mutation with no audit record, but nothing else wrong: the audit gap is reported + // as its own figure and must not pull the score down with it. + const report = buildReport([scanFile("api.v1.auth.tokens.ts", CLEAN)!], []); + expect(report.auditGap.sensitiveMutations).toBe(1); + expect(report.auditGap.withAudit).toBe(0); + expect(report.global).toBe(100); + }); + + it("records parse failures", () => { + const report = buildReport([scanFile("api.v1.a.ts", BUILDER)!], ["broken.ts"]); + expect(report.parseFailures).toEqual(["broken.ts"]); + }); + + it("excludes an unmeasured entry point from the global mean", () => { + const trivial = scanFile("resources.health.ts", TRIVIAL)!; + const busy = scanFile("api.v1.busy.ts", BUSY_AND_FAILING)!; + + const report = buildReport([trivial, busy], []); + + expect(report.measured).toBe(1); + expect(report.unmeasured).toBe(1); + // If the trivial entry's vacuous 100 counted toward the mean, the global score would be 50 + // instead of matching the one entry that was actually measured. + expect(report.global).toBe(scoreEntry(busy).score); + }); + + it("excludes an unmeasured entry point from its family mean too", () => { + const trivial = scanFile("resources.health.ts", TRIVIAL)!; + const busy = scanFile("resources.busy.ts", BUSY_AND_FAILING)!; + + const report = buildReport([trivial, busy], []); + + const family = report.byFamily["resources"]!; + expect(family.n).toBe(2); + expect(family.measured).toBe(1); + expect(family.mean).toBe(scoreEntry(busy).score); + }); +}); + +// A1. `measured` used to read post-suppression (`visible`) applicability, so suppressing an +// entry's only applicable checks removed it from the global mean, every family mean and the +// sensitive cohort, rather than keeping it at its capped score. That is how a suppression with no +// behavioural change moved the global from 17 to 33 tree-wide. +describe("A1: a suppression cannot raise the global", () => { + it("scoring 100 and 0, suppressing every check on the failing entry leaves the global at 50", () => { + const passing = scanFile("api.v1.auth.tokens.ts", CLEAN)!; + const failing = scanFile("api.v1.busy.ts", BUSY_AND_FAILING)!; + + const before = buildReport([passing, failing], []); + expect(before.global).toBe(50); + + const failingSuppressed = scanFile( + "api.v1.busy.ts", + `// obs-map-disable error-classification -- silence +// obs-map-disable request-context -- silence +${BUSY_AND_FAILING}` + )!; + const after = buildReport([passing, failingSuppressed], []); + + expect(after.global).toBe(50); + expect(after.measured).toBe(2); + }); + + // M9. A prior version of this test asserted suppressed.score <= plain.score for a check that was + // passing, which the pre-existing per-entry Math.min cap already guarantees on its own: removing + // a passing (maximal) result from a ratio can only lower or hold it, at every level, with or + // without A1's fix, so the assertion passed unchanged against the pre-fix code and proved + // nothing about the aggregate mechanism A1 actually changed. Deleted rather than kept as + // decoration; "does not raise the score" for a failing suppression is exercised, with real + // discriminating power, by the two tests above. +}); + +/** + * The invariant, both ways round. The README states one direction, removing error handling must + * lower the score, and that alone could not see the free-points path: adding a catch that only + * rethrows changes nothing about how the route behaves, and used to move it from not-applicable to + * pass, worth 50 points a route and 27 points across the tree. + */ +describe("no-op error handling must not pay", () => { + const BODY = `const rows = await prisma.thing.findMany(); + return json({ rows });`; + + const plain = `import { prisma } from "~/db.server"; +export async function loader() { + ${BODY} +}`; + + const wrappedInARethrow = `import { prisma } from "~/db.server"; +export async function loader() { + try { + ${BODY} + } catch (e) { + throw e; + } +}`; + + const handled = `import { logger } from "~/services/logger.server"; +import { prisma } from "~/db.server"; +export async function loader({ params }) { + try { + ${BODY} + } catch (error) { + if (error instanceof NotFoundError) return json({ error: "not found" }, { status: 404 }); + logger.error("thing lookup failed", { environmentId: params.envId, error }); + throw error; + } +}`; + + it("does not pay for wrapping a body in a catch that only rethrows", () => { + const before = scoreEntry(scanFile("api.v1.x.ts", plain)!); + const after = scoreEntry(scanFile("api.v1.x.ts", wrappedInARethrow)!); + expect(after.score).toBeLessThanOrEqual(before.score); + }); + + // A4. rethrows used to be set by any ThrowStatement anywhere in the clause, dead code included, + // so appending `throw e;` after a `return` in a swallowing catch flipped error-classification + // from fail to not-applicable: a mutation with no behavioural effect that hid the swallow. + it("does not improve the verdict when a dead throw follows a return in a swallowing catch", () => { + const swallows = `import { prisma } from "~/db.server"; +export async function loader() { + try { + ${BODY} + } catch (e) { + return null; + } +}`; + const swallowsWithDeadThrow = `import { prisma } from "~/db.server"; +export async function loader() { + try { + ${BODY} + } catch (e) { + return null; + throw e; + } +}`; + + const before = scoreEntry(scanFile("api.v1.x.ts", swallows)!); + const after = scoreEntry(scanFile("api.v1.x.ts", swallowsWithDeadThrow)!); + + expect(before.checks.find((c) => c.id === "error-classification")!.status).toBe("fail"); + expect(after.checks.find((c) => c.id === "error-classification")!.status).toBe("fail"); + expect(after.score).toBeLessThanOrEqual(before.score); + }); + + it("does not pay for wrapping a whole tree in catches that only rethrow", () => { + const before = buildReport( + [scanFile("api.v1.x.ts", plain)!, scanFile("api.v1.y.ts", plain)!], + [] + ); + const after = buildReport( + [scanFile("api.v1.x.ts", wrappedInARethrow)!, scanFile("api.v1.y.ts", wrappedInARethrow)!], + [] + ); + expect(after.global!).toBeLessThanOrEqual(before.global!); + }); + + it("does not pay for deleting error handling either", () => { + const before = scoreEntry(scanFile("api.v1.x.ts", handled)!); + const after = scoreEntry(scanFile("api.v1.x.ts", plain)!); + expect(after.score).toBeLessThanOrEqual(before.score); + // And the handled version is genuinely better, so the invariant is not holding by both being 0. + expect(before.score).toBeGreaterThan(after.score); + }); + + it("still credits a catch that decides something on its way through", () => { + const scored = scoreEntry(scanFile("api.v1.x.ts", handled)!); + expect(scored.checks.find((c) => c.id === "error-classification")!.status).toBe("pass"); + }); +}); + +// C4b. A route whose body is in another module used to be scored as if it were a redirect stub: +// zero statements, zero callees, `isTrivial` true, every check not-applicable, and a placeholder +// 100 that no mean ever used. The tool said nothing about it at all, so moving a body into a +// `.server.ts` file silently deleted the route from the metric. +describe("a route that delegates its body to another module", () => { + const DELEGATED = `export { action } from "./handler.server";`; + const entry = () => scoreEntry(scanFile("webhooks.v1.stripe.ts", DELEGATED)!); + + it("reports every check as not-applicable for the reason that is true", () => { + const e = entry(); + expect(e.rawChecks.every((c) => c.status === "not-applicable")).toBe(true); + expect(new Set(e.rawChecks.map((c) => c.detail))).toEqual( + new Set(["delegates its body to another module"]) + ); + }); + + it("is not measured, and says so on the entry", () => { + const e = entry(); + expect(e.delegating).toBe(true); + expect(e.measured).toBe(false); + }); + + // request-context would otherwise fail it for leaving its failures to the central handler, which + // is an accusation about a body this file does not contain. + it("is not accused of anything the scanner cannot see", () => { + expect(entry().rawChecks.find((c) => c.id === "request-context")!.status).toBe( + "not-applicable" + ); + }); + + it("is counted apart from the entries nothing happened to apply to", () => { + const r = buildReport( + [scanFile("webhooks.v1.stripe.ts", DELEGATED)!, scanFile("@.ts", TRIVIAL)!], + [] + ); + expect(r.delegating).toEqual(["webhooks.v1.stripe.ts"]); + expect(r.unmeasured).toBe(1); + expect(r.measured).toBe(0); + }); + + it("cannot raise the global, since it is in no mean", () => { + const withDelegate = buildReport( + [scanFile("api.v1.b.ts", BUSY_AND_FAILING)!, scanFile("webhooks.v1.stripe.ts", DELEGATED)!], + [] + ); + const without = buildReport([scanFile("api.v1.b.ts", BUSY_AND_FAILING)!], []); + expect(withDelegate.global).toBe(without.global); + }); +}); + +// C5. The composite is disclosed rather than weighted: the reader gets applicability, pass rate, +// how many entries rest on one check alone, and what the global would be without each check. +describe("per-check contribution", () => { + const r = () => + buildReport( + [ + scanFile("api.v1.a.ts", BUSY_AND_FAILING)!, + scanFile("api.v1.b.ts", RAW)!, + scanFile("@.ts", TRIVIAL)!, + ], + [] + ); + + it("has one row per check, in registry order", () => { + expect(r().checkContributions.map((c) => c.id)).toEqual([ + "error-classification", + "auth-boundary", + "auth-scope", + "request-context", + "audit-trail", + ]); + }); + + it("counts applicability and passes off the pre-suppression results", () => { + const context = r().checkContributions.find((c) => c.id === "request-context")!; + expect(context.applicable).toBe(2); + expect(context.passed).toBe(0); + }); + + // `RAW` has no catch, so request-context is the only scored check that applies to it. + it("counts the entries that rest on one check alone", () => { + const rows = r().checkContributions; + expect(rows.find((c) => c.id === "request-context")!.sole).toBe(1); + expect(rows.find((c) => c.id === "error-classification")!.sole).toBe(0); + }); + + it("says what the global would be without each scored check", () => { + const report = r(); + expect(report.global).toBe(0); + const errors = report.checkContributions.find((c) => c.id === "error-classification")!; + expect(errors.scored).toBe(true); + expect(errors.globalWithout).toBe(0); + }); + + it("gives no without-figure for a check that is not in the score", () => { + const audit = r().checkContributions.find((c) => c.id === "audit-trail")!; + expect(audit.scored).toBe(false); + expect(audit.globalWithout).toBeNull(); + }); + + // Taking the only applicable check away leaves nothing measured, which is an absence and not a + // perfect score. + it("gives a null without-figure when nothing would be left measured", () => { + const only = buildReport([scanFile("api.v1.b.ts", RAW)!], []); + expect( + only.checkContributions.find((c) => c.id === "request-context")!.globalWithout + ).toBeNull(); + }); +}); + +// C4b, stated as the property rather than as a shape. Moving a body into a `.server.ts` file is an +// ordinary refactor: it must not delete the route from the metric silently. The corpus cannot hold +// this one, because its per-route assertion reads "dropped out of the measured set" as a rise, and +// dropping out is the correct outcome here. What is forbidden is dropping out QUIETLY. +describe("refactoring a body out of the route file", () => { + const BEFORE = `import { prisma } from "~/db.server"; +export async function action() { + try { return await prisma.thing.create({ data: {} }); } catch (e) { return null; } +}`; + const AFTER = `export { action } from "./handler.server";`; + + it("leaves the mean, and is reported instead of being dropped", () => { + const before = buildReport([scanFile("webhooks.v1.stripe.ts", BEFORE)!], []); + const after = buildReport([scanFile("webhooks.v1.stripe.ts", AFTER)!], []); + + expect(before.measured).toBe(1); + expect(before.global).toBe(0); + expect(after.measured).toBe(0); + expect(after.global).toBeNull(); + expect(after.delegating).toEqual(["webhooks.v1.stripe.ts"]); + expect(after.unmeasured).toBe(0); + }); +}); + +/** + * `contextGap` and `auditGap` are the same arithmetic `checkContributions` already does for every + * check, written out again by hand for two named ids: `map(find).filter(status)` for the context + * figure, `filter(some)` for the audit one, and a third spelling of "passed" for each. Three + * implementations of "applicable, and how many of those passed", and nothing said they had to + * agree, on the two figures the report puts in front of a reader as headline numbers. + * + * Pinned rather than shared. Collapsing them would mean the gap figures reading their check's row + * out of `checkContributions`, which is a fine refactor and a wider blast radius than the property + * is worth: what matters is that they cannot disagree, and an assertion says that without moving + * any code the renderers read. + */ +describe("the hand-rolled gap figures agree with the per-check contributions", () => { + const SOURCE = `import { prisma } from "~/db.server"; +import { logger } from "~/services/logger.server"; +export async function action({ params }) { + try { + return await prisma.apiKey.create({ data: { orgId: params.orgId } }); + } catch (e) { + logger.error("failed", { orgId: params.orgId }); + return null; + } +}`; + + // A sensitive mutation that DOES record an audit event, so `withAudit` is not simply + // `sensitiveMutations`. Without it the audit assertion held whatever the numerator counted. + const AUDITED = `import { prisma } from "~/db.server"; +import { startImpersonation } from "~/models/admin.server"; +export async function action({ request, params }) { + const session = await startImpersonation(request, params.userId); + await prisma.apiKey.create({ data: { orgId: params.orgId } }); + return redirect("/", { headers: session }); +}`; + + const report = buildReport( + [ + scanFile("api.v1.orgs.$orgId.apikeys.ts", SOURCE)!, + scanFile("api.v1.tokens.ts", SOURCE)!, + scanFile("resources.impersonation.ts", AUDITED)!, + scanFile("healthcheck.ts", `export const loader = () => new Response("ok");`)!, + ], + [] + ); + + const contribution = (id: string) => report.checkContributions.find((c) => c.id === id)!; + + it("reports the same request-context denominator and numerator", () => { + expect(report.contextGap.applicable).toBe(contribution("request-context").applicable); + expect(report.contextGap.naming).toBe(contribution("request-context").passed); + }); + + it("reports the same audit-trail denominator and numerator", () => { + expect(report.auditGap.sensitiveMutations).toBe(contribution("audit-trail").applicable); + expect(report.auditGap.withAudit).toBe(contribution("audit-trail").passed); + }); + + // A denominator of zero would make both assertions above hold vacuously. + // Both assertions above hold vacuously on a zero denominator, and the audit one holds vacuously + // whenever every applicable route fails, since the two counts coincide. + it("measured something for both of them, with the audit numerator strictly between", () => { + expect(report.contextGap.applicable).toBeGreaterThan(0); + expect(report.auditGap.withAudit).toBeGreaterThan(0); + expect(report.auditGap.withAudit).toBeLessThan(report.auditGap.sensitiveMutations); + }); +}); diff --git a/internal-packages/observability-map/src/score.ts b/internal-packages/observability-map/src/score.ts new file mode 100644 index 00000000000..86f15402a51 --- /dev/null +++ b/internal-packages/observability-map/src/score.ts @@ -0,0 +1,293 @@ +import type { CheckResult, EntryPoint } from "./types.js"; +import { CHECKS, SCORED_CHECK_IDS } from "./checks/index.js"; +import { parseSuppressions } from "./suppression.js"; +import { familyOf, routePathOf, type Family } from "./adapters/remix.js"; +import { classifySensitivity } from "./sensitivity.js"; + +export type ScoredEntry = { + fileName: string; + routePath: string; + family: Family; + sensitive: boolean; + /** + * The route's body is in another module (`EntryPoint.delegating`), so every check here reads + * not-applicable for that reason and the entry is never measured. Carried separately from + * `measured` because "we could not see it" and "nothing happened to apply" are different facts + * and the report has to be able to say which. + */ + delegating: boolean; + /** Post-suppression: a suppressed check reads `not-applicable` here, with the reason in + * `detail`. This is the display view; every denominator below reads `rawChecks` instead, so a + * suppression is never invisible to a published figure just because its check is not scored. */ + checks: CheckResult[]; + /** Every check exactly as it ran, before a suppression comment can turn a result into + * `not-applicable`. The one true source for any figure that counts applicability: `measured` + * below, and `contextGap`/`auditGap` in `MapReport`, which read this rather than `checks` for + * exactly that reason. */ + rawChecks: CheckResult[]; + /** + * Whether at least one scored check (`SCORED_CHECK_IDS`, so never `audit-trail`) was applicable + * before suppression. A fully-suppressed entry stays measured, at its capped score, so a + * suppression cannot buy removal from every mean by way of removal from this one. + * `false` means nothing was measured here: the 100 in `score` is a vacuous default, not a + * finding, and `buildReport` excludes an unmeasured entry from every mean it computes so that + * default cannot inflate a figure nobody checked. + */ + measured: boolean; + /** Every check a comment in the source suppressed, scored or not, in `CHECKS` order. Includes + * `audit-trail`: a suppression is real regardless of whether its check feeds the score. */ + suppressed: string[]; + /** Ids in a suppression directive that name no check, so they suppress nothing. Carried here so + * the renderers can say so: dropping them silently is what made a typo look like an + * acknowledgement. */ + unknownSuppressions: string[]; + /** Passed over applicable, across scored checks only. 100 when nothing applies. */ + score: number; +}; + +/** + * What one check contributes to the composite, so a reader can see what the global is made of. + * + * The four-check framing presents a composite the number is not: `request-context` applies to + * nearly every entry point and the rest apply to a minority, so most entries score 0 or 100 on one + * boolean. Disclosed rather than weighted, deliberately. Weighting was rejected in the design + * because a coefficient nobody can explain invites argument about the number instead of the + * finding, and that reasoning has not changed. + */ +export type CheckContribution = { + id: string; + /** Entry points the check was applicable to, pre-suppression. */ + applicable: number; + /** Of those, how many passed. */ + passed: number; + /** Whether the check feeds `global` at all. `audit-trail` does not, see `buildReport`. */ + scored: boolean; + /** Entry points where this was the ONLY applicable scored check, so their score is this check's + * verdict and nothing else. Zero for a check that is not scored. */ + sole: number; + /** The global recomputed with this check taken out of the score, so the difference from `global` + * is what the check is worth. Null when the check is not scored, and null when taking it out + * would leave nothing measured. */ + globalWithout: number | null; +}; + +export type MapReport = { + /** Null when no entry point had an applicable scored check: an absent figure, not a perfect one. */ + global: number | null; + /** Entry points with at least one applicable scored check, i.e. those `global` is averaged over. */ + measured: number; + /** Entry points every scored check reported not-applicable for; excluded from `global`. Counts + * only routes the scanner could read: a delegating one is in `delegating` instead. */ + unmeasured: number; + /** + * Routes whose body is in another module, by file name. Excluded from `global` for the same + * reason a parse failure is, and reported for the same reason: the denominator is smaller than + * the entry point count and nothing about these routes has been checked. Moving a body into a + * `.server.ts` file is an ordinary refactor, and without this it silently deletes the route from + * the metric while the route reads as having nothing to fix. + */ + delegating: string[]; + /** Per-check applicability, pass rate and worth, in `CHECKS` order. */ + checkContributions: CheckContribution[]; + /** Suppressions in force: how many entry points carry one, and how many scored checks in total. */ + suppressions: { entries: number; checks: number }; + /** Suppression directives naming no check, per file, so a typo is reported rather than dropped. */ + unknownSuppressions: { fileName: string; ids: string[] }[]; + byFamily: Record; + sensitiveCohort: { n: number; measured: number; mean: number | null }; + auditGap: { sensitiveMutations: number; withAudit: number }; + /** + * `request-context` fails 401 of the 412 entry points it applies to, so it is reported as a + * figure rather than as hundreds of identical list entries, the same treatment `audit-trail` + * gets. It stays fully in the score: the gap is real and the score is meant to show it. + */ + contextGap: { applicable: number; naming: number }; + entries: ScoredEntry[]; + parseFailures: string[]; +}; + +/** + * What every check reports for a route whose body is in another module. Applied here rather than in + * each check, because it is a fact about what the scan could see and not about any one question: + * the file holds no handler function and no builder call, so there is nothing for a check to read + * and no check may claim a verdict. `request-context` would otherwise fail such a route for leaving + * its failures to the central handler, an accusation about a body this file does not contain. + * + * Because it is answered here, no check tests `ep.delegating` itself. Two did, and both branches + * were unreachable. + */ +const DELEGATED_CHECKS = (): CheckResult[] => + CHECKS.map((c) => ({ + id: c.id, + status: "not-applicable" as const, + detail: "delegates its body to another module", + })); + +export function scoreEntry(ep: EntryPoint): ScoredEntry { + const { byId: suppressed, unknown } = parseSuppressions(ep.source, ep.fileName); + const raw = ep.delegating ? DELEGATED_CHECKS() : CHECKS.map((c) => c.run(ep)); + const checks = raw.map((result) => { + const reason = suppressed.get(result.id); + return reason + ? { id: result.id, status: "not-applicable" as const, detail: `suppressed: ${reason}` } + : result; + }); + + const scored = raw.filter((c) => SCORED_CHECK_IDS.includes(c.id)); + const ratio = (of: CheckResult[]) => { + const applicable = of.filter((c) => c.status !== "not-applicable"); + if (applicable.length === 0) return 100; + return Math.round( + (applicable.filter((c) => c.status === "pass").length / applicable.length) * 100 + ); + }; + + const visible = scored.filter((c) => !suppressed.has(c.id)); + const scoredApplicable = scored.filter((c) => c.status !== "not-applicable"); + + return { + fileName: ep.fileName, + routePath: routePathOf(ep.fileName), + family: familyOf(ep.fileName), + sensitive: classifySensitivity(ep).sensitive, + delegating: ep.delegating, + checks, + rawChecks: raw, + suppressed: raw.filter((c) => suppressed.has(c.id)).map((c) => c.id), + unknownSuppressions: unknown, + measured: scoredApplicable.length > 0, + // Capped by the pre-suppression ratio: removing a failing check from both the numerator and + // the denominator otherwise raises the ratio, which is how 33 became 50 became 100 before this + // cap existed. See ScoredEntry.measured for why the denominator itself is pre-suppression too. + score: Math.min(ratio(visible), ratio(scored)), + }; +} + +/** + * Null for an empty group rather than 100. A family nothing was measured in has no score, and + * rendering the absence as a full green bar said the opposite of what the data said. + */ +const mean = (xs: number[]): number | null => + xs.length === 0 ? null : Math.round(xs.reduce((a, b) => a + b, 0) / xs.length); + +/** + * `n` is every entry point in the group; `mean` is taken over the measured subset only, so an + * entry point nothing applied to cannot drag a family's or cohort's figure toward 100. `measured` + * is reported alongside so a reader can tell a family scoring high because it is clean apart from + * a family scoring high because most of it was never measured. + */ +function groupStats(entries: ScoredEntry[]): { + n: number; + measured: number; + mean: number | null; +} { + const measuredEntries = entries.filter((e) => e.measured); + return { + n: entries.length, + measured: measuredEntries.length, + mean: mean(measuredEntries.map((e) => e.score)), + }; +} + +/** + * The global as it would read with `omitted` taken out of the scored set, so the difference from + * the published global is what that check is worth. Recomputed from `rawChecks` the same way + * `scoreEntry` computes a score, minus the suppression cap: a suppression can only lower an entry's + * score, and lowering both figures by the same rule would leave the difference between them saying + * something about suppressions rather than about the check. + */ +function globalWithout(entries: ScoredEntry[], omitted: string): number | null { + const scores: number[] = []; + for (const e of entries) { + const applicable = e.rawChecks.filter( + (c) => SCORED_CHECK_IDS.includes(c.id) && c.id !== omitted && c.status !== "not-applicable" + ); + if (applicable.length === 0) continue; + const passed = applicable.filter((c) => c.status === "pass").length; + scores.push(Math.round((passed / applicable.length) * 100)); + } + return mean(scores); +} + +function checkContributions(entries: ScoredEntry[]): CheckContribution[] { + return CHECKS.map((check) => { + const results = entries + .map((e) => e.rawChecks.find((c) => c.id === check.id)) + .filter((c): c is CheckResult => c !== undefined && c.status !== "not-applicable"); + const scored = SCORED_CHECK_IDS.includes(check.id); + return { + id: check.id, + applicable: results.length, + passed: results.filter((c) => c.status === "pass").length, + scored, + sole: scored + ? entries.filter((e) => { + const applicable = e.rawChecks.filter( + (c) => SCORED_CHECK_IDS.includes(c.id) && c.status !== "not-applicable" + ); + return applicable.length === 1 && applicable[0]!.id === check.id; + }).length + : 0, + globalWithout: scored ? globalWithout(entries, check.id) : null, + }; + }); +} + +export function buildReport(eps: EntryPoint[], parseFailures: string[]): MapReport { + const entries = eps.map(scoreEntry); + const measuredEntries = entries.filter((e) => e.measured); + + const byFamily: MapReport["byFamily"] = {}; + for (const family of new Set(entries.map((e) => e.family))) { + byFamily[family] = groupStats(entries.filter((e) => e.family === family)); + } + + const sensitive = entries.filter((e) => e.sensitive); + + // audit-trail is excluded from the score (see checks/index.ts and scoreEntry above), and is + // reported here as its own architectural figure instead: how many sensitive mutations have an + // audit record, out of how many. Folding it into the score would tank every sensitive route on a + // gap that is the same everywhere, and bury the routes that have their own, fixable problems. + // + // Both gaps read `rawChecks`, pre-suppression, the same reason `measured` does: suppressing the + // one request-context or audit-trail finding on an entry must not shrink these denominators and + // raise the printed percentage, on the same screen as a claim that suppression cannot do that. + const contextChecks = entries + .map((e) => e.rawChecks.find((c) => c.id === "request-context")) + .filter((c): c is CheckResult => c !== undefined && c.status !== "not-applicable"); + + const auditApplicable = entries.filter((e) => + e.rawChecks.some((c) => c.id === "audit-trail" && c.status !== "not-applicable") + ); + + const suppressing = entries.filter((e) => e.suppressed.length > 0); + + return { + global: mean(measuredEntries.map((e) => e.score)), + measured: measuredEntries.length, + unmeasured: entries.filter((e) => !e.measured && !e.delegating).length, + delegating: entries.filter((e) => e.delegating).map((e) => e.fileName), + checkContributions: checkContributions(entries), + suppressions: { + entries: suppressing.length, + checks: suppressing.reduce((n, e) => n + e.suppressed.length, 0), + }, + unknownSuppressions: entries + .filter((e) => e.unknownSuppressions.length > 0) + .map((e) => ({ fileName: e.fileName, ids: e.unknownSuppressions })), + byFamily, + sensitiveCohort: groupStats(sensitive), + auditGap: { + sensitiveMutations: auditApplicable.length, + withAudit: auditApplicable.filter((e) => + e.rawChecks.some((c) => c.id === "audit-trail" && c.status === "pass") + ).length, + }, + contextGap: { + applicable: contextChecks.length, + naming: contextChecks.filter((c) => c.status === "pass").length, + }, + entries, + parseFailures, + }; +} diff --git a/internal-packages/observability-map/src/sensitivity.test.ts b/internal-packages/observability-map/src/sensitivity.test.ts new file mode 100644 index 00000000000..bb89f427de1 --- /dev/null +++ b/internal-packages/observability-map/src/sensitivity.test.ts @@ -0,0 +1,238 @@ +import { classifySensitivity } from "./sensitivity.js"; +import { scanFile } from "./scan.js"; + +const ep = (fileName: string, source: string) => scanFile(fileName, source)!; + +describe("classifySensitivity", () => { + it("flags a route whose filename says nothing but which calls a sensitive helper", () => { + const e = ep( + "@.ts", + `import { clearImpersonation } from "~/models/admin.server"; + export async function loader({ request }) { return clearImpersonation(request, "/admin"); }` + ); + const s = classifySensitivity(e); + expect(s.sensitive).toBe(true); + expect(s.reasons.some((r) => r.includes("clearImpersonation"))).toBe(true); + }); + + it("flags on the path when the filename is explicit", () => { + const e = ep("api.v1.projects.$ref.envvars.ts", `export async function loader() { return 1; }`); + expect(classifySensitivity(e).sensitive).toBe(true); + }); + + it("does not flag an ordinary read route", () => { + const e = ep( + "api.v1.timezones.ts", + `import { json } from "@remix-run/server-runtime"; + export async function loader() { return json({ timezones: [] }); }` + ); + expect(classifySensitivity(e).sensitive).toBe(false); + }); + + it("does not flag on a substring that merely contains a sensitive word", () => { + const e = ep( + "api.v1.authorship.ts", + `export async function loader() { return { author: "x" }; }` + ); + expect(classifySensitivity(e).sensitive).toBe(false); + }); +}); + +// Directory routes: `fileName` can be `dirName/route.tsx` rather than a flat dotted name. A naive +// `fileName.split(".")` treats "billing/route" as one non-matching segment because the slash never +// gets split, so it misses the directory name entirely. The classifier must derive real path +// segments (via the same route-path logic the remix adapter uses) so both shapes work alike. +describe("classifySensitivity: directory routes", () => { + it("flags a single-segment directory route by its directory name", () => { + const e = ep("billing/route.tsx", `export async function loader() { return 1; }`); + const s = classifySensitivity(e); + expect(s.sensitive).toBe(true); + expect(s.reasons.some((r) => r.includes("billing"))).toBe(true); + }); + + it("flags a multi-segment directory route via a segment among dynamic params", () => { + const e = ep( + "_app.orgs.$slug.billing/route.tsx", + `export async function loader() { return 1; }` + ); + expect(classifySensitivity(e).sensitive).toBe(true); + }); + + it("does not flag a directory route on a substring that merely contains a sensitive word", () => { + const e = ep("api.v1.authorship/route.tsx", `export async function loader() { return 1; }`); + expect(classifySensitivity(e).sensitive).toBe(false); + }); +}); + +// calleeNames is scoped to the loader/action body, unlike importedNames which is file-wide. A +// sensitive symbol called only at module scope is invisible to calleeNames, so it is only caught +// when it also shows up as an import. +describe("classifySensitivity: calleeNames is body-scoped, importedNames is file-wide", () => { + it("flags a sensitive symbol invoked only at module scope, via the import rather than the callee", () => { + const e = ep( + "api.v1.setup.ts", + `import { startImpersonation } from "~/models/admin.server"; + startImpersonation(globalThis, "seed"); + export async function loader() { return 1; }` + ); + const s = classifySensitivity(e); + expect(s.sensitive).toBe(true); + expect(s.reasons.some((r) => r.includes("startImpersonation"))).toBe(true); + }); + + it("flags a sensitive callee defined locally and invoked inside the loader, even without an import", () => { + const e = ep( + "api.v1.local-admin.ts", + `async function regenerateApiKey() { return "x"; } + export async function loader() { return regenerateApiKey(); }` + ); + expect(classifySensitivity(e).sensitive).toBe(true); + }); + + it("does not flag a sensitive-named call made only at module scope outside the loader/action body", () => { + const e = ep( + "api.v1.module-scope.ts", + `function regenerateApiKey() { return "x"; } + regenerateApiKey(); + export async function loader() { return 1; }` + ); + expect(classifySensitivity(e).sensitive).toBe(false); + }); +}); + +describe("what sensitivity must not mean", () => { + const ep = (fileName: string, source: string) => scanFile(fileName, source)!; + + // C4. Calling the admin guard cannot be what makes a route risky: it is the mitigation, not the + // hazard. Counting it made 34 of 67 sensitive entries sensitive only because they were guarded, + // and `auth-boundary` then passed every one of them on the same call. Circular, and it was the + // fix list's primary sort key. + it("does not treat calling the admin guard as what makes a route sensitive", () => { + const s = classifySensitivity( + ep( + "admin.api.v1.queue-metrics.ts", + `import { requireAdminApiRequest } from "~/services/personalAccessToken.server"; + import { prisma } from "~/db.server"; + export async function loader({ request }) { + await requireAdminApiRequest(request); + return json(await prisma.queueMetric.findMany()); + }` + ) + ); + expect(s.sensitive).toBe(false); + }); + + it("still flags an admin route that does something sensitive on its own account", () => { + const s = classifySensitivity( + ep( + "admin.api.v1.impersonate.ts", + `import { requireAdminApiRequest } from "~/services/personalAccessToken.server"; + import { startImpersonation } from "~/models/admin.server"; + export async function action({ request }) { + await requireAdminApiRequest(request); + return startImpersonation(request, "user_1"); + }` + ) + ); + expect(s.sensitive).toBe(true); + expect(s.reasons).toContain("calls startImpersonation"); + }); + + // A waitpoint token is a run coordination handle, not a credential. Seven of the eight routes + // matching the `tokens` segment were waitpoint routes. + it("does not treat a waitpoint token route as a credential route", () => { + const s = classifySensitivity( + ep( + "api.v1.waitpoints.tokens.$waitpointFriendlyId.complete.ts", + `import { prisma } from "~/db.server"; + export async function action() { return prisma.waitpoint.update({ where: {}, data: {} }); }` + ) + ); + expect(s.sensitive).toBe(false); + }); + + it("still flags the personal access token routes", () => { + expect( + classifySensitivity( + ep( + "account.tokens/route.tsx", + `import { prisma } from "~/db.server"; + export async function loader() { return prisma.personalAccessToken.findMany(); }` + ) + ).reasons + ).toContain('path segment "tokens"'); + expect( + classifySensitivity( + ep( + "api.v1.token.ts", + `import { prisma } from "~/db.server"; + export async function action() { return prisma.token.create({ data: {} }); }` + ) + ).reasons + ).toContain('path segment "token"'); + }); +}); + +// C2. The classifier covered tokens, billing, impersonation and envvars and missed the entire +// surface where authorization bugs live, so `auth-boundary` and `audit-trail` never looked at +// membership, the login surface, API keys or the two billing settings the bare `billing` segment +// does not match. The vocabulary below is read off `apps/webapp/app/routes`; +// `test/webappSymbols.test.ts` is what holds it to that. +describe("classifySensitivity: the access-control surface", () => { + const flags = (fileName: string) => + classifySensitivity(ep(fileName, `export async function loader() { return 1; }`)).sensitive; + + it.each([ + ["api.v1.orgs.$orgParam.members.ts", "membership"], + ["api.v1.orgs.$orgParam.invites.ts", "invites"], + ["invite-accept.tsx", "an invite acceptance"], + ["_app.orgs.$organizationSlug.settings.roles/route.tsx", "roles"], + ["orgs.$organizationSlug.team.ts", "the team page"], + ["login._index/route.tsx", "the login page"], + ["login.mfa/route.tsx", "the mfa step"], + ["magic.tsx", "a magic link"], + ["auth.sso.ts", "sso"], + ["account.security/route.tsx", "account security"], + [ + "_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.apikeys/route.tsx", + "api keys", + ], + ["admin.api.v1.revoked-api-keys.$id.ts", "revoked api keys"], + ["_app.orgs.$organizationSlug.settings.billing-limits/route.tsx", "billing limits"], + ["_app.orgs.$organizationSlug.settings.billing-alerts/route.tsx", "billing alerts"], + ["admin.api.v1.orgs.$organizationId.billing-limit.hit.ts", "an admin billing limit"], + ["resources.account.session-duration/route.tsx", "session duration"], + ])("flags %s (%s)", (fileName) => { + expect(flags(fileName)).toBe(true); + }); + + // Remix's trailing underscore opts a route out of its parent layout and says nothing about what + // the route does, so the segment vocabulary is written without it. + it("flags a route whose segment carries the no-layout marker", () => { + expect(flags("resources.impersonation_.view-as.ts")).toBe(true); + expect(flags("resources.impersonation.ts")).toBe(true); + }); + + // Measured and rejected. Every `sessions` route in this tree is the realtime agent-session + // product, sixteen files of it, and reading the word as the auth session surface would have put + // all sixteen in front of the routes that mint credentials. + it("does not flag the agent-session product on the word sessions", () => { + expect(flags("api.v1.sessions.$session.close.ts")).toBe(false); + expect( + flags( + "_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.sessions._index/route.tsx" + ) + ).toBe(false); + }); + + // Measured and rejected. Logging out destroys the caller's own session: no other party's + // credential to guard, no actor to record beyond the one already leaving. + it("does not flag the logout route", () => { + expect(flags("logout.tsx")).toBe(false); + }); + + it("still does not flag an ordinary route that shares a prefix with the new vocabulary", () => { + expect(flags("api.v1.teams-directory.ts")).toBe(false); + expect(flags("resources.logins.ts")).toBe(false); + }); +}); diff --git a/internal-packages/observability-map/src/sensitivity.ts b/internal-packages/observability-map/src/sensitivity.ts new file mode 100644 index 00000000000..ba13454fe17 --- /dev/null +++ b/internal-packages/observability-map/src/sensitivity.ts @@ -0,0 +1,163 @@ +import type { EntryPoint } from "./types.js"; +import { routePathOf } from "./adapters/remix.js"; + +/** + * Symbols whose presence says the route does something risky: minting or revoking a credential, + * escalating to another user, destroying a tenant. Calling a guard is not one of them: + * `requireAdminApiRequest` was on this list and made 34 of the 67 sensitive entry points sensitive + * purely because they were guarded, which `auth-boundary` then passed them for. A mitigation + * cannot be the hazard, and this list feeds the fix list's primary sort key. + * + * Half of this list used to name nothing. `Set.has` is exact, so `setImpersonation`, `createJWT`, + * `signJWT` and `updateEnvVars`, none of which are exported anywhere in `apps/webapp/app`, matched + * no route at all, while the real escalation `startImpersonation` was absent. Every name here now + * resolves to a declaration in the webapp, and `webappSymbols.test.ts` fails if one stops + * doing so. + */ +export const SENSITIVE_SYMBOLS = [ + // Escalation: acting as another user. + "startImpersonation", + "clearImpersonation", + "generateImpersonationToken", + // Minting and revoking credentials. + "createPersonalAccessToken", + "createPersonalAccessTokenFromAuthorizationCode", + "revokePersonalAccessToken", + "createOrganizationAccessToken", + "revokeOrganizationAccessToken", + "createAuthorizationCode", + "createApiKeyForEnv", + "createPkApiKeyForEnv", + "regenerateApiKey", + "generateJWTTokenForEnvironment", + "generateRegistryCredentials", + "mintRunToken", + "mintSessionToken", + "mintDashboardAgentToken", + "mintDashboardAgentUserActorToken", + // Access control and tenant destruction. `DeleteOrganizationService`/`DeleteProjectService` are + // classes, reached through `importedNames`: four routes import one and none is named for it. + "removeTeamMember", + "revokeInvite", + "DeleteOrganizationService", + "DeleteProjectService", +]; + +/** + * Segments in `SENSITIVE_SEGMENTS` that match no route in the tree today. Kept because they are + * ordinary words for the thing they name, so a route called one of them would be sensitive the day + * it lands, and separated because the rest of the vocabulary is read off the tree and + * `webappSymbols.test.ts` holds it to that. Adding a word here is a deliberate statement that + * it names nothing yet, and shows up in review as one. + */ +export const ANTICIPATED_SEGMENTS = ["payment", "invoices", "secrets"]; + +/** + * Whole path segments only, so "authorship" does not match "auth". + * + * Every entry is a segment that exists in `apps/webapp/app/routes` today; the vocabulary was read + * off the tree rather than invented, and `webappSymbols.test.ts` fails if a segment stops + * appearing in a route name. Two consequences of that rule are worth stating rather than leaving + * to be rediscovered: there is no `transfer` segment because the webapp has no org or project + * transfer route, and org/project deletion is reached through `DeleteOrganizationService` above + * rather than through a segment, because the four routes that delete are named `orgs` and + * `projects` and `settings`. + * + * Two segments were measured and left out. + * + * `logout` is one route, and both questions the cohort exists to ask are meaningless on it: + * `logout.tsx` destroys the caller's own session, so there is no other party's credential to guard + * and no actor to record beyond the one already leaving. Including it produced one permanent + * `auth-boundary` failure that no change to the route could clear. + * + * `sessions` is the bigger one. It reads as the auth session surface and is not: every `sessions` + * route in this tree is the realtime agent-session product, + * `_app...env.$envParam.sessions._index/route.tsx` renders `SessionsTable`, and + * `api.v1.sessions.$session.close.ts` closes an agent session. Sixteen route files carry the + * segment. The genuine session-management surface is `session-duration`, which is here, and the + * credential minting inside those routes is caught by `mintSessionToken` in the symbol list above, + * which is why `api.v1.sessions.ts` is in the cohort and its fifteen siblings are not. + */ +export const SENSITIVE_SEGMENTS = [ + // Credentials, tokens and money: the original vocabulary. + "auth", + "jwt", + "token", + "tokens", + "envvars", + "billing", + ...ANTICIPATED_SEGMENTS, + "impersonate", + "authorization-code", + "regenerate-api-key", + // Access control: who is in a tenant and what they may do. + "members", + "invites", + "invite", + "invite-accept", + "invite-resend", + "invite-revoke", + "roles", + "team", + // Authentication and session management. The login surface handles credentials even though it + // is, by design, the one surface an unauthenticated caller may reach. + "login", + "magic", + "mfa", + "sso", + "security", + "session-duration", + // Credentials again, in the spellings the tree actually uses. + "apikeys", + "revoked-api-keys", + "impersonation", + // The bare `billing` segment matches neither of these, and it matches no admin route at all. + "billing-limit", + "billing-limits", + "billing-alerts", +]; + +/** + * A route-name segment with Remix's layout markers taken off, so the segment vocabulary can be + * written the way a reader would say it. A trailing underscore opts a route out of its parent + * layout (`resources.impersonation_.view-as.ts`) and changes nothing about what the route does. + */ +export function normalizeSegment(segment: string): string { + // Trimmed by hand rather than with /_+$/, which backtracks polynomially on a run of underscores + // and trips CodeQL. Nothing here is attacker-controlled (the input is a filename read off disk), + // so this is about not spending a reviewer's attention on the alert. + let end = segment.length; + while (end > 0 && segment[end - 1] === "_") end--; + return segment.slice(0, end); +} + +export type Sensitivity = { sensitive: boolean; reasons: string[] }; + +export function classifySensitivity(ep: EntryPoint): Sensitivity { + const reasons: string[] = []; + + // importedNames is file-wide; calleeNames is scoped to the loader/action body. A sensitive + // symbol called only at module scope is caught here only if it is also imported. + const symbols = new Set([...ep.importedNames, ...ep.calleeNames]); + for (const s of SENSITIVE_SYMBOLS) { + if (symbols.has(s)) reasons.push(`calls ${s}`); + } + + // `routePathOf` turns both flat routes (`api.v1.envvars.ts`) and directory routes + // (`billing/route.tsx`) into real `/`-separated path segments, so this matches whole segments in + // either shape rather than splitting the raw fileName on ".". + const segments = routePathOf(ep.fileName) + .split("/") + .filter((s) => s.length > 0) + .map(normalizeSegment); + for (const [i, seg] of segments.entries()) { + if (!SENSITIVE_SEGMENTS.includes(seg)) continue; + // A waitpoint token is a handle for resuming a run, not a credential. Seven of the eight + // `tokens` matches in the tree were waitpoint routes, so the segment on its own was mostly + // finding the wrong thing. + if ((seg === "token" || seg === "tokens") && segments[i - 1] === "waitpoints") continue; + reasons.push(`path segment "${seg}"`); + } + + return { sensitive: reasons.length > 0, reasons }; +} diff --git a/internal-packages/observability-map/src/suppression.test.ts b/internal-packages/observability-map/src/suppression.test.ts new file mode 100644 index 00000000000..62a29facbb4 --- /dev/null +++ b/internal-packages/observability-map/src/suppression.test.ts @@ -0,0 +1,302 @@ +import { parseSuppressions, suppressedChecks } from "./suppression.js"; + +describe("suppressedChecks", () => { + it("reads a suppression with its reason", () => { + const m = suppressedChecks( + `// obs-map-disable error-classification -- liveness probe, deliberately silent + export async function loader() { return { ok: true }; }` + ); + expect(m.get("error-classification")).toBe("liveness probe, deliberately silent"); + }); + + it("ignores a suppression with no reason", () => { + const m = suppressedChecks(`// obs-map-disable error-classification`); + expect(m.size).toBe(0); + }); + + it("ignores a suppression whose reason is only whitespace", () => { + const m = suppressedChecks(`// obs-map-disable error-classification -- `); + expect(m.size).toBe(0); + }); + + it("reads several suppressions in one file", () => { + const m = suppressedChecks( + `// obs-map-disable error-classification -- liveness probe + // obs-map-disable request-context -- no identifiers exist here + export async function loader() { return { ok: true }; }` + ); + expect(m.size).toBe(2); + expect(m.get("error-classification")).toBe("liveness probe"); + expect(m.get("request-context")).toBe("no identifiers exist here"); + }); + + it("returns nothing for a file with no suppressions", () => { + expect(suppressedChecks(`export async function loader() { return 1; }`).size).toBe(0); + }); + + it("does not carry a reason across lines", () => { + const m = suppressedChecks( + `// obs-map-disable error-classification + // some other comment -- with a dash + export async function loader() { return 1; }` + ); + expect(m.size).toBe(0); + }); + + // I2. The directive is a comment directive. Matching it file-wide meant a string literal that + // merely quotes it, in a test fixture or an error message, silently suppressed a real check. + it("ignores the directive inside a string literal", () => { + const m = suppressedChecks( + `const example = "obs-map-disable error-classification -- not a real suppression"; + export async function loader() { return 1; }` + ); + expect(m.size).toBe(0); + }); + + it("reads the directive from a block comment", () => { + const m = suppressedChecks( + `/* obs-map-disable auth-boundary -- public by design, see ADR 12 */ + export async function loader() { return 1; }` + ); + expect(m.get("auth-boundary")).toBe("public by design, see ADR 12"); + }); + + it("reads the directive from a jsdoc line", () => { + const m = suppressedChecks( + `/** + * obs-map-disable request-context -- nothing tenant-scoped here + */ + export async function loader() { return 1; }` + ); + expect(m.get("request-context")).toBe("nothing tenant-scoped here"); + }); + + it("ignores code that happens to follow a comment on the same line", () => { + const m = suppressedChecks( + `const x = 1; // obs-map-disable error-classification -- fine + export async function loader() { return x; }` + ); + expect(m.get("error-classification")).toBe("fine"); + }); + + // The directive was called `-next-line` while applying to the whole entry point, so a comment on + // the last line of a file switched a check off for everything above it. Renamed rather than + // scoped, because a finding has no line number to scope it to. The old spelling is not honoured. + it("does not honour the old -next-line spelling", () => { + const m = suppressedChecks( + `// obs-map-disable-next-line error-classification -- stale directive + export async function loader() { return 1; }` + ); + expect(m.size).toBe(0); + }); + + // A2. `indexOf("//")` against the raw text found the marker inside a string literal too, so a + // string that merely quotes the directive granted a suppression nobody wrote and silenced a real + // check. Reading comment ranges off the parsed source closes this: a string literal is one node + // with its own span, never trivia, so a directive inside it is content, not a comment. + it("does not suppress from a directive quoted inside a string literal", () => { + const m = suppressedChecks( + `const msg = "see // obs-map-disable error-classification -- because reasons"; + export async function loader() { return 1; }` + ); + expect(m.size).toBe(0); + }); + + it("does not suppress from a directive quoted inside a string with no real comment marker", () => { + const m = suppressedChecks( + `const u = "https://example.com obs-map-disable auth-boundary -- nope"; + export async function loader() { return 1; }` + ); + expect(m.size).toBe(0); + }); + + it("does not suppress from a directive inside a template literal", () => { + const m = suppressedChecks( + "const msg = `see // obs-map-disable error-classification -- template literal`;\n" + + "export async function loader() { return 1; }" + ); + expect(m.size).toBe(0); + }); + + // I3. A standalone `ts.createScanner` has no parser state, so it still granted two suppressions + // nobody wrote: it never rescans a template as a continuation after a `${...}` substitution, so + // text after the `}` reads as ordinary code and a `//` in it is a real comment to the scanner; + // and a scanner created in `LanguageVariant.Standard` has no JSX context, so a `//` inside JSX + // text reads as a line comment mid-URL. Reading comments off the actual parsed tree closes both: + // a template's literal segments and a JSX text node are real nodes, never trivia. + it("does not suppress from a directive after a template substitution", () => { + const m = suppressedChecks( + "const msg = `${name} // obs-map-disable error-classification -- via substitution`;\n" + + "export async function loader() { return 1; }" + ); + expect(m.size).toBe(0); + }); + + it("does not suppress from a directive inside JSX text", () => { + const m = suppressedChecks( + `export default function Page() { + return

docs at https://example.com obs-map-disable error-classification -- x

; + }`, + "route.tsx" + ); + expect(m.size).toBe(0); + }); + + // S4. The case above passes without any JSX handling at all, because the comment-range lexers + // only find a comment at the exact offset they are asked about and the `//` there is mid-text. + // These are the shapes that actually needed the fix: JSX text that BEGINS with a comment marker, + // which is what the lexers see when they are pointed at the start of a JsxText node. Removing + // `ts.isJsxText` from `isClaimedContent` makes all four fail and nothing else in the suite. + describe("jsx text is content, not a comment", () => { + const page = (body: string) => `const name = "x"; + export default function Page() { + return ${body}; + }`; + + it("does not suppress from JSX text beginning with a line comment marker", () => { + const m = suppressedChecks( + page(`

// obs-map-disable error-classification -- jsx line

`), + "route.tsx" + ); + expect(m.size).toBe(0); + }); + + it("does not suppress from JSX text beginning with a block comment marker", () => { + const m = suppressedChecks( + page(`

/* obs-map-disable audit-trail -- jsx block */

`), + "route.tsx" + ); + expect(m.size).toBe(0); + }); + + it("does not suppress from JSX text starting right after an expression container", () => { + const m = suppressedChecks( + page(`

{name}// obs-map-disable request-context -- after expression

`), + "route.tsx" + ); + expect(m.size).toBe(0); + }); + + // A fourth input, exercising the same mechanism at a different tree position: the text is not + // the first child of the outermost element, so the token boundary the lexer is pointed at is a + // different one again. + it("does not suppress from JSX text nested several elements deep", () => { + const m = suppressedChecks( + page(`
// obs-map-disable auth-boundary -- nested jsx
`), + "route.tsx" + ); + expect(m.size).toBe(0); + }); + + // Positive control for the same code path: a real comment inside a JSX expression container is + // not JSX text and must survive. A filter that dropped it would pass every test above for the + // wrong reason. + it("still reads a directive from a comment in a JSX expression container", () => { + const m = suppressedChecks( + page(`

{/* obs-map-disable audit-trail -- real comment */}

`), + "route.tsx" + ); + expect(m.get("audit-trail")).toBe("real comment"); + }); + }); + + // Extra inputs beyond the brief's two, exercising the same "not a real parse position" mechanism + // differently: a second substitution, and a string literal nested inside a JSX expression + // container, which is a different node kind again from either hole above. + it("does not suppress from a directive after a second template substitution", () => { + const m = suppressedChecks( + "const msg = `${a}${b} // obs-map-disable auth-boundary -- nested substitution`;\n" + + "export async function loader() { return 1; }" + ); + expect(m.size).toBe(0); + }); + + it("does not suppress from a directive inside a string literal nested in a JSX expression container", () => { + const m = suppressedChecks( + `export default function Page() { + return

{"see // obs-map-disable request-context -- nested string"}

; + }`, + "route.tsx" + ); + expect(m.size).toBe(0); + }); + + // Positive control: a genuine directive still works in a .tsx file, and a directive after a + // template with no substitution (already covered above) is not the only shape that must survive. + it("still reads a genuine directive in a .tsx file", () => { + const m = suppressedChecks( + `// obs-map-disable error-classification -- liveness probe + export default function Page() { return

hi

; }`, + "route.tsx" + ); + expect(m.get("error-classification")).toBe("liveness probe"); + }); + + // Regression control: a generic arrow function is only unambiguous when the file is parsed as + // plain TypeScript, not TSX (`` would otherwise start a JSX element). A .ts file must still + // parse sanely and keep reading a genuine trailing comment correctly. + it("still reads a genuine directive beside a generic arrow function in a .ts file", () => { + const m = suppressedChecks( + "const identity = (x: T): T => x; // obs-map-disable auth-boundary -- generic helper\n" + + "export async function loader() { return identity(1); }" + ); + expect(m.get("auth-boundary")).toBe("generic helper"); + }); +}); + +// B6. `// obs-map-disable eror-classification -- typo` used to parse, land in the map, match no +// check and appear nowhere, so the author read the finding as acknowledged. +describe("a suppression naming a check that does not exist", () => { + it("suppresses nothing and is reported as unknown", () => { + const r = parseSuppressions( + `// obs-map-disable eror-classification -- typo + export async function loader() { return 1; }` + ); + expect(r.byId.size).toBe(0); + expect(r.unknown).toEqual(["eror-classification"]); + }); + + it("does not swallow the real suppressions beside it", () => { + const r = parseSuppressions( + `// obs-map-disable auth-boundry -- typo + // obs-map-disable auth-boundary -- public by design + export async function loader() { return 1; }` + ); + expect(r.byId.get("auth-boundary")).toBe("public by design"); + expect(r.unknown).toEqual(["auth-boundry"]); + }); + + it("reports each unknown id once however many times it appears", () => { + const r = parseSuppressions( + `// obs-map-disable request-contex -- typo + /* obs-map-disable request-contex -- typo again */ + // obs-map-disable audit-trial -- another typo + export async function loader() { return 1; }` + ); + expect(r.unknown).toEqual(["request-contex", "audit-trial"]); + }); + + it("is not reported when the directive had no reason, since it was never a suppression", () => { + const r = parseSuppressions( + `// obs-map-disable eror-classification + export async function loader() { return 1; }` + ); + expect(r.unknown).toEqual([]); + }); + + it("is not read out of a string literal any more than a real one is", () => { + const r = parseSuppressions( + `const help = "// obs-map-disable eror-classification -- typo"; + export async function loader() { return help; }` + ); + expect(r.unknown).toEqual([]); + }); + + it("keeps suppressedChecks returning only the ids that name a check", () => { + const m = suppressedChecks( + `// obs-map-disable eror-classification -- typo + export async function loader() { return 1; }` + ); + expect(m.size).toBe(0); + }); +}); diff --git a/internal-packages/observability-map/src/suppression.ts b/internal-packages/observability-map/src/suppression.ts new file mode 100644 index 00000000000..463706dfa2c --- /dev/null +++ b/internal-packages/observability-map/src/suppression.ts @@ -0,0 +1,172 @@ +import ts from "typescript"; +import { CHECKS } from "./checks/index.js"; + +const KNOWN_CHECK_IDS = new Set(CHECKS.map((c) => c.id)); + +/** + * The directive, and the reason that must follow it. The reason runs to the end of the line: `.` + * does not match a newline, so a suppression on one line cannot pick up a reason from the next. + * + * It was `obs-map-disable-next-line`, which was a lie: a check applies to a whole entry point, so + * the directive did too, and one on the last line of a file switched a check off for everything + * above it. The honest options were to scope it to a line or to rename it, and scoping is not + * available: a `CheckResult` carries no line number, and neither does an `EntryPoint`, so there is + * nothing to match a line against. Scoping it would mean inventing a proximity rule that silently + * drops legitimate suppressions. So the name now says what it does. Real line scoping needs + * positions on the findings, which is scanner work. + */ +const PATTERN = /obs-map-disable\s+([a-z-]+)\s+--\s+(.+)/; + +/** + * Every leaf token in the parsed source: keeps descending through `.getChildren()` rather than + * `ts.forEachChild`, which only returns the child nodes a statement or expression models as its + * own properties and silently skips a bare punctuation or keyword token (a closing brace, a + * semicolon). A comment can sit directly before one of those with nothing else following it, the + * last line inside a block, and `.getChildren()` still reaches it because the token itself is + * still a node with a position. + */ +function leafTokens(node: ts.Node): ts.Node[] { + const children = node.getChildren(); + return children.length === 0 ? [node] : children.flatMap(leafTokens); +} + +/** + * Node kinds whose text the parser has already claimed as content, so nothing inside their span can + * be trivia however it is spelled. `getLeadingCommentRanges` and `getTrailingCommentRanges` are raw + * lexers over source text from an offset and consult no parse tree at all, so at a leaf-token + * boundary they will happily lex the inside of one of these as a comment: a JSX text node that + * BEGINS with `//` or `/*` is the shape that reached the real tree, in + * `resources.branches.create.tsx`'s `//`. + * + * The four cases in `jsx text is content, not a comment` (`suppression.test.ts`) are the ones + * that fail without `ts.isJsxText` here; the positive control beside them, `still reads a directive + * from a comment in a JSX expression container`, is what stops the filter being widened until it + * eats real comments. `does not suppress from a directive inside a template literal` and the two + * substitution cases cover the template kinds, and `ignores the directive inside a string literal` + * covers the string kind. + * + * The mutation corpus does NOT cover any of this, and cannot: a suppression can only lower an + * entry's score, because `scoreEntry` caps it at the pre-suppression ratio. Suppression bugs are + * invisible to a harness that watches for the score rising, so they need ordinary unit tests. + */ +function isClaimedContent(node: ts.Node): boolean { + return ( + ts.isJsxText(node) || + ts.isStringLiteral(node) || + ts.isNoSubstitutionTemplateLiteral(node) || + ts.isTemplateHead(node) || + ts.isTemplateMiddle(node) || + ts.isTemplateTail(node) || + ts.isRegularExpressionLiteral(node) + ); +} + +/** + * Every comment range in the source, read off a real parsed `ts.SourceFile` rather than a + * standalone `ts.createScanner`, and then filtered against the spans above. + * + * Both halves are needed. Parsing rather than scanning is what stops a template literal WITH a + * substitution being rescanned as ordinary code after `${x}`, and what makes JSX text a node at all. + * Filtering by span is what stops the two comment-range lexers reading the start of such a node as + * a comment anyway, which they do because they never see the tree the parser built. + * + * The filter is on the range's start offset falling inside a claimed span, not on the gap between a + * token's full start and its start. A gap filter was tried and rejected: it loses a same-line + * trailing comment and a comment inside a JSX expression container, both of which are real. + * + * Both lexers are called at every token boundary, because which one returns a given comment depends + * on whether it shares a line with the token before it (trailing) or comes after a line break + * (leading), not on which directive it happens to be. + */ +function commentRanges(source: string, sf: ts.SourceFile): ts.CommentRange[] { + const claimed: ts.TextRange[] = []; + const collectClaimed = (node: ts.Node) => { + if (isClaimedContent(node)) claimed.push({ pos: node.getStart(sf), end: node.end }); + ts.forEachChild(node, collectClaimed); + }; + collectClaimed(sf); + const inClaimedSpan = (pos: number) => claimed.some((s) => pos >= s.pos && pos < s.end); + + const seen = new Set(); + const ranges: ts.CommentRange[] = []; + const add = (found: ts.CommentRange[] | undefined) => { + for (const range of found ?? []) { + if (seen.has(range.pos)) continue; + seen.add(range.pos); + if (inClaimedSpan(range.pos)) continue; + ranges.push(range); + } + }; + for (const token of leafTokens(sf)) { + add(ts.getLeadingCommentRanges(source, token.getFullStart())); + add(ts.getTrailingCommentRanges(source, token.getEnd())); + } + return ranges; +} + +/** One physical line of comment content per range, the `//`, `/*`, `*​/` and a jsdoc `*` prefix + * stripped, so a multi-line block comment still matches the directive one line at a time. */ +function commentLines(source: string, sf: ts.SourceFile): string[] { + const lines: string[] = []; + for (const range of commentRanges(source, sf)) { + const text = source.slice(range.pos, range.end); + if (range.kind === ts.SyntaxKind.SingleLineCommentTrivia) { + lines.push(text.slice(2)); + continue; + } + const body = text.slice(2, text.length - 2); // drop the leading /* and the closing */ + for (const rawLine of body.split("\n")) { + const trimmed = rawLine.trimStart(); + lines.push(trimmed.startsWith("*") ? trimmed.slice(1) : rawLine); + } + } + return lines; +} + +export type Suppressions = { + /** Check id to reason, for ids that name a check in `CHECKS`. */ + byId: Map; + /** + * Ids that parsed as a directive but name no check, in source order and deduplicated. A typo + * (`eror-classification`) used to land in the map, match nothing and appear nowhere, so the + * author read the finding as acknowledged while the tool kept reporting it. + */ + unknown: string[]; +}; + +/** + * Every suppression directive in the source, split by whether its id names a real check. A + * directive without a reason, or outside a comment, is ignored either way. + * + * `fileName` picks the parser's script kind: JSX syntax is only legal, and only correctly + * distinguished from a generic type argument list (`(x) => x`), when the file is really a + * `.tsx`. Defaults to a plain `.ts` for callers that only have source text. + */ +export function parseSuppressions(source: string, fileName = "check.ts"): Suppressions { + const scriptKind = fileName.endsWith(".tsx") ? ts.ScriptKind.TSX : ts.ScriptKind.TS; + const sf = ts.createSourceFile( + fileName, + source, + ts.ScriptTarget.Latest, + /* setParentNodes */ true, + scriptKind + ); + + const byId = new Map(); + const unknown = new Set(); + for (const line of commentLines(source, sf)) { + const match = PATTERN.exec(line); + if (!match) continue; + const [, id, reason] = match; + const trimmedReason = reason?.trim(); + if (!id || !trimmedReason || trimmedReason.length === 0) continue; + if (KNOWN_CHECK_IDS.has(id)) byId.set(id, trimmedReason); + else unknown.add(id); + } + return { byId, unknown: [...unknown] }; +} + +/** The known half of `parseSuppressions`, for callers that only apply suppressions. */ +export function suppressedChecks(source: string, fileName = "check.ts"): Map { + return parseSuppressions(source, fileName).byId; +} diff --git a/internal-packages/observability-map/src/triviality.test.ts b/internal-packages/observability-map/src/triviality.test.ts new file mode 100644 index 00000000000..5cdb6200961 --- /dev/null +++ b/internal-packages/observability-map/src/triviality.test.ts @@ -0,0 +1,167 @@ +import { isTrivial } from "./triviality.js"; +import { scanFile } from "./scan.js"; + +const ep = (fileName: string, source: string) => scanFile(fileName, source)!; + +describe("isTrivial", () => { + it("treats a redirect-only route as trivial", () => { + const e = ep( + "@.ts", + `import { redirect } from "@remix-run/server-runtime"; + export async function loader() { return redirect("/admin"); }` + ); + expect(isTrivial(e)).toBe(true); + }); + + it("does not treat a route that queries the database as trivial", () => { + const e = ep( + "api.v1.things.ts", + `import { prisma } from "~/db.server"; + export async function loader() { + const rows = await prisma.thing.findMany(); + return rows; + }` + ); + expect(isTrivial(e)).toBe(false); + }); + + // The motivating case from the design: four lines, one delegating call, nothing to instrument. + it("treats the impersonation-clearing route as trivial", () => { + const e = ep( + "@.ts", + `import { clearImpersonation } from "~/models/admin.server"; + export async function loader({ request }) { return clearImpersonation(request, "/admin"); }` + ); + expect(isTrivial(e)).toBe(true); + }); + + it("treats a static-response route as trivial", () => { + const e = ep( + "internal.webhooks.slack.interactivity.ts", + `export function action() { return new Response(null, { status: 200 }); }` + ); + expect(isTrivial(e)).toBe(true); + }); + + it("treats a guard and two fixed responses as trivial", () => { + const e = ep( + "api.v1.mock.ts", + `export async function action() { + if (process.env.NODE_ENV === "production") { + return new Response("Not found", { status: 404 }); + } + return new Response(JSON.stringify({ id: "123" }), { status: 200 }); + }` + ); + expect(isTrivial(e)).toBe(true); + }); + + it("treats a params-parse and redirect as trivial", () => { + const e = ep( + "orgs.$organizationSlug.billing.ts", + `import { redirect } from "@remix-run/server-runtime"; + import { OrganizationParamsSchema, v3BillingPath } from "~/utils/pathBuilder"; + export const loader = async ({ params }) => { + const { organizationSlug } = OrganizationParamsSchema.parse(params); + return redirect(v3BillingPath({ slug: organizationSlug })); + };` + ); + expect(isTrivial(e)).toBe(true); + }); +}); + +// statementCount deliberately does not descend into inline callbacks, so a two-statement body can +// still hold a pile of work. calleeNames does descend, which is what catches these. +describe("isTrivial: work hidden from the statement count", () => { + it("does not treat a short body holding a busy callback as trivial", () => { + const e = ep( + "api.v1.remote-build-provider-status.ts", + `export async function loader() { + const result = await fromPromise( + (async () => { + const response = await callProvider(); + const parsed = ProviderStatus.safeParse(await response.json()); + if (!parsed.success) return err("bad-payload"); + return ok(parsed.data); + })() + ); + return result.match(toJson, toError); + }` + ); + expect(isTrivial(e)).toBe(false); + }); + + it("does not treat a builder-wrapped route with a one-line handler as trivial", () => { + const e = ep( + "api.v1.deployments.current.ts", + `import { json } from "@remix-run/server-runtime"; + import { createLoaderApiRoute } from "~/services/routeBuilders/apiBuilder.server"; + export const loader = createLoaderApiRoute( + { findResource: async (_params, auth) => lookup(auth) }, + async ({ resource }) => { return json(resource); } + );` + ); + expect(isTrivial(e)).toBe(false); + }); + + it("does not treat a body that delegates to a same-file helper as trivial", () => { + const e = ep( + "api.v1.proxy.ts", + `export async function loader({ request }) { return proxy(request); } + async function proxy(request) { + const url = buildUrl(request); + const response = await send(url); + const body = await response.text(); + return new Response(body); + }` + ); + expect(isTrivial(e)).toBe(false); + }); +}); + +// Calibrated against apps/webapp/app/routes: three statements is the widest window that holds only +// redirects, fixed responses and single hand-offs. The fourth statement is where routes start +// authenticating and then calling a presenter, which is work worth reporting on. +describe("isTrivial: the statement boundary", () => { + it("treats a three-statement redirect as trivial", () => { + const e = ep( + "schedules._index/route.tsx", + `import { redirect } from "@remix-run/server-runtime"; + import { EnvironmentParamSchema, v3EnvironmentPath } from "~/utils/pathBuilder"; + export async function loader({ params }) { + const { organizationSlug, projectParam, envParam } = EnvironmentParamSchema.parse(params); + const tasksPath = v3EnvironmentPath({ slug: organizationSlug }, { slug: projectParam }, { slug: envParam }); + return redirect(\`\${tasksPath}?types=SCHEDULED\`); + }` + ); + expect(isTrivial(e)).toBe(true); + }); + + it("does not treat an authenticated hand-off to a presenter as trivial", () => { + const e = ep( + "tasks.stream/route.tsx", + `import { TasksStreamPresenter } from "~/presenters/v3/TasksStreamPresenter.server"; + import { requireUserId } from "~/services/session.server"; + import { EnvironmentParamSchema } from "~/utils/pathBuilder"; + export async function loader({ request, params }) { + const userId = await requireUserId(request); + const { organizationSlug, projectParam, envParam } = EnvironmentParamSchema.parse(params); + const presenter = new TasksStreamPresenter(); + return presenter.call({ request, projectParam, envParam, organizationSlug, userId }); + }` + ); + expect(isTrivial(e)).toBe(false); + }); +}); + +describe("isTrivial: an error path is something to instrument", () => { + it("does not treat a short body with a try/catch as trivial", () => { + const e = ep( + "api.v1.ping.ts", + `export async function loader() { + try { return await ping(); } catch { return null; } + }` + ); + expect(isTrivial(e)).toBe(false); + }); +}); diff --git a/internal-packages/observability-map/src/triviality.ts b/internal-packages/observability-map/src/triviality.ts new file mode 100644 index 00000000000..15515d1ec36 --- /dev/null +++ b/internal-packages/observability-map/src/triviality.ts @@ -0,0 +1,122 @@ +import type { RouteExport } from "./routeExports.js"; +import type { EntryPoint } from "./types.js"; + +/** + * Substrings that say the route touches a service, a datastore or the network. Always matched + * against the callee names, and additionally against `TrivialityView.hintText`, which is the whole + * file for the entry-point-wide view and empty for a per-export one. + */ +const SIDE_EFFECT_HINTS = ["prisma", "logger", "fetch", "$transaction", "redis", "engine"]; + +/** + * Calls a genuinely trivial body makes: parse the params, build a path, hand back a response. Every + * shape found in the real tree stays at or below three, so anything busier is doing work. Allowing + * a fourth admits `_app.orgs.$organizationSlug.settings/route.tsx`, which awaits two service calls. + */ +const MAX_CALLS = 3; + +/** + * Parse the params, build a path, redirect. Or an environment guard and two returns. Both real + * shapes need three. Allowing a fourth admits the routes that authenticate and then hand off to a + * presenter (`...tasks.stream/route.tsx`), which have real work behind them and belong in the + * report; allowing a fifth admits an admin route that calls a service and hand-rolls its own error + * responses. + */ +const MAX_STATEMENTS = 3; + +/** + * What the rule reads, so the entry-point-wide answer and a single export's answer are the same + * rule over different bodies rather than two rules that can drift. + */ +type TrivialityView = { + statementCount: number; + calleeNames: string[]; + hasTryCatch: boolean; + /** Every builder call in scope of this view. A view with one is never trivial. */ + initializerCallees: (string | null)[]; + /** + * Text to match the side-effect hints against besides the callee names. + * + * The whole file for the entry-point-wide view, so an import of `prisma` disqualifies it even + * when the query sits somewhere the scanner does not walk. For a per-export view it is that + * export's own callee PATHS instead, and the difference is not a convenience: + * + * - The file's text is a fact about the file, so reading it into one export's verdict is the + * per-file-for-per-export substitution this rule exists to damp. It is also defeatable. + * `log-caller-scope-userid` in the mutation corpus prepends `logger.error(...)` to every body; + * with this term file-wide that put the word `logger` in `auth.github.ts` and turned its + * untouched one-line redirect loader from excused into accused, on a rewrite that changed + * nothing the loader does. + * - Emptying it instead is not the answer either, and that was measured: `calleeNames` keeps only + * a call's last segment, so `prisma.orgMember.findMany` reads as `findMany` and a + * three-statement body that queries the datastore matches no hint at all. Five existing + * `auth-boundary` fixtures went from `fail` to `not-applicable`, which is the check being + * switched off rather than fixed. + * + * The callee paths are body-scoped like the first option wants and name the receiver like the + * second needs. Comments and imports are not in them, which is deliberate: everything in this + * view is something the export actually does. + */ + hintText: string; +}; + +/** + * Nothing to instrument: a body of a statement or two that only redirects, returns a fixed + * response, or hands off in a single call. Checks report not-applicable for these rather than + * failing, which is what stops `@.ts` being a finding. + * + * Deliberately reluctant. A route wrongly called trivial is exempted and never shows up in the + * report again, so every signal that the body might be doing real work rules triviality out: + * + * - `statementCount` counts a nested function's statements but `calleeNames` descends further, into + * the callee of every call at any depth, so the call count still catches bodies the statement + * count reads as short. + * - An initializer callee means the route is wrapped in a builder, and the config passed to that + * builder (`findResource`, `authorization`) is work the scanner never walks. The visible body is + * not the whole route, so we cannot claim it is trivial. + * - A try/catch is exactly what the error-classification check reads, so a body with one has an + * error path worth reporting on however short it is. + */ +function isTrivialView(view: TrivialityView): boolean { + if (view.statementCount > MAX_STATEMENTS) return false; + if (view.calleeNames.length > MAX_CALLS) return false; + if (view.hasTryCatch) return false; + if (view.initializerCallees.some((c) => c !== null)) return false; + + const callees = view.calleeNames.join(" ").toLowerCase(); + const hints = view.hintText.toLowerCase(); + return !SIDE_EFFECT_HINTS.some((h) => callees.includes(h) || hints.includes(h)); +} + +/** The rule over everything the entry point does, both exports and their same-file helpers. */ +export function isTrivial(ep: EntryPoint): boolean { + return isTrivialView({ + statementCount: ep.statementCount, + calleeNames: ep.calleeNames, + hasTryCatch: ep.hasTryCatch, + initializerCallees: [ep.loaderInitializerCallee, ep.actionInitializerCallee], + hintText: ep.source, + }); +} + +/** + * The same rule over ONE export's handlers. + * + * Needed because a per-export verdict judged against an entry-point-wide triviality rule accuses + * the wrong half of a file. `auth.github.ts` and `auth.google.ts` are + * `export let loader = () => redirect("/login")` beside an action that calls + * `authenticator.authenticate`: per export the loader is unguarded, and the entry-point-wide rule + * calls the file non-trivial because the ACTION is not, so `auth-boundary` accused a one-line + * redirect stub of missing an auth guard. `checks/index.test.ts` pins both directions of that + * ("reports not-applicable for a redirect-stub loader beside a guarded action" and "fails an export + * whose own body does real work unguarded"). + */ +export function isTrivialExport(e: RouteExport): boolean { + return isTrivialView({ + statementCount: e.statementCount, + calleeNames: e.calleeNames, + hasTryCatch: e.hasTryCatch, + initializerCallees: [e.initializerCallee], + hintText: e.calleeTexts.join(" "), + }); +} diff --git a/internal-packages/observability-map/src/types.ts b/internal-packages/observability-map/src/types.ts new file mode 100644 index 00000000000..f11fb4a4a1f --- /dev/null +++ b/internal-packages/observability-map/src/types.ts @@ -0,0 +1,222 @@ +export type CheckStatus = "pass" | "fail" | "not-applicable"; + +export type CheckResult = { + id: string; + status: CheckStatus; + detail?: string; +}; + +/** + * One catch clause in a loader/action body, or in a same-file helper the body calls. Per clause + * rather than per entry point, so a narrow parse guard sitting beside a broad handler catch stays + * legible instead of collapsing into one boolean. + */ +export type CatchEvidence = { + /** + * Throwing is the clause's only way out. Two conditions: a `throw` is reached on the clause's + * guaranteed path (the positions certain to execute whenever the clause runs: its own + * statements, a bare nested block's, a `do` body's, a catchless `try`'s tryBlock, a + * single-default `switch`'s clause, an `if (true)` then-arm, and both arms of an `if`/`else` + * together, cut at the first statement that definitely exits, see `catchClauseEvidence` and + * `definitelyExits`), and the clause contains no live `return` anywhere. A throw guarded by a + * condition the walk cannot fold, a loop, a nested caught `try`, a finally block or a callback + * does not count, and neither does one written after something that has already returned. + */ + rethrows: boolean; + /** A `throw` is reached on that same guaranteed path, whether or not it is the only way out. + * `rethrows` is this AND no reachable `return`. Kept separately so a verdict can say what is true + * of a clause that both throws and returns. */ + throws: boolean; + /** + * The clause picks what to do from what it caught, on that same guaranteed path: an `if` or + * `switch` whose condition references the caught error binding AND at least one of whose arms + * returns or throws, or a conditional that is the whole value of a `return`/`throw`. + * `if (retries > 0)` does not count, `if (e instanceof Error) { }` does not count, and a + * bindingless `catch { ... }` cannot count at all. An `instanceof` used only to word a message, + * `json({ error: e instanceof Error ? e.message : String(e) })`, does not count either: every + * error still leaves by the same path. + */ + branches: boolean; + /** + * The guarded region parses something: `JSON.parse`, `request.json()`, a zod `parse`/`safeParse`, + * a `decode`, or a `new URL`/`URLSearchParams`/`RegExp`. Those three constructors are read here + * because a `new` expression is not a call, so the call-callee scan that feeds this check never + * sees them; other constructors do not count, or every `new SomePresenter()` in a try would excuse + * its catch. + */ + guardsParse: boolean; + /** + * The guarded region does something that could raise at all: a call, a construction, an `await`, + * a member access, a `throw`, an iteration, an `instanceof`. False means `try { 0; }` and little + * else: any call counts, including one that cannot throw, so `try { String(0); }` reads as true. + * See `canRaise` in `scan.ts` for both directions of that, including the destructuring + * declaration it misses. + */ + guardCanRaise: boolean; + /** + * The containment twin of `guardCanRaise`: false only when the guarded region provably cannot + * raise, i.e. every statement is an expression over a bare literal, which is `try { 0; }` and + * nothing else. Everything `canRaise`'s whitelist misses (a destructuring declaration, a + * temporal-dead-zone read) stays true here, so `guardCanRaise` implies `guardMayRaise`. What the + * refused-callback arm of `error-classification` reads: a route whose own classifying catch + * `canRaise` cannot see must never be told nothing it owns decides + * (`does not accuse a route that owns a catch of owning none`), while the provably dead + * `try { 0; }` clause still blocks nothing (`still fails a per-item swallow beside a deciding + * catch over a dead guard`). + */ + guardMayRaise: boolean; + /** + * Everything the guarded region waits for is one of those parses. What separates + * `try { const body = await request.json(); } catch { 400 }` from + * `try { const body = await request.json(); return await handleEverything(body); } catch { 500 }`, + * which the statement count reads as the same size. Synchronous work is not counted here: the + * calls that prepare a parse's input are synchronous, and the swallows this has to catch wait on + * a service. + */ + awaitsOnlyParse: boolean; + /** Statements in the guarded try block, counted as `statementCount` counts them. */ + tryStatementCount: number; +}; + +/** A logging call made from a loader/action body, or from a same-file helper the body calls. */ +export type LogCall = { + /** Full callee path, e.g. `logger.error`. */ + callee: string; + /** Property names on the first object-literal argument, e.g. `["environmentId", "error"]`. */ + fields: string[]; + /** Whether the call sits inside a catch clause, i.e. on the failure path. */ + inCatch: boolean; +}; + +export type EntryPoint = { + fileName: string; + source: string; + hasLoader: boolean; + hasAction: boolean; + /** Callee name when `loader`/`action` is assigned from a call, e.g. a route builder. */ + loaderInitializerCallee: string | null; + actionInitializerCallee: string | null; + /** + * Top-level keys of the object literals passed to that call, e.g. `["params", "authorization"]`. + * Empty when the export has no initializer call, and empty when the call takes no object + * literal, so an empty array is "nothing declared here" rather than "no builder". + */ + loaderBuilderOptions: string[]; + actionBuilderOptions: string[]; + /** + * The route declares a loader or an action, and the scan resolved neither a handler function nor + * a builder call for any of them: `export { action } from "./handler.server"`, + * `export const action = handleWebhook`. Nothing about the request handling is in this file, so + * every check reports not-applicable and `buildReport` counts the entry point separately from + * the ones nothing happened to apply to. Those are different facts: a redirect stub genuinely has + * nothing to instrument, a delegating route has work the scanner cannot see. + * + * A route that delegates one export and writes the other in the file is NOT delegating by this + * definition, and is judged on the half that is visible. + */ + delegating: boolean; + /** + * Whether THIS export's handler assigns the caller's own id to an object-literal property, the + * `where: { members: { some: { userId: authentication.userId } } }` and + * `presenter.call({ userId: user.id })` shapes. Read by `auth-scope` as evidence that the handler + * narrowed its work to whoever is asking. See `CALLER_ID_PATH` and `scopesByCallerIn` in + * `scan.ts`. + * + * Split per export because the exposure is per export: a loader that narrows itself to the caller + * says nothing about the action beside it. Property assignments only, so a value read into a + * local first (`const userId = user.id; ... { userId }`) is not seen. + */ + loaderScopesByCaller: boolean; + actionScopesByCaller: boolean; + /** + * Callees whose answer THIS export's handlers demonstrably looked at: the call's result was bound + * to a local and some `if`, `while`, `switch` or conditional in the same handlers reads that + * local. `const user = await getUser(request); if (!user) return redirect("/login");` puts + * `getUser` here; a call whose result is dropped, or bound and never tested, does not appear. + * + * Read by `auth-boundary` for the guards that answer with null instead of throwing, where being + * called is not evidence that the route acted on the answer. + * + * Split per export for the same reason `loaderScopesByCaller` is, and there is no entry-point-wide + * version on purpose: a loader that reads what `getUser` returned says nothing about the action + * beside it, so the union is not a fact any check should be able to reach for. + * + * Deliberately coarse. It does not check that the test guards anything, that the local is the one + * tested rather than a same-named one in another scope, or that the branch exits: a route + * that writes `if (!user) { logger.warn("anonymous"); }` and carries on is credited. It separates + * "looked at the answer" from "ignored it", which is the distinction the check needs, and not + * "acted correctly on the answer", which it cannot see. + */ + loaderCheckedCallees: string[]; + actionCheckedCallees: string[]; + /** Named and default imports, file-wide. */ + importedNames: string[]; + /** + * Names of functions called inside the loader/action bodies, or in a same-file helper they call. + * + * Entry-point-wide, and read only by the questions that are themselves entry-point-wide: + * `sensitivity.ts` asks what the file touches, `triviality.ts` counts how much the file does, + * `audit-trail` asks whether the file records anything. A question about ONE export's exposure + * must read `loaderCalleeNames`/`actionCalleeNames` instead. `auth-boundary` read this and + * credited a file whose loader called a guard for an action that called none. + */ + calleeNames: string[]; + /** + * The same callee names attributed to the export whose handlers made the call. A handler serving + * both exports (`const { loader, action } = createActionApiRoute({ handler })`) contributes to + * both, and a same-file helper contributes to whichever exports reach it. + * + * Every name here appears in `calleeNames` and every name in `calleeNames` appears in at least one + * of these, because all three are filled from one push in `scanFile`. `scan.test.ts` pins that + * ("every callee name is attributed to an export that exists") and `integration.test.ts` pins it + * again across the real route tree, so the split cannot drift away from the union it came from. + */ + loaderCalleeNames: string[]; + actionCalleeNames: string[]; + /** + * The same calls as `loaderCalleeNames`/`actionCalleeNames`, each as its whole dotted path + * (`prisma.organization.findFirst` rather than `findFirst`). Read by the per-export triviality + * rule, which has to know that a three-statement body reaches the datastore; the bare name that + * `auth-boundary` matches guards against throws that receiver away. + */ + loaderCalleeTexts: string[]; + actionCalleeTexts: string[]; + /** + * Whether a `try` appears in the loader/action bodies, or in a same-file helper they call. Note + * that this says a `try`, not a catch: a `try`/`finally` sets it while `catches` stays empty and + * every catch-shaped field stays false. Read `catches.length` to ask whether anything is caught. + */ + hasTryCatch: boolean; + /** The same fact for one export's handlers alone. Read by the per-export triviality rule. */ + loaderHasTryCatch: boolean; + actionHasTryCatch: boolean; + /** One entry per catch clause in those bodies, in source order. */ + catches: CatchEvidence[]; + /** + * Catch clauses the scan found but refused to attribute to the route, because they sit inside a + * per-item iteration callback. Still refused for attribution: they never join `catches`, never + * speak for the route's `tryStatementCount`, and never reach a pass. Kept WITH their evidence, + * built by the same `catchClauseEvidence` machinery as an own catch, so `error-classification` + * can judge what a refused catch does rather than where it sits: a refused swallow fails the + * route (`fails a per-item swallow even when the route owns an inert rethrow catch`), a refused + * catch that decides or rethrows caps at not-applicable (`sits out a route whose only catch is a + * deciding per-item boundary`). The count the old field carried is `.length`. + */ + callbackCatches: CatchEvidence[]; + /** Calls to a `logger.*` or `log.*` callee in those bodies, in source order. */ + logCalls: LogCall[]; + /** + * Statement count across loader/action bodies, used by the triviality rule. Includes the + * statements of functions written inline in those bodies, so wrapping a body in a callback does + * not shrink it. A body that delegates to a same-file helper counts that helper's statements too, + * one hop only: work in a helper's own helpers, or in an imported module, is not counted. + */ + statementCount: number; + /** + * The same count for one export's handlers alone, counted by the same walk. A handler serving both + * exports is counted once in `statementCount` and once in each of these, so the two do not sum to + * the entry point's total and must not be used as though they did. + */ + loaderStatementCount: number; + actionStatementCount: number; +}; diff --git a/internal-packages/observability-map/src/webappSymbols.test.ts b/internal-packages/observability-map/src/webappSymbols.test.ts new file mode 100644 index 00000000000..3a709d2d913 --- /dev/null +++ b/internal-packages/observability-map/src/webappSymbols.test.ts @@ -0,0 +1,210 @@ +import ts from "typescript"; +import { readdirSync, readFileSync } from "node:fs"; +import { join, resolve } from "node:path"; +import { AUDIT_SYMBOLS } from "./checks/auditTrail.js"; +import { GUARDS, SOFT_GUARDS } from "./checks/authBoundary.js"; +import { isScannableFile } from "./scan.js"; +import { + ANTICIPATED_SEGMENTS, + normalizeSegment, + SENSITIVE_SEGMENTS, + SENSITIVE_SYMBOLS, +} from "./sensitivity.js"; + +/** + * Every name and every path segment the tool matches on must exist in the codebase it is pointed + * at. + * + * This is the test the last round did not have, and the cost of not having it was measured: half of + * `SENSITIVE_SYMBOLS` named nothing. `Set.has` is exact, so `setImpersonation`, `createJWT`, + * `signJWT` and `updateEnvVars` matched no route in the tree, while `startImpersonation`, the real + * escalation, was absent from the list. Nothing failed, nothing was reported, and the symbol half + * of the classifier was quietly doing almost nothing. `auth-boundary`'s guard list has the same + * failure mode with a worse consequence, since a guard name that resolves to nothing turns into a + * route that can never pass rather than a route that can never fail. + * + * What is checked: + * + * - every guard name and every sensitive symbol is DECLARED somewhere under one of `ROOTS`. A + * declaration is a function, class, interface, type, enum or variable name, or a member name on a + * class, interface or object literal. Members count because several guards are reached through an + * object: `rbac.authenticateSession`, `authenticator.isAuthenticated`, and `calleeName` in + * `scan.ts` records the property for a member call, so that is the form the check sees. + * - every sensitive path segment appears as a segment of a real route file name. + * + * What is NOT checked, and each is a place a wrong entry can still hide: + * + * - that the declaration found is the one meant. `authenticateAdmin` is a local helper inside + * `admin.api.v1.platform-notifications.ts`; a second route declaring its own no-op function of + * that name would be credited by `auth-boundary`. Names cannot carry that guarantee, and the + * alternative, a module-resolving import graph, is a different kind of analysis from anything + * else in this package. + * - that a guard actually guards. `resolveAuthenticatedEnv` declares fine and authenticates + * nothing, which is why it is not on the list; keeping it off is a hand-read judgement this test + * cannot make. + * - anything outside `ROOTS`. A guard declared only by a dependency is listed in + * `EXTERNAL_GUARDS` and not resolved at all; see the comment there for why that is a list + * rather than a path into `node_modules`. + */ + +const REPO = resolve(__dirname, "../../.."); + +/** + * Where a guard or a sensitive symbol may be declared. The webapp first, then the two packages it + * authenticates through: `packages/plugins` declares the RBAC controller interface the dashboard + * and PAT builders call, and `internal-packages/rbac` declares its fallback and the user-actor + * token verifier that three routes import directly. + */ +const ROOTS = [ + resolve(REPO, "apps/webapp/app"), + resolve(REPO, "packages/plugins/src"), + resolve(REPO, "internal-packages/rbac/src"), +]; + +/** + * Guard names declared by a dependency rather than by us, and therefore deliberately unchecked. + * + * Both are methods on remix-auth's `Authenticator`, reached as `authenticator.authenticate(...)` + * and `authenticator.isAuthenticated(...)`; the whole login surface is built on them. An earlier + * version of this test resolved them by reading + * `apps/webapp/node_modules/remix-auth/build/authenticator.d.ts` directly. That is a path into an + * installed tree: a hoisting change, a version bump that moves `build/`, or a fresh clone with a + * different install layout turns a real assertion into a confusing environmental failure, and a + * test that fails for environmental reasons teaches people to ignore it. + * + * So they are listed instead, which is a smaller claim honestly made. The test still fails if a + * guard name is neither declared in first-party source nor on this list, so a name that resolves + * nowhere cannot be added silently; what it no longer does is prove these two exist. + */ +const EXTERNAL_GUARDS = new Set(["authenticate", "isAuthenticated"]); + +/** Two is the number of remix-auth methods on the guard list. A third entry means someone widened + * the unchecked set, which is the thing this bound exists to make visible in review. */ +const MAX_EXTERNAL_GUARDS = 2; + +const ROUTES = resolve(REPO, "apps/webapp/app/routes"); + +function walkFiles(dir: string, out: string[] = []): string[] { + for (const entry of readdirSync(dir, { withFileTypes: true })) { + const path = join(dir, entry.name); + if (entry.isDirectory()) walkFiles(path, out); + else if (isScannableFile(entry.name)) out.push(path); + } + return out; +} + +function declaredNames(): Set { + const names = new Set(); + const addBinding = (name: ts.BindingName) => { + if (ts.isIdentifier(name)) { + names.add(name.text); + return; + } + for (const element of name.elements) { + if (!ts.isOmittedExpression(element)) addBinding(element.name); + } + }; + + const files = ROOTS.flatMap((root) => walkFiles(root)); + for (const file of files) { + const sf = ts.createSourceFile(file, readFileSync(file, "utf8"), ts.ScriptTarget.Latest, false); + const visit = (node: ts.Node) => { + if (ts.isVariableDeclaration(node) || ts.isParameter(node)) addBinding(node.name); + else if ( + (ts.isFunctionDeclaration(node) || + ts.isClassDeclaration(node) || + ts.isInterfaceDeclaration(node) || + ts.isTypeAliasDeclaration(node) || + ts.isEnumDeclaration(node)) && + node.name + ) { + names.add(node.name.text); + } else if ( + (ts.isMethodDeclaration(node) || + ts.isMethodSignature(node) || + ts.isPropertyDeclaration(node) || + ts.isPropertySignature(node) || + ts.isPropertyAssignment(node) || + ts.isShorthandPropertyAssignment(node)) && + node.name && + ts.isIdentifier(node.name) + ) { + names.add(node.name.text); + } + ts.forEachChild(node, visit); + }; + visit(sf); + } + return names; +} + +/** Every dot-separated piece of every route name, flat file or directory, e.g. `billing-limits`. */ +function routeSegments(): Set { + const segments = new Set(); + for (const entry of readdirSync(ROUTES, { withFileTypes: true })) { + for (const part of entry.name.replace(/\.tsx?$/, "").split(".")) { + // `sensitivity.ts`'s own normalizer. This validates the vocabulary that file matches on, so + // a segment has to be trimmed here exactly as it is trimmed there; the local `/_+$/` was + // also the regex `normalizeSegment`'s own comment says not to use. + segments.add(normalizeSegment(part)); + } + } + return segments; +} + +describe("the names the tool matches on exist in the webapp", () => { + const declared = declaredNames(); + const segments = routeSegments(); + + it("found a codebase to check against", () => { + expect(declared.size).toBeGreaterThan(5000); + expect(segments.size).toBeGreaterThan(100); + }); + + it("every sensitive symbol is declared somewhere", () => { + expect(SENSITIVE_SYMBOLS.filter((s) => !declared.has(s))).toEqual([]); + }); + + // The list this test did not cover, and it had rotted completely: all three of `auditLog`, + // `recordAudit` and `writeAuditEvent` were exported nowhere, so `audit-trail`'s pass branch could + // not fire and the report said "No audit helper exists in the webapp" while + // `models/admin.server.ts` was writing `impersonationAuditLog` rows on two paths. + it("every audit symbol is declared somewhere", () => { + expect(AUDIT_SYMBOLS.filter((s) => !declared.has(s))).toEqual([]); + }); + + it("every auth guard is declared somewhere, or is a listed dependency method", () => { + const names = [...GUARDS, ...SOFT_GUARDS]; + expect(names.filter((g) => !declared.has(g) && !EXTERNAL_GUARDS.has(g))).toEqual([]); + }); + + // The escape hatch is only worth having while it stays small. + it("keeps the unchecked guard names to the two remix-auth methods", () => { + expect(EXTERNAL_GUARDS.size).toBeLessThanOrEqual(MAX_EXTERNAL_GUARDS); + expect([...EXTERNAL_GUARDS].filter((g) => !GUARDS.has(g))).toEqual([]); + }); + + it("every sensitive path segment names a real route segment", () => { + const live = SENSITIVE_SEGMENTS.filter((s) => !ANTICIPATED_SEGMENTS.includes(s)); + expect(live.filter((s) => !segments.has(s))).toEqual([]); + }); + + // The escape hatch is only worth having while it is small and honest about itself. + it("every anticipated segment really does name nothing yet", () => { + expect(ANTICIPATED_SEGMENTS.filter((s) => segments.has(s))).toEqual([]); + }); + + // The checker has to be able to fail. These run the same predicates over the names the last round + // shipped, which is what the test exists to have caught. + it("would reject the symbols that named nothing", () => { + for (const dead of ["setImpersonation", "createJWT", "signJWT", "updateEnvVars"]) { + expect(declared.has(dead)).toBe(false); + } + }); + + it("would reject a guard name and a path segment that name nothing", () => { + expect(declared.has("requireNothingAtAll")).toBe(false); + expect(EXTERNAL_GUARDS.has("requireNothingAtAll")).toBe(false); + expect(segments.has("no-such-route-segment")).toBe(false); + }); +}); diff --git a/internal-packages/observability-map/tsconfig.build.json b/internal-packages/observability-map/tsconfig.build.json new file mode 100644 index 00000000000..6e0d21f36bc --- /dev/null +++ b/internal-packages/observability-map/tsconfig.build.json @@ -0,0 +1,15 @@ +{ + "extends": "./tsconfig.json", + "include": ["src/**/*.ts"], + "exclude": ["src/**/*.test.ts", "src/mutations.ts"], + "compilerOptions": { + "noEmit": false, + // A build that does not compile must not leave a dist behind: three type errors shipped in the + // last wave while `build` still emitted, so the failure was only visible in the exit code. + "noEmitOnError": true, + "declaration": true, + "outDir": "dist", + "rootDir": ".", + "types": ["node"] + } +} diff --git a/internal-packages/observability-map/tsconfig.json b/internal-packages/observability-map/tsconfig.json new file mode 100644 index 00000000000..ea5663ae806 --- /dev/null +++ b/internal-packages/observability-map/tsconfig.json @@ -0,0 +1,21 @@ +{ + "compilerOptions": { + "target": "ES2019", + // ES2020 rather than the ES2019 most sibling packages use, because the tests call + // String.prototype.matchAll. It already resolved: @types/node carries a + // `/// `, so the program had es2020 whatever this line said. Stating + // it here stops the requirement resting on a transitive reference from a types package. + "lib": ["ES2020"], + "module": "ESNext", + "moduleResolution": "Bundler", + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true, + "isolatedModules": true, + "skipLibCheck": true, + "noEmit": true, + "strict": true, + "types": ["vitest/globals", "node"], + "customConditions": ["@triggerdotdev/source"] + }, + "exclude": ["node_modules", "dist"] +} diff --git a/internal-packages/observability-map/turbo.json b/internal-packages/observability-map/turbo.json new file mode 100644 index 00000000000..42fa065651d --- /dev/null +++ b/internal-packages/observability-map/turbo.json @@ -0,0 +1,31 @@ +{ + "$schema": "https://turborepo.org/schema.json", + "extends": ["//"], + "pipeline": { + // Uncacheable on purpose. This suite's real inputs live outside the package: it scans + // `apps/webapp/app`, `packages/plugins/src`, `internal-packages/rbac/src` and four files under + // `.github/workflows`. The root `test` task keys the cache on this package's own files, so a + // cached pass replayed after a route change broke the scan, which is a guard that stops + // guarding while still reading green. + // + // `inputs` was tried and rejected rather than assumed unworkable. Turbo 1.x does accept `..` + // in an input glob, and `../../apps/webapp/app/**` did bust the cache on a route change. What + // it also does is replace the default file set rather than add to it, so the same config + // silently dropped this package's own `vitest.config.ts` from the hash: editing it replayed a + // cached pass. The `$TURBO_DEFAULT$` token that adds rather than replaces is turbo 2.x only + // and matches nothing on the 1.10.3 here. Trading a stale-on-routes hole for a + // stale-on-own-config hole is not a fix, and an inputs list mirroring what the tests read is + // one more thing that drifts out of sync without saying so. + // + // Cost is about 23s per run, and no CI job pays it: the dedicated workflow calls vitest + // without turbo, and `unit-tests-internal.yml` runs cold. + // + // `it("keeps its test task out of the turbo cache")` in `src/integration.test.ts` fails if + // this is removed. + "test": { + "dependsOn": ["^build"], + "outputs": [], + "cache": false + } + } +} diff --git a/internal-packages/observability-map/vitest.config.ts b/internal-packages/observability-map/vitest.config.ts new file mode 100644 index 00000000000..c1680ce67f9 --- /dev/null +++ b/internal-packages/observability-map/vitest.config.ts @@ -0,0 +1,5 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { include: ["**/*.test.ts"], globals: true, isolate: true, testTimeout: 10_000 }, +}); diff --git a/package.json b/package.json index 2708b06be73..9fad94dd9d4 100644 --- a/package.json +++ b/package.json @@ -40,6 +40,7 @@ "clean": "turbo run clean", "clean:node_modules": "find . -name 'node_modules' -type d -prune -exec rm -rf '{}' +", "typecheck": "turbo run typecheck", + "map": "pnpm --filter @internal/observability-map run map", "test:e2e": "playwright test", "test:e2e:ui": "playwright test --ui", "test:e2e:dev": "turbo run test:e2e:dev", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f37e3250982..64f6fe98b18 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1139,6 +1139,25 @@ importers: specifier: 6.0.1 version: 6.0.1 + internal-packages/observability-map: + dependencies: + typescript: + specifier: 6.0.3 + version: 6.0.3 + devDependencies: + '@types/node': + specifier: 24.13.3 + version: 24.13.3 + rimraf: + specifier: 6.0.1 + version: 6.0.1 + tsx: + specifier: ^4.19.2 + version: 4.22.4 + vitest: + specifier: 4.1.7 + version: 4.1.7(@opentelemetry/api@1.9.1)(@types/node@24.13.3)(@vitest/coverage-v8@4.1.7)(vite@6.4.2(@types/node@24.13.3)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.22.4)(yaml@2.9.0)) + internal-packages/otlp-importer: dependencies: long: