feat(observability-map): static observability scorer for webapp route entry points

.github/workflows/observability-map.yml

@@ -0,0 +1,310 @@
+name: 🗺️ Observability Map
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened]
+    paths:
+      - "apps/webapp/app/routes/**"
+      - "internal-packages/observability-map/**"
+  # The corpus job below is gated to this package's own paths, so a scheduled run is what still
+  # scans the tree as it drifts. Nightly rather than per route pull request: a new route can make a
+  # known laundering shape start paying, but that is a property of the tree accumulating, not of any
+  # one pull request, and it does not need catching within five minutes of the merge.
+  schedule:
+    - cron: "0 3 * * *"
+  workflow_dispatch:
+
+concurrency:
+  group: observability-map-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  # The workflow's paths filter is the union of what the two jobs below want, because GitHub
+  # evaluates it once per workflow. This narrows it again for the corpus job alone.
+  changes:
+    name: 🔍 Which paths moved
+    # Only the pull request path reads this job's output. On a schedule the action has no base to
+    # diff, warns that `before` is missing and reports the files in the last commit on main, which
+    # nothing then consults. Skipping it there keeps the nightly off a job it does not need.
+    if: github.event_name == 'pull_request'
+    runs-on: warp-ubuntu-latest-x64-2x
+    outputs:
+      package: ${{ steps.filter.outputs.package }}
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          fetch-depth: 1
+          persist-credentials: false
+      - uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1
+        id: filter
+        with:
+          filters: |
+            package:
+              - 'internal-packages/observability-map/**'
+              - '.github/workflows/observability-map.yml'
+
+  # The tree-scale mutation corpus: every known laundering shape applied to the whole route tree,
+  # asserting the score does not rise. Roughly four and a half minutes for 45 entries, which is why
+  # it is gated out of the package's default `pnpm test` and run here instead. Unlike the report
+  # job below it has no token to lose, so it runs for fork PRs too, and unlike the report job it is
+  # allowed to fail the build.
+  #
+  # Gated to this package's own paths rather than running on every route pull request. What the
+  # corpus measures is the TOOL's resistance to laundering, and only an edit to the tool can weaken
+  # that, so a routes-only change was paying four and a half minutes of a 4x runner for a result
+  # that could not differ from the last one. It was also the worst kind of job to spend that on: a
+  # red x that fires on a large share of webapp pull requests, is allowed to fail, and gates
+  # nothing, which is the shape people learn to scroll past.
+  #
+  # What this gives up is real and small. A route landing a shape no corpus entry has seen can make
+  # a known laundering mutation start paying, and that is now caught by the nightly rather than by
+  # the pull request that caused it. Tree drift accrues over months, so a day is the right
+  # granularity for it; the tool's own regressions, which are the ones a single commit can cause,
+  # still gate per pull request.
+  mutation-corpus:
+    name: 🧬 Mutation corpus
+    needs: changes
+    # `!cancelled()` is here for the nightly, not for tidiness. `needs` carries an implicit
+    # success() on the job it names, and that implicit test outranks the `||` below: with a plain
+    # condition, a `changes` job that failed or was skipped skips this one, so the nightly would
+    # stop scanning for tree drift and report nothing about having stopped. A status-check function
+    # in the `if` is what drops the implicit success(), so the event test below decides alone.
+    # `!cancelled()` rather than `always()` because `cancel-in-progress` above is a real path and a
+    # superseded run should not finish this job.
+    #
+    # Pull request behaviour is deliberately unchanged: on a PR a failed `changes` leaves
+    # `needs.changes.outputs.package` empty, so the corpus still skips. The nightly is the backstop
+    # for that, which is the same trade the paths gate already makes for routes-only pull requests.
+    if: >-
+      !cancelled() &&
+      (github.event_name != 'pull_request' || needs.changes.outputs.package == 'true')
+    runs-on: warp-ubuntu-latest-x64-4x
+    steps:
+      - name: ⬇️ Checkout repo
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          fetch-depth: 1
+          persist-credentials: false
+
+      - name: ⎔ Setup pnpm
+        uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0
+        with:
+          version: 10.33.2
+
+      - name: ⎔ Setup node
+        uses: WarpBuilds/setup-node@bc639b444d583175926b588962199c247d23e8d3 # v6
+        with:
+          node-version: 24.18.0
+          cache: "pnpm"
+
+      - name: 📥 Download deps
+        run: pnpm install --frozen-lockfile
+
+      - name: 🧬 Run the corpus
+        env:
+          OBS_MAP_MUTATION_CORPUS: "1"
+        run: |
+          pnpm --filter @internal/observability-map exec vitest run \
+            src/mutationCorpus.test.ts --disable-console-intercept
+
+  # The package's own tests are NOT run here. They gate through pr_checks.yml, which is the only
+  # workflow the all-checks aggregate can see, so a job in this file would report a result nobody
+  # is required to wait for. See unit-tests-observability-map.yml and the obsmap filter.
+  report:
+    runs-on: warp-ubuntu-latest-x64-4x
+    # Only this job comments, so only this job gets the write.
+    permissions:
+      contents: read
+      pull-requests: write
+    # Fork PRs get a read-only token, so the comment cannot post. Skipping the job beats a red x.
+    # The event test is what keeps this job off the nightly, which has no pull request to comment on
+    # and only exists for the corpus job above.
+    if: >-
+      github.event_name == 'pull_request' &&
+      github.event.pull_request.head.repo.full_name == github.repository
+    steps:
+      - name: ⬇️ Checkout repo
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          fetch-depth: 0
+          persist-credentials: false
+
+      - name: ⎔ Setup pnpm
+        uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0
+        with:
+          version: 10.33.2
+
+      - name: ⎔ Setup node
+        uses: WarpBuilds/setup-node@bc639b444d583175926b588962199c247d23e8d3 # v6
+        with:
+          node-version: 24.18.0
+          cache: "pnpm"
+
+      - name: 📥 Download deps
+        run: pnpm install --frozen-lockfile
+
+      # Guarded rather than allowed to fail: this job must never block a pull request. The failure
+      # is not swallowed either, the render step below turns a missing head report into a comment
+      # saying so, because a swallowed failure with no comment is the outcome nobody wants.
+      #
+      # `--out` rather than a stdout redirect, so nothing a tool decides to print can end up inside
+      # the document `prCommentCli` parses. `pnpm --filter` takes its recursive path and some
+      # versions announce `Scope: N of M workspace projects` on the way; that line landing in
+      # head.json would fail the parse and degrade every run to the stale-report comment, which is
+      # a permanent quiet failure rather than a loud one. It does not reproduce on the 10.33.2
+      # pinned above, so this closes the class rather than a reproduction: the file is written by
+      # the process that owns it and stdout is left to be log output. Held by
+      # `it("let the scanner write its own report rather than capturing stdout")` in
+      # `internal-packages/observability-map/src/integration.test.ts`.
+      #
+      # `-s` keeps the partial dance honest now the redirect no longer creates the file: a scanner
+      # that exits 0 without writing takes the else branch and the stale-report comment, instead of
+      # failing the `mv` and turning the job red.
+      - name: 🔎 Scan head
+        run: |
+          if pnpm --filter @internal/observability-map exec tsx src/cli.ts \
+            --out=/tmp/head.json.partial && [ -s /tmp/head.json.partial ]; then
+            mv /tmp/head.json.partial /tmp/head.json
+          else
+            rm -f /tmp/head.json /tmp/head.json.partial
+            echo "head scan failed; the comment will say the report is stale for this run" >&2
+          fi
+
+      # base.sha, not a merge base, and two reviewers have now read that as a bug. The checkout
+      # above is the default for a pull_request event, so the working tree is GitHub's test merge
+      # commit, whose parents are base.sha and the PR head. The head tree therefore already contains
+      # the base branch up to base.sha, and diffing it against base.sha is what isolates this pull
+      # request's own work. A merge base would leave the intervening base-branch commits in the head
+      # tree and out of the base tree, and blame the pull request for all of them.
+      - name: 🔎 Scan base with the head's scanner
+        run: |
+          if git worktree add /tmp/base-tree ${{ github.event.pull_request.base.sha }} \
+            && pnpm --filter @internal/observability-map exec tsx src/cli.ts \
+              --routes=/tmp/base-tree/apps/webapp/app/routes --out=/tmp/base.json \
+            && [ -s /tmp/base.json ]; then
+            :
+          else
+            echo "-" > /tmp/base.json || true
+            echo "base scan failed or the worktree could not be added; falling back to no base" >&2
+          fi
+
+      # Looked up before the render step because the render decision needs it: with no delta to
+      # report, a pull request that already has a comment gets a resolved state rather than being
+      # left with findings that no longer exist, and one that does not gets nothing at all. The
+      # upsert step reuses the id rather than asking twice.
+      #
+      # On a failure that outlasts the retries, both steps below do nothing. Guessing is worse than
+      # silence here: this step is the only thing that knows which comment to PATCH, so a guess of
+      # "a comment exists" still reaches an upsert with no id to patch, which POSTs. That either
+      # adds a second marker comment beside the stale one, or says "the findings an earlier push
+      # reported are gone" on a pull request that never had findings. Worst case now is no comment
+      # this run, which the next push fixes.
+      - name: 🔍 Look for a comment from an earlier push
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ github.token }}
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+        run: |
+          rm -f /tmp/existing-comment-id /tmp/comment-lookup-failed
+          found=""
+          ok=""
+          for attempt in 1 2 3; do
+            if found=$(gh api "repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/comments" --paginate \
+              --jq '[.[] | select(.body | startswith("<!-- observability-map-report -->"))][0].id // empty'); then
+              ok=1
+              break
+            fi
+            echo "comment lookup attempt ${attempt} failed" >&2
+            sleep $((attempt * 5))
+          done
+
+          if [ -z "$ok" ]; then
+            touch /tmp/comment-lookup-failed
+            echo "comment lookup failed after 3 attempts; this run posts nothing" >&2
+            exit 0
+          fi
+
+          # --paginate runs the jq once per page, so a marker comment on more than one page yields
+          # one id per page. Unhandled, that puts a newline in the PATCH url and the step dies under
+          # continue-on-error. The oldest wins: it is the one the upsert has been updating.
+          count=$(printf '%s\n' "$found" | grep -c '[0-9]' || true)
+          if [ "$count" -gt 1 ]; then
+            echo "warning: ${count} marker comments on this pull request; updating the oldest" >&2
+          fi
+          printf '%s\n' "$found" | awk 'NF { print $1; exit }' > /tmp/existing-comment-id
+
+      # continue-on-error for the same reason as the scan: a rendering bug must not turn the job
+      # red. An empty /tmp/comment.md means there is nothing to post, which is a decision
+      # prCommentCli makes, not this shell.
+      - name: 📝 Render comment
+        continue-on-error: true
+        run: |
+          rm -f /tmp/comment.md
+          render() { pnpm --filter @internal/observability-map exec tsx src/report/prCommentCli.ts "$@"; }
+
+          # Every write goes through this, so a renderer that exits non-zero never leaves a 0-byte
+          # comment.md for the upsert to skip in silence.
+          emit() {
+            if render "$@" > /tmp/comment.md.partial; then
+              mv /tmp/comment.md.partial /tmp/comment.md
+              return 0
+            fi
+            rm -f /tmp/comment.md.partial
+            return 1
+          }
+
+          if [ -f /tmp/comment-lookup-failed ]; then
+            echo "the comment lookup failed, so this run posts nothing" >&2
+            exit 0
+          fi
+
+          if [ ! -s /tmp/head.json ]; then
+            emit --scan-failed || echo "could not render the stale-report comment either" >&2
+            exit 0
+          fi
+
+          base=/tmp/base.json
+          if [ ! -s /tmp/base.json ] || [ "$(cat /tmp/base.json)" = "-" ]; then
+            base="-"
+          fi
+
+          flags=()
+          if [ -s /tmp/existing-comment-id ]; then
+            flags=(--existing-comment)
+          fi
+
+          if ! emit /tmp/head.json "$base" "${flags[@]}"; then
+            echo "render failed; falling back to the stale-report comment" >&2
+            emit --scan-failed || echo "could not render the stale-report comment either" >&2
+          fi
+
+      # continue-on-error for the same reason: a transient gh api failure (rate limit, network)
+      # must not fail the job either. Worst case, the PR gets no comment this run.
+      - name: 💬 Upsert PR comment
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ github.token }}
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+        run: |
+          # The same sentinel the render step reads, so the two cannot disagree about what a failed
+          # lookup means. Without it this step reads a missing id as "no comment exists" and POSTs.
+          if [ -f /tmp/comment-lookup-failed ]; then
+            echo "the comment lookup failed, so this run posts nothing"
+            exit 0
+          fi
+          if [ ! -s /tmp/comment.md ]; then
+            echo "nothing to post: this pull request does not move the report"
+            exit 0
+          fi
+          existing=""
+          if [ -f /tmp/existing-comment-id ]; then
+            existing=$(cat /tmp/existing-comment-id)
+          fi
+          if [ -n "$existing" ]; then
+            gh api -X PATCH "repos/${GITHUB_REPOSITORY}/issues/comments/${existing}" -F body=@/tmp/comment.md
+          else
+            gh api "repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/comments" -F body=@/tmp/comment.md
+          fi

.github/workflows/pr_checks.yml

@@ -22,6 +22,7 @@ jobs:
       webapp: ${{ steps.filter.outputs.webapp }}
       packages: ${{ steps.filter.outputs.packages }}
       internal: ${{ steps.filter.outputs.internal }}
+      obsmap: ${{ steps.filter.outputs.obsmap }}
       cli: ${{ steps.filter.outputs.cli }}
       sdk: ${{ steps.filter.outputs.sdk }}
     steps:
@@ -81,6 +82,36 @@ jobs:
               - 'pnpm-lock.yaml'
               - 'pnpm-workspace.yaml'
               - 'turbo.json'
+            # The whole webapp app tree, not just its routes, and that is the whole reason this
+            # filter exists. Two tests in @internal/observability-map read it: integration.test.ts
+            # scans the live route tree, and webappSymbols.test.ts walks all of apps/webapp/app and
+            # fails when a guard, sensitive or audit symbol stops resolving. Routes-only was this
+            # filter's own bug: renaming e.g. requireUserId in app/services/session.server.ts
+            # matched `webapp` and nothing else, so no job ran the suite and the break landed on
+            # main, or on the next unrelated internal-packages PR.
+            #
+            # The cost of the wider set, measured over the last 400 commits on main: 31% touch
+            # routes, 52% touch apps/webapp/app, so the job goes from firing on roughly a third of
+            # PRs to roughly a half. It is the cheap one -- a single 4x runner, no containers, no
+            # database, no prisma generate -- which is what makes that affordable.
+            #
+            # observability-map.yml is here because integration.test.ts asserts on its text and no
+            # other filter watches it, so editing the report workflow alone ran nothing at all.
+            #
+            # Deliberately NOT here: this package's own paths, and packages/plugins/src and
+            # internal-packages/rbac/src, the other two trees webappSymbols.test.ts reads.
+            # `internal` above already matches `internal-packages/**` and `packages/**`, and
+            # `unit-tests-internal.yml` runs `turbo run test --filter "@internal/*"`, which picks up
+            # @internal/observability-map and runs the same vitest suite. Listing them here as well
+            # ran the suite twice on every PR touching them, which was this filter's own doing.
+            obsmap:
+              - 'apps/webapp/app/**'
+              - '.github/workflows/pr_checks.yml'
+              - '.github/workflows/unit-tests-observability-map.yml'
+              - '.github/workflows/observability-map.yml'
+              - 'package.json'
+              - 'pnpm-lock.yaml'
+              - 'pnpm-workspace.yaml'
             cli:
               - 'packages/cli-v3/**'
               - 'packages/build/**'
@@ -149,6 +180,11 @@ jobs:
       DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
       DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
 
+  obsmap:
+    needs: changes
+    if: needs.changes.outputs.obsmap == 'true'
+    uses: ./.github/workflows/unit-tests-observability-map.yml
+
   e2e:
     needs: changes
     if: needs.changes.outputs.cli == 'true'
@@ -172,6 +208,7 @@ jobs:
       - e2e-webapp
       - packages
       - internal
+      - obsmap
       - e2e
       - sdk-compat
     if: always()