From a1f3e241b788d9c76ec5403ed1a0bad1e4ea1207 Mon Sep 17 00:00:00 2001 From: DemchaAV Date: Mon, 27 Jul 2026 07:45:50 +0100 Subject: [PATCH 1/2] feat(benchmarks): gate the layout pass on exact measurement counts Nothing in CI could tell a slower engine from a slower machine. The timing smoke run compares against absolute millisecond ceilings set about three times above observed values, so a regression that stays under the ceiling passes, and no job compares a branch against its own base. MeasurementCountBenchmark already answered that question deterministically and nothing ran it: it wraps the text measurement system in a counting decorator and reports, as exact integers, how many width measurements a document costs. Those counts are identical on every machine and move only when the engine changes. MeasurementCountGateTest now asserts them per scenario. It lands in the module whose whole test suite CI already runs as the deterministic-gates step, so it gates every performance-relevant pull request without a workflow change. A second test requires each probe scenario to carry a recorded expectation, so a new scenario cannot be measured and silently ungated. The gate is complementary to the layout snapshots rather than a duplicate: two implementations that place fragments identically but measure a different number of times satisfy every snapshot, and only this separates them. Verified by measuring one token twice in the wrapping loop - long-text goes from 4472 to 8552 requests and the gate fails with both records printed. The probe's scenario list moves into a shared accessor so the probe and the gate cannot drift apart. Allocation stays ungated: it is stable to about 0.1% across runs on one JVM but is a property of the JDK as much as of the engine, so an expectation recorded on one machine would be a guess about another. --- .github/workflows/ci.yml | 6 +- .../compose/MeasurementCountBenchmark.java | 58 ++++++--- .../compose/MeasurementCountGateTest.java | 110 ++++++++++++++++++ docs/operations/benchmarks.md | 17 +++ 4 files changed, 172 insertions(+), 19 deletions(-) create mode 100644 benchmarks/src/test/java/com/demcha/compose/MeasurementCountGateTest.java diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 075d95b84..d91767501 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -362,8 +362,10 @@ jobs: - name: Run deterministic benchmark gates # Fast, machine-independent unit/gate tests (image-cache reuse, - # render-operator coalescing, scenario/threshold coverage, diff tooling). - # Catches structural regressions the timing smoke run cannot. + # render-operator coalescing, measurement counts, scenario/threshold + # coverage, diff tooling). Catches structural regressions the timing + # smoke run cannot: these are exact integers, so unlike a millisecond + # ceiling they separate a slower engine from a slower machine. run: ./mvnw -B -ntp -f benchmarks/pom.xml test - name: Run coarse performance smoke benchmark diff --git a/benchmarks/src/main/java/com/demcha/compose/MeasurementCountBenchmark.java b/benchmarks/src/main/java/com/demcha/compose/MeasurementCountBenchmark.java index 926921ad3..b26290cad 100644 --- a/benchmarks/src/main/java/com/demcha/compose/MeasurementCountBenchmark.java +++ b/benchmarks/src/main/java/com/demcha/compose/MeasurementCountBenchmark.java @@ -20,7 +20,9 @@ import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; import java.util.ArrayList; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; import java.util.function.Consumer; /** @@ -93,13 +95,7 @@ private void run() throws Exception { System.out.println("Thread allocation measurement: " + (allocationSupported() ? "enabled" : "UNAVAILABLE (Alloc KB = n/a)")); System.out.println(); - Consumer longText = flow -> - flow.addParagraph(p -> p.text(LONG_PARAGRAPH).textStyle(BODY_STYLE)); - Consumer longToken = flow -> - flow.addParagraph(p -> p.text(LONG_TOKEN_PARAGRAPH).textStyle(BODY_STYLE)); - Consumer accentedText = flow -> - flow.addParagraph(p -> p.text(ACCENTED_LATIN_PARAGRAPH).textStyle(BODY_STYLE)); - Consumer largeTable = MeasurementCountBenchmark::authorLargeTable; + Map> scenarios = scenarios(); // Warm up the JVM (class loading + JIT) BEFORE the allocation window so the // "Alloc KB" column reflects steady-state per-document layout allocation, not @@ -108,17 +104,15 @@ private void run() throws Exception { // layout cost (verified: cold first compile 36.6 MB vs warm 0.65 MB for the // same long-text document). The measurement-COUNT columns are exact either way. for (int warmup = 0; warmup < 5; warmup++) { - measureScenario("warmup", longText); - measureScenario("warmup", longToken); - measureScenario("warmup", accentedText); - measureScenario("warmup", largeTable); + for (Consumer author : scenarios.values()) { + measureScenario("warmup", author); + } } List results = new ArrayList<>(); - results.add(measureScenario("long-text", longText)); - results.add(measureScenario("long-token", longToken)); - results.add(measureScenario("accented-latin", accentedText)); - results.add(measureScenario("large-table", largeTable)); + for (Map.Entry> scenario : scenarios.entrySet()) { + results.add(measureScenario(scenario.getKey(), scenario.getValue())); + } System.out.printf("%-14s | %11s | %9s | %9s | %11s | %8s | %11s | %10s | %6s%n", "Scenario", "WidthReqs", "Distinct", "Repeat %", "Sum chars", "Max arg", "LineMetrics", "Alloc KB", "Pages"); @@ -140,7 +134,37 @@ private void run() throws Exception { writeReport(results); } - private Result measureScenario(String scenario, Consumer author) throws Exception { + /** + * The probe's scenarios, in report order. Single source for the probe and for + * {@code MeasurementCountGateTest}, so a scenario added here cannot slip past + * the gate unnoticed. + * + * @return scenario name to document author + */ + static Map> scenarios() { + Map> scenarios = new LinkedHashMap<>(); + scenarios.put("long-text", flow -> + flow.addParagraph(p -> p.text(LONG_PARAGRAPH).textStyle(BODY_STYLE))); + scenarios.put("long-token", flow -> + flow.addParagraph(p -> p.text(LONG_TOKEN_PARAGRAPH).textStyle(BODY_STYLE))); + scenarios.put("accented-latin", flow -> + flow.addParagraph(p -> p.text(ACCENTED_LATIN_PARAGRAPH).textStyle(BODY_STYLE))); + scenarios.put("large-table", MeasurementCountBenchmark::authorLargeTable); + return scenarios; + } + + /** + * Compiles one scenario and returns its exact measurement counts. + * + *

Package-private and static so the gate test can drive the real layout + * pass rather than re-implementing the instrumentation.

+ * + * @param scenario report label + * @param author document author for the scenario + * @return the scenario's counts, page/fragment totals and compile allocation + * @throws Exception when composition or measurement fails + */ + static Result measureScenario(String scenario, Consumer author) throws Exception { try (DocumentSession session = GraphCompose.document() .pageSize(DocumentPageSize.A4) .margin(24, 24, 24, 24) @@ -244,7 +268,7 @@ private void writeReport(List results) throws Exception { System.out.println("Saved CSV counter report to " + csvPath); } - private record Result(String scenario, + record Result(String scenario, CountingTextMeasurementSystem.Counts counts, int pages, int fragments, diff --git a/benchmarks/src/test/java/com/demcha/compose/MeasurementCountGateTest.java b/benchmarks/src/test/java/com/demcha/compose/MeasurementCountGateTest.java new file mode 100644 index 000000000..7d032f3c2 --- /dev/null +++ b/benchmarks/src/test/java/com/demcha/compose/MeasurementCountGateTest.java @@ -0,0 +1,110 @@ +package com.demcha.compose; + +import org.junit.jupiter.api.Test; + +import java.util.LinkedHashMap; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Deterministic regression gate for how much measurement work the layout pass + * does, driving {@link MeasurementCountBenchmark}'s real compile through a + * {@link CountingTextMeasurementSystem}. + * + *

These counts are exact integers, not timings. A wall-clock benchmark can + * only say "slower on this machine today"; it cannot separate a slower laptop + * from a slower engine, and an absolute millisecond ceiling generous enough to + * survive CI variance is generous enough to miss a real regression. Asking the + * layout pass how many width measurements it requested has neither problem: the + * answer is identical on every machine and moves only when the engine's + * behaviour actually changes.

+ * + *

Complementary to the layout snapshots rather than a duplicate of them. Two + * implementations that place fragments identically but measure a different + * number of times both satisfy every snapshot; only this gate separates + * them.

+ * + *

When this fails, the engine changed how it measures text. Decide + * whether that was intended: fewer requests for the same document is an + * improvement worth recording, more is a regression worth explaining. Either + * way the fix is to run {@code MeasurementCountBenchmark} and update the + * expectation below in the same commit as the behaviour change, so the number + * moves deliberately.

+ * + *

Allocation is deliberately not gated. It is stable to about 0.1% across + * runs on one JVM, but it is a property of the JDK as much as of the engine, so + * an expectation recorded on one machine would be a guess about another. The + * probe still reports it.

+ */ +class MeasurementCountGateTest { + + /** + * Exact measurement work per scenario, recorded from + * {@link MeasurementCountBenchmark} on the 2.1.1 development line. + */ + private static final Map EXPECTED = expected(); + + private static Map expected() { + Map work = new LinkedHashMap<>(); + work.put("long-text", new Work(4472, 32, 32457, 124, 1, 2)); + work.put("long-token", new Work(99, 55, 7739, 600, 1, 1)); + work.put("accented-latin", new Work(2499, 37, 14393, 123, 1, 1)); + work.put("large-table", new Work(1206, 211, 5916, 13, 0, 6)); + return work; + } + + /** + * The measurement work one scenario costs the layout pass. + * + * @param widthRequests total text-width measurements requested + * @param distinctWidthRequests distinct (style, text) pairs among them + * @param summedRequestChars characters across all requests + * @param maxRequestChars longest single measured string + * @param lineMetricsCalls line-metric lookups + * @param pages pages the document compiled to + */ + private record Work(long widthRequests, + long distinctWidthRequests, + long summedRequestChars, + long maxRequestChars, + long lineMetricsCalls, + int pages) { + + static Work of(MeasurementCountBenchmark.Result result) { + CountingTextMeasurementSystem.Counts counts = result.counts(); + return new Work(counts.widthRequests(), + counts.distinctWidthRequests(), + counts.summedRequestChars(), + counts.maxRequestChars(), + counts.lineMetricsCalls(), + result.pages()); + } + } + + @Test + void everyScenarioMeasuresExactlyAsMuchTextAsRecorded() throws Exception { + for (var scenario : MeasurementCountBenchmark.scenarios().entrySet()) { + Work actual = Work.of( + MeasurementCountBenchmark.measureScenario(scenario.getKey(), scenario.getValue())); + + assertThat(actual) + .describedAs("measurement work for '%s' changed - re-run " + + "MeasurementCountBenchmark and update the expectation " + + "in the same commit if the change was intended", + scenario.getKey()) + .isEqualTo(EXPECTED.get(scenario.getKey())); + } + } + + /** + * A scenario added to the probe without an expectation would be measured and + * silently ignored, which is the failure mode that lets coverage rot. + */ + @Test + void everyProbeScenarioIsGated() { + assertThat(MeasurementCountBenchmark.scenarios().keySet()) + .describedAs("probe scenarios missing a recorded expectation") + .containsExactlyInAnyOrderElementsOf(EXPECTED.keySet()); + } +} diff --git a/docs/operations/benchmarks.md b/docs/operations/benchmarks.md index 72e98ff2e..6b4a3e15c 100644 --- a/docs/operations/benchmarks.md +++ b/docs/operations/benchmarks.md @@ -200,6 +200,23 @@ Changing the engine (layout, pagination, render ordering, PDF session, text measurement, fonts) and want to see how it moves performance? Pick the view that fits, cheapest first: +- **"Did the engine change how much work it does?" — the deterministic gate.** + `MeasurementCountGateTest` compiles four fixtures and asserts the exact number + of text measurements each one costs. These are integers, identical on every + machine, so they answer the question a timing run cannot: whether the *engine* + changed or just the hardware it ran on. It runs on every performance-relevant + PR as part of the `Run deterministic benchmark gates` step, and locally in a + second: + + ```bash + ./mvnw -B -ntp test -Dtest=MeasurementCountGateTest -f benchmarks/pom.xml + ``` + + A failure prints the recorded and actual counts side by side. Fewer requests + for the same document is an improvement worth recording, more is a regression + worth explaining; either way, re-run `MeasurementCountBenchmark` and update the + expectation in the same commit as the behaviour change. + - **"Did I regress?" — gate against the committed baseline.** Run a median and let the `11-verdict-current-speed` step score each scenario IMPROVED / NEUTRAL / REGRESSED against `baselines/current-speed-full.json` (hard gate: From 8fbe8c1984d9b35f38f98c3e5f366f7eb2f43fb4 Mon Sep 17 00:00:00 2001 From: DemchaAV Date: Mon, 27 Jul 2026 09:42:09 +0100 Subject: [PATCH 2/2] test(benchmarks): report every changed scenario in one run Asserting inside the scenario loop stopped at the first mismatch, so a deliberate engine change surfaced one scenario's new numbers per run and the author had to re-run three more times to collect the rest. Soft assertions report all four together. --- .../java/com/demcha/compose/MeasurementCountGateTest.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/benchmarks/src/test/java/com/demcha/compose/MeasurementCountGateTest.java b/benchmarks/src/test/java/com/demcha/compose/MeasurementCountGateTest.java index 7d032f3c2..92310f7c7 100644 --- a/benchmarks/src/test/java/com/demcha/compose/MeasurementCountGateTest.java +++ b/benchmarks/src/test/java/com/demcha/compose/MeasurementCountGateTest.java @@ -1,5 +1,6 @@ package com.demcha.compose; +import org.assertj.core.api.SoftAssertions; import org.junit.jupiter.api.Test; import java.util.LinkedHashMap; @@ -84,17 +85,21 @@ static Work of(MeasurementCountBenchmark.Result result) { @Test void everyScenarioMeasuresExactlyAsMuchTextAsRecorded() throws Exception { + // Soft, so a deliberate engine change reports all four scenarios' new + // numbers in one run instead of surfacing them one failure at a time. + SoftAssertions soft = new SoftAssertions(); for (var scenario : MeasurementCountBenchmark.scenarios().entrySet()) { Work actual = Work.of( MeasurementCountBenchmark.measureScenario(scenario.getKey(), scenario.getValue())); - assertThat(actual) + soft.assertThat(actual) .describedAs("measurement work for '%s' changed - re-run " + "MeasurementCountBenchmark and update the expectation " + "in the same commit if the change was intended", scenario.getKey()) .isEqualTo(EXPECTED.get(scenario.getKey())); } + soft.assertAll(); } /**