diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 075d95b8..d9176750 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -362,8 +362,10 @@ jobs: - name: Run deterministic benchmark gates # Fast, machine-independent unit/gate tests (image-cache reuse, - # render-operator coalescing, scenario/threshold coverage, diff tooling). - # Catches structural regressions the timing smoke run cannot. + # render-operator coalescing, measurement counts, scenario/threshold + # coverage, diff tooling). Catches structural regressions the timing + # smoke run cannot: these are exact integers, so unlike a millisecond + # ceiling they separate a slower engine from a slower machine. run: ./mvnw -B -ntp -f benchmarks/pom.xml test - name: Run coarse performance smoke benchmark diff --git a/benchmarks/src/main/java/com/demcha/compose/MeasurementCountBenchmark.java b/benchmarks/src/main/java/com/demcha/compose/MeasurementCountBenchmark.java index 926921ad..b26290ca 100644 --- a/benchmarks/src/main/java/com/demcha/compose/MeasurementCountBenchmark.java +++ b/benchmarks/src/main/java/com/demcha/compose/MeasurementCountBenchmark.java @@ -20,7 +20,9 @@ import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; import java.util.ArrayList; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; import java.util.function.Consumer; /** @@ -93,13 +95,7 @@ private void run() throws Exception { System.out.println("Thread allocation measurement: " + (allocationSupported() ? "enabled" : "UNAVAILABLE (Alloc KB = n/a)")); System.out.println(); - Consumer longText = flow -> - flow.addParagraph(p -> p.text(LONG_PARAGRAPH).textStyle(BODY_STYLE)); - Consumer longToken = flow -> - flow.addParagraph(p -> p.text(LONG_TOKEN_PARAGRAPH).textStyle(BODY_STYLE)); - Consumer accentedText = flow -> - flow.addParagraph(p -> p.text(ACCENTED_LATIN_PARAGRAPH).textStyle(BODY_STYLE)); - Consumer largeTable = MeasurementCountBenchmark::authorLargeTable; + Map> scenarios = scenarios(); // Warm up the JVM (class loading + JIT) BEFORE the allocation window so the // "Alloc KB" column reflects steady-state per-document layout allocation, not @@ -108,17 +104,15 @@ private void run() throws Exception { // layout cost (verified: cold first compile 36.6 MB vs warm 0.65 MB for the // same long-text document). The measurement-COUNT columns are exact either way. for (int warmup = 0; warmup < 5; warmup++) { - measureScenario("warmup", longText); - measureScenario("warmup", longToken); - measureScenario("warmup", accentedText); - measureScenario("warmup", largeTable); + for (Consumer author : scenarios.values()) { + measureScenario("warmup", author); + } } List results = new ArrayList<>(); - results.add(measureScenario("long-text", longText)); - results.add(measureScenario("long-token", longToken)); - results.add(measureScenario("accented-latin", accentedText)); - results.add(measureScenario("large-table", largeTable)); + for (Map.Entry> scenario : scenarios.entrySet()) { + results.add(measureScenario(scenario.getKey(), scenario.getValue())); + } System.out.printf("%-14s | %11s | %9s | %9s | %11s | %8s | %11s | %10s | %6s%n", "Scenario", "WidthReqs", "Distinct", "Repeat %", "Sum chars", "Max arg", "LineMetrics", "Alloc KB", "Pages"); @@ -140,7 +134,37 @@ private void run() throws Exception { writeReport(results); } - private Result measureScenario(String scenario, Consumer author) throws Exception { + /** + * The probe's scenarios, in report order. Single source for the probe and for + * {@code MeasurementCountGateTest}, so a scenario added here cannot slip past + * the gate unnoticed. + * + * @return scenario name to document author + */ + static Map> scenarios() { + Map> scenarios = new LinkedHashMap<>(); + scenarios.put("long-text", flow -> + flow.addParagraph(p -> p.text(LONG_PARAGRAPH).textStyle(BODY_STYLE))); + scenarios.put("long-token", flow -> + flow.addParagraph(p -> p.text(LONG_TOKEN_PARAGRAPH).textStyle(BODY_STYLE))); + scenarios.put("accented-latin", flow -> + flow.addParagraph(p -> p.text(ACCENTED_LATIN_PARAGRAPH).textStyle(BODY_STYLE))); + scenarios.put("large-table", MeasurementCountBenchmark::authorLargeTable); + return scenarios; + } + + /** + * Compiles one scenario and returns its exact measurement counts. + * + *

Package-private and static so the gate test can drive the real layout + * pass rather than re-implementing the instrumentation.

+ * + * @param scenario report label + * @param author document author for the scenario + * @return the scenario's counts, page/fragment totals and compile allocation + * @throws Exception when composition or measurement fails + */ + static Result measureScenario(String scenario, Consumer author) throws Exception { try (DocumentSession session = GraphCompose.document() .pageSize(DocumentPageSize.A4) .margin(24, 24, 24, 24) @@ -244,7 +268,7 @@ private void writeReport(List results) throws Exception { System.out.println("Saved CSV counter report to " + csvPath); } - private record Result(String scenario, + record Result(String scenario, CountingTextMeasurementSystem.Counts counts, int pages, int fragments, diff --git a/benchmarks/src/test/java/com/demcha/compose/MeasurementCountGateTest.java b/benchmarks/src/test/java/com/demcha/compose/MeasurementCountGateTest.java new file mode 100644 index 00000000..92310f7c --- /dev/null +++ b/benchmarks/src/test/java/com/demcha/compose/MeasurementCountGateTest.java @@ -0,0 +1,115 @@ +package com.demcha.compose; + +import org.assertj.core.api.SoftAssertions; +import org.junit.jupiter.api.Test; + +import java.util.LinkedHashMap; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Deterministic regression gate for how much measurement work the layout pass + * does, driving {@link MeasurementCountBenchmark}'s real compile through a + * {@link CountingTextMeasurementSystem}. + * + *

These counts are exact integers, not timings. A wall-clock benchmark can + * only say "slower on this machine today"; it cannot separate a slower laptop + * from a slower engine, and an absolute millisecond ceiling generous enough to + * survive CI variance is generous enough to miss a real regression. Asking the + * layout pass how many width measurements it requested has neither problem: the + * answer is identical on every machine and moves only when the engine's + * behaviour actually changes.

+ * + *

Complementary to the layout snapshots rather than a duplicate of them. Two + * implementations that place fragments identically but measure a different + * number of times both satisfy every snapshot; only this gate separates + * them.

+ * + *

When this fails, the engine changed how it measures text. Decide + * whether that was intended: fewer requests for the same document is an + * improvement worth recording, more is a regression worth explaining. Either + * way the fix is to run {@code MeasurementCountBenchmark} and update the + * expectation below in the same commit as the behaviour change, so the number + * moves deliberately.

+ * + *

Allocation is deliberately not gated. It is stable to about 0.1% across + * runs on one JVM, but it is a property of the JDK as much as of the engine, so + * an expectation recorded on one machine would be a guess about another. The + * probe still reports it.

+ */ +class MeasurementCountGateTest { + + /** + * Exact measurement work per scenario, recorded from + * {@link MeasurementCountBenchmark} on the 2.1.1 development line. + */ + private static final Map EXPECTED = expected(); + + private static Map expected() { + Map work = new LinkedHashMap<>(); + work.put("long-text", new Work(4472, 32, 32457, 124, 1, 2)); + work.put("long-token", new Work(99, 55, 7739, 600, 1, 1)); + work.put("accented-latin", new Work(2499, 37, 14393, 123, 1, 1)); + work.put("large-table", new Work(1206, 211, 5916, 13, 0, 6)); + return work; + } + + /** + * The measurement work one scenario costs the layout pass. + * + * @param widthRequests total text-width measurements requested + * @param distinctWidthRequests distinct (style, text) pairs among them + * @param summedRequestChars characters across all requests + * @param maxRequestChars longest single measured string + * @param lineMetricsCalls line-metric lookups + * @param pages pages the document compiled to + */ + private record Work(long widthRequests, + long distinctWidthRequests, + long summedRequestChars, + long maxRequestChars, + long lineMetricsCalls, + int pages) { + + static Work of(MeasurementCountBenchmark.Result result) { + CountingTextMeasurementSystem.Counts counts = result.counts(); + return new Work(counts.widthRequests(), + counts.distinctWidthRequests(), + counts.summedRequestChars(), + counts.maxRequestChars(), + counts.lineMetricsCalls(), + result.pages()); + } + } + + @Test + void everyScenarioMeasuresExactlyAsMuchTextAsRecorded() throws Exception { + // Soft, so a deliberate engine change reports all four scenarios' new + // numbers in one run instead of surfacing them one failure at a time. + SoftAssertions soft = new SoftAssertions(); + for (var scenario : MeasurementCountBenchmark.scenarios().entrySet()) { + Work actual = Work.of( + MeasurementCountBenchmark.measureScenario(scenario.getKey(), scenario.getValue())); + + soft.assertThat(actual) + .describedAs("measurement work for '%s' changed - re-run " + + "MeasurementCountBenchmark and update the expectation " + + "in the same commit if the change was intended", + scenario.getKey()) + .isEqualTo(EXPECTED.get(scenario.getKey())); + } + soft.assertAll(); + } + + /** + * A scenario added to the probe without an expectation would be measured and + * silently ignored, which is the failure mode that lets coverage rot. + */ + @Test + void everyProbeScenarioIsGated() { + assertThat(MeasurementCountBenchmark.scenarios().keySet()) + .describedAs("probe scenarios missing a recorded expectation") + .containsExactlyInAnyOrderElementsOf(EXPECTED.keySet()); + } +} diff --git a/docs/operations/benchmarks.md b/docs/operations/benchmarks.md index 72e98ff2..6b4a3e15 100644 --- a/docs/operations/benchmarks.md +++ b/docs/operations/benchmarks.md @@ -200,6 +200,23 @@ Changing the engine (layout, pagination, render ordering, PDF session, text measurement, fonts) and want to see how it moves performance? Pick the view that fits, cheapest first: +- **"Did the engine change how much work it does?" — the deterministic gate.** + `MeasurementCountGateTest` compiles four fixtures and asserts the exact number + of text measurements each one costs. These are integers, identical on every + machine, so they answer the question a timing run cannot: whether the *engine* + changed or just the hardware it ran on. It runs on every performance-relevant + PR as part of the `Run deterministic benchmark gates` step, and locally in a + second: + + ```bash + ./mvnw -B -ntp test -Dtest=MeasurementCountGateTest -f benchmarks/pom.xml + ``` + + A failure prints the recorded and actual counts side by side. Fewer requests + for the same document is an improvement worth recording, more is a regression + worth explaining; either way, re-run `MeasurementCountBenchmark` and update the + expectation in the same commit as the behaviour change. + - **"Did I regress?" — gate against the committed baseline.** Run a median and let the `11-verdict-current-speed` step score each scenario IMPROVED / NEUTRAL / REGRESSED against `baselines/current-speed-full.json` (hard gate: