From a9a66e0ad725048bce084c480ecbabb6a793391c Mon Sep 17 00:00:00 2001 From: John Oliver <1615532+johnoliver@users.noreply.github.com> Date: Fri, 31 Jul 2026 14:45:47 +0000 Subject: [PATCH 01/11] Add continuous profiling and update telemetry to latest illuminate requirements --- .../diagnostics/DiagnosticEngine.java | 13 +++ .../{MachineStats.java => MachineInfo.java} | 25 +++-- .../CodeOptimizerDiagnosticEngineJfr.java | 62 +++++++++-- .../collection/calibration/Calibration.java | 17 --- .../collection/calibration/Calibrator.java | 9 -- .../calibration/CalibratorDefault.java | 100 ----------------- .../calibration/ContextSwitchingRunner.java | 97 ---------------- .../diagnostics/jfr/SystemStatsProvider.java | 33 +----- .../internal/configuration/Configuration.java | 12 ++ .../PerformanceMonitoringService.java | 7 ++ .../agent/internal/profiler/Profiler.java | 104 ++++++++++++++++++ .../triggers/AlertingSubsystemInit.java | 22 +++- .../profiler/diagnostic-cpu-profile.jfc | 2 +- .../profiler/diagnostic-memory-profile.jfc | 2 +- .../internal/profiler/reduced-cpu-profile.jfc | 2 +- .../profiler/reduced-memory-profile.jfc | 2 +- .../ProfilerContinuousProfilingTest.java | 99 +++++++++++++++++ .../jfrfile/JfrFileReader.java | 20 ++++ .../smoketestapp/TestController.java | 54 +++++++++ .../smoketest/DiagnosticsTest.java | 21 ++++ 20 files changed, 427 insertions(+), 276 deletions(-) rename agent/agent-profiler/agent-diagnostics-jfr/src/main/java/com/microsoft/applicationinsights/diagnostics/jfr/{MachineStats.java => MachineInfo.java} (71%) delete mode 100644 agent/agent-profiler/agent-diagnostics/src/main/java/com/microsoft/applicationinsights/diagnostics/collection/calibration/Calibration.java delete mode 100644 agent/agent-profiler/agent-diagnostics/src/main/java/com/microsoft/applicationinsights/diagnostics/collection/calibration/Calibrator.java delete mode 100644 agent/agent-profiler/agent-diagnostics/src/main/java/com/microsoft/applicationinsights/diagnostics/collection/calibration/CalibratorDefault.java delete mode 100644 agent/agent-profiler/agent-diagnostics/src/main/java/com/microsoft/applicationinsights/diagnostics/collection/calibration/ContextSwitchingRunner.java create mode 100644 agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/profiler/ProfilerContinuousProfilingTest.java diff --git a/agent/agent-profiler/agent-diagnostics-api/src/main/java/com/microsoft/applicationinsights/diagnostics/DiagnosticEngine.java b/agent/agent-profiler/agent-diagnostics-api/src/main/java/com/microsoft/applicationinsights/diagnostics/DiagnosticEngine.java index f47a8f4659b..fca33239223 100644 --- a/agent/agent-profiler/agent-diagnostics-api/src/main/java/com/microsoft/applicationinsights/diagnostics/DiagnosticEngine.java +++ b/agent/agent-profiler/agent-diagnostics-api/src/main/java/com/microsoft/applicationinsights/diagnostics/DiagnosticEngine.java @@ -17,4 +17,17 @@ public interface DiagnosticEngine { * alertBreach.alertConfiguration.profileDuration */ Future> performDiagnosis(AlertBreach alertBreach); + + /** + * Start collecting diagnostics continuously. + * + *

Used with continuous profiling, where a circular buffer is dumped on demand rather than a + * forward-looking recording being created per breach. Registering the periodic diagnostic + * emitters up front ensures diagnostic events populate the continuous recording buffer, so they + * are present in any snapshot that is dumped. Defaults to a no-op. + */ + default void startContinuousDiagnostics() {} + + /** Stop collecting diagnostics continuously. Defaults to a no-op. */ + default void stopContinuousDiagnostics() {} } diff --git a/agent/agent-profiler/agent-diagnostics-jfr/src/main/java/com/microsoft/applicationinsights/diagnostics/jfr/MachineStats.java b/agent/agent-profiler/agent-diagnostics-jfr/src/main/java/com/microsoft/applicationinsights/diagnostics/jfr/MachineInfo.java similarity index 71% rename from agent/agent-profiler/agent-diagnostics-jfr/src/main/java/com/microsoft/applicationinsights/diagnostics/jfr/MachineStats.java rename to agent/agent-profiler/agent-diagnostics-jfr/src/main/java/com/microsoft/applicationinsights/diagnostics/jfr/MachineInfo.java index 438a98ed9e3..5f3d815d250 100644 --- a/agent/agent-profiler/agent-diagnostics-jfr/src/main/java/com/microsoft/applicationinsights/diagnostics/jfr/MachineStats.java +++ b/agent/agent-profiler/agent-diagnostics-jfr/src/main/java/com/microsoft/applicationinsights/diagnostics/jfr/MachineInfo.java @@ -17,15 +17,20 @@ import jdk.jfr.StackTrace; @SuppressWarnings("Java8ApiChecker") // JFR APIs require Java 11+, but agent targets Java 8 bytecode -@Name("com.microsoft.applicationinsights.diagnostics.jfr.MachineStats") -@Label("MachineStats") +@Name("com.microsoft.applicationinsights.diagnostics.jfr.MachineInfo") +@Label("MachineInfo") @Category("Diagnostic") -@Description("MachineStats") +@Description("MachineInfo") @StackTrace(false) @Period("beginChunk") -public class MachineStats extends Event implements JsonSerializable { - public static final String NAME = - "com.microsoft.applicationinsights.diagnostics.jfr.MachineStats"; +public class MachineInfo extends Event implements JsonSerializable { + public static final String NAME = "com.microsoft.applicationinsights.diagnostics.jfr.MachineInfo"; + + /** + * Legacy calibration field. No longer populated (calibration was removed) but retained — along + * with its accessor and deserialization — so previously-recorded recordings can still be read and + * scored. New recordings emit it as 0. + */ private double contextSwitchesPerMs; private int coreCount; @@ -34,7 +39,7 @@ public double getContextSwitchesPerMs() { return contextSwitchesPerMs; } - public MachineStats setContextSwitchesPerMs(double contextSwitchesPerMs) { + public MachineInfo setContextSwitchesPerMs(double contextSwitchesPerMs) { this.contextSwitchesPerMs = contextSwitchesPerMs; return this; } @@ -43,7 +48,7 @@ public int getCoreCount() { return coreCount; } - public MachineStats setCoreCount(int coreCount) { + public MachineInfo setCoreCount(int coreCount) { this.coreCount = coreCount; return this; } @@ -57,10 +62,10 @@ public JsonWriter toJson(JsonWriter jsonWriter) throws IOException { .writeEndObject(); } - public static MachineStats fromJson(JsonReader jsonReader) throws IOException { + public static MachineInfo fromJson(JsonReader jsonReader) throws IOException { return jsonReader.readObject( reader -> { - MachineStats deserializedValue = new MachineStats(); + MachineInfo deserializedValue = new MachineInfo(); while (reader.nextToken() != JsonToken.END_OBJECT) { String fieldName = reader.getFieldName(); diff --git a/agent/agent-profiler/agent-diagnostics/src/main/java/com/microsoft/applicationinsights/diagnostics/appinsights/CodeOptimizerDiagnosticEngineJfr.java b/agent/agent-profiler/agent-diagnostics/src/main/java/com/microsoft/applicationinsights/diagnostics/appinsights/CodeOptimizerDiagnosticEngineJfr.java index de01d5d6901..fd60d5c4fe3 100644 --- a/agent/agent-profiler/agent-diagnostics/src/main/java/com/microsoft/applicationinsights/diagnostics/appinsights/CodeOptimizerDiagnosticEngineJfr.java +++ b/agent/agent-profiler/agent-diagnostics/src/main/java/com/microsoft/applicationinsights/diagnostics/appinsights/CodeOptimizerDiagnosticEngineJfr.java @@ -10,7 +10,7 @@ import com.microsoft.applicationinsights.diagnostics.DiagnosticEngine; import com.microsoft.applicationinsights.diagnostics.jfr.AlertBreachJfrEvent; import com.microsoft.applicationinsights.diagnostics.jfr.CodeOptimizerDiagnosticsJfrInit; -import com.microsoft.applicationinsights.diagnostics.jfr.MachineStats; +import com.microsoft.applicationinsights.diagnostics.jfr.MachineInfo; import com.microsoft.applicationinsights.diagnostics.jfr.SystemStatsProvider; import java.io.IOException; import java.io.StringWriter; @@ -20,6 +20,8 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -35,7 +37,11 @@ public class CodeOptimizerDiagnosticEngineJfr implements DiagnosticEngine { private final ScheduledExecutorService executorService; private final Semaphore semaphore = new Semaphore(1, false); private final Path cgroupBasePath; - private int thisPid; + private final AtomicInteger thisPid = new AtomicInteger(); + + // When true, periodic diagnostic emitters are registered continuously (for continuous profiling) + // and must not be torn down by an individual performDiagnosis cycle. + private final AtomicBoolean continuous = new AtomicBoolean(false); public CodeOptimizerDiagnosticEngineJfr( ScheduledExecutorService executorService, Path cgroupBasePath) { @@ -50,7 +56,7 @@ public void init(int thisPid) { return; } - this.thisPid = thisPid; + this.thisPid.set(thisPid); logger.debug("Initialising Code Optimizer Diagnostic Engine"); CodeOptimizerDiagnosticsJfrInit.initFeature(thisPid, cgroupBasePath); @@ -68,8 +74,48 @@ private static void endDiagnosticCycle() { CodeOptimizerDiagnosticsJfrInit.stop(); } + @Override + public void startContinuousDiagnostics() { + if (!CodeOptimizerDiagnosticsJfrInit.isOsSupported()) { + logger.warn("Code Optimizer diagnostics is not supported on this operating system"); + return; + } + + continuous.set(true); + logger.debug("Starting continuous Code Optimizer diagnostics"); + // Registers the periodic diagnostic emitters (Telemetry, CGroupData) so they continuously + // populate the continuous profiling circular buffer. + startDiagnosticCycle(thisPid.get(), cgroupBasePath); + } + + @Override + public void stopContinuousDiagnostics() { + if (!CodeOptimizerDiagnosticsJfrInit.isOsSupported()) { + return; + } + + continuous.set(false); + logger.debug("Stopping continuous Code Optimizer diagnostics"); + endDiagnosticCycle(); + } + @Override public Future> performDiagnosis(AlertBreach alert) { + if (continuous.get()) { + // Periodic diagnostics are already running continuously, so we must not start or stop the + // diagnostic cycle here (doing so would remove the continuously-registered emitters). Just + // emit the point-in-time breach information. + CompletableFuture> diagnosisResultCompletableFuture = + new CompletableFuture<>(); + try { + emitInfo(alert, cgroupBasePath); + diagnosisResultCompletableFuture.complete(null); + } catch (RuntimeException e) { + diagnosisResultCompletableFuture.completeExceptionally(e); + } + return diagnosisResultCompletableFuture; + } + CompletableFuture> diagnosisResultCompletableFuture = new CompletableFuture<>(); try { @@ -80,7 +126,7 @@ public Future> performDiagnosis(AlertBreach alert) { long end = profileDurationInSec - TIME_BEFORE_END_OF_PROFILE_TO_EMIT_EVENT; - startDiagnosticCycle(thisPid, cgroupBasePath); + startDiagnosticCycle(thisPid.get(), cgroupBasePath); scheduleEmittingAlertBreachEvent(alert, end); @@ -140,12 +186,12 @@ private static void emitInfo(AlertBreach alert, Path cgroupBasePath) { logger.debug("Emitting Code Optimizer Diagnostic Event"); emitAlertBreachJfrEvent(alert); CodeOptimizerDiagnosticsJfrInit.emitCGroupData(cgroupBasePath); - emitMachineStats(); + emitMachineInfo(); } - private static void emitMachineStats() { - MachineStats machineStats = SystemStatsProvider.getMachineStats(); - machineStats.commit(); + private static void emitMachineInfo() { + MachineInfo machineInfo = SystemStatsProvider.getMachineInfo(); + machineInfo.commit(); } private static void emitAlertBreachJfrEvent(AlertBreach alert) { diff --git a/agent/agent-profiler/agent-diagnostics/src/main/java/com/microsoft/applicationinsights/diagnostics/collection/calibration/Calibration.java b/agent/agent-profiler/agent-diagnostics/src/main/java/com/microsoft/applicationinsights/diagnostics/collection/calibration/Calibration.java deleted file mode 100644 index 8fef10c3e30..00000000000 --- a/agent/agent-profiler/agent-diagnostics/src/main/java/com/microsoft/applicationinsights/diagnostics/collection/calibration/Calibration.java +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.microsoft.applicationinsights.diagnostics.collection.calibration; - -public class Calibration { - public static final double UNKNOWN = -1; - private final double contextSwitchingRate; - - public Calibration(double contextSwitchingRate) { - this.contextSwitchingRate = contextSwitchingRate; - } - - public double getContextSwitchingRate() { - return contextSwitchingRate; - } -} diff --git a/agent/agent-profiler/agent-diagnostics/src/main/java/com/microsoft/applicationinsights/diagnostics/collection/calibration/Calibrator.java b/agent/agent-profiler/agent-diagnostics/src/main/java/com/microsoft/applicationinsights/diagnostics/collection/calibration/Calibrator.java deleted file mode 100644 index c2950e8ddd8..00000000000 --- a/agent/agent-profiler/agent-diagnostics/src/main/java/com/microsoft/applicationinsights/diagnostics/collection/calibration/Calibrator.java +++ /dev/null @@ -1,9 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.microsoft.applicationinsights.diagnostics.collection.calibration; - -public interface Calibrator { - - Calibration calibrate(); -} diff --git a/agent/agent-profiler/agent-diagnostics/src/main/java/com/microsoft/applicationinsights/diagnostics/collection/calibration/CalibratorDefault.java b/agent/agent-profiler/agent-diagnostics/src/main/java/com/microsoft/applicationinsights/diagnostics/collection/calibration/CalibratorDefault.java deleted file mode 100644 index 7045adb6edc..00000000000 --- a/agent/agent-profiler/agent-diagnostics/src/main/java/com/microsoft/applicationinsights/diagnostics/collection/calibration/CalibratorDefault.java +++ /dev/null @@ -1,100 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.microsoft.applicationinsights.diagnostics.collection.calibration; - -import com.microsoft.applicationinsights.diagnostics.collection.libos.OperatingSystemInteractionException; -import com.microsoft.applicationinsights.diagnostics.collection.libos.kernel.KernelMonitorDeviceDriver; -import com.microsoft.applicationinsights.diagnostics.collection.libos.process.Process; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Iterator; -import java.util.List; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class CalibratorDefault implements Calibrator { - - private static final Logger logger = LoggerFactory.getLogger(ContextSwitchingRunner.class); - - private final ContextSwitchingRunner contextSwitching; - private final KernelMonitorDeviceDriver kernel; - private final Process thisProcess; - - public CalibratorDefault( - ContextSwitchingRunner contextSwitching, - KernelMonitorDeviceDriver kernel, - Process thisProcess) { - this.contextSwitching = contextSwitching; - this.kernel = kernel; - this.thisProcess = thisProcess; - } - - @Override - public Calibration calibrate() { - try { - List contextSwitchesPerRun = new ArrayList<>(); - Iterator iterator = contextSwitching.iterator(); - - int runCount = contextSwitching.getRunCount(); - double[] diagnosticsTimes = new double[runCount]; - double[] cpuTimes = new double[runCount]; - - for (int i = 0; iterator.hasNext(); i++) { - long time = System.currentTimeMillis(); - - update(thisProcess); - - iterator.next(); - - update(thisProcess); - - time = System.currentTimeMillis() - time; - - // MIN_VALUE to Make sure its not 0 - diagnosticsTimes[i] = safeDiv(getProcessCpuTime(thisProcess), (double) time); - cpuTimes[i] = safeDiv(getCpuTime(), (double) time); - - double contextSwitches = getContextSwitches(); - - contextSwitchesPerRun.add(safeDiv(contextSwitches, (double) time)); - } - - double maxContextSwitches = Collections.max(contextSwitchesPerRun); - int index = contextSwitchesPerRun.indexOf(maxContextSwitches); - - double diagnosticsTime = diagnosticsTimes[index]; - double cpuTime = diagnosticsTimes[index]; - double timeDiagnosticsHadAvailable = safeDiv(diagnosticsTime, cpuTime); - double contextSwitchingRate = safeDiv(maxContextSwitches, timeDiagnosticsHadAvailable); - return new Calibration(contextSwitchingRate); - } catch (Throwable e) { - logger.debug("Completing exceptionally", e); - } - return new Calibration(Calibration.UNKNOWN); - } - - private static double safeDiv(double numerator, double denominator) { - return numerator / (denominator + Double.MIN_VALUE); - } - - private void update(Process process) throws OperatingSystemInteractionException { - process.poll(); - kernel.poll(); - - process.update(); - kernel.update(); - } - - private static double getProcessCpuTime(Process process) { - return process.getCpuStats().getTotalTime().doubleValue(); - } - - private long getContextSwitches() throws OperatingSystemInteractionException { - return kernel.getCounters().getContextSwitches(); - } - - private double getCpuTime() throws OperatingSystemInteractionException { - return 100D - kernel.getCounters().getIdleTime(); - } -} diff --git a/agent/agent-profiler/agent-diagnostics/src/main/java/com/microsoft/applicationinsights/diagnostics/collection/calibration/ContextSwitchingRunner.java b/agent/agent-profiler/agent-diagnostics/src/main/java/com/microsoft/applicationinsights/diagnostics/collection/calibration/ContextSwitchingRunner.java deleted file mode 100644 index 90ff94f45c5..00000000000 --- a/agent/agent-profiler/agent-diagnostics/src/main/java/com/microsoft/applicationinsights/diagnostics/collection/calibration/ContextSwitchingRunner.java +++ /dev/null @@ -1,97 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.microsoft.applicationinsights.diagnostics.collection.calibration; - -import java.util.Arrays; -import java.util.Iterator; -import java.util.List; - -public class ContextSwitchingRunner implements Iterable { - - private static final int NUMBER_OF_CALLS = 40000; - - private static final Object lock = new Object(); - - private final List threadCounts; - - public ContextSwitchingRunner() { - threadCounts = Arrays.asList(100, 1000, 10, 100); - } - - public int getRunCount() { - return threadCounts.size(); - } - - public static void main(String[] args) { - new ContextSwitchingRunner().runThreads(1000); - } - - @SuppressWarnings( - "ThreadJoinLoop") // thread join in loop is safe and necessary for proper synchronization - private void runThreads(int threadCount) { - Thread[] threads = new Thread[threadCount]; - - for (int i = 0; i < threads.length; i++) { - threads[i] = - new Thread() { - @SuppressWarnings("unused") // value not used but required by API signature or framework - private int value = 0; - - @Override - public void run() { - for (int i = 0; i < NUMBER_OF_CALLS; i++) { - synchronized (lock) { - value++; - } - } - } - }; - } - - int numThreads = 0; - // fork - for (; numThreads < threads.length; numThreads++) { - try { - threads[numThreads].start(); - } catch (Error error) { - // OS will not allow us to spawn enough threads, just use however many we have - break; - } - } - - // join - try { - for (int i = 0; i < numThreads; i++) { - if (threads[i] != null) { - threads[i].join(); - } - } - } catch (InterruptedException e) { - // Ignore - } - } - - @Override - public Iterator iterator() { - Iterator countIterator = threadCounts.iterator(); - return new Iterator() { - @Override - public boolean hasNext() { - return countIterator.hasNext(); - } - - @Override - public Void next() { - Integer threadCount = countIterator.next(); - runThreads(threadCount); - return null; - } - - @Override - public void remove() { - throw new UnsupportedOperationException("Doesn't make sense to remove from this iterable"); - } - }; - } -} diff --git a/agent/agent-profiler/agent-diagnostics/src/main/java/com/microsoft/applicationinsights/diagnostics/jfr/SystemStatsProvider.java b/agent/agent-profiler/agent-diagnostics/src/main/java/com/microsoft/applicationinsights/diagnostics/jfr/SystemStatsProvider.java index 953ffd970f8..8750b7d6056 100644 --- a/agent/agent-profiler/agent-diagnostics/src/main/java/com/microsoft/applicationinsights/diagnostics/jfr/SystemStatsProvider.java +++ b/agent/agent-profiler/agent-diagnostics/src/main/java/com/microsoft/applicationinsights/diagnostics/jfr/SystemStatsProvider.java @@ -4,10 +4,6 @@ package com.microsoft.applicationinsights.diagnostics.jfr; import com.microsoft.applicationinsights.diagnostics.collection.SystemStatsReader; -import com.microsoft.applicationinsights.diagnostics.collection.calibration.Calibration; -import com.microsoft.applicationinsights.diagnostics.collection.calibration.Calibrator; -import com.microsoft.applicationinsights.diagnostics.collection.calibration.CalibratorDefault; -import com.microsoft.applicationinsights.diagnostics.collection.calibration.ContextSwitchingRunner; import com.microsoft.applicationinsights.diagnostics.collection.cores.RuntimeCoreCounter; import com.microsoft.applicationinsights.diagnostics.collection.libos.OperatingSystemInteractionException; import com.microsoft.applicationinsights.diagnostics.collection.libos.hardware.MemoryInfoReader; @@ -61,10 +57,9 @@ public static void init(int thisPid, Path cgroupBasePath) { if (initialised.compareAndSet(false, true)) { singletons.put(ThisPidSupplier.class, new AtomicReference<>((ThisPidSupplier) () -> thisPid)); - if (singletons.get(Calibrator.class) == null) { + if (singletons.get(MachineInfo.class) == null) { try { - getCalibration(); - getMachineStats(); + getMachineInfo(); getCGroupData(cgroupBasePath); // Close until needed @@ -132,25 +127,10 @@ public static CGroupData getCGroupData(Path cgroupBasePath) { }); } - public static MachineStats getMachineStats() { + public static MachineInfo getMachineInfo() { return getSingleton( - MachineStats.class, - () -> - new MachineStats() - .setContextSwitchesPerMs(getCalibration().getContextSwitchingRate()) - .setCoreCount(new RuntimeCoreCounter().getCoreCount())); - } - - private static Calibration getCalibration() { - return getSingleton( - Calibration.class, - () -> { - Calibrator calibrator = - new CalibratorDefault( - new ContextSwitchingRunner(), getKernelMonitor(), getThisProcess()); - - return calibrator.calibrate(); - }); + MachineInfo.class, + () -> new MachineInfo().setCoreCount(new RuntimeCoreCounter().getCoreCount())); } private static Process getThisProcess() { @@ -158,9 +138,6 @@ private static Process getThisProcess() { Process.class, () -> { ProcessDumper processDumper = getProcessDumper(); - if (processDumper == null) { - return null; - } processDumper.poll(); Process thisProcess = processDumper.thisProcess(); diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/configuration/Configuration.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/configuration/Configuration.java index 57861e0a0d0..77517f7bb22 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/configuration/Configuration.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/configuration/Configuration.java @@ -1543,6 +1543,18 @@ public static class ProfilerConfiguration { // Whether to register a JMX MBean that allows triggering profiles via JMX tools. public boolean enableProfilerControlMBean = false; + + // When enabled, the profiler keeps a single JFR recording running continuously using a + // circular buffer (bounded by continuousProfilingMaxAgeSeconds) instead of starting a new + // timed recording for each trigger. When a profile is requested, the current contents of the + // circular buffer are dumped and uploaded immediately, capturing the most recent + // continuousProfilingMaxAgeSeconds of data. This lets profile requests be serviced without + // waiting for a recording duration to elapse. + public boolean enableContinuousProfiling = false; + + // The maximum age (in seconds) of data retained in the continuous profiling circular buffer. + // Only used when enableContinuousProfiling is true. + public int continuousProfilingMaxAgeSeconds = 120; } /** diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/PerformanceMonitoringService.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/PerformanceMonitoringService.java index d8af15d22db..5f00a46fdd4 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/PerformanceMonitoringService.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/PerformanceMonitoringService.java @@ -135,6 +135,13 @@ synchronized void enableProfiler( // Daemon remains alive permanently due to scheduling an update profiler.initialize(uploadService, serviceProfilerExecutorService); + // When continuous profiling is enabled the profiler dumps a backward-looking circular buffer + // on demand, so diagnostics must run continuously to populate that buffer with diagnostic + // events (rather than being started per-breach for a forward-looking recording). + if (configuration.enableContinuousProfiling && diagnosticEngine != null) { + diagnosticEngine.startContinuousDiagnostics(); + } + } catch (Throwable t) { logger.error( "Failed to initialise profiler service", diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/Profiler.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/Profiler.java index 6ad09d39bbb..cf6cf88e00a 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/Profiler.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/Profiler.java @@ -77,6 +77,14 @@ public class Profiler { private final RecordingConfiguration spanRecordingConfiguration; private final RecordingConfiguration manualRecordingConfiguration; + private final boolean continuousProfilingEnabled; + private final Duration continuousProfilingMaxAge; + private final RecordingConfiguration continuousRecordingConfiguration; + + // Long-running recording backed by a circular buffer (maxAge, no duration) used when + // continuous profiling is enabled. Guarded by activeRecordingLock. + @Nullable private Recording continuousRecording = null; + private final File temporaryDirectory; private final TimeSource timeSource; @@ -104,6 +112,9 @@ public Profiler(Configuration.ProfilerConfiguration config, File tempDir, TimeSo cpuRecordingConfiguration = AlternativeJfrConfigurations.getCpuProfileConfig(config); spanRecordingConfiguration = AlternativeJfrConfigurations.getSpanProfileConfig(config); manualRecordingConfiguration = AlternativeJfrConfigurations.getManualProfileConfig(config); + continuousProfilingEnabled = config.enableContinuousProfiling; + continuousProfilingMaxAge = Duration.ofSeconds(config.continuousProfilingMaxAgeSeconds); + continuousRecordingConfiguration = AlternativeJfrConfigurations.getCpuProfileConfig(config); temporaryDirectory = tempDir; } @@ -129,6 +140,8 @@ public void initialize( // Possibly an older JVM, try using Diagnostic command flightRecorderConnection = FlightRecorderConnection.diagnosticCommandConnection(mbeanServer); } + + startContinuousRecordingIfEnabled(); } // visible for testing @@ -140,6 +153,8 @@ void initialize( this.scheduledExecutorService = scheduledExecutorService; this.recordingOptionsBuilder = new RecordingOptions.Builder(); this.flightRecorderConnection = flightRecorderConnection; + + startContinuousRecordingIfEnabled(); } /** Apply new configuration settings obtained from Service Profiler. */ @@ -152,12 +167,101 @@ public void updateConfiguration(ProfilerConfiguration newConfig) { // visible for tests void profileAndUpload(AlertBreach alertBreach, Duration duration, UploadListener uploadListener) { Instant recordingStart = timeSource.getNow(); + if (continuousProfilingEnabled) { + captureContinuousRecording(alertBreach, recordingStart, uploadListener); + return; + } executeProfile( alertBreach.getType(), duration, uploadNewRecording(alertBreach, recordingStart, uploadListener)); } + private void startContinuousRecordingIfEnabled() { + if (!continuousProfilingEnabled) { + return; + } + synchronized (activeRecordingLock) { + if (continuousRecording != null) { + return; + } + try { + // A continuous recording uses a circular buffer bounded by maxAge and no duration, so it + // runs indefinitely while only retaining the most recent window of data on disk. + RecordingOptions recordingOptions = + recordingOptionsBuilder + .maxAge(continuousProfilingMaxAge.toMillis() + " ms") + .disk("true") + .build(); + continuousRecording = createRecording(recordingOptions, continuousRecordingConfiguration); + continuousRecording.start(); + logger.info( + "Started continuous JFR recording with circular buffer maxAge of {} seconds", + continuousProfilingMaxAge.getSeconds()); + } catch (IOException | JfrConnectionException e) { + logger.error("Failed to start continuous JFR recording", e); + continuousRecording = null; + } + } + } + + @SuppressWarnings( + "CatchingUnchecked") // catching unchecked exception is necessary for proper error handling + private void captureContinuousRecording( + AlertBreach alertBreach, Instant recordingStart, UploadListener uploadListener) { + synchronized (activeRecordingLock) { + if (continuousRecording == null) { + logger.warn("Profile requested but continuous recording is not running, ignoring request."); + return; + } + + // Enforce global cooldown across all trigger sources + if (globalCooldownSeconds > 0 && timeSource.getNow().isBefore(globalCooldownUntil)) { + logger.info( + "Profile requested (type={}), but global cooldown is active until {}. Ignoring request.", + alertBreach.getType(), + globalCooldownUntil); + return; + } + + File dumpFile; + try { + dumpFile = createJfrFile(continuousProfilingMaxAge); + } catch (IOException e) { + logger.error("Failed to create jfr file", e); + return; + } + + try { + // Dump the current state of the circular buffer, capturing up to maxAge of data. The + // continuous recording keeps running so future requests can be serviced immediately. + continuousRecording.dump(dumpFile.getAbsolutePath()); + } catch (IOException | JfrConnectionException e) { + logger.error("Failed to dump continuous recording", e); + if (dumpFile.exists() && !dumpFile.delete()) { + logger.error("Failed to remove file " + dumpFile.getAbsolutePath()); + } + return; + } + + activeRecordingFile = dumpFile; + } + + try { + logger.info("Uploading continuous recording snapshot"); + uploadService.upload( + alertBreach, recordingStart.toEpochMilli(), activeRecordingFile, uploadListener); + } catch (Exception e) { + logger.error("Failed to upload recording", e); + } catch (Error e) { + // rethrow errors + logger.error("Failed to upload recording", e); + throw e; + } finally { + clearActiveRecording(); + } + } + @Nullable private Recording startRecording(AlertMetricType alertType, Duration duration) { synchronized (activeRecordingLock) { diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/triggers/AlertingSubsystemInit.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/triggers/AlertingSubsystemInit.java index b7c5b3a5b85..3e20ea64dbd 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/triggers/AlertingSubsystemInit.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/triggers/AlertingSubsystemInit.java @@ -55,7 +55,13 @@ public static AlertingSubsystem create( // TODO (trask) delay creation of AlertingSubsystem until after Profiler is created and // initialized? Consumer alertAction = - alert -> alertAction(alert, profiler, diagnosticEngine, telemetryClient); + alert -> + alertAction( + alert, + profiler, + diagnosticEngine, + telemetryClient, + configuration.enableContinuousProfiling); alertingSubsystem = AlertingSubsystem.create( @@ -126,18 +132,28 @@ private static void alertAction( AlertBreach alert, Profiler profiler, DiagnosticEngine diagnosticEngine, - TelemetryClient telemetryClient) { + TelemetryClient telemetryClient, + boolean continuousProfilingEnabled) { if (profiler != null) { // This is an event that the backend specifically looks for to track when a profile is // started sendMessageTelemetry(telemetryClient, "StartProfiler triggered."); + // With continuous profiling the profiler immediately dumps a backward-looking snapshot of the + // circular buffer, so the breach diagnostics (AlertBreach, CGroupData, MachineInfo) must be + // emitted before the dump in order to be captured in the recording. + if (continuousProfilingEnabled && diagnosticEngine != null) { + diagnosticEngine.performDiagnosis(alert); + } + profiler.accept( alert, serviceProfilerIndex -> sendServiceProfilerIndex(serviceProfilerIndex, telemetryClient)); - if (diagnosticEngine != null) { + // With traditional profiling a new forward-looking recording is created, so diagnostics are + // emitted after the recording has started. + if (!continuousProfilingEnabled && diagnosticEngine != null) { diagnosticEngine.performDiagnosis(alert); } } diff --git a/agent/agent-tooling/src/main/resources/com/microsoft/applicationinsights/agent/internal/profiler/diagnostic-cpu-profile.jfc b/agent/agent-tooling/src/main/resources/com/microsoft/applicationinsights/agent/internal/profiler/diagnostic-cpu-profile.jfc index c63ef693d1f..25b792bd7a9 100644 --- a/agent/agent-tooling/src/main/resources/com/microsoft/applicationinsights/agent/internal/profiler/diagnostic-cpu-profile.jfc +++ b/agent/agent-tooling/src/main/resources/com/microsoft/applicationinsights/agent/internal/profiler/diagnostic-cpu-profile.jfc @@ -699,7 +699,7 @@ beginChunk - + true beginChunk diff --git a/agent/agent-tooling/src/main/resources/com/microsoft/applicationinsights/agent/internal/profiler/diagnostic-memory-profile.jfc b/agent/agent-tooling/src/main/resources/com/microsoft/applicationinsights/agent/internal/profiler/diagnostic-memory-profile.jfc index 3a3b373ac6d..1786375188b 100644 --- a/agent/agent-tooling/src/main/resources/com/microsoft/applicationinsights/agent/internal/profiler/diagnostic-memory-profile.jfc +++ b/agent/agent-tooling/src/main/resources/com/microsoft/applicationinsights/agent/internal/profiler/diagnostic-memory-profile.jfc @@ -698,7 +698,7 @@ beginChunk - + true beginChunk diff --git a/agent/agent-tooling/src/main/resources/com/microsoft/applicationinsights/agent/internal/profiler/reduced-cpu-profile.jfc b/agent/agent-tooling/src/main/resources/com/microsoft/applicationinsights/agent/internal/profiler/reduced-cpu-profile.jfc index 472f1b8d78d..4642c09351e 100644 --- a/agent/agent-tooling/src/main/resources/com/microsoft/applicationinsights/agent/internal/profiler/reduced-cpu-profile.jfc +++ b/agent/agent-tooling/src/main/resources/com/microsoft/applicationinsights/agent/internal/profiler/reduced-cpu-profile.jfc @@ -698,7 +698,7 @@ beginChunk - + true beginChunk diff --git a/agent/agent-tooling/src/main/resources/com/microsoft/applicationinsights/agent/internal/profiler/reduced-memory-profile.jfc b/agent/agent-tooling/src/main/resources/com/microsoft/applicationinsights/agent/internal/profiler/reduced-memory-profile.jfc index 3ac17539e95..aedeecc63e7 100644 --- a/agent/agent-tooling/src/main/resources/com/microsoft/applicationinsights/agent/internal/profiler/reduced-memory-profile.jfc +++ b/agent/agent-tooling/src/main/resources/com/microsoft/applicationinsights/agent/internal/profiler/reduced-memory-profile.jfc @@ -698,7 +698,7 @@ beginChunk - + true beginChunk diff --git a/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/profiler/ProfilerContinuousProfilingTest.java b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/profiler/ProfilerContinuousProfilingTest.java new file mode 100644 index 00000000000..a51cc2cc064 --- /dev/null +++ b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/profiler/ProfilerContinuousProfilingTest.java @@ -0,0 +1,99 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.microsoft.applicationinsights.agent.internal.profiler; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; + +import com.microsoft.applicationinsights.agent.internal.configuration.Configuration; +import com.microsoft.applicationinsights.agent.internal.profiler.testutil.TestTimeSource; +import com.microsoft.applicationinsights.agent.internal.profiler.upload.UploadListener; +import com.microsoft.applicationinsights.agent.internal.profiler.upload.UploadService; +import com.microsoft.applicationinsights.alerting.alert.AlertBreach; +import com.microsoft.applicationinsights.alerting.config.AlertConfiguration; +import com.microsoft.applicationinsights.alerting.config.AlertMetricType; +import io.opentelemetry.contrib.jfr.connection.FlightRecorderConnection; +import io.opentelemetry.contrib.jfr.connection.Recording; +import io.opentelemetry.contrib.jfr.connection.RecordingConfiguration; +import io.opentelemetry.contrib.jfr.connection.RecordingOptions; +import java.io.File; +import java.time.Duration; +import java.time.Instant; +import java.util.UUID; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class ProfilerContinuousProfilingTest { + @TempDir File tempDir; + + private final TestTimeSource timeSource = new TestTimeSource(); + private ScheduledExecutorService executor; + + @AfterEach + void tearDown() { + if (executor != null) { + executor.shutdownNow(); + } + } + + private static AlertBreach manualBreach() { + return AlertBreach.builder() + .setType(AlertMetricType.MANUAL) + .setAlertValue(0.0) + .setAlertConfiguration( + AlertConfiguration.builder() + .setType(AlertMetricType.MANUAL) + .setEnabled(true) + .setProfileDurationSeconds(1) + .build()) + .setProfileId(UUID.randomUUID().toString()) + .setCpuMetric(0) + .setMemoryUsage(0) + .build(); + } + + @Test + void profileRequestDumpsRunningContinuousRecordingImmediately() throws Exception { + Configuration.ProfilerConfiguration config = new Configuration.ProfilerConfiguration(); + config.enableContinuousProfiling = true; + config.continuousProfilingMaxAgeSeconds = 60; + config.globalCooldownSeconds = 0; + + Recording continuousRecording = mock(Recording.class); + Profiler profiler = + new Profiler(config, tempDir, timeSource) { + @Override + protected Recording createRecording(RecordingOptions o, RecordingConfiguration c) { + return continuousRecording; + } + }; + + UploadService uploadService = mock(UploadService.class); + FlightRecorderConnection frc = mock(FlightRecorderConnection.class); + executor = Executors.newScheduledThreadPool(1); + profiler.initialize(uploadService, executor, frc); + + // Continuous recording is started up-front and kept running. + verify(continuousRecording).start(); + + timeSource.setNow(Instant.parse("2025-01-01T00:00:00Z")); + UploadListener noOp = index -> {}; + profiler.profileAndUpload(manualBreach(), Duration.ofSeconds(1), noOp); + + // A profile request dumps the current circular buffer and uploads immediately, without + // starting/stopping the recording. + verify(continuousRecording).dump(anyString()); + verify(continuousRecording, never()).stop(); + verify(uploadService).upload(any(), anyLong(), any(File.class), any()); + assertThat(profiler.isRecordingActive()).isFalse(); + } +} diff --git a/smoke-tests/apps/Diagnostics/JfrFileReader/src/main/java/com/microsoft/applicationinsights/jfrfile/JfrFileReader.java b/smoke-tests/apps/Diagnostics/JfrFileReader/src/main/java/com/microsoft/applicationinsights/jfrfile/JfrFileReader.java index 6af5701aad2..6df6ac368ac 100644 --- a/smoke-tests/apps/Diagnostics/JfrFileReader/src/main/java/com/microsoft/applicationinsights/jfrfile/JfrFileReader.java +++ b/smoke-tests/apps/Diagnostics/JfrFileReader/src/main/java/com/microsoft/applicationinsights/jfrfile/JfrFileReader.java @@ -7,6 +7,7 @@ import java.io.IOException; import java.nio.file.Path; import jdk.jfr.EventType; +import jdk.jfr.consumer.RecordedEvent; import jdk.jfr.consumer.RecordingFile; public class JfrFileReader { @@ -17,4 +18,23 @@ public static boolean hasEventOfType(Path jfrFile, String event) throws IOExcept .stream().map(EventType::getName) .anyMatch(event::equals); } + + /** + * Returns true if the recording contains at least one actual event instance of the given type. + * + *

Unlike {@link #hasEventOfType(Path, String)}, which only checks whether the event type is + * registered in the recording metadata, this reads the recorded events and confirms one was + * actually captured. + */ + public static boolean hasEventInstanceOfType(Path jfrFile, String event) throws IOException { + try (RecordingFile recordingFile = new RecordingFile(jfrFile)) { + while (recordingFile.hasMoreEvents()) { + RecordedEvent recordedEvent = recordingFile.readEvent(); + if (recordedEvent.getEventType().getName().equals(event)) { + return true; + } + } + } + return false; + } } diff --git a/smoke-tests/apps/Diagnostics/src/main/java/com/microsoft/applicationinsights/smoketestapp/TestController.java b/smoke-tests/apps/Diagnostics/src/main/java/com/microsoft/applicationinsights/smoketestapp/TestController.java index 4ff77440f75..74b8a711125 100644 --- a/smoke-tests/apps/Diagnostics/src/main/java/com/microsoft/applicationinsights/smoketestapp/TestController.java +++ b/smoke-tests/apps/Diagnostics/src/main/java/com/microsoft/applicationinsights/smoketestapp/TestController.java @@ -59,6 +59,60 @@ public String jfrFileHasDiagnostics() throws Exception { return String.valueOf(false); } + /** + * Verifies that a continuous-profiling recording captured the breach diagnostic events. When a + * profile is requested the profiler immediately dumps the backward-looking circular buffer, so + * {@code AlertBreach}, {@code MachineInfo} and {@code CGroupData} must be emitted before the dump + * in order to be captured in the recording. + */ + @GetMapping("/continuousJfrFileHasDiagnostics") + public String continuousJfrFileHasDiagnostics() throws Exception { + String[] requiredEvents = { + "com.microsoft.applicationinsights.diagnostics.jfr.AlertBreach", + "com.microsoft.applicationinsights.diagnostics.jfr.MachineInfo", + "com.microsoft.applicationinsights.diagnostics.jfr.CGroupData", + }; + + for (int i = 0; i < 60; i++) { + try { + Optional jfrFile = + Files.walk(new File("/tmp/root/applicationinsights").toPath()) + .filter(Files::isRegularFile) + .filter(it -> it.toFile().getName().contains(".jfr")) + .findFirst(); + + if (!jfrFile.isPresent()) { + Thread.sleep(1000, 0); + continue; + } + + Path decompressedFile = decompressFile(jfrFile.get()); + + boolean hasAll = true; + for (String event : requiredEvents) { + if (!com.microsoft.applicationinsights.jfrfile.JfrFileReader.hasEventInstanceOfType( + decompressedFile, event)) { + hasAll = false; + break; + } + } + + if (hasAll) { + return String.valueOf(true); + } + } catch (Exception e) { + // Ignore early exceptions, as to be expected, throw them if they are still happening + // towards the end + if (i > 55) { + throw e; + } + } + Thread.sleep(1000, 0); + } + + return String.valueOf(false); + } + private Path decompressFile(Path jfrFile) { try { byte[] buffer = new byte[1024]; diff --git a/smoke-tests/apps/Diagnostics/src/smokeTest/java/com/microsoft/applicationinsights/smoketest/DiagnosticsTest.java b/smoke-tests/apps/Diagnostics/src/smokeTest/java/com/microsoft/applicationinsights/smoketest/DiagnosticsTest.java index d43ab793d0c..e6641d1e625 100644 --- a/smoke-tests/apps/Diagnostics/src/smokeTest/java/com/microsoft/applicationinsights/smoketest/DiagnosticsTest.java +++ b/smoke-tests/apps/Diagnostics/src/smokeTest/java/com/microsoft/applicationinsights/smoketest/DiagnosticsTest.java @@ -41,4 +41,25 @@ static class Java11Test extends DiagnosticsTest { super(testing); } } + + @UseAgent("applicationinsights-continuous.json") + @Environment(JAVA_11) + static class ContinuousProfilingJava11Test extends DiagnosticsTest { + + @RegisterExtension + static final SmokeTestExtension testing = + BASE_BUILDER.setProfilerEndpoint(ProfilerState.manualprofile).build(); + + ContinuousProfilingJava11Test() { + super(testing); + } + + @Test + @TargetUri("/") + void continuousRecordingHasBreachDiagnostics() throws Exception { + String url = testing.getBaseUrl() + "/continuousJfrFileHasDiagnostics"; + String response = HttpHelper.get(url, "", emptyMap()); + Assertions.assertTrue(Boolean.parseBoolean(response)); + } + } } From a1d4f01be2ab825cae43488d56bac5616c40f458 Mon Sep 17 00:00:00 2001 From: John Oliver <1615532+johnoliver@users.noreply.github.com> Date: Fri, 31 Jul 2026 19:40:45 +0000 Subject: [PATCH 02/11] add MachineStats schema version --- .../diagnostics/jfr/MachineInfo.java | 44 ++++++++++++------- 1 file changed, 27 insertions(+), 17 deletions(-) diff --git a/agent/agent-profiler/agent-diagnostics-jfr/src/main/java/com/microsoft/applicationinsights/diagnostics/jfr/MachineInfo.java b/agent/agent-profiler/agent-diagnostics-jfr/src/main/java/com/microsoft/applicationinsights/diagnostics/jfr/MachineInfo.java index 5f3d815d250..a9944c77830 100644 --- a/agent/agent-profiler/agent-diagnostics-jfr/src/main/java/com/microsoft/applicationinsights/diagnostics/jfr/MachineInfo.java +++ b/agent/agent-profiler/agent-diagnostics-jfr/src/main/java/com/microsoft/applicationinsights/diagnostics/jfr/MachineInfo.java @@ -27,22 +27,23 @@ public class MachineInfo extends Event implements JsonSerializable public static final String NAME = "com.microsoft.applicationinsights.diagnostics.jfr.MachineInfo"; /** - * Legacy calibration field. No longer populated (calibration was removed) but retained — along - * with its accessor and deserialization — so previously-recorded recordings can still be read and - * scored. New recordings emit it as 0. + * Event name emitted by agents prior to the MachineStats->MachineInfo rename. Readers fall + * back to this so previously-recorded recordings (which carry a "MachineStats" event) can still + * be located and scored. */ - private double contextSwitchesPerMs; + public static final String LEGACY_NAME = + "com.microsoft.applicationinsights.diagnostics.jfr.MachineStats"; - private int coreCount; + /** + * Current schema version. Version 2 drops the legacy {@code contextSwitchesPerMs} field; + * recordings produced before the schemaVersion field was added carry schemaVersion 1 (the + * implicit legacy version) and still serialize {@code contextSwitchesPerMs}. + */ + public static final int SCHEMA_VERSION = 2; - public double getContextSwitchesPerMs() { - return contextSwitchesPerMs; - } + private int coreCount; - public MachineInfo setContextSwitchesPerMs(double contextSwitchesPerMs) { - this.contextSwitchesPerMs = contextSwitchesPerMs; - return this; - } + private int schemaVersion; public int getCoreCount() { return coreCount; @@ -53,12 +54,21 @@ public MachineInfo setCoreCount(int coreCount) { return this; } + public int getSchemaVersion() { + return schemaVersion; + } + + public MachineInfo setSchemaVersion(int schemaVersion) { + this.schemaVersion = schemaVersion; + return this; + } + @Override public JsonWriter toJson(JsonWriter jsonWriter) throws IOException { + jsonWriter.writeStartObject(); return jsonWriter - .writeStartObject() - .writeDoubleField("contextSwitchesPerMs", contextSwitchesPerMs) .writeIntField("coreCount", coreCount) + .writeIntField("schemaVersion", schemaVersion) .writeEndObject(); } @@ -70,10 +80,10 @@ public static MachineInfo fromJson(JsonReader jsonReader) throws IOException { while (reader.nextToken() != JsonToken.END_OBJECT) { String fieldName = reader.getFieldName(); reader.nextToken(); - if ("contextSwitchesPerMs".equals(fieldName)) { - deserializedValue.setContextSwitchesPerMs(reader.getDouble()); - } else if ("coreCount".equals(fieldName)) { + if ("coreCount".equals(fieldName)) { deserializedValue.setCoreCount(reader.getInt()); + } else if ("schemaVersion".equals(fieldName)) { + deserializedValue.setSchemaVersion(reader.getInt()); } else { reader.skipChildren(); } From 8acde2437dd0762fd31859cf1b831238902b80b2 Mon Sep 17 00:00:00 2001 From: John Oliver <1615532+johnoliver@users.noreply.github.com> Date: Mon, 3 Aug 2026 11:54:45 +0000 Subject: [PATCH 03/11] Add missing applicationinsights-continuous.json smoke test resource The DiagnosticsTest ContinuousProfilingJava11Test references applicationinsights-continuous.json via @UseAgent, but the resource was never committed. On CI the file is absent, so the continuous profiling config fails to load and all three DiagnosticsTest cases fail (the missing config also leaves the shared test setup in a state that breaks the subsequently-run Java11Test.getJfr). Committing the resource fixes the Diagnostics:DiagnosticsTest build failure. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../applicationinsights-continuous.json | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 smoke-tests/apps/Diagnostics/src/smokeTest/resources/applicationinsights-continuous.json diff --git a/smoke-tests/apps/Diagnostics/src/smokeTest/resources/applicationinsights-continuous.json b/smoke-tests/apps/Diagnostics/src/smokeTest/resources/applicationinsights-continuous.json new file mode 100644 index 00000000000..7ae10f1226c --- /dev/null +++ b/smoke-tests/apps/Diagnostics/src/smokeTest/resources/applicationinsights-continuous.json @@ -0,0 +1,17 @@ +{ + "role" : { + "name" : "testrolename", + "instance" : "testroleinstance" + }, + "sampling" : { + "percentage" : 100 + }, + "preview" : { + "profiler" : { + "enabled" : true, + "enableContinuousProfiling" : true, + "continuousProfilingMaxAgeSeconds" : 120, + "enableDiagnostics" : true + } + } +} From 44b0a922dd5d732aea06b535de04a8d261bda44a Mon Sep 17 00:00:00 2001 From: John Oliver <1615532+johnoliver@users.noreply.github.com> Date: Mon, 3 Aug 2026 12:18:54 +0000 Subject: [PATCH 04/11] Honor configured profile duration for continuous profiling dumps Previously a profile request against a continuous (circular-buffer) recording always dumped the full local continuousProfilingMaxAge window, silently ignoring the portal-/JMX-configured profile duration carried on the alert breach. captureContinuousRecording now resolves the capture window as min(requestedDuration, maxAge): when the requested duration is shorter than the buffer it streams out only the trailing requested window via Recording.getStream(start, end); when it covers (or exceeds) the buffer it keeps using the more robust dump() path. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../agent/internal/profiler/Profiler.java | 60 ++++++++++++++++--- .../ProfilerContinuousProfilingTest.java | 59 ++++++++++++++++-- 2 files changed, 106 insertions(+), 13 deletions(-) diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/Profiler.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/Profiler.java index cf6cf88e00a..1720f360577 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/Profiler.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/Profiler.java @@ -168,7 +168,7 @@ public void updateConfiguration(ProfilerConfiguration newConfig) { void profileAndUpload(AlertBreach alertBreach, Duration duration, UploadListener uploadListener) { Instant recordingStart = timeSource.getNow(); if (continuousProfilingEnabled) { - captureContinuousRecording(alertBreach, recordingStart, uploadListener); + captureContinuousRecording(alertBreach, duration, recordingStart, uploadListener); return; } executeProfile( @@ -208,7 +208,10 @@ private void startContinuousRecordingIfEnabled() { @SuppressWarnings( "CatchingUnchecked") // catching unchecked exception is necessary for proper error handling private void captureContinuousRecording( - AlertBreach alertBreach, Instant recordingStart, UploadListener uploadListener) { + AlertBreach alertBreach, + Duration requestedDuration, + Instant recordingStart, + UploadListener uploadListener) { synchronized (activeRecordingLock) { if (continuousRecording == null) { logger.warn("Profile requested but continuous recording is not running, ignoring request."); @@ -224,18 +227,23 @@ private void captureContinuousRecording( return; } + // The circular buffer only retains up to maxAge of data, so the captured window can never + // exceed maxAge. Honor a shorter portal-/JMX-configured profile duration when one is + // provided, otherwise fall back to the full maxAge window. + Duration captureWindow = resolveContinuousCaptureWindow(requestedDuration); + File dumpFile; try { - dumpFile = createJfrFile(continuousProfilingMaxAge); + dumpFile = createJfrFile(captureWindow); } catch (IOException e) { logger.error("Failed to create jfr file", e); return; } try { - // Dump the current state of the circular buffer, capturing up to maxAge of data. The - // continuous recording keeps running so future requests can be serviced immediately. - continuousRecording.dump(dumpFile.getAbsolutePath()); + // Dump the trailing captureWindow of the circular buffer. The continuous recording keeps + // running so future requests can be serviced immediately. + dumpContinuousRecording(dumpFile, recordingStart, captureWindow); } catch (IOException | JfrConnectionException e) { logger.error("Failed to dump continuous recording", e); if (dumpFile.exists() && !dumpFile.delete()) { @@ -262,6 +270,38 @@ private void captureContinuousRecording( } } + /** + * Resolves the window of buffered data to capture from the continuous recording. The window is + * bounded by the configured continuous profiling maxAge (the circular buffer can hold no more + * than that) but is otherwise driven by the requested profile duration so that a shorter + * portal-/JMX-configured duration is honored rather than silently ignored. + */ + private Duration resolveContinuousCaptureWindow(@Nullable Duration requestedDuration) { + if (requestedDuration == null + || requestedDuration.isZero() + || requestedDuration.isNegative() + || requestedDuration.compareTo(continuousProfilingMaxAge) > 0) { + return continuousProfilingMaxAge; + } + return requestedDuration; + } + + /** + * Writes the trailing {@code captureWindow} of the continuous recording's circular buffer to + * {@code dumpFile}. When the requested window covers the whole buffer the more robust {@link + * Recording#dump(String)} path is used; otherwise only the requested trailing window is streamed + * out so the configured profile duration is respected. + */ + private void dumpContinuousRecording(File dumpFile, Instant recordingEnd, Duration captureWindow) + throws IOException, JfrConnectionException { + if (captureWindow.compareTo(continuousProfilingMaxAge) >= 0) { + continuousRecording.dump(dumpFile.getAbsolutePath()); + return; + } + writeFileFromStream( + continuousRecording, dumpFile, recordingEnd.minus(captureWindow), recordingEnd); + } + @Nullable private Recording startRecording(AlertMetricType alertType, Duration duration) { synchronized (activeRecordingLock) { @@ -409,12 +449,18 @@ private static void closeRecording(Recording recording, File recordingFile) { private static void writeFileFromStream(Recording recording, File recordingFile) throws IOException, JfrConnectionException { + writeFileFromStream(recording, recordingFile, null, null); + } + + private static void writeFileFromStream( + Recording recording, File recordingFile, @Nullable Instant start, @Nullable Instant end) + throws IOException, JfrConnectionException { if (recordingFile.exists()) { recordingFile.delete(); } recordingFile.createNewFile(); - try (BufferedInputStream stream = new BufferedInputStream(recording.getStream(null, null)); + try (BufferedInputStream stream = new BufferedInputStream(recording.getStream(start, end)); FileOutputStream fos = new FileOutputStream(recordingFile)) { int read; byte[] buffer = new byte[10 * 1024]; diff --git a/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/profiler/ProfilerContinuousProfilingTest.java b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/profiler/ProfilerContinuousProfilingTest.java index a51cc2cc064..8010636459a 100644 --- a/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/profiler/ProfilerContinuousProfilingTest.java +++ b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/profiler/ProfilerContinuousProfilingTest.java @@ -7,9 +7,11 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; import com.microsoft.applicationinsights.agent.internal.configuration.Configuration; import com.microsoft.applicationinsights.agent.internal.profiler.testutil.TestTimeSource; @@ -22,7 +24,9 @@ import io.opentelemetry.contrib.jfr.connection.Recording; import io.opentelemetry.contrib.jfr.connection.RecordingConfiguration; import io.opentelemetry.contrib.jfr.connection.RecordingOptions; +import java.io.ByteArrayInputStream; import java.io.File; +import java.nio.charset.StandardCharsets; import java.time.Duration; import java.time.Instant; import java.util.UUID; @@ -45,7 +49,7 @@ void tearDown() { } } - private static AlertBreach manualBreach() { + private static AlertBreach manualBreach(int profileDurationSeconds) { return AlertBreach.builder() .setType(AlertMetricType.MANUAL) .setAlertValue(0.0) @@ -53,7 +57,7 @@ private static AlertBreach manualBreach() { AlertConfiguration.builder() .setType(AlertMetricType.MANUAL) .setEnabled(true) - .setProfileDurationSeconds(1) + .setProfileDurationSeconds(profileDurationSeconds) .build()) .setProfileId(UUID.randomUUID().toString()) .setCpuMetric(0) @@ -62,13 +66,16 @@ private static AlertBreach manualBreach() { } @Test - void profileRequestDumpsRunningContinuousRecordingImmediately() throws Exception { + void profileRequestCapturesRequestedWindowOfContinuousRecording() throws Exception { Configuration.ProfilerConfiguration config = new Configuration.ProfilerConfiguration(); config.enableContinuousProfiling = true; config.continuousProfilingMaxAgeSeconds = 60; config.globalCooldownSeconds = 0; Recording continuousRecording = mock(Recording.class); + when(continuousRecording.getStream(any(), any())) + .thenReturn(new ByteArrayInputStream("jfr".getBytes(StandardCharsets.UTF_8))); + Profiler profiler = new Profiler(config, tempDir, timeSource) { @Override @@ -85,13 +92,53 @@ protected Recording createRecording(RecordingOptions o, RecordingConfiguration c // Continuous recording is started up-front and kept running. verify(continuousRecording).start(); + Instant now = Instant.parse("2025-01-01T00:00:00Z"); + timeSource.setNow(now); + UploadListener noOp = index -> {}; + + // The portal-/JMX-configured duration (10s) is shorter than the 60s buffer, so only the + // trailing 10s window of the circular buffer is captured. + profiler.profileAndUpload(manualBreach(10), Duration.ofSeconds(10), noOp); + + verify(continuousRecording).getStream(eq(now.minusSeconds(10)), eq(now)); + verify(continuousRecording, never()).dump(anyString()); + verify(continuousRecording, never()).stop(); + verify(uploadService).upload(any(), anyLong(), any(File.class), any()); + assertThat(profiler.isRecordingActive()).isFalse(); + } + + @Test + void profileRequestDumpsWholeBufferWhenRequestedDurationExceedsMaxAge() throws Exception { + Configuration.ProfilerConfiguration config = new Configuration.ProfilerConfiguration(); + config.enableContinuousProfiling = true; + config.continuousProfilingMaxAgeSeconds = 60; + config.globalCooldownSeconds = 0; + + Recording continuousRecording = mock(Recording.class); + Profiler profiler = + new Profiler(config, tempDir, timeSource) { + @Override + protected Recording createRecording(RecordingOptions o, RecordingConfiguration c) { + return continuousRecording; + } + }; + + UploadService uploadService = mock(UploadService.class); + FlightRecorderConnection frc = mock(FlightRecorderConnection.class); + executor = Executors.newScheduledThreadPool(1); + profiler.initialize(uploadService, executor, frc); + + verify(continuousRecording).start(); + timeSource.setNow(Instant.parse("2025-01-01T00:00:00Z")); UploadListener noOp = index -> {}; - profiler.profileAndUpload(manualBreach(), Duration.ofSeconds(1), noOp); - // A profile request dumps the current circular buffer and uploads immediately, without - // starting/stopping the recording. + // The requested duration (90s) exceeds the 60s buffer, so the whole circular buffer is dumped + // via the more robust dump() path. + profiler.profileAndUpload(manualBreach(90), Duration.ofSeconds(90), noOp); + verify(continuousRecording).dump(anyString()); + verify(continuousRecording, never()).getStream(any(), any()); verify(continuousRecording, never()).stop(); verify(uploadService).upload(any(), anyLong(), any(File.class), any()); assertThat(profiler.isRecordingActive()).isFalse(); From ff50bee430cdb144870e2510de661684ff4bac11 Mon Sep 17 00:00:00 2001 From: John Oliver <1615532+johnoliver@users.noreply.github.com> Date: Mon, 3 Aug 2026 12:28:25 +0000 Subject: [PATCH 05/11] Fix startup ordering race that tears down continuous diagnostic emitters A breach processed during startup, before startContinuousDiagnostics has run, would schedule a diagnostic-cycle shutdown. Once continuous profiling started and registered the continuous emitters, that stale scheduled shutdown fired and permanently tore them down. Guard the scheduled teardown under a lifecycle lock that checks whether continuous diagnostics has since been enabled, and synchronize the start/stop transitions so the check-and-stop cannot interleave with a concurrent startContinuousDiagnostics. Refactor the JFR-touching helpers into overridable instance methods so the race can be covered by a deterministic unit test. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../agent-diagnostics/build.gradle.kts | 5 + .../CodeOptimizerDiagnosticEngineJfr.java | 76 +++++++--- .../CodeOptimizerDiagnosticEngineJfrTest.java | 141 ++++++++++++++++++ 3 files changed, 199 insertions(+), 23 deletions(-) create mode 100644 agent/agent-profiler/agent-diagnostics/src/test/java/com/microsoft/applicationinsights/diagnostics/appinsights/CodeOptimizerDiagnosticEngineJfrTest.java diff --git a/agent/agent-profiler/agent-diagnostics/build.gradle.kts b/agent/agent-profiler/agent-diagnostics/build.gradle.kts index 8aedc1966c9..3e7350428a3 100644 --- a/agent/agent-profiler/agent-diagnostics/build.gradle.kts +++ b/agent/agent-profiler/agent-diagnostics/build.gradle.kts @@ -18,4 +18,9 @@ dependencies { compileOnly("com.google.auto.service:auto-service") annotationProcessor("com.google.auto.service:auto-service") + + testImplementation("org.assertj:assertj-core") + testImplementation("org.mockito:mockito-core") + testImplementation("org.gradle.jfr.polyfill:jfr-polyfill:1.0.2") + testImplementation("com.azure:azure-json") } diff --git a/agent/agent-profiler/agent-diagnostics/src/main/java/com/microsoft/applicationinsights/diagnostics/appinsights/CodeOptimizerDiagnosticEngineJfr.java b/agent/agent-profiler/agent-diagnostics/src/main/java/com/microsoft/applicationinsights/diagnostics/appinsights/CodeOptimizerDiagnosticEngineJfr.java index fd60d5c4fe3..2a6483e4aca 100644 --- a/agent/agent-profiler/agent-diagnostics/src/main/java/com/microsoft/applicationinsights/diagnostics/appinsights/CodeOptimizerDiagnosticEngineJfr.java +++ b/agent/agent-profiler/agent-diagnostics/src/main/java/com/microsoft/applicationinsights/diagnostics/appinsights/CodeOptimizerDiagnosticEngineJfr.java @@ -43,6 +43,11 @@ public class CodeOptimizerDiagnosticEngineJfr implements DiagnosticEngine { // and must not be torn down by an individual performDiagnosis cycle. private final AtomicBoolean continuous = new AtomicBoolean(false); + // Guards transitions of the continuous flag against the teardown performed at the end of a + // (non-continuous) diagnostic cycle, so that a breach processed during startup cannot tear down + // the continuously-registered emitters once continuous profiling has been enabled. + private final Object continuousLifecycleLock = new Object(); + public CodeOptimizerDiagnosticEngineJfr( ScheduledExecutorService executorService, Path cgroupBasePath) { this.executorService = executorService; @@ -51,7 +56,7 @@ public CodeOptimizerDiagnosticEngineJfr( @Override public void init(int thisPid) { - if (!CodeOptimizerDiagnosticsJfrInit.isOsSupported()) { + if (!isOsSupported()) { logger.warn("Code Optimizer diagnostics is not supported on this operating system"); return; } @@ -63,40 +68,52 @@ public void init(int thisPid) { logger.debug("Code Optimizer Diagnostic Engine Initialised"); } - private static void startDiagnosticCycle(int thisPid, Path cgroupBasePath) { + // visible for testing + protected boolean isOsSupported() { + return CodeOptimizerDiagnosticsJfrInit.isOsSupported(); + } + + // visible for testing + protected void startDiagnosticCycle() { logger.debug("Starting Code Optimizer Diagnostic Cycle"); - CodeOptimizerDiagnosticsJfrInit.initFeature(thisPid, cgroupBasePath); - CodeOptimizerDiagnosticsJfrInit.start(thisPid, cgroupBasePath); + int pid = thisPid.get(); + CodeOptimizerDiagnosticsJfrInit.initFeature(pid, cgroupBasePath); + CodeOptimizerDiagnosticsJfrInit.start(pid, cgroupBasePath); } - private static void endDiagnosticCycle() { + // visible for testing + protected void endDiagnosticCycle() { logger.debug("Ending Code Optimizer Diagnostic Cycle"); CodeOptimizerDiagnosticsJfrInit.stop(); } @Override public void startContinuousDiagnostics() { - if (!CodeOptimizerDiagnosticsJfrInit.isOsSupported()) { + if (!isOsSupported()) { logger.warn("Code Optimizer diagnostics is not supported on this operating system"); return; } - continuous.set(true); - logger.debug("Starting continuous Code Optimizer diagnostics"); - // Registers the periodic diagnostic emitters (Telemetry, CGroupData) so they continuously - // populate the continuous profiling circular buffer. - startDiagnosticCycle(thisPid.get(), cgroupBasePath); + synchronized (continuousLifecycleLock) { + continuous.set(true); + logger.debug("Starting continuous Code Optimizer diagnostics"); + // Registers the periodic diagnostic emitters (Telemetry, CGroupData) so they continuously + // populate the continuous profiling circular buffer. + startDiagnosticCycle(); + } } @Override public void stopContinuousDiagnostics() { - if (!CodeOptimizerDiagnosticsJfrInit.isOsSupported()) { + if (!isOsSupported()) { return; } - continuous.set(false); - logger.debug("Stopping continuous Code Optimizer diagnostics"); - endDiagnosticCycle(); + synchronized (continuousLifecycleLock) { + continuous.set(false); + logger.debug("Stopping continuous Code Optimizer diagnostics"); + endDiagnosticCycle(); + } } @Override @@ -108,7 +125,7 @@ public Future> performDiagnosis(AlertBreach alert) { CompletableFuture> diagnosisResultCompletableFuture = new CompletableFuture<>(); try { - emitInfo(alert, cgroupBasePath); + emitInfo(alert); diagnosisResultCompletableFuture.complete(null); } catch (RuntimeException e) { diagnosisResultCompletableFuture.completeExceptionally(e); @@ -120,13 +137,13 @@ public Future> performDiagnosis(AlertBreach alert) { new CompletableFuture<>(); try { if (semaphore.tryAcquire(SEMAPHORE_TIMEOUT_IN_SEC, TimeUnit.SECONDS)) { - emitInfo(alert, cgroupBasePath); + emitInfo(alert); long profileDurationInSec = alert.getAlertConfiguration().getProfileDurationSeconds(); long end = profileDurationInSec - TIME_BEFORE_END_OF_PROFILE_TO_EMIT_EVENT; - startDiagnosticCycle(thisPid.get(), cgroupBasePath); + startDiagnosticCycle(); scheduleEmittingAlertBreachEvent(alert, end); @@ -151,13 +168,25 @@ private void scheduleShutdown( executorService.schedule( () -> { try { - emitInfo(alert, cgroupBasePath); + emitInfo(alert); // We do not return a result atm diagnosisResultCompletableFuture.complete(null); - logger.debug("Shutting down diagnostic cycle"); - endDiagnosticCycle(); + // Only tear down the diagnostic cycle if continuous diagnostics has not been enabled in + // the meantime. If a breach is processed during startup, before + // startContinuousDiagnostics + // has run, this shutdown would otherwise permanently stop the continuously-registered + // emitters once continuous profiling starts. The lock ensures the check-and-stop cannot + // interleave with a concurrent startContinuousDiagnostics. + synchronized (continuousLifecycleLock) { + if (continuous.get()) { + logger.debug("Continuous diagnostics is active; leaving diagnostic cycle running"); + } else { + logger.debug("Shutting down diagnostic cycle"); + endDiagnosticCycle(); + } + } } catch (RuntimeException e) { logger.error("Failed to shutdown cleanly", e); } finally { @@ -173,7 +202,7 @@ private void scheduleEmittingAlertBreachEvent(AlertBreach alert, long end) { executorService.schedule( () -> { try { - emitInfo(alert, cgroupBasePath); + emitInfo(alert); } catch (RuntimeException e) { logger.error("Failed to emit breach", e); } @@ -182,7 +211,8 @@ private void scheduleEmittingAlertBreachEvent(AlertBreach alert, long end) { TimeUnit.SECONDS); } - private static void emitInfo(AlertBreach alert, Path cgroupBasePath) { + // visible for testing + protected void emitInfo(AlertBreach alert) { logger.debug("Emitting Code Optimizer Diagnostic Event"); emitAlertBreachJfrEvent(alert); CodeOptimizerDiagnosticsJfrInit.emitCGroupData(cgroupBasePath); diff --git a/agent/agent-profiler/agent-diagnostics/src/test/java/com/microsoft/applicationinsights/diagnostics/appinsights/CodeOptimizerDiagnosticEngineJfrTest.java b/agent/agent-profiler/agent-diagnostics/src/test/java/com/microsoft/applicationinsights/diagnostics/appinsights/CodeOptimizerDiagnosticEngineJfrTest.java new file mode 100644 index 00000000000..1f13cf42895 --- /dev/null +++ b/agent/agent-profiler/agent-diagnostics/src/test/java/com/microsoft/applicationinsights/diagnostics/appinsights/CodeOptimizerDiagnosticEngineJfrTest.java @@ -0,0 +1,141 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.microsoft.applicationinsights.diagnostics.appinsights; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.microsoft.applicationinsights.alerting.alert.AlertBreach; +import com.microsoft.applicationinsights.alerting.config.AlertConfiguration; +import com.microsoft.applicationinsights.alerting.config.AlertMetricType; +import java.nio.file.Paths; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; + +class CodeOptimizerDiagnosticEngineJfrTest { + + /** + * Engine subclass that avoids the real JFR subsystem and simply records how many times the + * diagnostic cycle is started/ended so tests can assert on lifecycle behavior. + */ + private static class RecordingEngine extends CodeOptimizerDiagnosticEngineJfr { + final AtomicInteger startCycleCount = new AtomicInteger(); + final AtomicInteger endCycleCount = new AtomicInteger(); + final AtomicInteger emitInfoCount = new AtomicInteger(); + + RecordingEngine(ScheduledExecutorService executorService) { + super(executorService, Paths.get("/")); + } + + @Override + protected boolean isOsSupported() { + return true; + } + + @Override + protected void startDiagnosticCycle() { + startCycleCount.incrementAndGet(); + } + + @Override + protected void endDiagnosticCycle() { + endCycleCount.incrementAndGet(); + } + + @Override + protected void emitInfo(AlertBreach alert) { + emitInfoCount.incrementAndGet(); + } + } + + private static AlertBreach manualBreach(int profileDurationSeconds) { + return AlertBreach.builder() + .setType(AlertMetricType.MANUAL) + .setAlertValue(0.0) + .setAlertConfiguration( + AlertConfiguration.builder() + .setType(AlertMetricType.MANUAL) + .setEnabled(true) + .setProfileDurationSeconds(profileDurationSeconds) + .build()) + .setProfileId(UUID.randomUUID().toString()) + .setCpuMetric(0) + .setMemoryUsage(0) + .build(); + } + + private static Runnable captureScheduledShutdown( + ScheduledExecutorService executor, long expectedDelaySeconds) { + ArgumentCaptor runnableCaptor = ArgumentCaptor.forClass(Runnable.class); + ArgumentCaptor delayCaptor = ArgumentCaptor.forClass(Long.class); + verify(executor, times(2)) + .schedule(runnableCaptor.capture(), delayCaptor.capture(), eq(TimeUnit.SECONDS)); + List runnables = runnableCaptor.getAllValues(); + List delays = delayCaptor.getAllValues(); + for (int i = 0; i < delays.size(); i++) { + if (delays.get(i) == expectedDelaySeconds) { + return runnables.get(i); + } + } + throw new AssertionError("No task scheduled with delay " + expectedDelaySeconds); + } + + @Test + void breachDuringStartupDoesNotTearDownContinuousEmitters() { + ScheduledExecutorService executor = mock(ScheduledExecutorService.class); + when(executor.schedule(any(Runnable.class), anyLong(), any(TimeUnit.class))).thenReturn(null); + + RecordingEngine engine = new RecordingEngine(executor); + + // A breach is processed during startup, before continuous diagnostics has been enabled. + int profileDurationSeconds = 60; + long end = + profileDurationSeconds + - CodeOptimizerDiagnosticEngineJfr.TIME_BEFORE_END_OF_PROFILE_TO_EMIT_EVENT; + engine.performDiagnosis(manualBreach(profileDurationSeconds)); + + Runnable shutdown = captureScheduledShutdown(executor, end); + + // Continuous diagnostics is enabled after the breach was scheduled but before the stale + // shutdown fires. + engine.startContinuousDiagnostics(); + + // The stale shutdown from the initial breach now fires. + shutdown.run(); + + // It must NOT tear down the continuously-registered emitters. + assertThat(engine.endCycleCount.get()).isZero(); + } + + @Test + void breachWithoutContinuousDiagnosticsTearsDownCycle() { + ScheduledExecutorService executor = mock(ScheduledExecutorService.class); + when(executor.schedule(any(Runnable.class), anyLong(), any(TimeUnit.class))).thenReturn(null); + + RecordingEngine engine = new RecordingEngine(executor); + + int profileDurationSeconds = 60; + long end = + profileDurationSeconds + - CodeOptimizerDiagnosticEngineJfr.TIME_BEFORE_END_OF_PROFILE_TO_EMIT_EVENT; + engine.performDiagnosis(manualBreach(profileDurationSeconds)); + + Runnable shutdown = captureScheduledShutdown(executor, end); + shutdown.run(); + + // With no continuous diagnostics, the cycle is torn down as before. + assertThat(engine.endCycleCount.get()).isEqualTo(1); + } +} From a2493a8222291ab8e064785121765515f7794f4c Mon Sep 17 00:00:00 2001 From: John Oliver <1615532+johnoliver@users.noreply.github.com> Date: Mon, 3 Aug 2026 12:52:45 +0000 Subject: [PATCH 06/11] Restore null guard for NoOpProcessDumper in getThisProcess NoOpProcessDumper.thisProcess() returns null on operating systems that do not support diagnostics. The continuous profiling refactor dropped the guard around the process dumper result, so getThisProcess() would NPE on thisProcess.getPid(). Restore a guard that returns null when no process is available, matching the previous defensive behavior. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../diagnostics/jfr/SystemStatsProvider.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/agent/agent-profiler/agent-diagnostics/src/main/java/com/microsoft/applicationinsights/diagnostics/jfr/SystemStatsProvider.java b/agent/agent-profiler/agent-diagnostics/src/main/java/com/microsoft/applicationinsights/diagnostics/jfr/SystemStatsProvider.java index 8750b7d6056..dd4b4acbd6a 100644 --- a/agent/agent-profiler/agent-diagnostics/src/main/java/com/microsoft/applicationinsights/diagnostics/jfr/SystemStatsProvider.java +++ b/agent/agent-profiler/agent-diagnostics/src/main/java/com/microsoft/applicationinsights/diagnostics/jfr/SystemStatsProvider.java @@ -141,6 +141,10 @@ private static Process getThisProcess() { processDumper.poll(); Process thisProcess = processDumper.thisProcess(); + if (thisProcess == null) { + // e.g. NoOpProcessDumper on unsupported operating systems returns no process + return null; + } processDumper.closeProcesses(Collections.singletonList(thisProcess.getPid())); return thisProcess; }); From 60e7e0416ae42979101d9bed3480f0efff447653 Mon Sep 17 00:00:00 2001 From: John Oliver <1615532+johnoliver@users.noreply.github.com> Date: Mon, 3 Aug 2026 13:05:18 +0000 Subject: [PATCH 07/11] Fix continuous profiling timestamp, concurrency and config robustness Addresses several review findings on the continuous profiling dump path: - Timestamp the uploaded profile (and encode the file name) at the start of the captured window (now - captureWindow) instead of at dump time, so the profile lands at the correct point on the portal timeline (B4). - Keep the dumped file in a local variable and advance the global cooldown inside the recording lock, so concurrent triggers cannot publish or delete each other's files via the shared activeRecordingFile field (B7). - Use a dedicated RecordingOptions.Builder for the continuous recording so the maxAge/disk options no longer permanently mutate the builder shared with on-demand recordings (S4). - Clamp a non-positive continuousProfilingMaxAgeSeconds to the default rather than passing '0 ms'/'-1000 ms' straight to JFR (C4). - Reuse the CPU recording configuration for the continuous recording instead of opening a second stream on the same .jfc resource (B2, partial). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../agent/internal/profiler/Profiler.java | 73 ++++++++++++++----- .../ProfilerContinuousProfilingTest.java | 10 ++- 2 files changed, 62 insertions(+), 21 deletions(-) diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/Profiler.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/Profiler.java index 1720f360577..bb0d814ddaf 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/Profiler.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/Profiler.java @@ -48,6 +48,8 @@ public class Profiler { private static final Logger logger = LoggerFactory.getLogger(Profiler.class); + private static final Duration DEFAULT_CONTINUOUS_PROFILING_MAX_AGE = Duration.ofMinutes(2); + // service execution context private ScheduledExecutorService scheduledExecutorService; @@ -113,11 +115,25 @@ public Profiler(Configuration.ProfilerConfiguration config, File tempDir, TimeSo spanRecordingConfiguration = AlternativeJfrConfigurations.getSpanProfileConfig(config); manualRecordingConfiguration = AlternativeJfrConfigurations.getManualProfileConfig(config); continuousProfilingEnabled = config.enableContinuousProfiling; - continuousProfilingMaxAge = Duration.ofSeconds(config.continuousProfilingMaxAgeSeconds); - continuousRecordingConfiguration = AlternativeJfrConfigurations.getCpuProfileConfig(config); + continuousProfilingMaxAge = + resolveContinuousProfilingMaxAge(config.continuousProfilingMaxAgeSeconds); + // Continuous profiling uses a single always-on recording, so it can only carry one JFC. Reuse + // the CPU configuration rather than opening a second stream on the same resource. + continuousRecordingConfiguration = cpuRecordingConfiguration; temporaryDirectory = tempDir; } + private static Duration resolveContinuousProfilingMaxAge(int continuousProfilingMaxAgeSeconds) { + if (continuousProfilingMaxAgeSeconds <= 0) { + logger.warn( + "continuousProfilingMaxAgeSeconds must be positive but was {}; falling back to {}s", + continuousProfilingMaxAgeSeconds, + DEFAULT_CONTINUOUS_PROFILING_MAX_AGE.getSeconds()); + return DEFAULT_CONTINUOUS_PROFILING_MAX_AGE; + } + return Duration.ofSeconds(continuousProfilingMaxAgeSeconds); + } + /** * Call init before run. * @@ -188,8 +204,10 @@ private void startContinuousRecordingIfEnabled() { try { // A continuous recording uses a circular buffer bounded by maxAge and no duration, so it // runs indefinitely while only retaining the most recent window of data on disk. + // Use a dedicated builder so the maxAge/disk options for the continuous circular buffer do + // not mutate the shared recordingOptionsBuilder used by on-demand recordings. RecordingOptions recordingOptions = - recordingOptionsBuilder + new RecordingOptions.Builder() .maxAge(continuousProfilingMaxAge.toMillis() + " ms") .disk("true") .build(); @@ -210,8 +228,10 @@ private void startContinuousRecordingIfEnabled() { private void captureContinuousRecording( AlertBreach alertBreach, Duration requestedDuration, - Instant recordingStart, + Instant recordingEnd, UploadListener uploadListener) { + File dumpFile; + Instant bufferStart; synchronized (activeRecordingLock) { if (continuousRecording == null) { logger.warn("Profile requested but continuous recording is not running, ignoring request."); @@ -232,9 +252,14 @@ private void captureContinuousRecording( // provided, otherwise fall back to the full maxAge window. Duration captureWindow = resolveContinuousCaptureWindow(requestedDuration); - File dumpFile; + // The dumped buffer covers [recordingEnd - captureWindow, recordingEnd]. Use the start of + // that + // window as the profile timestamp and file name so the profile is indexed at the point the + // data actually begins rather than at dump time. + bufferStart = recordingEnd.minus(captureWindow); + try { - dumpFile = createJfrFile(captureWindow); + dumpFile = createJfrFile(bufferStart, recordingEnd); } catch (IOException e) { logger.error("Failed to create jfr file", e); return; @@ -243,7 +268,7 @@ private void captureContinuousRecording( try { // Dump the trailing captureWindow of the circular buffer. The continuous recording keeps // running so future requests can be serviced immediately. - dumpContinuousRecording(dumpFile, recordingStart, captureWindow); + dumpContinuousRecording(dumpFile, recordingEnd, captureWindow); } catch (IOException | JfrConnectionException e) { logger.error("Failed to dump continuous recording", e); if (dumpFile.exists() && !dumpFile.delete()) { @@ -252,13 +277,14 @@ private void captureContinuousRecording( return; } - activeRecordingFile = dumpFile; + // Start the global cooldown while still holding the lock so concurrent triggers are rejected + // before they can dump another snapshot. + startGlobalCooldown(); } try { logger.info("Uploading continuous recording snapshot"); - uploadService.upload( - alertBreach, recordingStart.toEpochMilli(), activeRecordingFile, uploadListener); + uploadService.upload(alertBreach, bufferStart.toEpochMilli(), dumpFile, uploadListener); } catch (Exception e) { logger.error("Failed to upload recording", e); } catch (Error e) { @@ -266,7 +292,9 @@ private void captureContinuousRecording( logger.error("Failed to upload recording", e); throw e; } finally { - clearActiveRecording(); + if (dumpFile.exists() && !dumpFile.delete()) { + logger.error("Failed to remove file " + dumpFile.getAbsolutePath()); + } } } @@ -476,10 +504,7 @@ void clearActiveRecording() { activeRecording = null; // Start global cooldown now that the recording is complete - if (globalCooldownSeconds > 0) { - globalCooldownUntil = timeSource.getNow().plusSeconds(globalCooldownSeconds); - logger.debug("Global profile cooldown active until {}", globalCooldownUntil); - } + startGlobalCooldown(); // delete uploaded profile if (activeRecordingFile != null && activeRecordingFile.exists()) { @@ -491,6 +516,14 @@ void clearActiveRecording() { } } + // Advances the global cooldown window. Callers must hold activeRecordingLock. + private void startGlobalCooldown() { + if (globalCooldownSeconds > 0) { + globalCooldownUntil = timeSource.getNow().plusSeconds(globalCooldownSeconds); + logger.debug("Global profile cooldown active until {}", globalCooldownUntil); + } + } + // visible for testing Instant getGlobalCooldownUntil() { return globalCooldownUntil; @@ -506,6 +539,13 @@ boolean isRecordingActive() { /** Dump JFR profile to file. */ // visible for testing protected File createJfrFile(Duration duration) throws IOException { + Instant recordingStart = timeSource.getNow(); + return createJfrFile(recordingStart, recordingStart.plus(duration)); + } + + /** Create a JFR file whose name encodes the window of data it contains. */ + // visible for testing + protected File createJfrFile(Instant recordingStart, Instant recordingEnd) throws IOException { if (!temporaryDirectory.exists()) { if (!temporaryDirectory.mkdirs()) { throw new IOException( @@ -513,9 +553,6 @@ protected File createJfrFile(Duration duration) throws IOException { } } - Instant recordingStart = timeSource.getNow(); - Instant recordingEnd = recordingStart.plus(duration); - return new File( temporaryDirectory, "recording_" + recordingStart.toEpochMilli() + "-" + recordingEnd.toEpochMilli() + ".jfr"); diff --git a/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/profiler/ProfilerContinuousProfilingTest.java b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/profiler/ProfilerContinuousProfilingTest.java index 8010636459a..a3c91f6e08b 100644 --- a/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/profiler/ProfilerContinuousProfilingTest.java +++ b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/profiler/ProfilerContinuousProfilingTest.java @@ -5,7 +5,6 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; @@ -103,7 +102,9 @@ protected Recording createRecording(RecordingOptions o, RecordingConfiguration c verify(continuousRecording).getStream(eq(now.minusSeconds(10)), eq(now)); verify(continuousRecording, never()).dump(anyString()); verify(continuousRecording, never()).stop(); - verify(uploadService).upload(any(), anyLong(), any(File.class), any()); + // The profile is timestamped at the start of the captured window (now - 10s), not at dump time. + verify(uploadService) + .upload(any(), eq(now.minusSeconds(10).toEpochMilli()), any(File.class), any()); assertThat(profiler.isRecordingActive()).isFalse(); } @@ -131,6 +132,7 @@ protected Recording createRecording(RecordingOptions o, RecordingConfiguration c verify(continuousRecording).start(); timeSource.setNow(Instant.parse("2025-01-01T00:00:00Z")); + Instant now = Instant.parse("2025-01-01T00:00:00Z"); UploadListener noOp = index -> {}; // The requested duration (90s) exceeds the 60s buffer, so the whole circular buffer is dumped @@ -140,7 +142,9 @@ protected Recording createRecording(RecordingOptions o, RecordingConfiguration c verify(continuousRecording).dump(anyString()); verify(continuousRecording, never()).getStream(any(), any()); verify(continuousRecording, never()).stop(); - verify(uploadService).upload(any(), anyLong(), any(File.class), any()); + // The captured window is the whole 60s buffer, so the profile is timestamped at now - 60s. + verify(uploadService) + .upload(any(), eq(now.minusSeconds(60).toEpochMilli()), any(File.class), any()); assertThat(profiler.isRecordingActive()).isFalse(); } } From 634d365e80d6c96909812c0722a93464b53404c5 Mon Sep 17 00:00:00 2001 From: John Oliver <1615532+johnoliver@users.noreply.github.com> Date: Mon, 3 Aug 2026 13:05:28 +0000 Subject: [PATCH 08/11] Address continuous profiling review follow-ups (logging, docs, test app) - Log the failure when emitting continuous diagnostic breach information, since the returned future is discarded by the caller and the error would otherwise be swallowed (S3). - Drop the redundant inner init() guard in SystemStatsProvider that keyed off a singleton it populates itself; init() is already single-shot via the initialised flag (R5). - Close the Files.walk stream via try-with-resources and select the most recently modified .jfr file in the Diagnostics smoke-test controller, so continuous profiling's extra dumps cannot cause a stale-file false green; extract the shared polling loop (S5). - Document enableContinuousProfiling / continuousProfilingMaxAgeSeconds in docs/README.md, including their limitations, and add CHANGELOG entries for the feature and the MachineStats -> MachineInfo event rename (D1, D2). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- CHANGELOG.md | 13 +++ .../CodeOptimizerDiagnosticEngineJfr.java | 3 + .../diagnostics/jfr/SystemStatsProvider.java | 19 ++-- docs/README.md | 21 ++++ .../smoketestapp/TestController.java | 100 +++++++++--------- 5 files changed, 94 insertions(+), 62 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 471be016d55..5325ecc93b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,19 @@ ## Unreleased +### Enhancements + +* Add continuous profiling (`enableContinuousProfiling`, `continuousProfilingMaxAgeSeconds`) which + keeps a single JFR recording running in a circular buffer so profile requests dump the most recent + window of data immediately + ([#4807](https://github.com/microsoft/ApplicationInsights-Java/pull/4807)) + +### Breaking changes + +* Rename the `MachineStats` JFR diagnostic event to `MachineInfo`. Custom `.jfc` files that enable + `com.microsoft.applicationinsights.diagnostics.jfr.MachineStats` must be updated to the new event + name ([#4807](https://github.com/microsoft/ApplicationInsights-Java/pull/4807)) + ## Version 3.7.9 GA (06/18/2026) ### Enhancements diff --git a/agent/agent-profiler/agent-diagnostics/src/main/java/com/microsoft/applicationinsights/diagnostics/appinsights/CodeOptimizerDiagnosticEngineJfr.java b/agent/agent-profiler/agent-diagnostics/src/main/java/com/microsoft/applicationinsights/diagnostics/appinsights/CodeOptimizerDiagnosticEngineJfr.java index 2a6483e4aca..540f71c5372 100644 --- a/agent/agent-profiler/agent-diagnostics/src/main/java/com/microsoft/applicationinsights/diagnostics/appinsights/CodeOptimizerDiagnosticEngineJfr.java +++ b/agent/agent-profiler/agent-diagnostics/src/main/java/com/microsoft/applicationinsights/diagnostics/appinsights/CodeOptimizerDiagnosticEngineJfr.java @@ -128,6 +128,9 @@ public Future> performDiagnosis(AlertBreach alert) { emitInfo(alert); diagnosisResultCompletableFuture.complete(null); } catch (RuntimeException e) { + // The caller discards the returned future, so log here to avoid silently swallowing the + // failure to emit breach diagnostics. + logger.error("Failed to emit continuous diagnostic breach information", e); diagnosisResultCompletableFuture.completeExceptionally(e); } return diagnosisResultCompletableFuture; diff --git a/agent/agent-profiler/agent-diagnostics/src/main/java/com/microsoft/applicationinsights/diagnostics/jfr/SystemStatsProvider.java b/agent/agent-profiler/agent-diagnostics/src/main/java/com/microsoft/applicationinsights/diagnostics/jfr/SystemStatsProvider.java index dd4b4acbd6a..87830bd1819 100644 --- a/agent/agent-profiler/agent-diagnostics/src/main/java/com/microsoft/applicationinsights/diagnostics/jfr/SystemStatsProvider.java +++ b/agent/agent-profiler/agent-diagnostics/src/main/java/com/microsoft/applicationinsights/diagnostics/jfr/SystemStatsProvider.java @@ -57,16 +57,15 @@ public static void init(int thisPid, Path cgroupBasePath) { if (initialised.compareAndSet(false, true)) { singletons.put(ThisPidSupplier.class, new AtomicReference<>((ThisPidSupplier) () -> thisPid)); - if (singletons.get(MachineInfo.class) == null) { - try { - getMachineInfo(); - getCGroupData(cgroupBasePath); - - // Close until needed - close(); - } catch (RuntimeException e) { - logger.error("Failed to initialise Code Optimizer", e); - } + try { + // Eagerly warm up the machine and cgroup singletons, then release resources until needed. + getMachineInfo(); + getCGroupData(cgroupBasePath); + + // Close until needed + close(); + } catch (RuntimeException e) { + logger.error("Failed to initialise Code Optimizer", e); } } } diff --git a/docs/README.md b/docs/README.md index 0574b30ca65..2e9d6a7c352 100644 --- a/docs/README.md +++ b/docs/README.md @@ -96,6 +96,8 @@ Additionally, a number of parameters can be configured using environment variabl "manualTriggeredSettings": "profile-without-env-data", "globalCooldownSeconds": 120, "enableProfilerControlMBean": false, + "enableContinuousProfiling": false, + "continuousProfilingMaxAgeSeconds": 120, "manualTrigger": { "enabled": false, "filePath": "applicationinsights-agent-profile-trigger", @@ -135,6 +137,25 @@ cooldowns still apply). (`com.microsoft:type=AI-alert,name=ProfilerControl`) that allows triggering profiles via JMX tools. See [Manual Profile Triggering](#manual-profile-triggering) for usage. +`enableContinuousProfiling` - (default: false) When enabled, the profiler keeps a single JFR +recording running continuously using a circular buffer (bounded by `continuousProfilingMaxAgeSeconds`) +instead of starting a new timed recording for each trigger. When a profile is requested, the current +contents of the circular buffer are dumped and uploaded immediately, so requests are serviced without +waiting for a recording duration to elapse. The uploaded profile is timestamped at the start of the +captured window. Note the following limitations while this feature is in preview: + +- The continuous recording uses the `cpuTriggeredSettings` JFC for all trigger types, so + `memoryTriggeredSettings` and `manualTriggeredSettings` are not applied to continuous captures. +- A requested profile duration (from the portal, JMX, or a file trigger) is honored but clamped to + `continuousProfilingMaxAgeSeconds`, since the circular buffer cannot retain more than that. +- Because JFR runs for the lifetime of the JVM rather than in short bursts, expect a steady-state + increase in CPU, memory and disk I/O compared to on-demand profiling. + +`continuousProfilingMaxAgeSeconds` - (default: 120) The maximum age, in seconds, of data retained in +the continuous profiling circular buffer. Only used when `enableContinuousProfiling` is `true`. +Non-positive values fall back to the default. Larger values increase both the on-disk retention and +the size of each uploaded profile. + `manualTrigger` - Configuration for the file-based manual profile trigger: - `enabled` - (default: false) Whether the file-based manual trigger is enabled. diff --git a/smoke-tests/apps/Diagnostics/src/main/java/com/microsoft/applicationinsights/smoketestapp/TestController.java b/smoke-tests/apps/Diagnostics/src/main/java/com/microsoft/applicationinsights/smoketestapp/TestController.java index 74b8a711125..05f05e713a0 100644 --- a/smoke-tests/apps/Diagnostics/src/main/java/com/microsoft/applicationinsights/smoketestapp/TestController.java +++ b/smoke-tests/apps/Diagnostics/src/main/java/com/microsoft/applicationinsights/smoketestapp/TestController.java @@ -5,9 +5,12 @@ import java.io.File; import java.io.OutputStream; +import java.io.UncheckedIOException; import java.nio.file.Files; import java.nio.file.Path; +import java.util.Comparator; import java.util.Optional; +import java.util.stream.Stream; import java.util.zip.GZIPInputStream; import java.util.zip.ZipException; import org.springframework.web.bind.annotation.GetMapping; @@ -22,41 +25,12 @@ public String root() { @GetMapping("/jfrFileHasDiagnostics") public String jfrFileHasDiagnostics() throws Exception { - Optional jfrFile; - for (int i = 0; i < 60; i++) { - try { - jfrFile = - Files.walk(new File("/tmp/root/applicationinsights").toPath()) - .filter(Files::isRegularFile) - .filter(it -> it.toFile().getName().contains(".jfr")) - .findFirst(); - - if (!jfrFile.isPresent()) { - Thread.sleep(1000, 0); - continue; - } - - Path decompressedFile = decompressFile(jfrFile.get()); - - boolean hasTelemetry = - com.microsoft.applicationinsights.jfrfile.JfrFileReader.hasEventOfType( - decompressedFile, "com.microsoft.applicationinsights.diagnostics.jfr.Telemetry"); - - if (hasTelemetry) { - return String.valueOf(true); - } - } catch (Exception e) { - // Ignore early exceptions, as to be expected, throw them if they are still happening - // towards the end - if (i > 55) { - throw e; - } else { - Thread.sleep(1000, 0); - } - } - } - - return String.valueOf(false); + return String.valueOf( + pollForJfrFileMatching( + decompressedFile -> + com.microsoft.applicationinsights.jfrfile.JfrFileReader.hasEventOfType( + decompressedFile, + "com.microsoft.applicationinsights.diagnostics.jfr.Telemetry"))); } /** @@ -73,13 +47,28 @@ public String continuousJfrFileHasDiagnostics() throws Exception { "com.microsoft.applicationinsights.diagnostics.jfr.CGroupData", }; + return String.valueOf( + pollForJfrFileMatching( + decompressedFile -> { + for (String event : requiredEvents) { + if (!com.microsoft.applicationinsights.jfrfile.JfrFileReader.hasEventInstanceOfType( + decompressedFile, event)) { + return false; + } + } + return true; + })); + } + + /** + * Polls for up to 60 seconds for the most recently modified {@code .jfr} file produced by the + * agent and applies {@code predicate} to it. Continuous profiling can produce several dumps, so + * the newest file is used rather than an arbitrary one. + */ + private boolean pollForJfrFileMatching(JfrFilePredicate predicate) throws Exception { for (int i = 0; i < 60; i++) { try { - Optional jfrFile = - Files.walk(new File("/tmp/root/applicationinsights").toPath()) - .filter(Files::isRegularFile) - .filter(it -> it.toFile().getName().contains(".jfr")) - .findFirst(); + Optional jfrFile = findNewestJfrFile(); if (!jfrFile.isPresent()) { Thread.sleep(1000, 0); @@ -88,17 +77,8 @@ public String continuousJfrFileHasDiagnostics() throws Exception { Path decompressedFile = decompressFile(jfrFile.get()); - boolean hasAll = true; - for (String event : requiredEvents) { - if (!com.microsoft.applicationinsights.jfrfile.JfrFileReader.hasEventInstanceOfType( - decompressedFile, event)) { - hasAll = false; - break; - } - } - - if (hasAll) { - return String.valueOf(true); + if (predicate.test(decompressedFile)) { + return true; } } catch (Exception e) { // Ignore early exceptions, as to be expected, throw them if they are still happening @@ -110,7 +90,23 @@ public String continuousJfrFileHasDiagnostics() throws Exception { Thread.sleep(1000, 0); } - return String.valueOf(false); + return false; + } + + @FunctionalInterface + private interface JfrFilePredicate { + boolean test(Path decompressedFile) throws Exception; + } + + private static Optional findNewestJfrFile() { + try (Stream files = Files.walk(new File("/tmp/root/applicationinsights").toPath())) { + return files + .filter(Files::isRegularFile) + .filter(it -> it.toFile().getName().contains(".jfr")) + .max(Comparator.comparingLong(it -> it.toFile().lastModified())); + } catch (java.io.IOException e) { + throw new UncheckedIOException(e); + } } private Path decompressFile(Path jfrFile) { From 43e24c5fada3852576ab2f752249a65c06aedf7d Mon Sep 17 00:00:00 2001 From: John Oliver <1615532+johnoliver@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:22:55 +0000 Subject: [PATCH 09/11] Fix continuous profiling dump and harden Diagnostics smoke test The continuous circular buffer is always in the RECORDING state, but the attempt to honor a shorter requested profile duration streamed a sub-window via Recording.getStream(start, end), which the JFR connection only permits on a STOPPED recording. Every manual/portal/JMX trigger therefore threw IllegalStateException ("Recording state RECORDING not in [STOPPED]"), so no continuous profile was ever produced and the DiagnosticsTest smoke tests failed. A live recording can only be dumped in its entirety, so always use Recording.dump() and drop the infeasible sub-window streaming path. The uploaded profile is still timestamped at the start of the captured window (recordingEnd - maxAge). Update the README limitation note accordingly. Also harden the Diagnostics smoke test controller: continuous profiling can produce several dumps and a dump may be observed mid-write, so iterate every candidate .jfr file (newest first) and skip any that cannot be decompressed yet instead of letting an EOFException fail the whole request. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../agent/internal/profiler/Profiler.java | 69 ++++--------------- .../ProfilerContinuousProfilingTest.java | 20 +++--- docs/README.md | 6 +- .../smoketestapp/TestController.java | 52 +++++++------- 4 files changed, 51 insertions(+), 96 deletions(-) diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/Profiler.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/Profiler.java index bb0d814ddaf..58bce2f0693 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/Profiler.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/Profiler.java @@ -184,7 +184,7 @@ public void updateConfiguration(ProfilerConfiguration newConfig) { void profileAndUpload(AlertBreach alertBreach, Duration duration, UploadListener uploadListener) { Instant recordingStart = timeSource.getNow(); if (continuousProfilingEnabled) { - captureContinuousRecording(alertBreach, duration, recordingStart, uploadListener); + captureContinuousRecording(alertBreach, recordingStart, uploadListener); return; } executeProfile( @@ -226,10 +226,7 @@ private void startContinuousRecordingIfEnabled() { @SuppressWarnings( "CatchingUnchecked") // catching unchecked exception is necessary for proper error handling private void captureContinuousRecording( - AlertBreach alertBreach, - Duration requestedDuration, - Instant recordingEnd, - UploadListener uploadListener) { + AlertBreach alertBreach, Instant recordingEnd, UploadListener uploadListener) { File dumpFile; Instant bufferStart; synchronized (activeRecordingLock) { @@ -247,16 +244,12 @@ private void captureContinuousRecording( return; } - // The circular buffer only retains up to maxAge of data, so the captured window can never - // exceed maxAge. Honor a shorter portal-/JMX-configured profile duration when one is - // provided, otherwise fall back to the full maxAge window. - Duration captureWindow = resolveContinuousCaptureWindow(requestedDuration); - - // The dumped buffer covers [recordingEnd - captureWindow, recordingEnd]. Use the start of - // that - // window as the profile timestamp and file name so the profile is indexed at the point the - // data actually begins rather than at dump time. - bufferStart = recordingEnd.minus(captureWindow); + // A live circular buffer can only be dumped in its entirety; the JFR connection only supports + // streaming a sub-window from a stopped recording, so a shorter portal-/JMX-configured profile + // duration cannot be honored without tearing down continuous profiling. The dump therefore + // always covers the full retained window, [recordingEnd - maxAge, recordingEnd]. Index the + // profile at the start of that window rather than at dump time. + bufferStart = recordingEnd.minus(continuousProfilingMaxAge); try { dumpFile = createJfrFile(bufferStart, recordingEnd); @@ -266,9 +259,9 @@ private void captureContinuousRecording( } try { - // Dump the trailing captureWindow of the circular buffer. The continuous recording keeps - // running so future requests can be serviced immediately. - dumpContinuousRecording(dumpFile, recordingEnd, captureWindow); + // Dump the current state of the circular buffer, capturing up to maxAge of data. The + // continuous recording keeps running so future requests can be serviced immediately. + continuousRecording.dump(dumpFile.getAbsolutePath()); } catch (IOException | JfrConnectionException e) { logger.error("Failed to dump continuous recording", e); if (dumpFile.exists() && !dumpFile.delete()) { @@ -298,38 +291,6 @@ private void captureContinuousRecording( } } - /** - * Resolves the window of buffered data to capture from the continuous recording. The window is - * bounded by the configured continuous profiling maxAge (the circular buffer can hold no more - * than that) but is otherwise driven by the requested profile duration so that a shorter - * portal-/JMX-configured duration is honored rather than silently ignored. - */ - private Duration resolveContinuousCaptureWindow(@Nullable Duration requestedDuration) { - if (requestedDuration == null - || requestedDuration.isZero() - || requestedDuration.isNegative() - || requestedDuration.compareTo(continuousProfilingMaxAge) > 0) { - return continuousProfilingMaxAge; - } - return requestedDuration; - } - - /** - * Writes the trailing {@code captureWindow} of the continuous recording's circular buffer to - * {@code dumpFile}. When the requested window covers the whole buffer the more robust {@link - * Recording#dump(String)} path is used; otherwise only the requested trailing window is streamed - * out so the configured profile duration is respected. - */ - private void dumpContinuousRecording(File dumpFile, Instant recordingEnd, Duration captureWindow) - throws IOException, JfrConnectionException { - if (captureWindow.compareTo(continuousProfilingMaxAge) >= 0) { - continuousRecording.dump(dumpFile.getAbsolutePath()); - return; - } - writeFileFromStream( - continuousRecording, dumpFile, recordingEnd.minus(captureWindow), recordingEnd); - } - @Nullable private Recording startRecording(AlertMetricType alertType, Duration duration) { synchronized (activeRecordingLock) { @@ -477,18 +438,12 @@ private static void closeRecording(Recording recording, File recordingFile) { private static void writeFileFromStream(Recording recording, File recordingFile) throws IOException, JfrConnectionException { - writeFileFromStream(recording, recordingFile, null, null); - } - - private static void writeFileFromStream( - Recording recording, File recordingFile, @Nullable Instant start, @Nullable Instant end) - throws IOException, JfrConnectionException { if (recordingFile.exists()) { recordingFile.delete(); } recordingFile.createNewFile(); - try (BufferedInputStream stream = new BufferedInputStream(recording.getStream(start, end)); + try (BufferedInputStream stream = new BufferedInputStream(recording.getStream(null, null)); FileOutputStream fos = new FileOutputStream(recordingFile)) { int read; byte[] buffer = new byte[10 * 1024]; diff --git a/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/profiler/ProfilerContinuousProfilingTest.java b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/profiler/ProfilerContinuousProfilingTest.java index a3c91f6e08b..26be48ab0fd 100644 --- a/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/profiler/ProfilerContinuousProfilingTest.java +++ b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/profiler/ProfilerContinuousProfilingTest.java @@ -10,7 +10,6 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; import com.microsoft.applicationinsights.agent.internal.configuration.Configuration; import com.microsoft.applicationinsights.agent.internal.profiler.testutil.TestTimeSource; @@ -23,9 +22,7 @@ import io.opentelemetry.contrib.jfr.connection.Recording; import io.opentelemetry.contrib.jfr.connection.RecordingConfiguration; import io.opentelemetry.contrib.jfr.connection.RecordingOptions; -import java.io.ByteArrayInputStream; import java.io.File; -import java.nio.charset.StandardCharsets; import java.time.Duration; import java.time.Instant; import java.util.UUID; @@ -65,15 +62,13 @@ private static AlertBreach manualBreach(int profileDurationSeconds) { } @Test - void profileRequestCapturesRequestedWindowOfContinuousRecording() throws Exception { + void profileRequestAlwaysDumpsWholeBufferEvenForShorterRequestedDuration() throws Exception { Configuration.ProfilerConfiguration config = new Configuration.ProfilerConfiguration(); config.enableContinuousProfiling = true; config.continuousProfilingMaxAgeSeconds = 60; config.globalCooldownSeconds = 0; Recording continuousRecording = mock(Recording.class); - when(continuousRecording.getStream(any(), any())) - .thenReturn(new ByteArrayInputStream("jfr".getBytes(StandardCharsets.UTF_8))); Profiler profiler = new Profiler(config, tempDir, timeSource) { @@ -95,16 +90,17 @@ protected Recording createRecording(RecordingOptions o, RecordingConfiguration c timeSource.setNow(now); UploadListener noOp = index -> {}; - // The portal-/JMX-configured duration (10s) is shorter than the 60s buffer, so only the - // trailing 10s window of the circular buffer is captured. + // A live circular buffer can only be dumped in its entirety; a shorter portal-/JMX-configured + // duration (10s) cannot be honored by streaming a sub-window from the still-running recording, + // so the whole 60s buffer is dumped via the robust dump() path. profiler.profileAndUpload(manualBreach(10), Duration.ofSeconds(10), noOp); - verify(continuousRecording).getStream(eq(now.minusSeconds(10)), eq(now)); - verify(continuousRecording, never()).dump(anyString()); + verify(continuousRecording).dump(anyString()); + verify(continuousRecording, never()).getStream(any(), any()); verify(continuousRecording, never()).stop(); - // The profile is timestamped at the start of the captured window (now - 10s), not at dump time. + // The captured window is the whole 60s buffer, so the profile is timestamped at now - 60s. verify(uploadService) - .upload(any(), eq(now.minusSeconds(10).toEpochMilli()), any(File.class), any()); + .upload(any(), eq(now.minusSeconds(60).toEpochMilli()), any(File.class), any()); assertThat(profiler.isRecordingActive()).isFalse(); } diff --git a/docs/README.md b/docs/README.md index 2e9d6a7c352..56615a8d9af 100644 --- a/docs/README.md +++ b/docs/README.md @@ -146,8 +146,10 @@ captured window. Note the following limitations while this feature is in preview - The continuous recording uses the `cpuTriggeredSettings` JFC for all trigger types, so `memoryTriggeredSettings` and `manualTriggeredSettings` are not applied to continuous captures. -- A requested profile duration (from the portal, JMX, or a file trigger) is honored but clamped to - `continuousProfilingMaxAgeSeconds`, since the circular buffer cannot retain more than that. +- A requested profile duration (from the portal, JMX, or a file trigger) is ignored: each request + dumps the whole retained circular buffer (up to `continuousProfilingMaxAgeSeconds`), because a + live JFR recording can only be dumped in its entirety and cannot be streamed for a sub-window + without being stopped. - Because JFR runs for the lifetime of the JVM rather than in short bursts, expect a steady-state increase in CPU, memory and disk I/O compared to on-demand profiling. diff --git a/smoke-tests/apps/Diagnostics/src/main/java/com/microsoft/applicationinsights/smoketestapp/TestController.java b/smoke-tests/apps/Diagnostics/src/main/java/com/microsoft/applicationinsights/smoketestapp/TestController.java index 05f05e713a0..f354810149f 100644 --- a/smoke-tests/apps/Diagnostics/src/main/java/com/microsoft/applicationinsights/smoketestapp/TestController.java +++ b/smoke-tests/apps/Diagnostics/src/main/java/com/microsoft/applicationinsights/smoketestapp/TestController.java @@ -8,8 +8,10 @@ import java.io.UncheckedIOException; import java.nio.file.Files; import java.nio.file.Path; +import java.util.Collections; import java.util.Comparator; -import java.util.Optional; +import java.util.List; +import java.util.stream.Collectors; import java.util.stream.Stream; import java.util.zip.GZIPInputStream; import java.util.zip.ZipException; @@ -61,30 +63,25 @@ public String continuousJfrFileHasDiagnostics() throws Exception { } /** - * Polls for up to 60 seconds for the most recently modified {@code .jfr} file produced by the - * agent and applies {@code predicate} to it. Continuous profiling can produce several dumps, so - * the newest file is used rather than an arbitrary one. + * Polls for up to 60 seconds for a {@code .jfr} file produced by the agent that satisfies {@code + * predicate}. Continuous profiling can produce several dumps and a dump may be observed + * mid-write, so every candidate file is examined (newest first) and any file that cannot be read + * yet is skipped rather than failing the whole request. */ private boolean pollForJfrFileMatching(JfrFilePredicate predicate) throws Exception { for (int i = 0; i < 60; i++) { - try { - Optional jfrFile = findNewestJfrFile(); - - if (!jfrFile.isPresent()) { - Thread.sleep(1000, 0); - continue; - } - - Path decompressedFile = decompressFile(jfrFile.get()); - - if (predicate.test(decompressedFile)) { - return true; - } - } catch (Exception e) { - // Ignore early exceptions, as to be expected, throw them if they are still happening - // towards the end - if (i > 55) { - throw e; + for (Path jfrFile : findJfrFilesNewestFirst()) { + try { + Path decompressedFile = decompressFile(jfrFile); + if (predicate.test(decompressedFile)) { + return true; + } + } catch (Exception e) { + // A dump may still be being written, or otherwise unreadable; skip it and try the next + // candidate / next poll iteration. + if (i > 55) { + e.printStackTrace(); + } } } Thread.sleep(1000, 0); @@ -98,12 +95,17 @@ private interface JfrFilePredicate { boolean test(Path decompressedFile) throws Exception; } - private static Optional findNewestJfrFile() { - try (Stream files = Files.walk(new File("/tmp/root/applicationinsights").toPath())) { + private static List findJfrFilesNewestFirst() { + Path root = new File("/tmp/root/applicationinsights").toPath(); + if (!Files.isDirectory(root)) { + return Collections.emptyList(); + } + try (Stream files = Files.walk(root)) { return files .filter(Files::isRegularFile) .filter(it -> it.toFile().getName().contains(".jfr")) - .max(Comparator.comparingLong(it -> it.toFile().lastModified())); + .sorted(Comparator.comparingLong((Path it) -> it.toFile().lastModified()).reversed()) + .collect(Collectors.toList()); } catch (java.io.IOException e) { throw new UncheckedIOException(e); } From d06e8b256d5f3a4f906308ae6d220498cccd4506 Mon Sep 17 00:00:00 2001 From: John Oliver <1615532+johnoliver@users.noreply.github.com> Date: Mon, 3 Aug 2026 15:13:15 +0000 Subject: [PATCH 10/11] Spotless --- .../applicationinsights/agent/internal/profiler/Profiler.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/Profiler.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/Profiler.java index 58bce2f0693..531f836eabe 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/Profiler.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/Profiler.java @@ -245,7 +245,8 @@ private void captureContinuousRecording( } // A live circular buffer can only be dumped in its entirety; the JFR connection only supports - // streaming a sub-window from a stopped recording, so a shorter portal-/JMX-configured profile + // streaming a sub-window from a stopped recording, so a shorter portal-/JMX-configured + // profile // duration cannot be honored without tearing down continuous profiling. The dump therefore // always covers the full retained window, [recordingEnd - maxAge, recordingEnd]. Index the // profile at the start of that window rather than at dump time. From 3ed430b7bf3e7733145c780f8bd89827020745b9 Mon Sep 17 00:00:00 2001 From: John Oliver <1615532+johnoliver@users.noreply.github.com> Date: Mon, 3 Aug 2026 16:53:38 +0000 Subject: [PATCH 11/11] Fixes from review --- .../diagnostics/jfr/SystemStatsProvider.java | 13 ++++++++++--- .../smoketestapp/TestController.java | 13 ++++++++++++- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/agent/agent-profiler/agent-diagnostics/src/main/java/com/microsoft/applicationinsights/diagnostics/jfr/SystemStatsProvider.java b/agent/agent-profiler/agent-diagnostics/src/main/java/com/microsoft/applicationinsights/diagnostics/jfr/SystemStatsProvider.java index 87830bd1819..ef6757e0dbb 100644 --- a/agent/agent-profiler/agent-diagnostics/src/main/java/com/microsoft/applicationinsights/diagnostics/jfr/SystemStatsProvider.java +++ b/agent/agent-profiler/agent-diagnostics/src/main/java/com/microsoft/applicationinsights/diagnostics/jfr/SystemStatsProvider.java @@ -129,7 +129,10 @@ public static CGroupData getCGroupData(Path cgroupBasePath) { public static MachineInfo getMachineInfo() { return getSingleton( MachineInfo.class, - () -> new MachineInfo().setCoreCount(new RuntimeCoreCounter().getCoreCount())); + () -> + new MachineInfo() + .setCoreCount(new RuntimeCoreCounter().getCoreCount()) + .setSchemaVersion(MachineInfo.SCHEMA_VERSION)); } private static Process getThisProcess() { @@ -141,8 +144,12 @@ private static Process getThisProcess() { Process thisProcess = processDumper.thisProcess(); if (thisProcess == null) { - // e.g. NoOpProcessDumper on unsupported operating systems returns no process - return null; + // e.g. NoOpProcessDumper on unsupported operating systems returns no process. The only + // caller (buildSystemStatsReader) dereferences the result, so fail fast with a clear + // message rather than returning null and triggering an opaque NPE later. + throw new IllegalStateException( + "Current process information is unavailable on this operating system; " + + "system stats cannot be collected"); } processDumper.closeProcesses(Collections.singletonList(thisProcess.getPid())); return thisProcess; diff --git a/smoke-tests/apps/Diagnostics/src/main/java/com/microsoft/applicationinsights/smoketestapp/TestController.java b/smoke-tests/apps/Diagnostics/src/main/java/com/microsoft/applicationinsights/smoketestapp/TestController.java index f354810149f..2867736cf44 100644 --- a/smoke-tests/apps/Diagnostics/src/main/java/com/microsoft/applicationinsights/smoketestapp/TestController.java +++ b/smoke-tests/apps/Diagnostics/src/main/java/com/microsoft/applicationinsights/smoketestapp/TestController.java @@ -71,8 +71,9 @@ public String continuousJfrFileHasDiagnostics() throws Exception { private boolean pollForJfrFileMatching(JfrFilePredicate predicate) throws Exception { for (int i = 0; i < 60; i++) { for (Path jfrFile : findJfrFilesNewestFirst()) { + Path decompressedFile = null; try { - Path decompressedFile = decompressFile(jfrFile); + decompressedFile = decompressFile(jfrFile); if (predicate.test(decompressedFile)) { return true; } @@ -82,6 +83,16 @@ private boolean pollForJfrFileMatching(JfrFilePredicate predicate) throws Except if (i > 55) { e.printStackTrace(); } + } finally { + // decompressFile returns the original file (not a temp copy) when decompression fails, so + // only delete files we actually created to avoid removing the agent's dump. + if (decompressedFile != null && !decompressedFile.equals(jfrFile)) { + try { + Files.deleteIfExists(decompressedFile); + } catch (java.io.IOException ignored) { + // best effort cleanup + } + } } } Thread.sleep(1000, 0);