Updated jit stage to account for OSR.

Don't jit overhead methods if the job is configured to not measure it.
Remove extra call counting delay for in-process benchmarks.
Set CallCountingDelayMs env var if DisassemblyDiagnoser is not used.
Added a test for very long first invocation time.

Author: timcassell

src/BenchmarkDotNet/Configs/ImmutableConfig.cs

@@ -120,6 +120,8 @@ internal ImmutableConfig(
 
         internal bool HasPerfCollectProfiler() => diagnosers.OfType<PerfCollectProfiler>().Any();
 
+        internal bool HasDisassemblyDiagnoser() => diagnosers.OfType<DisassemblyDiagnoser>().Any();
+
         public bool HasExtraStatsDiagnoser() => HasMemoryDiagnoser() || HasThreadingDiagnoser() || HasExceptionDiagnoser();
 
         public IDiagnoser? GetCompositeDiagnoser(BenchmarkCase benchmarkCase, RunMode runMode)

src/BenchmarkDotNet/Engines/Engine.cs

@@ -5,6 +5,7 @@
 using System.Threading;
 using BenchmarkDotNet.Characteristics;
 using BenchmarkDotNet.Environments;
+using BenchmarkDotNet.Helpers;
 using BenchmarkDotNet.Jobs;
 using BenchmarkDotNet.Portability;
 using BenchmarkDotNet.Reports;
@@ -213,12 +214,9 @@ private ClockSpan Measure(Action<long> action, long invokeCount)
             ForceGcCollect();
 
             // #1542
-            if (JitInfo.BackgroundCompilationDelay > TimeSpan.Zero)
-            {
-                // We put the current thread to sleep so tiered jit can kick in, compile its stuff,
-                // and NOT allocate anything on the background thread when we are measuring allocations.
-                Thread.Sleep(JitInfo.BackgroundCompilationDelay);
-            }
+            // If the jit is tiered, we put the current thread to sleep so it can kick in, compile its stuff,
+            // and NOT allocate anything on the background thread when we are measuring allocations.
+            SleepHelper.SleepIfPositive(JitInfo.BackgroundCompilationDelay);
 
             GcStats gcStats;
             using (FinalizerBlocker.MaybeStart())

src/BenchmarkDotNet/Engines/EngineJitStage.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Collections.Generic;
 using System.Threading;
+using BenchmarkDotNet.Helpers;
 using BenchmarkDotNet.Jobs;
 using BenchmarkDotNet.Portability;
 using BenchmarkDotNet.Reports;
@@ -27,27 +28,35 @@ internal sealed class EngineFirstJitStage : EngineJitStage
         // It is not worth spending a long time in jit stage for macro-benchmarks.
         private static readonly TimeInterval MaxTieringTime = TimeInterval.FromSeconds(10);
 
+        // Jit call counting delay is only for when the app starts up. We don't need to wait for every benchmark if multiple benchmarks are ran in-process.
+        private static TimeSpan tieredDelay = JitInfo.TieredDelay;
+
         internal bool didStopEarly = false;
         internal Measurement lastMeasurement;
 
         private readonly IEnumerator<IterationData> enumerator;
+        private readonly bool evaluateOverhead;
 
-        internal EngineFirstJitStage(EngineParameters parameters) : base(parameters)
+        internal EngineFirstJitStage(bool evaluateOverhead, EngineParameters parameters) : base(parameters)
         {
             enumerator = EnumerateIterations();
+            this.evaluateOverhead = evaluateOverhead;
         }
 
         internal override List<Measurement> GetMeasurementList() => new(GetMaxMeasurementCount());
 
-        private static int GetMaxMeasurementCount()
+        private int GetMaxMeasurementCount()
         {
-            int tieredCallCountThreshold = JitInfo.TieredCallCountThreshold;
-            if (JitInfo.IsDPGO)
+            if (!JitInfo.IsTiered)
             {
-                tieredCallCountThreshold *= 2;
+                return 1;
             }
-            // +1 for first jit, x2 for overhead + workload
-            return (tieredCallCountThreshold + 1) * 2;
+            int count = JitInfo.MaxTierPromotions* JitInfo.TieredCallCountThreshold + 2;
+            if (evaluateOverhead)
+            {
+                count *= 2;
+            }
+            return count;
         }
 
         internal override bool GetShouldRunIteration(List<Measurement> measurements, out IterationData iterationData)
@@ -73,8 +82,11 @@ internal override bool GetShouldRunIteration(List<Measurement> measurements, out
         private IEnumerator<IterationData> EnumerateIterations()
         {
             ++iterationIndex;
-            yield return GetDummyIterationData(dummy1Action);
-            yield return GetOverheadIterationData();
+            if (evaluateOverhead)
+            {
+                yield return GetDummyIterationData(dummy1Action);
+                yield return GetOverheadIterationData();
+            }
             yield return GetDummyIterationData(dummy2Action);
             yield return GetWorkloadIterationData();
             yield return GetDummyIterationData(dummy3Action);
@@ -86,12 +98,14 @@ private IEnumerator<IterationData> EnumerateIterations()
             }
 
             // Wait enough time for jit call counting to begin.
-            MaybeSleep(JitInfo.TieredDelay);
+            SleepHelper.SleepIfPositive(tieredDelay);
+            // Don't make the next jit stage wait if it's ran in the same process.
+            tieredDelay = TimeSpan.Zero;
 
             // Attempt to promote methods to tier1, but don't spend too much time in jit stage.
             StartedClock startedClock = parameters.TargetJob.ResolveValue(InfrastructureMode.ClockCharacteristic, parameters.Resolver).Start();
 
-            int remainingTiers = JitInfo.IsDPGO ? 2 : 1;
+            int remainingTiers = JitInfo.MaxTierPromotions;
             while (remainingTiers > 0)
             {
                 --remainingTiers;
@@ -100,7 +114,10 @@ private IEnumerator<IterationData> EnumerateIterations()
                 {
                     --remainingCalls;
                     ++iterationIndex;
-                    yield return GetOverheadIterationData();
+                    if (evaluateOverhead)
+                    {
+                        yield return GetOverheadIterationData();
+                    }
                     yield return GetWorkloadIterationData();
 
                     if ((remainingTiers + remainingCalls) > 0
@@ -111,13 +128,16 @@ private IEnumerator<IterationData> EnumerateIterations()
                     }
                 }
 
-                MaybeSleep(JitInfo.BackgroundCompilationDelay);
+                SleepHelper.SleepIfPositive(JitInfo.BackgroundCompilationDelay);
             }
 
-            // Empirical evidence shows that the first call after the method is tiered up takes longer,
+            // Empirical evidence shows that the first call after the method is tiered up may take longer,
             // so we run an extra iteration to ensure the next stage gets a stable measurement.
             ++iterationIndex;
-            yield return GetOverheadIterationData();
+            if (evaluateOverhead)
+            {
+                yield return GetOverheadIterationData();
+            }
             yield return GetWorkloadIterationData();
         }
 
@@ -126,32 +146,22 @@ private IterationData GetOverheadIterationData()
 
         private IterationData GetWorkloadIterationData()
             => new(IterationMode.Workload, IterationStage.Jitting, iterationIndex, 1, 1, parameters.IterationSetupAction, parameters.IterationCleanupAction, parameters.WorkloadActionNoUnroll);
-
-        private static void MaybeSleep(TimeSpan timeSpan)
-        {
-            if (timeSpan > TimeSpan.Zero)
-            {
-                Thread.Sleep(timeSpan);
-            }
-        }
     }
 
-    internal sealed class EngineSecondJitStage(int unrollFactor, EngineParameters parameters) : EngineJitStage(parameters)
+    internal sealed class EngineSecondJitStage : EngineJitStage
     {
-        private readonly int unrollFactor = unrollFactor;
+        private readonly int unrollFactor;
+        private readonly bool evaluateOverhead;
 
-        internal override List<Measurement> GetMeasurementList() => new(GetMaxCallCount());
-
-        private static int GetMaxCallCount()
+        public EngineSecondJitStage(int unrollFactor, bool evaluateOverhead, EngineParameters parameters) : base(parameters)
         {
-            int tieredCallCountThreshold = JitInfo.TieredCallCountThreshold;
-            if (JitInfo.IsDPGO)
-            {
-                tieredCallCountThreshold *= 2;
-            }
-            return tieredCallCountThreshold + 1;
+            this.unrollFactor = unrollFactor;
+            this.evaluateOverhead = evaluateOverhead;
+            iterationIndex = evaluateOverhead ? 0 : 2;
         }
 
+        internal override List<Measurement> GetMeasurementList() => new(evaluateOverhead ? 5 : 3);
+
         // The benchmark method has already been jitted via *NoUnroll, we only need to jit the *Unroll methods here, which aren't tiered.
         internal override bool GetShouldRunIteration(List<Measurement> measurements, out IterationData iterationData)
         {

src/BenchmarkDotNet/Engines/EngineStage.cs

@@ -33,7 +33,7 @@ internal static IEnumerable<EngineStage> EnumerateStages(EngineParameters parame
                     int minInvokeCount = parameters.TargetJob.ResolveValue(AccuracyMode.MinInvokeCountCharacteristic, parameters.Resolver);
 
                     // AOT technically doesn't have a JIT, but we run jit stage regardless because of static constructors. #2004
-                    var jitStage = new EngineFirstJitStage(parameters);
+                    var jitStage = new EngineFirstJitStage(evaluateOverhead, parameters);
                     yield return jitStage;
 
                     bool hasUnrollFactor = parameters.TargetJob.HasValue(RunMode.UnrollFactorCharacteristic);
@@ -62,7 +62,7 @@ internal static IEnumerable<EngineStage> EnumerateStages(EngineParameters parame
                     // TODO: This stage can be removed after we refactor the engine/codegen to pass the clock into the delegates.
                     if (!RuntimeInformation.IsAot && unrollFactor != 1)
                     {
-                        yield return new EngineSecondJitStage(unrollFactor, parameters);
+                        yield return new EngineSecondJitStage(unrollFactor, evaluateOverhead, parameters);
                     }
 
                     if (!skipPilotStage)

src/BenchmarkDotNet/Extensions/ProcessExtensions.cs

@@ -150,8 +150,11 @@ internal static void SetEnvironmentVariables(this ProcessStartInfo start, Benchm
             // disable ReSharper's Dynamic Program Analysis (see https://github.com/dotnet/BenchmarkDotNet/issues/1871 for details)
             start.EnvironmentVariables["JETBRAINS_DPA_AGENT_ENABLE"] = "0";
 
-            // TODO: set CallCountingDelayMs without breaking DisassemblyDiagnoser. https://github.com/dotnet/runtime/issues/117339
-            //SetClrEnvironmentVariables(start, JitInfo.CallCountingDelayMsEnv, "0");
+            // CallCountingDelayMs=0 breaks DisassemblyDiagnoser, so we only set it if the job doesn't need disassembly. https://github.com/dotnet/runtime/issues/117339
+            if (!benchmarkCase.Config.HasDisassemblyDiagnoser())
+            {
+                SetClrEnvironmentVariables(start, JitInfo.EnvCallCountingDelayMs, "0");
+            }
 
             if (!benchmarkCase.Job.HasValue(EnvironmentMode.EnvironmentVariablesCharacteristic))
                 return;

src/BenchmarkDotNet/Helpers/SleepHelper.cs

@@ -0,0 +1,16 @@
+using System;
+using System.Threading;
+
+namespace BenchmarkDotNet.Helpers
+{
+    internal static class SleepHelper
+    {
+        public static void SleepIfPositive(TimeSpan timeSpan)
+        {
+            if (timeSpan > TimeSpan.Zero)
+            {
+                Thread.Sleep(timeSpan);
+            }
+        }
+    }
+}