feat(minor): Support thresholds update for range and relative thresholds [4/4]

Plugins/BenchmarkHelpGenerator/BenchmarkHelpGenerator.swift

@@ -157,6 +157,42 @@ struct Benchmark: AsyncParsableCommand {
     )
     var checkAbsolute = false
 
+    @Flag(
+        name: .long,
+        help: """
+            Specifies that thresholds check command should skip loading benchmark targets.
+            Use this flag to skip unnecessary building of benchmark targets and loading of benchmark results, to save time.
+            This flag is specially useful when combined with static threshold files that contain the newly supported relative or range thresholds.
+            With such a set up, you'll save the time needed to build the benchmark targets and the thresholds check operation
+            will only read the threshold tolerance values from the static files.
+            """
+    )
+    var skipLoadingBenchmarks = false
+
+    @Option(
+        name: .long,
+        help: """
+            The number of times to run each benchmark in thresholds update operation.
+            This is only valid when --relative or --range are also specified.
+            When combined with --relative or --range flags, this option will run the benchmarks multiple times to calculate
+            relative or range thresholds, and each time it'll widen the threshold tolerances according to the new result.
+            Defaults to 1.
+            """
+    )
+    var runCount: Int?
+
+    @Flag(
+        name: .long,
+        help: "Specifies that thresholds update command should output relative thresholds to the static files."
+    )
+    var relative = false
+
+    @Flag(
+        name: .long,
+        help: "Specifies that thresholds update command should output min-max range thresholds to the static files."
+    )
+    var range = false
+
     @Option(
         name: .long,
         help:

Plugins/BenchmarkTool/BenchmarkTool+Baselines.swift

@@ -451,8 +451,7 @@ extension BenchmarkBaseline: Equatable {
                             benchmarks,
                             name: lhsBenchmarkIdentifier.name,
                             target: lhsBenchmarkIdentifier.target,
-                            metric: lhsBenchmarkResult.metric,
-                            defaultThresholds: lhsBenchmarkResult.thresholds ?? BenchmarkThresholds.default
+                            metric: lhsBenchmarkResult.metric
                         )
 
                         let deviationResults = lhsBenchmarkResult.deviationsComparedWith(
@@ -485,7 +484,7 @@ extension BenchmarkBaseline: Equatable {
     public func failsAbsoluteThresholdChecks(
         benchmarks: [Benchmark],
         p90Thresholds: [BenchmarkIdentifier:
-            [BenchmarkMetric: BenchmarkThresholds.AbsoluteThreshold]]
+            [BenchmarkMetric: BenchmarkThreshold]]
     ) -> BenchmarkResult.ThresholdDeviations {
         var allDeviationResults = BenchmarkResult.ThresholdDeviations()
 

Plugins/BenchmarkTool/BenchmarkTool+Export+InfluxCSVFormatter.swift

@@ -54,7 +54,8 @@ class InfluxCSVFormatter {
         let memory = machine.memory
 
         if header {
-            let dataTypeHeader = "#datatype tag,tag,tag,tag,tag,tag,tag,tag,tag,double,double,double,long,long,dateTime\n"
+            let dataTypeHeader =
+                "#datatype tag,tag,tag,tag,tag,tag,tag,tag,tag,double,double,double,long,long,dateTime\n"
             finalFileFormat.append(dataTypeHeader)
             let headers =
                 "measurement,hostName,processoryType,processors,memory,kernelVersion,metric,unit,test,percentile,value,test_average,iterations,warmup_iterations,time\n"

Plugins/BenchmarkTool/BenchmarkTool+Export+JMHFormatter.swift

@@ -34,23 +34,23 @@ extension JMHPrimaryMetric {
         let factor = result.metric.countable == false ? 1_000 : 1
 
         for p in percentiles {
-            percentileValues[String(p)] = Statistics.roundToDecimalplaces(
+            percentileValues[String(p)] = Statistics.roundToDecimalPlaces(
                 Double(histogram.valueAtPercentile(p)) / Double(factor),
                 3
             )
         }
 
         for value in histogram.recordedValues() {
             for _ in 0..<value.count {
-                recordedValues.append(Statistics.roundToDecimalplaces(Double(value.value) / Double(factor), 3))
+                recordedValues.append(Statistics.roundToDecimalPlaces(Double(value.value) / Double(factor), 3))
             }
         }
 
-        self.score = Statistics.roundToDecimalplaces(score / Double(factor), 3)
-        scoreError = Statistics.roundToDecimalplaces(error / Double(factor), 3)
+        self.score = Statistics.roundToDecimalPlaces(score / Double(factor), 3)
+        scoreError = Statistics.roundToDecimalPlaces(error / Double(factor), 3)
         scoreConfidence = [
-            Statistics.roundToDecimalplaces(score - error) / Double(factor),
-            Statistics.roundToDecimalplaces(score + error) / Double(factor),
+            Statistics.roundToDecimalPlaces(score - error) / Double(factor),
+            Statistics.roundToDecimalPlaces(score + error) / Double(factor),
         ]
         scorePercentiles = percentileValues
         if result.metric.countable {

Plugins/BenchmarkTool/BenchmarkTool+Export.swift

@@ -23,18 +23,17 @@ import Musl
 #endif
 
 extension BenchmarkTool {
-    func write(
-        exportData: String,
-        hostIdentifier: String? = nil,
-        fileName: String = "results.txt"
-    ) throws {
-        // Set up desired output path and create any intermediate directories for structure as required:
+    enum OutputPath {
+        case stdout
+        case file(FilePath)
+    }
+
+    func outputPath(hostIdentifier: String? = nil, fileName: String) -> OutputPath {
         var outputPath: FilePath
 
         if let path = (thresholdsOperation == nil) ? path : thresholdsPath {
             if path == "stdout" {
-                print(exportData)
-                return
+                return .stdout
             }
 
             let subPath = FilePath(path).removingRoot()
@@ -59,6 +58,24 @@ extension BenchmarkTool {
 
         outputPath.append(csvFile.components)
 
+        return .file(outputPath)
+    }
+
+    func write(
+        exportData: String,
+        hostIdentifier: String? = nil,
+        fileName: String = "results.txt"
+    ) throws {
+        // Set up desired output path and create any intermediate directories for structure as required:
+        let outputPath: FilePath
+        switch self.outputPath(hostIdentifier: hostIdentifier, fileName: fileName) {
+        case .stdout:
+            print(exportData)
+            return
+        case .file(let path):
+            outputPath = path
+        }
+
         print("Writing to \(outputPath)")
 
         printFailedBenchmarks()
@@ -256,23 +273,132 @@ extension BenchmarkTool {
                 }
             }
         case .metricP90AbsoluteThresholds:
+            let jsonEncoder = JSONEncoder()
+            jsonEncoder.outputFormatting = [.prettyPrinted, .sortedKeys]
+
             try baseline.results.forEach { key, results in
-                let jsonEncoder = JSONEncoder()
-                jsonEncoder.outputFormatting = [.prettyPrinted, .sortedKeys]
+                let fileName = cleanupStringForShellSafety("\(key.target).\(key.name).p90.json")
+
+                var outputResults: [BenchmarkMetric: BenchmarkThreshold] = [:]
+
+                let wantsRelative = self.wantsRelativeThresholds
+                let wantsRange = self.wantsRangeThresholds
+                let wantsRelativeOrRange = wantsRelative || wantsRange
+
+                /// If it's the first run or if relative/range are not specified, then
+                /// override the thresholds file with the new results we have.
+                /// If runNumber is zero that'd mean this is not part of a multi-run benchmark,
+                /// so we'll still try to update thresholds instead of overriding them.
+                if runNumber == 1 || !wantsRelativeOrRange {
+                    for values in results {
+                        outputResults[values.metric] = .absolute(
+                            Int(values.statistics.histogram.valueAtPercentile(90.0))
+                        )
+                    }
 
-                var outputResults: [String: BenchmarkThresholds.AbsoluteThreshold] = [:]
-                results.forEach { values in
-                    outputResults[values.metric.rawDescription] = Int(
-                        values.statistics.histogram.valueAtPercentile(90.0)
-                    )
+                } else {
+                    /// If it's not the first run and any of relative/range are specified, then
+                    /// merge the new results with the existing thresholds.
+
+                    var currentThresholds: [BenchmarkMetric: BenchmarkThreshold]?
+
+                    switch self.outputPath(fileName: fileName) {
+                    case .stdout:
+                        currentThresholds = nil
+                    case .file(let path):
+                        currentThresholds = Self.makeBenchmarkThresholds(
+                            path: path,
+                            benchmarkIdentifier: key
+                        )
+                    }
+
+                    outputResults = currentThresholds ?? [:]
+
+                    for values in results {
+                        let metric = values.metric
+                        let newValue = values.statistics.histogram.valueAtPercentile(90.0)
+
+                        var relativeResult: BenchmarkThreshold.RelativeOrRange.Relative?
+                        var rangeResult: BenchmarkThreshold.RelativeOrRange.Range?
+                        if wantsRelativeOrRange {
+                            let newValue = Double(Int(truncatingIfNeeded: newValue))
+                            /// Prefer Double to keep precision
+                            var min = Double(newValue)
+                            var max = Double(newValue)
+
+                            /// Load current min/max values from static thresholds file
+                            switch currentThresholds?[metric] {
+                            case .absolute(let value):
+                                min = Double(value)
+                                max = Double(value)
+                            case .relativeOrRange(let relativeOrRange):
+                                /// If for "wantsRelative", we prefer to use the min/max
+                                if let range = relativeOrRange.range {
+                                    min = Double(range.min)
+                                    max = Double(range.max)
+                                } else if let relative = relativeOrRange.relative {
+                                    let base = Double(relative.base)
+                                    let diff = (base / 100) * relative.tolerancePercentage
+                                    min = base - diff
+                                    max = base + diff
+                                }
+                            case .none: break
+                            }
+
+                            /// Update the min/max values
+                            min = Swift.min(min, Double(newValue))
+                            max = Swift.max(max, Double(newValue))
+
+                            /// If min == max, it won't make a difference than using .absolute
+                            if min != max {
+                                if wantsRange {
+                                    rangeResult = .init(min: Int(min), max: Int(max))
+                                }
+
+                                if wantsRelative {
+                                    /// Calculate base and tolerancePercentage
+                                    let base = (min + max) / 2
+                                    let diff = max - base
+                                    let diffPercentage = (base == 0) ? 0 : (diff / base * 100)
+                                    let tolerancePercentage = Statistics.roundToDecimalPlaces(diffPercentage, 2, .up)
+
+                                    relativeResult = .init(
+                                        base: Int(base),
+                                        tolerancePercentage: tolerancePercentage
+                                    )
+                                }
+                            }
+                        }
+
+                        if relativeResult == nil && rangeResult == nil {
+                            outputResults[metric] = .absolute(Int(truncatingIfNeeded: newValue))
+                        } else {
+                            /// If we have a relative/range threshold but it's not specified in the command for
+                            /// this run to update it, we still would like to keep the non-updated existing threshold.
+                            switch currentThresholds?[metric] {
+                            case .relativeOrRange(let currentRelativeOrRange):
+                                relativeResult = relativeResult ?? currentRelativeOrRange.relative
+                                rangeResult = rangeResult ?? currentRelativeOrRange.range
+                            case .absolute, .none:
+                                break
+                            }
+
+                            outputResults[metric] = .relativeOrRange(
+                                BenchmarkThreshold.RelativeOrRange(
+                                    relative: relativeResult,
+                                    range: rangeResult
+                                )
+                            )
+                        }
+                    }
                 }
 
                 let jsonResultData = try jsonEncoder.encode(outputResults)
 
                 if let stringOutput = String(data: jsonResultData, encoding: .utf8) {
                     try write(
                         exportData: stringOutput,
-                        fileName: cleanupStringForShellSafety("\(key.target).\(key.name).p90.json")
+                        fileName: fileName
                     )
                 } else {
                     print("Failed to encode json for \(outputResults)")

Plugins/BenchmarkTool/BenchmarkTool+Operations.swift

@@ -81,12 +81,17 @@ extension BenchmarkTool {
         return cleanedString
     }
 
-    struct NameAndTarget: Hashable {
+    struct NameAndTarget: Hashable, Comparable {
         let name: String
         let target: String
+
+        static func < (lhs: NameAndTarget, rhs: NameAndTarget) -> Bool {
+            (lhs.target, lhs.name) < (rhs.target, rhs.name)
+        }
     }
 
     mutating func postProcessBenchmarkResults() throws {
+
         // Turn on buffering again for output
         setvbuf(stdout, nil, _IOFBF, Int(BUFSIZ))
 
@@ -102,7 +107,7 @@ extension BenchmarkTool {
             case .read:
                 print("Reading thresholds from \"\(thresholdsPath)\"")
 
-                var p90Thresholds: [BenchmarkIdentifier: [BenchmarkMetric: BenchmarkThresholds.AbsoluteThreshold]] = [:]
+                var p90Thresholds: [BenchmarkIdentifier: [BenchmarkMetric: BenchmarkThreshold]] = [:]
                 try benchmarks.forEach { benchmark in
                     if try shouldIncludeBenchmark(benchmark.baseName) {
                         if let thresholds = BenchmarkTool.makeBenchmarkThresholds(
@@ -148,7 +153,7 @@ extension BenchmarkTool {
                     }
                 }
 
-                var p90Thresholds: [BenchmarkIdentifier: [BenchmarkMetric: BenchmarkThresholds.AbsoluteThreshold]] = [:]
+                var p90Thresholds: [BenchmarkIdentifier: [BenchmarkMetric: BenchmarkThreshold]] = [:]
 
                 if noProgress == false {
                     print("")
@@ -301,7 +306,7 @@ extension BenchmarkTool {
                         }
                     }
 
-                    var p90Thresholds: [BenchmarkIdentifier: [BenchmarkMetric: BenchmarkThresholds.AbsoluteThreshold]] =
+                    var p90Thresholds: [BenchmarkIdentifier: [BenchmarkMetric: BenchmarkThreshold]] =
                         [:]
 
                     if let benchmarkPath = checkAbsolutePath { // load statically defined thresholds for .p90