diff --git a/Sources/swcomp/Benchmarks/BenchmarkResult.swift b/Sources/swcomp/Benchmarks/BenchmarkResult.swift index 2a5eb46e..f5ef820c 100644 --- a/Sources/swcomp/Benchmarks/BenchmarkResult.swift +++ b/Sources/swcomp/Benchmarks/BenchmarkResult.swift @@ -13,7 +13,42 @@ struct BenchmarkResult: Codable { var avg: Double var std: Double - func compare(with other: BenchmarkResult) -> Int? { + func printComparison(with other: BenchmarkResult) { + let diff = (self.avg / other.avg - 1) * 100 + let comparison = self.compare(with: other) + if diff < 0 { + switch comparison { + case 1: + print(String(format: "OK %f%% (p-value > 0.05)", diff)) + case nil: + print("Cannot compare due to unsupported iteration count.") + case -1: + print(String(format: "REG %f%% (p-value < 0.05)", diff)) + case 0: + print(String(format: "REG %f%% (p-value = 0.05)", diff)) + default: + swcompExit(.benchmarkUnknownCompResult) + } + } + else if diff > 0 { + switch comparison { + case 1: + print(String(format: "OK %f%% (p-value > 0.05)", diff)) + case nil: + print("Cannot compare due to unsupported iteration count.") + case -1: + print(String(format: "IMP %f%% (p-value < 0.05)", diff)) + case 0: + print(String(format: "IMP %f%% (p-value = 0.05)", diff)) + default: + swcompExit(.benchmarkUnknownCompResult) + } + } else { + print("OK (exact match of averages)") + } + } + + private func compare(with other: BenchmarkResult) -> Int? { let degreesOfFreedom = Double(self.iterCount + other.iterCount - 2) let t1: Double = Double(self.iterCount - 1) * pow(self.std, 2) let t2: Double = Double(other.iterCount - 1) * pow(other.std, 2) diff --git a/Sources/swcomp/Benchmarks/RunBenchmarkCommand.swift b/Sources/swcomp/Benchmarks/RunBenchmarkCommand.swift index 65cacf0c..402e1530 100644 --- a/Sources/swcomp/Benchmarks/RunBenchmarkCommand.swift +++ b/Sources/swcomp/Benchmarks/RunBenchmarkCommand.swift @@ -23,7 +23,7 @@ final class RunBenchmarkCommand: Command { @Key("-s", "--save", description: "Saves the results into the specified file") var savePath: String? - @Key("-c", "--compare", description: "Compares the results with the results saved in the specified file") + @Key("-c", "--compare", description: "Compares the results with other results saved in the specified file") var comparePath: String? @Flag("-W", "--no-warmup", description: "Disables warmup iteration") @@ -87,38 +87,7 @@ final class RunBenchmarkCommand: Command { let result = BenchmarkResult(name: self.selectedBenchmark.rawValue, input: input, iterCount: iterationCount, avg: avgSpeed, std: std) if let other = otherResults?.first(where: { $0.name == result.name && $0.input == result.input }) { - let comparison = result.compare(with: other) - let diff = (result.avg / other.avg - 1) * 100 - if diff < 0 { - switch comparison { - case 1: - print(String(format: "OK %f%% (p-value > 0.05)", diff)) - case nil: - print("Cannot compare due to unsupported iteration count.") - case -1: - print(String(format: "REG %f%% (p-value < 0.05)", diff)) - case 0: - print(String(format: "REG %f%% (p-value = 0.05)", diff)) - default: - swcompExit(.benchmarkUnknownCompResult) - } - } - else if diff > 0 { - switch comparison { - case 1: - print(String(format: "OK %f%% (p-value > 0.05)", diff)) - case nil: - print("Cannot compare due to unsupported iteration count.") - case -1: - print(String(format: "IMP %f%% (p-value < 0.05)", diff)) - case 0: - print(String(format: "IMP %f%% (p-value = 0.05)", diff)) - default: - swcompExit(.benchmarkUnknownCompResult) - } - } else { - print("OK (exact match of averages)") - } + result.printComparison(with: other) } results.append(result)