From 7892e6d74fa1a86aa6c2b776d8b7cfb95da122ec Mon Sep 17 00:00:00 2001 From: Brian Floersch Date: Thu, 26 Mar 2026 19:23:05 -0400 Subject: [PATCH 1/5] Improve benchmark harness --- Benchmarks/.gitignore | 3 +- Benchmarks/Readme.md | 22 +- Benchmarks/Sources/Benchmarks/main.swift | 780 ++++++++++++++++------- Benchmarks/compare_swift_benchmarks.py | 49 ++ Benchmarks/run_swift_benchmarks.sh | 47 ++ 5 files changed, 666 insertions(+), 235 deletions(-) create mode 100755 Benchmarks/compare_swift_benchmarks.py create mode 100755 Benchmarks/run_swift_benchmarks.sh diff --git a/Benchmarks/.gitignore b/Benchmarks/.gitignore index 4ab79eb..e47d58b 100644 --- a/Benchmarks/.gitignore +++ b/Benchmarks/.gitignore @@ -8,5 +8,6 @@ DerivedData/ .swiftpm/xcode/package.xcworkspace/contents.xcworkspacedata .swiftpm/xcode/xcshareddata .netrc +/results -Examples \ No newline at end of file +Examples diff --git a/Benchmarks/Readme.md b/Benchmarks/Readme.md index 4786d4c..8ab8ef4 100644 --- a/Benchmarks/Readme.md +++ b/Benchmarks/Readme.md @@ -4,7 +4,26 @@ This sub-project attempts to compare async channels with golang. ## Setup -All swift tests were run with `10` rounds (averaged) with default release optimizations.\ +The Swift benchmark harness now supports configurable rounds, warmups, and JSON output for version-to-version comparisons. + +Recommended workflow: + +```bash +BENCHMARK_NAME=swift-6.2.3 Benchmarks/run_swift_benchmarks.sh +BENCHMARK_NAME=swift-6.3.0 SWIFT_BIN=$HOME/.swiftly/bin/swift Benchmarks/run_swift_benchmarks.sh +Benchmarks/compare_swift_benchmarks.py \ + Benchmarks/results/swift-6.2.3-.json \ + Benchmarks/results/swift-6.3.0-.json +``` + +Useful knobs: + +```bash +xcrun swift run -c release --package-path Benchmarks Benchmarks --help +``` + +Results written to `Benchmarks/results/` are ignored by git so local benchmark output does not dirty the repository. + All Go tests were written as go micro benchmarks.\ All tests performed on an M1 max @@ -118,4 +137,3 @@ Async algorithms channel seems to fall apart with write contention on non intege ## Future work I have not published benchmarks on linux. Linux uses `pthread_mutex_t` and from my limited testing, performance is slightly worse than MacOS, in line compared to AsyncAlgorithms, yet still significantly slower than go. - diff --git a/Benchmarks/Sources/Benchmarks/main.swift b/Benchmarks/Sources/Benchmarks/main.swift index 356998f..8c5ffc5 100644 --- a/Benchmarks/Sources/Benchmarks/main.swift +++ b/Benchmarks/Sources/Benchmarks/main.swift @@ -1,317 +1,633 @@ import Foundation -import AsyncChannels -import CoreFoundation +import Dispatch import AsyncAlgorithms +import AsyncChannels -let iterations = 10 - -protocol Initializable: Sendable { +protocol BenchmarkPayload: Sendable { init() } -extension Int : Initializable {} -extension String : Initializable { +extension Int: BenchmarkPayload {} + +extension String: BenchmarkPayload { init() { self.init("My test string for benchmarking") } } -struct ValueData: Initializable { +struct ValueData: BenchmarkPayload { let foo: String let bar: Int + init() { foo = "My test string for benchmarking" bar = 1234 } } -final class RefData: Initializable { +final class RefData: BenchmarkPayload { let foo: String let bar: Int + required init() { foo = "My test string for benchmarking" bar = 1234 } } -// MARK: Run Tests +enum BenchmarkError: Error, CustomStringConvertible { + case invalidArgument(String) + case missingValue(String) -print() -print("Starting benchmarks with \(iterations) rounds") -print() -print("Test | Type | Execution Time(ms)") -print("-----|------|---------------") - -await run(Int.self) -await run(String.self) -await run(ValueData.self) -await run(RefData.self) - -await runAsyncAlg(Int.self) -await runAsyncAlg(String.self) -await runAsyncAlg(ValueData.self) -await runAsyncAlg(RefData.self) - -class TestData { - let foo: String - let bar: Int - - init() { - self.foo = "foo" - self.bar = 123 + var description: String { + switch self { + case .invalidArgument(let argument): + return "Invalid argument: \(argument)" + case .missingValue(let flag): + return "Missing value for \(flag)" + } } } -class TestDataDeinit: @unchecked Sendable { - let foo: String - let bar: Int - - init() { - self.foo = "foo" - self.bar = 123 - } - - deinit { - print("deinited") - } +enum OutputFormat: String, Codable { + case markdown + case json } -struct TestStruct { - var foo: String - var bar: Int - - init() { - self.foo = "foo" - self.bar = 123 +struct BenchmarkConfig: Codable { + let rounds: Int + let warmupRounds: Int + let writes: Int + let syncWrites: Int + let selectWritesPerChannel: Int + let buffer: Int + let includeAsyncAlgorithms: Bool + let outputFormat: OutputFormat + + static let `default` = BenchmarkConfig( + rounds: 10, + warmupRounds: 1, + writes: 1_000_000, + syncWrites: 5_000_000, + selectWritesPerChannel: 100_000, + buffer: 100, + includeAsyncAlgorithms: true, + outputFormat: .markdown + ) + + init( + rounds: Int, + warmupRounds: Int, + writes: Int, + syncWrites: Int, + selectWritesPerChannel: Int, + buffer: Int, + includeAsyncAlgorithms: Bool, + outputFormat: OutputFormat + ) { + self.rounds = rounds + self.warmupRounds = warmupRounds + self.writes = writes + self.syncWrites = syncWrites + self.selectWritesPerChannel = selectWritesPerChannel + self.buffer = buffer + self.includeAsyncAlgorithms = includeAsyncAlgorithms + self.outputFormat = outputFormat } -} -func run(_ type: T.Type) async { - formatResult(await testSPSC(type)) - formatResult(await testMPSC(type)) - formatResult(await testSPMC(type)) - formatResult(await testMPMC(type)) - formatResult(await testMPSCWriteContention(type)) - - formatResult(await testSPSCBuffered(type)) - formatResult(await testMPSCBuffered(type)) - formatResult(await testSPMCBuffered(type)) - formatResult(await testMPMCBuffered(type)) - formatResult(await testMPSCWriteContentionBuffered(type)) - - formatResult(await testSyncRw(type)) - formatResult(await testMultiSelect(type)) -} - -func runAsyncAlg(_ type: T.Type) async { - formatResult(await testAsyncAlgSPSC(type)) - formatResult(await testAsyncAlgMPSC(type)) - formatResult(await testAsyncAlgSPMC(type)) - formatResult(await testAsyncAlgMPMC(type)) - formatResult(await testAsyncAlgMPSCWriteContention(type)) -} + init(arguments: [String]) throws { + var rounds = Self.default.rounds + var warmupRounds = Self.default.warmupRounds + var writes = Self.default.writes + var syncWrites = Self.default.syncWrites + var selectWritesPerChannel = Self.default.selectWritesPerChannel + var buffer = Self.default.buffer + var includeAsyncAlgorithms = Self.default.includeAsyncAlgorithms + var outputFormat = Self.default.outputFormat + + var index = 0 + while index < arguments.count { + let argument = arguments[index] + switch argument { + case "--rounds": + rounds = try Self.parseInt(arguments, index: &index, flag: argument) + case "--warmup": + warmupRounds = try Self.parseInt(arguments, index: &index, flag: argument) + case "--writes": + writes = try Self.parseInt(arguments, index: &index, flag: argument) + case "--sync-writes": + syncWrites = try Self.parseInt(arguments, index: &index, flag: argument) + case "--select-writes": + selectWritesPerChannel = try Self.parseInt(arguments, index: &index, flag: argument) + case "--buffer": + buffer = try Self.parseInt(arguments, index: &index, flag: argument) + case "--skip-async-algorithms": + includeAsyncAlgorithms = false + case "--format": + index += 1 + guard index < arguments.count else { + throw BenchmarkError.missingValue(argument) + } + guard let format = OutputFormat(rawValue: arguments[index]) else { + throw BenchmarkError.invalidArgument("\(argument) \(arguments[index])") + } + outputFormat = format + case "--help", "-h": + printUsage() + Foundation.exit(0) + default: + throw BenchmarkError.invalidArgument(argument) + } + index += 1 + } -func timeIt(iterations: Int, block: () async -> ()) async -> Double { - var sum: CFAbsoluteTime = 0 - for _ in 0..(_ type: T.Type, writes: Int = 1_000_000) async -> (String, String, Double) { - let result = await runMPMC(type, producers: 1, consumers: 1, writes: writes) - return ("SPSC", "\(T.self)", result) + private static func parseInt(_ arguments: [String], index: inout Int, flag: String) throws -> Int { + index += 1 + guard index < arguments.count else { + throw BenchmarkError.missingValue(flag) + } + guard let value = Int(arguments[index]) else { + throw BenchmarkError.invalidArgument("\(flag) \(arguments[index])") + } + return value + } } -func testMPSC(_ type: T.Type, producers: Int = 5, writes: Int = 1_000_000) async -> (String, String, Double) { - let result = await runMPMC(type, producers: producers, consumers: 1, writes: writes) - return ("MPSC", "\(T.self)", result) +struct BenchmarkResult: Codable { + let library: String + let name: String + let type: String + let operations: Int + let rounds: Int + let warmupRounds: Int + let averageMs: Double + let medianMs: Double + let minMs: Double + let maxMs: Double + let operationsPerSecond: Double + let samplesMs: [Double] } -func testSPMC(_ type: T.Type, consumers: Int = 5, writes: Int = 1_000_000) async -> (String, String, Double) { - let result = await runMPMC(type, producers: 1, consumers: consumers, writes: writes) - return ("SPMC", "\(T.self)", result) +struct BenchmarkReport: Codable { + let generatedAt: String + let toolchain: String + let host: String + let config: BenchmarkConfig + let results: [BenchmarkResult] } -func testMPMC(_ type: T.Type, producers: Int = 5, consumers: Int = 5, writes: Int = 1_000_000, buffer: Int = 100) async -> (String, String, Double) { - let result = await runMPMC(type, producers: producers, consumers: consumers, writes: writes, buffer: buffer) - return ("MPMC", "\(T.self)", result) +func printUsage() { + let usage = """ + Usage: Benchmarks [options] + + --format markdown|json + --rounds + --warmup + --writes + --sync-writes + --select-writes + --buffer + --skip-async-algorithms + --help + """ + print(usage) } -func testMPSCWriteContention(_ type: T.Type, producers: Int = 1000, writes: Int = 1_000_000) async -> (String, String, Double) { - let result = await runMPMC(type, producers: producers, consumers: 1, writes: writes) - return ("MPSC Write Contention", "\(T.self)", result) -} +func measureScenario( + library: String, + name: String, + type: T.Type, + operations: Int, + config: BenchmarkConfig, + block: () async -> Void +) async -> BenchmarkResult { + for _ in 0..(_ type: T.Type, writes: Int = 1_000_000, buffer: Int = 100) async -> (String, String, Double) { - let result = await runMPMC(type, producers: 1, consumers: 1, writes: writes, buffer: buffer) - return ("SPSC Buffered(\(buffer))", "\(T.self)", result) -} + for _ in 0..(_ type: T.Type, producers: Int = 5, writes: Int = 1_000_000, buffer: Int = 100) async -> (String, String, Double) { - let result = await runMPMC(type, producers: producers, consumers: 1, writes: writes, buffer: buffer) - return ("MPSC Buffered(\(buffer))", "\(T.self)", result) -} + let averageMs = samplesMs.reduce(0, +) / Double(samplesMs.count) + let sorted = samplesMs.sorted() + let medianMs: Double + if sorted.count.isMultiple(of: 2) { + let mid = sorted.count / 2 + medianMs = (sorted[mid - 1] + sorted[mid]) / 2 + } else { + medianMs = sorted[sorted.count / 2] + } -func testSPMCBuffered(_ type: T.Type, consumers: Int = 5, writes: Int = 1_000_000, buffer: Int = 100) async -> (String, String, Double) { - let result = await runMPMC(type, producers: 1, consumers: consumers, writes: writes, buffer: buffer) - return ("SPMC Buffered(\(buffer))", "\(T.self)", result) + let operationsPerSecond = Double(operations) / (averageMs / 1000) + + return BenchmarkResult( + library: library, + name: name, + type: String(describing: type), + operations: operations, + rounds: config.rounds, + warmupRounds: config.warmupRounds, + averageMs: averageMs, + medianMs: medianMs, + minMs: sorted.first ?? 0, + maxMs: sorted.last ?? 0, + operationsPerSecond: operationsPerSecond, + samplesMs: samplesMs + ) } -func testMPMCBuffered(_ type: T.Type, producers: Int = 5, consumers: Int = 5, writes: Int = 1_000_000, buffer: Int = 100) async -> (String, String, Double) { - let result = await runMPMC(type, producers: producers, consumers: consumers, writes: writes, buffer: buffer) - return ("MPMC Buffered(\(buffer))", "\(T.self)", result) +func run(_ type: T.Type, config: BenchmarkConfig) async -> [BenchmarkResult] { + var results = [BenchmarkResult]() + + results.append(await measureScenario( + library: "AsyncChannels", + name: "SPSC", + type: type, + operations: config.writes, + config: config + ) { + await runMPMC(type, producers: 1, consumers: 1, writes: config.writes) + }) + + results.append(await measureScenario( + library: "AsyncChannels", + name: "MPSC", + type: type, + operations: config.writes, + config: config + ) { + await runMPMC(type, producers: 5, consumers: 1, writes: config.writes) + }) + + results.append(await measureScenario( + library: "AsyncChannels", + name: "SPMC", + type: type, + operations: config.writes, + config: config + ) { + await runMPMC(type, producers: 1, consumers: 5, writes: config.writes) + }) + + results.append(await measureScenario( + library: "AsyncChannels", + name: "MPMC", + type: type, + operations: config.writes, + config: config + ) { + await runMPMC(type, producers: 5, consumers: 5, writes: config.writes) + }) + + results.append(await measureScenario( + library: "AsyncChannels", + name: "MPSC Write Contention", + type: type, + operations: config.writes, + config: config + ) { + await runMPMC(type, producers: 1000, consumers: 1, writes: config.writes) + }) + + results.append(await measureScenario( + library: "AsyncChannels", + name: "SPSC Buffered(\(config.buffer))", + type: type, + operations: config.writes, + config: config + ) { + await runMPMC(type, producers: 1, consumers: 1, writes: config.writes, buffer: config.buffer) + }) + + results.append(await measureScenario( + library: "AsyncChannels", + name: "MPSC Buffered(\(config.buffer))", + type: type, + operations: config.writes, + config: config + ) { + await runMPMC(type, producers: 5, consumers: 1, writes: config.writes, buffer: config.buffer) + }) + + results.append(await measureScenario( + library: "AsyncChannels", + name: "SPMC Buffered(\(config.buffer))", + type: type, + operations: config.writes, + config: config + ) { + await runMPMC(type, producers: 1, consumers: 5, writes: config.writes, buffer: config.buffer) + }) + + results.append(await measureScenario( + library: "AsyncChannels", + name: "MPMC Buffered(\(config.buffer))", + type: type, + operations: config.writes, + config: config + ) { + await runMPMC(type, producers: 5, consumers: 5, writes: config.writes, buffer: config.buffer) + }) + + results.append(await measureScenario( + library: "AsyncChannels", + name: "MPSC Write Contention Buffered(\(config.buffer))", + type: type, + operations: config.writes, + config: config + ) { + await runMPMC(type, producers: 1000, consumers: 1, writes: config.writes, buffer: config.buffer) + }) + + results.append(await measureScenario( + library: "AsyncChannels", + name: "SyncRW", + type: type, + operations: config.syncWrites, + config: config + ) { + await runSyncRw(type, writes: config.syncWrites) + }) + + results.append(await measureScenario( + library: "AsyncChannels", + name: "Channel multi-select", + type: type, + operations: config.selectWritesPerChannel * 6, + config: config + ) { + await runMultiSelect(type, writesPerChannel: config.selectWritesPerChannel) + }) + + return results } -func testMPSCWriteContentionBuffered(_ type: T.Type, producers: Int = 1000, writes: Int = 1_000_000, buffer: Int = 100) async -> (String, String, Double) { - let result = await runMPMC(type, producers: producers, consumers: 1, writes: writes, buffer: buffer) - return ("MPSC Write Contention Buffered(\(buffer))", "\(T.self)", result) +func runAsyncAlgorithms(_ type: T.Type, config: BenchmarkConfig) async -> [BenchmarkResult] { + var results = [BenchmarkResult]() + + results.append(await measureScenario( + library: "AsyncAlgorithms", + name: "SPSC", + type: type, + operations: config.writes, + config: config + ) { + await runMPMCAsyncAlg(type, producers: 1, consumers: 1, writes: config.writes) + }) + + results.append(await measureScenario( + library: "AsyncAlgorithms", + name: "MPSC", + type: type, + operations: config.writes, + config: config + ) { + await runMPMCAsyncAlg(type, producers: 5, consumers: 1, writes: config.writes) + }) + + results.append(await measureScenario( + library: "AsyncAlgorithms", + name: "SPMC", + type: type, + operations: config.writes, + config: config + ) { + await runMPMCAsyncAlg(type, producers: 1, consumers: 5, writes: config.writes) + }) + + results.append(await measureScenario( + library: "AsyncAlgorithms", + name: "MPMC", + type: type, + operations: config.writes, + config: config + ) { + await runMPMCAsyncAlg(type, producers: 5, consumers: 5, writes: config.writes) + }) + + results.append(await measureScenario( + library: "AsyncAlgorithms", + name: "MPSC Write Contention", + type: type, + operations: config.writes, + config: config + ) { + await runMPMCAsyncAlg(type, producers: 1000, consumers: 1, writes: config.writes) + }) + + return results } -// MARK: Async alg comparison tests +func runMPMC( + _ type: T.Type, + producers: Int, + consumers: Int, + writes: Int, + buffer: Int = 0 +) async { + let channel = Channel(capacity: buffer) + + async let writeGroup: () = withTaskGroup(of: Void.self) { group in + for _ in 0..(_ type: T.Type, writes: Int = 1_000_000) async -> (String, String, Double) { - let result = await runMPMCAsyncAlg(type, producers: 1, consumers: 1, writes: writes) - return ("SPSC Async alg", "\(T.self)", result) -} + async let readGroup: () = withTaskGroup(of: Void.self) { group in + for _ in 0..(_ type: T.Type, producers: Int = 5, writes: Int = 1_000_000) async -> (String, String, Double) { - let result = await runMPMCAsyncAlg(type, producers: producers, consumers: 1, writes: writes) - return ("MPSC Async alg", "\(T.self)", result) + await writeGroup + channel.close() + await readGroup } -func testAsyncAlgSPMC(_ type: T.Type, consumers: Int = 5, writes: Int = 1_000_000) async -> (String, String, Double) { - let result = await runMPMCAsyncAlg(type, producers: 1, consumers: consumers, writes: writes) - return ("SPMC Async alg", "\(T.self)", result) +func runSyncRw(_ type: T.Type, writes: Int) async { + let channel = Channel(capacity: 1) + for _ in 0..(_ type: T.Type, producers: Int = 5, consumers: Int = 5, writes: Int = 1_000_000, buffer: Int = 100) async -> (String, String, Double) { - let result = await runMPMCAsyncAlg(type, producers: producers, consumers: consumers, writes: writes, buffer: buffer) - return ("MPMC Async alg", "\(T.self)", result) -} +func runMultiSelect(_ type: T.Type, writesPerChannel: Int) async { + let channels = [ + Channel(), + Channel(), + Channel(), + Channel(), + Channel(), + Channel() + ] + + for channel in channels { + Task { + for _ in 0..(_ type: T.Type, producers: Int = 1000, writes: Int = 1_000_000) async -> (String, String, Double) { - let result = await runMPMCAsyncAlg(type, producers: producers, consumers: 1, writes: writes) - return ("MPSC Async alg Write Contention", "\(T.self)", result) + var received = 0 + let totalWrites = writesPerChannel * channels.count + + while received < totalWrites { + await select { + receive(channels[0]) { received += 1 } + receive(channels[1]) { received += 1 } + receive(channels[2]) { received += 1 } + receive(channels[3]) { received += 1 } + receive(channels[4]) { received += 1 } + receive(channels[5]) { received += 1 } + } + } } - -func runMPMC(_ type: T.Type, producers: Int, consumers: Int, writes: Int, buffer: Int = 0) async -> Double { - return await timeIt(iterations: iterations) { - let a = Channel(capacity: buffer) - - async let writeGroup: () = withTaskGroup(of: Void.self) { group in - for _ in 0..( + _ type: T.Type, + producers: Int, + consumers: Int, + writes: Int +) async { + let channel = AsyncChannel() + + async let writeGroup: () = withTaskGroup(of: Void.self) { group in + for _ in 0..(_ type: T.Type, writes: Int = 5_000_000) async -> (String, String, Double) { - let result = await timeIt(iterations: iterations) { - let a = Channel(capacity: 1) - - for _ in 0..(_ type: T.Type) async -> (String, String, Double) { - let result = await timeIt(iterations: iterations) { - let a = Channel() - let b = Channel() - let c = Channel() - let d = Channel() - let e = Channel() - let f = Channel() - - for chan in [a, b, c, d, e, f] { - Task { - for _ in (0..<100_000) { - await chan <- T() - } +func generateReport(config: BenchmarkConfig) async -> BenchmarkReport { + var results = [BenchmarkResult]() + + let payloadTypes: [any BenchmarkPayload.Type] = [ + Int.self, + String.self, + ValueData.self, + RefData.self + ] + + for payloadType in payloadTypes { + switch payloadType { + case is Int.Type: + results.append(contentsOf: await run(Int.self, config: config)) + if config.includeAsyncAlgorithms { + results.append(contentsOf: await runAsyncAlgorithms(Int.self, config: config)) } - } - - var sum = 0 - - while sum < 6 * 100_000 { - await select { - receive(a) { sum += 1 } - receive(b) { sum += 1 } - receive(c) { sum += 1 } - receive(d) { sum += 1 } - receive(e) { sum += 1 } - receive(f) { sum += 1 } + case is String.Type: + results.append(contentsOf: await run(String.self, config: config)) + if config.includeAsyncAlgorithms { + results.append(contentsOf: await runAsyncAlgorithms(String.self, config: config)) + } + case is ValueData.Type: + results.append(contentsOf: await run(ValueData.self, config: config)) + if config.includeAsyncAlgorithms { + results.append(contentsOf: await runAsyncAlgorithms(ValueData.self, config: config)) + } + case is RefData.Type: + results.append(contentsOf: await run(RefData.self, config: config)) + if config.includeAsyncAlgorithms { + results.append(contentsOf: await runAsyncAlgorithms(RefData.self, config: config)) } + default: + break } } - return ("Channel multi-select", "\(T.self)", result) + + let formatter = ISO8601DateFormatter() + let toolchain = ProcessInfo.processInfo.environment["BENCHMARK_TOOLCHAIN_LABEL"] ?? "unknown" + + return BenchmarkReport( + generatedAt: formatter.string(from: Date()), + toolchain: toolchain, + host: ProcessInfo.processInfo.hostName, + config: config, + results: results + ) } +func formatMarkdown(_ report: BenchmarkReport) -> String { + var lines = [String]() + lines.append("") + lines.append("Toolchain: \(report.toolchain)") + lines.append("Host: \(report.host)") + lines.append("Rounds: \(report.config.rounds), warmup: \(report.config.warmupRounds)") + lines.append("Writes: \(report.config.writes), sync writes: \(report.config.syncWrites), select writes/channel: \(report.config.selectWritesPerChannel)") + lines.append("") + lines.append("Library | Test | Type | Avg (ms) | Median (ms) | Min (ms) | Max (ms) | Ops/s") + lines.append("--------|------|------|----------|-------------|----------|----------|------") + + for result in report.results { + lines.append( + "\(result.library) | \(result.name) | `\(result.type)` | `\(String(format: "%.2f", result.averageMs))` | `\(String(format: "%.2f", result.medianMs))` | `\(String(format: "%.2f", result.minMs))` | `\(String(format: "%.2f", result.maxMs))` | `\(String(format: "%.0f", result.operationsPerSecond))`" + ) + } + return lines.joined(separator: "\n") +} -func runMPMCAsyncAlg(_ type: T.Type, producers: Int, consumers: Int, writes: Int, buffer: Int = 0) async -> Double { - return await timeIt(iterations: iterations) { - let a = AsyncChannel() - - async let writeGroup: () = withTaskGroup(of: Void.self) { group in - for _ in 0.. dict: + with open(path, "r", encoding="utf-8") as handle: + return json.load(handle) + + +def key(result: dict) -> tuple[str, str, str]: + return (result["library"], result["name"], result["type"]) + + +def main() -> int: + if len(sys.argv) != 3: + print("usage: compare_swift_benchmarks.py ", file=sys.stderr) + return 1 + + baseline = load(sys.argv[1]) + candidate = load(sys.argv[2]) + + baseline_results = {key(result): result for result in baseline["results"]} + candidate_results = {key(result): result for result in candidate["results"]} + + shared_keys = sorted(set(baseline_results) & set(candidate_results)) + + print(f"Baseline: {Path(sys.argv[1]).name} ({baseline['toolchain']})") + print(f"Candidate: {Path(sys.argv[2]).name} ({candidate['toolchain']})") + print() + print("Library | Test | Type | Baseline Avg (ms) | Candidate Avg (ms) | Delta %") + print("--------|------|------|-------------------|--------------------|--------") + + for current_key in shared_keys: + before = baseline_results[current_key]["averageMs"] + after = candidate_results[current_key]["averageMs"] + delta = ((after - before) / before) * 100 + library, name, result_type = current_key + print( + f"{library} | {name} | `{result_type}` | `{before:.2f}` | `{after:.2f}` | `{delta:+.2f}%`" + ) + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/Benchmarks/run_swift_benchmarks.sh b/Benchmarks/run_swift_benchmarks.sh new file mode 100755 index 0000000..21b9675 --- /dev/null +++ b/Benchmarks/run_swift_benchmarks.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SWIFT_BIN="${SWIFT_BIN:-swift}" +ROUNDS="${ROUNDS:-6}" +WARMUP="${WARMUP:-1}" +WRITES="${WRITES:-1000000}" +SYNC_WRITES="${SYNC_WRITES:-5000000}" +SELECT_WRITES="${SELECT_WRITES:-100000}" +BUFFER="${BUFFER:-100}" +OUTPUT_DIR="${OUTPUT_DIR:-Benchmarks/results}" +SCRATCH_ROOT="${SCRATCH_ROOT:-/tmp/asyncchannels-benchmarks}" +BENCHMARK_NAME="${BENCHMARK_NAME:-}" +SKIP_ASYNC_ALGORITHMS="${SKIP_ASYNC_ALGORITHMS:-0}" + +mkdir -p "${OUTPUT_DIR}" + +if [[ -z "${BENCHMARK_NAME}" ]]; then + BENCHMARK_NAME="$("${SWIFT_BIN}" --version | head -n1 | tr ' /()' '_' | tr -cd '[:alnum:]_.-')" +fi + +TIMESTAMP="$(date +%Y%m%d-%H%M%S)" +OUTPUT_PATH="${OUTPUT_DIR}/${BENCHMARK_NAME}-${TIMESTAMP}.json" +SCRATCH_PATH="${SCRATCH_ROOT}/${BENCHMARK_NAME}" + +CMD=( + "${SWIFT_BIN}" run -c release --package-path Benchmarks --scratch-path "${SCRATCH_PATH}" Benchmarks + --format json + --rounds "${ROUNDS}" + --warmup "${WARMUP}" + --writes "${WRITES}" + --sync-writes "${SYNC_WRITES}" + --select-writes "${SELECT_WRITES}" + --buffer "${BUFFER}" +) + +if [[ "${SKIP_ASYNC_ALGORITHMS}" == "1" ]]; then + CMD+=(--skip-async-algorithms) +fi + +echo "Running benchmark suite with ${SWIFT_BIN}" +echo "Writing ${OUTPUT_PATH}" + +BENCHMARK_TOOLCHAIN_LABEL="${BENCHMARK_NAME}" "${CMD[@]}" > "${OUTPUT_PATH}" + +echo "Finished: ${OUTPUT_PATH}" From c8094340c799ea0a7d32d6f3d09b6f44f9c4ebca Mon Sep 17 00:00:00 2001 From: Brian Floersch Date: Thu, 26 Mar 2026 19:34:39 -0400 Subject: [PATCH 2/5] Add Go benchmark runner --- Benchmarks/Readme.md | 10 ++ Benchmarks/compare_library_benchmarks.py | 60 +++++++++ Benchmarks/parse_go_benchmarks.py | 156 +++++++++++++++++++++++ Benchmarks/run_go_benchmarks.sh | 40 ++++++ 4 files changed, 266 insertions(+) create mode 100755 Benchmarks/compare_library_benchmarks.py create mode 100755 Benchmarks/parse_go_benchmarks.py create mode 100755 Benchmarks/run_go_benchmarks.sh diff --git a/Benchmarks/Readme.md b/Benchmarks/Readme.md index 8ab8ef4..efec888 100644 --- a/Benchmarks/Readme.md +++ b/Benchmarks/Readme.md @@ -16,6 +16,16 @@ Benchmarks/compare_swift_benchmarks.py \ Benchmarks/results/swift-6.3.0-.json ``` +For cross-library comparison on the same machine: + +```bash +BENCHMARK_NAME=swift-6.3.0 SWIFT_BIN=$HOME/.swiftly/bin/swift Benchmarks/run_swift_benchmarks.sh +BENCHMARK_NAME=go-1.26.1 Benchmarks/run_go_benchmarks.sh +Benchmarks/compare_library_benchmarks.py \ + Benchmarks/results/swift-6.3.0-.json \ + Benchmarks/results/go-1.26.1-.json +``` + Useful knobs: ```bash diff --git a/Benchmarks/compare_library_benchmarks.py b/Benchmarks/compare_library_benchmarks.py new file mode 100755 index 0000000..3319600 --- /dev/null +++ b/Benchmarks/compare_library_benchmarks.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 + +import json +import sys +from collections import defaultdict + + +def load(path: str) -> dict: + with open(path, "r", encoding="utf-8") as handle: + return json.load(handle) + + +def main() -> int: + if len(sys.argv) < 2: + print("usage: compare_library_benchmarks.py [report.json ...]", file=sys.stderr) + return 1 + + merged = defaultdict(dict) + for report_path in sys.argv[1:]: + report = load(report_path) + for result in report["results"]: + merged[(result["name"], result["type"])][result["library"]] = result + + libraries = sorted({library for values in merged.values() for library in values}) + if not libraries: + return 0 + + header = ["Test", "Type"] + [f"{library} Avg (ms)" for library in libraries] + if "Go" in libraries: + header += [f"{library} vs Go" for library in libraries if library != "Go"] + print(" | ".join(header)) + print(" | ".join(["---"] * len(header))) + + for (name, result_type), values in sorted(merged.items()): + if len(values) < 2: + continue + row = [name, f"`{result_type}`"] + for library in libraries: + result = values.get(library) + row.append(f"`{result['averageMs']:.2f}`" if result else "") + + go_result = values.get("Go") + if go_result: + for library in libraries: + if library == "Go": + continue + result = values.get(library) + if result: + delta = ((result["averageMs"] - go_result["averageMs"]) / go_result["averageMs"]) * 100 + row.append(f"`{delta:+.2f}%`") + else: + row.append("") + + print(" | ".join(row)) + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/Benchmarks/parse_go_benchmarks.py b/Benchmarks/parse_go_benchmarks.py new file mode 100755 index 0000000..22d63ee --- /dev/null +++ b/Benchmarks/parse_go_benchmarks.py @@ -0,0 +1,156 @@ +#!/usr/bin/env python3 + +import json +import re +import statistics +import sys +from pathlib import Path + + +BENCHMARK_NAME_MAP = { + "SPSC": "SPSC", + "MPSC": "MPSC", + "SPMC": "SPMC", + "MPMC": "MPMC", + "MPSCWriteContention": "MPSC Write Contention", + "SPSCBuffered": "SPSC Buffered({buffer})", + "MPSCBuffered": "MPSC Buffered({buffer})", + "SPMCBuffered": "SPMC Buffered({buffer})", + "MPMCBuffered": "MPMC Buffered({buffer})", + "MPSCWriteContentionBuffered": "MPSC Write Contention Buffered({buffer})", + "SyncRw": "SyncRW", + "MultiSelect": "Channel multi-select", +} + +OPERATIONS_MAP = { + "SPSC": "writes", + "MPSC": "writes", + "SPMC": "writes", + "MPMC": "writes", + "MPSCWriteContention": "writes", + "SPSCBuffered": "writes", + "MPSCBuffered": "writes", + "SPMCBuffered": "writes", + "MPMCBuffered": "writes", + "MPSCWriteContentionBuffered": "writes", + "SyncRw": "sync_writes", + "MultiSelect": "select_total", +} + +LINE_RE = re.compile( + r"^Benchmark(?P[A-Za-z0-9]+)-\d+\s+" + r"(?P\d+)\s+" + r"(?P\d+(?:\.\d+)?) ns/op\s+" + r"(?P\d+) B/op\s+" + r"(?P\d+) allocs/op$" +) + + +def usage() -> int: + print( + "usage: parse_go_benchmarks.py ", + file=sys.stderr, + ) + return 1 + + +def median(values: list[float]) -> float: + return statistics.median(values) + + +def main() -> int: + if len(sys.argv) != 7: + return usage() + + raw_path = Path(sys.argv[1]) + toolchain = sys.argv[2] + writes = int(sys.argv[3]) + sync_writes = int(sys.argv[4]) + select_writes = int(sys.argv[5]) + buffer = int(sys.argv[6]) + + metadata: dict[str, str] = {} + grouped: dict[str, list[dict[str, float]]] = {} + + for line in raw_path.read_text(encoding="utf-8").splitlines(): + if line.startswith("goos:"): + metadata["goos"] = line.split(":", 1)[1].strip() + elif line.startswith("goarch:"): + metadata["goarch"] = line.split(":", 1)[1].strip() + elif line.startswith("pkg:"): + metadata["pkg"] = line.split(":", 1)[1].strip() + elif line.startswith("cpu:"): + metadata["cpu"] = line.split(":", 1)[1].strip() + + match = LINE_RE.match(line.strip()) + if not match: + continue + + grouped.setdefault(match.group("name"), []).append( + { + "ns_op": float(match.group("ns_op")), + "bytes_op": float(match.group("bytes_op")), + "allocs_op": float(match.group("allocs_op")), + } + ) + + config = { + "rounds": max((len(samples) for samples in grouped.values()), default=0), + "warmupRounds": 0, + "writes": writes, + "syncWrites": sync_writes, + "selectWritesPerChannel": select_writes, + "buffer": buffer, + "includeAsyncAlgorithms": False, + "outputFormat": "json", + } + + operation_values = { + "writes": writes, + "sync_writes": sync_writes, + "select_total": select_writes * 6, + } + + results = [] + for raw_name, samples in sorted(grouped.items()): + scenario_name_template = BENCHMARK_NAME_MAP[raw_name] + scenario_name = scenario_name_template.format(buffer=buffer) + sample_ms = [sample["ns_op"] / 1_000_000 for sample in samples] + avg_ms = statistics.fmean(sample_ms) + ops_count = operation_values[OPERATIONS_MAP[raw_name]] + + results.append( + { + "library": "Go", + "name": scenario_name, + "type": "Int", + "operations": ops_count, + "rounds": len(samples), + "warmupRounds": 0, + "averageMs": avg_ms, + "medianMs": median(sample_ms), + "minMs": min(sample_ms), + "maxMs": max(sample_ms), + "operationsPerSecond": ops_count / (avg_ms / 1000), + "samplesMs": sample_ms, + "bytesPerOp": statistics.fmean(sample["bytes_op"] for sample in samples), + "allocsPerOp": statistics.fmean(sample["allocs_op"] for sample in samples), + } + ) + + report = { + "generatedAt": None, + "toolchain": toolchain, + "host": metadata.get("cpu", "unknown"), + "config": config, + "goMetadata": metadata, + "results": results, + } + + json.dump(report, sys.stdout, indent=2, sort_keys=True) + print() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/Benchmarks/run_go_benchmarks.sh b/Benchmarks/run_go_benchmarks.sh new file mode 100755 index 0000000..1ffbe65 --- /dev/null +++ b/Benchmarks/run_go_benchmarks.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash + +set -euo pipefail + +GO_BIN="${GO_BIN:-go}" +COUNT="${COUNT:-5}" +WRITES="${WRITES:-1000000}" +SYNC_WRITES="${SYNC_WRITES:-5000000}" +SELECT_WRITES="${SELECT_WRITES:-100000}" +BUFFER="${BUFFER:-100}" +OUTPUT_DIR="${OUTPUT_DIR:-Benchmarks/results}" +BENCHMARK_NAME="${BENCHMARK_NAME:-}" + +mkdir -p "${OUTPUT_DIR}" + +if [[ -z "${BENCHMARK_NAME}" ]]; then + BENCHMARK_NAME="$("${GO_BIN}" version | tr ' /()' '_' | tr -cd '[:alnum:]_.-')" +fi + +TIMESTAMP="$(date +%Y%m%d-%H%M%S)" +RAW_OUTPUT_PATH="${OUTPUT_DIR}/${BENCHMARK_NAME}-${TIMESTAMP}.txt" +JSON_OUTPUT_PATH="${OUTPUT_DIR}/${BENCHMARK_NAME}-${TIMESTAMP}.json" + +echo "Running Go benchmark suite with ${GO_BIN}" +echo "Writing raw output to ${RAW_OUTPUT_PATH}" + +( + cd Benchmarks/golang + "${GO_BIN}" test -run '^$' -bench . -benchmem -count "${COUNT}" +) | tee "${RAW_OUTPUT_PATH}" + +Benchmarks/parse_go_benchmarks.py \ + "${RAW_OUTPUT_PATH}" \ + "${BENCHMARK_NAME}" \ + "${WRITES}" \ + "${SYNC_WRITES}" \ + "${SELECT_WRITES}" \ + "${BUFFER}" > "${JSON_OUTPUT_PATH}" + +echo "Finished: ${JSON_OUTPUT_PATH}" From 4640e93a330d8d59aa66765dfb2560bc8d68407c Mon Sep 17 00:00:00 2001 From: Brian Floersch Date: Thu, 26 Mar 2026 19:45:50 -0400 Subject: [PATCH 3/5] Generate benchmark README from results --- Benchmarks/Readme.md | 378 +++++++++++++++++++------- Benchmarks/run_go_benchmarks.sh | 5 + Benchmarks/run_swift_benchmarks.sh | 5 + Benchmarks/update_benchmark_readme.py | 270 ++++++++++++++++++ Benchmarks/update_benchmark_readme.sh | 30 ++ 5 files changed, 590 insertions(+), 98 deletions(-) create mode 100755 Benchmarks/update_benchmark_readme.py create mode 100755 Benchmarks/update_benchmark_readme.sh diff --git a/Benchmarks/Readme.md b/Benchmarks/Readme.md index efec888..ac9a8cf 100644 --- a/Benchmarks/Readme.md +++ b/Benchmarks/Readme.md @@ -35,114 +35,296 @@ xcrun swift run -c release --package-path Benchmarks Benchmarks --help Results written to `Benchmarks/results/` are ignored by git so local benchmark output does not dirty the repository. All Go tests were written as go micro benchmarks.\ -All tests performed on an M1 max +All tests performed on the local machine used to generate the current results. -## Swift vs Go +The benchmark README is generated from local result files: -### Raw results +```bash +Benchmarks/update_benchmark_readme.sh +``` -First are the baseline go tests. +Or as part of a benchmark run: +```bash +UPDATE_README=1 BENCHMARK_NAME=swift-6.3.0 SWIFT_BIN=$HOME/.swiftly/bin/swift Benchmarks/run_swift_benchmarks.sh +UPDATE_README=1 BENCHMARK_NAME=go-1.26.1 Benchmarks/run_go_benchmarks.sh ``` + + +## Generated Results + +These sections are generated from benchmark result files in `Benchmarks/results/`. + +**Environment** +- Swift toolchain: `swift-6.3.0` +- Swift host: `brians-macbook-air.local` +- Swift rounds: `5` measured, `1` warmup +- Swift writes: `1000000`, sync writes: `5000000`, select writes/channel: `100000`, buffer: `100` +- Baseline Swift toolchain: `swift-6.2.3` +- Baseline Swift host: `brians-macbook-air.local` +- Baseline Swift rounds: `5` measured, `1` warmup +- Baseline Swift writes: `1000000`, sync writes: `5000000`, select writes/channel: `100000`, buffer: `100` + +Baseline Swift report: `Benchmarks/results/swift-6.2.3-20260326-185306.json` +Candidate Swift report: `Benchmarks/results/swift-6.3.0-20260326-191022.json` + +**Swift Toolchain Comparison** +Library | Test | Type | Baseline Avg (ms) | Candidate Avg (ms) | Delta % +--- | --- | --- | --- | --- | --- +AsyncAlgorithms | MPMC | `Int` | `7027.25` | `5215.65` | `-25.78%` +AsyncAlgorithms | MPMC | `RefData` | `7504.62` | `5588.75` | `-25.53%` +AsyncAlgorithms | MPMC | `String` | `8709.50` | `5587.76` | `-35.84%` +AsyncAlgorithms | MPMC | `ValueData` | `9590.54` | `5014.12` | `-47.72%` +AsyncAlgorithms | MPSC | `Int` | `4788.59` | `3761.30` | `-21.45%` +AsyncAlgorithms | MPSC | `RefData` | `5180.12` | `3945.61` | `-23.83%` +AsyncAlgorithms | MPSC | `String` | `6355.52` | `4179.43` | `-34.24%` +AsyncAlgorithms | MPSC | `ValueData` | `6757.28` | `4236.05` | `-37.31%` +AsyncAlgorithms | MPSC Write Contention | `Int` | `9298.62` | `5591.63` | `-39.87%` +AsyncAlgorithms | MPSC Write Contention | `RefData` | `8655.29` | `6262.41` | `-27.65%` +AsyncAlgorithms | MPSC Write Contention | `String` | `10685.25` | `6139.55` | `-42.54%` +AsyncAlgorithms | MPSC Write Contention | `ValueData` | `10456.41` | `5252.34` | `-49.77%` +AsyncAlgorithms | SPMC | `Int` | `4789.23` | `3622.70` | `-24.36%` +AsyncAlgorithms | SPMC | `RefData` | `5223.19` | `4114.84` | `-21.22%` +AsyncAlgorithms | SPMC | `String` | `6353.19` | `3890.67` | `-38.76%` +AsyncAlgorithms | SPMC | `ValueData` | `6499.68` | `3986.87` | `-38.66%` +AsyncAlgorithms | SPSC | `Int` | `2871.85` | `2305.84` | `-19.71%` +AsyncAlgorithms | SPSC | `RefData` | `2927.92` | `2418.31` | `-17.41%` +AsyncAlgorithms | SPSC | `String` | `3467.80` | `2350.12` | `-32.23%` +AsyncAlgorithms | SPSC | `ValueData` | `4047.56` | `2420.69` | `-40.19%` +AsyncChannels | Channel multi-select | `Int` | `1695.74` | `1381.14` | `-18.55%` +AsyncChannels | Channel multi-select | `RefData` | `1893.57` | `1176.29` | `-37.88%` +AsyncChannels | Channel multi-select | `String` | `1878.38` | `1362.78` | `-27.45%` +AsyncChannels | Channel multi-select | `ValueData` | `2183.19` | `1449.17` | `-33.62%` +AsyncChannels | MPMC | `Int` | `933.62` | `773.33` | `-17.17%` +AsyncChannels | MPMC | `RefData` | `983.93` | `725.90` | `-26.22%` +AsyncChannels | MPMC | `String` | `930.62` | `818.39` | `-12.06%` +AsyncChannels | MPMC | `ValueData` | `977.47` | `815.51` | `-16.57%` +AsyncChannels | MPMC Buffered(100) | `Int` | `454.85` | `459.06` | `+0.92%` +AsyncChannels | MPMC Buffered(100) | `RefData` | `649.37` | `501.98` | `-22.70%` +AsyncChannels | MPMC Buffered(100) | `String` | `495.95` | `445.53` | `-10.17%` +AsyncChannels | MPMC Buffered(100) | `ValueData` | `602.29` | `499.51` | `-17.06%` +AsyncChannels | MPSC | `Int` | `476.00` | `332.49` | `-30.15%` +AsyncChannels | MPSC | `RefData` | `638.52` | `323.41` | `-49.35%` +AsyncChannels | MPSC | `String` | `599.85` | `330.18` | `-44.96%` +AsyncChannels | MPSC | `ValueData` | `640.65` | `315.16` | `-50.81%` +AsyncChannels | MPSC Buffered(100) | `Int` | `264.37` | `157.45` | `-40.44%` +AsyncChannels | MPSC Buffered(100) | `RefData` | `474.13` | `147.87` | `-68.81%` +AsyncChannels | MPSC Buffered(100) | `String` | `365.77` | `212.38` | `-41.94%` +AsyncChannels | MPSC Buffered(100) | `ValueData` | `481.41` | `230.82` | `-52.05%` +AsyncChannels | MPSC Write Contention | `Int` | `511.44` | `385.93` | `-24.54%` +AsyncChannels | MPSC Write Contention | `RefData` | `730.74` | `376.60` | `-48.46%` +AsyncChannels | MPSC Write Contention | `String` | `681.47` | `390.94` | `-42.63%` +AsyncChannels | MPSC Write Contention | `ValueData` | `697.42` | `419.93` | `-39.79%` +AsyncChannels | MPSC Write Contention Buffered(100) | `Int` | `520.99` | `395.09` | `-24.17%` +AsyncChannels | MPSC Write Contention Buffered(100) | `RefData` | `788.64` | `356.57` | `-54.79%` +AsyncChannels | MPSC Write Contention Buffered(100) | `String` | `650.07` | `435.39` | `-33.02%` +AsyncChannels | MPSC Write Contention Buffered(100) | `ValueData` | `725.20` | `424.92` | `-41.41%` +AsyncChannels | SPMC | `Int` | `589.31` | `320.46` | `-45.62%` +AsyncChannels | SPMC | `RefData` | `631.42` | `302.77` | `-52.05%` +AsyncChannels | SPMC | `String` | `510.73` | `297.05` | `-41.84%` +AsyncChannels | SPMC | `ValueData` | `632.23` | `307.19` | `-51.41%` +AsyncChannels | SPMC Buffered(100) | `Int` | `283.86` | `219.90` | `-22.53%` +AsyncChannels | SPMC Buffered(100) | `RefData` | `428.43` | `180.02` | `-57.98%` +AsyncChannels | SPMC Buffered(100) | `String` | `319.52` | `228.32` | `-28.54%` +AsyncChannels | SPMC Buffered(100) | `ValueData` | `424.22` | `217.26` | `-48.79%` +AsyncChannels | SPSC | `Int` | `432.39` | `300.47` | `-30.51%` +AsyncChannels | SPSC | `RefData` | `594.98` | `355.01` | `-40.33%` +AsyncChannels | SPSC | `String` | `705.70` | `332.97` | `-52.82%` +AsyncChannels | SPSC | `ValueData` | `600.47` | `346.81` | `-42.24%` +AsyncChannels | SPSC Buffered(100) | `Int` | `494.76` | `439.99` | `-11.07%` +AsyncChannels | SPSC Buffered(100) | `RefData` | `622.45` | `368.84` | `-40.74%` +AsyncChannels | SPSC Buffered(100) | `String` | `609.59` | `469.59` | `-22.97%` +AsyncChannels | SPSC Buffered(100) | `ValueData` | `821.87` | `485.11` | `-40.97%` +AsyncChannels | SyncRW | `Int` | `487.80` | `327.48` | `-32.87%` +AsyncChannels | SyncRW | `RefData` | `669.29` | `363.38` | `-45.71%` +AsyncChannels | SyncRW | `String` | `651.87` | `357.71` | `-45.13%` +AsyncChannels | SyncRW | `ValueData` | `702.92` | `356.54` | `-49.28%` + +**AsyncChannels Results** +Test | Type | Avg (ms) | Median (ms) | Ops/s +--- | --- | --- | --- | --- +SPSC | `Int` | `300.47` | `310.60` | `3328111` +MPSC | `Int` | `332.49` | `332.27` | `3007626` +SPMC | `Int` | `320.46` | `319.93` | `3120474` +MPMC | `Int` | `773.33` | `771.20` | `1293105` +MPSC Write Contention | `Int` | `385.93` | `382.52` | `2591141` +SPSC Buffered(100) | `Int` | `439.99` | `439.22` | `2272779` +MPSC Buffered(100) | `Int` | `157.45` | `159.26` | `6351100` +SPMC Buffered(100) | `Int` | `219.90` | `225.18` | `4547510` +MPMC Buffered(100) | `Int` | `459.06` | `457.86` | `2178367` +MPSC Write Contention Buffered(100) | `Int` | `395.09` | `396.82` | `2531067` +SyncRW | `Int` | `327.48` | `324.22` | `15267918` +Channel multi-select | `Int` | `1381.14` | `1373.23` | `434425` +SPSC | `String` | `332.97` | `332.17` | `3003284` +MPSC | `String` | `330.18` | `332.04` | `3028678` +SPMC | `String` | `297.05` | `299.69` | `3366381` +MPMC | `String` | `818.39` | `822.43` | `1221912` +MPSC Write Contention | `String` | `390.94` | `385.97` | `2557931` +SPSC Buffered(100) | `String` | `469.59` | `475.72` | `2129497` +MPSC Buffered(100) | `String` | `212.38` | `212.69` | `4708434` +SPMC Buffered(100) | `String` | `228.32` | `230.53` | `4379802` +MPMC Buffered(100) | `String` | `445.53` | `444.08` | `2244504` +MPSC Write Contention Buffered(100) | `String` | `435.39` | `412.69` | `2296794` +SyncRW | `String` | `357.71` | `354.91` | `13977834` +Channel multi-select | `String` | `1362.78` | `1348.23` | `440275` +SPSC | `ValueData` | `346.81` | `344.38` | `2883405` +MPSC | `ValueData` | `315.16` | `315.01` | `3172962` +SPMC | `ValueData` | `307.19` | `312.51` | `3255301` +MPMC | `ValueData` | `815.51` | `803.69` | `1226221` +MPSC Write Contention | `ValueData` | `419.93` | `416.36` | `2381321` +SPSC Buffered(100) | `ValueData` | `485.11` | `505.86` | `2061394` +MPSC Buffered(100) | `ValueData` | `230.82` | `228.44` | `4332345` +SPMC Buffered(100) | `ValueData` | `217.26` | `206.35` | `4602739` +MPMC Buffered(100) | `ValueData` | `499.51` | `507.02` | `2001955` +MPSC Write Contention Buffered(100) | `ValueData` | `424.92` | `424.22` | `2353377` +SyncRW | `ValueData` | `356.54` | `353.95` | `14023849` +Channel multi-select | `ValueData` | `1449.17` | `1433.59` | `414029` +SPSC | `RefData` | `355.01` | `347.87` | `2816820` +MPSC | `RefData` | `323.41` | `326.96` | `3092084` +SPMC | `RefData` | `302.77` | `303.59` | `3302870` +MPMC | `RefData` | `725.90` | `720.67` | `1377600` +MPSC Write Contention | `RefData` | `376.60` | `371.72` | `2655310` +SPSC Buffered(100) | `RefData` | `368.84` | `376.02` | `2711186` +MPSC Buffered(100) | `RefData` | `147.87` | `142.17` | `6762521` +SPMC Buffered(100) | `RefData` | `180.02` | `179.84` | `5554794` +MPMC Buffered(100) | `RefData` | `501.98` | `502.03` | `1992126` +MPSC Write Contention Buffered(100) | `RefData` | `356.57` | `355.07` | `2804511` +SyncRW | `RefData` | `363.38` | `362.71` | `13759573` +Channel multi-select | `RefData` | `1176.29` | `1155.01` | `510079` + +**AsyncAlgorithms Results** +Test | Type | Avg (ms) | Median (ms) | Ops/s +--- | --- | --- | --- | --- +SPSC | `Int` | `2305.84` | `2305.39` | `433681` +MPSC | `Int` | `3761.30` | `3758.22` | `265865` +SPMC | `Int` | `3622.70` | `3625.11` | `276037` +MPMC | `Int` | `5215.65` | `5232.01` | `191731` +MPSC Write Contention | `Int` | `5591.63` | `5600.81` | `178839` +SPSC | `String` | `2350.12` | `2351.79` | `425510` +MPSC | `String` | `4179.43` | `4111.78` | `239267` +SPMC | `String` | `3890.67` | `3896.53` | `257025` +MPMC | `String` | `5587.76` | `5588.08` | `178963` +MPSC Write Contention | `String` | `6139.55` | `6254.25` | `162878` +SPSC | `ValueData` | `2420.69` | `2420.30` | `413106` +MPSC | `ValueData` | `4236.05` | `4238.90` | `236069` +SPMC | `ValueData` | `3986.87` | `4006.63` | `250824` +MPMC | `ValueData` | `5014.12` | `5013.27` | `199437` +MPSC Write Contention | `ValueData` | `5252.34` | `5050.63` | `190391` +SPSC | `RefData` | `2418.31` | `2292.62` | `413512` +MPSC | `RefData` | `3945.61` | `3793.76` | `253446` +SPMC | `RefData` | `4114.84` | `4196.80` | `243023` +MPMC | `RefData` | `5588.75` | `5980.78` | `178931` +MPSC Write Contention | `RefData` | `6262.41` | `6268.68` | `159683` + +- Go toolchain: `go_version_go1.26.1_darwin_arm64` +- Go host: `Apple M5` +- Go rounds: `5` measured, `0` warmup +- Go writes: `1000000`, sync writes: `5000000`, select writes/channel: `100000`, buffer: `100` +Go report: `Benchmarks/results/go_version_go1.26.1_darwin_arm64-20260326-193146.json` + +**Cross-Library Comparison** +Test | Type | AsyncAlgorithms Avg (ms) | AsyncChannels Avg (ms) | Go Avg (ms) | AsyncAlgorithms vs Go | AsyncChannels vs Go +--- | --- | --- | --- | --- | --- | --- +Channel multi-select | `Int` | | `1381.14` | `552.71` | | `+149.88%` +MPMC | `Int` | `5215.65` | `773.33` | `241.73` | `+2057.62%` | `+219.91%` +MPMC | `RefData` | `5588.75` | `725.90` | +MPMC | `String` | `5587.76` | `818.39` | +MPMC | `ValueData` | `5014.12` | `815.51` | +MPMC Buffered(100) | `Int` | | `459.06` | `138.23` | | `+232.10%` +MPSC | `Int` | `3761.30` | `332.49` | `244.90` | `+1435.87%` | `+35.77%` +MPSC | `RefData` | `3945.61` | `323.41` | +MPSC | `String` | `4179.43` | `330.18` | +MPSC | `ValueData` | `4236.05` | `315.16` | +MPSC Buffered(100) | `Int` | | `157.45` | `80.55` | | `+95.48%` +MPSC Write Contention | `Int` | `5591.63` | `385.93` | `503.68` | `+1010.15%` | `-23.38%` +MPSC Write Contention | `RefData` | `6262.41` | `376.60` | +MPSC Write Contention | `String` | `6139.55` | `390.94` | +MPSC Write Contention | `ValueData` | `5252.34` | `419.93` | +MPSC Write Contention Buffered(100) | `Int` | | `395.09` | `642.26` | | `-38.48%` +SPMC | `Int` | `3622.70` | `320.46` | `259.52` | `+1295.95%` | `+23.49%` +SPMC | `RefData` | `4114.84` | `302.77` | +SPMC | `String` | `3890.67` | `297.05` | +SPMC | `ValueData` | `3986.87` | `307.19` | +SPMC Buffered(100) | `Int` | | `219.90` | `124.92` | | `+76.03%` +SPSC | `Int` | `2305.84` | `300.47` | `99.82` | `+2209.98%` | `+201.01%` +SPSC | `RefData` | `2418.31` | `355.01` | +SPSC | `String` | `2350.12` | `332.97` | +SPSC | `ValueData` | `2420.69` | `346.81` | +SPSC Buffered(100) | `Int` | | `439.99` | `32.32` | | `+1261.22%` +SyncRW | `Int` | | `327.48` | `205.30` | | `+59.52%` + +**Raw Go Benchmark Output** +```text goos: darwin goarch: arm64 pkg: benchmarks -BenchmarkSPSC-10 7 158625381 ns/op 274 B/op 3 allocs/op -BenchmarkMPSC-10 4 262326958 ns/op 1160 B/op 6 allocs/op -BenchmarkSPMC-10 4 273297438 ns/op 992 B/op 7 allocs/op -BenchmarkMPMC-10 4 288092292 ns/op 1080 B/op 5 allocs/op -BenchmarkMPSCWriteContention-10 4 319210156 ns/op 19288 B/op 136 allocs/op -BenchmarkSPSCBuffered-10 15 72298667 ns/op 928 B/op 3 allocs/op -BenchmarkMPSCBuffered-10 13 90277561 ns/op 928 B/op 3 allocs/op -BenchmarkSPMCBuffered-10 13 90140349 ns/op 928 B/op 3 allocs/op -BenchmarkMPMCBuffered-10 12 92401344 ns/op 928 B/op 3 allocs/op -BenchmarkMPSCWriteContentionBuffered-10 4 321178198 ns/op 13696 B/op 136 allocs/op -BenchmarkSyncRw-10 8 131667010 ns/op 112 B/op 1 allocs/op -BenchmarkMultiSelect-10 4 310069656 ns/op 576 B/op 6 allocs/op +cpu: Apple M5 +BenchmarkSPSC-10 10 100272954 ns/op 393 B/op 5 allocs/op +BenchmarkSPSC-10 12 102025347 ns/op 316 B/op 5 allocs/op +BenchmarkSPSC-10 12 98038990 ns/op 225 B/op 5 allocs/op +BenchmarkSPSC-10 12 98737108 ns/op 356 B/op 5 allocs/op +BenchmarkSPSC-10 12 100029587 ns/op 216 B/op 5 allocs/op +BenchmarkMPSC-10 5 237266517 ns/op 907 B/op 11 allocs/op +BenchmarkMPSC-10 5 250905517 ns/op 1643 B/op 11 allocs/op +BenchmarkMPSC-10 5 244221625 ns/op 1473 B/op 12 allocs/op +BenchmarkMPSC-10 5 248658942 ns/op 2100 B/op 13 allocs/op +BenchmarkMPSC-10 5 243436267 ns/op 1620 B/op 12 allocs/op +BenchmarkSPMC-10 5 258839358 ns/op 920 B/op 11 allocs/op +BenchmarkSPMC-10 4 259902323 ns/op 1368 B/op 13 allocs/op +BenchmarkSPMC-10 4 260214886 ns/op 1560 B/op 12 allocs/op +BenchmarkSPMC-10 4 260728906 ns/op 1368 B/op 13 allocs/op +BenchmarkSPMC-10 4 257892062 ns/op 368 B/op 9 allocs/op +BenchmarkMPMC-10 5 245182675 ns/op 1473 B/op 16 allocs/op +BenchmarkMPMC-10 5 228625383 ns/op 1169 B/op 15 allocs/op +BenchmarkMPMC-10 5 231445842 ns/op 926 B/op 14 allocs/op +BenchmarkMPMC-10 5 237985958 ns/op 1502 B/op 16 allocs/op +BenchmarkMPMC-10 5 265417000 ns/op 1368 B/op 14 allocs/op +BenchmarkMPSCWriteContention-10 3 473304681 ns/op 64168 B/op 1144 allocs/op +BenchmarkMPSCWriteContention-10 3 450217056 ns/op 66301 B/op 1151 allocs/op +BenchmarkMPSCWriteContention-10 3 462066264 ns/op 66872 B/op 1171 allocs/op +BenchmarkMPSCWriteContention-10 3 479075514 ns/op 63026 B/op 1136 allocs/op +BenchmarkMPSCWriteContention-10 3 653751097 ns/op 81730 B/op 1303 allocs/op +BenchmarkSPSCBuffered-10 36 45888530 ns/op 1128 B/op 5 allocs/op +BenchmarkSPSCBuffered-10 33 38004106 ns/op 1128 B/op 5 allocs/op +BenchmarkSPSCBuffered-10 45 25035363 ns/op 1128 B/op 5 allocs/op +BenchmarkSPSCBuffered-10 46 25337199 ns/op 1128 B/op 5 allocs/op +BenchmarkSPSCBuffered-10 42 27350754 ns/op 1128 B/op 5 allocs/op +BenchmarkMPSCBuffered-10 18 77956752 ns/op 1320 B/op 9 allocs/op +BenchmarkMPSCBuffered-10 16 80845818 ns/op 1320 B/op 9 allocs/op +BenchmarkMPSCBuffered-10 12 90167514 ns/op 1320 B/op 9 allocs/op +BenchmarkMPSCBuffered-10 13 80640811 ns/op 1320 B/op 9 allocs/op +BenchmarkMPSCBuffered-10 15 73124811 ns/op 1320 B/op 9 allocs/op +BenchmarkSPMCBuffered-10 15 80036100 ns/op 1224 B/op 9 allocs/op +BenchmarkSPMCBuffered-10 12 115830795 ns/op 1224 B/op 9 allocs/op +BenchmarkSPMCBuffered-10 9 115885287 ns/op 1224 B/op 9 allocs/op +BenchmarkSPMCBuffered-10 12 178193066 ns/op 1224 B/op 9 allocs/op +BenchmarkSPMCBuffered-10 9 134650676 ns/op 1224 B/op 9 allocs/op +BenchmarkMPMCBuffered-10 9 128738537 ns/op 1416 B/op 13 allocs/op +BenchmarkMPMCBuffered-10 9 117296884 ns/op 1416 B/op 13 allocs/op +BenchmarkMPMCBuffered-10 14 158546688 ns/op 1416 B/op 13 allocs/op +BenchmarkMPMCBuffered-10 10 139077846 ns/op 1416 B/op 13 allocs/op +BenchmarkMPMCBuffered-10 8 147492698 ns/op 1416 B/op 13 allocs/op +BenchmarkMPSCWriteContentionBuffered-10 2 622269584 ns/op 88952 B/op 1360 allocs/op +BenchmarkMPSCWriteContentionBuffered-10 2 733462500 ns/op 70808 B/op 1198 allocs/op +BenchmarkMPSCWriteContentionBuffered-10 3 606159264 ns/op 76445 B/op 1248 allocs/op +BenchmarkMPSCWriteContentionBuffered-10 6 619955701 ns/op 64312 B/op 1140 allocs/op +BenchmarkMPSCWriteContentionBuffered-10 3 629465417 ns/op 81858 B/op 1296 allocs/op +BenchmarkSyncRw-10 5 316503150 ns/op 128 B/op 1 allocs/op +BenchmarkSyncRw-10 19 173398162 ns/op 128 B/op 1 allocs/op +BenchmarkSyncRw-10 10 147316417 ns/op 128 B/op 1 allocs/op +BenchmarkSyncRw-10 6 197205194 ns/op 128 B/op 1 allocs/op +BenchmarkSyncRw-10 9 192063287 ns/op 128 B/op 1 allocs/op +BenchmarkMultiSelect-10 3 555280958 ns/op 816 B/op 12 allocs/op +BenchmarkMultiSelect-10 2 560279688 ns/op 816 B/op 12 allocs/op +BenchmarkMultiSelect-10 2 593197979 ns/op 816 B/op 12 allocs/op +BenchmarkMultiSelect-10 3 544327736 ns/op 816 B/op 12 allocs/op +BenchmarkMultiSelect-10 3 510480069 ns/op 816 B/op 12 allocs/op +PASS +ok benchmarks 142.460s ``` + + -The below results are from running the whole benchmark suit which covers multiple data types. - -Test | Type | Execution Time(ms) ------|------|--------------- -SPSC | `Int` | `985` -MPSC | `Int` | `560` -SPMC | `Int` | `619` -MPMC | `Int` | `522` -MPSC Write Contention | `Int` | `734` -SPSC Buffered(100) | `Int` | `254` -MPSC Buffered(100) | `Int` | `379` -SPMC Buffered(100) | `Int` | `377` -MPMC Buffered(100) | `Int` | `529` -MPSC Write Contention Buffered(100) | `Int` | `729` -SyncRW | `Int` | `1039` -Channel multi-select | `Int` | `1846` -SPSC | `String` | `1008` -MPSC | `String` | `565` -SPMC | `String` | `613` -MPMC | `String` | `538` -MPSC Write Contention | `String` | `737` -SPSC Buffered(100) | `String` | `231` -MPSC Buffered(100) | `String` | `381` -SPMC Buffered(100) | `String` | `370` -MPMC Buffered(100) | `String` | `536` -MPSC Write Contention Buffered(100) | `String` | `740` -SyncRW | `String` | `1136` -Channel multi-select | `String` | `1860` -SPSC | `ValueData` | `1000` -MPSC | `ValueData` | `558` -SPMC | `ValueData` | `612` -MPMC | `ValueData` | `533` -MPSC Write Contention | `ValueData` | `736` -SPSC Buffered(100) | `ValueData` | `229` -MPSC Buffered(100) | `ValueData` | `375` -SPMC Buffered(100) | `ValueData` | `351` -MPMC Buffered(100) | `ValueData` | `516` -MPSC Write Contention Buffered(100) | `ValueData` | `745` -SyncRW | `ValueData` | `1116` -Channel multi-select | `ValueData` | `1862` -SPSC | `RefData` | `1028` -MPSC | `RefData` | `604` -SPMC | `RefData` | `616` -MPMC | `RefData` | `565` -MPSC Write Contention | `RefData` | `769` -SPSC Buffered(100) | `RefData` | `165` -MPSC Buffered(100) | `RefData` | `346` -SPMC Buffered(100) | `RefData` | `334` -MPMC Buffered(100) | `RefData` | `565` -MPSC Write Contention Buffered(100) | `RefData` | `754` -SyncRW | `RefData` | `1162` -Channel multi-select | `RefData` | `1894` - - -## Async Channels vs Async Algorithms AsyncChannel - -Apple has their own channel implementation in the [swift-async-algorithms package](https://github.com/apple/swift-async-algorithms/blob/main/Sources/AsyncAlgorithms/AsyncAlgorithms.docc/Guides/Channel.md). We cannot compare every benchmark since it does not support buffering or select. Below are the results from comparable tests on the same data types as above. - -Test | Type | Execution Time(ms) ------|------|--------------- -SPSC Async alg | `Int` | `3000` -MPSC Async alg | `Int` | `4030` -SPMC Async alg | `Int` | `3951` -MPMC Async alg | `Int` | `4231` -MPSC Async alg Write Contention | `Int` | `7343` -SPSC Async alg | `String` | `3021` -MPSC Async alg | `String` | `4083` -SPMC Async alg | `String` | `3937` -MPMC Async alg | `String` | `4313` -MPSC Async alg Write Contention | `String` | `21004` -SPSC Async alg | `ValueData` | `3006` -MPSC Async alg | `ValueData` | `4052` -SPMC Async alg | `ValueData` | `3911` -MPMC Async alg | `ValueData` | `4275` -MPSC Async alg Write Contention | `ValueData` | `19684` -SPSC Async alg | `RefData` | `3026` -MPSC Async alg | `RefData` | `4064` -SPMC Async alg | `RefData` | `3929` -MPMC Async alg | `RefData` | `4285` -MPSC Async alg Write Contention | `RefData` | `20992` - -Async algorithms channel seems to fall apart with write contention on non integer types. ## Future work diff --git a/Benchmarks/run_go_benchmarks.sh b/Benchmarks/run_go_benchmarks.sh index 1ffbe65..c9d0a38 100755 --- a/Benchmarks/run_go_benchmarks.sh +++ b/Benchmarks/run_go_benchmarks.sh @@ -10,6 +10,7 @@ SELECT_WRITES="${SELECT_WRITES:-100000}" BUFFER="${BUFFER:-100}" OUTPUT_DIR="${OUTPUT_DIR:-Benchmarks/results}" BENCHMARK_NAME="${BENCHMARK_NAME:-}" +UPDATE_README="${UPDATE_README:-0}" mkdir -p "${OUTPUT_DIR}" @@ -38,3 +39,7 @@ Benchmarks/parse_go_benchmarks.py \ "${BUFFER}" > "${JSON_OUTPUT_PATH}" echo "Finished: ${JSON_OUTPUT_PATH}" + +if [[ "${UPDATE_README}" == "1" ]]; then + GO_REPORT="${JSON_OUTPUT_PATH}" GO_RAW_REPORT="${RAW_OUTPUT_PATH}" Benchmarks/update_benchmark_readme.sh +fi diff --git a/Benchmarks/run_swift_benchmarks.sh b/Benchmarks/run_swift_benchmarks.sh index 21b9675..84fc565 100755 --- a/Benchmarks/run_swift_benchmarks.sh +++ b/Benchmarks/run_swift_benchmarks.sh @@ -13,6 +13,7 @@ OUTPUT_DIR="${OUTPUT_DIR:-Benchmarks/results}" SCRATCH_ROOT="${SCRATCH_ROOT:-/tmp/asyncchannels-benchmarks}" BENCHMARK_NAME="${BENCHMARK_NAME:-}" SKIP_ASYNC_ALGORITHMS="${SKIP_ASYNC_ALGORITHMS:-0}" +UPDATE_README="${UPDATE_README:-0}" mkdir -p "${OUTPUT_DIR}" @@ -45,3 +46,7 @@ echo "Writing ${OUTPUT_PATH}" BENCHMARK_TOOLCHAIN_LABEL="${BENCHMARK_NAME}" "${CMD[@]}" > "${OUTPUT_PATH}" echo "Finished: ${OUTPUT_PATH}" + +if [[ "${UPDATE_README}" == "1" ]]; then + SWIFT_REPORT="${OUTPUT_PATH}" Benchmarks/update_benchmark_readme.sh +fi diff --git a/Benchmarks/update_benchmark_readme.py b/Benchmarks/update_benchmark_readme.py new file mode 100755 index 0000000..27882dc --- /dev/null +++ b/Benchmarks/update_benchmark_readme.py @@ -0,0 +1,270 @@ +#!/usr/bin/env python3 + +import argparse +import json +from pathlib import Path +from typing import Iterable, Optional + + +START_MARKER = "" +END_MARKER = "" + + +def load_json(path: Path) -> dict: + with path.open("r", encoding="utf-8") as handle: + return json.load(handle) + + +def latest_result(results_dir: Path, prefix: str, suffix: str) -> Optional[Path]: + matches = sorted( + ( + path + for path in results_dir.glob(f"{prefix}*{suffix}") + if "smoke" not in path.name + ), + key=lambda path: path.stat().st_mtime, + reverse=True, + ) + return matches[0] if matches else None + + +def markdown_table(headers: list[str], rows: Iterable[list[str]]) -> str: + lines = [" | ".join(headers), " | ".join(["---"] * len(headers))] + lines.extend(" | ".join(row) for row in rows) + return "\n".join(lines) + + +def render_report_table(report: dict, library: str) -> str: + rows = [] + for result in report["results"]: + if result["library"] != library: + continue + rows.append( + [ + result["name"], + f"`{result['type']}`", + f"`{result['averageMs']:.2f}`", + f"`{result['medianMs']:.2f}`", + f"`{result['operationsPerSecond']:.0f}`", + ] + ) + return markdown_table( + ["Test", "Type", "Avg (ms)", "Median (ms)", "Ops/s"], + rows, + ) + + +def render_swift_comparison(baseline: dict, candidate: dict) -> str: + baseline_results = { + (result["library"], result["name"], result["type"]): result + for result in baseline["results"] + } + candidate_results = { + (result["library"], result["name"], result["type"]): result + for result in candidate["results"] + } + + rows = [] + for key in sorted(set(baseline_results) & set(candidate_results)): + before = baseline_results[key]["averageMs"] + after = candidate_results[key]["averageMs"] + delta = ((after - before) / before) * 100 + library, name, result_type = key + rows.append( + [ + library, + name, + f"`{result_type}`", + f"`{before:.2f}`", + f"`{after:.2f}`", + f"`{delta:+.2f}%`", + ] + ) + + return markdown_table( + ["Library", "Test", "Type", "Baseline Avg (ms)", "Candidate Avg (ms)", "Delta %"], + rows, + ) + + +def render_library_comparison(reports: list[dict]) -> str: + merged: dict[tuple[str, str], dict[str, dict]] = {} + for report in reports: + for result in report["results"]: + merged.setdefault((result["name"], result["type"]), {})[result["library"]] = result + + libraries = sorted({result["library"] for report in reports for result in report["results"]}) + headers = ["Test", "Type"] + [f"{library} Avg (ms)" for library in libraries] + if "Go" in libraries: + headers += [f"{library} vs Go" for library in libraries if library != "Go"] + + rows = [] + for (name, result_type), values in sorted(merged.items()): + if len(values) < 2: + continue + row = [name, f"`{result_type}`"] + for library in libraries: + result = values.get(library) + row.append(f"`{result['averageMs']:.2f}`" if result else "") + + go_result = values.get("Go") + if go_result: + for library in libraries: + if library == "Go": + continue + result = values.get(library) + row.append( + f"`{((result['averageMs'] - go_result['averageMs']) / go_result['averageMs']) * 100:+.2f}%`" + if result + else "" + ) + + rows.append(row) + + return markdown_table(headers, rows) + + +def render_environment(report: dict, label: str) -> str: + config = report["config"] + lines = [ + f"- {label} toolchain: `{report['toolchain']}`", + f"- {label} host: `{report['host']}`", + f"- {label} rounds: `{config['rounds']}` measured, `{config['warmupRounds']}` warmup", + f"- {label} writes: `{config['writes']}`, sync writes: `{config['syncWrites']}`, select writes/channel: `{config['selectWritesPerChannel']}`, buffer: `{config['buffer']}`", + ] + return "\n".join(lines) + + +def render_generated_section( + swift_report: dict, + baseline_swift_report: Optional[dict], + go_report: Optional[dict], + go_raw_text: Optional[str], + source_paths: dict[str, Optional[Path]], +) -> str: + parts = [ + START_MARKER, + "## Generated Results", + "", + "These sections are generated from benchmark result files in `Benchmarks/results/`.", + "", + "**Environment**", + render_environment(swift_report, "Swift"), + ] + + if baseline_swift_report: + parts.extend( + [ + render_environment(baseline_swift_report, "Baseline Swift"), + "", + f"Baseline Swift report: `{source_paths['baseline_swift'].as_posix()}`", + f"Candidate Swift report: `{source_paths['swift'].as_posix()}`", + "", + "**Swift Toolchain Comparison**", + render_swift_comparison(baseline_swift_report, swift_report), + ] + ) + + parts.extend( + [ + "", + "**AsyncChannels Results**", + render_report_table(swift_report, "AsyncChannels"), + ] + ) + + if any(result["library"] == "AsyncAlgorithms" for result in swift_report["results"]): + parts.extend( + [ + "", + "**AsyncAlgorithms Results**", + render_report_table(swift_report, "AsyncAlgorithms"), + ] + ) + + if go_report: + parts.extend( + [ + "", + render_environment(go_report, "Go"), + f"Go report: `{source_paths['go'].as_posix()}`", + "", + "**Cross-Library Comparison**", + render_library_comparison([swift_report, go_report]), + ] + ) + + if go_raw_text is not None: + parts.extend( + [ + "", + "**Raw Go Benchmark Output**", + "```text", + go_raw_text.rstrip(), + "```", + ] + ) + + parts.append(END_MARKER) + return "\n".join(parts) + + +def replace_generated_section(readme_text: str, generated_text: str) -> str: + start = readme_text.find(START_MARKER) + end = readme_text.find(END_MARKER) + if start == -1 or end == -1: + return f"{readme_text.rstrip()}\n\n{generated_text}\n" + end += len(END_MARKER) + return f"{readme_text[:start].rstrip()}\n\n{generated_text}\n{readme_text[end:]}" + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--readme", default="Benchmarks/Readme.md") + parser.add_argument("--results-dir", default="Benchmarks/results") + parser.add_argument("--swift") + parser.add_argument("--baseline-swift") + parser.add_argument("--go") + parser.add_argument("--go-raw") + return parser.parse_args() + + +def main() -> int: + args = parse_args() + readme_path = Path(args.readme) + results_dir = Path(args.results_dir) + + swift_path = Path(args.swift) if args.swift else latest_result(results_dir, "swift-", ".json") + baseline_swift_path = Path(args.baseline_swift) if args.baseline_swift else latest_result(results_dir, "swift-6.2", ".json") + go_path = Path(args.go) if args.go else latest_result(results_dir, "go_", ".json") + go_raw_path = Path(args.go_raw) if args.go_raw else latest_result(results_dir, "go_", ".txt") + + if swift_path is None: + raise SystemExit("No Swift benchmark report found") + + swift_report = load_json(swift_path) + baseline_swift_report = load_json(baseline_swift_path) if baseline_swift_path and baseline_swift_path != swift_path else None + go_report = load_json(go_path) if go_path else None + go_raw_text = go_raw_path.read_text(encoding="utf-8") if go_raw_path else None + + generated = render_generated_section( + swift_report=swift_report, + baseline_swift_report=baseline_swift_report, + go_report=go_report, + go_raw_text=go_raw_text, + source_paths={ + "swift": swift_path, + "baseline_swift": baseline_swift_path, + "go": go_path, + "go_raw": go_raw_path, + }, + ) + + updated = replace_generated_section(readme_path.read_text(encoding="utf-8"), generated) + readme_path.write_text(updated, encoding="utf-8") + print(f"Updated {readme_path}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/Benchmarks/update_benchmark_readme.sh b/Benchmarks/update_benchmark_readme.sh new file mode 100755 index 0000000..d6870ba --- /dev/null +++ b/Benchmarks/update_benchmark_readme.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +set -euo pipefail + +RESULTS_DIR="${RESULTS_DIR:-Benchmarks/results}" +README_PATH="${README_PATH:-Benchmarks/Readme.md}" +SWIFT_REPORT="${SWIFT_REPORT:-}" +BASELINE_SWIFT_REPORT="${BASELINE_SWIFT_REPORT:-}" +GO_REPORT="${GO_REPORT:-}" +GO_RAW_REPORT="${GO_RAW_REPORT:-}" + +CMD=(Benchmarks/update_benchmark_readme.py --readme "${README_PATH}" --results-dir "${RESULTS_DIR}") + +if [[ -n "${SWIFT_REPORT}" ]]; then + CMD+=(--swift "${SWIFT_REPORT}") +fi + +if [[ -n "${BASELINE_SWIFT_REPORT}" ]]; then + CMD+=(--baseline-swift "${BASELINE_SWIFT_REPORT}") +fi + +if [[ -n "${GO_REPORT}" ]]; then + CMD+=(--go "${GO_REPORT}") +fi + +if [[ -n "${GO_RAW_REPORT}" ]]; then + CMD+=(--go-raw "${GO_RAW_REPORT}") +fi + +"${CMD[@]}" From 0bdddc07b98827ca484f03fde1cbe80af18715e3 Mon Sep 17 00:00:00 2001 From: Brian Floersch Date: Thu, 26 Mar 2026 19:55:32 -0400 Subject: [PATCH 4/5] Make Go the primary benchmark baseline --- Benchmarks/Readme.md | 213 +++++++++++------------ Benchmarks/compare_library_benchmarks.py | 10 +- Benchmarks/update_benchmark_readme.py | 39 +++-- 3 files changed, 137 insertions(+), 125 deletions(-) diff --git a/Benchmarks/Readme.md b/Benchmarks/Readme.md index ac9a8cf..a74fed5 100644 --- a/Benchmarks/Readme.md +++ b/Benchmarks/Readme.md @@ -60,85 +60,31 @@ These sections are generated from benchmark result files in `Benchmarks/results/ - Swift host: `brians-macbook-air.local` - Swift rounds: `5` measured, `1` warmup - Swift writes: `1000000`, sync writes: `5000000`, select writes/channel: `100000`, buffer: `100` -- Baseline Swift toolchain: `swift-6.2.3` -- Baseline Swift host: `brians-macbook-air.local` -- Baseline Swift rounds: `5` measured, `1` warmup -- Baseline Swift writes: `1000000`, sync writes: `5000000`, select writes/channel: `100000`, buffer: `100` -Baseline Swift report: `Benchmarks/results/swift-6.2.3-20260326-185306.json` -Candidate Swift report: `Benchmarks/results/swift-6.3.0-20260326-191022.json` +Swift report: `Benchmarks/results/swift-6.3.0-20260326-191022.json` +- Go toolchain: `go_version_go1.26.1_darwin_arm64` +- Go host: `Apple M5` +- Go rounds: `5` measured, `0` warmup +- Go writes: `1000000`, sync writes: `5000000`, select writes/channel: `100000`, buffer: `100` +Go report: `Benchmarks/results/go_version_go1.26.1_darwin_arm64-20260326-193146.json` -**Swift Toolchain Comparison** -Library | Test | Type | Baseline Avg (ms) | Candidate Avg (ms) | Delta % ---- | --- | --- | --- | --- | --- -AsyncAlgorithms | MPMC | `Int` | `7027.25` | `5215.65` | `-25.78%` -AsyncAlgorithms | MPMC | `RefData` | `7504.62` | `5588.75` | `-25.53%` -AsyncAlgorithms | MPMC | `String` | `8709.50` | `5587.76` | `-35.84%` -AsyncAlgorithms | MPMC | `ValueData` | `9590.54` | `5014.12` | `-47.72%` -AsyncAlgorithms | MPSC | `Int` | `4788.59` | `3761.30` | `-21.45%` -AsyncAlgorithms | MPSC | `RefData` | `5180.12` | `3945.61` | `-23.83%` -AsyncAlgorithms | MPSC | `String` | `6355.52` | `4179.43` | `-34.24%` -AsyncAlgorithms | MPSC | `ValueData` | `6757.28` | `4236.05` | `-37.31%` -AsyncAlgorithms | MPSC Write Contention | `Int` | `9298.62` | `5591.63` | `-39.87%` -AsyncAlgorithms | MPSC Write Contention | `RefData` | `8655.29` | `6262.41` | `-27.65%` -AsyncAlgorithms | MPSC Write Contention | `String` | `10685.25` | `6139.55` | `-42.54%` -AsyncAlgorithms | MPSC Write Contention | `ValueData` | `10456.41` | `5252.34` | `-49.77%` -AsyncAlgorithms | SPMC | `Int` | `4789.23` | `3622.70` | `-24.36%` -AsyncAlgorithms | SPMC | `RefData` | `5223.19` | `4114.84` | `-21.22%` -AsyncAlgorithms | SPMC | `String` | `6353.19` | `3890.67` | `-38.76%` -AsyncAlgorithms | SPMC | `ValueData` | `6499.68` | `3986.87` | `-38.66%` -AsyncAlgorithms | SPSC | `Int` | `2871.85` | `2305.84` | `-19.71%` -AsyncAlgorithms | SPSC | `RefData` | `2927.92` | `2418.31` | `-17.41%` -AsyncAlgorithms | SPSC | `String` | `3467.80` | `2350.12` | `-32.23%` -AsyncAlgorithms | SPSC | `ValueData` | `4047.56` | `2420.69` | `-40.19%` -AsyncChannels | Channel multi-select | `Int` | `1695.74` | `1381.14` | `-18.55%` -AsyncChannels | Channel multi-select | `RefData` | `1893.57` | `1176.29` | `-37.88%` -AsyncChannels | Channel multi-select | `String` | `1878.38` | `1362.78` | `-27.45%` -AsyncChannels | Channel multi-select | `ValueData` | `2183.19` | `1449.17` | `-33.62%` -AsyncChannels | MPMC | `Int` | `933.62` | `773.33` | `-17.17%` -AsyncChannels | MPMC | `RefData` | `983.93` | `725.90` | `-26.22%` -AsyncChannels | MPMC | `String` | `930.62` | `818.39` | `-12.06%` -AsyncChannels | MPMC | `ValueData` | `977.47` | `815.51` | `-16.57%` -AsyncChannels | MPMC Buffered(100) | `Int` | `454.85` | `459.06` | `+0.92%` -AsyncChannels | MPMC Buffered(100) | `RefData` | `649.37` | `501.98` | `-22.70%` -AsyncChannels | MPMC Buffered(100) | `String` | `495.95` | `445.53` | `-10.17%` -AsyncChannels | MPMC Buffered(100) | `ValueData` | `602.29` | `499.51` | `-17.06%` -AsyncChannels | MPSC | `Int` | `476.00` | `332.49` | `-30.15%` -AsyncChannels | MPSC | `RefData` | `638.52` | `323.41` | `-49.35%` -AsyncChannels | MPSC | `String` | `599.85` | `330.18` | `-44.96%` -AsyncChannels | MPSC | `ValueData` | `640.65` | `315.16` | `-50.81%` -AsyncChannels | MPSC Buffered(100) | `Int` | `264.37` | `157.45` | `-40.44%` -AsyncChannels | MPSC Buffered(100) | `RefData` | `474.13` | `147.87` | `-68.81%` -AsyncChannels | MPSC Buffered(100) | `String` | `365.77` | `212.38` | `-41.94%` -AsyncChannels | MPSC Buffered(100) | `ValueData` | `481.41` | `230.82` | `-52.05%` -AsyncChannels | MPSC Write Contention | `Int` | `511.44` | `385.93` | `-24.54%` -AsyncChannels | MPSC Write Contention | `RefData` | `730.74` | `376.60` | `-48.46%` -AsyncChannels | MPSC Write Contention | `String` | `681.47` | `390.94` | `-42.63%` -AsyncChannels | MPSC Write Contention | `ValueData` | `697.42` | `419.93` | `-39.79%` -AsyncChannels | MPSC Write Contention Buffered(100) | `Int` | `520.99` | `395.09` | `-24.17%` -AsyncChannels | MPSC Write Contention Buffered(100) | `RefData` | `788.64` | `356.57` | `-54.79%` -AsyncChannels | MPSC Write Contention Buffered(100) | `String` | `650.07` | `435.39` | `-33.02%` -AsyncChannels | MPSC Write Contention Buffered(100) | `ValueData` | `725.20` | `424.92` | `-41.41%` -AsyncChannels | SPMC | `Int` | `589.31` | `320.46` | `-45.62%` -AsyncChannels | SPMC | `RefData` | `631.42` | `302.77` | `-52.05%` -AsyncChannels | SPMC | `String` | `510.73` | `297.05` | `-41.84%` -AsyncChannels | SPMC | `ValueData` | `632.23` | `307.19` | `-51.41%` -AsyncChannels | SPMC Buffered(100) | `Int` | `283.86` | `219.90` | `-22.53%` -AsyncChannels | SPMC Buffered(100) | `RefData` | `428.43` | `180.02` | `-57.98%` -AsyncChannels | SPMC Buffered(100) | `String` | `319.52` | `228.32` | `-28.54%` -AsyncChannels | SPMC Buffered(100) | `ValueData` | `424.22` | `217.26` | `-48.79%` -AsyncChannels | SPSC | `Int` | `432.39` | `300.47` | `-30.51%` -AsyncChannels | SPSC | `RefData` | `594.98` | `355.01` | `-40.33%` -AsyncChannels | SPSC | `String` | `705.70` | `332.97` | `-52.82%` -AsyncChannels | SPSC | `ValueData` | `600.47` | `346.81` | `-42.24%` -AsyncChannels | SPSC Buffered(100) | `Int` | `494.76` | `439.99` | `-11.07%` -AsyncChannels | SPSC Buffered(100) | `RefData` | `622.45` | `368.84` | `-40.74%` -AsyncChannels | SPSC Buffered(100) | `String` | `609.59` | `469.59` | `-22.97%` -AsyncChannels | SPSC Buffered(100) | `ValueData` | `821.87` | `485.11` | `-40.97%` -AsyncChannels | SyncRW | `Int` | `487.80` | `327.48` | `-32.87%` -AsyncChannels | SyncRW | `RefData` | `669.29` | `363.38` | `-45.71%` -AsyncChannels | SyncRW | `String` | `651.87` | `357.71` | `-45.13%` -AsyncChannels | SyncRW | `ValueData` | `702.92` | `356.54` | `-49.28%` +**Go Baseline Comparison** +Go is the baseline. Negative percentages mean the Swift library is faster than Go for that scenario. + +Test | Type | Go Avg (ms) | AsyncChannels Avg (ms) | AsyncAlgorithms Avg (ms) | AsyncChannels vs Go | AsyncAlgorithms vs Go +--- | --- | --- | --- | --- | --- | --- +Channel multi-select | `Int` | `552.71` | `1381.14` | | `+149.88%` | +MPMC | `Int` | `241.73` | `773.33` | `5215.65` | `+219.91%` | `+2057.62%` +MPMC Buffered(100) | `Int` | `138.23` | `459.06` | | `+232.10%` | +MPSC | `Int` | `244.90` | `332.49` | `3761.30` | `+35.77%` | `+1435.87%` +MPSC Buffered(100) | `Int` | `80.55` | `157.45` | | `+95.48%` | +MPSC Write Contention | `Int` | `503.68` | `385.93` | `5591.63` | `-23.38%` | `+1010.15%` +MPSC Write Contention Buffered(100) | `Int` | `642.26` | `395.09` | | `-38.48%` | +SPMC | `Int` | `259.52` | `320.46` | `3622.70` | `+23.49%` | `+1295.95%` +SPMC Buffered(100) | `Int` | `124.92` | `219.90` | | `+76.03%` | +SPSC | `Int` | `99.82` | `300.47` | `2305.84` | `+201.01%` | `+2209.98%` +SPSC Buffered(100) | `Int` | `32.32` | `439.99` | | `+1261.22%` | +SyncRW | `Int` | `205.30` | `327.48` | | `+59.52%` | **AsyncChannels Results** Test | Type | Avg (ms) | Median (ms) | Ops/s @@ -216,42 +162,83 @@ SPMC | `RefData` | `4114.84` | `4196.80` | `243023` MPMC | `RefData` | `5588.75` | `5980.78` | `178931` MPSC Write Contention | `RefData` | `6262.41` | `6268.68` | `159683` -- Go toolchain: `go_version_go1.26.1_darwin_arm64` -- Go host: `Apple M5` -- Go rounds: `5` measured, `0` warmup -- Go writes: `1000000`, sync writes: `5000000`, select writes/channel: `100000`, buffer: `100` -Go report: `Benchmarks/results/go_version_go1.26.1_darwin_arm64-20260326-193146.json` +- Baseline Swift toolchain: `swift-6.2.3` +- Baseline Swift host: `brians-macbook-air.local` +- Baseline Swift rounds: `5` measured, `1` warmup +- Baseline Swift writes: `1000000`, sync writes: `5000000`, select writes/channel: `100000`, buffer: `100` +Baseline Swift report: `Benchmarks/results/swift-6.2.3-20260326-185306.json` -**Cross-Library Comparison** -Test | Type | AsyncAlgorithms Avg (ms) | AsyncChannels Avg (ms) | Go Avg (ms) | AsyncAlgorithms vs Go | AsyncChannels vs Go ---- | --- | --- | --- | --- | --- | --- -Channel multi-select | `Int` | | `1381.14` | `552.71` | | `+149.88%` -MPMC | `Int` | `5215.65` | `773.33` | `241.73` | `+2057.62%` | `+219.91%` -MPMC | `RefData` | `5588.75` | `725.90` | -MPMC | `String` | `5587.76` | `818.39` | -MPMC | `ValueData` | `5014.12` | `815.51` | -MPMC Buffered(100) | `Int` | | `459.06` | `138.23` | | `+232.10%` -MPSC | `Int` | `3761.30` | `332.49` | `244.90` | `+1435.87%` | `+35.77%` -MPSC | `RefData` | `3945.61` | `323.41` | -MPSC | `String` | `4179.43` | `330.18` | -MPSC | `ValueData` | `4236.05` | `315.16` | -MPSC Buffered(100) | `Int` | | `157.45` | `80.55` | | `+95.48%` -MPSC Write Contention | `Int` | `5591.63` | `385.93` | `503.68` | `+1010.15%` | `-23.38%` -MPSC Write Contention | `RefData` | `6262.41` | `376.60` | -MPSC Write Contention | `String` | `6139.55` | `390.94` | -MPSC Write Contention | `ValueData` | `5252.34` | `419.93` | -MPSC Write Contention Buffered(100) | `Int` | | `395.09` | `642.26` | | `-38.48%` -SPMC | `Int` | `3622.70` | `320.46` | `259.52` | `+1295.95%` | `+23.49%` -SPMC | `RefData` | `4114.84` | `302.77` | -SPMC | `String` | `3890.67` | `297.05` | -SPMC | `ValueData` | `3986.87` | `307.19` | -SPMC Buffered(100) | `Int` | | `219.90` | `124.92` | | `+76.03%` -SPSC | `Int` | `2305.84` | `300.47` | `99.82` | `+2209.98%` | `+201.01%` -SPSC | `RefData` | `2418.31` | `355.01` | -SPSC | `String` | `2350.12` | `332.97` | -SPSC | `ValueData` | `2420.69` | `346.81` | -SPSC Buffered(100) | `Int` | | `439.99` | `32.32` | | `+1261.22%` -SyncRW | `Int` | | `327.48` | `205.30` | | `+59.52%` +**Historical Swift Toolchain Comparison** +Library | Test | Type | Baseline Avg (ms) | Candidate Avg (ms) | Delta % +--- | --- | --- | --- | --- | --- +AsyncAlgorithms | MPMC | `Int` | `7027.25` | `5215.65` | `-25.78%` +AsyncAlgorithms | MPMC | `RefData` | `7504.62` | `5588.75` | `-25.53%` +AsyncAlgorithms | MPMC | `String` | `8709.50` | `5587.76` | `-35.84%` +AsyncAlgorithms | MPMC | `ValueData` | `9590.54` | `5014.12` | `-47.72%` +AsyncAlgorithms | MPSC | `Int` | `4788.59` | `3761.30` | `-21.45%` +AsyncAlgorithms | MPSC | `RefData` | `5180.12` | `3945.61` | `-23.83%` +AsyncAlgorithms | MPSC | `String` | `6355.52` | `4179.43` | `-34.24%` +AsyncAlgorithms | MPSC | `ValueData` | `6757.28` | `4236.05` | `-37.31%` +AsyncAlgorithms | MPSC Write Contention | `Int` | `9298.62` | `5591.63` | `-39.87%` +AsyncAlgorithms | MPSC Write Contention | `RefData` | `8655.29` | `6262.41` | `-27.65%` +AsyncAlgorithms | MPSC Write Contention | `String` | `10685.25` | `6139.55` | `-42.54%` +AsyncAlgorithms | MPSC Write Contention | `ValueData` | `10456.41` | `5252.34` | `-49.77%` +AsyncAlgorithms | SPMC | `Int` | `4789.23` | `3622.70` | `-24.36%` +AsyncAlgorithms | SPMC | `RefData` | `5223.19` | `4114.84` | `-21.22%` +AsyncAlgorithms | SPMC | `String` | `6353.19` | `3890.67` | `-38.76%` +AsyncAlgorithms | SPMC | `ValueData` | `6499.68` | `3986.87` | `-38.66%` +AsyncAlgorithms | SPSC | `Int` | `2871.85` | `2305.84` | `-19.71%` +AsyncAlgorithms | SPSC | `RefData` | `2927.92` | `2418.31` | `-17.41%` +AsyncAlgorithms | SPSC | `String` | `3467.80` | `2350.12` | `-32.23%` +AsyncAlgorithms | SPSC | `ValueData` | `4047.56` | `2420.69` | `-40.19%` +AsyncChannels | Channel multi-select | `Int` | `1695.74` | `1381.14` | `-18.55%` +AsyncChannels | Channel multi-select | `RefData` | `1893.57` | `1176.29` | `-37.88%` +AsyncChannels | Channel multi-select | `String` | `1878.38` | `1362.78` | `-27.45%` +AsyncChannels | Channel multi-select | `ValueData` | `2183.19` | `1449.17` | `-33.62%` +AsyncChannels | MPMC | `Int` | `933.62` | `773.33` | `-17.17%` +AsyncChannels | MPMC | `RefData` | `983.93` | `725.90` | `-26.22%` +AsyncChannels | MPMC | `String` | `930.62` | `818.39` | `-12.06%` +AsyncChannels | MPMC | `ValueData` | `977.47` | `815.51` | `-16.57%` +AsyncChannels | MPMC Buffered(100) | `Int` | `454.85` | `459.06` | `+0.92%` +AsyncChannels | MPMC Buffered(100) | `RefData` | `649.37` | `501.98` | `-22.70%` +AsyncChannels | MPMC Buffered(100) | `String` | `495.95` | `445.53` | `-10.17%` +AsyncChannels | MPMC Buffered(100) | `ValueData` | `602.29` | `499.51` | `-17.06%` +AsyncChannels | MPSC | `Int` | `476.00` | `332.49` | `-30.15%` +AsyncChannels | MPSC | `RefData` | `638.52` | `323.41` | `-49.35%` +AsyncChannels | MPSC | `String` | `599.85` | `330.18` | `-44.96%` +AsyncChannels | MPSC | `ValueData` | `640.65` | `315.16` | `-50.81%` +AsyncChannels | MPSC Buffered(100) | `Int` | `264.37` | `157.45` | `-40.44%` +AsyncChannels | MPSC Buffered(100) | `RefData` | `474.13` | `147.87` | `-68.81%` +AsyncChannels | MPSC Buffered(100) | `String` | `365.77` | `212.38` | `-41.94%` +AsyncChannels | MPSC Buffered(100) | `ValueData` | `481.41` | `230.82` | `-52.05%` +AsyncChannels | MPSC Write Contention | `Int` | `511.44` | `385.93` | `-24.54%` +AsyncChannels | MPSC Write Contention | `RefData` | `730.74` | `376.60` | `-48.46%` +AsyncChannels | MPSC Write Contention | `String` | `681.47` | `390.94` | `-42.63%` +AsyncChannels | MPSC Write Contention | `ValueData` | `697.42` | `419.93` | `-39.79%` +AsyncChannels | MPSC Write Contention Buffered(100) | `Int` | `520.99` | `395.09` | `-24.17%` +AsyncChannels | MPSC Write Contention Buffered(100) | `RefData` | `788.64` | `356.57` | `-54.79%` +AsyncChannels | MPSC Write Contention Buffered(100) | `String` | `650.07` | `435.39` | `-33.02%` +AsyncChannels | MPSC Write Contention Buffered(100) | `ValueData` | `725.20` | `424.92` | `-41.41%` +AsyncChannels | SPMC | `Int` | `589.31` | `320.46` | `-45.62%` +AsyncChannels | SPMC | `RefData` | `631.42` | `302.77` | `-52.05%` +AsyncChannels | SPMC | `String` | `510.73` | `297.05` | `-41.84%` +AsyncChannels | SPMC | `ValueData` | `632.23` | `307.19` | `-51.41%` +AsyncChannels | SPMC Buffered(100) | `Int` | `283.86` | `219.90` | `-22.53%` +AsyncChannels | SPMC Buffered(100) | `RefData` | `428.43` | `180.02` | `-57.98%` +AsyncChannels | SPMC Buffered(100) | `String` | `319.52` | `228.32` | `-28.54%` +AsyncChannels | SPMC Buffered(100) | `ValueData` | `424.22` | `217.26` | `-48.79%` +AsyncChannels | SPSC | `Int` | `432.39` | `300.47` | `-30.51%` +AsyncChannels | SPSC | `RefData` | `594.98` | `355.01` | `-40.33%` +AsyncChannels | SPSC | `String` | `705.70` | `332.97` | `-52.82%` +AsyncChannels | SPSC | `ValueData` | `600.47` | `346.81` | `-42.24%` +AsyncChannels | SPSC Buffered(100) | `Int` | `494.76` | `439.99` | `-11.07%` +AsyncChannels | SPSC Buffered(100) | `RefData` | `622.45` | `368.84` | `-40.74%` +AsyncChannels | SPSC Buffered(100) | `String` | `609.59` | `469.59` | `-22.97%` +AsyncChannels | SPSC Buffered(100) | `ValueData` | `821.87` | `485.11` | `-40.97%` +AsyncChannels | SyncRW | `Int` | `487.80` | `327.48` | `-32.87%` +AsyncChannels | SyncRW | `RefData` | `669.29` | `363.38` | `-45.71%` +AsyncChannels | SyncRW | `String` | `651.87` | `357.71` | `-45.13%` +AsyncChannels | SyncRW | `ValueData` | `702.92` | `356.54` | `-49.28%` **Raw Go Benchmark Output** ```text @@ -326,6 +313,8 @@ ok benchmarks 142.460s + + ## Future work I have not published benchmarks on linux. Linux uses `pthread_mutex_t` and from my limited testing, performance is slightly worse than MacOS, in line compared to AsyncAlgorithms, yet still significantly slower than go. diff --git a/Benchmarks/compare_library_benchmarks.py b/Benchmarks/compare_library_benchmarks.py index 3319600..0e7ce16 100755 --- a/Benchmarks/compare_library_benchmarks.py +++ b/Benchmarks/compare_library_benchmarks.py @@ -5,6 +5,12 @@ from collections import defaultdict +def sort_libraries(libraries: set[str]) -> list[str]: + preferred = ["Go", "AsyncChannels", "AsyncAlgorithms"] + remainder = sorted(library for library in libraries if library not in preferred) + return [library for library in preferred if library in libraries] + remainder + + def load(path: str) -> dict: with open(path, "r", encoding="utf-8") as handle: return json.load(handle) @@ -21,7 +27,7 @@ def main() -> int: for result in report["results"]: merged[(result["name"], result["type"])][result["library"]] = result - libraries = sorted({library for values in merged.values() for library in values}) + libraries = sort_libraries({library for values in merged.values() for library in values}) if not libraries: return 0 @@ -34,6 +40,8 @@ def main() -> int: for (name, result_type), values in sorted(merged.items()): if len(values) < 2: continue + if "Go" in libraries and "Go" not in values: + continue row = [name, f"`{result_type}`"] for library in libraries: result = values.get(library) diff --git a/Benchmarks/update_benchmark_readme.py b/Benchmarks/update_benchmark_readme.py index 27882dc..dd4b4bc 100755 --- a/Benchmarks/update_benchmark_readme.py +++ b/Benchmarks/update_benchmark_readme.py @@ -34,6 +34,12 @@ def markdown_table(headers: list[str], rows: Iterable[list[str]]) -> str: return "\n".join(lines) +def sort_libraries(libraries: set[str]) -> list[str]: + preferred = ["Go", "AsyncChannels", "AsyncAlgorithms"] + remainder = sorted(library for library in libraries if library not in preferred) + return [library for library in preferred if library in libraries] + remainder + + def render_report_table(report: dict, library: str) -> str: rows = [] for result in report["results"]: @@ -93,7 +99,7 @@ def render_library_comparison(reports: list[dict]) -> str: for result in report["results"]: merged.setdefault((result["name"], result["type"]), {})[result["library"]] = result - libraries = sorted({result["library"] for report in reports for result in report["results"]}) + libraries = sort_libraries({result["library"] for report in reports for result in report["results"]}) headers = ["Test", "Type"] + [f"{library} Avg (ms)" for library in libraries] if "Go" in libraries: headers += [f"{library} vs Go" for library in libraries if library != "Go"] @@ -102,6 +108,8 @@ def render_library_comparison(reports: list[dict]) -> str: for (name, result_type), values in sorted(merged.items()): if len(values) < 2: continue + if "Go" in libraries and "Go" not in values: + continue row = [name, f"`{result_type}`"] for library in libraries: result = values.get(library) @@ -152,16 +160,23 @@ def render_generated_section( render_environment(swift_report, "Swift"), ] - if baseline_swift_report: + parts.extend( + [ + "", + f"Swift report: `{source_paths['swift'].as_posix()}`", + ] + ) + + if go_report: parts.extend( [ - render_environment(baseline_swift_report, "Baseline Swift"), + render_environment(go_report, "Go"), + f"Go report: `{source_paths['go'].as_posix()}`", "", - f"Baseline Swift report: `{source_paths['baseline_swift'].as_posix()}`", - f"Candidate Swift report: `{source_paths['swift'].as_posix()}`", + "**Go Baseline Comparison**", + "Go is the baseline. Negative percentages mean the Swift library is faster than Go for that scenario.", "", - "**Swift Toolchain Comparison**", - render_swift_comparison(baseline_swift_report, swift_report), + render_library_comparison([swift_report, go_report]), ] ) @@ -182,15 +197,15 @@ def render_generated_section( ] ) - if go_report: + if baseline_swift_report: parts.extend( [ "", - render_environment(go_report, "Go"), - f"Go report: `{source_paths['go'].as_posix()}`", + render_environment(baseline_swift_report, "Baseline Swift"), + f"Baseline Swift report: `{source_paths['baseline_swift'].as_posix()}`", "", - "**Cross-Library Comparison**", - render_library_comparison([swift_report, go_report]), + "**Historical Swift Toolchain Comparison**", + render_swift_comparison(baseline_swift_report, swift_report), ] ) From aeb70b20b81cdd60412ee25fb88b6353c171c94e Mon Sep 17 00:00:00 2001 From: Brian Floersch Date: Thu, 26 Mar 2026 20:07:23 -0400 Subject: [PATCH 5/5] Add repo agent guide --- AGENTS.md | 164 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 164 insertions(+) create mode 100644 AGENTS.md diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..5ba9c0f --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,164 @@ +# AGENTS.md + +This repository is a Swift package implementing async channel primitives and a benchmark suite used to drive performance work. + +## Priorities + +- Go is the performance baseline. +- `AsyncChannels` is the implementation under optimization. +- `AsyncAlgorithms` is a useful secondary comparison, but not the golden target. +- Prefer changes that improve the full benchmark suite, not just one synthetic case. + +## Repository Layout + +- [Package.swift](Package.swift): root Swift package manifest. +- [Sources/AsyncChannels](Sources/AsyncChannels): library implementation. +- [Tests/AsyncChannelsTests](Tests/AsyncChannelsTests): behavior, type, and README examples. +- [Benchmarks](Benchmarks): Swift and Go benchmark harnesses and benchmark docs. +- [Examples/ImageConverter](Examples/ImageConverter): example package using the library. + +## Code Navigation + +Start here when working on channel performance or behavior: + +- [Channel.swift](Sources/AsyncChannels/Channel.swift): public `Channel` API, send/receive operators, sync helpers, close. +- [ThrowingChannel.swift](Sources/AsyncChannels/ThrowingChannel.swift): throwing variant of the channel API. +- [ChannelInternal.swift](Sources/AsyncChannels/ChannelInternal.swift): core queueing, continuation handoff, buffering, close behavior, and payload boxing. +- [Select.swift](Sources/AsyncChannels/Select.swift): select implementation and dynamic case handling. +- [FastLock.swift](Sources/AsyncChannels/FastLock.swift): platform-specific lock implementation. On Darwin it uses `os_unfair_lock`; on Linux it uses `pthread_mutex_t`. + +Useful test files: + +- [BehaviorTests.swift](Tests/AsyncChannelsTests/BehaviorTests.swift): stress and concurrency behavior tests. +- [TypeTests.swift](Tests/AsyncChannelsTests/TypeTests.swift): type behavior and sendability-focused tests. +- [ReadmeTests.swift](Tests/AsyncChannelsTests/ReadmeTests.swift): examples mirrored from the README. + +## Running Tests + +From the repo root: + +```bash +swift test +``` + +If you are validating a specific installed Swift toolchain: + +```bash +xcrun swift test +$HOME/.swiftly/bin/swift test +``` + +## Benchmark Layout + +Swift benchmark files: + +- [Benchmarks/Sources/Benchmarks/main.swift](Benchmarks/Sources/Benchmarks/main.swift): configurable Swift benchmark executable. +- [Benchmarks/run_swift_benchmarks.sh](Benchmarks/run_swift_benchmarks.sh): wrapper that runs the Swift benchmark suite and writes JSON reports. +- [Benchmarks/compare_swift_benchmarks.py](Benchmarks/compare_swift_benchmarks.py): compares two Swift JSON reports. + +Go benchmark files: + +- [Benchmarks/golang/benchmark_test.go](Benchmarks/golang/benchmark_test.go): Go benchmark suite. +- [Benchmarks/run_go_benchmarks.sh](Benchmarks/run_go_benchmarks.sh): wrapper that runs Go benchmarks and writes raw text plus normalized JSON. +- [Benchmarks/parse_go_benchmarks.py](Benchmarks/parse_go_benchmarks.py): converts `go test -bench` output to JSON. +- [Benchmarks/compare_library_benchmarks.py](Benchmarks/compare_library_benchmarks.py): compares Go vs Swift reports. + +README generation: + +- [Benchmarks/update_benchmark_readme.py](Benchmarks/update_benchmark_readme.py): rebuilds the generated benchmark section in the README from result files. +- [Benchmarks/update_benchmark_readme.sh](Benchmarks/update_benchmark_readme.sh): wrapper for the README generator. +- [Benchmarks/Readme.md](Benchmarks/Readme.md): benchmark documentation with generated result tables. + +## Running Benchmarks + +Run Swift benchmarks with the system toolchain: + +```bash +BENCHMARK_NAME=swift-current Benchmarks/run_swift_benchmarks.sh +``` + +Run Swift benchmarks with a specific Swiftly toolchain: + +```bash +BENCHMARK_NAME=swift-6.3.0 SWIFT_BIN=$HOME/.swiftly/bin/swift Benchmarks/run_swift_benchmarks.sh +``` + +Run Go benchmarks: + +```bash +BENCHMARK_NAME=go-1.26.1 Benchmarks/run_go_benchmarks.sh +``` + +Important knobs: + +- `ROUNDS`: measured rounds for Swift benchmarks. +- `WARMUP`: warmup rounds for Swift benchmarks. +- `COUNT`: benchmark count for Go. +- `WRITES` +- `SYNC_WRITES` +- `SELECT_WRITES` +- `BUFFER` +- `UPDATE_README=1`: refresh [Benchmarks/Readme.md](Benchmarks/Readme.md) after the run. + +Examples: + +```bash +UPDATE_README=1 BENCHMARK_NAME=swift-6.3.0 SWIFT_BIN=$HOME/.swiftly/bin/swift Benchmarks/run_swift_benchmarks.sh +UPDATE_README=1 BENCHMARK_NAME=go-1.26.1 Benchmarks/run_go_benchmarks.sh +``` + +## Comparing Results + +Compare two Swift toolchains: + +```bash +Benchmarks/compare_swift_benchmarks.py \ + Benchmarks/results/swift-6.2.3-.json \ + Benchmarks/results/swift-6.3.0-.json +``` + +Compare Swift vs Go on the same machine: + +```bash +Benchmarks/compare_library_benchmarks.py \ + Benchmarks/results/swift-6.3.0-.json \ + Benchmarks/results/go-1.26.1-.json +``` + +The generated README treats Go as the primary baseline near the top. + +## Updating Benchmark Docs + +Regenerate the benchmark README from the latest local result files: + +```bash +Benchmarks/update_benchmark_readme.sh +``` + +You can also pin exact result files: + +```bash +SWIFT_REPORT=Benchmarks/results/swift-6.3.0-.json \ +GO_REPORT=Benchmarks/results/go-1.26.1-.json \ +GO_RAW_REPORT=Benchmarks/results/go-1.26.1-.txt \ +Benchmarks/update_benchmark_readme.sh +``` + +`Benchmarks/results/` is gitignored. Result files are intentionally local unless someone explicitly decides to commit them elsewhere. + +## Tooling Notes + +- Go is installed via Homebrew on this machine. +- If `go` is missing, install it with: + +```bash +brew install go +``` + +- Swift benchmark runs use separate scratch paths per toolchain in `/tmp/asyncchannels-benchmarks` to avoid mixed-build artifacts. + +## Practical Advice For Performance Work + +- If you change queueing, continuation handoff, locking, or select behavior, rerun the full benchmark suite, not only one scenario. +- When benchmarking, keep the machine and workload constant. Cross-machine comparisons are not useful. +- The most important files for hot-path performance work are [ChannelInternal.swift](Sources/AsyncChannels/ChannelInternal.swift), [Select.swift](Sources/AsyncChannels/Select.swift), and [FastLock.swift](Sources/AsyncChannels/FastLock.swift).