Files
swift-nio/Sources/NIOConcurrencyHelpers/NIOThreadPoolWorkAvailable.swift
Kushal PisavadiaandGitHub 9b92dcd5c2 Replace ConditionLock with wake-one signalling NIOThreadPoolWorkAvailable (#3507)
**TL;DR** This change leads to a ~90% reduction in observed system CPU
time for some use cases by waking a single thread, instead of all idle
threads.

# Changes

Inlining the commit messages here.

## Add NIOThreadPool submit throughput benchmarks

### Motivation

`NIOThreadPool` had no benchmarks measuring submit overhead. This makes
it difficult to evaluate the cost of signalling changes or to catch
latency regressions.

### Modifications

Add thread pool submit benchmarks, covering use cases with 4-thread and
16-thread pools.

### Result

`NIOThreadPool` submit throughput and context-switch overhead are now
tracked by benchmarks.

### Benchmark Results

<details>

```
NIOThreadPool.serial_wakeup(16 threads)
╒══════════════════════════╤═══════════╤═══════════╤═══════════╤═══════════╤═══════════╤═══════════╤═══════════╤═══════════╕
│ Metric                   │        p0 │       p25 │       p50 │       p75 │       p90 │       p99 │      p100 │   Samples │
╞══════════════════════════╪═══════════╪═══════════╪═══════════╪═══════════╪═══════════╪═══════════╪═══════════╪═══════════╡
│ Context switches (K)     │        77 │        78 │        79 │        80 │        80 │        92 │        92 │        30 │
├──────────────────────────┼───────────┼───────────┼───────────┼───────────┼───────────┼───────────┼───────────┼───────────┤
│ Syscalls (total) (K) *   │       106 │       107 │       108 │       109 │       110 │       116 │       116 │        30 │
├──────────────────────────┼───────────┼───────────┼───────────┼───────────┼───────────┼───────────┼───────────┼───────────┤
│ Time (system CPU) (ms) * │      1649 │      1752 │      1768 │      1795 │      1826 │      1929 │      1929 │        30 │
├──────────────────────────┼───────────┼───────────┼───────────┼───────────┼───────────┼───────────┼───────────┼───────────┤
│ Time (total CPU) (ms) *  │      1701 │      1805 │      1821 │      1849 │      1879 │      1987 │      1987 │        30 │
├──────────────────────────┼───────────┼───────────┼───────────┼───────────┼───────────┼───────────┼───────────┼───────────┤
│ Time (user CPU) (ms) *   │        51 │        52 │        52 │        53 │        53 │        57 │        57 │        30 │
├──────────────────────────┼───────────┼───────────┼───────────┼───────────┼───────────┼───────────┼───────────┼───────────┤
│ Time (wall clock) (ms) * │       167 │       177 │       178 │       181 │       183 │       200 │       200 │        30 │
╘══════════════════════════╧═══════════╧═══════════╧═══════════╧═══════════╧═══════════╧═══════════╧═══════════╧═══════════╛

NIOThreadPool.serial_wakeup(4 threads)
╒══════════════════════════╤═══════════╤═══════════╤═══════════╤═══════════╤═══════════╤═══════════╤═══════════╤═══════════╕
│ Metric                   │        p0 │       p25 │       p50 │       p75 │       p90 │       p99 │      p100 │   Samples │
╞══════════════════════════╪═══════════╪═══════════╪═══════════╪═══════════╪═══════════╪═══════════╪═══════════╪═══════════╡
│ Context switches (K)     │        44 │        44 │        44 │        45 │        45 │        45 │        45 │        30 │
├──────────────────────────┼───────────┼───────────┼───────────┼───────────┼───────────┼───────────┼───────────┼───────────┤
│ Syscalls (total) (K) *   │        65 │        65 │        65 │        66 │        66 │        67 │        67 │        30 │
├──────────────────────────┼───────────┼───────────┼───────────┼───────────┼───────────┼───────────┼───────────┼───────────┤
│ Time (system CPU) (ms) * │       159 │       162 │       163 │       165 │       166 │       169 │       169 │        30 │
├──────────────────────────┼───────────┼───────────┼───────────┼───────────┼───────────┼───────────┼───────────┼───────────┤
│ Time (total CPU) (ms) *  │       178 │       182 │       183 │       185 │       186 │       190 │       190 │        30 │
├──────────────────────────┼───────────┼───────────┼───────────┼───────────┼───────────┼───────────┼───────────┼───────────┤
│ Time (user CPU) (ms) *   │        19 │        19 │        20 │        20 │        20 │        21 │        21 │        30 │
├──────────────────────────┼───────────┼───────────┼───────────┼───────────┼───────────┼───────────┼───────────┼───────────┤
│ Time (wall clock) (ms) * │        76 │        79 │        79 │        80 │        80 │        82 │        82 │        30 │
╘══════════════════════════╧═══════════╧═══════════╧═══════════╧═══════════╧═══════════╧═══════════╧═══════════╧═══════════╛
```

</details>

## Replace `ConditionLock` with wake-one signalling
`NIOThreadPoolWorkAvailable`

### Motivation

`NIOThreadPool` used `ConditionLock` which calls
`pthread_cond_broadcast` on every state change, waking all threads when
only one work item is enqueued. This causes a thundering-herd problem.

### Modifications

Add `NIOThreadPoolWorkAvailable` in NIOConcurrencyHelpers that uses
`pthread_cond_signal` (wake-one) for work submission and
`pthread_cond_broadcast` only for shutdown. Replace
`ConditionLock<_WorkState>` and the `_WorkState` enum in `NIOThreadPool`
with this new primitive.

### Result

Submitting a work item wakes exactly **one** thread instead of all
threads.

### Benchmark Results

<details>

```
NIOThreadPool.serial_wakeup(16 threads)
╒══════════════════════════╤═══════════╤═══════════╤═══════════╤═══════════╤═══════════╤═══════════╤═══════════╤═══════════╕
│ Metric                   │        p0 │       p25 │       p50 │       p75 │       p90 │       p99 │      p100 │   Samples │
╞══════════════════════════╪═══════════╪═══════════╪═══════════╪═══════════╪═══════════╪═══════════╪═══════════╪═══════════╡
│ Context switches (K)     │        20 │        20 │        20 │        20 │        20 │        20 │        20 │        30 │
├──────────────────────────┼───────────┼───────────┼───────────┼───────────┼───────────┼───────────┼───────────┼───────────┤
│ Syscalls (total) (K) *   │        40 │        40 │        40 │        40 │        40 │        40 │        40 │        30 │
├──────────────────────────┼───────────┼───────────┼───────────┼───────────┼───────────┼───────────┼───────────┼───────────┤
│ Time (system CPU) (ms) * │        47 │        49 │        49 │        50 │        50 │        55 │        55 │        30 │
├──────────────────────────┼───────────┼───────────┼───────────┼───────────┼───────────┼───────────┼───────────┼───────────┤
│ Time (total CPU) (ms) *  │        57 │        58 │        59 │        60 │        61 │        67 │        67 │        30 │
├──────────────────────────┼───────────┼───────────┼───────────┼───────────┼───────────┼───────────┼───────────┼───────────┤
│ Time (user CPU) (ms) *   │        10 │        10 │        10 │        10 │        10 │        12 │        12 │        30 │
├──────────────────────────┼───────────┼───────────┼───────────┼───────────┼───────────┼───────────┼───────────┼───────────┤
│ Time (wall clock) (ms) * │        54 │        55 │        56 │        56 │        57 │        65 │        65 │        30 │
╘══════════════════════════╧═══════════╧═══════════╧═══════════╧═══════════╧═══════════╧═══════════╧═══════════╧═══════════╛

NIOThreadPool.serial_wakeup(4 threads)
╒══════════════════════════╤═══════════╤═══════════╤═══════════╤═══════════╤═══════════╤═══════════╤═══════════╤═══════════╕
│ Metric                   │        p0 │       p25 │       p50 │       p75 │       p90 │       p99 │      p100 │   Samples │
╞══════════════════════════╪═══════════╪═══════════╪═══════════╪═══════════╪═══════════╪═══════════╪═══════════╪═══════════╡
│ Context switches (K)     │        20 │        20 │        20 │        20 │        20 │        20 │        20 │        30 │
├──────────────────────────┼───────────┼───────────┼───────────┼───────────┼───────────┼───────────┼───────────┼───────────┤
│ Syscalls (total) (K) *   │        40 │        40 │        40 │        40 │        40 │        40 │        40 │        30 │
├──────────────────────────┼───────────┼───────────┼───────────┼───────────┼───────────┼───────────┼───────────┼───────────┤
│ Time (system CPU) (ms) * │        45 │        46 │        46 │        47 │        57 │        75 │        75 │        30 │
├──────────────────────────┼───────────┼───────────┼───────────┼───────────┼───────────┼───────────┼───────────┼───────────┤
│ Time (total CPU) (ms) *  │        54 │        55 │        56 │        57 │        68 │        87 │        87 │        30 │
├──────────────────────────┼───────────┼───────────┼───────────┼───────────┼───────────┼───────────┼───────────┼───────────┤
│ Time (user CPU) (μs) *   │      9055 │      9372 │      9478 │      9765 │     10887 │     12585 │     12585 │        30 │
├──────────────────────────┼───────────┼───────────┼───────────┼───────────┼───────────┼───────────┼───────────┼───────────┤
│ Time (wall clock) (ms) * │        52 │        53 │        53 │        54 │        64 │       124 │       124 │        30 │
╘══════════════════════════╧═══════════╧═══════════╧═══════════╧═══════════╧═══════════╧═══════════╧═══════════╧═══════════╛
```

</details>
2026-02-10 20:01:35 +00:00

188 lines
6.7 KiB
Swift

//===----------------------------------------------------------------------===//
//
// This source file is part of the SwiftNIO open source project
//
// Copyright (c) 2026 Apple Inc. and the SwiftNIO project authors
// Licensed under Apache License v2.0
//
// See LICENSE.txt for license information
// See CONTRIBUTORS.txt for the list of SwiftNIO project authors
//
// SPDX-License-Identifier: Apache-2.0
//
//===----------------------------------------------------------------------===//
#if canImport(Darwin)
import Darwin
#elseif os(Windows)
import ucrt
import WinSDK
#elseif canImport(Glibc)
@preconcurrency import Glibc
#elseif canImport(Musl)
@preconcurrency import Musl
#elseif canImport(Bionic)
@preconcurrency import Bionic
#elseif canImport(WASILibc)
@preconcurrency import WASILibc
#if canImport(wasi_pthread)
import wasi_pthread
#endif
#else
#error("The NIOThreadPoolWorkAvailable module was unable to identify your C library.")
#endif
/// A specialized synchronization primitive for ``NIOThreadPool`` that uses
/// `pthread_cond_signal` (wake-one) instead of `pthread_cond_broadcast` (wake-all)
/// when work is enqueued, eliminating the thundering-herd problem.
///
/// This type manages a `workAvailable` counter under a mutex, paired with a
/// condition variable. Threads waiting for work block until `workAvailable > 0`.
@usableFromInline
package struct NIOThreadPoolWorkAvailable: @unchecked Sendable {
@usableFromInline
final class _Storage {
@usableFromInline
let lock: NIOLock
@usableFromInline
var workAvailable: Int
#if os(Windows)
@usableFromInline
let cond: UnsafeMutablePointer<CONDITION_VARIABLE> =
UnsafeMutablePointer.allocate(capacity: 1)
#elseif os(OpenBSD)
@usableFromInline
let cond: UnsafeMutablePointer<pthread_cond_t?> =
UnsafeMutablePointer.allocate(capacity: 1)
#elseif (compiler(<6.1) && !os(WASI)) || (compiler(>=6.1) && _runtime(_multithreaded))
@usableFromInline
let cond: UnsafeMutablePointer<pthread_cond_t> =
UnsafeMutablePointer.allocate(capacity: 1)
#endif
@usableFromInline
init() {
self.lock = NIOLock()
self.workAvailable = 0
#if os(Windows)
InitializeConditionVariable(self.cond)
#elseif (compiler(<6.1) && !os(WASI)) || (compiler(>=6.1) && _runtime(_multithreaded))
let err = pthread_cond_init(self.cond, nil)
precondition(err == 0, "\(#function) failed in pthread_cond_init with error \(err)")
#endif
}
deinit {
#if os(Windows)
// condition variables do not need to be explicitly destroyed
self.cond.deallocate()
#elseif (compiler(<6.1) && !os(WASI)) || (compiler(>=6.1) && _runtime(_multithreaded))
let err = pthread_cond_destroy(self.cond)
precondition(err == 0, "\(#function) failed in pthread_cond_destroy with error \(err)")
self.cond.deallocate()
#endif
}
}
@usableFromInline
let _storage: _Storage
@usableFromInline
package enum Signal: Sendable {
/// No signal after unlock.
case none
/// Wake one waiting thread (``pthread_cond_signal``).
case signalOne
/// Wake all waiting threads (``pthread_cond_broadcast``).
case broadcastAll
}
@inlinable
package init() {
self._storage = _Storage()
}
/// Lock, run `body`, apply the returned delta to `workAvailable`,
/// unlock, then signal as indicated by the return value.
@inlinable
package func withLock<Result>(
_ body: () -> (workDelta: Int, signal: Signal, result: Result)
) -> Result {
self._storage.lock.lock()
let (workDelta, signal, result) = body()
self._storage.workAvailable += workDelta
self._storage.lock.unlock()
self._signal(signal)
return result
}
/// Lock, run `body`, set `workAvailable` to the returned value if non-nil,
/// unlock, then signal as indicated. Use for shutdown where an
/// absolute value is needed rather than a delta.
@inlinable
package func withLockSettingWorkAvailable<Result>(
_ body: () -> (workAvailable: Int?, signal: Signal, result: Result)
) -> Result {
self._storage.lock.lock()
let (workAvailable, signal, result) = body()
if let workAvailable = workAvailable {
self._storage.workAvailable = workAvailable
}
self._storage.lock.unlock()
self._signal(signal)
return result
}
/// Lock, wait while `workAvailable <= 0`, run `body`, apply the
/// returned delta to `workAvailable`, unlock, then signal as indicated.
@inlinable
package func withLockWaitingForWork<Result>(
_ body: () -> (workDelta: Int, signal: Signal, result: Result)
) -> Result {
self._storage.lock.lock()
while self._storage.workAvailable <= 0 {
self._storage.lock.withLockPrimitive { mutex in
#if os(Windows)
let ok = SleepConditionVariableSRW(self._storage.cond, mutex, INFINITE, 0)
precondition(
ok,
"\(#function) failed in SleepConditionVariableSRW with error \(GetLastError())"
)
#elseif (compiler(<6.1) && !os(WASI)) || (compiler(>=6.1) && _runtime(_multithreaded))
let err = pthread_cond_wait(self._storage.cond, mutex)
precondition(err == 0, "\(#function) failed in pthread_cond_wait with error \(err)")
#endif
}
}
let (workDelta, signal, result) = body()
self._storage.workAvailable += workDelta
self._storage.lock.unlock()
self._signal(signal)
return result
}
@inlinable
func _signal(_ signal: Signal) {
switch signal {
case .none:
break
case .signalOne:
#if os(Windows)
WakeConditionVariable(self._storage.cond)
#elseif (compiler(<6.1) && !os(WASI)) || (compiler(>=6.1) && _runtime(_multithreaded))
let err = pthread_cond_signal(self._storage.cond)
precondition(err == 0, "\(#function) failed in pthread_cond_signal with error \(err)")
#endif
case .broadcastAll:
#if os(Windows)
WakeAllConditionVariable(self._storage.cond)
#elseif (compiler(<6.1) && !os(WASI)) || (compiler(>=6.1) && _runtime(_multithreaded))
let err = pthread_cond_broadcast(self._storage.cond)
precondition(err == 0, "\(#function) failed in pthread_cond_broadcast with error \(err)")
#endif
}
}
}