Files
swift-nio/Sources/NIOCore/Linux.swift
e3d5c560e0 Fix coreCount on Linux when using cgroup v2 with CFS throttling disabled (#3462)
Fix coreCount on Linux when using cgroup v2 with CFS throttling disabled

### Motivation:

When using `swift-nio` on Linux with cgroup v2 enabled, but with CFS
throttling disabled, it falls back to attempting to read the cpuset file
at the cgroup v1 path. This does not exist, which in turns falls back to
returning `_SC_NPROCESSORS_ONLN`, which will return the total number of
cores available (ignoring cgroup assignments).

This has unexpected effects, including the default behaviour of starting
the `MultiThreadedEventLoopGroup.singleton` with significantly more
event loops than cores available to the workload.

### Modifications:

- Adds `SystemCalls.statfs`, and associated constants, to determine the
cgroup version.
- Adds `Linux.cgroupVersion()` API to expose cgroup version.
- Adds `Linux.cgroupV2MountPoint` variable to determine the cgroup v2
mount point.
- Adds `Linux.cpuSetPathV1` & `Linux.cpuSetPathV2` (and
`Linux.cpuSetPath` convenience) variables to determine the correct cpu
set path.
- Alters `System.coreCount` to use the appropriate logic from above to
ensure that `cpuset.cpus` is parsed from the correct location.

### Result:

`Linux.coreCount` should correctly parse and return the core count on
Linux cgroup v2 enabled systems (when CFS throttling is disabled), while
maintaining correctness for other configurations.

---------

Co-authored-by: Johannes Weiss <johannesweiss@apple.com>
Co-authored-by: Cory Benfield <lukasa@apple.com>
2026-01-05 14:26:21 +00:00

173 lines
6.1 KiB
Swift

//===----------------------------------------------------------------------===//
//
// This source file is part of the SwiftNIO open source project
//
// Copyright (c) 2017-2023 Apple Inc. and the SwiftNIO project authors
// Licensed under Apache License v2.0
//
// See LICENSE.txt for license information
// See CONTRIBUTORS.txt for the list of SwiftNIO project authors
//
// SPDX-License-Identifier: Apache-2.0
//
//===----------------------------------------------------------------------===//
// This is a companion to System.swift that provides only Linux specials: either things that exist
// only on Linux, or things that have Linux-specific extensions.
#if os(Linux) || os(Android)
import CNIOLinux
#if canImport(Android)
@preconcurrency import Android
#endif
enum Linux {
static let cfsQuotaPath = "/sys/fs/cgroup/cpu/cpu.cfs_quota_us"
static let cfsPeriodPath = "/sys/fs/cgroup/cpu/cpu.cfs_period_us"
static let cfsCpuMaxPath = "/sys/fs/cgroup/cpu.max"
static let cpuSetPathV1 = "/sys/fs/cgroup/cpuset/cpuset.cpus"
static let cpuSetPathV2: String? = {
if let cgroupV2MountPoint = Self.cgroupV2MountPoint {
return "\(cgroupV2MountPoint)/cpuset.cpus"
}
return nil
}()
static let cgroupV2MountPoint: String? = {
guard
let fd = try? SystemCalls.open(file: "/proc/self/cgroup", oFlag: O_RDONLY, mode: NIOPOSIXFileMode(S_IRUSR))
else { return nil }
defer { try! SystemCalls.close(descriptor: fd) }
guard let lines = try? Self.readLines(descriptor: fd) else { return nil }
// Parse each line looking for cgroup v2 format: "0::/path"
for line in lines {
if let cgroupPath = Self.parseV2CgroupLine(line) {
return "/sys/fs/cgroup\(cgroupPath)"
}
}
return nil
}()
/// Returns the appropriate cpuset path based on the detected cgroup version
static let cpuSetPath: String? = {
guard let version = Self.cgroupVersion else { return nil }
switch version {
case .v1:
return cpuSetPathV1
case .v2:
return cpuSetPathV2
}
}()
/// Detects whether we're using cgroup v1 or v2
static let cgroupVersion: CgroupVersion? = {
guard let type = try? SystemCalls.statfs_ftype("/sys/fs/cgroup") else { return nil }
switch type {
case CNIOLinux_TMPFS_MAGIC:
return .v1
case CNIOLinux_CGROUP2_SUPER_MAGIC:
return .v2
default:
return nil
}
}()
enum CgroupVersion {
case v1
case v2
}
/// Parses a single line from /proc/self/cgroup to extract cgroup v2 path
internal static func parseV2CgroupLine(_ line: Substring) -> String? {
// Expected format is "0::/path"
let parts = line.split(separator: ":", maxSplits: 2, omittingEmptySubsequences: false)
guard parts.count == 3,
parts[0] == "0",
parts[1] == ""
else {
return nil
}
// Extract the path from parts[2]
return String(parts[2])
}
private static func readLines(descriptor: CInt) throws -> [Substring] {
// linux doesn't properly report /sys/fs/cgroup/* files lengths so we use a reasonable limit
var buf = ByteBufferAllocator().buffer(capacity: 1024)
try buf.writeWithUnsafeMutableBytes(minimumWritableBytes: buf.capacity) { ptr in
let res = try SystemCalls.read(descriptor: descriptor, pointer: ptr.baseAddress!, size: ptr.count)
switch res {
case .processed(let n):
return n
case .wouldBlock:
preconditionFailure("read returned EWOULDBLOCK despite a blocking fd")
}
}
return String(buffer: buf).split(separator: "\n")
}
private static func firstLineOfFile(path: String) throws -> Substring? {
guard let fd = try? SystemCalls.open(file: path, oFlag: O_RDONLY, mode: NIOPOSIXFileMode(S_IRUSR)) else {
return nil
}
defer { try! SystemCalls.close(descriptor: fd) }
return try? Self.readLines(descriptor: fd).first
}
private static func countCoreIds(cores: Substring) -> Int {
let ids = cores.split(separator: "-", maxSplits: 1)
guard
let first = ids.first.flatMap({ Int($0, radix: 10) }),
let last = ids.last.flatMap({ Int($0, radix: 10) }),
last >= first
else { preconditionFailure("cpuset format is incorrect") }
return 1 + last - first
}
static func coreCount(cpuset cpusetPath: String) -> Int? {
guard
let cpuset = try? firstLineOfFile(path: cpusetPath).flatMap({ $0.split(separator: ",") }),
!cpuset.isEmpty
else { return nil }
return cpuset.map(countCoreIds).reduce(0, +)
}
/// Get the available core count according to cgroup1 restrictions.
/// Round up to the next whole number.
static func coreCountCgroup1Restriction(
quota quotaPath: String = Linux.cfsQuotaPath,
period periodPath: String = Linux.cfsPeriodPath
) -> Int? {
guard
let quota = try? firstLineOfFile(path: quotaPath).flatMap({ Int($0) }),
quota > 0
else { return nil }
guard
let period = try? firstLineOfFile(path: periodPath).flatMap({ Int($0) }),
period > 0
else { return nil }
return (quota - 1 + period) / period // always round up if fractional CPU quota requested
}
/// Get the available core count according to cgroup2 restrictions.
/// Round up to the next whole number.
static func coreCountCgroup2Restriction(cpuMaxPath: String = Linux.cfsCpuMaxPath) -> Int? {
guard let maxDetails = try? firstLineOfFile(path: cpuMaxPath),
let spaceIndex = maxDetails.firstIndex(of: " "),
let quota = Int(maxDetails[maxDetails.startIndex..<spaceIndex]),
let period = Int(maxDetails[maxDetails.index(after: spaceIndex)..<maxDetails.endIndex])
else { return nil }
return (quota - 1 + period) / period // always round up if fractional CPU quota requested
}
}
#endif