Fix a few simple sendability issues (#832)

Motivation:

We're about to go on a sendability journey. Let's pick some low hanging
fruit to get started.

Modifications:

- Add a few assume-isolated calls
- Stop using static var
- Use a dispatch group instead of a work item to wait for work to be
done.

Result:

Fewer warnings

Sources/AsyncHTTPClient/ConnectionPool/ChannelHandler/HTTP1ProxyConnectHandler.swift

@@ -137,7 +137,7 @@ final class HTTP1ProxyConnectHandler: ChannelDuplexHandler, RemovableChannelHand
             return
         }
 
-        let timeout = context.eventLoop.scheduleTask(deadline: self.deadline) {
+        let timeout = context.eventLoop.assumeIsolated().scheduleTask(deadline: self.deadline) {
             switch self.state {
             case .initialized:
                 preconditionFailure("How can we have a scheduled timeout, if the connection is not even up?")

Sources/AsyncHTTPClient/ConnectionPool/ChannelHandler/SOCKSEventsHandler.swift

@@ -99,7 +99,7 @@ final class SOCKSEventsHandler: ChannelInboundHandler, RemovableChannelHandler {
             return
         }
 
-        let scheduled = context.eventLoop.scheduleTask(deadline: self.deadline) {
+        let scheduled = context.eventLoop.assumeIsolated().scheduleTask(deadline: self.deadline) {
             switch self.state {
             case .initialized, .channelActive:
                 // close the connection, if the handshake timed out

Sources/AsyncHTTPClient/ConnectionPool/ChannelHandler/TLSEventsHandler.swift

@@ -104,7 +104,7 @@ final class TLSEventsHandler: ChannelInboundHandler, RemovableChannelHandler {
 
         var scheduled: Scheduled<Void>?
         if let deadline = deadline {
-            scheduled = context.eventLoop.scheduleTask(deadline: deadline) {
+            scheduled = context.eventLoop.assumeIsolated().scheduleTask(deadline: deadline) {
                 switch self.state {
                 case .initialized, .channelActive:
                     // close the connection, if the handshake timed out

Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+StateMachine.swift

@@ -26,7 +26,9 @@ extension HTTPConnectionPool {
                 self.connection = connection
             }
 
-            static let none = Action(request: .none, connection: .none)
+            static var none: Action {
+                Action(request: .none, connection: .none)
+            }
         }
 
         enum ConnectionAction {
@@ -397,7 +399,9 @@ extension HTTPConnectionPool.StateMachine {
     }
 
     struct EstablishedAction {
-        static let none: Self = .init(request: .none, connection: .none)
+        static var none: Self {
+            Self(request: .none, connection: .none)
+        }
         let request: HTTPConnectionPool.StateMachine.RequestAction
         let connection: EstablishedConnectionAction
     }

Sources/AsyncHTTPClient/HTTPClient+HTTPCookie.swift

@@ -216,7 +216,7 @@ extension String.UTF8View.SubSequence {
     }
 }
 
-private let posixLocale: UnsafeMutableRawPointer = {
+nonisolated(unsafe) private let posixLocale: UnsafeMutableRawPointer = {
     // All POSIX systems must provide a "POSIX" locale, and its date/time formats are US English.
     // https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap07.html#tag_07_03_05
     let _posixLocale = newlocale(LC_TIME_MASK | LC_NUMERIC_MASK, "POSIX", nil)!

Sources/AsyncHTTPClient/HTTPClient.swift

@@ -222,22 +222,49 @@ public class HTTPClient {
                 """
             )
         }
-        let errorStorage: NIOLockedValueBox<Error?> = NIOLockedValueBox(nil)
-        let continuation = DispatchWorkItem {}
-        self.shutdown(requiresCleanClose: requiresCleanClose, queue: DispatchQueue(label: "async-http-client.shutdown"))
-        { error in
-            if let error = error {
-                errorStorage.withLockedValue { errorStorage in
-                    errorStorage = error
+
+        final class ShutdownError: @unchecked Sendable {
+            // @unchecked because error is protected by lock.
+
+            // Stores whether the shutdown has happened or not.
+            private let lock: ConditionLock<Bool>
+            private var error: Error?
+
+            init() {
+                self.error = nil
+                self.lock = ConditionLock(value: false)
+            }
+
+            func didShutdown(_ error: (any Error)?) {
+                self.lock.lock(whenValue: false)
+                defer {
+                    self.lock.unlock(withValue: true)
                 }
+                self.error = error
+            }
+
+            func blockUntilShutdown() -> (any Error)? {
+                self.lock.lock(whenValue: true)
+                defer {
+                    self.lock.unlock(withValue: true)
+                }
+                return self.error
             }
-            continuation.perform()
         }
-        continuation.wait()
-        try errorStorage.withLockedValue { errorStorage in
-            if let error = errorStorage {
-                throw error
-            }
+
+        let shutdownError = ShutdownError()
+
+        self.shutdown(
+            requiresCleanClose: requiresCleanClose,
+            queue: DispatchQueue(label: "async-http-client.shutdown")
+        ) { error in
+            shutdownError.didShutdown(error)
+        }
+
+        let error = shutdownError.blockUntilShutdown()
+
+        if let error = error {
+            throw error
         }
     }
 
@@ -756,14 +783,13 @@ public class HTTPClient {
                 delegate: delegate
             )
 
-            var deadlineSchedule: Scheduled<Void>?
             if let deadline = deadline {
-                deadlineSchedule = taskEL.scheduleTask(deadline: deadline) {
+                let deadlineSchedule = taskEL.scheduleTask(deadline: deadline) {
                     requestBag.deadlineExceeded()
                 }
 
                 task.promise.futureResult.whenComplete { _ in
-                    deadlineSchedule?.cancel()
+                    deadlineSchedule.cancel()
                 }
             }
 

Sources/AsyncHTTPClient/NIOTransportServices/TLSConfiguration.swift

@@ -60,7 +60,7 @@ extension TLSVersion {
 @available(macOS 10.14, iOS 12.0, tvOS 12.0, watchOS 5.0, *)
 extension TLSConfiguration {
     /// Dispatch queue used by Network framework TLS to control certificate verification
-    static var tlsDispatchQueue = DispatchQueue(label: "TLSDispatch")
+    static let tlsDispatchQueue = DispatchQueue(label: "TLSDispatch")
 
     /// create NWProtocolTLS.Options for use with NIOTransportServices from the NIOSSL TLSConfiguration
     ///