Merge pull request #157 from CognitiveDisson/addUploadRetryDelay

Add a delay between network request retries

README.md

@@ -298,6 +298,7 @@ _Note that for the `producer` mode, the prebuild build phase and `xccc`, `xcld`,
 | `stats_dir` | Directory where all XCRemoteCache statistics (e.g. counters) are stored | `~/.xccache` | ⬜️ |
 | `download_retries` | Number of retries for download requests | `0` | ⬜️ |
 | `upload_retries` | Number of retries for upload requests | `3` | ⬜️ |
+| `retry_delay` | Delay between retries in seconds | `10` | ⬜️ |
 | `request_custom_headers` | Dictionary of extra HTTP headers for all remote server requests | `[]` | ⬜️ |
 | `thin_target_mock_filename` | Filename (without an extension) of the compilation input file that is used as a fake compilation for the forced-cached target (aka thin target) | `standin` | ⬜️ |
 | `focused_targets` | A list of all targets that are not thinned. If empty, all targets are meant to be non-thin | `[]` | ⬜️ |

Sources/XCRemoteCache/Commands/Postbuild/XCPostbuild.swift

@@ -123,6 +123,7 @@ public class XCPostbuild {
             let networkClient = NetworkClientImpl(
                 session: sessionFactory.build(),
                 retries: config.uploadRetries,
+                retryDelay: config.retryDelay,
                 fileManager: fileManager,
                 awsV4Signature: awsV4Signature
             )

Sources/XCRemoteCache/Commands/Prebuild/XCPrebuild.swift

@@ -109,6 +109,7 @@ public class XCPrebuild {
             let networkClient = NetworkClientImpl(
                 session: sessionFactory.build(),
                 retries: config.downloadRetries,
+                retryDelay: config.retryDelay,
                 fileManager: fileManager,
                 awsV4Signature: awsV4Signature
             )

Sources/XCRemoteCache/Commands/Prepare/XCPrepare.swift

@@ -87,6 +87,7 @@ public class XCPrepare {
             let networkClient = NetworkClientImpl(
                 session: sessionFactory.build(),
                 retries: config.downloadRetries,
+                retryDelay: config.retryDelay,
                 fileManager: fileManager,
                 awsV4Signature: awsV4Signature
             )

Sources/XCRemoteCache/Commands/Prepare/XCPrepareMark.swift

@@ -69,6 +69,7 @@ public class XCPrepareMark {
             let networkClient = NetworkClientImpl(
                 session: sessionFactory.build(),
                 retries: config.uploadRetries,
+                retryDelay: config.retryDelay,
                 fileManager: fileManager,
                 awsV4Signature: awsV4Signature
             )

Sources/XCRemoteCache/Config/XCRemoteCacheConfig.swift

@@ -83,6 +83,8 @@ public struct XCRemoteCacheConfig: Encodable {
     var downloadRetries: Int = 0
     /// Number of retries for upload requests
     var uploadRetries: Int = 3
+    /// Delay between retries in seconds
+    var retryDelay: Double = 10.0
     /// Extra headers appended to all remote HTTP(S) requests
     var requestCustomHeaders: [String: String] = [:]
     /// Filename (without an extension) of the compilation input file that is used
@@ -175,6 +177,7 @@ extension XCRemoteCacheConfig {
         merge.statsDir = scheme.statsDir ?? statsDir
         merge.downloadRetries = scheme.downloadRetries ?? downloadRetries
         merge.uploadRetries = scheme.uploadRetries ?? uploadRetries
+        merge.retryDelay = scheme.retryDelay ?? retryDelay
         merge.requestCustomHeaders = scheme.requestCustomHeaders ?? requestCustomHeaders
         merge.thinTargetMockFilename = scheme.thinTargetMockFilename ?? thinTargetMockFilename
         merge.focusedTargets = scheme.focusedTargets ?? focusedTargets
@@ -243,6 +246,7 @@ struct ConfigFileScheme: Decodable {
     let statsDir: String?
     let downloadRetries: Int?
     let uploadRetries: Int?
+    let retryDelay: Double?
     let requestCustomHeaders: [String: String]?
     let thinTargetMockFilename: String?
     let focusedTargets: [String]?
@@ -291,6 +295,7 @@ struct ConfigFileScheme: Decodable {
         case statsDir = "stats_dir"
         case downloadRetries = "download_retries"
         case uploadRetries = "upload_retries"
+        case retryDelay = "retry_delay"
         case requestCustomHeaders = "request_custom_headers"
         case thinTargetMockFilename = "thin_target_mock_filename"
         case focusedTargets = "focused_targets"
@@ -357,6 +362,7 @@ class XCRemoteCacheConfigReader {
                 extraConfURL = URL(fileURLWithPath: config.extraConfigurationFile, relativeTo: rootURL)
             } catch {
                 infoLog("Extra config override failed with \(error). Skipping extra configuration")
+                // swiftlint:disable:next unneeded_break_in_switch
                 break
             }
         }

Sources/XCRemoteCache/Network/NetworkClientImpl.swift

@@ -29,12 +29,14 @@ class NetworkClientImpl: NetworkClient {
     private let session: URLSession
     private let fileManager: FileManager
     private let maxRetries: Int
+    private let retryDelay: TimeInterval
     private let awsV4Signature: AWSV4Signature?
 
-    init(session: URLSession, retries: Int, fileManager: FileManager, awsV4Signature: AWSV4Signature?) {
+    init(session: URLSession, retries: Int, retryDelay: TimeInterval, fileManager: FileManager, awsV4Signature: AWSV4Signature?) {
         self.session = session
         self.fileManager = fileManager
-        maxRetries = retries
+        self.maxRetries = retries
+        self.retryDelay = retryDelay
         self.awsV4Signature = awsV4Signature
     }
 
@@ -173,7 +175,13 @@ class NetworkClientImpl: NetworkClient {
             if let error = responseError {
                 if retries > 0 {
                     infoLog("Upload request failed with \(error). Left retries: \(retries).")
-                    self.makeUploadRequest(request, input: input, retries: retries - 1, completion: completion)
+                    self.retryUpload(
+                        request,
+                        input: input,
+                        retries: retries,
+                        completion: completion,
+                        after: self.retryDelay
+                    )
                     return
                 }
                 errorLog("Upload request failed: \(error)")
@@ -184,6 +192,13 @@ class NetworkClientImpl: NetworkClient {
         }
         dataTask.resume()
     }
+
+    private func retryUpload(_ request: URLRequest, input: URL, retries: Int, completion: @escaping (Result<Void, NetworkClientError>) -> Void, after: TimeInterval) {
+        DispatchQueue.global().asyncAfter(deadline: .now() + after) { [weak self] in
+            guard let self = self else { return }
+            self.makeUploadRequest(request, input: input, retries: retries - 1, completion: completion)
+        }
+    }
 }
 
 private extension NetworkClientError {

Tests/XCRemoteCacheTests/Network/NetworkClientImplTests.swift

@@ -72,7 +72,13 @@ class NetworkClientImplTests: XCTestCase {
         configuration.protocolClasses = [URLProtocolStub.self]
         session = URLSession(configuration: configuration)
         fileManager = FileManager.default
-        client = NetworkClientImpl(session: session, retries: 0, fileManager: fileManager, awsV4Signature: nil)
+        client = NetworkClientImpl(
+            session: session,
+            retries: 0,
+            retryDelay: 0,
+            fileManager: fileManager,
+            awsV4Signature: nil
+        )
     }
 
     override func tearDown() {
@@ -87,7 +93,7 @@ class NetworkClientImplTests: XCTestCase {
         super.tearDown()
     }
 
-    func waitForResponse<R>(_ action: (@escaping Completion<R>) -> Void) throws -> Result<R, NetworkClientError> {
+    func waitForResponse<R>(_ action: (@escaping Completion<R>) -> Void, timeout: TimeInterval = 0.1) throws -> Result<R, NetworkClientError> {
         let responseExpectation = expectation(description: "RequestResponse")
         var receivedResponse: Result<R, NetworkClientError>?
 
@@ -95,7 +101,7 @@ class NetworkClientImplTests: XCTestCase {
             receivedResponse = response
             responseExpectation.fulfill()
         }
-        waitForExpectations(timeout: 0.1)
+        waitForExpectations(timeout: timeout)
         return try receivedResponse.unwrap()
     }
 
@@ -141,9 +147,15 @@ class NetworkClientImplTests: XCTestCase {
     }
 
     func testUploadFilureWith400Retries() throws {
-        client = NetworkClientImpl(session: session, retries: 2, fileManager: fileManager, awsV4Signature: nil)
+        client = NetworkClientImpl(
+            session: session,
+            retries: 2,
+            retryDelay: 0,
+            fileManager: fileManager,
+            awsV4Signature: nil
+        )
         responses[url] = .success(failureResponse, Data())
-        _ = try waitForResponse { client.upload(fileURL, as: url, completion: $0) }
+        _ = try waitForResponse({ client.upload(fileURL, as: url, completion: $0) }, timeout: 0.5)
 
         XCTAssertEqual(
             requests.map(\.url),
@@ -153,7 +165,13 @@ class NetworkClientImplTests: XCTestCase {
     }
 
     func testUploadSuccessDoesntRetry() throws {
-        client = NetworkClientImpl(session: session, retries: 0, fileManager: fileManager, awsV4Signature: nil)
+        client = NetworkClientImpl(
+            session: session,
+            retries: 0,
+            retryDelay: 0,
+            fileManager: fileManager,
+            awsV4Signature: nil
+        )
         responses[url] = .success(successResponse, Data())
         _ = try waitForResponse { client.upload(fileURL, as: url, completion: $0) }
 
@@ -208,7 +226,13 @@ class NetworkClientImplTests: XCTestCase {
             service: "iam",
             date: Date(timeIntervalSince1970: 1_440_938_160)
         )
-        client = NetworkClientImpl(session: session, retries: 0, fileManager: fileManager, awsV4Signature: signature)
+        client = NetworkClientImpl(
+            session: session,
+            retries: 0,
+            retryDelay: 0,
+            fileManager: fileManager,
+            awsV4Signature: signature
+        )
         responses[url] = .success(successResponse, Data())
         _ = try waitForResponse { client.fetch(url, completion: $0) }