Write to the shared frontend lock early

README.md

@@ -292,7 +292,7 @@ where
 
 ```shell
 ditto "${SCRIPT_INPUT_FILE_0}" "${SCRIPT_OUTPUT_FILE_0}"
-[ -f "${SCRIPT_INPUT_FILE_1}" ] && ditto "${SCRIPT_INPUT_FILE_1}" "${SCRIPT_OUTPUT_FILE_1}" || rm "${SCRIPT_OUTPUT_FILE_1}"
+[ -f "${SCRIPT_INPUT_FILE_1}" ] && ditto "${SCRIPT_INPUT_FILE_1}" "${SCRIPT_OUTPUT_FILE_1}" || rm -f "${SCRIPT_OUTPUT_FILE_1}"
 ```
 
 where

Sources/XCRemoteCache/Commands/Postbuild/PostbuildContext.swift

@@ -153,6 +153,6 @@ extension PostbuildContext {
         }
         disabled = try env.readEnv(key: "XCRC_DISABLED") ?? false
         let llbuildId: String = try env.readEnv(key: "LLBUILD_BUILD_ID")
-        llbuildIdLockFile = XCSwiftFrontend.generateLlbuildIdSharedLockUrl(llbuildId: llbuildId, tmpDir: targetTempDir)
+        llbuildIdLockFile = XCSwiftFrontend.buildLlbuildIdSharedLockUrl(llbuildId: llbuildId, tmpDir: targetTempDir)
     }
 }

Sources/XCRemoteCache/Commands/Prebuild/PrebuildContext.swift

@@ -76,6 +76,6 @@ extension PrebuildContext {
         overlayHeadersPath = targetTempDir.appendingPathComponent("all-product-headers.yaml")
         disabled = try env.readEnv(key: "XCRC_DISABLED") ?? false
         let llbuildId: String = try env.readEnv(key: "LLBUILD_BUILD_ID")
-        llbuildIdLockFile = XCSwiftFrontend.generateLlbuildIdSharedLockUrl(llbuildId: llbuildId, tmpDir: targetTempDir)
+        llbuildIdLockFile = XCSwiftFrontend.buildLlbuildIdSharedLockUrl(llbuildId: llbuildId, tmpDir: targetTempDir)
     }
 }

Sources/XCRemoteCache/Commands/SwiftFrontend/SwiftFrontendOrchestrator.swift

@@ -91,13 +91,15 @@ class CommonSwiftFrontendOrchestrator {
     /// as we know, the remote cache cannot be used. Then all other compilation process (-c) can run
     /// in parallel with emit-module
     private func validateEmitModuleStep(criticalSection: () throws -> Void) throws {
+        debugLog("starting the emit-module step: locking")
         try lockAccessor.exclusiveAccess { handle in
-            defer {
-                handle.write(Self.self.emitModuleContent)
-            }
-            do {
-                try criticalSection()
-            }
+            debugLog("starting the emit-module step: locked")
+            // writing to the file content proactively - incase the critical section never returns
+            // (in case of a fallback to the local compilation), all awaiting swift-frontent processes
+            // will be immediatelly unblocked
+            handle.write(Self.self.emitModuleContent)
+            try criticalSection()
+            debugLog("lock file emit-module criticial end")
         }
     }
 
@@ -109,11 +111,15 @@ class CommonSwiftFrontendOrchestrator {
         var executed = false
         let startingDate = Date()
         while !executed {
+            debugLog("lock file compilation trying to acquire a lock ....")
             try lockAccessor.exclusiveAccess { handle in
                 if !handle.availableData.isEmpty {
                     // the file is not empty so the emit-module process is done with the "check"
+                    debugLog("swift-frontend lock file is unlocked for compilation")
                     try criticalSection()
                     executed = true
+                } else {
+                    debugLog("swift-frontend lock file is not ready for compilation")
                 }
             }
             // When a max locking time is achieved, execute anyway

Sources/XCRemoteCache/Commands/SwiftFrontend/XCSwiftFrontend.swift

@@ -54,13 +54,26 @@ public class XCSwiftFrontend: XCSwiftAbstract<SwiftFrontendArgInput> {
         return (config, context)
     }
 
+    override func fallbackCommand(config: XCRemoteCacheConfig) throws -> String {
+        // Do not invoke raw swift-frontend because that would lead to the invifnite loop
+        // swift-frontent -> xcswift-frontent -> swift-frontent
+        //
+        // Note: Returning the `swiftc` executaion here because it is possible to pass all arguments
+        // from swift-frontent to `swiftc` and swiftc will be able to redirect to swift-frontend
+        // (because the first argument is `-frontend`). If that is not a case (might change in
+        // future swift compiler versions), invoke swift-frontent from the Xcode, but that introduces
+        // a limitation that disallows custom toolchains in Xcode:
+        // $DEVELOPER_DIR/Toolchains/XcodeDefault.xctoolchain/usr/bin/swift-frontend
+        return config.swiftcCommand
+    }
+
     override public func run() throws {
         do {
             /// The LLBUILD_BUILD_ID ENV that describes the swiftc (parent) invocation
             let llbuildId: String = try env.readEnv(key: "LLBUILD_BUILD_ID")
             let (_, context) = try buildContext()
 
-            let sharedLockFileURL = XCSwiftFrontend.generateLlbuildIdSharedLockUrl(
+            let sharedLockFileURL = XCSwiftFrontend.buildLlbuildIdSharedLockUrl(
                 llbuildId: llbuildId,
                 tmpDir: context.tempDir
             )
@@ -87,7 +100,7 @@ public class XCSwiftFrontend: XCSwiftAbstract<SwiftFrontendArgInput> {
 extension XCSwiftFrontend {
     /// Generate the filename to be used to sycnhronize mutliple swift-frontend invocations
     /// The same file is used in prebuild, xcswift-frontend and postbuild (to clean it up)
-    static func generateLlbuildIdSharedLockUrl(llbuildId: String, tmpDir: URL) -> URL {
+    static func buildLlbuildIdSharedLockUrl(llbuildId: String, tmpDir: URL) -> URL {
         return tmpDir.appendingPathComponent(llbuildId).appendingPathExtension("lock")
     }
 }

Sources/XCRemoteCache/Commands/Swiftc/XCSwiftc.swift

@@ -67,12 +67,18 @@ public class XCSwiftAbstract<InputArgs> {
         fatalError("Need to override in \(Self.self)")
     }
 
+    // Return the fallback command that should be invoked in case of a cache miss
+    // Expected that swift-frontend invokcations will override it
+    func fallbackCommand(config: XCRemoteCacheConfig) throws -> String {
+        config.swiftcCommand
+    }
+
     // swiftlint:disable:next function_body_length
     public func run() throws {
         let fileManager = FileManager.default
         let (config, context) = try buildContext()
 
-        let swiftcCommand = config.swiftcCommand
+        let swiftcCommand = try fallbackCommand(config: config)
         let markerURL = context.tempDir.appendingPathComponent(config.modeMarkerPath)
         let markerReader = FileMarkerReader(markerURL, fileManager: fileManager)
         let markerWriter = FileMarkerWriter(markerURL, fileAccessor: fileManager)

Sources/XCRemoteCache/Dependencies/CacheModeController.swift

@@ -120,6 +120,12 @@ class PhaseCacheModeController: CacheModeController {
         return false
     }
 
+    // cleanup the build lock file (if exists) as the very last step of this controller
+    // this is just a non-critical cleanup step to not leave {{LLBUILD_BUILD_ID}}.lock
+    // files in $TARGET_TEMP_DIR. It is expected that both prebuild and postbuild will
+    // invoke it, to ensure:
+    // - swift-frontent synchronization is done per-target build
+    // - no .lock leftover files
     private func cleanupLlBuildLock() throws {
         if fileManager.fileExists(atPath: llbuildLockFile.path) {
             do {

e2eTests/StandaloneSampleApp/StandaloneApp.xcodeproj/project.pbxproj

@@ -376,7 +376,7 @@
 			);
 			runOnlyForDeploymentPostprocessing = 0;
 			shellPath = /bin/sh;
-			shellScript = "ditto \"${SCRIPT_INPUT_FILE_0}\" \"${SCRIPT_OUTPUT_FILE_0}\"\n[ -f \"${SCRIPT_INPUT_FILE_1}\" ] && ditto \"${SCRIPT_INPUT_FILE_1}\" \"${SCRIPT_OUTPUT_FILE_1}\" || rm \"${SCRIPT_OUTPUT_FILE_1}\"\n\n";
+			shellScript = "ditto \"${SCRIPT_INPUT_FILE_0}\" \"${SCRIPT_OUTPUT_FILE_0}\"\n[ -f \"${SCRIPT_INPUT_FILE_1}\" ] && ditto \"${SCRIPT_INPUT_FILE_1}\" \"${SCRIPT_OUTPUT_FILE_1}\" || rm -f \"${SCRIPT_OUTPUT_FILE_1}\"\n\n";
 		};
 /* End PBXShellScriptBuildPhase section */