Add producerFast mode

2021-11-22 21:59:19 +01:00
parent fa82f920ad
commit 1c94e51059
11 changed files with 111 additions and 23 deletions
@@ -33,13 +33,16 @@ enum ThinningCreatorPluginError: Error {
 /// Warning! This plugin assumes that producer's DerivedData are always cleaned before a build
 class ThinningCreatorPlugin: ArtifactCreatorPlugin {
    private let targetTempDir: URL
+    private let modeMarkerPath: String
    private let dirScanner: DirScanner

    /// Default Initializer
    /// - Parameter targetTempDir: Location of current target-specific temp dir (TARGET_TEMP_DIR)
+    /// - Parameter modeMarkerPath: path of maker file that informs if a given target can reuse remote artifacts.
    /// - Parameter dirScanner: scanner to access disk and read files and directories hierarchy
-    init(targetTempDir: URL, dirScanner: DirScanner) {
+    init(targetTempDir: URL, modeMarkerPath: String, dirScanner: DirScanner) {
        self.targetTempDir = targetTempDir
+        self.modeMarkerPath = modeMarkerPath
        self.dirScanner = dirScanner
    }

@@ -57,20 +60,8 @@ class ThinningCreatorPlugin: ArtifactCreatorPlugin {
            let fileKey: String
        }
        let uploadedTargetArtifacts = try allURLs.compactMap { tempDir -> TargetTuple? in
-            // All targets that uploaded their artifacts, have it placed in the
-            // `$(TARGET_TEMP_DIR)/xccache/produced/{{fileKey}}.zip` location. Find all targets that have such a file
-
-            let targetGeneratedArtifactRootDir = tempDir
-                .appendingPathComponent("xccache")
-                .appendingPathComponent("produced")
-            guard try dirScanner.itemType(atPath: targetGeneratedArtifactRootDir.path) == ItemType.dir else {
-                // given target didn't generate any artifacts (e.g. it is never cached with XCRemoteCache)
-                return nil
-            }
-
-            let allFilesProduced = try dirScanner.items(at: targetGeneratedArtifactRootDir)
-            let allArtifacts = allFilesProduced.filter { $0.pathExtension == "zip" }
-            guard !allArtifacts.isEmpty else {
+            let potentialArtifacts = try findTargetPackageZip(tempDir: tempDir)
+            guard let allArtifacts = potentialArtifacts, !allArtifacts.isEmpty else {
                // there is no generated *.zip file, so given target didn't create an artifact - it could be
                // just a helper target (like the target we integrate this plugin with)
                return nil
@@ -78,7 +69,7 @@ class ThinningCreatorPlugin: ArtifactCreatorPlugin {
            // Find {{fileKey}} based on the .zip file basename
            guard allArtifacts.count == 1 else {
                throw ThinningCreatorPluginError.noSingleTargetArtifactsGenerated(
-                    rootDir: targetGeneratedArtifactRootDir
+                    rootDir: tempDir
                )
            }
            let fileKey = allArtifacts[0].deletingPathExtension().lastPathComponent
@@ -96,6 +87,38 @@ class ThinningCreatorPlugin: ArtifactCreatorPlugin {
        return Dictionary(uniqueKeysWithValues: extraKeysTuples)
    }

+    private func findTargetPackageZip(tempDir: URL) throws -> [URL]? {
+        // Producer mode:
+        // All targets that uploaded their artifacts, have it placed in the
+        // `$(TARGET_TEMP_DIR)/xccache/produced/{{fileKey}}.zip` location. Find all targets that have such a file
+        // ProducerFull mode:
+        // If a target reused already existing artifact, it still has `$(TARGET_TEMP_DIR)/rc.enabled` marker file
+        // and the reused zip is placed in:
+        // `$(TARGET_TEMP_DIR)/xccache/{{fileKey}}.zip` location.
+
+        let targetEnabledMarker = tempDir.appendingPathComponent(modeMarkerPath)
+        let targetReusedArtifactRootDir = tempDir.appendingPathComponent("xccache")
+        let targetGeneratedArtifactRootDir = tempDir
+            .appendingPathComponent("xccache")
+            .appendingPathComponent("produced")
+
+        let pathToDirWithZipArtifacts: URL
+        // try the ProducerFull mode path first
+        if try dirScanner.itemType(atPath: targetEnabledMarker.path) == ItemType.file {
+            pathToDirWithZipArtifacts = targetReusedArtifactRootDir
+        } else {
+            guard try dirScanner.itemType(atPath: targetGeneratedArtifactRootDir.path) == ItemType.dir else {
+                // given target didn't generate any artifacts (e.g. it is never cached with XCRemoteCache)
+                return nil
+            }
+            pathToDirWithZipArtifacts = targetGeneratedArtifactRootDir
+        }
+
+        let allFilesProduced = try dirScanner.items(at: pathToDirWithZipArtifacts)
+        let allArtifacts = allFilesProduced.filter { $0.pathExtension == "zip" }
+        return allArtifacts
+    }
+
    func artifactToUpload(main: MainArtifactMeta) throws -> [Artifact] {
        return []
    }
@@ -73,6 +73,7 @@ public struct PostbuildContext {
    var thinnedTargets: [String]
    /// Action type: build, indexbuild etc.
    var action: BuildActionType
+    let modeMarkerPath: String
 }

 extension PostbuildContext {
@@ -115,5 +116,6 @@ extension PostbuildContext {
        let thinFocusedTargetsString: String = env.readEnv(key: "SPT_XCREMOTE_CACHE_THINNED_TARGETS") ?? ""
        thinnedTargets = thinFocusedTargetsString.split(separator: ",").map(String.init)
        action = (try? BuildActionType(rawValue: env.readEnv(key: "ACTION"))) ?? .unknown
+        modeMarkerPath = config.modeMarkerPath
    }
 }
@@ -205,9 +205,10 @@ public class XCPostbuild {
                            worker: DispatchGroupParallelizationWorker(qos: .userInitiated)
                        )
                        consumerPlugins.append(thinningPlugin)
-                    case .producer:
+                    case .producer, .producerFast:
                        let thinningPlugin = ThinningCreatorPlugin(
                            targetTempDir: context.targetTempDir,
+                            modeMarkerPath: context.modeMarkerPath,
                            dirScanner: fileManager
                        )
                        creatorPlugins.append(thinningPlugin)
@@ -120,9 +120,9 @@ class Swiftc: SwiftcProtocol {
                let prebuildDiscoveryURL = context.tempDir.appendingPathComponent(context.prebuildDependenciesPath)
                let prebuildDiscoverWriter = dependenciesWriterFactory(prebuildDiscoveryURL, fileManager)
                try prebuildDiscoverWriter.write(skipForSha: remoteCommit)
-            case .consumer, .producer:
-                // Never skips prebuild phase and fallbacks to the swiftc compilation for:
-                // 1) Not enabled remote cache or 2) producer
+            case .consumer, .producer, .producerFast:
+                // Never skip prebuild phase and fallback to the swiftc compilation for:
+                // 1) Not enabled remote cache, 2) producer(s)
                break
            }
            return .forceFallback
@@ -24,6 +24,8 @@ public struct SwiftcContext {
        case producer
        /// Commit sha of the commit to use during remote cache
        case consumer(commit: RemoteCommitInfo)
+        /// Remote artifact exists and can be optimistically used in lieu of a local compilation
+        case producerFast
    }

    let objcHeaderOutput: URL
@@ -74,6 +76,14 @@ public struct SwiftcContext {
            mode = .consumer(commit: remoteCommit)
        case .producer:
            mode = .producer
+        case .producerFast:
+            let remoteCommit = RemoteCommitInfo(try? String(contentsOf: remoteCommitLocation).trim())
+            switch remoteCommit {
+            case .unavailable:
+                mode = .producer
+            case .available:
+                mode = .producerFast
+            }
        }
        invocationHistoryFile = URL(fileURLWithPath: config.compilationHistoryFile, relativeTo: tempDir)
    }
@@ -116,6 +116,12 @@ class SwiftcOrchestrator {
            }
        case .consumer:
            fallbackToDefault(command: swiftcCommand)
+        case .producerFast:
+            let compileStepResult = try swiftc.mockCompilation()
+            if case .forceFallback = compileStepResult {
+                // cannot reuse cached artifact. Build it locally and upload to the server just as for the producer
+                fallthrough
+            }
        case .producer:
            var swiftcArgs = ProcessInfo().arguments
            swiftcArgs = try producerFallbackCommandProcessors.reduce(swiftcArgs) { args, processor in
@@ -20,4 +20,5 @@
 public enum Mode: String, Codable, CaseIterable {
    case consumer
    case producer
+    case producerFast
 }
@@ -44,7 +44,7 @@ class RemoteNetworkClientAbstractFactory {
            return RemoteNetworkClientImpl(networkClient, downloadURLBuilder)
        }
        switch mode {
-        case .producer:
+        case .producer, .producerFast:
            let upstreamBuilders = try upstreamStreamURL.map(urlBuilderFactory)
            return ReplicatedRemotesNetworkClient(
                networkClient,
@@ -33,7 +33,10 @@ class ThinningCreatorPluginTests: FileXCTestCase {
        targetTempDirRoot = workingDir.appendingPathComponent("Root")
        currentTargetTempDir = targetTempDirRoot.appendingPathComponent("Current.build")
        try fileManager.spt_createEmptyDir(currentTargetTempDir)
-        plugin = ThinningCreatorPlugin(targetTempDir: currentTargetTempDir, dirScanner: FileManager.default)
+        plugin = ThinningCreatorPlugin(
+            targetTempDir: currentTargetTempDir,
+            modeMarkerPath: "rc.enabled",
+            dirScanner: FileManager.default)
    }

    func testReturnsEmptyExtraKeysForNoArtifacts() throws {
@@ -50,7 +50,8 @@ class PostbuildTests: FileXCTestCase {
        bundleDir: nil,
        derivedSourcesDir: "",
        thinnedTargets: [],
-        action: .build
+        action: .build,
+        modeMarkerPath: ""
    )
    private var network = RemoteNetworkClientImpl(
        NetworkClientFake(fileManager: .default),
@@ -190,4 +190,45 @@ class SwiftcOrchestratorTests: XCTestCase {

        XCTAssertNotNil(shellOutSpy.switchedProcess)
    }
+
+    func testForFailedCompilationMockInProducerFastModeBuildsArtifactObjCHeader() throws {
+        let swiftc = SwiftcMock(mockingResult: .forceFallback)
+        let orchestrator = SwiftcOrchestrator(
+            mode: .producerFast,
+            swiftc: swiftc,
+            swiftcCommand: "",
+            objcHeaderOutput: objcHeaderURL,
+            moduleOutput: moduleOutputURL,
+            arch: "archTest",
+            artifactBuilder: artifactBuilder,
+            producerFallbackCommandProcessors: [],
+            invocationStorage: invocationStorage,
+            shellOut: shellOutSpy
+        )
+
+        try orchestrator.run()
+
+        XCTAssertEqual(artifactBuilder.addedObjCHeaders, ["archTest": [objcHeaderURL]])
+    }
+
+    func testSuccessedMockInProducerFastModeModeDoesntFillObjCHeader() throws {
+        let swiftc = SwiftcMock(mockingResult: .success)
+        let orchestrator = SwiftcOrchestrator(
+            mode: .producerFast,
+            swiftc: swiftc,
+            swiftcCommand: "",
+            objcHeaderOutput: objcHeaderURL,
+            moduleOutput: moduleOutputURL,
+            arch: "arch",
+            artifactBuilder: artifactBuilder,
+            producerFallbackCommandProcessors: [],
+            invocationStorage: invocationStorage,
+            shellOut: shellOutSpy
+        )
+
+        try orchestrator.run()
+
+        XCTAssertEqual(artifactBuilder.addedObjCHeaders, [:])
+    }
+
 }