diff --git a/Sources/XCRemoteCache/Commands/Plugins/Thinning/ThinningCreatorPlugin.swift b/Sources/XCRemoteCache/Commands/Plugins/Thinning/ThinningCreatorPlugin.swift index e5b0354..c8b431b 100644 --- a/Sources/XCRemoteCache/Commands/Plugins/Thinning/ThinningCreatorPlugin.swift +++ b/Sources/XCRemoteCache/Commands/Plugins/Thinning/ThinningCreatorPlugin.swift @@ -33,13 +33,16 @@ enum ThinningCreatorPluginError: Error { /// Warning! This plugin assumes that producer's DerivedData are always cleaned before a build class ThinningCreatorPlugin: ArtifactCreatorPlugin { private let targetTempDir: URL + private let modeMarkerPath: String private let dirScanner: DirScanner /// Default Initializer /// - Parameter targetTempDir: Location of current target-specific temp dir (TARGET_TEMP_DIR) + /// - Parameter modeMarkerPath: path of maker file that informs if a given target can reuse remote artifacts. /// - Parameter dirScanner: scanner to access disk and read files and directories hierarchy - init(targetTempDir: URL, dirScanner: DirScanner) { + init(targetTempDir: URL, modeMarkerPath: String, dirScanner: DirScanner) { self.targetTempDir = targetTempDir + self.modeMarkerPath = modeMarkerPath self.dirScanner = dirScanner } @@ -57,20 +60,8 @@ class ThinningCreatorPlugin: ArtifactCreatorPlugin { let fileKey: String } let uploadedTargetArtifacts = try allURLs.compactMap { tempDir -> TargetTuple? in - // All targets that uploaded their artifacts, have it placed in the - // `$(TARGET_TEMP_DIR)/xccache/produced/{{fileKey}}.zip` location. Find all targets that have such a file - - let targetGeneratedArtifactRootDir = tempDir - .appendingPathComponent("xccache") - .appendingPathComponent("produced") - guard try dirScanner.itemType(atPath: targetGeneratedArtifactRootDir.path) == ItemType.dir else { - // given target didn't generate any artifacts (e.g. it is never cached with XCRemoteCache) - return nil - } - - let allFilesProduced = try dirScanner.items(at: targetGeneratedArtifactRootDir) - let allArtifacts = allFilesProduced.filter { $0.pathExtension == "zip" } - guard !allArtifacts.isEmpty else { + let potentialArtifacts = try findTargetPackageZip(tempDir: tempDir) + guard let allArtifacts = potentialArtifacts, !allArtifacts.isEmpty else { // there is no generated *.zip file, so given target didn't create an artifact - it could be // just a helper target (like the target we integrate this plugin with) return nil @@ -78,7 +69,7 @@ class ThinningCreatorPlugin: ArtifactCreatorPlugin { // Find {{fileKey}} based on the .zip file basename guard allArtifacts.count == 1 else { throw ThinningCreatorPluginError.noSingleTargetArtifactsGenerated( - rootDir: targetGeneratedArtifactRootDir + rootDir: tempDir ) } let fileKey = allArtifacts[0].deletingPathExtension().lastPathComponent @@ -96,6 +87,38 @@ class ThinningCreatorPlugin: ArtifactCreatorPlugin { return Dictionary(uniqueKeysWithValues: extraKeysTuples) } + private func findTargetPackageZip(tempDir: URL) throws -> [URL]? { + // Producer mode: + // All targets that uploaded their artifacts, have it placed in the + // `$(TARGET_TEMP_DIR)/xccache/produced/{{fileKey}}.zip` location. Find all targets that have such a file + // ProducerFull mode: + // If a target reused already existing artifact, it still has `$(TARGET_TEMP_DIR)/rc.enabled` marker file + // and the reused zip is placed in: + // `$(TARGET_TEMP_DIR)/xccache/{{fileKey}}.zip` location. + + let targetEnabledMarker = tempDir.appendingPathComponent(modeMarkerPath) + let targetReusedArtifactRootDir = tempDir.appendingPathComponent("xccache") + let targetGeneratedArtifactRootDir = tempDir + .appendingPathComponent("xccache") + .appendingPathComponent("produced") + + let pathToDirWithZipArtifacts: URL + // try the ProducerFull mode path first + if try dirScanner.itemType(atPath: targetEnabledMarker.path) == ItemType.file { + pathToDirWithZipArtifacts = targetReusedArtifactRootDir + } else { + guard try dirScanner.itemType(atPath: targetGeneratedArtifactRootDir.path) == ItemType.dir else { + // given target didn't generate any artifacts (e.g. it is never cached with XCRemoteCache) + return nil + } + pathToDirWithZipArtifacts = targetGeneratedArtifactRootDir + } + + let allFilesProduced = try dirScanner.items(at: pathToDirWithZipArtifacts) + let allArtifacts = allFilesProduced.filter { $0.pathExtension == "zip" } + return allArtifacts + } + func artifactToUpload(main: MainArtifactMeta) throws -> [Artifact] { return [] } diff --git a/Sources/XCRemoteCache/Commands/Postbuild/PostbuildContext.swift b/Sources/XCRemoteCache/Commands/Postbuild/PostbuildContext.swift index 4aaa134..9a87dae 100644 --- a/Sources/XCRemoteCache/Commands/Postbuild/PostbuildContext.swift +++ b/Sources/XCRemoteCache/Commands/Postbuild/PostbuildContext.swift @@ -73,6 +73,7 @@ public struct PostbuildContext { var thinnedTargets: [String] /// Action type: build, indexbuild etc. var action: BuildActionType + let modeMarkerPath: String } extension PostbuildContext { @@ -115,5 +116,6 @@ extension PostbuildContext { let thinFocusedTargetsString: String = env.readEnv(key: "SPT_XCREMOTE_CACHE_THINNED_TARGETS") ?? "" thinnedTargets = thinFocusedTargetsString.split(separator: ",").map(String.init) action = (try? BuildActionType(rawValue: env.readEnv(key: "ACTION"))) ?? .unknown + modeMarkerPath = config.modeMarkerPath } } diff --git a/Sources/XCRemoteCache/Commands/Postbuild/XCPostbuild.swift b/Sources/XCRemoteCache/Commands/Postbuild/XCPostbuild.swift index 92123de..20a045d 100644 --- a/Sources/XCRemoteCache/Commands/Postbuild/XCPostbuild.swift +++ b/Sources/XCRemoteCache/Commands/Postbuild/XCPostbuild.swift @@ -205,9 +205,10 @@ public class XCPostbuild { worker: DispatchGroupParallelizationWorker(qos: .userInitiated) ) consumerPlugins.append(thinningPlugin) - case .producer: + case .producer, .producerFast: let thinningPlugin = ThinningCreatorPlugin( targetTempDir: context.targetTempDir, + modeMarkerPath: context.modeMarkerPath, dirScanner: fileManager ) creatorPlugins.append(thinningPlugin) diff --git a/Sources/XCRemoteCache/Commands/Swiftc/Swiftc.swift b/Sources/XCRemoteCache/Commands/Swiftc/Swiftc.swift index 231e3cc..ef30547 100644 --- a/Sources/XCRemoteCache/Commands/Swiftc/Swiftc.swift +++ b/Sources/XCRemoteCache/Commands/Swiftc/Swiftc.swift @@ -120,9 +120,9 @@ class Swiftc: SwiftcProtocol { let prebuildDiscoveryURL = context.tempDir.appendingPathComponent(context.prebuildDependenciesPath) let prebuildDiscoverWriter = dependenciesWriterFactory(prebuildDiscoveryURL, fileManager) try prebuildDiscoverWriter.write(skipForSha: remoteCommit) - case .consumer, .producer: - // Never skips prebuild phase and fallbacks to the swiftc compilation for: - // 1) Not enabled remote cache or 2) producer + case .consumer, .producer, .producerFast: + // Never skip prebuild phase and fallback to the swiftc compilation for: + // 1) Not enabled remote cache, 2) producer(s) break } return .forceFallback diff --git a/Sources/XCRemoteCache/Commands/Swiftc/SwiftcContext.swift b/Sources/XCRemoteCache/Commands/Swiftc/SwiftcContext.swift index d0f0ac3..80ef0cd 100644 --- a/Sources/XCRemoteCache/Commands/Swiftc/SwiftcContext.swift +++ b/Sources/XCRemoteCache/Commands/Swiftc/SwiftcContext.swift @@ -24,6 +24,8 @@ public struct SwiftcContext { case producer /// Commit sha of the commit to use during remote cache case consumer(commit: RemoteCommitInfo) + /// Remote artifact exists and can be optimistically used in lieu of a local compilation + case producerFast } let objcHeaderOutput: URL @@ -74,6 +76,14 @@ public struct SwiftcContext { mode = .consumer(commit: remoteCommit) case .producer: mode = .producer + case .producerFast: + let remoteCommit = RemoteCommitInfo(try? String(contentsOf: remoteCommitLocation).trim()) + switch remoteCommit { + case .unavailable: + mode = .producer + case .available: + mode = .producerFast + } } invocationHistoryFile = URL(fileURLWithPath: config.compilationHistoryFile, relativeTo: tempDir) } diff --git a/Sources/XCRemoteCache/Commands/Swiftc/SwiftcOrchestrator.swift b/Sources/XCRemoteCache/Commands/Swiftc/SwiftcOrchestrator.swift index 43721c8..3f35aaf 100644 --- a/Sources/XCRemoteCache/Commands/Swiftc/SwiftcOrchestrator.swift +++ b/Sources/XCRemoteCache/Commands/Swiftc/SwiftcOrchestrator.swift @@ -116,6 +116,12 @@ class SwiftcOrchestrator { } case .consumer: fallbackToDefault(command: swiftcCommand) + case .producerFast: + let compileStepResult = try swiftc.mockCompilation() + if case .forceFallback = compileStepResult { + // cannot reuse cached artifact. Build it locally and upload to the server just as for the producer + fallthrough + } case .producer: var swiftcArgs = ProcessInfo().arguments swiftcArgs = try producerFallbackCommandProcessors.reduce(swiftcArgs) { args, processor in diff --git a/Sources/XCRemoteCache/Config/Mode.swift b/Sources/XCRemoteCache/Config/Mode.swift index eb3ffd7..931b2dd 100644 --- a/Sources/XCRemoteCache/Config/Mode.swift +++ b/Sources/XCRemoteCache/Config/Mode.swift @@ -20,4 +20,5 @@ public enum Mode: String, Codable, CaseIterable { case consumer case producer + case producerFast } diff --git a/Sources/XCRemoteCache/Network/RemoteNetworkClientAbstractFactory.swift b/Sources/XCRemoteCache/Network/RemoteNetworkClientAbstractFactory.swift index 1808522..b1d8930 100644 --- a/Sources/XCRemoteCache/Network/RemoteNetworkClientAbstractFactory.swift +++ b/Sources/XCRemoteCache/Network/RemoteNetworkClientAbstractFactory.swift @@ -44,7 +44,7 @@ class RemoteNetworkClientAbstractFactory { return RemoteNetworkClientImpl(networkClient, downloadURLBuilder) } switch mode { - case .producer: + case .producer, .producerFast: let upstreamBuilders = try upstreamStreamURL.map(urlBuilderFactory) return ReplicatedRemotesNetworkClient( networkClient, diff --git a/Tests/XCRemoteCacheTests/Commands/Plugins/Thinning/ThinningCreatorPluginTests.swift b/Tests/XCRemoteCacheTests/Commands/Plugins/Thinning/ThinningCreatorPluginTests.swift index dcc16f2..6f49fea 100644 --- a/Tests/XCRemoteCacheTests/Commands/Plugins/Thinning/ThinningCreatorPluginTests.swift +++ b/Tests/XCRemoteCacheTests/Commands/Plugins/Thinning/ThinningCreatorPluginTests.swift @@ -33,7 +33,10 @@ class ThinningCreatorPluginTests: FileXCTestCase { targetTempDirRoot = workingDir.appendingPathComponent("Root") currentTargetTempDir = targetTempDirRoot.appendingPathComponent("Current.build") try fileManager.spt_createEmptyDir(currentTargetTempDir) - plugin = ThinningCreatorPlugin(targetTempDir: currentTargetTempDir, dirScanner: FileManager.default) + plugin = ThinningCreatorPlugin( + targetTempDir: currentTargetTempDir, + modeMarkerPath: "rc.enabled", + dirScanner: FileManager.default) } func testReturnsEmptyExtraKeysForNoArtifacts() throws { diff --git a/Tests/XCRemoteCacheTests/Commands/PostbuildTests.swift b/Tests/XCRemoteCacheTests/Commands/PostbuildTests.swift index a67b846..5b66b19 100644 --- a/Tests/XCRemoteCacheTests/Commands/PostbuildTests.swift +++ b/Tests/XCRemoteCacheTests/Commands/PostbuildTests.swift @@ -50,7 +50,8 @@ class PostbuildTests: FileXCTestCase { bundleDir: nil, derivedSourcesDir: "", thinnedTargets: [], - action: .build + action: .build, + modeMarkerPath: "" ) private var network = RemoteNetworkClientImpl( NetworkClientFake(fileManager: .default), diff --git a/Tests/XCRemoteCacheTests/Commands/SwiftcOrchestratorTests.swift b/Tests/XCRemoteCacheTests/Commands/SwiftcOrchestratorTests.swift index 8d175aa..b16e2a5 100644 --- a/Tests/XCRemoteCacheTests/Commands/SwiftcOrchestratorTests.swift +++ b/Tests/XCRemoteCacheTests/Commands/SwiftcOrchestratorTests.swift @@ -190,4 +190,45 @@ class SwiftcOrchestratorTests: XCTestCase { XCTAssertNotNil(shellOutSpy.switchedProcess) } + + func testForFailedCompilationMockInProducerFastModeBuildsArtifactObjCHeader() throws { + let swiftc = SwiftcMock(mockingResult: .forceFallback) + let orchestrator = SwiftcOrchestrator( + mode: .producerFast, + swiftc: swiftc, + swiftcCommand: "", + objcHeaderOutput: objcHeaderURL, + moduleOutput: moduleOutputURL, + arch: "archTest", + artifactBuilder: artifactBuilder, + producerFallbackCommandProcessors: [], + invocationStorage: invocationStorage, + shellOut: shellOutSpy + ) + + try orchestrator.run() + + XCTAssertEqual(artifactBuilder.addedObjCHeaders, ["archTest": [objcHeaderURL]]) + } + + func testSuccessedMockInProducerFastModeModeDoesntFillObjCHeader() throws { + let swiftc = SwiftcMock(mockingResult: .success) + let orchestrator = SwiftcOrchestrator( + mode: .producerFast, + swiftc: swiftc, + swiftcCommand: "", + objcHeaderOutput: objcHeaderURL, + moduleOutput: moduleOutputURL, + arch: "arch", + artifactBuilder: artifactBuilder, + producerFallbackCommandProcessors: [], + invocationStorage: invocationStorage, + shellOut: shellOutSpy + ) + + try orchestrator.run() + + XCTAssertEqual(artifactBuilder.addedObjCHeaders, [:]) + } + }