From cf0b27d03caefe052cea71cb544dc17fb9a64129 Mon Sep 17 00:00:00 2001 From: Bartosz Polaczyk Date: Fri, 4 Feb 2022 21:48:43 +0100 Subject: [PATCH] Add integration This reverts commit b2d47760cf6900d11d50455a3e4e61954433c0eb. --- .../Commands/Postbuild/PostbuildContext.swift | 4 ++ .../Commands/Postbuild/XCPostbuild.swift | 14 +++- .../Commands/Prebuild/PrebuildContext.swift | 4 ++ .../Commands/Prebuild/XCPrebuild.swift | 14 +++- .../OverlayDependenciesRemapper.swift | 55 ++++++++++++++++ .../Commands/PostbuildTests.swift | 3 +- .../Commands/PrebuildTests.swift | 15 +++-- .../OverlayDependenciesRemapperTests.swift | 65 +++++++++++++++++++ .../TestDoubles/OverlayReaderFake.swift | 32 +++++++++ 9 files changed, 198 insertions(+), 8 deletions(-) create mode 100644 Sources/XCRemoteCache/Dependencies/OverlayDependenciesRemapper.swift create mode 100644 Tests/XCRemoteCacheTests/Dependencies/OverlayDependenciesRemapperTests.swift create mode 100644 Tests/XCRemoteCacheTests/TestDoubles/OverlayReaderFake.swift diff --git a/Sources/XCRemoteCache/Commands/Postbuild/PostbuildContext.swift b/Sources/XCRemoteCache/Commands/Postbuild/PostbuildContext.swift index 12078fb..40dbdac 100644 --- a/Sources/XCRemoteCache/Commands/Postbuild/PostbuildContext.swift +++ b/Sources/XCRemoteCache/Commands/Postbuild/PostbuildContext.swift @@ -79,6 +79,8 @@ public struct PostbuildContext { /// Action type: build, indexbuild etc. var action: BuildActionType let modeMarkerPath: String + /// location of the json file that define virtual files system overlay (mappings of the virtual location file -> local file path) + let overlayHeadersPath: URL } extension PostbuildContext { @@ -127,5 +129,7 @@ extension PostbuildContext { thinnedTargets = thinFocusedTargetsString.split(separator: ",").map(String.init) action = (try? BuildActionType(rawValue: env.readEnv(key: "ACTION"))) ?? .unknown modeMarkerPath = config.modeMarkerPath + /// The files has yaml extension, even it is in the json format + overlayHeadersPath = targetTempDir.appendingPathComponent("all-product-headers.yaml") } } diff --git a/Sources/XCRemoteCache/Commands/Postbuild/XCPostbuild.swift b/Sources/XCRemoteCache/Commands/Postbuild/XCPostbuild.swift index 0420781..2cc89bb 100644 --- a/Sources/XCRemoteCache/Commands/Postbuild/XCPostbuild.swift +++ b/Sources/XCRemoteCache/Commands/Postbuild/XCPostbuild.swift @@ -66,7 +66,7 @@ public class XCPostbuild { // Initialize dependencies let primaryGitBranch = GitBranch(repoLocation: config.primaryRepo, branch: config.primaryBranch) let gitClient = GitClientImpl(repoRoot: config.repoRoot, primary: primaryGitBranch, shell: shellGetStdout) - let pathRemapper = try StringDependenciesRemapperFactory().build( + let envsRemapper = try StringDependenciesRemapperFactory().build( orderKeys: DependenciesMapping.rewrittenEnvs, envs: env, customMappings: config.outOfBandMappings @@ -145,6 +145,18 @@ public class XCPostbuild { fileDependeciesReaderFactory: fileReaderFactory, dirScanner: fileManager ) + // As the PostbuildContext assumes file format location and filename (`all-product-headers.yaml`) + // do not fail in case of a missing headers overlay file. In the future, all overlay files should be + // captured from the swiftc invocation similarly is stored in the `history.compile` for the consumer mode. + let overlayReader = JsonOverlayReader( + context.overlayHeadersPath, + mode: .bestEffort, + fileReader: fileManager + ) + let overlayRemapper = try OverlayDependenciesRemapper( + overlayReader: overlayReader + ) + let pathRemapper = DependenciesRemapperComposite([overlayRemapper, envsRemapper]) let dependencyProcessor = DependencyProcessorImpl( xcode: context.xcodeDir, product: context.productsDir, diff --git a/Sources/XCRemoteCache/Commands/Prebuild/PrebuildContext.swift b/Sources/XCRemoteCache/Commands/Prebuild/PrebuildContext.swift index 1109a79..97a8265 100644 --- a/Sources/XCRemoteCache/Commands/Prebuild/PrebuildContext.swift +++ b/Sources/XCRemoteCache/Commands/Prebuild/PrebuildContext.swift @@ -43,6 +43,8 @@ public struct PrebuildContext { let targetName: String /// List of all targets to downloaded from the thinning aggregation target var thinnedTargets: [String]? + /// location of the json file that define virtual files system overlay (mappings of the virtual location file -> local file path) + let overlayHeadersPath: URL } extension PrebuildContext { @@ -64,5 +66,7 @@ extension PrebuildContext { self.targetName = targetName let thinFocusedTargetsString: String? = env.readEnv(key: "SPT_XCREMOTE_CACHE_THINNED_TARGETS") thinnedTargets = thinFocusedTargetsString?.split(separator: ",").map(String.init) + /// The files has yaml extension, even it is in the json format + overlayHeadersPath = targetTempDir.appendingPathComponent("all-product-headers.yaml") } } diff --git a/Sources/XCRemoteCache/Commands/Prebuild/XCPrebuild.swift b/Sources/XCRemoteCache/Commands/Prebuild/XCPrebuild.swift index 9eba5d8..9b8140c 100644 --- a/Sources/XCRemoteCache/Commands/Prebuild/XCPrebuild.swift +++ b/Sources/XCRemoteCache/Commands/Prebuild/XCPrebuild.swift @@ -115,11 +115,23 @@ public class XCPrebuild { ) let client: NetworkClient = config.disableHttpCache ? networkClient : cacheNetworkClient let remoteNetworkClient = RemoteNetworkClientImpl(client, urlBuilder) - let pathRemapper = try StringDependenciesRemapperFactory().build( + let envsRemapper = try StringDependenciesRemapperFactory().build( orderKeys: DependenciesMapping.rewrittenEnvs, envs: env, customMappings: config.outOfBandMappings ) + // As the PostbuildContext assumes file format location and filename (`all-product-headers.yaml`) + // do not fail in case of a missing headers overlay file. In the future, all overlay files should be + // captured from the swiftc invocation similarly is stored in the `history.compile` for the consumer mode. + let overlayReader = JsonOverlayReader( + context.overlayHeadersPath, + mode: .bestEffort, + fileReader: fileManager + ) + let overlayRemapper = try OverlayDependenciesRemapper( + overlayReader: overlayReader + ) + let pathRemapper = DependenciesRemapperComposite([overlayRemapper, envsRemapper]) let filesFingerprintGenerator = FingerprintAccumulatorImpl( algorithm: MD5Algorithm(), fileManager: fileManager diff --git a/Sources/XCRemoteCache/Dependencies/OverlayDependenciesRemapper.swift b/Sources/XCRemoteCache/Dependencies/OverlayDependenciesRemapper.swift new file mode 100644 index 0000000..cda9432 --- /dev/null +++ b/Sources/XCRemoteCache/Dependencies/OverlayDependenciesRemapper.swift @@ -0,0 +1,55 @@ +// Copyright (c) 2021 Spotify AB. +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import Foundation + +/// File paths remapper according the virtual file system mappings +/// Warning: this class is not thread safe +class OverlayDependenciesRemapper: DependenciesRemapper { + private var mappings: [OverlayMapping] + + init(overlayReader: OverlayReader) throws { + mappings = try overlayReader.provideMappings() + } + + private func mapPath( + _ path: String, + source: KeyPath, + destination: KeyPath + ) -> String { + guard let mapping = mappings.first(where: { $0[keyPath: source].path == path }) else { + // TODO: support partial mappings, where a directory path can be replaced with some other directory + // no direct mapping found + return path + } + return mapping[keyPath: destination].path + } + + func replace(genericPaths: [String]) -> [String] { + Set(genericPaths.map { + mapPath($0, source: \.virtual, destination: \.local) + }).sorted() + } + + func replace(localPaths: [String]) -> [String] { + Set(localPaths.map { + mapPath($0, source: \.local, destination: \.virtual) + }).sorted() + } +} diff --git a/Tests/XCRemoteCacheTests/Commands/PostbuildTests.swift b/Tests/XCRemoteCacheTests/Commands/PostbuildTests.swift index a35d14a..fd0621d 100644 --- a/Tests/XCRemoteCacheTests/Commands/PostbuildTests.swift +++ b/Tests/XCRemoteCacheTests/Commands/PostbuildTests.swift @@ -51,7 +51,8 @@ class PostbuildTests: FileXCTestCase { derivedSourcesDir: "", thinnedTargets: [], action: .build, - modeMarkerPath: "" + modeMarkerPath: "", + overlayHeadersPath: "" ) private var network = RemoteNetworkClientImpl( NetworkClientFake(fileManager: .default), diff --git a/Tests/XCRemoteCacheTests/Commands/PrebuildTests.swift b/Tests/XCRemoteCacheTests/Commands/PrebuildTests.swift index c6fd4a9..ccdce58 100644 --- a/Tests/XCRemoteCacheTests/Commands/PrebuildTests.swift +++ b/Tests/XCRemoteCacheTests/Commands/PrebuildTests.swift @@ -62,7 +62,8 @@ class PrebuildTests: FileXCTestCase { forceCached: false, compilationHistoryFile: compilationHistory, turnOffRemoteCacheOnFirstTimeout: true, - targetName: "" + targetName: "", + overlayHeadersPath: "" ) contextCached = PrebuildContext( targetTempDir: sampleURL, @@ -74,7 +75,8 @@ class PrebuildTests: FileXCTestCase { forceCached: true, compilationHistoryFile: compilationHistory, turnOffRemoteCacheOnFirstTimeout: true, - targetName: "" + targetName: "", + overlayHeadersPath: "" ) organizer = ArtifactOrganizerFake(artifactRoot: artifactsRoot, unzippedExtension: "unzip") globalCacheSwitcher = InMemoryGlobalCacheSwitcher() @@ -238,7 +240,8 @@ class PrebuildTests: FileXCTestCase { forceCached: false, compilationHistoryFile: compilationHistory, turnOffRemoteCacheOnFirstTimeout: true, - targetName: "" + targetName: "", + overlayHeadersPath: "" ) let prebuild = Prebuild( @@ -268,7 +271,8 @@ class PrebuildTests: FileXCTestCase { forceCached: false, compilationHistoryFile: compilationHistory, turnOffRemoteCacheOnFirstTimeout: true, - targetName: "" + targetName: "", + overlayHeadersPath: "" ) metaContent = try generateMeta(fingerprint: generator.generate(), filekey: "1") let downloadedArtifactPackage = artifactsRoot.appendingPathComponent("1") @@ -330,7 +334,8 @@ class PrebuildTests: FileXCTestCase { forceCached: false, compilationHistoryFile: compilationHistory, turnOffRemoteCacheOnFirstTimeout: false, - targetName: "" + targetName: "", + overlayHeadersPath: "" ) try globalCacheSwitcher.enable(sha: "1") let prebuild = Prebuild( diff --git a/Tests/XCRemoteCacheTests/Dependencies/OverlayDependenciesRemapperTests.swift b/Tests/XCRemoteCacheTests/Dependencies/OverlayDependenciesRemapperTests.swift new file mode 100644 index 0000000..6d707a6 --- /dev/null +++ b/Tests/XCRemoteCacheTests/Dependencies/OverlayDependenciesRemapperTests.swift @@ -0,0 +1,65 @@ +// Copyright (c) 2021 Spotify AB. +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +@testable import XCRemoteCache +import XCTest + +class OverlayDependenciesRemapperTests: XCTestCase { + private let overlayReader = OverlayReaderFake( + mappings: [.init(virtual: "/file.h", local: "/Intermediate/Some/file.h")] + ) + + func testMappingFromLocalToGeneric() throws { + let reader = try OverlayDependenciesRemapper( + overlayReader: overlayReader + ) + + let dependencies = reader.replace(localPaths: ["/Intermediate/Some/file.h"]) + XCTAssertEqual(dependencies, ["/file.h"]) + } + + func testMappingFromGenericToLocal() throws { + let reader = try OverlayDependenciesRemapper( + overlayReader: overlayReader + ) + + let dependencies = reader.replace(genericPaths: ["/file.h"]) + XCTAssertEqual(dependencies, ["/Intermediate/Some/file.h"]) + } + + func testGenericDependenciesAreMerged() throws { + + let reader = try OverlayDependenciesRemapper( + overlayReader: overlayReader + ) + + let dependencies = reader.replace(localPaths: ["/Intermediate/Some/file.h", "/file.h"]) + XCTAssertEqual(dependencies, ["/file.h"]) + } + + func testLocalDependenciesAreMerged() throws { + let reader = try OverlayDependenciesRemapper( + overlayReader: overlayReader + ) + + let dependencies = reader.replace(genericPaths: ["/Intermediate/Some/file.h", "/file.h"]) + XCTAssertEqual(dependencies, ["/Intermediate/Some/file.h"]) + } +} + diff --git a/Tests/XCRemoteCacheTests/TestDoubles/OverlayReaderFake.swift b/Tests/XCRemoteCacheTests/TestDoubles/OverlayReaderFake.swift new file mode 100644 index 0000000..ec716b8 --- /dev/null +++ b/Tests/XCRemoteCacheTests/TestDoubles/OverlayReaderFake.swift @@ -0,0 +1,32 @@ +// Copyright (c) 2021 Spotify AB. +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import Foundation +@testable import XCRemoteCache + +class OverlayReaderFake: OverlayReader { + private let mappings: [OverlayMapping] + init(mappings: [OverlayMapping]) { + self.mappings = mappings + } + + func provideMappings() throws -> [OverlayMapping] { + return mappings + } +}