From e7d1e905cfa47e169e75d91f3b7dd2f5338019a0 Mon Sep 17 00:00:00 2001 From: Bartosz Polaczyk Date: Thu, 10 Feb 2022 20:57:05 +0100 Subject: [PATCH 1/2] Do not throw overlay for the best-effort mode --- .../Dependencies/OverlayReader.swift | 48 +++++++++++-------- .../Dependencies/OverlayReaderTests.swift | 21 +++++++- 2 files changed, 48 insertions(+), 21 deletions(-) diff --git a/Sources/XCRemoteCache/Dependencies/OverlayReader.swift b/Sources/XCRemoteCache/Dependencies/OverlayReader.swift index 266cbde..6d0fdf7 100644 --- a/Sources/XCRemoteCache/Dependencies/OverlayReader.swift +++ b/Sources/XCRemoteCache/Dependencies/OverlayReader.swift @@ -95,31 +95,39 @@ class JsonOverlayReader: OverlayReader { return [] } } - + do { let overlay: Overlay = try jsonDecoder.decode(Overlay.self, from: jsonContent) - let mappings: [OverlayMapping] = try overlay.roots.reduce([]) { prev, root in - switch root.type { - case .directory: - //iterate all contents - let dir = URL(fileURLWithPath: root.name) - let mappings: [OverlayMapping] = try root.contents.map { content in - switch content.type { - case .file: - let virtual = dir.appendingPathComponent(content.name) - let local = URL(fileURLWithPath: content.externalContents) - return .init(virtual: virtual, local: local) - case .directory: - throw JsonOverlayReaderError.unsupportedFormat - } + let mappings: [OverlayMapping] = try overlay.roots.reduce([]) { prev, root in + switch root.type { + case .directory: + //iterate all contents + let dir = URL(fileURLWithPath: root.name) + let mappings: [OverlayMapping] = try root.contents.map { content in + switch content.type { + case .file: + let virtual = dir.appendingPathComponent(content.name) + let local = URL(fileURLWithPath: content.externalContents) + return .init(virtual: virtual, local: local) + case .directory: + throw JsonOverlayReaderError.unsupportedFormat + } + } + return prev + mappings + case .file: + throw JsonOverlayReaderError.unsupportedFormat } - return prev + mappings - case .file: - throw JsonOverlayReaderError.unsupportedFormat + } + return mappings + } catch { + switch mode { + case .strict: + throw error + case .bestEffort: + printWarning("Overlay reader has failed with an error \(error). Best-effort mode - skipping an overlay.") + return [] } } - - return mappings } } diff --git a/Tests/XCRemoteCacheTests/Dependencies/OverlayReaderTests.swift b/Tests/XCRemoteCacheTests/Dependencies/OverlayReaderTests.swift index fbc854a..8cccf99 100644 --- a/Tests/XCRemoteCacheTests/Dependencies/OverlayReaderTests.swift +++ b/Tests/XCRemoteCacheTests/Dependencies/OverlayReaderTests.swift @@ -20,7 +20,7 @@ @testable import XCRemoteCache import XCTest -class JsonOverlayReaderTests: XCTestCase { +class JsonOverlayReaderTests: FileXCTestCase { private static let resourcesSubdirectory = "TestData/Dependencies/JsonOverlayReaderTests" func testParsingWithSuccess() throws { @@ -59,6 +59,25 @@ class JsonOverlayReaderTests: XCTestCase { XCTAssertEqual(mappings, []) } + func testInvalidJsonDoesntThrowForBestEffortMode() throws { + let workingDir = try prepareTempDir() + let file = workingDir.appendingPathExtension("overlay.json") + try fileManager.spt_createEmptyFile(file) + let reader = JsonOverlayReader(file, mode: .bestEffort, fileReader: fileManager) + let mappings = try reader.provideMappings() + + XCTAssertEqual(mappings, []) + } + + func testInvalidJsonThrowsForStrictMode() throws { + let workingDir = try prepareTempDir() + let file = workingDir.appendingPathExtension("overlay.json") + try fileManager.spt_createEmptyFile(file) + let reader = JsonOverlayReader(file, mode: .strict, fileReader: fileManager) + + XCTAssertThrowsError(try reader.provideMappings()) + } + private func pathForTestData(name: String) throws -> URL { return try XCTUnwrap(Bundle.module.url(forResource: name, withExtension: "json", subdirectory: JsonOverlayReaderTests.resourcesSubdirectory)) } From cccae0d9f690417dd3dd245008091b2c2fbefdab Mon Sep 17 00:00:00 2001 From: Bartosz Polaczyk Date: Thu, 10 Feb 2022 21:03:53 +0100 Subject: [PATCH 2/2] Add disable_vfs_overlay feature flag --- README.md | 1 + .../Commands/Postbuild/XCPostbuild.swift | 29 +++++++++++-------- .../Commands/Prebuild/XCPrebuild.swift | 27 ++++++++++------- .../Config/XCRemoteCacheConfig.swift | 5 ++++ 4 files changed, 39 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index db63f14..9b6f2c0 100755 --- a/README.md +++ b/README.md @@ -312,6 +312,7 @@ _Note that for the `producer` mode, the prebuild build phase and `xccc`, `xcld`, | `aws_service` | Service for AWS V4 Signature Authorization. E.g. `storage`. | `""` | ⬜️ | | `out_of_band_mappings` | A dictionary of files path remapping that should be applied to make it absolute path agnostic on a list of dependencies. Useful if a project refers files out of repo root, either compilation files or precompiled dependencies. Keys represent generic replacement and values are substrings that should be replaced. Example: for mapping `["COOL_LIBRARY": "/CoolLibrary"]` `/CoolLibrary/main.swift`will be represented as `$(COOL_LIBRARY)/main.swift`). Warning: remapping order is not-deterministic so avoid remappings with multiple matchings. | `[:]` | ⬜️ | | `disable_certificate_verification` | A Boolean value that opts-in SSL certificate validation is disabled | `false` | ⬜️ | +| `disable_vfs_overlay` | A feature flag to disable virtual file system overlay support (temporary) | `false` | ⬜️ | ## Backend cache server diff --git a/Sources/XCRemoteCache/Commands/Postbuild/XCPostbuild.swift b/Sources/XCRemoteCache/Commands/Postbuild/XCPostbuild.swift index 63a19e9..ab08e4c 100644 --- a/Sources/XCRemoteCache/Commands/Postbuild/XCPostbuild.swift +++ b/Sources/XCRemoteCache/Commands/Postbuild/XCPostbuild.swift @@ -145,18 +145,23 @@ public class XCPostbuild { fileDependeciesReaderFactory: fileReaderFactory, dirScanner: fileManager ) - // As the PostbuildContext assumes file location and filename (`all-product-headers.yaml`) - // do not fail in case of a missing headers overlay file. In the future, all overlay files could be - // captured from the swiftc invocation similarly is stored in the `history.compile` for the consumer mode. - let overlayReader = JsonOverlayReader( - context.overlayHeadersPath, - mode: .bestEffort, - fileReader: fileManager - ) - let overlayRemapper = OverlayDependenciesRemapper( - overlayReader: overlayReader - ) - let pathRemapper = DependenciesRemapperComposite([overlayRemapper, envsRemapper]) + var remappers: [DependenciesRemapper] = [] + if !config.disableVFSOverlay { + // As the PostbuildContext assumes file location and filename (`all-product-headers.yaml`) + // do not fail in case of a missing headers overlay file. In the future, all overlay files could be + // captured from the swiftc invocation similarly is stored in the `history.compile` for the consumer mode. + let overlayReader = JsonOverlayReader( + context.overlayHeadersPath, + mode: .bestEffort, + fileReader: fileManager + ) + let overlayRemapper = OverlayDependenciesRemapper( + overlayReader: overlayReader + ) + remappers.append(overlayRemapper) + } + remappers.append(envsRemapper) + let pathRemapper = DependenciesRemapperComposite(remappers) let dependencyProcessor = DependencyProcessorImpl( xcode: context.xcodeDir, product: context.productsDir, diff --git a/Sources/XCRemoteCache/Commands/Prebuild/XCPrebuild.swift b/Sources/XCRemoteCache/Commands/Prebuild/XCPrebuild.swift index f5c70e1..7fc43b8 100644 --- a/Sources/XCRemoteCache/Commands/Prebuild/XCPrebuild.swift +++ b/Sources/XCRemoteCache/Commands/Prebuild/XCPrebuild.swift @@ -120,17 +120,22 @@ public class XCPrebuild { envs: env, customMappings: config.outOfBandMappings ) - // As PrebuildContext assumes file location and its filename (`all-product-headers.yaml`) - // do not fail in case of a missing headers overlay file. - let overlayReader = JsonOverlayReader( - context.overlayHeadersPath, - mode: .bestEffort, - fileReader: fileManager - ) - let overlayRemapper = OverlayDependenciesRemapper( - overlayReader: overlayReader - ) - let pathRemapper = DependenciesRemapperComposite([overlayRemapper, envsRemapper]) + var remappers: [DependenciesRemapper] = [] + if !config.disableVFSOverlay { + // As PrebuildContext assumes file location and its filename (`all-product-headers.yaml`) + // do not fail in case of a missing headers overlay file. + let overlayReader = JsonOverlayReader( + context.overlayHeadersPath, + mode: .bestEffort, + fileReader: fileManager + ) + let overlayRemapper = OverlayDependenciesRemapper( + overlayReader: overlayReader + ) + remappers.append(overlayRemapper) + } + remappers.append(envsRemapper) + let pathRemapper = DependenciesRemapperComposite(remappers) let filesFingerprintGenerator = FingerprintAccumulatorImpl( algorithm: MD5Algorithm(), fileManager: fileManager diff --git a/Sources/XCRemoteCache/Config/XCRemoteCacheConfig.swift b/Sources/XCRemoteCache/Config/XCRemoteCacheConfig.swift index 776b4f0..04b7191 100644 --- a/Sources/XCRemoteCache/Config/XCRemoteCacheConfig.swift +++ b/Sources/XCRemoteCache/Config/XCRemoteCacheConfig.swift @@ -131,6 +131,8 @@ public struct XCRemoteCacheConfig: Encodable { var outOfBandMappings: [String: String] = [:] /// If true, SSL certificate validation is disabled var disableCertificateVerification: Bool = false + /// A feature flag to disable virtual file system overlay support (temporary) + var disableVFSOverlay: Bool = false } extension XCRemoteCacheConfig { @@ -183,6 +185,7 @@ extension XCRemoteCacheConfig { merge.AWSService = scheme.AWSService ?? AWSService merge.outOfBandMappings = scheme.outOfBandMappings ?? outOfBandMappings merge.disableCertificateVerification = scheme.disableCertificateVerification ?? disableCertificateVerification + merge.disableVFSOverlay = scheme.disableVFSOverlay ?? disableVFSOverlay return merge } @@ -244,6 +247,7 @@ struct ConfigFileScheme: Decodable { let AWSService: String? let outOfBandMappings: [String: String]? let disableCertificateVerification: Bool? + let disableVFSOverlay: Bool? // Yams library doesn't support encoding strategy, see https://github.com/jpsim/Yams/issues/84 enum CodingKeys: String, CodingKey { @@ -288,6 +292,7 @@ struct ConfigFileScheme: Decodable { case AWSService = "aws_service" case outOfBandMappings = "out_of_band_mappings" case disableCertificateVerification = "disable_certificate_verification" + case disableVFSOverlay = "disable_vfs_overlay" } }