Merge pull request #82 from polac24/20220210-best-effort-overlay

Do not fail prebuild/postbuild for invalid vfs overlay

README.md

@@ -312,6 +312,7 @@ _Note that for the `producer` mode, the prebuild build phase and `xccc`, `xcld`,
 | `aws_service` | Service for AWS V4 Signature Authorization. E.g. `storage`. | `""` | ⬜️ |
 | `out_of_band_mappings` | A dictionary of files path remapping that should be applied to make it absolute path agnostic on a list of dependencies. Useful if a project refers files out of repo root, either compilation files or precompiled dependencies. Keys represent generic replacement and values are substrings that should be replaced. Example: for mapping `["COOL_LIBRARY": "/CoolLibrary"]` `/CoolLibrary/main.swift`will be represented as `$(COOL_LIBRARY)/main.swift`). Warning: remapping order is not-deterministic so avoid remappings with multiple matchings. | `[:]` | ⬜️ |
 | `disable_certificate_verification` | A Boolean value that opts-in SSL certificate validation is disabled | `false` | ⬜️ |
+| `disable_vfs_overlay` | A feature flag to disable virtual file system overlay support (temporary) | `false` | ⬜️ |
 
 ## Backend cache server
 

Sources/XCRemoteCache/Commands/Postbuild/XCPostbuild.swift

@@ -145,18 +145,23 @@ public class XCPostbuild {
                 fileDependeciesReaderFactory: fileReaderFactory,
                 dirScanner: fileManager
             )
-            // As the PostbuildContext assumes file location and filename (`all-product-headers.yaml`)
-            // do not fail in case of a missing headers overlay file. In the future, all overlay files could be
-            // captured from the swiftc invocation similarly is stored in the `history.compile` for the consumer mode.
-            let overlayReader = JsonOverlayReader(
-                context.overlayHeadersPath,
-                mode: .bestEffort,
-                fileReader: fileManager
-            )
-            let overlayRemapper = OverlayDependenciesRemapper(
-                overlayReader: overlayReader
-            )
-            let pathRemapper = DependenciesRemapperComposite([overlayRemapper, envsRemapper])
+            var remappers: [DependenciesRemapper] = []
+            if !config.disableVFSOverlay {
+                // As the PostbuildContext assumes file location and filename (`all-product-headers.yaml`)
+                // do not fail in case of a missing headers overlay file. In the future, all overlay files could be
+                // captured from the swiftc invocation similarly is stored in the `history.compile` for the consumer mode.
+                let overlayReader = JsonOverlayReader(
+                    context.overlayHeadersPath,
+                    mode: .bestEffort,
+                    fileReader: fileManager
+                )
+                let overlayRemapper = OverlayDependenciesRemapper(
+                    overlayReader: overlayReader
+                )
+                remappers.append(overlayRemapper)
+            }
+            remappers.append(envsRemapper)
+            let pathRemapper = DependenciesRemapperComposite(remappers)
             let dependencyProcessor = DependencyProcessorImpl(
                 xcode: context.xcodeDir,
                 product: context.productsDir,

Sources/XCRemoteCache/Commands/Prebuild/XCPrebuild.swift

@@ -120,17 +120,22 @@ public class XCPrebuild {
                 envs: env,
                 customMappings: config.outOfBandMappings
             )
-            // As PrebuildContext assumes file location and its filename (`all-product-headers.yaml`)
-            // do not fail in case of a missing headers overlay file. 
-            let overlayReader = JsonOverlayReader(
-                context.overlayHeadersPath,
-                mode: .bestEffort,
-                fileReader: fileManager
-            )
-            let overlayRemapper = OverlayDependenciesRemapper(
-                overlayReader: overlayReader
-            )
-            let pathRemapper = DependenciesRemapperComposite([overlayRemapper, envsRemapper])
+            var remappers: [DependenciesRemapper] = []
+            if !config.disableVFSOverlay {
+                // As PrebuildContext assumes file location and its filename (`all-product-headers.yaml`)
+                // do not fail in case of a missing headers overlay file.
+                let overlayReader = JsonOverlayReader(
+                    context.overlayHeadersPath,
+                    mode: .bestEffort,
+                    fileReader: fileManager
+                )
+                let overlayRemapper = OverlayDependenciesRemapper(
+                    overlayReader: overlayReader
+                )
+                remappers.append(overlayRemapper)
+            }
+            remappers.append(envsRemapper)
+            let pathRemapper = DependenciesRemapperComposite(remappers)
             let filesFingerprintGenerator = FingerprintAccumulatorImpl(
                 algorithm: MD5Algorithm(),
                 fileManager: fileManager

Sources/XCRemoteCache/Config/XCRemoteCacheConfig.swift

@@ -131,6 +131,8 @@ public struct XCRemoteCacheConfig: Encodable {
     var outOfBandMappings: [String: String] = [:]
     /// If true, SSL certificate validation is disabled
     var disableCertificateVerification: Bool = false
+    /// A feature flag to disable virtual file system overlay support (temporary)
+    var disableVFSOverlay: Bool = false
 }
 
 extension XCRemoteCacheConfig {
@@ -183,6 +185,7 @@ extension XCRemoteCacheConfig {
         merge.AWSService = scheme.AWSService ?? AWSService
         merge.outOfBandMappings = scheme.outOfBandMappings ?? outOfBandMappings
         merge.disableCertificateVerification = scheme.disableCertificateVerification ?? disableCertificateVerification
+        merge.disableVFSOverlay = scheme.disableVFSOverlay ?? disableVFSOverlay
         return merge
     }
 
@@ -244,6 +247,7 @@ struct ConfigFileScheme: Decodable {
     let AWSService: String?
     let outOfBandMappings: [String: String]?
     let disableCertificateVerification: Bool?
+    let disableVFSOverlay: Bool?
 
     // Yams library doesn't support encoding strategy, see https://github.com/jpsim/Yams/issues/84
     enum CodingKeys: String, CodingKey {
@@ -288,6 +292,7 @@ struct ConfigFileScheme: Decodable {
         case AWSService = "aws_service"
         case outOfBandMappings = "out_of_band_mappings"
         case disableCertificateVerification = "disable_certificate_verification"
+        case disableVFSOverlay = "disable_vfs_overlay"
     }
 }
 

Sources/XCRemoteCache/Dependencies/OverlayReader.swift

@@ -95,31 +95,39 @@ class JsonOverlayReader: OverlayReader {
                 return []
             }
         }
-
+        do {
         let overlay: Overlay = try jsonDecoder.decode(Overlay.self, from: jsonContent)
-        let mappings: [OverlayMapping] = try overlay.roots.reduce([]) { prev, root in
-            switch root.type {
-            case .directory:
-                //iterate all contents
-                let dir = URL(fileURLWithPath: root.name)
-                let mappings: [OverlayMapping] = try root.contents.map { content in
-                    switch content.type {
-                    case .file:
-                        let virtual = dir.appendingPathComponent(content.name)
-                        let local = URL(fileURLWithPath: content.externalContents)
-                        return .init(virtual: virtual, local: local)
-                    case .directory:
-                        throw JsonOverlayReaderError.unsupportedFormat
-                    }
+            let mappings: [OverlayMapping] = try overlay.roots.reduce([]) { prev, root in
+                switch root.type {
+                case .directory:
+                    //iterate all contents
+                    let dir = URL(fileURLWithPath: root.name)
+                    let mappings: [OverlayMapping] = try root.contents.map { content in
+                        switch content.type {
+                        case .file:
+                            let virtual = dir.appendingPathComponent(content.name)
+                            let local = URL(fileURLWithPath: content.externalContents)
+                            return .init(virtual: virtual, local: local)
+                        case .directory:
+                            throw JsonOverlayReaderError.unsupportedFormat
+                        }
 
+                    }
+                    return prev + mappings
+                case .file:
+                    throw JsonOverlayReaderError.unsupportedFormat
                 }
-                return prev + mappings
-            case .file:
-                throw JsonOverlayReaderError.unsupportedFormat
+            }
+            return mappings
+        } catch {
+            switch mode {
+            case .strict:
+                throw error
+            case .bestEffort:
+                printWarning("Overlay reader has failed with an error \(error). Best-effort mode - skipping an overlay.")
+                return []
             }
         }
-
-        return mappings
     }
 
 }

Tests/XCRemoteCacheTests/Dependencies/OverlayReaderTests.swift

@@ -20,7 +20,7 @@
 @testable import XCRemoteCache
 import XCTest
 
-class JsonOverlayReaderTests: XCTestCase {
+class JsonOverlayReaderTests: FileXCTestCase {
     private static let resourcesSubdirectory = "TestData/Dependencies/JsonOverlayReaderTests"
 
     func testParsingWithSuccess() throws {
@@ -59,6 +59,25 @@ class JsonOverlayReaderTests: XCTestCase {
         XCTAssertEqual(mappings, [])
     }
 
+    func testInvalidJsonDoesntThrowForBestEffortMode() throws {
+        let workingDir = try prepareTempDir()
+        let file = workingDir.appendingPathExtension("overlay.json")
+        try fileManager.spt_createEmptyFile(file)
+        let reader = JsonOverlayReader(file, mode: .bestEffort, fileReader: fileManager)
+        let mappings = try reader.provideMappings()
+
+        XCTAssertEqual(mappings, [])
+    }
+
+    func testInvalidJsonThrowsForStrictMode() throws {
+        let workingDir = try prepareTempDir()
+        let file = workingDir.appendingPathExtension("overlay.json")
+        try fileManager.spt_createEmptyFile(file)
+        let reader = JsonOverlayReader(file, mode: .strict, fileReader: fileManager)
+
+        XCTAssertThrowsError(try reader.provideMappings())
+    }
+
     private func pathForTestData(name: String) throws -> URL {
         return try XCTUnwrap(Bundle.module.url(forResource: name, withExtension: "json", subdirectory: JsonOverlayReaderTests.resourcesSubdirectory))
     }