Merge pull request #130 from polac24/irrelevant-dependency

Add irrelevant dependencies property

README.md

@@ -316,6 +316,7 @@ _Note that for the `producer` mode, the prebuild build phase and `xccc`, `xcld`,
 | `disable_certificate_verification` | A Boolean value that opts-in SSL certificate validation is disabled | `false` | ⬜️ |
 | `disable_vfs_overlay` | A feature flag to disable virtual file system overlay support (temporary) | `false` | ⬜️ |
 | `custom_rewrite_envs` | A list of extra ENVs that should be used as placeholders in the dependency list. ENV rewrite process is optimistic - does nothing if an ENV is not defined in the pre/postbuild process. | `[]` | ⬜️ |
+| `irrelevant_dependencies_paths` | Regexes of files that should not be included in a list of dependencies. Warning! Add entries here with caution - excluding dependencies that are relevant might lead to a target overcaching. The regex can match either partially or fully the filepath, e.g. `\\.modulemap$` will exclude all `.modulemap` files. | `[]` | ⬜️ |
 
 ## Backend cache server
 

Sources/XCRemoteCache/Commands/Postbuild/PostbuildContext.swift

@@ -83,6 +83,8 @@ public struct PostbuildContext {
     /// location of the json file that define virtual files system overlay
     /// (mappings of the virtual location file -> local file path)
     let overlayHeadersPath: URL
+    /// Regexes of files that should not be included in the dependency list
+    let irrelevantDependenciesPaths: [String]
 }
 
 extension PostbuildContext {
@@ -135,5 +137,6 @@ extension PostbuildContext {
         modeMarkerPath = config.modeMarkerPath
         /// Note: The file has yaml extension, even it is in the json format
         overlayHeadersPath = targetTempDir.appendingPathComponent("all-product-headers.yaml")
+        irrelevantDependenciesPaths = config.irrelevantDependenciesPaths
     }
 }

Sources/XCRemoteCache/Commands/Postbuild/XCPostbuild.swift

@@ -172,7 +172,8 @@ public class XCPostbuild {
                 source: context.srcRoot,
                 intermediate: context.targetTempDir,
                 derivedFiles: context.derivedFilesDir,
-                bundle: context.bundleDir
+                bundle: context.bundleDir,
+                skippedRegexes: context.irrelevantDependenciesPaths
             )
             // Override fingerprints for all produced '.swiftmodule' files
             let fingerprintOverrideManager = FingerprintOverrideManagerImpl(

Sources/XCRemoteCache/Config/XCRemoteCacheConfig.swift

@@ -138,6 +138,11 @@ public struct XCRemoteCacheConfig: Encodable {
     /// A list of extra ENVs that should be used as placeholders in the dependency list
     /// ENV rewrite process is optimistic - does nothing if an ENV is not defined in the pre/postbuild process
     var customRewriteEnvs: [String] = []
+    /// Regexes of files that should not be included in a list of dependencies. Warning! Add entries here
+    /// with caution - excluding dependencies that are relevant might lead to a target overcaching
+    /// Note: The regex can match either partially or fully the filepath, e.g. `\\.modulemap$` will exclude
+    /// all `.modulemap` files
+    var irrelevantDependenciesPaths: [String] = []
 }
 
 extension XCRemoteCacheConfig {
@@ -193,6 +198,7 @@ extension XCRemoteCacheConfig {
         merge.disableCertificateVerification = scheme.disableCertificateVerification ?? disableCertificateVerification
         merge.disableVFSOverlay = scheme.disableVFSOverlay ?? disableVFSOverlay
         merge.customRewriteEnvs = scheme.customRewriteEnvs ?? customRewriteEnvs
+        merge.irrelevantDependenciesPaths = scheme.irrelevantDependenciesPaths ?? irrelevantDependenciesPaths
         return merge
     }
 
@@ -257,6 +263,7 @@ struct ConfigFileScheme: Decodable {
     let disableCertificateVerification: Bool?
     let disableVFSOverlay: Bool?
     let customRewriteEnvs: [String]?
+    let irrelevantDependenciesPaths: [String]?
 
     // Yams library doesn't support encoding strategy, see https://github.com/jpsim/Yams/issues/84
     enum CodingKeys: String, CodingKey {
@@ -304,6 +311,7 @@ struct ConfigFileScheme: Decodable {
         case disableCertificateVerification = "disable_certificate_verification"
         case disableVFSOverlay = "disable_vfs_overlay"
         case customRewriteEnvs = "custom_rewrite_envs"
+        case irrelevantDependenciesPaths = "irrelevant_dependencies_paths"
     }
 }
 

Sources/XCRemoteCache/Dependencies/DependencyProcessor.swift

@@ -30,6 +30,8 @@ public struct Dependency: Equatable {
         case derivedFile
         // Product of the target itself
         case ownProduct
+        // User-excluded path
+        case userExcluded
         case unknown
     }
 
@@ -58,14 +60,16 @@ class DependencyProcessorImpl: DependencyProcessor {
     private let intermediatePath: String
     private let derivedFilesPath: String
     private let bundlePath: String?
+    private let skippedRegexes: [String]
 
-    init(xcode: URL, product: URL, source: URL, intermediate: URL, derivedFiles: URL, bundle: URL?) {
+    init(xcode: URL, product: URL, source: URL, intermediate: URL, derivedFiles: URL, bundle: URL?, skippedRegexes: [String]) {
         xcodePath = xcode.path.dirPath()
         productPath = product.path.dirPath()
         sourcePath = source.path.dirPath()
         intermediatePath = intermediate.path.dirPath()
         derivedFilesPath = derivedFiles.path.dirPath()
         bundlePath = bundle?.path.dirPath()
+        self.skippedRegexes = skippedRegexes
     }
 
     func process(_ files: [URL]) -> [Dependency] {
@@ -76,7 +80,9 @@ class DependencyProcessorImpl: DependencyProcessor {
     private func classify(_ files: [URL]) -> [Dependency] {
         return files.map { file -> Dependency in
             let filePath = file.resolvingSymlinksInPath().path
-            if filePath.hasPrefix(xcodePath) {
+            if skippedRegexes.contains(where: { filePath.range(of: $0, options: .regularExpression) != nil }) {
+                return Dependency(url: file, type: .userExcluded)
+            } else if filePath.hasPrefix(xcodePath) {
                 return Dependency(url: file, type: .xcode)
             } else if filePath.hasPrefix(intermediatePath) {
                 return Dependency(url: file, type: .intermediate)
@@ -114,7 +120,10 @@ class DependencyProcessorImpl: DependencyProcessor {
         //   because in case of a hit, these will be taken from the artifact
         // - Customized DERIVED_FILE_DIR may change a directory of
         //   derived files, which by default is under `*/Interemediates`
-        let irrelevantDependenciesType: [Dependency.Kind] = [.xcode, .intermediate, .ownProduct, .derivedFile]
+        // - User-specified (in .rcinfo) files to exclude
+        let irrelevantDependenciesType: [Dependency.Kind] = [
+            .xcode, .intermediate, .ownProduct, .derivedFile, .userExcluded,
+        ]
         return !irrelevantDependenciesType.contains(dependency.type)
     }
 }

Tests/XCRemoteCacheTests/Commands/PostbuildTests.swift

@@ -54,7 +54,8 @@ class PostbuildTests: FileXCTestCase {
         thinnedTargets: [],
         action: .build,
         modeMarkerPath: "",
-        overlayHeadersPath: ""
+        overlayHeadersPath: "",
+        irrelevantDependenciesPaths: []
     )
     private var network = RemoteNetworkClientImpl(
         NetworkClientFake(fileManager: .default),
@@ -80,7 +81,8 @@ class PostbuildTests: FileXCTestCase {
         source: "/Source",
         intermediate: "/Intermediate",
         derivedFiles: "/DerivedFiles",
-        bundle: nil
+        bundle: nil,
+        skippedRegexes: []
     )
     private var overrideManager = FingerprintOverrideManagerImpl(
         overridingFileExtensions: ["swiftmodule"],

Tests/XCRemoteCacheTests/Dependencies/DependencyProcessorImplTests.swift

@@ -28,10 +28,11 @@ class DependencyProcessorImplTests: FileXCTestCase {
         source: "/Source",
         intermediate: "/Intermediate",
         derivedFiles: "/DerivedFiles",
-        bundle: "/Bundle"
+        bundle: "/Bundle",
+        skippedRegexes: []
     )
 
-    func testIntermediateFileIsSkippedForProductAndSourceSubdirectory() {
+    func testIntermediateFileIsskippedRegexesForProductAndSourceSubdirectory() {
         let intermediateFile: URL = "/Intermediate/some"
         let processor = DependencyProcessorImpl(
             xcode: "/Xcode",
@@ -39,7 +40,8 @@ class DependencyProcessorImplTests: FileXCTestCase {
             source: "/",
             intermediate: "/Intermediate",
             derivedFiles: "/DerivedFiles",
-            bundle: nil
+            bundle: nil,
+            skippedRegexes: []
         )
 
         XCTAssertEqual(
@@ -48,7 +50,7 @@ class DependencyProcessorImplTests: FileXCTestCase {
         )
     }
 
-    func testBundleFileIsSkippedForProductAndSourceSubdirectory() {
+    func testBundleFileIsskippedRegexesForProductAndSourceSubdirectory() {
         let bundleFile: URL = "/Bundle/some"
         let processor = DependencyProcessorImpl(
             xcode: "/Xcode",
@@ -56,7 +58,8 @@ class DependencyProcessorImplTests: FileXCTestCase {
             source: "/",
             intermediate: "/Intermediate",
             derivedFiles: "/DerivedFiles",
-            bundle: "/Bundle"
+            bundle: "/Bundle",
+            skippedRegexes: []
         )
 
         XCTAssertEqual(
@@ -150,7 +153,8 @@ class DependencyProcessorImplTests: FileXCTestCase {
             source: "/Source",
             intermediate: intermediateDirReal,
             derivedFiles: "/DerivedFiles",
-            bundle: "/Bundle"
+            bundle: "/Bundle",
+            skippedRegexes: []
         )
 
         let intermediateFileSymlink = createSymlink(
@@ -179,7 +183,8 @@ class DependencyProcessorImplTests: FileXCTestCase {
             source: sourceDirReal,
             intermediate: "/Intermediate",
             derivedFiles: "/DerivedFiles",
-            bundle: "/Bundle"
+            bundle: "/Bundle",
+            skippedRegexes: []
         )
 
         let sourceFileSymlink = createSymlink(
@@ -219,7 +224,8 @@ class DependencyProcessorImplTests: FileXCTestCase {
             source: "/",
             intermediate: "/Intermediate",
             derivedFiles: "/DerivedFiles",
-            bundle: nil
+            bundle: nil,
+            skippedRegexes: []
         )
 
         XCTAssertEqual(
@@ -227,4 +233,58 @@ class DependencyProcessorImplTests: FileXCTestCase {
             []
         )
     }
+
+    func testSkippsFilesWithFullMatch() {
+        let source: URL = "/someFile.m"
+        let processor = DependencyProcessorImpl(
+            xcode: "/Xcode",
+            product: "/",
+            source: "/",
+            intermediate: "/Intermediate",
+            derivedFiles: "/DerivedFiles",
+            bundle: nil,
+            skippedRegexes: ["/someFile\\.m"]
+        )
+
+        XCTAssertEqual(
+            processor.process([source]),
+            []
+        )
+    }
+
+    func testSkippsFilesWithPartialMatch() {
+        let derivedModulemap: URL = "/module.modulemap"
+        let processor = DependencyProcessorImpl(
+            xcode: "/Xcode",
+            product: "/product",
+            source: "/",
+            intermediate: "/Intermediate",
+            derivedFiles: "/DerivedFiles",
+            bundle: nil,
+            skippedRegexes: ["\\.modulemap$"]
+        )
+
+        XCTAssertEqual(
+            processor.process([derivedModulemap]),
+            []
+        )
+    }
+
+    func testDoesntSkipFileIfInvalidRegex() {
+        let source: URL = "/someFile.m"
+        let processor = DependencyProcessorImpl(
+            xcode: "/Xcode",
+            product: "/product",
+            source: "/",
+            intermediate: "/Intermediate",
+            derivedFiles: "/DerivedFiles",
+            bundle: nil,
+            skippedRegexes: ["\\"]
+        )
+
+        XCTAssertEqual(
+            processor.process([source]),
+            [.init(url: source, type: .source)]
+        )
+    }
 }