diff --git a/README.md b/README.md index ebe3d32..55d42e8 100755 --- a/README.md +++ b/README.md @@ -316,6 +316,7 @@ _Note that for the `producer` mode, the prebuild build phase and `xccc`, `xcld`, | `disable_certificate_verification` | A Boolean value that opts-in SSL certificate validation is disabled | `false` | ⬜️ | | `disable_vfs_overlay` | A feature flag to disable virtual file system overlay support (temporary) | `false` | ⬜️ | | `custom_rewrite_envs` | A list of extra ENVs that should be used as placeholders in the dependency list. ENV rewrite process is optimistic - does nothing if an ENV is not defined in the pre/postbuild process. | `[]` | ⬜️ | +| `irrelevant_dependencies_paths` | Regexes of files that should not be included in a list of dependencies. Warning! Add entries here with caution - excluding dependencies that are relevant might lead to a target overcaching. The regex can match either partially or fully the filepath, e.g. `\\.modulemap$` will exclude all `.modulemap` files. | `[]` | ⬜️ | ## Backend cache server diff --git a/Sources/XCRemoteCache/Commands/Postbuild/PostbuildContext.swift b/Sources/XCRemoteCache/Commands/Postbuild/PostbuildContext.swift index abd9f0e..3c88bfa 100644 --- a/Sources/XCRemoteCache/Commands/Postbuild/PostbuildContext.swift +++ b/Sources/XCRemoteCache/Commands/Postbuild/PostbuildContext.swift @@ -83,6 +83,8 @@ public struct PostbuildContext { /// location of the json file that define virtual files system overlay /// (mappings of the virtual location file -> local file path) let overlayHeadersPath: URL + /// Regexes of files that should not be included in the dependency list + let irrelevantDependenciesPaths: [String] } extension PostbuildContext { @@ -135,5 +137,6 @@ extension PostbuildContext { modeMarkerPath = config.modeMarkerPath /// Note: The file has yaml extension, even it is in the json format overlayHeadersPath = targetTempDir.appendingPathComponent("all-product-headers.yaml") + irrelevantDependenciesPaths = config.irrelevantDependenciesPaths } } diff --git a/Sources/XCRemoteCache/Commands/Postbuild/XCPostbuild.swift b/Sources/XCRemoteCache/Commands/Postbuild/XCPostbuild.swift index 0656af3..ea40dc5 100644 --- a/Sources/XCRemoteCache/Commands/Postbuild/XCPostbuild.swift +++ b/Sources/XCRemoteCache/Commands/Postbuild/XCPostbuild.swift @@ -172,7 +172,8 @@ public class XCPostbuild { source: context.srcRoot, intermediate: context.targetTempDir, derivedFiles: context.derivedFilesDir, - bundle: context.bundleDir + bundle: context.bundleDir, + skippedRegexes: context.irrelevantDependenciesPaths ) // Override fingerprints for all produced '.swiftmodule' files let fingerprintOverrideManager = FingerprintOverrideManagerImpl( diff --git a/Sources/XCRemoteCache/Config/XCRemoteCacheConfig.swift b/Sources/XCRemoteCache/Config/XCRemoteCacheConfig.swift index be5bc27..80ab797 100644 --- a/Sources/XCRemoteCache/Config/XCRemoteCacheConfig.swift +++ b/Sources/XCRemoteCache/Config/XCRemoteCacheConfig.swift @@ -138,6 +138,11 @@ public struct XCRemoteCacheConfig: Encodable { /// A list of extra ENVs that should be used as placeholders in the dependency list /// ENV rewrite process is optimistic - does nothing if an ENV is not defined in the pre/postbuild process var customRewriteEnvs: [String] = [] + /// Regexes of files that should not be included in a list of dependencies. Warning! Add entries here + /// with caution - excluding dependencies that are relevant might lead to a target overcaching + /// Note: The regex can match either partially or fully the filepath, e.g. `\\.modulemap$` will exclude + /// all `.modulemap` files + var irrelevantDependenciesPaths: [String] = [] } extension XCRemoteCacheConfig { @@ -193,6 +198,7 @@ extension XCRemoteCacheConfig { merge.disableCertificateVerification = scheme.disableCertificateVerification ?? disableCertificateVerification merge.disableVFSOverlay = scheme.disableVFSOverlay ?? disableVFSOverlay merge.customRewriteEnvs = scheme.customRewriteEnvs ?? customRewriteEnvs + merge.irrelevantDependenciesPaths = scheme.irrelevantDependenciesPaths ?? irrelevantDependenciesPaths return merge } @@ -257,6 +263,7 @@ struct ConfigFileScheme: Decodable { let disableCertificateVerification: Bool? let disableVFSOverlay: Bool? let customRewriteEnvs: [String]? + let irrelevantDependenciesPaths: [String]? // Yams library doesn't support encoding strategy, see https://github.com/jpsim/Yams/issues/84 enum CodingKeys: String, CodingKey { @@ -304,6 +311,7 @@ struct ConfigFileScheme: Decodable { case disableCertificateVerification = "disable_certificate_verification" case disableVFSOverlay = "disable_vfs_overlay" case customRewriteEnvs = "custom_rewrite_envs" + case irrelevantDependenciesPaths = "irrelevant_dependencies_paths" } } diff --git a/Sources/XCRemoteCache/Dependencies/DependencyProcessor.swift b/Sources/XCRemoteCache/Dependencies/DependencyProcessor.swift index 4308f17..fc20375 100644 --- a/Sources/XCRemoteCache/Dependencies/DependencyProcessor.swift +++ b/Sources/XCRemoteCache/Dependencies/DependencyProcessor.swift @@ -30,6 +30,8 @@ public struct Dependency: Equatable { case derivedFile // Product of the target itself case ownProduct + // User-excluded path + case userExcluded case unknown } @@ -58,14 +60,16 @@ class DependencyProcessorImpl: DependencyProcessor { private let intermediatePath: String private let derivedFilesPath: String private let bundlePath: String? + private let skippedRegexes: [String] - init(xcode: URL, product: URL, source: URL, intermediate: URL, derivedFiles: URL, bundle: URL?) { + init(xcode: URL, product: URL, source: URL, intermediate: URL, derivedFiles: URL, bundle: URL?, skippedRegexes: [String]) { xcodePath = xcode.path.dirPath() productPath = product.path.dirPath() sourcePath = source.path.dirPath() intermediatePath = intermediate.path.dirPath() derivedFilesPath = derivedFiles.path.dirPath() bundlePath = bundle?.path.dirPath() + self.skippedRegexes = skippedRegexes } func process(_ files: [URL]) -> [Dependency] { @@ -76,7 +80,9 @@ class DependencyProcessorImpl: DependencyProcessor { private func classify(_ files: [URL]) -> [Dependency] { return files.map { file -> Dependency in let filePath = file.resolvingSymlinksInPath().path - if filePath.hasPrefix(xcodePath) { + if skippedRegexes.contains(where: { filePath.range(of: $0, options: .regularExpression) != nil }) { + return Dependency(url: file, type: .userExcluded) + } else if filePath.hasPrefix(xcodePath) { return Dependency(url: file, type: .xcode) } else if filePath.hasPrefix(intermediatePath) { return Dependency(url: file, type: .intermediate) @@ -114,7 +120,10 @@ class DependencyProcessorImpl: DependencyProcessor { // because in case of a hit, these will be taken from the artifact // - Customized DERIVED_FILE_DIR may change a directory of // derived files, which by default is under `*/Interemediates` - let irrelevantDependenciesType: [Dependency.Kind] = [.xcode, .intermediate, .ownProduct, .derivedFile] + // - User-specified (in .rcinfo) files to exclude + let irrelevantDependenciesType: [Dependency.Kind] = [ + .xcode, .intermediate, .ownProduct, .derivedFile, .userExcluded, + ] return !irrelevantDependenciesType.contains(dependency.type) } } diff --git a/Tests/XCRemoteCacheTests/Commands/PostbuildTests.swift b/Tests/XCRemoteCacheTests/Commands/PostbuildTests.swift index cf95ba9..33cb631 100644 --- a/Tests/XCRemoteCacheTests/Commands/PostbuildTests.swift +++ b/Tests/XCRemoteCacheTests/Commands/PostbuildTests.swift @@ -54,7 +54,8 @@ class PostbuildTests: FileXCTestCase { thinnedTargets: [], action: .build, modeMarkerPath: "", - overlayHeadersPath: "" + overlayHeadersPath: "", + irrelevantDependenciesPaths: [] ) private var network = RemoteNetworkClientImpl( NetworkClientFake(fileManager: .default), @@ -80,7 +81,8 @@ class PostbuildTests: FileXCTestCase { source: "/Source", intermediate: "/Intermediate", derivedFiles: "/DerivedFiles", - bundle: nil + bundle: nil, + skippedRegexes: [] ) private var overrideManager = FingerprintOverrideManagerImpl( overridingFileExtensions: ["swiftmodule"], diff --git a/Tests/XCRemoteCacheTests/Dependencies/DependencyProcessorImplTests.swift b/Tests/XCRemoteCacheTests/Dependencies/DependencyProcessorImplTests.swift index a0ef4e2..6ae2daa 100644 --- a/Tests/XCRemoteCacheTests/Dependencies/DependencyProcessorImplTests.swift +++ b/Tests/XCRemoteCacheTests/Dependencies/DependencyProcessorImplTests.swift @@ -28,10 +28,11 @@ class DependencyProcessorImplTests: FileXCTestCase { source: "/Source", intermediate: "/Intermediate", derivedFiles: "/DerivedFiles", - bundle: "/Bundle" + bundle: "/Bundle", + skippedRegexes: [] ) - func testIntermediateFileIsSkippedForProductAndSourceSubdirectory() { + func testIntermediateFileIsskippedRegexesForProductAndSourceSubdirectory() { let intermediateFile: URL = "/Intermediate/some" let processor = DependencyProcessorImpl( xcode: "/Xcode", @@ -39,7 +40,8 @@ class DependencyProcessorImplTests: FileXCTestCase { source: "/", intermediate: "/Intermediate", derivedFiles: "/DerivedFiles", - bundle: nil + bundle: nil, + skippedRegexes: [] ) XCTAssertEqual( @@ -48,7 +50,7 @@ class DependencyProcessorImplTests: FileXCTestCase { ) } - func testBundleFileIsSkippedForProductAndSourceSubdirectory() { + func testBundleFileIsskippedRegexesForProductAndSourceSubdirectory() { let bundleFile: URL = "/Bundle/some" let processor = DependencyProcessorImpl( xcode: "/Xcode", @@ -56,7 +58,8 @@ class DependencyProcessorImplTests: FileXCTestCase { source: "/", intermediate: "/Intermediate", derivedFiles: "/DerivedFiles", - bundle: "/Bundle" + bundle: "/Bundle", + skippedRegexes: [] ) XCTAssertEqual( @@ -150,7 +153,8 @@ class DependencyProcessorImplTests: FileXCTestCase { source: "/Source", intermediate: intermediateDirReal, derivedFiles: "/DerivedFiles", - bundle: "/Bundle" + bundle: "/Bundle", + skippedRegexes: [] ) let intermediateFileSymlink = createSymlink( @@ -179,7 +183,8 @@ class DependencyProcessorImplTests: FileXCTestCase { source: sourceDirReal, intermediate: "/Intermediate", derivedFiles: "/DerivedFiles", - bundle: "/Bundle" + bundle: "/Bundle", + skippedRegexes: [] ) let sourceFileSymlink = createSymlink( @@ -219,7 +224,8 @@ class DependencyProcessorImplTests: FileXCTestCase { source: "/", intermediate: "/Intermediate", derivedFiles: "/DerivedFiles", - bundle: nil + bundle: nil, + skippedRegexes: [] ) XCTAssertEqual( @@ -227,4 +233,58 @@ class DependencyProcessorImplTests: FileXCTestCase { [] ) } + + func testSkippsFilesWithFullMatch() { + let source: URL = "/someFile.m" + let processor = DependencyProcessorImpl( + xcode: "/Xcode", + product: "/", + source: "/", + intermediate: "/Intermediate", + derivedFiles: "/DerivedFiles", + bundle: nil, + skippedRegexes: ["/someFile\\.m"] + ) + + XCTAssertEqual( + processor.process([source]), + [] + ) + } + + func testSkippsFilesWithPartialMatch() { + let derivedModulemap: URL = "/module.modulemap" + let processor = DependencyProcessorImpl( + xcode: "/Xcode", + product: "/product", + source: "/", + intermediate: "/Intermediate", + derivedFiles: "/DerivedFiles", + bundle: nil, + skippedRegexes: ["\\.modulemap$"] + ) + + XCTAssertEqual( + processor.process([derivedModulemap]), + [] + ) + } + + func testDoesntSkipFileIfInvalidRegex() { + let source: URL = "/someFile.m" + let processor = DependencyProcessorImpl( + xcode: "/Xcode", + product: "/product", + source: "/", + intermediate: "/Intermediate", + derivedFiles: "/DerivedFiles", + bundle: nil, + skippedRegexes: ["\\"] + ) + + XCTAssertEqual( + processor.process([source]), + [.init(url: source, type: .source)] + ) + } }