Merge pull request #122 from polac24/20220418-derived

Fix building target dependencies list with custom DERIVED_FILE_DIR

.github/workflows/ci.yaml

@@ -9,6 +9,8 @@ jobs:
       - uses: actions/checkout@v1
       - name: SwiftLint
         uses: norio-nomura/action-swiftlint@3.1.0
+        with:
+          args: --strict
 
   macOS:
     runs-on: macOS-latest

.swiftlint.yml

@@ -5,7 +5,6 @@ disabled_rules:
   - superfluous_disable_command # Disabled since we disable some rules pre-emptively to avoid issues in the future
   - todo                        # Temporarily disabled. We have too many right now hiding real issues :(
   - nesting                     # Does not make sense anymore since Swift 4 uses nested `CodingKeys` enums for example
-  - trailing_dot_in_comments    # Triggers warnings for generated file headers
 
 opt_in_rules:
   - anyobject_protocol
@@ -64,6 +63,7 @@ excluded:
   - docs/
   - fastlane/
   - DerivedData/
+  - e2eTests/XCRemoteCacheSample/Pods
 
 attributes:
   always_on_same_line:
@@ -88,6 +88,25 @@ file_header:
                      \/\/  Created by .*? on .*\.
                      \/\/  Copyright © \d{4} .*\. All rights reserved\.
                      \/\/
+  required_pattern: |
+                     \/\/ Copyright \(c\) \d{4} Spotify AB\.
+                     \/\/
+                     \/\/ Licensed to the Apache Software Foundation \(ASF\) under one
+                     \/\/ or more contributor license agreements\.  See the NOTICE file
+                     \/\/ distributed with this work for additional information
+                     \/\/ regarding copyright ownership\.  The ASF licenses this file
+                     \/\/ to you under the Apache License, Version 2.0 \(the
+                     \/\/ "License"\); you may not use this file except in compliance
+                     \/\/ with the License\.  You may obtain a copy of the License at
+                     \/\/
+                     \/\/ http:\/\/www.apache.org\/licenses\/LICENSE-2\.0
+                     \/\/
+                     \/\/ Unless required by applicable law or agreed to in writing,
+                     \/\/ software distributed under the License is distributed on an
+                     \/\/ \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+                     \/\/ KIND, either express or implied\.  See the License for the
+                     \/\/ specific language governing permissions and limitations
+                     \/\/ under the License\.
 force_cast: warning
 force_try: warning
 implicit_getter: warning
@@ -124,6 +143,6 @@ custom_rules:
     severity: warning
   trailing_dot_in_comments:
     name: "Trailing dot in comments"
-    regex: '^[ ]*///?[^\n]*\.\n'
+    regex: '^(?!\/\/\ Copyright\ \(c\)\ \d{4}\ Spotify AB\.|\/\/\ under\ the\ License\.)[ ]*///?[^\n]*\.\n'
     message: "There shouldn't be trailing dot in comments"
     severity: warning

Package.resolved

@@ -3,16 +3,16 @@
     "pins": [
       {
         "package": "AEXML",
-        "repositoryURL": "https://github.com/tadija/AEXML",
+        "repositoryURL": "https://github.com/tadija/AEXML.git",
         "state": {
           "branch": null,
-          "revision": "8623e73b193386909566a9ca20203e33a09af142",
-          "version": "4.5.0"
+          "revision": "38f7d00b23ecd891e1ee656fa6aeebd6ba04ecc3",
+          "version": "4.6.1"
         }
       },
       {
         "package": "PathKit",
-        "repositoryURL": "https://github.com/kylef/PathKit",
+        "repositoryURL": "https://github.com/kylef/PathKit.git",
         "state": {
           "branch": null,
           "revision": "73f8e9dca9b7a3078cb79128217dc8f2e585a511",
@@ -42,8 +42,8 @@
         "repositoryURL": "https://github.com/tuist/XcodeProj.git",
         "state": {
           "branch": null,
-          "revision": "0b18c3e7a10c241323397a80cb445051f4494971",
-          "version": "8.0.0"
+          "revision": "c75c3acc25460195cfd203a04dde165395bf00e0",
+          "version": "8.7.1"
         }
       },
       {
@@ -51,8 +51,8 @@
         "repositoryURL": "https://github.com/jpsim/Yams.git",
         "state": {
           "branch": null,
-          "revision": "53741ba55ecca5c7149d8c9f810913ec80845c69",
-          "version": "3.0.0"
+          "revision": "00c403debcd0a007b854bb35e598466207a2d58c",
+          "version": "5.0.0"
         }
       },
       {
@@ -60,8 +60,8 @@
         "repositoryURL": "https://github.com/marmelroy/Zip.git",
         "state": {
           "branch": null,
-          "revision": "80b1c3005ee25b4c7ce46c4029ac3347e8d5e37e",
-          "version": "2.0.0"
+          "revision": "67fa55813b9e7b3b9acee9c0ae501def28746d76",
+          "version": "2.1.2"
         }
       }
     ]

Package.swift

@@ -1,4 +1,5 @@
 // swift-tools-version:5.3
+// swiftlint:disable:previous file_header
 // The swift-tools-version declares the minimum version of Swift required to build this package
 
 import PackageDescription
@@ -12,10 +13,10 @@ let package = Package(
         .executable(name: "xcprebuild", targets: ["xcprebuild"]),
     ],
     dependencies: [
-        .package(url: "https://github.com/marmelroy/Zip.git", from: "2.0.0"),
-        .package(url: "https://github.com/jpsim/Yams.git", from: "3.0.0"),
+        .package(url: "https://github.com/marmelroy/Zip.git", from: "2.1.2"),
+        .package(url: "https://github.com/jpsim/Yams.git", from: "5.0.0"),
         .package(url: "https://github.com/apple/swift-argument-parser", from: "0.0.1"),
-        .package(url: "https://github.com/tuist/XcodeProj.git", from: "8.0.0"),
+        .package(url: "https://github.com/tuist/XcodeProj.git", from: "8.7.1"),
     ],
     targets: [
         .target(

README.md

@@ -288,7 +288,7 @@ _Note that for the `producer` mode, the prebuild build phase and `xccc`, `xcld`,
 | `cache_commit_history` | Number of historical git commits to look for cache artifacts | `10` | ⬜️ |
 | `source_root` | Source root of the Xcode project | `""` | ⬜️ |
 | `fingerprint_override_extension` | Fingerprint override extension (sample override `Module.swiftmodule/x86_64.swiftmodule.md5`) | `md5` | ⬜️ |
-| `extra_configuration_file` | Configuration file that overrides project configuration | `user.rcinfo` | ⬜️ |
+| `extra_configuration_file` | Configuration file that overrides project configuration (this property can be overriden multiple times in different files to chain extra configuration files) | `user.rcinfo` | ⬜️ |
 | `publishing_sha` | Custom commit sha to publish artifact (producer only) | `nil` | ⬜️ |
 | `artifact_maximum_age` | Maximum age in days HTTP response should be locally cached before being evicted | `30` | ⬜️ |
 | `custom_fingerprint_envs` | Extra ENV keys that should be convoluted into the environment fingerprint | `[]` | ⬜️ |
@@ -308,11 +308,13 @@ _Note that for the `producer` mode, the prebuild build phase and `xccc`, `xcld`,
 | `prettify_meta_files` | A Boolean value that opts-in pretty JSON formatting for meta files | `false` | ⬜️ |
 | `aws_secret_key` | Secret key for AWS V4 Signature Authorization. If this is set to a non-empty String - an Authentication Header will be added based on this and the other `aws_*` parameters.| `""` | ⬜️ |
 | `aws_access_key` | Access key for AWS V4 Signature Authorization. | `""` | ⬜️ |
+| `aws_security_token` | Temporary security token provided by the AWS Security Token Service. | `nil` | ⬜️ |
 | `aws_region` | Region for AWS V4 Signature Authorization. E.g. `eu`.  | `""` | ⬜️ |
 | `aws_service` | Service for AWS V4 Signature Authorization. E.g. `storage`. | `""` | ⬜️ |
 | `out_of_band_mappings` | A dictionary of files path remapping that should be applied to make it absolute path agnostic on a list of dependencies. Useful if a project refers files out of repo root, either compilation files or precompiled dependencies. Keys represent generic replacement and values are substrings that should be replaced. Example: for mapping `["COOL_LIBRARY": "/CoolLibrary"]` `/CoolLibrary/main.swift`will be represented as `$(COOL_LIBRARY)/main.swift`). Warning: remapping order is not-deterministic so avoid remappings with multiple matchings. | `[:]` | ⬜️ |
 | `disable_certificate_verification` | A Boolean value that opts-in SSL certificate validation is disabled | `false` | ⬜️ |
 | `disable_vfs_overlay` | A feature flag to disable virtual file system overlay support (temporary) | `false` | ⬜️ |
+| `custom_rewrite_envs` | A list of extra ENVs that should be used as placeholders in the dependency list. ENV rewrite process is optimistic - does nothing if an ENV is not defined in the pre/postbuild process. | `[]` | ⬜️ |
 
 ## Backend cache server
 
@@ -350,6 +352,8 @@ XCRemoteCache supports Amazon S3 and Google Cloud Storage buckets to be used as
 
 To set it up use the configuration parameters `aws_secret_key`, `aws_access_key`, `aws_region`, and `aws_service` in the `.rcinfo` file. Specify the URL to the bucket in cache-addresses field in the same file.
 
+XCRemoteCache also supports [AWS Temporary Access Keys](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#temporary-access-keys). Use additional `aws_security_token` parameter combined with `aws_secret_key`, `aws_access_key` to set it up. [This page](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/configuring-instance-metadata-service.html) describes how to receive a security token. 
+
 Example
 ```yaml
 ...
@@ -403,6 +407,7 @@ Note: This setup is not recommended and may not be supported in future XCRemoteC
 * Recommended: multi-targets Xcode project
 * Recommended: do not use fast-forward PR strategy (use merge or squash instead)
 * Recommended: avoid `DWARF with dSYM File` "Debug Information Format" build setting. Use `DWARF` instead
+* Recommended: avoid having a symbolic link in the source root (e.g. placing a project in `/tmp`) 
 
 ## Limitations
 

Sources/XCRemoteCache/Commands/Libtool/XCLibtoolCreateUniversalBinary.swift

@@ -52,7 +52,7 @@ class XCLibtoolCreateUniversalBinary: XCLibtoolLogic {
         let config: XCRemoteCacheConfig
         do {
             let srcRoot: URL = URL(fileURLWithPath: fileManager.currentDirectoryPath)
-            config = try XCRemoteCacheConfigReader(srcRootPath: srcRoot.path, fileManager: fileManager)
+            config = try XCRemoteCacheConfigReader(srcRootPath: srcRoot.path, fileReader: fileManager)
                 .readConfiguration()
         } catch {
             errorLog("Libtool initialization failed with error: \(error). Fallbacking to libtool")

Sources/XCRemoteCache/Commands/Plugins/Thinning/ThinningCreatorPlugin.swift

@@ -38,7 +38,7 @@ class ThinningCreatorPlugin: ArtifactCreatorPlugin {
 
     /// Default Initializer
     /// - Parameter targetTempDir: Location of current target-specific temp dir (TARGET_TEMP_DIR)
-    /// - Parameter modeMarkerPath: path of maker file that informs if a given target can reuse remote artifacts.
+    /// - Parameter modeMarkerPath: path of maker file that informs if a given target can reuse remote artifacts
     /// - Parameter dirScanner: scanner to access disk and read files and directories hierarchy
     init(targetTempDir: URL, modeMarkerPath: String, dirScanner: DirScanner) {
         self.targetTempDir = targetTempDir
@@ -94,7 +94,7 @@ class ThinningCreatorPlugin: ArtifactCreatorPlugin {
         // ProducerFast mode:
         // If a target reused already existing artifact, it still has `$(TARGET_TEMP_DIR)/rc.enabled` marker file
         // and the reused zip is placed in:
-        // `$(TARGET_TEMP_DIR)/xccache/{{fileKey}}.zip` location.
+        // `$(TARGET_TEMP_DIR)/xccache/{{fileKey}}.zip` location
 
         let targetEnabledMarker = tempDir.appendingPathComponent(modeMarkerPath)
         let targetReusedArtifactRootDir = tempDir.appendingPathComponent("xccache")

Sources/XCRemoteCache/Commands/Plugins/Thinning/ThinningPrebuildPlugin.swift

@@ -25,7 +25,7 @@ class ThinningConsumerPlugin {
 
     deinit {
         // initialised but never run plugin suggests that standard target fallbacks to the local development
-        // and DerivedData still misses build artifacts.
+        // and DerivedData still misses build artifacts
         guard wasRun else {
             let errorMessage = """
             \(type(of: self)) plugin has never been run, thinning cannot be supported. Verify you \

Sources/XCRemoteCache/Commands/Postbuild/PostbuildContext.swift

@@ -40,6 +40,7 @@ public struct PostbuildContext {
     var mode: Mode
     var targetName: String
     var targetTempDir: URL
+    var derivedFilesDir: URL
     /// Location where all compilation outputs (.o) are placed
     var compilationTempDir: URL
     var configuration: String
@@ -76,19 +77,22 @@ public struct PostbuildContext {
     let derivedSourcesDir: URL
     /// List of all targets to downloaded from the thinning aggregation target
     var thinnedTargets: [String]
-    /// Action type: build, indexbuild etc.
+    /// Action type: build, indexbuild etc
     var action: BuildActionType
     let modeMarkerPath: String
-    /// location of the json file that define virtual files system overlay (mappings of the virtual location file -> local file path)
+    /// location of the json file that define virtual files system overlay
+    /// (mappings of the virtual location file -> local file path)
     let overlayHeadersPath: URL
 }
 
 extension PostbuildContext {
+    // swiftlint:disable:next function_body_length
     init(_ config: XCRemoteCacheConfig, env: [String: String]) throws {
         mode = config.mode
         let targetNameValue: String = try env.readEnv(key: "TARGET_NAME")
         targetName = targetNameValue
         targetTempDir = try env.readEnv(key: "TARGET_TEMP_DIR")
+        derivedFilesDir = try env.readEnv(key: "DERIVED_FILE_DIR")
         let archs: [String] = try env.readEnv(key: "ARCHS").split(separator: " ").map(String.init)
         guard let firstArch = archs.first, !firstArch.isEmpty else {
             throw PostbuildContextError.missingArchitecture
@@ -119,7 +123,7 @@ extension PostbuildContext {
         dSYMPath = try env.readEnv(key: "DWARF_DSYM_FOLDER_PATH")
             .appendingPathComponent(env.readEnv(key: "DWARF_DSYM_FILE_NAME"))
         builtProductsDir = try env.readEnv(key: "BUILT_PRODUCTS_DIR")
-        if let contentsFolderPath = env.readEnv(key: "CONTENTS_FOLDER_PATH") {
+        if let contentsFolderPath: String = env.readEnv(key: "CONTENTS_FOLDER_PATH") {
             bundleDir = productsDir.appendingPathComponent(contentsFolderPath)
         } else {
             bundleDir = nil

Sources/XCRemoteCache/Commands/Postbuild/XCPostbuild.swift

@@ -19,6 +19,7 @@
 
 import Foundation
 
+// swiftlint:disable type_body_length
 /// Checks current mode from a configuration and based on that:
 /// * triggers build completion
 /// * triggers uploading artifacts to the server for a 'producer' mode
@@ -33,8 +34,9 @@ public class XCPostbuild {
         let context: PostbuildContext
         let cacheHitLogger: CacheHitLogger
         do {
-            config = try XCRemoteCacheConfigReader(env: env, fileManager: fileManager).readConfiguration()
+            config = try XCRemoteCacheConfigReader(env: env, fileReader: fileManager).readConfiguration()
             context = try PostbuildContext(config, env: env)
+            updateProcessTag(context.targetName)
             let counterFactory: FileStatsCoordinator.CountersFactory = { file, count in
                 ExclusiveFileCounter(ExclusiveFile(file, mode: .override), countersCount: count)
             }
@@ -66,8 +68,8 @@ public class XCPostbuild {
             // Initialize dependencies
             let primaryGitBranch = GitBranch(repoLocation: config.primaryRepo, branch: config.primaryBranch)
             let gitClient = GitClientImpl(repoRoot: config.repoRoot, primary: primaryGitBranch, shell: shellGetStdout)
-            let envsRemapper = try StringDependenciesRemapperFactory().build(
-                orderKeys: DependenciesMapping.rewrittenEnvs,
+            let envsRemapper = try PathDependenciesRemapperFactory().build(
+                orderKeys: DependenciesMapping.rewrittenEnvs + config.customRewriteEnvs,
                 envs: env,
                 customMappings: config.outOfBandMappings
             )
@@ -112,6 +114,7 @@ public class XCPostbuild {
                 awsV4Signature = AWSV4Signature(
                     secretKey: config.AWSSecretKey,
                     accessKey: config.AWSAccessKey,
+                    securityToken: config.AWSSecurityToken,
                     region: config.AWSRegion,
                     service: config.AWSService,
                     date: Date(timeIntervalSinceNow: 0)
@@ -149,7 +152,8 @@ public class XCPostbuild {
             if !config.disableVFSOverlay {
                 // As the PostbuildContext assumes file location and filename (`all-product-headers.yaml`)
                 // do not fail in case of a missing headers overlay file. In the future, all overlay files could be
-                // captured from the swiftc invocation similarly is stored in the `history.compile` for the consumer mode.
+                // captured from the swiftc invocation similarly is stored in the `history.compile`
+                // for the consumer mode
                 let overlayReader = JsonOverlayReader(
                     context.overlayHeadersPath,
                     mode: .bestEffort,
@@ -167,6 +171,7 @@ public class XCPostbuild {
                 product: context.productsDir,
                 source: context.srcRoot,
                 intermediate: context.targetTempDir,
+                derivedFiles: context.derivedFilesDir,
                 bundle: context.bundleDir
             )
             // Override fingerprints for all produced '.swiftmodule' files
@@ -318,3 +323,4 @@ public class XCPostbuild {
         }
     }
 }
+// swiftlint:enable type_body_length

Sources/XCRemoteCache/Commands/Prebuild/Prebuild.swift

@@ -63,7 +63,9 @@ class Prebuild {
         do {
             let metaData = try networkClient.fetch(.meta(commit: commit))
             let meta = try metaReader.read(data: metaData)
-            let localDependencies = try remapper.replace(genericPaths: meta.dependencies).map(URL.init(fileURLWithPath:))
+            let localDependencies = try remapper.replace(
+                genericPaths: meta.dependencies
+            ).map(URL.init(fileURLWithPath:))
             let localFingerprint = try generateFingerprint(for: localDependencies)
             if localFingerprint.raw != meta.rawFingerprint {
                 if context.forceCached {

Sources/XCRemoteCache/Commands/Prebuild/PrebuildContext.swift

@@ -43,7 +43,8 @@ public struct PrebuildContext {
     let targetName: String
     /// List of all targets to downloaded from the thinning aggregation target
     var thinnedTargets: [String]?
-    /// location of the json file that define virtual files system overlay (mappings of the virtual location file -> local file path)
+    /// location of the json file that define virtual files system overlay
+    /// (mappings of the virtual location file -> local file path)
     let overlayHeadersPath: URL
 }
 

Sources/XCRemoteCache/Commands/Prebuild/XCPrebuild.swift

@@ -29,8 +29,9 @@ public class XCPrebuild {
         let config: XCRemoteCacheConfig
         let context: PrebuildContext
         do {
-            config = try XCRemoteCacheConfigReader(env: env, fileManager: fileManager).readConfiguration()
+            config = try XCRemoteCacheConfigReader(env: env, fileReader: fileManager).readConfiguration()
             context = try PrebuildContext(config, env: env)
+            updateProcessTag(context.targetName)
         } catch {
             // Fatal error:
             exit(1, "FATAL: Prebuild initialization failed with error: \(error)")
@@ -95,6 +96,7 @@ public class XCPrebuild {
                 awsV4Signature = AWSV4Signature(
                     secretKey: config.AWSSecretKey,
                     accessKey: config.AWSAccessKey,
+                    securityToken: config.AWSSecurityToken,
                     region: config.AWSRegion,
                     service: config.AWSService,
                     date: Date(timeIntervalSinceNow: 0)
@@ -115,15 +117,15 @@ public class XCPrebuild {
             )
             let client: NetworkClient = config.disableHttpCache ? networkClient : cacheNetworkClient
             let remoteNetworkClient = RemoteNetworkClientImpl(client, urlBuilder)
-            let envsRemapper = try StringDependenciesRemapperFactory().build(
-                orderKeys: DependenciesMapping.rewrittenEnvs,
+            let envsRemapper = try PathDependenciesRemapperFactory().build(
+                orderKeys: DependenciesMapping.rewrittenEnvs + config.customRewriteEnvs,
                 envs: env,
                 customMappings: config.outOfBandMappings
             )
             var remappers: [DependenciesRemapper] = []
             if !config.disableVFSOverlay {
                 // As PrebuildContext assumes file location and its filename (`all-product-headers.yaml`)
-                // do not fail in case of a missing headers overlay file.
+                // do not fail in case of a missing headers overlay file
                 let overlayReader = JsonOverlayReader(
                     context.overlayHeadersPath,
                     mode: .bestEffort,

Sources/XCRemoteCache/Commands/Prepare/CCWrapperBuilder.swift

@@ -72,7 +72,17 @@ class TemplateBasedCCWrapperBuilder: CCWrapperBuilder {
         )
         infoLog("ClangWrapperBuilder compiles file at \(compilationFile).")
         // -O3: optimize for faster execution
-        let args = [clangCommand, "-arch", "arm64", "-arch", "x86_64", "-O3", compilationFile.path, "-o", destination.path]
+        let args = [
+            clangCommand,
+            "-arch",
+            "arm64",
+            "-arch",
+            "x86_64",
+            "-O3",
+            compilationFile.path,
+            "-o",
+            destination.path,
+        ]
         let compilationOutput = try shell("xcrun", args, URL(fileURLWithPath: "").path, nil)
         infoLog("Clang compilation output: \(compilationOutput)")
     }
@@ -385,6 +395,8 @@ class TemplateBasedCCWrapperBuilder: CCWrapperBuilder {
             const char *dependency_arg_name = "-MF";
             const char *output_arg_name = "-o";
             const char *serialize_diagnostics_arg_name = "--serialize-diagnostics";
+            const char *language_mode_arg_name = "-x";
+            const char *precompile_header_arg_value = "objective-c-header";
             const char *clang_cmd = "\(clangCommand)";
             const char *markerFile = "\(markerFilename)";
             const char *compilationHistoryFile = "\(compilationHistoryFilename)";
@@ -399,6 +411,7 @@ class TemplateBasedCCWrapperBuilder: CCWrapperBuilder {
             const char *output_file= NULL;
             const char *input_file = NULL;
             const char *diagnostics_file = NULL;
+            const char *language_mode = NULL;
 
             for (int i = 1; i < argc; i++) {
                 if (strcmp(argv[i], dependency_arg_name) == 0 && i < (argc - 1) ) {
@@ -419,6 +432,12 @@ class TemplateBasedCCWrapperBuilder: CCWrapperBuilder {
                     i += 1;
                     clang_args[i] = argv[i];
                     diagnostics_file = argv[i];
+                } if (strcmp(argv[i], language_mode_arg_name) == 0 && i < (argc - 1) ) {
+                    // called with "-x path" pattern and not the last argument
+                    clang_args[i] = argv[i];
+                    i += 1;
+                    clang_args[i] = argv[i];
+                    language_mode = argv[i];
                 } else if (
                     isSuffixed(argv[i],".m") ||
                     isSuffixed(argv[i],".mm") ||
@@ -426,10 +445,13 @@ class TemplateBasedCCWrapperBuilder: CCWrapperBuilder {
                     isSuffixed(argv[i],".cc") ||
                     isSuffixed(argv[i],".cpp") ||
                     isSuffixed(argv[i],".c++") ||
-                    isSuffixed(argv[i],".cxx")
+                    isSuffixed(argv[i],".cxx") ||
+                    isSuffixed(argv[i],".S") ||
+                    isSuffixed(argv[i],".s")
                 ) {
                     // a full list of extensions is taken from https://clang.llvm.org/docs/ClangFormatStyleOptions.html
                     // support for .m,.mm,.c,.cc,.cpp,.c++,.cxx input files
+                    // .s and .S are assembly files
                     clang_args[i] = argv[i];
                     input_file = argv[i];
                 } else {
@@ -438,6 +460,14 @@ class TemplateBasedCCWrapperBuilder: CCWrapperBuilder {
                 }
             }
 
+           // null-terminating the args array needed for local compilation fallback
+           clang_args[argc] = NULL;
+
+           // Verify mode. Even a target is cached, pch mode is not supported. Fallback to the local compilation
+           if (language_mode != NULL && strcmp(language_mode, precompile_header_arg_value) == 0) {
+               return execvp(clang_cmd, (char *const*) clang_args);
+           }
+
            // Verify all input arguments
            if (dependency_file == NULL) {
                fprintf(stderr, "error: missing %s input\\n", dependency_arg_name);
@@ -508,8 +538,6 @@ class TemplateBasedCCWrapperBuilder: CCWrapperBuilder {
                }
            }
 
-           // null-terminating the args array
-           clang_args[argc] = NULL;
            #pragma GCC diagnostic push
            #pragma GCC diagnostic ignored "-Wincompatible-pointer-types-discards-qualifiers"
            /// execvp takes $PATH to consideration

Sources/XCRemoteCache/Commands/Prepare/Integrate/BuildSettingsIntegrateAppender.swift

@@ -33,10 +33,12 @@ protocol BuildSettingsIntegrateAppender {
 class XcodeProjBuildSettingsIntegrateAppender: BuildSettingsIntegrateAppender {
     private let mode: Mode
     private let repoRoot: URL
+    private let fakeSrcRoot: URL
 
-    init(mode: Mode, repoRoot: URL) {
+    init(mode: Mode, repoRoot: URL, fakeSrcRoot: URL) {
         self.mode = mode
         self.repoRoot = repoRoot
+        self.fakeSrcRoot = fakeSrcRoot
     }
 
     func appendToBuildSettings(buildSettings: BuildSettings, wrappers: XCRCBinariesPaths) -> BuildSettings {
@@ -61,8 +63,11 @@ class XcodeProjBuildSettingsIntegrateAppender: BuildSettingsIntegrateAppender {
         result["OTHER_SWIFT_FLAGS"] = swiftFlags.settingValue
         result["OTHER_CFLAGS"] = clangFlags.settingValue
 
-        result["XCRC_FAKE_SRCROOT"] = "/\(String(repeating: "x", count: 10))"
-        result["XCRC_PLATFORM_PREFERRED_ARCH"] = "$(LINK_FILE_LIST_$(CURRENT_VARIANT)_$(PLATFORM_PREFERRED_ARCH):dir:standardizepath:file:default=arm64)"
+        result["XCRC_FAKE_SRCROOT"] = "\(fakeSrcRoot.path)"
+        result["XCRC_PLATFORM_PREFERRED_ARCH"] =
+        """
+        $(LINK_FILE_LIST_$(CURRENT_VARIANT)_$(PLATFORM_PREFERRED_ARCH):dir:standardizepath:file:default=arm64)
+        """
         return result
     }
 }

Sources/XCRemoteCache/Commands/Prepare/Integrate/IncludeOracle.swift

@@ -35,7 +35,7 @@ struct IncludeExcludeOracle: IncludeOracle {
 
 
     func shouldInclude(identifier: OracleIdentifierType) -> Bool {
-        // exclude array has precedence.
+        // exclude array has precedence
         if excludes.contains(identifier) {
             return false
         }

Sources/XCRemoteCache/Commands/Prepare/Integrate/XCIntegrate.swift

@@ -74,7 +74,7 @@ public class XCIntegrate {
             let binariesDir = commandURL.deletingLastPathComponent()
 
             let srcRoot: URL = URL(fileURLWithPath: projectPath).deletingLastPathComponent()
-            let config = try XCRemoteCacheConfigReader(srcRootPath: srcRoot.path, fileManager: fileManager)
+            let config = try XCRemoteCacheConfigReader(srcRootPath: srcRoot.path, fileReader: fileManager)
                 .readConfiguration()
 
             let context = try IntegrateContext(
@@ -97,7 +97,8 @@ public class XCIntegrate {
             )
             let buildSettingsAppender = XcodeProjBuildSettingsIntegrateAppender(
                 mode: context.mode,
-                repoRoot: context.repoRoot
+                repoRoot: context.repoRoot,
+                fakeSrcRoot: context.fakeSrcRoot
             )
             let lldbPatcher: LLDBInitPatcher
             switch lldbMode {

Sources/XCRemoteCache/Commands/Prepare/XCConfig.swift

@@ -32,7 +32,7 @@ public class XCConfig {
         let fileManager = FileManager.default
         let config: XCRemoteCacheConfig
         do {
-            config = try XCRemoteCacheConfigReader(env: env, fileManager: fileManager).readConfiguration()
+            config = try XCRemoteCacheConfigReader(env: env, fileReader: fileManager).readConfiguration()
         } catch {
             exit(1, "FATAL: Prepare initialization failed with error: \(error)")
         }

Sources/XCRemoteCache/Commands/Prepare/XCPrepare.swift

@@ -61,7 +61,7 @@ public class XCPrepare {
         var context: PrepareContext
         let xcodeVersion: String
         do {
-            config = try XCRemoteCacheConfigReader(env: env, fileManager: fileManager).readConfiguration()
+            config = try XCRemoteCacheConfigReader(env: env, fileReader: fileManager).readConfiguration()
             context = try PrepareContext(config, offline: offline)
             xcodeVersion = try customXcodeBuildNumber ?? XcodeProbeImpl(shell: shellGetStdout).read().buildVersion
         } catch {
@@ -78,6 +78,7 @@ public class XCPrepare {
                 awsV4Signature = AWSV4Signature(
                     secretKey: config.AWSSecretKey,
                     accessKey: config.AWSAccessKey,
+                    securityToken: config.AWSSecurityToken,
                     region: config.AWSRegion,
                     service: config.AWSService,
                     date: Date(timeIntervalSinceNow: 0)

Sources/XCRemoteCache/Commands/Prepare/XCPrepareMark.swift

@@ -46,7 +46,7 @@ public class XCPrepareMark {
         let context: PrepareMarkContext
         let xcodeVersion: String
         do {
-            config = try XCRemoteCacheConfigReader(env: env, fileManager: fileManager).readConfiguration()
+            config = try XCRemoteCacheConfigReader(env: env, fileReader: fileManager).readConfiguration()
             context = try PrepareMarkContext(config)
             xcodeVersion = try xcode ?? XcodeProbeImpl(shell: shellGetStdout).read().buildVersion
         } catch {
@@ -60,6 +60,7 @@ public class XCPrepareMark {
                 awsV4Signature = AWSV4Signature(
                     secretKey: config.AWSSecretKey,
                     accessKey: config.AWSAccessKey,
+                    securityToken: config.AWSSecurityToken,
                     region: config.AWSRegion,
                     service: config.AWSService,
                     date: Date(timeIntervalSinceNow: 0)

Sources/XCRemoteCache/Commands/Prepare/XCStats.swift

@@ -36,7 +36,7 @@ public class XCStats {
         let config: XCRemoteCacheConfig
         let context: XCStatsContext
         do {
-            config = try XCRemoteCacheConfigReader(env: env, fileManager: fileManager).readConfiguration()
+            config = try XCRemoteCacheConfigReader(env: env, fileReader: fileManager).readConfiguration()
             try context = XCStatsContext(config, fileManager: fileManager)
         } catch {
             exit(1, "FATAL: Prepare initialization failed with error: \(error)")

Sources/XCRemoteCache/Commands/ProductBinaryCreator/XCCreateBinary.swift

@@ -70,7 +70,7 @@ public class XCCreateBinary {
         let config: XCRemoteCacheConfig
         do {
             let srcRoot: URL = URL(fileURLWithPath: fileManager.currentDirectoryPath)
-            config = try XCRemoteCacheConfigReader(srcRootPath: srcRoot.path, fileManager: fileManager)
+            config = try XCRemoteCacheConfigReader(srcRootPath: srcRoot.path, fileReader: fileManager)
                 .readConfiguration()
         } catch {
             errorLog("\(stepDescription) initialization failed with error: \(error). Fallbacking to \(fallbackCommand)")

Sources/XCRemoteCache/Commands/Swiftc/SwiftcOrchestrator.swift

@@ -110,8 +110,8 @@ class SwiftcOrchestrator {
                     try invocationStorage.store(args: invocationArgs)
                 }
             } catch {
-                // The critical section is protected by a lock. Some other process already called compilation history.
-                // We only need to call our current step then.
+                // The critical section is protected by a lock. Some other process already called compilation history
+                // We only need to call our current step then
                 fallbackToDefault(command: swiftcCommand)
             }
         case .consumer:

Sources/XCRemoteCache/Commands/Swiftc/SwiftcProductsGenerator.swift

@@ -20,7 +20,7 @@
 import Foundation
 
 enum DiskSwiftcProductsGeneratorError: Error {
-    /// When a generator was asked to generate unknown swiftmodule extension file.
+    /// When a generator was asked to generate unknown swiftmodule extension file
     /// Probably a programmer error: asking to generate excessive extensions, not listed in
     /// `SwiftmoduleFileExtension.SwiftmoduleExtensions`
     case unknownSwiftmoduleFile

Sources/XCRemoteCache/Commands/Swiftc/XCSwiftc.swift

@@ -70,7 +70,7 @@ public class XCSwiftc {
         let context: SwiftcContext
         do {
             let srcRoot: URL = URL(fileURLWithPath: fileManager.currentDirectoryPath)
-            config = try XCRemoteCacheConfigReader(srcRootPath: srcRoot.path, fileManager: fileManager)
+            config = try XCRemoteCacheConfigReader(srcRootPath: srcRoot.path, fileReader: fileManager)
                 .readConfiguration()
             context = try SwiftcContext(config: config, input: inputArgs)
         } catch {

Sources/XCRemoteCache/Config/XCRemoteCacheConfig.swift

@@ -95,10 +95,10 @@ public struct XCRemoteCacheConfig: Encodable {
     ///  Disable cache for http requests to fecth metadata and download artifacts
     var disableHttpCache: Bool = false
     /// Path, relative to $TARGET_TEMP_DIR which gathers all compilation commands that should be e
-    /// xecuted if a target switches to local compilation.
+    /// xecuted if a target switches to local compilation
     /// Example: A new `.swift` file invalidates remote arXcodeProjIntegrate.swifttifact and triggers local compilation
     /// When that happens, all previously skipped clang build steps
-    /// need to be eventually called locally - this file lists all these commands.
+    /// need to be eventually called locally - this file lists all these commands
     var compilationHistoryFile: String = "history.compile"
     /// Timeout for remote response data interval (in seconds). If an interval between data chunks is
     /// longer than a timeout, a request fails
@@ -118,21 +118,26 @@ public struct XCRemoteCacheConfig: Encodable {
     var AWSSecretKey: String = ""
     /// Access key for AWS V4 Signature
     var AWSAccessKey: String = ""
+    /// Temporary security token provided by the AWS Security Token Service
+    var AWSSecurityToken: String?
     /// Region for AWS V4 Signature (e.g. `eu`)
     var AWSRegion: String = ""
     /// Service for AWS V4 Signature (e.g. `storage`)
     var AWSService: String = ""
-    /// A dictionary of files path remapping that should be applied to make it absolute path agnostic on a list of dependencies.
-    /// Useful if a project refers files out of repo root, either compilation files or precompiled dependencies.
-    /// Keys represent generic replacement and values are substrings that should be replaced.
+    /// A dictionary of files path remapping that should be applied to make it absolute path agnostic on a list of
+    /// dependencies. Useful if a project refers files out of repo root, either compilation files or precompiled
+    /// dependencies. Keys represent generic replacement and values are substrings that should be replaced
     /// Example: for mapping `["COOL_LIBRARY": "/CoolLibrary"]`
-    /// `/CoolLibrary/main.swift`will be represented as `$(COOL_LIBRARY)/main.swift`).
-    /// Warning: remapping order is not-deterministic so avoid remappings with multiple matchings.
+    /// `/CoolLibrary/main.swift`will be represented as `$(COOL_LIBRARY)/main.swift`)
+    /// Warning: remapping order is not-deterministic so avoid remappings with multiple matchings
     var outOfBandMappings: [String: String] = [:]
     /// If true, SSL certificate validation is disabled
     var disableCertificateVerification: Bool = false
     /// A feature flag to disable virtual file system overlay support (temporary)
     var disableVFSOverlay: Bool = false
+    /// A list of extra ENVs that should be used as placeholders in the dependency list
+    /// ENV rewrite process is optimistic - does nothing if an ENV is not defined in the pre/postbuild process
+    var customRewriteEnvs: [String] = []
 }
 
 extension XCRemoteCacheConfig {
@@ -181,11 +186,13 @@ extension XCRemoteCacheConfig {
         merge.prettifyMetaFiles = scheme.prettifyMetaFiles ?? prettifyMetaFiles
         merge.AWSAccessKey = scheme.AWSAccessKey ?? AWSAccessKey
         merge.AWSSecretKey = scheme.AWSSecretKey ?? AWSSecretKey
+        merge.AWSSecurityToken = scheme.AWSSecurityToken ?? AWSSecurityToken
         merge.AWSRegion = scheme.AWSRegion ?? AWSRegion
         merge.AWSService = scheme.AWSService ?? AWSService
         merge.outOfBandMappings = scheme.outOfBandMappings ?? outOfBandMappings
         merge.disableCertificateVerification = scheme.disableCertificateVerification ?? disableCertificateVerification
         merge.disableVFSOverlay = scheme.disableVFSOverlay ?? disableVFSOverlay
+        merge.customRewriteEnvs = scheme.customRewriteEnvs ?? customRewriteEnvs
         return merge
     }
 
@@ -243,11 +250,13 @@ struct ConfigFileScheme: Decodable {
     let prettifyMetaFiles: Bool?
     let AWSSecretKey: String?
     let AWSAccessKey: String?
+    let AWSSecurityToken: String?
     let AWSRegion: String?
     let AWSService: String?
     let outOfBandMappings: [String: String]?
     let disableCertificateVerification: Bool?
     let disableVFSOverlay: Bool?
+    let customRewriteEnvs: [String]?
 
     // Yams library doesn't support encoding strategy, see https://github.com/jpsim/Yams/issues/84
     enum CodingKeys: String, CodingKey {
@@ -288,11 +297,13 @@ struct ConfigFileScheme: Decodable {
         case prettifyMetaFiles = "prettify_meta_files"
         case AWSSecretKey = "aws_secret_key"
         case AWSAccessKey = "aws_access_key"
+        case AWSSecurityToken = "aws_security_token"
         case AWSRegion = "aws_region"
         case AWSService = "aws_service"
         case outOfBandMappings = "out_of_band_mappings"
         case disableCertificateVerification = "disable_certificate_verification"
         case disableVFSOverlay = "disable_vfs_overlay"
+        case customRewriteEnvs = "custom_rewrite_envs"
     }
 }
 
@@ -305,31 +316,41 @@ class XCRemoteCacheConfigReader {
     /// Name of the configuration file, required in $(SRCROOT) location
     private static let configurationFile = ".rcinfo"
     private let srcRoot: String
-    private let fileManager: FileManager
+    private let fileReader: FileReader
     private lazy var yamlDecorer = YAMLDecoder(encoding: .utf8)
 
-    init(env: [String: String], fileManager: FileManager) throws {
+    init(env: [String: String], fileReader: FileReader) throws {
         let explicitSrcRoot: String? = env.readEnv(key: "SRCROOT")
-        srcRoot = explicitSrcRoot ?? fileManager.currentDirectoryPath
-        self.fileManager = fileManager
+        srcRoot = explicitSrcRoot ?? FileManager.default.currentDirectoryPath
+        self.fileReader = fileReader
     }
 
-    init(srcRootPath srcRoot: String, fileManager: FileManager) {
+    init(srcRootPath srcRoot: String, fileReader: FileReader) {
         self.srcRoot = srcRoot
-        self.fileManager = fileManager
+        self.fileReader = fileReader
     }
 
+    // Reads the final configuration by loading all extra configs
+    // until reaching a config that doesn't override `extraConfigurationFile`
     func readConfiguration() throws -> XCRemoteCacheConfig {
         let rootURL = URL(fileURLWithPath: srcRoot)
         let configURL = URL(fileURLWithPath: Self.configurationFile, relativeTo: rootURL)
         let userConfigs = try readUserConfig(configURL)
         var config = XCRemoteCacheConfig(sourceRoot: srcRoot).merged(with: userConfigs)
-        let extraConfURL = URL(fileURLWithPath: config.extraConfigurationFile, relativeTo: rootURL)
-        do {
-            let extraConfig = try readUserConfig(extraConfURL)
-            config = config.merged(with: extraConfig)
-        } catch {
-            infoLog("Extra config override failed with \(error). Skipping extra configuration")
+        var extraConfURL = URL(fileURLWithPath: config.extraConfigurationFile, relativeTo: rootURL)
+        var visitedFiles = Set([configURL])
+        while !visitedFiles.contains(extraConfURL) {
+            do {
+                let extraConfig = try readUserConfig(extraConfURL)
+                debugLog("Reading extra configuration from \(extraConfURL)")
+                config = config.merged(with: extraConfig)
+                visitedFiles.insert(extraConfURL)
+                // Advance extra configuration
+                extraConfURL = URL(fileURLWithPath: config.extraConfigurationFile, relativeTo: rootURL)
+            } catch {
+                infoLog("Extra config override failed with \(error). Skipping extra configuration")
+                break
+            }
         }
 
         return try config.verifyAndApplyDefaults()
@@ -337,7 +358,7 @@ class XCRemoteCacheConfigReader {
 
     /// Reads user configuration from a file
     private func readUserConfig(_ file: URL) throws -> ConfigFileScheme {
-        let configurationContent = fileManager.contents(atPath: file.path)
+        let configurationContent = try fileReader.contents(atPath: file.path)
         guard let configurationData = configurationContent else {
             throw XCRemoteCacheConfigReaderError.missingConfigurationFile(file)
         }

Sources/XCRemoteCache/Dependencies/DependenciesReader.swift

@@ -72,7 +72,7 @@ public class FileDependenciesReader: DependenciesReader {
                 return Set(splitDependencyFileList(value))
             case let s where s.hasSuffix(".o") || s.hasSuffix(".bc"):
                 // 'swiftc' output formatting
-                // take dependencies from any .o or .bc file.
+                // take dependencies from any .o or .bc file
                 // Note: For WMO, all .{o|bc} files have the same dependencies
                 return Set(splitDependencyFileList(value))
             default:

Sources/XCRemoteCache/Dependencies/DependencyProcessor.swift

@@ -27,6 +27,7 @@ public struct Dependency: Equatable {
         case source
         case fingerprint
         case intermediate
+        case derivedFile
         // Product of the target itself
         case ownProduct
         case unknown
@@ -55,13 +56,15 @@ class DependencyProcessorImpl: DependencyProcessor {
     private let productPath: String
     private let sourcePath: String
     private let intermediatePath: String
+    private let derivedFilesPath: String
     private let bundlePath: String?
 
-    init(xcode: URL, product: URL, source: URL, intermediate: URL, bundle: URL?) {
+    init(xcode: URL, product: URL, source: URL, intermediate: URL, derivedFiles: URL, bundle: URL?) {
         xcodePath = xcode.path.dirPath()
         productPath = product.path.dirPath()
         sourcePath = source.path.dirPath()
         intermediatePath = intermediate.path.dirPath()
+        derivedFilesPath = derivedFiles.path.dirPath()
         bundlePath = bundle?.path.dirPath()
     }
 
@@ -77,6 +80,8 @@ class DependencyProcessorImpl: DependencyProcessor {
                 return Dependency(url: file, type: .xcode)
             } else if filePath.hasPrefix(intermediatePath) {
                 return Dependency(url: file, type: .intermediate)
+            } else if filePath.hasPrefix(derivedFilesPath) {
+                return Dependency(url: file, type: .derivedFile)
             } else if let bundle = bundlePath, filePath.hasPrefix(bundle) {
                 // If a target produces a bundle, explicitly classify all
                 // of products to distinguish from other targets products
@@ -107,7 +112,9 @@ class DependencyProcessorImpl: DependencyProcessor {
         // - All files in `*/Interemediates/*` - this file are created on-fly for a given target
         // - Some files may depend on its own product (e.g. .m may #include *-Swift.h) - we know products will match
         //   because in case of a hit, these will be taken from the artifact
-        let irrelevantDependenciesType: [Dependency.Kind] = [.xcode, .intermediate, .ownProduct]
+        // - Customized DERIVED_FILE_DIR may change a directory of
+        //   derived files, which by default is under `*/Interemediates`
+        let irrelevantDependenciesType: [Dependency.Kind] = [.xcode, .intermediate, .ownProduct, .derivedFile]
         return !irrelevantDependenciesType.contains(dependency.type)
     }
 }

Sources/XCRemoteCache/Dependencies/OverlayDependenciesRemapper.swift

@@ -42,8 +42,8 @@ class OverlayDependenciesRemapper: DependenciesRemapper {
 
     private func mapPath(
         _ path: String,
-        source: KeyPath<OverlayMapping,URL>,
-        destination: KeyPath<OverlayMapping,URL>
+        source: KeyPath<OverlayMapping, URL>,
+        destination: KeyPath<OverlayMapping, URL>
     ) throws -> String {
         guard let mapping = try getMappings().first(where: { $0[keyPath: source].path == path }) else {
             // TODO: support partial mappings, where a directory path can be replaced with some other directory
@@ -54,14 +54,14 @@ class OverlayDependenciesRemapper: DependenciesRemapper {
     }
 
     func replace(genericPaths: [String]) throws -> [String] {
-        try Set(genericPaths.map {
+        try genericPaths.map {
             try mapPath($0, source: \.virtual, destination: \.local)
-        }).sorted()
+        }
     }
 
     func replace(localPaths: [String]) throws -> [String] {
-        try Set(localPaths.map {
+        try localPaths.map {
             try mapPath($0, source: \.local, destination: \.virtual)
-        }).sorted()
+        }
     }
 }

Sources/XCRemoteCache/Dependencies/OverlayReader.swift

@@ -91,7 +91,7 @@ class JsonOverlayReader: OverlayReader {
             case .strict:
                 throw JsonOverlayReaderError.missingSourceFile(json)
             case .bestEffort:
-                printWarning("overlay mapping file \(json) doesn't exist. Skipping overlay for the best-effort mode.")
+                debugLog("overlay mapping file \(json) doesn't exist. Skipping overlay for the best-effort mode.")
                 return []
             }
         }
@@ -100,7 +100,7 @@ class JsonOverlayReader: OverlayReader {
             let mappings: [OverlayMapping] = try overlay.roots.reduce([]) { prev, root in
                 switch root.type {
                 case .directory:
-                    //iterate all contents
+                    // iterate all contents
                     let dir = URL(fileURLWithPath: root.name)
                     let mappings: [OverlayMapping] = try root.contents.map { content in
                         switch content.type {
@@ -124,7 +124,7 @@ class JsonOverlayReader: OverlayReader {
             case .strict:
                 throw error
             case .bestEffort:
-                printWarning("Overlay reader has failed with an error \(error). Best-effort mode - skipping an overlay.")
+                errorLog("Overlay reader has failed with an error \(error). Best-effort mode - skipping an overlay.")
                 return []
             }
         }

Sources/XCRemoteCache/Dependencies/PathDependenciesRemapperFactory.swift

@@ -19,24 +19,28 @@
 
 import Foundation
 
-enum StringDependenciesRemapperFactoryError: Error {
+enum PathDependenciesRemapperFactoryError: Error {
     /// Remapping keys are duplicated and can lead to undetermined results
     case mappingKeyDuplication
 }
 
-class StringDependenciesRemapperFactory {
+class PathDependenciesRemapperFactory {
     func build(
         orderKeys: [String],
         envs: [String: String],
         customMappings: [String: String]
     ) throws -> StringDependenciesRemapper {
-        let mappingMap = try envs.merging(customMappings) { envValue, outOfBandMapping in
-            throw StringDependenciesRemapperFactoryError.mappingKeyDuplication
+        let mappingMap = try envs.merging(customMappings) { _, _ in
+            throw PathDependenciesRemapperFactoryError.mappingKeyDuplication
         }
-        let mappingOrderKeys =  orderKeys + customMappings.keys
-        let mappings: [StringDependenciesRemapper.Mapping] = try mappingOrderKeys.map { key in
-            let localValue: String = try mappingMap.readEnv(key: key)
-            return StringDependenciesRemapper.Mapping(generic: "$(\(key))", local: localValue)
+        let mappingOrderKeys = orderKeys + customMappings.keys
+        let mappings: [StringDependenciesRemapper.Mapping] = mappingOrderKeys.compactMap { key in
+            guard let localURL: URL = mappingMap.readEnv(key: key) else {
+                debugLog("\(key) ENV to map a dependency is not defined")
+                return nil
+            }
+            infoLog("Found url to remapp: \(localURL). Remapping: \(localURL.standardized.path)")
+            return StringDependenciesRemapper.Mapping(generic: "$(\(key))", local: localURL.standardized.path)
         }
         return StringDependenciesRemapper(mappings: mappings)
     }

Sources/XCRemoteCache/Dependencies/TargetDepdenciesReader.swift

@@ -42,7 +42,7 @@ class TargetDependenciesReader: DependenciesReader {
         let allURLs = try dirScanner.items(at: directory)
         let mergedDependencies = try allURLs.reduce(Set<String>()) { (prev: Set<String>, file) in
             // include only these .d files that either have corresponding .o file (incremental) or end
-            // with '-master' (whole-module).
+            // with '-master' (whole-module)
             // Otherwise .d is probably just a leftover from previous builds
             let correspondingOutputURL = file.deletingPathExtension().appendingPathExtension("o")
             let isDependencyFile = file.pathExtension == "d"

Sources/XCRemoteCache/Fingerprint/EnvironmentFingerprint.swift

@@ -31,7 +31,7 @@ class EnvironmentFingerprintGenerator {
         "DYLIB_COMPATIBILITY_VERSION",
         "DYLIB_CURRENT_VERSION",
         "PRODUCT_MODULE_NAME",
-        "ARCHS"
+        "ARCHS",
     ]
     private let version: String
     private let customFingerprintEnvs: [String]

Sources/XCRemoteCache/Logger/Logger.swift

@@ -22,34 +22,36 @@ import Foundation
 import os.log
 
 
+private var processTag: String = ""
+
 public func exit(_ exitCode: Int32, _ message: String) -> Never {
-    os_log("%{public}@", log: OSLog.default, type: .error, message)
+    os_log("%{public}@%{public}@", log: OSLog.default, type: .error, processTag, message)
     printError(errorMessage: message)
     exit(exitCode)
 }
 
 func defaultLog(_ message: String) {
-    os_log("%{public}@", log: OSLog.default, type: .default, message)
+    os_log("%{public}@%{public}@", log: OSLog.default, type: .default, processTag, message)
 }
 
 func errorLog(_ message: String) {
-    os_log("%{public}@", log: OSLog.default, type: .error, message)
+    os_log("%{public}@%{public}@", log: OSLog.default, type: .error, processTag, message)
 }
 
 func infoLog(_ message: String) {
-    os_log("%{public}@", log: OSLog.default, type: .info, message)
+    os_log("%{public}@%{public}@", log: OSLog.default, type: .info, processTag, message)
 }
 
 func debugLog(_ message: String) {
-    os_log("%{public}@", log: OSLog.default, type: .debug, message)
+    os_log("%{public}@%{public}@", log: OSLog.default, type: .debug, processTag, message)
 }
 
 func printError(errorMessage: String) {
-    fputs("error: \(errorMessage)\n", stderr)
+    fputs("error: \(processTag)\(errorMessage)\n", stderr)
 }
 
 func printWarning(_ message: String) {
-    print("warning: \(message)")
+    print("warning: \(processTag)\(message)")
 }
 
 /// Prints a message to the user. It shows in Xcode (if applies) or console output
@@ -57,3 +59,7 @@ func printWarning(_ message: String) {
 func printToUser(_ message: String) {
     print("[RC] \(message)")
 }
+
+func updateProcessTag(_ tag: String) {
+    processTag = "(\(tag)) "
+}

Sources/XCRemoteCache/Models/MetaWriter.swift

@@ -20,7 +20,7 @@
 import Foundation
 
 protocol MetaWriter {
-    func write<T>(_ meta: T, locationDir : URL) throws -> URL where T : Meta
+    func write<T>(_ meta: T, locationDir: URL) throws -> URL where T: Meta
 }
 
 class JsonMetaWriter: MetaWriter {
@@ -36,7 +36,7 @@ class JsonMetaWriter: MetaWriter {
         self.metaEncoder = encoder
     }
 
-    func write<T>(_ meta: T, locationDir : URL) throws -> URL where T : Meta {
+    func write<T>(_ meta: T, locationDir: URL) throws -> URL where T: Meta {
         let metaURL = locationDir.appendingPathComponent(meta.fileKey).appendingPathExtension("json")
         let metaData = try metaEncoder.encode(meta)
         try fileWriter.write(toPath: metaURL.path, contents: metaData)

Sources/XCRemoteCache/Network/Authentication/AWSV4Signature.swift

@@ -23,17 +23,21 @@ struct AWSV4Signature {
 
     let secretKey: String
     let accessKey: String
+    let securityToken: String?
     let region: String
     let service: String
     let date: Date
 
-
     func addSignatureHeaderTo(request: inout URLRequest) {
 
         request.setValue(request.url?.host, forHTTPHeaderField: "host")
         request.setValue(StringToSign.ISO8601BasicFormatter.string(from: date), forHTTPHeaderField: "x-amz-date")
         request.setValue((request.httpBody ?? Data()).sha256(), forHTTPHeaderField: "x-amz-content-sha256")
 
+        if let securityToken = securityToken {
+            request.setValue(securityToken, forHTTPHeaderField: "x-amz-security-token")
+        }
+
         let canonicalRequest = CanonicalRequest(request: request)
         let stringToSign = StringToSign(
             region: region,

Sources/XCRemoteCache/Network/IgnoringCertificatesTrustManager.swift

@@ -25,7 +25,7 @@ final class IgnoringCertificatesTrustManager: NSObject, URLSessionDelegate {
             completionHandler(.performDefaultHandling, nil)
             return
         }
-        
+
         let urlCredential = URLCredential(trust: serverTrust)
         completionHandler(.useCredential, urlCredential)
     }

Sources/XCRemoteCache/Network/URLSessionFactory.swift

@@ -36,6 +36,8 @@ class DefaultURLSessionFactory: URLSessionFactory {
         let configuration = URLSessionConfiguration.default
         configuration.httpAdditionalHeaders = config.requestCustomHeaders
         configuration.timeoutIntervalForRequest = config.timeoutResponseDataChunksInterval
+        configuration.urlCache?.memoryCapacity = 0
+        configuration.urlCache?.diskCapacity = 0
         switch config.disableCertificateVerification {
         case true:
             return URLSession(

Sources/XCRemoteCache/Stats/ExclusiveFileAccessor.swift

@@ -76,8 +76,8 @@ class ExclusiveFile: ExclusiveFileAccessor {
         guard flock(fd, LOCK_EX) == 0 else {
             throw FileAccessorError.lockingFailure
         }
-        // While having a lock, make sure the file still exists.
-        // It might delete it while we were waiting for a lock.
+        // While having a lock, make sure the file still exists
+        // It might delete it while we were waiting for a lock
         guard access(fileURL.path, F_OK) == 0 else {
             throw FileAccessorError.lockingFailure
         }

Sources/XCRemoteCache/Utils/ENVReader.swift

@@ -24,8 +24,15 @@ enum EnvironmentError: Error {
 }
 
 extension Dictionary where Key == String, Value == String {
-    func readEnv(key: String) throws -> URL {
+    func readEnv(key: String) -> URL? {
         guard let value = self[key].map(URL.init(fileURLWithPath:)) else {
+            return nil
+        }
+        return value
+    }
+
+    func readEnv(key: String) throws -> URL {
+        guard let value: URL = readEnv(key: key) else {
             throw EnvironmentError.missingEnv(key)
         }
         return value

Tests/XCRemoteCacheTests/Commands/Plugins/Thinning/ThinningCreatorPluginTests.swift

@@ -131,7 +131,7 @@ class ThinningCreatorPluginTests: FileXCTestCase {
 
         XCTAssertEqual(extraKeys, [
             "thinning_Generated": "000",
-            "thinning_Reused": "999"
+            "thinning_Reused": "999",
         ])
     }
 }

Tests/XCRemoteCacheTests/Commands/PostbuildContextTests.swift

@@ -26,6 +26,7 @@ class PostbuildContextTests: FileXCTestCase {
     private static let SampleEnvs = [
         "TARGET_NAME": "TARGET_NAME",
         "TARGET_TEMP_DIR": "TARGET_TEMP_DIR",
+        "DERIVED_FILE_DIR": "DERIVED_FILE_DIR",
         "ARCHS": "x86_64",
         "OBJECT_FILE_DIR_normal": "/OBJECT_FILE_DIR_normal" ,
         "CONFIGURATION": "CONFIGURATION",
@@ -42,7 +43,7 @@ class PostbuildContextTests: FileXCTestCase {
         "DWARF_DSYM_FILE_NAME": "DWARF_DSYM_FILE_NAME",
         "BUILT_PRODUCTS_DIR": "BUILT_PRODUCTS_DIR",
         "DERIVED_SOURCES_DIR": "DERIVED_SOURCES_DIR",
-        "CURRENT_VARIANT": "normal"
+        "CURRENT_VARIANT": "normal",
     ]
 
     override func setUpWithError() throws {

Tests/XCRemoteCacheTests/Commands/PostbuildTests.swift

@@ -20,12 +20,14 @@
 @testable import XCRemoteCache
 import XCTest
 
+// swiftlint:disable file_length
 // swiftlint:disable:next type_body_length
 class PostbuildTests: FileXCTestCase {
     private var postbuildContext = PostbuildContext(
         mode: .producer,
         targetName: "",
         targetTempDir: "",
+        derivedFilesDir: "",
         compilationTempDir: "",
         configuration: "",
         platform: "",
@@ -77,6 +79,7 @@ class PostbuildTests: FileXCTestCase {
         product: "/Product",
         source: "/Source",
         intermediate: "/Intermediate",
+        derivedFiles: "/DerivedFiles",
         bundle: nil
     )
     private var overrideManager = FingerprintOverrideManagerImpl(
@@ -639,4 +642,3 @@ class PostbuildTests: FileXCTestCase {
         XCTAssertEqual(downloadedMeta, expectedMeta)
     }
 }
-// swiftlint:disable:next file_length

Tests/XCRemoteCacheTests/Commands/Prepare/Integrate/XcodeProjBuildSettingsIntegrateAppenderTests.swift

@@ -0,0 +1,62 @@
+// Copyright (c) 2021 Spotify AB.
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+@testable import XCRemoteCache
+import XCTest
+
+class XcodeProjBuildSettingsIntegrateAppenderTests: XCTestCase {
+    private let rootURL: URL = "/root"
+    private let binariesDir: URL = "/binaries"
+    private var buildSettings: BuildSettings!
+    private var binaries: XCRCBinariesPaths!
+
+    override func setUp() {
+        super.setUp()
+        buildSettings = BuildSettings()
+        binaries = XCRCBinariesPaths(
+            prepare: binariesDir.appendingPathComponent("xcprepare"),
+            cc: binariesDir.appendingPathComponent("xccc"),
+            swiftc: binariesDir.appendingPathComponent("xcswiftc"),
+            libtool: binariesDir.appendingPathComponent("xclibtool"),
+            ld: binariesDir.appendingPathComponent("xcld"),
+            prebuild: binariesDir.appendingPathComponent("xcprebuild"),
+            postbuild: binariesDir.appendingPathComponent("xcpostbuild")
+        )
+    }
+
+    func testProducerSettingFakeSrcRoot() throws {
+        let mode: Mode = .producer
+        let fakeRootURL: URL = "/xxxxxxxxxxP"
+        let appender = XcodeProjBuildSettingsIntegrateAppender(mode: mode, repoRoot: rootURL, fakeSrcRoot: fakeRootURL)
+        let result = appender.appendToBuildSettings(buildSettings: buildSettings, wrappers: binaries)
+        let resultURL = try XCTUnwrap(result["XCRC_FAKE_SRCROOT"] as? String)
+
+        XCTAssertEqual(resultURL, fakeRootURL.path)
+    }
+
+    func testConsumerSettingFakeSrcRoot() throws {
+        let mode: Mode = .consumer
+        let fakeRootURL: URL = "/xxxxxxxxxxC"
+        let appender = XcodeProjBuildSettingsIntegrateAppender(mode: mode, repoRoot: rootURL, fakeSrcRoot: fakeRootURL)
+        let result = appender.appendToBuildSettings(buildSettings: buildSettings, wrappers: binaries)
+        let resultURL: String = try XCTUnwrap(result["XCRC_FAKE_SRCROOT"] as? String)
+
+        XCTAssertEqual(resultURL, fakeRootURL.path)
+    }
+}

Tests/XCRemoteCacheTests/Commands/XCCCTests.swift

@@ -47,7 +47,8 @@ class TemplateBasedCCWrapperBuilderTests: FileXCTestCase {
         let app = appDir.appendingPathComponent("xccc")
         try? fileManager.removeItem(at: appDir)
         try? FileManager.default.createDirectory(at: appDir, withIntermediateDirectories: true, attributes: nil)
-        try? builder.compile(to: app, commitSha: commitSha)
+        // swiftlint:disable:next force_try
+        try! builder.compile(to: app, commitSha: commitSha)
         return app
     }()
 
@@ -332,11 +333,31 @@ class TemplateBasedCCWrapperBuilderTests: FileXCTestCase {
         XCTAssertNotEqual(newFileOutputData, Data())
     }
 
+    func testPCHCompilationFallbacks() throws {
+        // Marker is empty to mimic the new file scenario
+        let pchFile = directory.appendingPathComponent("input.pch")
+        createValidPCHFile(pchFile)
+        arguments = ["-x", "objective-c-header", "-MF", dependencyFile.path, "-o", outputFile.path, pchFile.path]
+
+        try shellExec(Self.xccc.path, args: arguments, inDir: directory.path)
+
+        XCTAssertTrue(fileManager.fileExists(atPath: outputFile.path))
+    }
+
     /// Creates a simple C code in the location
     private func createValidCFile(_ location: URL) {
         fileManager.createFile(atPath: location.path, contents: "int main(){}".data(using: .utf8), attributes: nil)
     }
 
+    /// Creates a simple PCH code in the location
+    private func createValidPCHFile(_ location: URL) {
+        fileManager.createFile(
+            atPath: location.path,
+            contents: "#import <Availability.h>".data(using: .utf8),
+            attributes: nil
+        )
+    }
+
     /// Creates a C code that requires extra CUSTOM_STR clang macro to compile
     private func createInvalidCFile(_ location: URL) {
         fileManager.createFile(

Tests/XCRemoteCacheTests/Config/XCRemoteCacheConfigReaderTests.swift

@@ -0,0 +1,88 @@
+// Copyright (c) 2022 Spotify AB.
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+@testable import XCRemoteCache
+import XCTest
+
+class XCRemoteCacheConfigReaderTests: XCTestCase {
+
+    private var fileReader: FileAccessorFake!
+    private var reader: XCRemoteCacheConfigReader!
+
+    override func setUp() {
+        super.setUp()
+        fileReader = FileAccessorFake(mode: .normal)
+        reader = XCRemoteCacheConfigReader(srcRootPath: "/", fileReader: fileReader)
+    }
+
+    func testReadsFromExtraConfig() throws {
+        try fileReader.write(toPath: "/.rcinfo", contents: "cache_addresses: [test]")
+
+        let config = try reader.readConfiguration()
+
+        XCTAssertEqual(config.cacheAddresses, ["test"])
+    }
+
+    func testOverridesExtraConfigFromExtraFile() throws {
+        try fileReader.write(toPath: "/.rcinfo", contents: "cache_addresses: [test]")
+        try fileReader.write(toPath: "/user.rcinfo", contents: "cache_addresses: [user]")
+
+        let config = try reader.readConfiguration()
+
+        XCTAssertEqual(config.cacheAddresses, ["user"])
+    }
+
+    func testReadsExtraConfigMultipleTimes() throws {
+        try fileReader.write(toPath: "/.rcinfo", contents: "cache_addresses: [test]")
+        try fileReader.write(toPath: "/user.rcinfo", contents: """
+        cache_addresses: [user]
+        extra_configuration_file: user2.rcinfo
+        """)
+        try fileReader.write(toPath: "/user2.rcinfo", contents: "cache_addresses: [user2]")
+
+        let config = try reader.readConfiguration()
+
+        XCTAssertEqual(config.cacheAddresses, ["user2"])
+    }
+
+    func testBreaksImportingExtraConfigIfReachingALoop() throws {
+        try fileReader.write(toPath: "/.rcinfo", contents: "cache_addresses: [test]")
+        try fileReader.write(toPath: "/user.rcinfo", contents: """
+        cache_addresses: [user]
+        extra_configuration_file: .rcinfo
+        """)
+
+        let config = try reader.readConfiguration()
+
+        XCTAssertEqual(config.cacheAddresses, ["user"])
+    }
+
+    func testBreaksImportingExtraConfigIfFileDoesntExist() throws {
+        try fileReader.write(toPath: "/.rcinfo", contents: "cache_addresses: [test]")
+        try fileReader.write(toPath: "/user.rcinfo", contents: """
+        cache_addresses: [user]
+        extra_configuration_file: nonexisting.rcinfo
+        """)
+
+        let config = try reader.readConfiguration()
+
+        XCTAssertEqual(config.cacheAddresses, ["user"])
+        XCTAssertEqual(config.extraConfigurationFile, "nonexisting.rcinfo")
+    }
+}

Tests/XCRemoteCacheTests/Dependencies/DependenciesRemapperCompositeTests.swift

@@ -81,8 +81,18 @@ class DependenciesRemapperCompositeTests: XCTestCase {
 
     func testRemapsMultipleMatchingMappers() throws {
         let remapper = DependenciesRemapperComposite([
-            StringDependenciesRemapper(mappings: [StringDependenciesRemapper.Mapping(generic: "$(ROOT)", local: "/root")]),
-            StringDependenciesRemapper(mappings: [StringDependenciesRemapper.Mapping(generic: "$(SPECIFIC)", local: "$(ROOT)/specific")])
+            StringDependenciesRemapper(mappings: [
+                StringDependenciesRemapper.Mapping(
+                    generic: "$(ROOT)",
+                    local: "/root"
+                ),
+            ]),
+            StringDependenciesRemapper(mappings: [
+                StringDependenciesRemapper.Mapping(
+                    generic: "$(SPECIFIC)",
+                    local: "$(ROOT)/specific"
+                ),
+            ]),
         ])
         let localPaths = ["/root/specific/file"]
 
@@ -93,8 +103,18 @@ class DependenciesRemapperCompositeTests: XCTestCase {
 
     func testRemapsBackToLocalWithRevertedRemappersOrder() throws {
         let remapper = DependenciesRemapperComposite([
-            StringDependenciesRemapper(mappings: [StringDependenciesRemapper.Mapping(generic: "$(ROOT)", local: "/root")]),
-            StringDependenciesRemapper(mappings: [StringDependenciesRemapper.Mapping(generic: "$(SPECIFIC)", local: "$(ROOT)/specific")])
+            StringDependenciesRemapper(mappings: [
+                StringDependenciesRemapper.Mapping(
+                    generic: "$(ROOT)",
+                    local: "/root"
+                ),
+            ]),
+            StringDependenciesRemapper(mappings: [
+                StringDependenciesRemapper.Mapping(
+                    generic: "$(SPECIFIC)",
+                    local: "$(ROOT)/specific"
+                ),
+            ]),
         ])
         let genericPaths = ["$(SPECIFIC)/file"]
 
@@ -105,8 +125,18 @@ class DependenciesRemapperCompositeTests: XCTestCase {
 
     func testRemappingTwoMappingsBackAndForthIsIdentical() throws {
         let remapper = DependenciesRemapperComposite([
-            StringDependenciesRemapper(mappings: [StringDependenciesRemapper.Mapping(generic: "$(ROOT)", local: "/root")]),
-            StringDependenciesRemapper(mappings: [StringDependenciesRemapper.Mapping(generic: "$(SPECIFIC)", local: "$(ROOT)/specific")])
+            StringDependenciesRemapper(mappings: [
+                StringDependenciesRemapper.Mapping(
+                    generic: "$(ROOT)",
+                    local: "/root"
+                ),
+            ]),
+            StringDependenciesRemapper(mappings: [
+                StringDependenciesRemapper.Mapping(
+                    generic: "$(SPECIFIC)",
+                    local: "$(ROOT)/specific"
+                ),
+            ]),
         ])
         let localPaths = ["/root/specific/file"]
 

Tests/XCRemoteCacheTests/Dependencies/DependencyProcessorImplTests.swift

@@ -27,6 +27,7 @@ class DependencyProcessorImplTests: FileXCTestCase {
         product: "/Product",
         source: "/Source",
         intermediate: "/Intermediate",
+        derivedFiles: "/DerivedFiles",
         bundle: "/Bundle"
     )
 
@@ -37,6 +38,7 @@ class DependencyProcessorImplTests: FileXCTestCase {
             product: "/",
             source: "/",
             intermediate: "/Intermediate",
+            derivedFiles: "/DerivedFiles",
             bundle: nil
         )
 
@@ -53,6 +55,7 @@ class DependencyProcessorImplTests: FileXCTestCase {
             product: "/",
             source: "/",
             intermediate: "/Intermediate",
+            derivedFiles: "/DerivedFiles",
             bundle: "/Bundle"
         )
 
@@ -62,6 +65,22 @@ class DependencyProcessorImplTests: FileXCTestCase {
         )
     }
 
+    func testFiltersOutGeneratedSwiftHeaders() throws {
+        let dependencies = processor.process([
+            "/DerivedFiles/ModuleName-Swift.h",
+        ])
+
+        XCTAssertEqual(dependencies, [])
+    }
+
+    func testFiltersOutDerivedFile() throws {
+        let dependencies = processor.process([
+            "/DerivedFiles/output.h",
+        ])
+
+        XCTAssertEqual(dependencies, [])
+    }
+
     func testFiltersOutProductModulemap() throws {
         let dependencies = processor.process([
             "/Product/some.modulemap",
@@ -130,13 +149,18 @@ class DependencyProcessorImplTests: FileXCTestCase {
             product: "/Product",
             source: "/Source",
             intermediate: intermediateDirReal,
+            derivedFiles: "/DerivedFiles",
             bundle: "/Bundle"
         )
 
-        let intermediateFileSymlink = createSymlink(filename: someFilename, sourceDir: symlink, destinationDir: intermediateDirReal)
+        let intermediateFileSymlink = createSymlink(
+            filename: someFilename,
+            sourceDir: symlink,
+            destinationDir: intermediateDirReal
+        )
 
         let dependencies = processor.process([
-            intermediateFileSymlink
+            intermediateFileSymlink,
         ])
 
         XCTAssertEqual(dependencies, [])
@@ -154,13 +178,18 @@ class DependencyProcessorImplTests: FileXCTestCase {
             product: "/Product",
             source: sourceDirReal,
             intermediate: "/Intermediate",
+            derivedFiles: "/DerivedFiles",
             bundle: "/Bundle"
         )
 
-        let sourceFileSymlink = createSymlink(filename: someFilename, sourceDir: symlink, destinationDir: sourceDirReal)
+        let sourceFileSymlink = createSymlink(
+            filename: someFilename,
+            sourceDir: symlink,
+            destinationDir: sourceDirReal
+        )
 
         let dependencies = processor.process([
-            sourceFileSymlink
+            sourceFileSymlink,
         ])
 
         XCTAssertEqual(dependencies, [.init(url: sourceFileSymlink, type: .source)])
@@ -173,10 +202,29 @@ class DependencyProcessorImplTests: FileXCTestCase {
     fileprivate func createSymlink(filename: String, sourceDir: URL, destinationDir: URL) -> URL {
         let fileMng = FileManager.default
 
-        XCTAssertNoThrow(try fileMng.spt_forceSymbolicLink(at: sourceDir,
-                                                           withDestinationURL: destinationDir))
+        XCTAssertNoThrow(try fileMng.spt_forceSymbolicLink(
+            at: sourceDir,
+            withDestinationURL: destinationDir
+        ))
         XCTAssertNoThrow(try fileMng.spt_createEmptyFile(destinationDir.appendingPathComponent(filename)))
 
         return sourceDir.appendingPathComponent(filename)
     }
+
+    func testSkipsCustomizedDerivedDirFileUnderSources() {
+        let derivedFile: URL = "/DerivedFiles/Module-Swift.h"
+        let processor = DependencyProcessorImpl(
+            xcode: "/Xcode",
+            product: "/",
+            source: "/",
+            intermediate: "/Intermediate",
+            derivedFiles: "/DerivedFiles",
+            bundle: nil
+        )
+
+        XCTAssertEqual(
+            processor.process([derivedFile]),
+            []
+        )
+    }
 }

Tests/XCRemoteCacheTests/Dependencies/OverlayDependenciesRemapperTests.swift

@@ -40,20 +40,20 @@ class OverlayDependenciesRemapperTests: XCTestCase {
         XCTAssertEqual(dependencies, ["/Intermediate/Some/file.h"])
     }
 
-    func testGenericDependenciesAreMerged() throws {
+    func testGenericDependenciesAreNotMerged() throws {
         let remapper = OverlayDependenciesRemapper(overlayReader: overlayReader)
 
 
         let dependencies = try remapper.replace(localPaths: ["/Intermediate/Some/file.h", "/file.h"])
-        XCTAssertEqual(dependencies, ["/file.h"])
+        XCTAssertEqual(dependencies, ["/file.h", "/file.h"])
     }
 
-    func testLocalDependenciesAreMerged() throws {
+    func testLocalDependenciesAreNotMerged() throws {
         let remapper = OverlayDependenciesRemapper(overlayReader: overlayReader)
 
 
         let dependencies = try remapper.replace(genericPaths: ["/Intermediate/Some/file.h", "/file.h"])
-        XCTAssertEqual(dependencies, ["/Intermediate/Some/file.h"])
+        XCTAssertEqual(dependencies, ["/Intermediate/Some/file.h", "/Intermediate/Some/file.h"])
     }
 
     func testMappingsAreReadOnce() throws {
@@ -68,4 +68,3 @@ class OverlayDependenciesRemapperTests: XCTestCase {
         XCTAssertEqual(secondDependencies, ["/file.h"])
     }
 }
-

Tests/XCRemoteCacheTests/Dependencies/OverlayReaderTests.swift

@@ -29,8 +29,14 @@ class JsonOverlayReaderTests: FileXCTestCase {
         let mappings = try reader.provideMappings()
 
         let expectedMappings = [
-            OverlayMapping(virtual: "/DerivedDataProducts/Target1.framework/Headers/Target1.h", local: "/Path/Target1/Target1.h"),
-            OverlayMapping(virtual: "/DerivedDataProducts/Target2.framework/Modules/module.modulemap", local: "/DerivedDataIntermediate/Target2.build/module.modulemap")
+            OverlayMapping(
+                virtual: "/DerivedDataProducts/Target1.framework/Headers/Target1.h",
+                local: "/Path/Target1/Target1.h"
+            ),
+            OverlayMapping(
+                virtual: "/DerivedDataProducts/Target2.framework/Modules/module.modulemap",
+                local: "/DerivedDataIntermediate/Target2.build/module.modulemap"
+            ),
         ]
         XCTAssertEqual(Set(mappings), Set(expectedMappings))
     }
@@ -79,6 +85,10 @@ class JsonOverlayReaderTests: FileXCTestCase {
     }
 
     private func pathForTestData(name: String) throws -> URL {
-        return try XCTUnwrap(Bundle.module.url(forResource: name, withExtension: "json", subdirectory: JsonOverlayReaderTests.resourcesSubdirectory))
+        return try XCTUnwrap(Bundle.module.url(
+            forResource: name,
+            withExtension: "json",
+            subdirectory: JsonOverlayReaderTests.resourcesSubdirectory
+        ))
     }
 }

Tests/XCRemoteCacheTests/Dependencies/PathDependenciesRemapperFactoryTests.swift

@@ -20,11 +20,12 @@
 @testable import XCRemoteCache
 import XCTest
 
-class StringDependenciesRemapperFactoryTests: XCTestCase {
-    private var factory: StringDependenciesRemapperFactory!
+class PathDependenciesRemapperFactoryTests: XCTestCase {
+    private var factory: PathDependenciesRemapperFactory!
 
     override func setUp() {
-        factory = StringDependenciesRemapperFactory()
+        super.setUp()
+        factory = PathDependenciesRemapperFactory()
     }
 
     func testMappingsFromEnvMaps() throws {
@@ -38,8 +39,30 @@ class StringDependenciesRemapperFactoryTests: XCTestCase {
         XCTAssertEqual(localPaths, ["/tmp/root/some.swift"])
     }
 
-    func testInvalidMappingsFromEnvFails() throws {
-        XCTAssertThrowsError(
+    func testMappingsGenericWhenMappingHasParentDir() throws {
+        let remapper = try factory.build(
+            orderKeys: ["SRC_ROOT"],
+            envs: ["SRC_ROOT": "/tmp/root/extra/.."],
+            customMappings: [:]
+        )
+
+        let localPaths = try remapper.replace(genericPaths: ["$(SRC_ROOT)/some.swift"])
+        XCTAssertEqual(localPaths, ["/tmp/root/some.swift"])
+    }
+
+    func testMappingsLocalWhenMappingHasParentDir() throws {
+        let remapper = try factory.build(
+            orderKeys: ["SRC_ROOT"],
+            envs: ["SRC_ROOT": "/tmp/root/excessive/.."],
+            customMappings: [:]
+        )
+
+        let localPaths = try remapper.replace(localPaths: ["/tmp/root/some.swift"])
+        XCTAssertEqual(localPaths, ["$(SRC_ROOT)/some.swift"])
+    }
+
+    func testMissingEnvIsSkipped() throws {
+        XCTAssertNoThrow(
             try factory.build(
                 orderKeys: ["SRC_ROOT"],
                 envs: ["NO_SRC_ROOT": ""],

Tests/XCRemoteCacheTests/Fingerprint/EnvironmentFingerprintGeneratorTests.swift

@@ -33,7 +33,7 @@ class EnvironmentFingerprintGeneratorTests: XCTestCase {
         "DYLIB_COMPATIBILITY_VERSION": "2",
         "DYLIB_CURRENT_VERSION": "3",
         "PRODUCT_MODULE_NAME": "4",
-        "ARCHS": "AR"
+        "ARCHS": "AR",
     ]
     /// Corresponds to EnvironmentFingerprintGenerator.version
     private static let currentVersion = "5"

Tests/XCRemoteCacheTests/Network/Authentication/AWSV4SignatureTest.swift

@@ -35,10 +35,12 @@ class AWSV4SignatureTest: XCTestCase {
 
         let key = "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY"
         let accessKey = "AKIDEXAMPLE"
+        let securityToken = "IQoJb3JpZ2luX2VjENv//////////wEaCXVzLWVhc3Q+bsHwqnovXtl/1JVe61XHMnAw3AIXwOAOxqMvhw=="
 
         AWSV4Signature(
             secretKey: key,
             accessKey: accessKey,
+            securityToken: securityToken,
             region: "us-east-1",
             service: "iam",
             date: Date(timeIntervalSince1970: 1_440_938_160)
@@ -49,8 +51,8 @@ class AWSV4SignatureTest: XCTestCase {
     func testAuthHeaderContainsCorrectSignature() throws {
         XCTAssertEqual(
             "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/iam/aws4_request, " +
-                "SignedHeaders=content-type;host;x-amz-content-sha256;x-amz-date, " +
-                "Signature=dd479fa8a80364edf2119ec24bebde66712ee9c9cb2b0d92eb3ab9ccdc0c3947",
+                "SignedHeaders=content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token, " +
+                "Signature=d26ab974bba1b248f041ea1120064e1fa672d6f06cac2cff42b38acea87b76e5",
             request.allHTTPHeaderFields?["Authorization"]
         )
     }

Tests/XCRemoteCacheTests/Network/NetworkClientImplTests.swift

@@ -203,6 +203,7 @@ class NetworkClientImplTests: XCTestCase {
         let signature = AWSV4Signature(
             secretKey: "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY",
             accessKey: "AKIDEXAMPLE",
+            securityToken: "IQoJb3JpZ2luX2VjENv//////////wEaCXVzLWVhc3Q+bsHwqnovXtl/1JVe61XHMnAw3AIXwOAOxqMvhw==",
             region: "us-east-1",
             service: "iam",
             date: Date(timeIntervalSince1970: 1_440_938_160)
@@ -213,8 +214,8 @@ class NetworkClientImplTests: XCTestCase {
 
         XCTAssertEqual(
             "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/iam/aws4_request, " +
-                "SignedHeaders=host;x-amz-content-sha256;x-amz-date, " +
-                "Signature=acdb223475463b6ce711b063f199387a7a12bd638e4dccaaeb5efcbf159b6454",
+                "SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-security-token, " +
+                "Signature=e5578464567fb97fd26e871702e4ec4ff7d61cb87eb72a40d22b80e12da30c34",
             try requests[0].allHTTPHeaderFields?["Authorization"].unwrap()
         )
     }

Tests/XCRemoteCacheTests/TestDoubles/OverlayReaderFake.swift

@@ -22,7 +22,7 @@ import Foundation
 
 class OverlayReaderFake: OverlayReader {
     var mappings: [OverlayMapping]
-    
+
     init(mappings: [OverlayMapping]) {
         self.mappings = mappings
     }

cocoapods-plugin/lib/cocoapods-xcremotecache/command/hooks.rb

@@ -146,8 +146,10 @@ module CocoapodsXCRemoteCacheModifier
           prebuild_script.dependency_file = "$(TARGET_TEMP_DIR)/prebuild.d"
 
           # Move prebuild (last element) to the position before compile sources phase (to make it real 'prebuild')
-          compile_phase_index = target.build_phases.index(target.source_build_phase)
-          target.build_phases.insert(compile_phase_index, target.build_phases.delete(prebuild_script))
+          if !existing_prebuild_script 
+            compile_phase_index = target.build_phases.index(target.source_build_phase)
+            target.build_phases.insert(compile_phase_index, target.build_phases.delete(prebuild_script))
+          end
         elsif mode == 'producer' || mode == 'producer-fast'
           # Delete existing prebuild build phase (to support switching between modes)
           target.build_phases.delete_if do |phase|
@@ -180,7 +182,7 @@ module CocoapodsXCRemoteCacheModifier
             end
           end
           mark_script = existing_mark_script || target.new_shell_script_build_phase("[XCRC] Mark")
-          mark_script.shell_script = "\"$SCRIPT_INPUT_FILE_0\" mark --configuration $CONFIGURATION --platform $PLATFORM_NAME"
+          mark_script.shell_script = "\"$SCRIPT_INPUT_FILE_0\" mark --configuration \"$CONFIGURATION\" --platform $PLATFORM_NAME"
           mark_script.input_paths = ["$SRCROOT/#{srcroot_relative_xc_location}/xcprepare"]
         else
           # Delete existing mark build phase (to support switching between modes or changing the final target)

cocoapods-plugin/lib/cocoapods-xcremotecache/gem_version.rb

@@ -13,5 +13,5 @@
 # limitations under the License.
 
 module CocoapodsXcremotecache
-  VERSION = "0.0.6"
+  VERSION = "0.0.9"
 end

docs/Development.md

@@ -23,6 +23,34 @@ If you prefer to edit in Xcode, run `swift package generate-xcodeproj`. Do **not
 
 The generated Xcode project contains schemes for each output application (like `xcswiftc`, `xcprebuild` etc.) so to build a single app, just select the appropriate scheme and build (⌘+B). If you want to build all applications at once, select `Aggregator` scheme that automatically builds all apps. `Aggregator` target in `Package.swift` is defined only for development convenience, it shouldn't be ever used as a dependency.   
 
+#### Debugging the app in a real project
+
+Debugging XCRemoteCache in a real project is simple:
+* Open XCRemoteCache's `Package.swift` in Xcode and select a scheme that corresponds to the process you want to debug, e.g. `xcpostbuild`
+
+![Select scheme](./img/debug-scheme.png)
+
+* Build the app with Product->Build (or ⌘+B)
+* Open the scheme's settings (⌘+⇧+<) and for the "Run" action, select "Wait for the executable to be launched"
+
+![Configure scheme](./img/debug-scheme-wait.png)
+
+* Find the produced binary in your DerivedData. The default location would be `~/Library/Developer/Xcode/DerivedData/XCRemoteCache-{hash}/Build/Products/Debug/xcpostbuild`
+* In the project you want to debug, pick the target you want to inspect and expand the build phase that calls the process
+* Replace the Input Files path so it points the locally built binary placed in DerivedData
+
+![Update phase](./img/debug-phase-update.png)
+
+* Now, you can run the **XCRemoteCache** project (not the project you want to debug). Because Xcode will not initiate a new process, it will just wait until someone else triggers it.
+
+![LLDB waiting](./img/debug-wait.png)
+
+* Finally, build the project to want to debug and expect your XCRemoteCache breakpoints to hit
+
+![Breakpoint hit](./img/debug-breakpoint.png)
+
+> Tip: If your breakpoints don't hit, try adding a dummy `sleep(1)` in the process entry point (e.g. at the top of `XCPostbuild.main` function)  
+
 #### Running tests in Xcode
 
 All unit tests are placed in `XCRemoteCacheTests`. To run them from Xcode, just pick any application scheme and run tests (⌘+U).

docs/FAQ.md

@@ -50,3 +50,18 @@ log show --predicate 'sender == "xcprepare"' --style compact --info --debug -las
 log show --predicate 'sender BEGINSWITH "xc"' --style compact --info --debug -last 10m
 ```
 </details>
+
+### Troubleshooting cache misses
+
+Here is a non-exhaustive list of steps that may help with troubleshooting poor cache hit rate.
+
+1. ***Producer&Consumer:*** Review XCRemoteCache [Requirements](../#Requirements) and [Limitations](../#limitations)
+1. ***Producer&Consumer:*** Make sure a producer build uses the same architecture(s) as a consumer. You can inspect `ARCHS` Build Setting in Xcode's Script Phase output logs. Navigate to the report navigator (⌘+9) and expand XCRemoteCache's `prebuild` step output using the "collapsed menu icon" (aka hamburger menu)
+1. ***Producer:*** Verify that all Xcode targets have a Build Phase called `postbuild`
+1. ***Producer:*** If you are using optional XCRemoteCache auto-marking feature (`--final-producer-target` or `final_target`) verify an extra Build Phase called `mark` is added to the specified target
+1. ***Producer:*** After a full build, review logs according to [docs](#how-can-i-find-xcremotecache-logs)
+1. ***Consumer:*** Verify that all Xcode targets have extra XCRemoteCache Build Phase called `prebuild` and `postbuild`
+1. ***Consumer:*** After a full build, review according to [docs](#how-can-i-find-xcremotecache-logs). Find a ***first:*** target that reports a cache miss with a message like `Prebuild step failed with error: ...`. If a target reports faces a cache miss, it may have a knock-on effect where a lot of its consumers (dependant targets) need to be built locally too
+1. ***Consumer:*** ***After a full build, review all meta files placed in `~/Library/Caches/XCRemoteCache/{your_host_path}/meta/*.json` and make sure no absolute paths are used in its `dependencies`. All paths should start a placeholder, like `$(SRCROOT)` or `$(BUILD_DIR)`***
+1. ***Consumer:*** If you are integrating XCRemoteCache and rebuild artifacts for the same sha, previously downloaded artifacts placed in a local cache may still be used on a consumer side. You can either manually delete your local cache at `~/Library/Caches/XCRemoteCache/` before any consumer build or disable a local cache with `artifact_maximum_age: 0` property in `.rcinfo`
+1. ***Consumer:*** To find an actual cache hit, before building in Xcode reset statistics with `xcprepare stats --reset` and once it is done, call `xcprepare stats` to find a cache hit rate

e2eTests/XCRemoteCacheSample/XCRemoteCacheSample/AppDelegate.swift

@@ -23,7 +23,6 @@ import UIKit
 class AppDelegate: UIResponder, UIApplicationDelegate {
 
 
-
     func application(_ application: UIApplication, didFinishLaunchingWithOptions launchOptions: [UIApplication.LaunchOptionsKey: Any]?) -> Bool {
         return true
     }
@@ -38,4 +37,3 @@ class AppDelegate: UIResponder, UIApplicationDelegate {
     }
 
 }
-

e2eTests/XCRemoteCacheSample/XCRemoteCacheSample/SceneDelegate.swift

@@ -41,4 +41,3 @@ class SceneDelegate: UIResponder, UIWindowSceneDelegate {
     func sceneDidEnterBackground(_ scene: UIScene) {
     }
 }
-

e2eTests/XCRemoteCacheSample/XCRemoteCacheSample/ViewController.swift

@@ -27,4 +27,3 @@ class ViewController: UIViewController {
 
 
 }
-