Merge pull request #199 from polac24/add-graceful-missing-common-sha

Add support for graceful handling missing common sha

.github/ISSUE_TEMPLATE/BUG_REPORT.md

@@ -0,0 +1,65 @@
+---
+name: ⚠️ Bug Report
+about: Something isn't working as expected
+
+---
+
+<!--
+PLEASE HELP US PROCESS GITHUB ISSUES FASTER BY PROVIDING THE FOLLOWING INFORMATION.
+-->
+
+**My integration setup**
+
+[ ] CocoaPods cocoapods-xcremotecache plugin
+[ ] Automatic integration using `xcprepare integrate ...`
+[ ] Manual integration
+[ ] Carthage
+
+**Expected/desired behavior**
+<!-- Describe what the desired behavior would be. -->
+
+**Minimal reproduction of the problem with instructions**
+<!-- Please provide the *STEPS TO REPRODUCE*. -->
+
+**Producer Logs**
+<!-- Capture logs from 10 minutes: `log show --predicate 'sender BEGINSWITH "xc"' --style compact --info --debug -last 10m` -->
+
+<details>
+  <pre> [REPLACE THIS WITH YOUR INFORMATION] </pre>
+</details>
+
+**Consumer Logs**
+<!-- Capture logs from 10 minutes: `log show --predicate 'sender BEGINSWITH "xc"' --style compact --info --debug -last 10m` -->
+
+<details>
+  <pre> [REPLACE THIS WITH YOUR INFORMATION] </pre>
+</details>
+
+**Pods/Carthage file**
+<!-- Delete if you don't use CocoaPods or Carthage -->
+
+<details>
+  <pre> [REPLACE THIS WITH YOUR INFORMATION] </pre>
+</details>
+
+**Environment**
+
+* **XCRemoteCache:** X.Y.Z
+* **cocoapods-xcremotecache:** X.Y.Z <!-- check with `gem list cocoapods-xcremotecache` >
+* **HTTP cache server:** ... <!-- e.g. demo docker, nginx, AWS etc. >
+* **Xcode:** X.Y.Z
+
+**Post build stats**
+<!-- 
+To capture build statistics: 
+* call `xcprepare stats --reset` (or `XCRC/xcprepare stats --reset` for CocoaPods)
+* Build a project in Xcode
+* `xcprepare stats` (or `XCRC/xcprepare stats` for CocoaPods) 
+-->
+
+<details>
+  <pre> [REPLACE THIS WITH YOUR INFORMATION] </pre>
+</details>
+
+**Others**
+<!-- Anything else relevant?  Operating system version, , ... -->

.github/ISSUE_TEMPLATE/DOCUMENTATION_ISSUE.md

@@ -0,0 +1,16 @@
+---
+name: 📕 Documentation Issue
+about: Suggestion for a change in a documentation
+
+---
+
+<!--
+PLEASE HELP US PROCESS GITHUB ISSUES FASTER BY PROVIDING THE FOLLOWING INFORMATION.
+-->
+
+**A suggestion**
+<!-- Describe how could the documentation be improved. -->
+
+**Which file, section, line**
+<!-- Provide a section it relates (if exist). -->
+

.github/ISSUE_TEMPLATE/FEATURE_REQUEST.md

@@ -0,0 +1,20 @@
+---
+name: 🙏 Future Request
+about: Suggestion for an improvement, either behaviour or implementation
+
+---
+
+<!--
+PLEASE HELP US PROCESS GITHUB ISSUES FASTER BY PROVIDING THE FOLLOWING INFORMATION.
+-->
+
+
+**Expected/desired behavior**
+<!-- Describe what the desired behavior would be. -->
+
+**Relevant integration setup**
+
+[ ] CocoaPods cocoapods-xcremotecache plugin
+[ ] Automatic integration using `xcprepare integrate ...`
+[ ] Manual integration
+[ ] Carthage

.github/workflows/ci.yaml

@@ -9,11 +9,13 @@ jobs:
       - uses: actions/checkout@v1
       - name: SwiftLint
         uses: norio-nomura/action-swiftlint@3.1.0
+        with:
+          args: --strict
 
   macOS:
-    runs-on: macOS-latest
+    runs-on: macos-12
     env:
-      XCODE_VERSION: ${{ '12.4' }}
+      XCODE_VERSION: ${{ '14.2' }}
     steps:
       - name: Select Xcode
         run: "sudo xcode-select -s /Applications/Xcode_$XCODE_VERSION.app"
@@ -23,3 +25,5 @@ jobs:
         run: rake build[release]
       - name: Test
         run: rake test
+      - name: E2ETests
+        run: rake e2e_only

.github/workflows/docs.yaml

@@ -0,0 +1,24 @@
+name: Docs
+
+on:
+  push:
+    branches:
+      - master
+
+jobs:
+  docs:
+    runs-on: macos-12
+    env:
+      XCODE_VERSION: ${{ '14.2' }}
+    steps:
+      - uses: actions/checkout@v2
+      - name: Select Xcode
+        run: "sudo xcode-select -s /Applications/Xcode_$XCODE_VERSION.app"
+      - name: "Generate documentation"
+        run: "swift package --allow-writing-to-directory ./docs  generate-documentation --target XCRemoteCache --disable-indexing --transform-for-static-hosting --output-path ./docs  --hosting-base-path XCRemoteCache/"
+      - name: Deploy GH-pages
+        uses: peaceiris/actions-gh-pages@v3
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          publish_dir: ./docs
+          keep_files: false

.github/workflows/release.yaml

@@ -6,9 +6,9 @@ on:
 jobs:
   macOS:
     name: Add macOS binaries to release
-    runs-on: macOS-latest
+    runs-on: macos-12
     env:
-      XCODE_VERSION: ${{ '12.4' }}
+      XCODE_VERSION: ${{ '14.2' }}
     steps:
       - name: Select Xcode
         run: "sudo xcode-select -s /Applications/Xcode_$XCODE_VERSION.app"
@@ -44,3 +44,29 @@ jobs:
           file_glob: true
           tag: ${{ github.ref }}
           overwrite: true
+  cocoapods:
+    name: Publish CocoaPods plugin
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    defaults:
+      run:
+        working-directory: cocoapods-plugin
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: '3.0'
+      - run: bundle install
+      - name: Publish to RubyGems
+        run: |
+          mkdir -p $HOME/.gem
+          touch $HOME/.gem/credentials
+          chmod 0600 $HOME/.gem/credentials
+          printf -- "---\n:rubygems_api_key: ${GEM_HOST_API_KEY}\n" > $HOME/.gem/credentials
+          gem build *.gemspec
+          CURRENT_VERSION=$(gem list cocoapods-xcremotecache  --remote  -q | sed 's/[^0-9\.]//g')
+          [ -f cocoapods-xcremotecache-$CURRENT_VERSION.gem ] && echo "Version $CURRENT_VERSION already exists" || gem push *.gem
+        env:
+          GEM_HOST_API_KEY: "${{secrets.RUBYGEMS_AUTH_TOKEN}}"

.gitignore

@@ -1,8 +1,13 @@
 .DS_Store
 /.build
 /Packages
-/*.xcodeproj
+*.xcodeproj/
+*.xcworkspace/
 DerivedData
 /.swiftpm/
 releases
-tmp/
+tmp/
+.idea/
+xcuserdata
+*.gem
+Pods/

.swiftlint.yml

@@ -63,6 +63,8 @@ excluded:
   - docs/
   - fastlane/
   - DerivedData/
+  - e2eTests/XCRemoteCacheSample/Pods
+  - e2eTests/StandaloneSampleApp
 
 attributes:
   always_on_same_line:
@@ -87,6 +89,25 @@ file_header:
                      \/\/  Created by .*? on .*\.
                      \/\/  Copyright © \d{4} .*\. All rights reserved\.
                      \/\/
+  required_pattern: |
+                     \/\/ Copyright \(c\) \d{4} Spotify AB\.
+                     \/\/
+                     \/\/ Licensed to the Apache Software Foundation \(ASF\) under one
+                     \/\/ or more contributor license agreements\.  See the NOTICE file
+                     \/\/ distributed with this work for additional information
+                     \/\/ regarding copyright ownership\.  The ASF licenses this file
+                     \/\/ to you under the Apache License, Version 2.0 \(the
+                     \/\/ "License"\); you may not use this file except in compliance
+                     \/\/ with the License\.  You may obtain a copy of the License at
+                     \/\/
+                     \/\/ http:\/\/www.apache.org\/licenses\/LICENSE-2\.0
+                     \/\/
+                     \/\/ Unless required by applicable law or agreed to in writing,
+                     \/\/ software distributed under the License is distributed on an
+                     \/\/ \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+                     \/\/ KIND, either express or implied\.  See the License for the
+                     \/\/ specific language governing permissions and limitations
+                     \/\/ under the License\.
 force_cast: warning
 force_try: warning
 implicit_getter: warning
@@ -123,6 +144,6 @@ custom_rules:
     severity: warning
   trailing_dot_in_comments:
     name: "Trailing dot in comments"
-    regex: '^[ ]*///?[^\n]*\.\n'
+    regex: '^(?!\/\/\ Copyright\ \(c\)\ \d{4}\ Spotify AB\.|\/\/\ under\ the\ License\.)[ ]*///?[^\n]*\.\n'
     message: "There shouldn't be trailing dot in comments"
     severity: warning

Package.resolved

@@ -3,16 +3,16 @@
     "pins": [
       {
         "package": "AEXML",
-        "repositoryURL": "https://github.com/tadija/AEXML",
+        "repositoryURL": "https://github.com/tadija/AEXML.git",
         "state": {
           "branch": null,
-          "revision": "8623e73b193386909566a9ca20203e33a09af142",
-          "version": "4.5.0"
+          "revision": "38f7d00b23ecd891e1ee656fa6aeebd6ba04ecc3",
+          "version": "4.6.1"
         }
       },
       {
         "package": "PathKit",
-        "repositoryURL": "https://github.com/kylef/PathKit",
+        "repositoryURL": "https://github.com/kylef/PathKit.git",
         "state": {
           "branch": null,
           "revision": "73f8e9dca9b7a3078cb79128217dc8f2e585a511",
@@ -37,13 +37,22 @@
           "version": "0.0.5"
         }
       },
+      {
+        "package": "SwiftDocCPlugin",
+        "repositoryURL": "https://github.com/apple/swift-docc-plugin",
+        "state": {
+          "branch": null,
+          "revision": "3303b164430d9a7055ba484c8ead67a52f7b74f6",
+          "version": "1.0.0"
+        }
+      },
       {
         "package": "XcodeProj",
         "repositoryURL": "https://github.com/tuist/XcodeProj.git",
         "state": {
           "branch": null,
-          "revision": "0b18c3e7a10c241323397a80cb445051f4494971",
-          "version": "8.0.0"
+          "revision": "fae27b48bc14ff3fd9b02902e48c4665ce5a0793",
+          "version": "8.9.0"
         }
       },
       {
@@ -51,8 +60,8 @@
         "repositoryURL": "https://github.com/jpsim/Yams.git",
         "state": {
           "branch": null,
-          "revision": "53741ba55ecca5c7149d8c9f810913ec80845c69",
-          "version": "3.0.0"
+          "revision": "00c403debcd0a007b854bb35e598466207a2d58c",
+          "version": "5.0.0"
         }
       },
       {
@@ -60,8 +69,8 @@
         "repositoryURL": "https://github.com/marmelroy/Zip.git",
         "state": {
           "branch": null,
-          "revision": "80b1c3005ee25b4c7ce46c4029ac3347e8d5e37e",
-          "version": "2.0.0"
+          "revision": "67fa55813b9e7b3b9acee9c0ae501def28746d76",
+          "version": "2.1.2"
         }
       }
     ]

Package.swift

@@ -1,4 +1,5 @@
-// swift-tools-version:5.1
+// swift-tools-version:5.3
+// swiftlint:disable:previous file_header
 // The swift-tools-version declares the minimum version of Swift required to build this package
 
 import PackageDescription
@@ -12,10 +13,11 @@ let package = Package(
         .executable(name: "xcprebuild", targets: ["xcprebuild"]),
     ],
     dependencies: [
-        .package(url: "https://github.com/marmelroy/Zip.git", from: "2.0.0"),
-        .package(url: "https://github.com/jpsim/Yams.git", from: "3.0.0"),
+        .package(url: "https://github.com/marmelroy/Zip.git", from: "2.1.2"),
+        .package(url: "https://github.com/jpsim/Yams.git", from: "5.0.0"),
         .package(url: "https://github.com/apple/swift-argument-parser", from: "0.0.1"),
-        .package(url: "https://github.com/tuist/XcodeProj.git", from: "8.0.0"),
+        .package(url: "https://github.com/tuist/XcodeProj.git", from: "8.9.0"),
+        .package(url: "https://github.com/apple/swift-docc-plugin", from: "1.0.0"),
     ],
     targets: [
         .target(
@@ -30,8 +32,16 @@ let package = Package(
             name: "xcswiftc",
             dependencies: ["XCRemoteCache"]
         ),
+        .target(
+            name: "xclibtoolSupport",
+            dependencies: ["XCRemoteCache"]
+        ),
         .target(
             name: "xclibtool",
+            dependencies: ["XCRemoteCache", "xclibtoolSupport"]
+        ),
+        .target(
+            name: "xclipo",
             dependencies: ["XCRemoteCache"]
         ),
         .target(
@@ -49,14 +59,32 @@ let package = Package(
             name: "xcld",
             dependencies: ["XCRemoteCache"]
         ),
+        .target(
+            name: "xcldplusplus",
+            dependencies: ["XCRemoteCache"]
+        ),
         .target(
             // Wrapper target that builds all binaries but does nothing in runtime
             name: "Aggregator",
-            dependencies: ["xcprebuild", "xcswiftc", "xclibtool", "xcpostbuild", "xcprepare", "xcld"]
+            dependencies: [
+                "xcprebuild",
+                "xcswiftc",
+                "xclibtool",
+                "xcpostbuild",
+                "xcprepare",
+                "xcld",
+                "xcldplusplus",
+                "xclipo",
+            ]
         ),
         .testTarget(
             name: "XCRemoteCacheTests",
-            dependencies: ["XCRemoteCache"]
+            dependencies: ["XCRemoteCache"],
+            resources: [.copy("TestData")]
+        ),
+        .testTarget(
+            name: "xclibtoolSupportTests",
+            dependencies: ["xclibtoolSupport"]
         ),
     ]
 )

README.md

@@ -1,13 +1,16 @@
 <p align="center">
-    <img src="docs/img/logo.png" width="75%">
+    <img src="docs/img/logo.png#gh-light-mode-only" width="75%">
+    <img src="docs/img/logo-dark.png#gh-dark-mode-only" width="75%">
 </p>
 
 _XCRemoteCache is a remote cache tool for Xcode projects. It reuses target artifacts generated on a remote machine, served from a simple REST server._
 
 [![Build Status](https://github.com/spotify/XCRemoteCache/workflows/CI/badge.svg)](https://github.com/spotify/XCRemoteCache/workflows/CI/badge.svg)
 [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](LICENSE)
+[![Slack](https://slackin.spotify.com/badge.svg)](https://slackin.spotify.com)
+[![Docs](https://github.com/spotify/XCRemoteCache/workflows/Docs/badge.svg)](https://spotify.github.io/XCRemoteCache/documentation/xcremotecache/)
 
-- [How and Why?](#how-and-why-)
+- [How and Why?](#how-and-why)
   * [Accurate target input files](#accurate-target-input-files)
     + [New file added to the target](#new-file-added-to-the-target)
   * [Debug symbols](#debug-symbols)
@@ -36,10 +39,14 @@ _XCRemoteCache is a remote cache tool for Xcode projects. It reuses target artif
   * [Amazon S3 and Google Cloud Storage](#amazon-s3-and-google-cloud-storage)
 - [CocoaPods plugin](#cocoapods-plugin)
 - [Requirements](#requirements)
+- [Apple silicon support](#apple-silicon-support)
+  * [Artifacts per architecture (Recommended)](#artifacts-per-architecture-recommended)
+  * [Fat artifacts](#fat-artifacts)
 - [Limitations](#limitations)
 - [FAQ](#faq)
 - [Development](#development)
 - [Release](#release)
+  * [Releasing CocoaPods plugin](#releasing-cocoapods-plugin)
   * [Building release package](#building-release-package)
 - [Contributing](#contributing)
 - [Code of conduct](#code-of-conduct)
@@ -50,7 +57,7 @@ _XCRemoteCache is a remote cache tool for Xcode projects. It reuses target artif
 
 The caching mechanism is based on remote artifacts that should be generated and uploaded to the cache server for each commit on a `master` branch, preferably as a part of CI/CD step. Xcode products are not portable between different Xcode versions, each XCRemoteCache artifact is linked with a specific Xcode build number that generated it. To support multiple Xcode versions, artifacts generation should happen for each Xcode version.
 
-The artifact resue flow is as follows: XCRemoteCache performs a target precheck (aka prebuild) and if a fingerprint for local sources matches the one computed on a generation side, several compilation steps wrappers (e.g. `xcswiftc`, `xccc`, `xclibtool`) mock corresponding compilation step(s) and linking (or archiving) moves the cached build artifact to the expected location.
+The artifact reuse flow is as follows: XCRemoteCache performs a target precheck (aka prebuild) and if a fingerprint for local sources matches the one computed on a generation side, several compilation steps wrappers (e.g. `xcswiftc`, `xccc`, `xclibtool`) mock corresponding compilation step(s) and linking (or archiving) moves the cached build artifact to the expected location.
 
 > Multiple commits that have the same target sources reuse artifact package on a remote server.
 
@@ -134,7 +141,7 @@ xcremotecache/xcprepare integrate --input <yourProject.xcodeproj> --mode consume
 | Argument | Description | Default | Required |
 | ------------- | ------------- | ------------- | ------------- |
 | `--input` | .xcodeproj location  | N/A | ✅ |
-| `--mode` | mode. Supported values: `consumer`, `producer` | N/A | ✅ |
+| `--mode` | mode. Supported values: `consumer`, `producer`, `producer-fast`(experimental) | N/A | ✅ |
 | `--targets-include` | comma-separated list of targets to integrate XCRemoteCache. | `""` | ⬜️ |
 | `--targets-exclude` | comma-separated list of targets to not integrate XCRemoteCache. Takes priority over --targets-include.  | `""` | ⬜️ |
 | `--configurations-include` | comma-separated list of configurations to integrate XCRemoteCache.  | `""` | ⬜️ |
@@ -145,6 +152,7 @@ xcremotecache/xcprepare integrate --input <yourProject.xcodeproj> --mode consume
 | `--lldb-init` | LLDBInit mode. Appends to .lldbinit a command required for debugging. Supported values: 'none' (do not append to .lldbinit), 'user' (append to ~/.lldbinit) | `user` | ⬜️ |
 | `--fake-src-root` | An arbitrary source location shared between producers and consumers. Should be unique for a project. | `/xxxxxxxxxx` | ⬜️ |
 | `--output` | Save the project with integrated XCRemoteCache to a separate location.  | N/A | ⬜️ |
+| `--sdks-exclude` | comma separated list of sdks to not integrate XCRemoteCache (e.g. "watchos*, watchsimulator*"). (Experimental)  | `""` | ⬜️ |
 
 </details>
 
@@ -188,7 +196,11 @@ Configure Xcode targets that **should use** XCRemoteCache:
 * `CC` - `xccc_file` from your `.rcinfo` configuration (e.g. `xcremotecache/xccc`)
 * `SWIFT_EXEC` - location of `xcprepare` (e.g. `xcremotecache/xcswiftc`)
 * `LIBTOOL` - location of `xclibtool` (e.g. `xcremotecache/xclibtool`)
+* `LIPO` - location of `xclipo` (e.g. `xcremotecache/xclipo`)
 * `LD` - location of `xcld` (e.g. `xcremotecache/xcld`)
+* `LDPLUSPLUS` - location of `xcldplusplus` (e.g. `xcremotecache/xcldplusplus`)
+* `XCRC_PLATFORM_PREFERRED_ARCH` - `$(LINK_FILE_LIST_$(CURRENT_VARIANT)_$(PLATFORM_PREFERRED_ARCH):dir:standardizepath:file:default=arm64)`
+* `SWIFT_USE_INTEGRATED_DRIVER` - `NO` (required in Xcode 14.0+)
 
 <details>
   <summary>Screenshot</summary>
@@ -208,8 +220,8 @@ Configure Xcode targets that **should use** XCRemoteCache:
 * command: `"$SCRIPT_INPUT_FILE_0"`
 * input files: location of `xcpostbuild` command (e.g. `xcremotecache/xcpostbuild`)
 * output files: 
-  * `$(TARGET_BUILD_DIR)/$(MODULES_FOLDER_PATH)/$(PRODUCT_MODULE_NAME).swiftmodule/$(PLATFORM_PREFERRED_ARCH).swiftmodule.md5`
-  * `$(TARGET_BUILD_DIR)/$(MODULES_FOLDER_PATH)/$(PRODUCT_MODULE_NAME).swiftmodule/$(PLATFORM_PREFERRED_ARCH)-$(LLVM_TARGET_TRIPLE_VENDOR)-$(SWIFT_PLATFORM_TARGET_PREFIX)$(LLVM_TARGET_TRIPLE_SUFFIX).swiftmodule.md5`
+  * `$(TARGET_BUILD_DIR)/$(MODULES_FOLDER_PATH)/$(PRODUCT_MODULE_NAME).swiftmodule/$(XCRC_PLATFORM_PREFERRED_ARCH).swiftmodule.md5`
+  * `$(TARGET_BUILD_DIR)/$(MODULES_FOLDER_PATH)/$(PRODUCT_MODULE_NAME).swiftmodule/$(XCRC_PLATFORM_PREFERRED_ARCH)-$(LLVM_TARGET_TRIPLE_VENDOR)-$(SWIFT_PLATFORM_TARGET_PREFIX)$(LLVM_TARGET_TRIPLE_SUFFIX).swiftmodule.md5`
 * discovery dependency file: `$(TARGET_TEMP_DIR)/postbuild.d`
 
 <details>
@@ -257,7 +269,42 @@ $ xcremotecache/xcprepare mark --configuration Debug --platform iphonesimulator
 
 That command creates an empty file on a remote server which informs that for given sha, configuration, platform, Xcode versions etc. all artifacts are available.
 
-_Note that for the `producer` mode, the prebuild build phase and `xccc`, `xcld`, `xclibtool` wrappers become no-op, so it is recommended to not add them for the `producer` mode._
+_Note that for the `producer` mode, the prebuild build phase and `xccc`, `xcld`, `xcldplusplus`, `xclibtool`, `xclipo` wrappers become no-op, so it is recommended to not add them for the `producer` mode._
+
+##### 7. Generalize `-Swift.h` (Optional only if using static library with a bridging header with public `NS_ENUM` exposed from ObjC)
+
+If a static library target contains a mixed target with a bridging header exposing an enum from ObjC in a public Swift API, your custom script that moves `*-Swift.h` to the shared location, it should also move `*-Swift.h.md5` next to it.
+
+Example:
+
+##### Existing script (Before):
+
+```shell
+ditto "${SCRIPT_INPUT_FILE_0}" "${SCRIPT_OUTPUT_FILE_0}"
+```
+
+where
+* `SCRIPT_INPUT_FILE_0="$(DERIVED_SOURCES_DIR)/$(SWIFT_OBJC_INTERFACE_HEADER_NAME)"`
+* `SCRIPT_OUTPUT_FILE_0="$(BUILT_PRODUCTS_DIR)/include/$(PRODUCT_MODULE_NAME)/$(SWIFT_OBJC_INTERFACE_HEADER_NAME)"`
+
+##### Correct script (After):
+
+```shell
+ditto "${SCRIPT_INPUT_FILE_0}" "${SCRIPT_OUTPUT_FILE_0}"
+[ -f "${SCRIPT_INPUT_FILE_1}" ] && ditto "${SCRIPT_INPUT_FILE_1}" "${SCRIPT_OUTPUT_FILE_1}" || rm "${SCRIPT_OUTPUT_FILE_1}"
+```
+
+where
+* `SCRIPT_INPUT_FILE_0="$(DERIVED_SOURCES_DIR)/$(SWIFT_OBJC_INTERFACE_HEADER_NAME)"`
+* `SCRIPT_INPUT_FILE_1="$(DERIVED_SOURCES_DIR)/$(SWIFT_OBJC_INTERFACE_HEADER_NAME).md5"`
+* `SCRIPT_OUTPUT_FILE_0="$(BUILT_PRODUCTS_DIR)/include/$(PRODUCT_MODULE_NAME)/$(SWIFT_OBJC_INTERFACE_HEADER_NAME)"`
+* `SCRIPT_OUTPUT_FILE_1="$(BUILT_PRODUCTS_DIR)/include/$(PRODUCT_MODULE_NAME)/$(SWIFT_OBJC_INTERFACE_HEADER_NAME).md5"`
+
+Note: This step is not required if at least one of these is true:
+
+* you build a framework (not a static library)
+* you don't expose `NS_ENUM` type from ObjC to Swift via a bridging header
+
 
 ## A full list of configuration parameters:
 
@@ -281,13 +328,15 @@ _Note that for the `producer` mode, the prebuild build phase and `xccc`, `xcld`,
 | `cache_commit_history` | Number of historical git commits to look for cache artifacts | `10` | ⬜️ |
 | `source_root` | Source root of the Xcode project | `""` | ⬜️ |
 | `fingerprint_override_extension` | Fingerprint override extension (sample override `Module.swiftmodule/x86_64.swiftmodule.md5`) | `md5` | ⬜️ |
-| `extra_configuration_file` | Configuration file that overrides project configuration | `user.rcinfo` | ⬜️ |
+| `extra_configuration_file` | Configuration file that overrides project configuration (this property can be overriden multiple times in different files to chain extra configuration files) | `user.rcinfo` | ⬜️ |
 | `publishing_sha` | Custom commit sha to publish artifact (producer only) | `nil` | ⬜️ |
 | `artifact_maximum_age` | Maximum age in days HTTP response should be locally cached before being evicted | `30` | ⬜️ |
 | `custom_fingerprint_envs` | Extra ENV keys that should be convoluted into the environment fingerprint | `[]` | ⬜️ |
 | `stats_dir` | Directory where all XCRemoteCache statistics (e.g. counters) are stored | `~/.xccache` | ⬜️ |
 | `download_retries` | Number of retries for download requests | `0` | ⬜️ |
 | `upload_retries` | Number of retries for upload requests | `3` | ⬜️ |
+| `retry_delay` | Delay between retries in seconds | `10` | ⬜️ |
+| `upload_batch_size` | Maximum number of simultaneous requests. 0 means no limits | `0` | ⬜️ |
 | `request_custom_headers` | Dictionary of extra HTTP headers for all remote server requests | `[]` | ⬜️ |
 | `thin_target_mock_filename` | Filename (without an extension) of the compilation input file that is used as a fake compilation for the forced-cached target (aka thin target) | `standin` | ⬜️ |
 | `focused_targets` | A list of all targets that are not thinned. If empty, all targets are meant to be non-thin | `[]` | ⬜️ |
@@ -298,11 +347,18 @@ _Note that for the `producer` mode, the prebuild build phase and `xccc`, `xcld`,
 | `product_files_extensions_with_content_override ` | List of all extensions that should carry over source fingerprints. Extensions of all product files that contain non-deterministic content (absolute paths, timestamp, etc) should be included. | `["swiftmodule"]` | ⬜️ |
 | `thinning_enabled ` | If true, support for thin projects is enabled | `false` | ⬜️ |
 | `thinning_target_module_name ` | Module name of a target that works as a helper for thinned targets | `"ThinningRemoteCacheModule"` | ⬜️ |
+| `prettify_meta_files` | A Boolean value that opts-in pretty JSON formatting for meta files | `false` | ⬜️ |
 | `aws_secret_key` | Secret key for AWS V4 Signature Authorization. If this is set to a non-empty String - an Authentication Header will be added based on this and the other `aws_*` parameters.| `""` | ⬜️ |
 | `aws_access_key` | Access key for AWS V4 Signature Authorization. | `""` | ⬜️ |
+| `aws_security_token` | Temporary security token provided by the AWS Security Token Service. | `nil` | ⬜️ |
 | `aws_region` | Region for AWS V4 Signature Authorization. E.g. `eu`.  | `""` | ⬜️ |
 | `aws_service` | Service for AWS V4 Signature Authorization. E.g. `storage`. | `""` | ⬜️ |
-
+| `out_of_band_mappings` | A dictionary of files path remapping that should be applied to make it absolute path agnostic on a list of dependencies. Useful if a project refers files out of repo root, either compilation files or precompiled dependencies. Keys represent generic replacement and values are substrings that should be replaced. Example: for mapping `["COOL_LIBRARY": "/CoolLibrary"]` `/CoolLibrary/main.swift`will be represented as `$(COOL_LIBRARY)/main.swift`). Warning: remapping order is not-deterministic so avoid remappings with multiple matchings. | `[:]` | ⬜️ |
+| `disable_certificate_verification` | A Boolean value that opts-in SSL certificate validation is disabled | `false` | ⬜️ |
+| `disable_vfs_overlay` | A feature flag to disable virtual file system overlay support (temporary) | `false` | ⬜️ |
+| `custom_rewrite_envs` | A list of extra ENVs that should be used as placeholders in the dependency list. ENV rewrite process is optimistic - does nothing if an ENV is not defined in the pre/postbuild process. | `[]` | ⬜️ |
+| `irrelevant_dependencies_paths` | Regexes of files that should not be included in a list of dependencies. Warning! Add entries here with caution - excluding dependencies that are relevant might lead to a target overcaching. The regex can match either partially or fully the filepath, e.g. `\\.modulemap$` will exclude all `.modulemap` files. | `[]` | ⬜️ |
+| `gracefully_handle_missing_common_sha` | If true, do not fail `prepare` if cannot find the most recent common commits with the primary branch. That might be useful on CI, where a shallow clone is used and cloning depth is not big enough to fetch a commit from a primary branch | `false` | ⬜️ |
 
 ## Backend cache server
 
@@ -340,6 +396,8 @@ XCRemoteCache supports Amazon S3 and Google Cloud Storage buckets to be used as
 
 To set it up use the configuration parameters `aws_secret_key`, `aws_access_key`, `aws_region`, and `aws_service` in the `.rcinfo` file. Specify the URL to the bucket in cache-addresses field in the same file.
 
+XCRemoteCache also supports [AWS Temporary Access Keys](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#temporary-access-keys). Use additional `aws_security_token` parameter combined with `aws_secret_key`, `aws_access_key` to set it up. [This page](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/configuring-instance-metadata-service.html) describes how to receive a security token. 
+
 Example
 ```yaml
 ...
@@ -358,6 +416,31 @@ Retention Policy: Buckets usually have a retention policy option which ensures o
 
 Head over to our [cocoapods-plugin](cocoapods-plugin/README.md) docs to see how to integrate XCRemoteCache in your CocoaPods project.
 
+## Apple silicon support
+
+### Artifacts per architecture (Recommended)
+
+_If all of your machines (both producer and all consumers have the same architecture, either Intel or Apple Silicon), you don't have to do anything._
+
+XCRemoteCache supports building artifacts for Apple silicon consumers. Is it recommended to build separately for `x86_64` and `arm64` architectures to have single-architecture artifacts that do not require downloading irrelevant binaries. Here are required steps if you want to support both Intel and Apple silicon consumers.
+
+* Building for a simulator on a producer: run a first build for `x86_64`, clean a build and build again for `arm64`, e.g.:
+```
+xcodebuild ARCHS=x86_64 ONLY_ACTIVE_ARCH=NO build ...
+xcodebuild clean
+xcodebuild ARCHS=arm64 ONLY_ACTIVE_ARCH=NO build ...
+```
+
+### Fat artifacts
+
+If you prefer to generate far artifacts (with both Intel and Apple silicon binaries), you can disable "Build Archive Architecture Only" on a producer side, e.g.
+
+```
+xcodebuild ONLY_ACTIVE_ARCH=NO build ...
+```
+
+Note: This setup is not recommended and may not be supported in future XCRemoteCache releases.
+
 ## Requirements
 
 * The repo under `git` version control
@@ -368,12 +451,14 @@ Head over to our [cocoapods-plugin](cocoapods-plugin/README.md) docs to see how
 * Recommended: multi-targets Xcode project
 * Recommended: do not use fast-forward PR strategy (use merge or squash instead)
 * Recommended: avoid `DWARF with dSYM File` "Debug Information Format" build setting. Use `DWARF` instead
+* Recommended: avoid having a symbolic link in the source root (e.g. placing a project in `/tmp`) 
 
 ## Limitations
 
-* Swift Package Manager (SPM) projects are not supported
+* Swift Package Manager (SPM) dependencies are not supported. _Because SPM does not allow customizing Build Settings, XCRemoteCache cannot specify `clang` and `swiftc` wrappers that control if the local compilation should be skipped (cache hit) or not (cache miss)_
 * Filenames with `_vers.c` suffix are reserved and cannot be used as a source file
 * All compilation files should be referenced via the git repo root. Referencing `/AbsolutePath/someOther.swift` or `../../someOther.swift` that resolve to the location outside of the git repo root is prohibited.
+* The new Swift driver (introduced by default in Xcode 14.0) is not supported and has to be disabled when using XCRemoteCache
 
 ## FAQ
 
@@ -388,6 +473,12 @@ Follow the [Development](docs/Development.md) guide. It has all the information
 To release a version, in [Releases](https://github.com/spotify/XCRemoteCache/releases) draft a new release with `v0.3.0{-rc0}` tag format. 
 Packages with binaries will be automatically uploaded to the GitHub [Releases](https://github.com/spotify/XCRemoteCache/releases) page.
 
+### Releasing CocoaPods plugin
+
+Bump a gem version defined in [gem_version.rb](cocoapods-plugin/lib/cocoapods-xcremotecache/gem_version.rb) and create a new release described above.
+
+A plugin is automatically uploaded to [RubyGems](https://rubygems.org/gems/cocoapods-xcremotecache) if a given version doesn't exist yet.
+
 ### Building release package
 
 To build a release zip package for a single platform (e.g. `x86_64-apple-macosx`, `arm64-apple-macosx`), call:
@@ -398,6 +489,12 @@ rake 'build[release, x86_64-apple-macosx]'
 
 The zip package will be generated at `releases/XCRemoteCache.zip`.
 
+## Support
+
+Create a [new issue](https://github.com/spotify/XCRemoteCache/issues/new) with as many details as possible.
+
+Reach us at the `#xcremotecache` channel in [Slack](https://slackin.spotify.com/).
+
 ## Contributing
 
 We feel that a welcoming community is important and we ask that you follow Spotify's 

Rakefile

@@ -1,4 +1,5 @@
 # encoding: utf-8
+require_relative 'tasks/e2e'
 
 ################################
 # Rake configuration
@@ -9,7 +10,7 @@ DERIVED_DATA_DIR = File.join('.build').freeze
 RELEASES_ROOT_DIR = File.join('releases').freeze
 
 EXECUTABLE_NAME = 'XCRemoteCache'
-EXECUTABLE_NAMES = ['xclibtool', 'xcpostbuild', 'xcprebuild', 'xcprepare', 'xcswiftc', 'xcld'] 
+EXECUTABLE_NAMES = ['xclibtool', 'xcpostbuild', 'xcprebuild', 'xcprepare', 'xcswiftc', 'xcld', 'xcldplusplus', 'xclipo']
 PROJECT_NAME = 'XCRemoteCache'
 
 SWIFTLINT_ENABLED = true
@@ -28,7 +29,7 @@ task :lint => [:prepare] do
   puts 'Run linting'
 
   system("swiftformat --lint --config .swiftformat --cache ignore .") or abort "swiftformat failure" if SWIFTFORMAT_ENABLED
-  system("swiftlint lint --config .swiftlint.yml") or abort "swiftlint failure" if SWIFTLINT_ENABLED
+  system("swiftlint lint --config .swiftlint.yml --strict") or abort "swiftlint failure" if SWIFTLINT_ENABLED
 end
 
 task :autocorrect => [:prepare]  do 
@@ -71,12 +72,34 @@ task :build, [:configuration, :arch, :sdks, :is_archive] do |task, args|
   end
 end
 
+desc 'Build release artifacts'
+task :prepare_release do
+  system("rm -rf releases && rm -rf tmp")
+  Rake::Task['build'].invoke("release", "x86_64-apple-macosx")
+  system("mkdir -p tmp && unzip releases/XCRemoteCache.zip -d tmp/xcremotecache-x86_64")
+  system("rm -rf releases")
+  Rake::Task['build'].invoke("release", "arm64-apple-macosx")
+  system("rake 'build[release, arm64-apple-macosx]'")
+  system("mkdir -p tmp && unzip releases/XCRemoteCache.zip -d tmp/xcremotecache-arm64")
+  system("rm -rf releases")
+  system("mkdir -p releases && zip -jr releases/XCRemoteCache-macOS-x86_64.zip LICENSE README.md tmp/xcremotecache-x86_64")
+  system("zip -jr releases/XCRemoteCache-macOS-arm64.zip LICENSE README.md tmp/xcremotecache-arm64")
+  system("mkdir -p tmp/xcremotecache && ls tmp/xcremotecache-x86_64 | xargs -I {} lipo -create -output tmp/xcremotecache/{} tmp/xcremotecache-x86_64/{} tmp/xcremotecache-arm64/{}")
+  system("zip -jr releases/XCRemoteCache-macOS-arm64-x86_64.zip LICENSE README.md tmp/xcremotecache")
+end
+
 desc 'run tests with SPM'
 task :test do
   # Running tests
   spm_test()
 end
 
+desc 'build and run E2E tests'
+task :e2e => [:build, :e2e_only]
+
+desc 'run E2E tests without building the XCRemoteCache binary'
+task :e2e_only => ['e2e:run']
+
 ################################
 # Helper functions
 ################################

Sources/XCRemoteCache/Artifacts/ArtifactCreator.swift

@@ -41,6 +41,8 @@ class BuildArtifactCreator: ArtifactSwiftProductsBuilderImpl, ArtifactCreator {
     private let moduleName: String?
     private let modulesFolderPath: String
     private let dSYMPath: URL
+    private let metaWriter: MetaWriter
+    private let artifactProcessor: ArtifactProcessor
     private let fileManager: FileManager
 
     init(
@@ -50,6 +52,8 @@ class BuildArtifactCreator: ArtifactSwiftProductsBuilderImpl, ArtifactCreator {
         moduleName: String?,
         modulesFolderPath: String,
         dSYMPath: URL,
+        metaWriter: MetaWriter,
+        artifactProcessor: ArtifactProcessor,
         fileManager: FileManager
     ) {
         self.buildDir = buildDir
@@ -59,6 +63,8 @@ class BuildArtifactCreator: ArtifactSwiftProductsBuilderImpl, ArtifactCreator {
         self.moduleName = moduleName
         self.fileManager = fileManager
         self.dSYMPath = dSYMPath
+        self.metaWriter = metaWriter
+        self.artifactProcessor = artifactProcessor
         super.init(workingDir: tempDir, moduleName: moduleName, fileManager: fileManager)
     }
 
@@ -72,7 +78,11 @@ class BuildArtifactCreator: ArtifactSwiftProductsBuilderImpl, ArtifactCreator {
         let dynamicLibraryArtifacts = try prepareDynamicLibraryArtifacts()
         zipPaths.append(contentsOf: dynamicLibraryArtifacts)
 
-        let creator = ZipArtifactCreator(workingDir: zipWorkingDir, fileManager: fileManager)
+        let creator = ZipArtifactCreator(
+            workingDir: zipWorkingDir,
+            metaWriter: metaWriter,
+            fileManager: fileManager
+        )
         return try creator.createArtifact(zipContent: zipPaths, artifactKey: artifactKey, meta: meta)
     }
 
@@ -80,6 +90,7 @@ class BuildArtifactCreator: ArtifactSwiftProductsBuilderImpl, ArtifactCreator {
     /// - Parameter tempDir: Temp location to organize file hierarchy in the artifact
     /// - returns: URLs to include into the artifact package
     fileprivate func prepareSwiftArtifacts(tempDir: URL) throws -> [URL] {
+        try artifactProcessor.process(localArtifact: tempDir)
         var artifacts: [URL] = []
 
         // Add optional directory with generated ObjC headers

Sources/XCRemoteCache/Artifacts/ArtifactOrganizer.swift

@@ -31,7 +31,7 @@ enum ArtifactOrganizerLocationPreparationResult: Equatable {
     case preparedForArtifact(artifact: URL)
 }
 
-/// Prepares .zip artifact for the local operations
+/// Prepares existing .zip artifact for the local operations
 protocol ArtifactOrganizer {
     /// Prepares the location for the artifact unzipping
     /// - Parameter fileKey: artifact fileKey that corresponds to the zip filename on the remote cache server
@@ -48,10 +48,13 @@ protocol ArtifactOrganizer {
 
 class ZipArtifactOrganizer: ArtifactOrganizer {
     private let cacheDir: URL
+    // all processors that should "prepare" the unzipped raw artifact
+    private let artifactProcessors: [ArtifactProcessor]
     private let fileManager: FileManager
 
-    init(targetTempDir: URL, fileManager: FileManager) {
+    init(targetTempDir: URL, artifactProcessors: [ArtifactProcessor], fileManager: FileManager) {
         cacheDir = targetTempDir.appendingPathComponent("xccache")
+        self.artifactProcessors = artifactProcessors
         self.fileManager = fileManager
     }
 
@@ -93,6 +96,10 @@ class ZipArtifactOrganizer: ArtifactOrganizer {
         // when the command was interrupted (internal crash or `kill -9` signal)
         let tempDestination = destinationURL.appendingPathExtension("tmp")
         try Zip.unzipFile(artifact, destination: tempDestination, overwrite: true, password: nil)
+
+        try artifactProcessors.forEach { processor in
+            try processor.process(rawArtifact: tempDestination)
+        }
         try fileManager.moveItem(at: tempDestination, to: destinationURL)
         return destinationURL
     }

Sources/XCRemoteCache/Artifacts/ArtifactProcessor.swift

@@ -0,0 +1,80 @@
+// Copyright (c) 2022 Spotify AB.
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import Foundation
+
+
+/// Performs a pre/postprocessing on an artifact package
+/// Could be a place for file reorganization (to support legacy package formats) and/or
+/// remapp absolute paths in some package files
+protocol ArtifactProcessor {
+    /// Processes a raw artifact in a directory. Raw artifact is a format of an artifact
+    /// that is stored in a remote cache server (generic)
+    /// - Parameter rawArtifact: directory that contains raw artifact content
+    func process(rawArtifact: URL) throws
+
+    /// Processes a local artifact in a directory
+    /// - Parameter localArtifact: directory that contains local (machine-specific) artifact content
+    func process(localArtifact: URL) throws
+}
+
+/// Processes downloaded artifact by replacing generic paths in generated ObjC headers placed in ./include
+class UnzippedArtifactProcessor: ArtifactProcessor {
+    /// All directories in an artifact that should be processed by path remapping
+    private static let remappingDirs = ["include"]
+    private let fileRemapper: FileDependenciesRemapper
+    private let dirScanner: DirScanner
+
+    init(fileRemapper: FileDependenciesRemapper, dirScanner: DirScanner) {
+        self.fileRemapper = fileRemapper
+        self.dirScanner = dirScanner
+    }
+
+    private func findProcessingEligableFiles(path: String) throws -> [URL] {
+        let remappingURL = URL(fileURLWithPath: path)
+        let allFiles = try dirScanner.recursiveItems(at: remappingURL)
+        return allFiles.filter({ !$0.isHidden })
+    }
+
+    /// Replaces all generic paths in a raw artifact's `include` dir with
+    /// absolute paths, specific for a given machine and configuration
+    /// - Parameter rawArtifact: raw artifact location
+    func process(rawArtifact url: URL) throws {
+        for remappingDir in Self.remappingDirs {
+            let remappingPath = url.appendingPathComponent(remappingDir).path
+            let allFiles = try findProcessingEligableFiles(path: remappingPath)
+            try allFiles.forEach(fileRemapper.remap(fromGeneric:))
+        }
+    }
+
+    func process(localArtifact url: URL) throws {
+        for remappingDir in Self.remappingDirs {
+            let remappingPath = url.appendingPathComponent(remappingDir).path
+            let allFiles = try findProcessingEligableFiles(path: remappingPath)
+            try allFiles.forEach(fileRemapper.remap(fromLocal:))
+        }
+    }
+}
+
+fileprivate extension URL {
+    // Recognize hidden files starting with a dot
+    var isHidden: Bool {
+        lastPathComponent.hasPrefix(".")
+    }
+}

Sources/XCRemoteCache/Artifacts/ArtifactSwiftProductsBuilder.swift

@@ -51,8 +51,6 @@ protocol ArtifactSwiftProductsBuilder {
 /// # {workingDir}/xccache/produced/include/#{moduleName} (if `moduleName` is defined)
 class ArtifactSwiftProductsBuilderImpl: ArtifactSwiftProductsBuilder {
 
-    /// List of all required swiftmodule related extensions that should be copied to the artifact
-    private static let swiftmoduleExtensionsToInclude = ["swiftmodule", "swiftdoc", "swiftsourceinfo"]
     private let workingDir: URL
     private let moduleName: String?
     private let fileManager: FileManager
@@ -90,7 +88,7 @@ class ArtifactSwiftProductsBuilderImpl: ArtifactSwiftProductsBuilder {
             throw ArtifactSwiftProductsBuilderError.populatingNonExistingObjCHeader
         }
         try fileManager.createDirectory(at: moduleObjCURL, withIntermediateDirectories: true, attributes: nil)
-        try fileManager.spt_forceLinkItem(at: headerURL, to: headerArtifactURL)
+        try fileManager.spt_forceCopyItem(at: headerURL, to: headerArtifactURL)
     }
 
     func includeModuleDefinitionsToTheArtifact(arch: String, moduleURL: URL) throws {

Sources/XCRemoteCache/Artifacts/FileDependenciesRemapper.swift

@@ -0,0 +1,81 @@
+// Copyright (c) 2022 Spotify AB.
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import Foundation
+
+
+enum FileDependenciesRemapperError: Error {
+    /// Thrown when the file to remap is invalid (e.g. doesn't exist or has unexpected format)
+    case invalidRemappingFile(URL)
+}
+
+/// Replaces paths in a file content between generic (placeholder-based)
+/// and local formats
+protocol FileDependenciesRemapper {
+    /// Replaces all generic paths (with placeholders) to a local, machine
+    /// specific absolute paths
+    /// - Parameter url: location of a file that should be remapped in-place
+    func remap(fromGeneric url: URL) throws
+    /// Replaces all local, machine specific absolute paths to
+    /// generic ones
+    /// - Parameter url: location of a file that should be remapped in-place
+    func remap(fromLocal url: URL) throws
+}
+
+/// Remaps absolute paths in a text files stored on a disk
+/// Note: That class can be used only for text-based files, not binaries
+class TextFileDependenciesRemapper: FileDependenciesRemapper {
+    private static let linesSeparator = "\n"
+    private let remapper: DependenciesRemapper
+    private let fileAccessor: FileAccessor
+
+    init(remapper: DependenciesRemapper, fileAccessor: FileAccessor) {
+        self.remapper = remapper
+        self.fileAccessor = fileAccessor
+    }
+
+    private func readFileLines(_ url: URL) throws -> [String] {
+        guard let content = try fileAccessor.contents(atPath: url.path) else {
+            // the file is empty
+            return []
+        }
+        guard let contentString = String(data: content, encoding: .utf8) else {
+            throw FileDependenciesRemapperError.invalidRemappingFile(url)
+        }
+        return contentString.components(separatedBy: .newlines)
+    }
+
+    private func storeFileLines(lines: [String], url: URL) throws {
+        let contentString = lines.joined(separator: "\n")
+        let contentData = contentString.data(using: String.Encoding.utf8)
+        try fileAccessor.write(toPath: url.path, contents: contentData)
+    }
+
+    func remap(fromGeneric url: URL) throws {
+        let contentLines = try readFileLines(url)
+        let remappedContent = try remapper.replace(genericPaths: contentLines)
+        try storeFileLines(lines: remappedContent, url: url)
+    }
+
+    func remap(fromLocal url: URL) throws {
+        let contentLines = try readFileLines(url)
+        let remappedContent = try remapper.replace(localPaths: contentLines)
+        try storeFileLines(lines: remappedContent, url: url)
+    }
+}

Sources/XCRemoteCache/Artifacts/SwiftmoduleFileExtension.swift

@@ -30,6 +30,9 @@ enum SwiftmoduleFileExtension: String {
     case swiftmodule
     case swiftdoc
     case swiftsourceinfo
+    case swiftinterface
+    case privateSwiftinterface = "private.swiftinterface"
+    case abiJson = "abi.json"
 }
 
 extension SwiftmoduleFileExtension {
@@ -38,5 +41,8 @@ extension SwiftmoduleFileExtension {
         .swiftmodule: .required,
         .swiftdoc: .required,
         .swiftsourceinfo: .optional,
+        .swiftinterface: .optional,
+        .privateSwiftinterface: .optional,
+        .abiJson: .optional,
     ]
 }

Sources/XCRemoteCache/Artifacts/ZipArtifactCreator.swift

@@ -23,11 +23,12 @@ import Zip
 class ZipArtifactCreator {
     /// Location where zip file should be generated
     private let workingDir: URL
+    private let metaWriter: MetaWriter
     private let fileManager: FileManager
-    private let metaEncoder = JSONEncoder()
 
-    init(workingDir: URL, fileManager: FileManager) {
+    init(workingDir: URL, metaWriter: MetaWriter, fileManager: FileManager) {
         self.workingDir = workingDir
+        self.metaWriter = metaWriter
         self.fileManager = fileManager
     }
 
@@ -35,18 +36,10 @@ class ZipArtifactCreator {
         let zipURL = workingDir.appendingPathComponent("\(artifactKey).zip")
         try fileManager.createDirectory(at: workingDir, withIntermediateDirectories: true, attributes: nil)
         // Include meta json to the artifact
-        let metaURL = try dumpMeta(meta)
+        let metaURL = try metaWriter.write(meta, locationDir: workingDir)
         let zipPaths = zipContent + [metaURL]
 
         try Zip.zipFiles(paths: zipPaths, zipFilePath: zipURL, password: nil, progress: nil)
         return Artifact(id: artifactKey, package: zipURL, meta: metaURL)
     }
-
-    // Save meta to a local file
-    private func dumpMeta<T: Meta>(_ meta: T) throws -> URL {
-        let metaURL = workingDir.appendingPathComponent(meta.fileKey).appendingPathExtension("json")
-        let metaData = try metaEncoder.encode(meta)
-        try fileManager.spt_writeToFile(atPath: metaURL.path, contents: metaData)
-        return metaURL
-    }
 }

Sources/XCRemoteCache/Commands/Libtool/XCCreateUniversalBinary.swift

@@ -19,31 +19,42 @@
 
 import Foundation
 
-enum XCLibtoolCreateUniversalBinaryError: Error {
+enum XCCreateUniversalBinaryError: Error {
     ///  Missing ar libraries that should constitute an universal build
     case missingInputLibrary
 }
 
-/// Wrapper for `libtool` call for creating an universal binary
-class XCLibtoolCreateUniversalBinary: XCLibtoolLogic {
+/// Wrapper for `libtool`/`lipo` call for creating an universal binary
+class XCCreateUniversalBinary: XCLibtoolLogic {
     private let output: URL
     private let tempDir: URL
     private let firstInputURL: URL
+    private let toolName: String
+    private let fallbackCommand: String
 
-    init(output: String, inputs: [String]) throws {
+    init(
+        output: String,
+        inputs: [String],
+        toolName: String,
+        fallbackCommand: String
+    ) throws {
         self.output = URL(fileURLWithPath: output)
         guard let firstInput = inputs.first else {
-            throw XCLibtoolCreateUniversalBinaryError.missingInputLibrary
+            throw XCCreateUniversalBinaryError.missingInputLibrary
         }
         let firstInputURL = URL(fileURLWithPath: firstInput)
         // inputs are place in $TARGET_TEMP_DIR/Objects-normal/$ARCH/Binary/$TARGET_NAME.a
         // TODO: find better (stable) technique to determine `$TARGET_TEMP_DIR`
+        errorLog("\(firstInputURL.absoluteString)")
+
         tempDir = firstInputURL
             .deletingLastPathComponent()
             .deletingLastPathComponent()
             .deletingLastPathComponent()
             .deletingLastPathComponent()
         self.firstInputURL = firstInputURL
+        self.toolName = toolName
+        self.fallbackCommand = fallbackCommand
     }
 
     func run() {
@@ -52,10 +63,10 @@ class XCLibtoolCreateUniversalBinary: XCLibtoolLogic {
         let config: XCRemoteCacheConfig
         do {
             let srcRoot: URL = URL(fileURLWithPath: fileManager.currentDirectoryPath)
-            config = try XCRemoteCacheConfigReader(srcRootPath: srcRoot.path, fileManager: fileManager)
+            config = try XCRemoteCacheConfigReader(srcRootPath: srcRoot.path, fileReader: fileManager)
                 .readConfiguration()
         } catch {
-            errorLog("Libtool initialization failed with error: \(error). Fallbacking to libtool")
+            errorLog("\(toolName) initialization failed with error: \(error). Fallbacking to \(fallbackCommand)")
             fallbackToDefault()
         }
 
@@ -74,22 +85,21 @@ class XCLibtoolCreateUniversalBinary: XCLibtoolLogic {
             // that these are already an universal binary
             try fileManager.spt_forceLinkItem(at: firstInputURL, to: output)
         } catch {
-            errorLog("Libtool failed with error: \(error). Fallbacking to libtool")
+            errorLog("\(toolName) failed with error: \(error). Fallbacking to \(fallbackCommand)")
             do {
                 try fileManager.removeItem(at: markerURL)
                 fallbackToDefault()
             } catch {
-                exit(1, "FATAL: libtool failed with error: \(error)")
+                exit(1, "FATAL: \(fallbackCommand) failed with error: \(error)")
             }
         }
     }
 
     private func fallbackToDefault() -> Never {
         let args = ProcessInfo().arguments
-        let command = "libtool"
-        let paramList = [command] + args.dropFirst()
+        let paramList = [fallbackCommand] + args.dropFirst()
         let cargs = paramList.map { strdup($0) } + [nil]
-        execvp(command, cargs)
+        execvp(fallbackCommand, cargs)
 
         /// C-function `execv` returns only when the command fails
         exit(1)

Sources/XCRemoteCache/Commands/Libtool/XCLibtool.swift

@@ -20,7 +20,7 @@
 import Foundation
 
 /// Represents a mode that libtool was called
-public enum XCLibtoolMode {
+public enum XCLibtoolMode: Equatable {
     /// Creating a static library (ar format) from a set of .o input files
     case createLibrary(output: String, filelist: String, dependencyInfo: String)
     /// Creating a universal library (multiple-architectures) from a set of input .a static libraries
@@ -44,7 +44,12 @@ public class XCLibtool {
                 stepDescription: "Libtool"
             )
         case .createUniversalBinary(let output, let inputs):
-            logic = try XCLibtoolCreateUniversalBinary(output: output, inputs: inputs)
+            logic = try XCCreateUniversalBinary(
+                output: output,
+                inputs: inputs,
+                toolName: "Libtool",
+                fallbackCommand: "libtool"
+            )
         }
     }
 

Sources/XCRemoteCache/Commands/Libtool/XCLipo.swift

@@ -0,0 +1,50 @@
+// Copyright (c) 2023 Spotify AB.
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import Foundation
+
+/// Wrapper for a `lipo` tool that creates a fat archive
+public class XCLipo {
+    private let logic: XCLibtoolLogic
+
+    public init(
+        output: String,
+        inputs: [String],
+        fallbackCommand: String,
+        stepDescription: String
+    ) throws {
+        errorLog("\(output)")
+        errorLog("\(inputs.joined(separator: ","))")
+        logic = try XCCreateUniversalBinary(
+            output: output,
+            inputs: inputs,
+            toolName: stepDescription,
+            fallbackCommand: fallbackCommand
+        )
+    }
+
+    /// Handles a `-create` action which is responsible to create a fat archive
+    /// If remote cache can reuse artifacts from a remote cache, it just links any of input
+    /// files to the destination (output) location because the binary in XCRC artifact already
+    /// contains a fat library
+    /// If a remote artifact cannot be reused, a fallback to the `lipo` command is performed
+    public func run() {
+        logic.run()
+    }
+}

Sources/XCRemoteCache/Commands/Plugins/Thinning/Factories/ThinningConsumerArtifactsOrganizerFactory.swift

@@ -27,13 +27,19 @@ protocol ThinningConsumerArtifactsOrganizerFactory {
 }
 
 class ThinningConsumerZipArtifactsOrganizerFactory: ThinningConsumerArtifactsOrganizerFactory {
+    private let processors: [ArtifactProcessor]
     private let fileManager: FileManager
 
-    init(fileManager: FileManager) {
+    init(processors: [ArtifactProcessor], fileManager: FileManager) {
+        self.processors = processors
         self.fileManager = fileManager
     }
 
     func build(targetTempDir: URL) -> ArtifactOrganizer {
-        ZipArtifactOrganizer(targetTempDir: targetTempDir, fileManager: fileManager)
+        ZipArtifactOrganizer(
+            targetTempDir: targetTempDir,
+            artifactProcessors: processors,
+            fileManager: fileManager
+        )
     }
 }

Sources/XCRemoteCache/Commands/Plugins/Thinning/ThinningCreatorPlugin.swift

@@ -33,13 +33,16 @@ enum ThinningCreatorPluginError: Error {
 /// Warning! This plugin assumes that producer's DerivedData are always cleaned before a build
 class ThinningCreatorPlugin: ArtifactCreatorPlugin {
     private let targetTempDir: URL
+    private let modeMarkerPath: String
     private let dirScanner: DirScanner
 
     /// Default Initializer
     /// - Parameter targetTempDir: Location of current target-specific temp dir (TARGET_TEMP_DIR)
+    /// - Parameter modeMarkerPath: path of maker file that informs if a given target can reuse remote artifacts
     /// - Parameter dirScanner: scanner to access disk and read files and directories hierarchy
-    init(targetTempDir: URL, dirScanner: DirScanner) {
+    init(targetTempDir: URL, modeMarkerPath: String, dirScanner: DirScanner) {
         self.targetTempDir = targetTempDir
+        self.modeMarkerPath = modeMarkerPath
         self.dirScanner = dirScanner
     }
 
@@ -57,31 +60,19 @@ class ThinningCreatorPlugin: ArtifactCreatorPlugin {
             let fileKey: String
         }
         let uploadedTargetArtifacts = try allURLs.compactMap { tempDir -> TargetTuple? in
-            // All targets that uploaded their artifacts, have it placed in the
-            // `$(TARGET_TEMP_DIR)/xccache/produced/{{fileKey}}.zip` location. Find all targets that have such a file
-
-            let targetGeneratedArtifactRootDir = tempDir
-                .appendingPathComponent("xccache")
-                .appendingPathComponent("produced")
-            guard try dirScanner.itemType(atPath: targetGeneratedArtifactRootDir.path) == ItemType.dir else {
-                // given target didn't generate any artifacts (e.g. it is never cached with XCRemoteCache)
-                return nil
-            }
-
-            let allFilesProduced = try dirScanner.items(at: targetGeneratedArtifactRootDir)
-            let allArtifacts = allFilesProduced.filter { $0.pathExtension == "zip" }
-            guard !allArtifacts.isEmpty else {
+            let potentialArtifacts = try findTargetPackageZip(tempDir: tempDir)
+            guard !potentialArtifacts.isEmpty else {
                 // there is no generated *.zip file, so given target didn't create an artifact - it could be
                 // just a helper target (like the target we integrate this plugin with)
                 return nil
             }
             // Find {{fileKey}} based on the .zip file basename
-            guard allArtifacts.count == 1 else {
+            guard potentialArtifacts.count == 1 else {
                 throw ThinningCreatorPluginError.noSingleTargetArtifactsGenerated(
-                    rootDir: targetGeneratedArtifactRootDir
+                    rootDir: tempDir
                 )
             }
-            let fileKey = allArtifacts[0].deletingPathExtension().lastPathComponent
+            let fileKey = potentialArtifacts[0].deletingPathExtension().lastPathComponent
             // Taking target name from tempDir, which has a structures "*.build"
             let targetName = tempDir.deletingPathExtension().lastPathComponent
             return TargetTuple(targetName: targetName, fileKey: fileKey)
@@ -96,6 +87,41 @@ class ThinningCreatorPlugin: ArtifactCreatorPlugin {
         return Dictionary(uniqueKeysWithValues: extraKeysTuples)
     }
 
+    private func findTargetPackageZip(tempDir: URL) throws -> [URL] {
+        // Producer mode:
+        // All targets that uploaded their artifacts, have it placed in the
+        // `$(TARGET_TEMP_DIR)/xccache/produced/{{fileKey}}.zip` location. Find all targets that have such a file
+        // ProducerFast mode:
+        // If a target reused already existing artifact, it still has `$(TARGET_TEMP_DIR)/rc.enabled` marker file
+        // and the reused zip is placed in:
+        // `$(TARGET_TEMP_DIR)/xccache/{{fileKey}}.zip` location
+
+        let targetEnabledMarker = tempDir.appendingPathComponent(modeMarkerPath)
+        let targetReusedArtifactRootDir = tempDir.appendingPathComponent("xccache")
+        let targetGeneratedArtifactRootDir = tempDir
+            .appendingPathComponent("xccache")
+            .appendingPathComponent("produced")
+
+        let pathToDirWithZipArtifacts: URL
+        // try the `.producerFast` scenario first (the artifact was not locally
+        // generated but just reused from the remote cache)
+        if try dirScanner.itemType(atPath: targetEnabledMarker.path) == ItemType.file {
+            pathToDirWithZipArtifacts = targetReusedArtifactRootDir
+        } else {
+            // cover a case when a target was build locally and an artifact
+            // has just been created (locally)
+            guard try dirScanner.itemType(atPath: targetGeneratedArtifactRootDir.path) == ItemType.dir else {
+                // given target didn't generate any artifacts (e.g. it is never cached with XCRemoteCache)
+                return []
+            }
+            pathToDirWithZipArtifacts = targetGeneratedArtifactRootDir
+        }
+
+        let allFilesProduced = try dirScanner.items(at: pathToDirWithZipArtifacts)
+        let allArtifacts = allFilesProduced.filter { $0.pathExtension == "zip" }
+        return allArtifacts
+    }
+
     func artifactToUpload(main: MainArtifactMeta) throws -> [Artifact] {
         return []
     }

Sources/XCRemoteCache/Commands/Plugins/Thinning/ThinningDiskSwiftcProductsGenerator.swift

@@ -40,7 +40,7 @@ class ThinningDiskSwiftcProductsGenerator: SwiftcProductsGenerator {
         destinationSwiftmodulePaths = Dictionary(
             uniqueKeysWithValues: SwiftmoduleFileExtension.SwiftmoduleExtensions
                 .map { ext, _ in
-                    switch (ext) {
+                    switch ext {
                     case .swiftsourceinfo:
                         let dest = modulePathDir.appendingPathComponent("Project")
                             .appendingPathComponent(moduleName)
@@ -58,7 +58,7 @@ class ThinningDiskSwiftcProductsGenerator: SwiftcProductsGenerator {
     func generateFrom(
         artifactSwiftModuleFiles sourceAtifactSwiftModuleFiles: [SwiftmoduleFileExtension: URL],
         artifactSwiftModuleObjCFile: URL
-    ) throws -> URL {
+    ) throws -> SwiftcProductsGeneratorOutput {
         // Move cached -Swift.h file to the expected location
         try diskCopier.copy(file: artifactSwiftModuleObjCFile, destination: objcHeaderOutput)
         for (ext, url) in sourceAtifactSwiftModuleFiles {
@@ -79,6 +79,9 @@ class ThinningDiskSwiftcProductsGenerator: SwiftcProductsGenerator {
         }
 
         // Build parent dir of the .swiftmodule file that contains a module
-        return modulePathOutput.deletingLastPathComponent()
+        return SwiftcProductsGeneratorOutput(
+            swiftmoduleDir: modulePathOutput.deletingLastPathComponent(),
+            objcHeaderFile: objcHeaderOutput
+        )
     }
 }

Sources/XCRemoteCache/Commands/Plugins/Thinning/ThinningPrebuildPlugin.swift

@@ -25,7 +25,7 @@ class ThinningConsumerPlugin {
 
     deinit {
         // initialised but never run plugin suggests that standard target fallbacks to the local development
-        // and DerivedData still misses build artifacts.
+        // and DerivedData still misses build artifacts
         guard wasRun else {
             let errorMessage = """
             \(type(of: self)) plugin has never been run, thinning cannot be supported. Verify you \

Sources/XCRemoteCache/Commands/Plugins/Thinning/UnzippedArtifactSwiftProductsOrganizer.swift

@@ -73,11 +73,12 @@ class UnzippedArtifactSwiftProductsOrganizer: SwiftProductsOrganizer {
             .appendingPathComponent(moduleName)
             .appendingPathComponent("\(moduleName)-Swift.h")
 
-        let generatedModuleDir = try productsGenerator.generateFrom(
+        let generatedModule = try productsGenerator.generateFrom(
             artifactSwiftModuleFiles: artifactSwiftmoduleFiles,
             artifactSwiftModuleObjCFile: artifactSwiftModuleObjCFile
         )
 
-        try fingerprintSyncer.decorate(sourceDir: generatedModuleDir, fingerprint: fingerprint)
+        try fingerprintSyncer.decorate(sourceDir: generatedModule.swiftmoduleDir, fingerprint: fingerprint)
+        try fingerprintSyncer.decorate(file: generatedModule.objcHeaderFile, fingerprint: fingerprint)
     }
 }

Sources/XCRemoteCache/Commands/Postbuild/Postbuild.swift

@@ -41,6 +41,7 @@ class Postbuild {
     private let dSYMOrganizer: DSYMOrganizer
     private let modeController: CacheModeController
     private let metaReader: MetaReader
+    private let metaWriter: MetaWriter
     private let creatorPlugins: [ArtifactCreatorPlugin]
     private let consumerPlugins: [ArtifactConsumerPostbuildPlugin]
 
@@ -58,6 +59,7 @@ class Postbuild {
         dSYMOrganizer: DSYMOrganizer,
         modeController: CacheModeController,
         metaReader: MetaReader,
+        metaWriter: MetaWriter,
         creatorPlugins: [ArtifactCreatorPlugin],
         consumerPlugins: [ArtifactConsumerPostbuildPlugin]
     ) {
@@ -74,6 +76,7 @@ class Postbuild {
         self.dSYMOrganizer = dSYMOrganizer
         self.modeController = modeController
         self.metaReader = metaReader
+        self.metaWriter = metaWriter
         self.creatorPlugins = creatorPlugins
         self.consumerPlugins = consumerPlugins
     }
@@ -125,6 +128,22 @@ class Postbuild {
         try generateFingerprintOverrides(contextSpecificFingerprint: fingerprint.contextSpecific)
     }
 
+    /// Uploads only a meta to the remote server - useful when the file artifact (.zip) already exists on a remote
+    /// server and only a meta for a current commit sha has to be uploaded
+    public func performMetaUpload(meta: MainArtifactMeta, for commit: String) throws {
+        // Reset plugins keys as these are unique to each
+        var meta = meta
+        meta.pluginsKeys = [:]
+        meta = try creatorPlugins.reduce(meta) { prevMeta, plugin in
+            var meta = prevMeta
+            // add extra keys from the plugin. A plugin overrides previously defined keys in case of duplication
+            meta.pluginsKeys = try meta.pluginsKeys.merging(plugin.extraMetaKeys(prevMeta), uniquingKeysWith: { $1 })
+            return meta
+        }
+        let metaPath = try metaWriter.write(meta, locationDir: context.targetTempDir)
+        try networkClient.uploadSynchronously(metaPath, as: .meta(commit: commit))
+    }
+
     /// Builds an artifact package and uploads it to the remote server
     public func performBuildUpload(for commit: String) throws {
         let dependencies = try generateDependencies()
@@ -134,7 +153,7 @@ class Postbuild {
         // Replace all local paths to the generic ones (e.g. $SRCROOT)
         let remappers = [remapper] + creatorPlugins.compactMap(\.customPathsRemapper)
         let remapper = DependenciesRemapperComposite(remappers)
-        let abstractFingerprintFiles = remapper.replace(localPaths: dependencies.map(\.path))
+        let abstractFingerprintFiles = try remapper.replace(localPaths: dependencies.map(\.path))
         // TODO: use `inputs` read by dependenciesReader
         var meta = MainArtifactMeta(
             dependencies: abstractFingerprintFiles,
@@ -219,13 +238,32 @@ class Postbuild {
         let moduleSwiftProductURL = context.productsDir
             .appendingPathComponent(context.modulesFolderPath)
             .appendingPathComponent("\(modulename).swiftmodule")
+        let objcHeaderSwiftProductURL = context.derivedSourcesDir
+            .appendingPathComponent("\(modulename)-Swift.h")
+        // This header is obly valid if building a frameworks
+        let objcHeaderSwiftPublicPathURL = context.publicHeadersFolderPath?
+            .appendingPathComponent("\(modulename)-Swift.h")
         if let fingerprint = contextSpecificFingerprint {
             try fingerprintSyncer.decorate(
                 sourceDir: moduleSwiftProductURL,
                 fingerprint: fingerprint
             )
+            try fingerprintSyncer.decorate(
+                file: objcHeaderSwiftProductURL,
+                fingerprint: fingerprint
+            )
+            if let objcPublic = objcHeaderSwiftPublicPathURL {
+                try fingerprintSyncer.decorate(
+                    file: objcPublic,
+                    fingerprint: fingerprint
+                )
+            }
         } else {
             try fingerprintSyncer.delete(sourceDir: moduleSwiftProductURL)
+            try fingerprintSyncer.delete(sourceDir: objcHeaderSwiftProductURL)
+            if let objcPublic = objcHeaderSwiftPublicPathURL {
+                try fingerprintSyncer.delete(file: objcPublic)
+            }
         }
     }
 }

Sources/XCRemoteCache/Commands/Postbuild/PostbuildContext.swift

@@ -32,12 +32,15 @@ enum MachOType: String, Codable {
 enum PostbuildContextError: Error {
     /// URL address is not a valid URL
     case invalidAddress(String)
+    /// ARCHS env does not contain any architecture to build
+    case missingArchitecture
 }
 
 public struct PostbuildContext {
     var mode: Mode
     var targetName: String
     var targetTempDir: URL
+    var derivedFilesDir: URL
     /// Location where all compilation outputs (.o) are placed
     var compilationTempDir: URL
     var configuration: String
@@ -64,25 +67,45 @@ public struct PostbuildContext {
     var machOType: MachOType
     var wasDsymGenerated: Bool
     var dSYMPath: URL
+    // building architecture. Used to find all dependencies from *.d files
+    // Warning: if two architectures are built (e.g. for disabled "Build Archive
+    // Architecture Only"), a first architecture one is picked
     let arch: String
     let builtProductsDir: URL
     /// Location to the product bundle. Can be nil for libraries
     let bundleDir: URL?
-    let derivedSourcesDir: URL
+    var derivedSourcesDir: URL
     /// List of all targets to downloaded from the thinning aggregation target
     var thinnedTargets: [String]
-    /// Action type: build, indexbuild etc.
+    /// Action type: build, indexbuild etc
     var action: BuildActionType
+    let modeMarkerPath: String
+    /// location of the json file that define virtual files system overlay
+    /// (mappings of the virtual location file -> local file path)
+    let overlayHeadersPath: URL
+    /// Regexes of files that should not be included in the dependency list
+    let irrelevantDependenciesPaths: [String]
+    /// Location of public headers. Not always available (e.g. static libraries)
+    var publicHeadersFolderPath: URL?
+    /// XCRemoteCache is explicitly disabled
+    let disabled: Bool
 }
 
 extension PostbuildContext {
+    // swiftlint:disable:next function_body_length
     init(_ config: XCRemoteCacheConfig, env: [String: String]) throws {
         mode = config.mode
         let targetNameValue: String = try env.readEnv(key: "TARGET_NAME")
         targetName = targetNameValue
         targetTempDir = try env.readEnv(key: "TARGET_TEMP_DIR")
-        arch = try env.readEnv(key: "PLATFORM_PREFERRED_ARCH")
-        compilationTempDir = try env.readEnv(key: "OBJECT_FILE_DIR_normal").appendingPathComponent(arch)
+        derivedFilesDir = try env.readEnv(key: "DERIVED_FILE_DIR")
+        let archs: [String] = try env.readEnv(key: "ARCHS").split(separator: " ").map(String.init)
+        guard let firstArch = archs.first, !firstArch.isEmpty else {
+            throw PostbuildContextError.missingArchitecture
+        }
+        arch = firstArch
+        let variant: String = try env.readEnv(key: "CURRENT_VARIANT")
+        compilationTempDir = try env.readEnv(key: "OBJECT_FILE_DIR_\(variant)").appendingPathComponent(arch)
         configuration = try env.readEnv(key: "CONFIGURATION")
         platform = try env.readEnv(key: "PLATFORM_NAME")
         xcodeBuildNumber = try env.readEnv(key: "XCODE_PRODUCT_BUILD_VERSION")
@@ -106,7 +129,7 @@ extension PostbuildContext {
         dSYMPath = try env.readEnv(key: "DWARF_DSYM_FOLDER_PATH")
             .appendingPathComponent(env.readEnv(key: "DWARF_DSYM_FILE_NAME"))
         builtProductsDir = try env.readEnv(key: "BUILT_PRODUCTS_DIR")
-        if let contentsFolderPath = env.readEnv(key: "CONTENTS_FOLDER_PATH") {
+        if let contentsFolderPath: String = env.readEnv(key: "CONTENTS_FOLDER_PATH") {
             bundleDir = productsDir.appendingPathComponent(contentsFolderPath)
         } else {
             bundleDir = nil
@@ -115,5 +138,16 @@ extension PostbuildContext {
         let thinFocusedTargetsString: String = env.readEnv(key: "SPT_XCREMOTE_CACHE_THINNED_TARGETS") ?? ""
         thinnedTargets = thinFocusedTargetsString.split(separator: ",").map(String.init)
         action = (try? BuildActionType(rawValue: env.readEnv(key: "ACTION"))) ?? .unknown
+        modeMarkerPath = config.modeMarkerPath
+        /// Note: The file has yaml extension, even it is in the json format
+        overlayHeadersPath = targetTempDir.appendingPathComponent("all-product-headers.yaml")
+        irrelevantDependenciesPaths = config.irrelevantDependenciesPaths
+        let publicHeadersPathEnv: String? = env.readEnv(key: "PUBLIC_HEADERS_FOLDER_PATH")
+        if let publicHeadersPath = publicHeadersPathEnv, publicHeadersPathEnv != "/usr/local/include" {
+            // '/usr/local/include' is a value of PUBLIC_HEADERS_FOLDER_PATH when no public headers are automatically
+            // generated and it is up to a project configuration to place it in a common location (e.g. static library)
+            publicHeadersFolderPath = builtProductsDir.appendingPathComponent(publicHeadersPath)
+        }
+        disabled = try env.readEnv(key: "XCRC_DISABLED") ?? false
     }
 }

Sources/XCRemoteCache/Commands/Postbuild/XCPostbuild.swift

@@ -19,6 +19,7 @@
 
 import Foundation
 
+// swiftlint:disable type_body_length
 /// Checks current mode from a configuration and based on that:
 /// * triggers build completion
 /// * triggers uploading artifacts to the server for a 'producer' mode
@@ -33,8 +34,9 @@ public class XCPostbuild {
         let context: PostbuildContext
         let cacheHitLogger: CacheHitLogger
         do {
-            config = try XCRemoteCacheConfigReader(env: env, fileManager: fileManager).readConfiguration()
+            config = try XCRemoteCacheConfigReader(env: env, fileReader: fileManager).readConfiguration()
             context = try PostbuildContext(config, env: env)
+            updateProcessTag(context.targetName)
             let counterFactory: FileStatsCoordinator.CountersFactory = { file, count in
                 ExclusiveFileCounter(ExclusiveFile(file, mode: .override), countersCount: count)
             }
@@ -66,9 +68,10 @@ public class XCPostbuild {
             // Initialize dependencies
             let primaryGitBranch = GitBranch(repoLocation: config.primaryRepo, branch: config.primaryBranch)
             let gitClient = GitClientImpl(repoRoot: config.repoRoot, primary: primaryGitBranch, shell: shellGetStdout)
-            let pathRemapper = try StringDependenciesRemapper.buildFromEnvs(
-                keys: DependenciesMapping.rewrittenEnvs,
-                envs: env
+            let envsRemapper = try PathDependenciesRemapperFactory().build(
+                orderKeys: DependenciesMapping.rewrittenEnvs + config.customRewriteEnvs,
+                envs: env,
+                customMappings: config.outOfBandMappings
             )
             let envFingerprint = try EnvironmentFingerprintGenerator(
                 configuration: config,
@@ -84,7 +87,14 @@ public class XCPostbuild {
                 fingerprintFilesGenerator,
                 algorithm: MD5Algorithm()
             )
-            let organizer = ZipArtifactOrganizer(targetTempDir: context.targetTempDir, fileManager: fileManager)
+            let organizer = ZipArtifactOrganizer(
+                targetTempDir: context.targetTempDir,
+                // In postbuild we don't preprocess artifacts (no need to replace path placeholders)
+                artifactProcessors: [],
+                fileManager: fileManager
+            )
+            let metaWriter = JsonMetaWriter(fileWriter: fileManager, pretty: config.prettifyMetaFiles)
+            let fileRemapper = TextFileDependenciesRemapper(remapper: envsRemapper, fileAccessor: fileManager)
             let artifactCreator = BuildArtifactCreator(
                 buildDir: context.productsDir,
                 tempDir: context.targetTempDir,
@@ -92,6 +102,8 @@ public class XCPostbuild {
                 moduleName: context.moduleName,
                 modulesFolderPath: context.modulesFolderPath,
                 dSYMPath: context.dSYMPath,
+                metaWriter: metaWriter,
+                artifactProcessor: UnzippedArtifactProcessor(fileRemapper: fileRemapper, dirScanner: fileManager),
                 fileManager: fileManager
             )
             let dirAccessor = DirAccessorComposer(
@@ -109,6 +121,7 @@ public class XCPostbuild {
                 awsV4Signature = AWSV4Signature(
                     secretKey: config.AWSSecretKey,
                     accessKey: config.AWSAccessKey,
+                    securityToken: config.AWSSecurityToken,
                     region: config.AWSRegion,
                     service: config.AWSService,
                     date: Date(timeIntervalSinceNow: 0)
@@ -117,6 +130,7 @@ public class XCPostbuild {
             let networkClient = NetworkClientImpl(
                 session: sessionFactory.build(),
                 retries: config.uploadRetries,
+                retryDelay: config.retryDelay,
                 fileManager: fileManager,
                 awsV4Signature: awsV4Signature
             )
@@ -124,6 +138,7 @@ public class XCPostbuild {
                 mode: context.mode,
                 downloadStreamURL: context.recommendedCacheAddress,
                 upstreamStreamURL: context.cacheAddresses,
+                uploadBatchSize: config.uploadBatchSize,
                 networkClient: networkClient,
                 urlBuilderFactory: {
                     try URLBuilderImpl(
@@ -142,12 +157,32 @@ public class XCPostbuild {
                 fileDependeciesReaderFactory: fileReaderFactory,
                 dirScanner: fileManager
             )
+            var remappers: [DependenciesRemapper] = []
+            if !config.disableVFSOverlay {
+                // As the PostbuildContext assumes file location and filename (`all-product-headers.yaml`)
+                // do not fail in case of a missing headers overlay file. In the future, all overlay files could be
+                // captured from the swiftc invocation similarly is stored in the `history.compile`
+                // for the consumer mode
+                let overlayReader = JsonOverlayReader(
+                    context.overlayHeadersPath,
+                    mode: .bestEffort,
+                    fileReader: fileManager
+                )
+                let overlayRemapper = OverlayDependenciesRemapper(
+                    overlayReader: overlayReader
+                )
+                remappers.append(overlayRemapper)
+            }
+            remappers.append(envsRemapper)
+            let pathRemapper = DependenciesRemapperComposite(remappers)
             let dependencyProcessor = DependencyProcessorImpl(
                 xcode: context.xcodeDir,
                 product: context.productsDir,
                 source: context.srcRoot,
                 intermediate: context.targetTempDir,
-                bundle: context.bundleDir
+                derivedFiles: context.derivedFilesDir,
+                bundle: context.bundleDir,
+                skippedRegexes: context.irrelevantDependenciesPaths
             )
             // Override fingerprints for all produced '.swiftmodule' files
             let fingerprintOverrideManager = FingerprintOverrideManagerImpl(
@@ -174,7 +209,10 @@ public class XCPostbuild {
                 if context.moduleName == config.thinningTargetModuleName {
                     switch context.mode {
                     case .consumer:
+                        // no need to process artifacts in postbuild. Prebuild has already
+                        // run a processor on a downloaded artifact
                         let artifactOrganizerFactory = ThinningConsumerZipArtifactsOrganizerFactory(
+                            processors: [],
                             fileManager: fileManager
                         )
                         let swiftProductsLocationProvider =
@@ -205,9 +243,10 @@ public class XCPostbuild {
                             worker: DispatchGroupParallelizationWorker(qos: .userInitiated)
                         )
                         consumerPlugins.append(thinningPlugin)
-                    case .producer:
+                    case .producer, .producerFast:
                         let thinningPlugin = ThinningCreatorPlugin(
                             targetTempDir: context.targetTempDir,
+                            modeMarkerPath: context.modeMarkerPath,
                             dirScanner: fileManager
                         )
                         creatorPlugins.append(thinningPlugin)
@@ -229,12 +268,18 @@ public class XCPostbuild {
                 dSYMOrganizer: dSYMOrganizer,
                 modeController: modeController,
                 metaReader: metaReader,
+                metaWriter: metaWriter,
                 creatorPlugins: creatorPlugins,
                 consumerPlugins: consumerPlugins
             )
 
             // Trigger build completion
-            if try modeController.isEnabled() {
+            if context.disabled {
+                infoLog("XCRC fully disabled for \(context.targetName), \(context.platform), \(context.configuration)")
+                // Cutoff the process is disabled, but generate an "empty" list of dependencies
+                try? modeController.disable()
+                return
+            } else if try modeController.isEnabled() {
                 // Decorate .swiftmodule in the product dir with fingerprint(s) overrides from a cache artifact
                 try postbuildAction.performBuildCompletion()
             } else if context.mode == .consumer {
@@ -243,11 +288,22 @@ public class XCPostbuild {
                 try postbuildAction.deleteFingerprintOverrides()
             }
 
+
             // Trigger uploading the artifact
-            if context.mode == .producer {
+            switch (context.mode, try modeController.isEnabled(), context.remoteCommit) {
+            case (.producerFast, true, .available(commit: let commitToReuse)):
+                // Upload only updated meta. Artifact zip is already on a remote server
+                let referenceCommit = try config.publishingSha ?? gitClient.getCurrentSha()
+                let metaData = try remoteNetworkClient.fetch(.meta(commit: commitToReuse))
+                let meta = try metaReader.read(data: metaData)
+                try postbuildAction.performMetaUpload(meta: meta, for: referenceCommit)
+            case (.producer, _, _), (.producerFast, _, _):
                 // Generate artifacts and upload to the remote server for a reference sha
                 let referenceCommit = try config.publishingSha ?? gitClient.getCurrentSha()
                 try postbuildAction.performBuildUpload(for: referenceCommit)
+            default:
+                // Consumer does not upload anything
+                break
             }
 
             let executableURL = context.productsDir.appendingPathComponent(context.executablePath)
@@ -261,9 +317,8 @@ public class XCPostbuild {
             } else {
                 try postbuildAction.performBuildCleanup()
                 try cacheHitLogger.logMiss()
-                // Producer mode doesn't use cached artifacts so modeController is not enabled. If producer
-                // reaches this point, there were no issues with publishing
-                let actionName = context.mode == .producer ? "Published" : "Disabled"
+                // If producers reach this point, there were no issues with publishing
+                let actionName = context.mode == .consumer ? "Disabled" : "Published"
                 printToUser("\(actionName) remote cache for \(context.targetName)")
             }
         } catch PluginError.unrecoverableError(let error) {
@@ -286,3 +341,4 @@ public class XCPostbuild {
         }
     }
 }
+// swiftlint:enable type_body_length

Sources/XCRemoteCache/Commands/Prebuild/Prebuild.swift

@@ -20,6 +20,7 @@
 import Foundation
 
 enum PrebuildResult: Equatable {
+    case disabled
     case incompatible
     case compatible(localDependencies: [URL])
 }
@@ -55,14 +56,20 @@ class Prebuild {
         self.artifactConsumerPrebuildPlugins = artifactConsumerPrebuildPlugins
     }
 
+    // swiftlint:disable:next function_body_length
     public func perform() throws -> PrebuildResult {
+        guard !context.disabled else {
+            return .disabled
+        }
         guard case .available(let commit) = context.remoteCommit else {
             return .incompatible
         }
         do {
             let metaData = try networkClient.fetch(.meta(commit: commit))
             let meta = try metaReader.read(data: metaData)
-            let localDependencies = remapper.replace(genericPaths: meta.dependencies).map(URL.init(fileURLWithPath:))
+            let localDependencies = try remapper.replace(
+                genericPaths: meta.dependencies
+            ).map(URL.init(fileURLWithPath:))
             let localFingerprint = try generateFingerprint(for: localDependencies)
             if localFingerprint.raw != meta.rawFingerprint {
                 if context.forceCached {

Sources/XCRemoteCache/Commands/Prebuild/PrebuildContext.swift

@@ -43,6 +43,11 @@ public struct PrebuildContext {
     let targetName: String
     /// List of all targets to downloaded from the thinning aggregation target
     var thinnedTargets: [String]?
+    /// location of the json file that define virtual files system overlay
+    /// (mappings of the virtual location file -> local file path)
+    let overlayHeadersPath: URL
+    /// XCRemoteCache is explicitly disabled
+    let disabled: Bool
 }
 
 extension PrebuildContext {
@@ -64,5 +69,8 @@ extension PrebuildContext {
         self.targetName = targetName
         let thinFocusedTargetsString: String? = env.readEnv(key: "SPT_XCREMOTE_CACHE_THINNED_TARGETS")
         thinnedTargets = thinFocusedTargetsString?.split(separator: ",").map(String.init)
+        /// Note: The file has yaml extension, even it is in the json format
+        overlayHeadersPath = targetTempDir.appendingPathComponent("all-product-headers.yaml")
+        disabled = try env.readEnv(key: "XCRC_DISABLED") ?? false
     }
 }

Sources/XCRemoteCache/Commands/Prebuild/XCPrebuild.swift

@@ -29,8 +29,9 @@ public class XCPrebuild {
         let config: XCRemoteCacheConfig
         let context: PrebuildContext
         do {
-            config = try XCRemoteCacheConfigReader(env: env, fileManager: fileManager).readConfiguration()
+            config = try XCRemoteCacheConfigReader(env: env, fileReader: fileManager).readConfiguration()
             context = try PrebuildContext(config, env: env)
+            updateProcessTag(context.targetName)
         } catch {
             // Fatal error:
             exit(1, "FATAL: Prebuild initialization failed with error: \(error)")
@@ -77,6 +78,10 @@ public class XCPrebuild {
             exit(0)
         }
 
+        let compilationHistoryOrganizer = CompilationHistoryFileOrganizer(
+            context.compilationHistoryFile,
+            fileManager: fileManager
+        )
         do {
             let envFingerprint = try EnvironmentFingerprintGenerator(
                 configuration: config,
@@ -95,6 +100,7 @@ public class XCPrebuild {
                 awsV4Signature = AWSV4Signature(
                     secretKey: config.AWSSecretKey,
                     accessKey: config.AWSAccessKey,
+                    securityToken: config.AWSSecurityToken,
                     region: config.AWSRegion,
                     service: config.AWSService,
                     date: Date(timeIntervalSinceNow: 0)
@@ -103,6 +109,7 @@ public class XCPrebuild {
             let networkClient = NetworkClientImpl(
                 session: sessionFactory.build(),
                 retries: config.downloadRetries,
+                retryDelay: config.retryDelay,
                 fileManager: fileManager,
                 awsV4Signature: awsV4Signature
             )
@@ -115,10 +122,27 @@ public class XCPrebuild {
             )
             let client: NetworkClient = config.disableHttpCache ? networkClient : cacheNetworkClient
             let remoteNetworkClient = RemoteNetworkClientImpl(client, urlBuilder)
-            let pathRemapper = try StringDependenciesRemapper.buildFromEnvs(
-                keys: DependenciesMapping.rewrittenEnvs,
-                envs: env
+            let envsRemapper = try PathDependenciesRemapperFactory().build(
+                orderKeys: DependenciesMapping.rewrittenEnvs + config.customRewriteEnvs,
+                envs: env,
+                customMappings: config.outOfBandMappings
             )
+            var remappers: [DependenciesRemapper] = []
+            if !config.disableVFSOverlay {
+                // As PrebuildContext assumes file location and its filename (`all-product-headers.yaml`)
+                // do not fail in case of a missing headers overlay file
+                let overlayReader = JsonOverlayReader(
+                    context.overlayHeadersPath,
+                    mode: .bestEffort,
+                    fileReader: fileManager
+                )
+                let overlayRemapper = OverlayDependenciesRemapper(
+                    overlayReader: overlayReader
+                )
+                remappers.append(overlayRemapper)
+            }
+            remappers.append(envsRemapper)
+            let pathRemapper = DependenciesRemapperComposite(remappers)
             let filesFingerprintGenerator = FingerprintAccumulatorImpl(
                 algorithm: MD5Algorithm(),
                 fileManager: fileManager
@@ -128,14 +152,25 @@ public class XCPrebuild {
                 filesFingerprintGenerator,
                 algorithm: MD5Algorithm()
             )
-            let organizer = ZipArtifactOrganizer(targetTempDir: context.targetTempDir, fileManager: fileManager)
-            let compilationHistoryOrganizer = CompilationHistoryFileOrganizer(context.compilationHistoryFile, fileManager: fileManager)
+            let fileRemapper = TextFileDependenciesRemapper(
+                remapper: envsRemapper,
+                fileAccessor: fileManager
+            )
+            let artifactProcessor = UnzippedArtifactProcessor(fileRemapper: fileRemapper, dirScanner: fileManager)
+            let organizer = ZipArtifactOrganizer(
+                targetTempDir: context.targetTempDir,
+                artifactProcessors: [artifactProcessor],
+                fileManager: fileManager
+            )
             let metaReader = JsonMetaReader(fileAccessor: fileManager)
             var consumerPlugins: [ArtifactConsumerPrebuildPlugin] = []
 
             if config.thinningEnabled {
                 if context.moduleName == config.thinningTargetModuleName, let thinnedTarget = context.thinnedTargets {
-                    let organizerFactory = ThinningConsumerZipArtifactsOrganizerFactory(fileManager: .default)
+                    let organizerFactory = ThinningConsumerZipArtifactsOrganizerFactory(
+                        processors: [artifactProcessor],
+                        fileManager: fileManager
+                    )
                     let aggregationPlugin = ThinningConsumerPrebuildPlugin(
                         targetName: context.targetName,
                         tempDir: context.targetTempDir,
@@ -167,7 +202,9 @@ public class XCPrebuild {
             case .compatible(localDependencies: let dependencies):
                 // TODO: pass `allowedInputFiles` observed in the build time
                 try modeController.enable(allowedInputFiles: dependencies, dependencies: dependencies)
-                compilationHistoryOrganizer.reset()
+            case .disabled:
+                infoLog("XCRemoteCache is explicitly disabled")
+                try modeController.disable()
             }
         } catch {
             disableRemoteCache(
@@ -175,6 +212,7 @@ public class XCPrebuild {
                 errorMessage: "Prebuild step failed with error: \(error)"
             )
         }
+        compilationHistoryOrganizer.reset()
     }
 
     private func disableRemoteCache(modeController: PhaseCacheModeController, errorMessage: String?) {

Sources/XCRemoteCache/Commands/Prepare/CCWrapperBuilder.swift

@@ -72,13 +72,24 @@ class TemplateBasedCCWrapperBuilder: CCWrapperBuilder {
         )
         infoLog("ClangWrapperBuilder compiles file at \(compilationFile).")
         // -O3: optimize for faster execution
-        let args = [clangCommand, "-O3", compilationFile.path, "-o", destination.path]
+        let args = [
+            clangCommand,
+            "-arch",
+            "arm64",
+            "-arch",
+            "x86_64",
+            "-O3",
+            compilationFile.path,
+            "-o",
+            destination.path,
+        ]
         let compilationOutput = try shell("xcrun", args, URL(fileURLWithPath: "").path, nil)
         infoLog("Clang compilation output: \(compilationOutput)")
     }
 
 
     /// Generates source of the cc wrapper
+    // swiftlint:disable line_length
     // swiftlint:disable:next function_body_length
     private func buildWrapperSource(clangCommand: String, markerFilename: String, commitSha: String) -> String {
         return """
@@ -384,6 +395,9 @@ class TemplateBasedCCWrapperBuilder: CCWrapperBuilder {
             const char *dependency_arg_name = "-MF";
             const char *output_arg_name = "-o";
             const char *serialize_diagnostics_arg_name = "--serialize-diagnostics";
+            const char *language_mode_arg_name = "-x";
+            const char *precompile_objc_header_arg_value = "objective-c-header";
+            const char *precompile_c_header_arg_value = "c-header";
             const char *clang_cmd = "\(clangCommand)";
             const char *markerFile = "\(markerFilename)";
             const char *compilationHistoryFile = "\(compilationHistoryFilename)";
@@ -398,6 +412,7 @@ class TemplateBasedCCWrapperBuilder: CCWrapperBuilder {
             const char *output_file= NULL;
             const char *input_file = NULL;
             const char *diagnostics_file = NULL;
+            const char *language_mode = NULL;
 
             for (int i = 1; i < argc; i++) {
                 if (strcmp(argv[i], dependency_arg_name) == 0 && i < (argc - 1) ) {
@@ -418,6 +433,12 @@ class TemplateBasedCCWrapperBuilder: CCWrapperBuilder {
                     i += 1;
                     clang_args[i] = argv[i];
                     diagnostics_file = argv[i];
+                } if (strcmp(argv[i], language_mode_arg_name) == 0 && i < (argc - 1) ) {
+                    // called with "-x path" pattern and not the last argument
+                    clang_args[i] = argv[i];
+                    i += 1;
+                    clang_args[i] = argv[i];
+                    language_mode = argv[i];
                 } else if (
                     isSuffixed(argv[i],".m") ||
                     isSuffixed(argv[i],".mm") ||
@@ -425,10 +446,13 @@ class TemplateBasedCCWrapperBuilder: CCWrapperBuilder {
                     isSuffixed(argv[i],".cc") ||
                     isSuffixed(argv[i],".cpp") ||
                     isSuffixed(argv[i],".c++") ||
-                    isSuffixed(argv[i],".cxx")
+                    isSuffixed(argv[i],".cxx") ||
+                    isSuffixed(argv[i],".S") ||
+                    isSuffixed(argv[i],".s")
                 ) {
                     // a full list of extensions is taken from https://clang.llvm.org/docs/ClangFormatStyleOptions.html
                     // support for .m,.mm,.c,.cc,.cpp,.c++,.cxx input files
+                    // .s and .S are assembly files
                     clang_args[i] = argv[i];
                     input_file = argv[i];
                 } else {
@@ -437,6 +461,14 @@ class TemplateBasedCCWrapperBuilder: CCWrapperBuilder {
                 }
             }
 
+           // null-terminating the args array needed for local compilation fallback
+           clang_args[argc] = NULL;
+
+           // Verify mode. Even a target is cached, pch mode is not supported. Fallback to the local compilation
+           if (language_mode != NULL && (strcmp(language_mode, precompile_objc_header_arg_value) == 0 || strcmp(language_mode, precompile_c_header_arg_value) == 0)) {
+               return execvp(clang_cmd, (char *const*) clang_args);
+           }
+
            // Verify all input arguments
            if (dependency_file == NULL) {
                fprintf(stderr, "error: missing %s input\\n", dependency_arg_name);
@@ -507,8 +539,6 @@ class TemplateBasedCCWrapperBuilder: CCWrapperBuilder {
                }
            }
 
-           // null-terminating the args array
-           clang_args[argc] = NULL;
            #pragma GCC diagnostic push
            #pragma GCC diagnostic ignored "-Wincompatible-pointer-types-discards-qualifiers"
            /// execvp takes $PATH to consideration
@@ -516,5 +546,5 @@ class TemplateBasedCCWrapperBuilder: CCWrapperBuilder {
            #pragma GCC diagnostic pop
         }
         """
-    } // swiftlint:disable:next file_length
-}
+    } // swiftlint:disable:next file_length line_length
+} // swiftlint:enable line_length

Sources/XCRemoteCache/Commands/Prepare/Integrate/BuildSettingsIntegrateAppender.swift

@@ -33,20 +33,34 @@ protocol BuildSettingsIntegrateAppender {
 class XcodeProjBuildSettingsIntegrateAppender: BuildSettingsIntegrateAppender {
     private let mode: Mode
     private let repoRoot: URL
+    private let fakeSrcRoot: URL
+    private let sdksExclude: [String]
 
-    init(mode: Mode, repoRoot: URL) {
+    init(mode: Mode, repoRoot: URL, fakeSrcRoot: URL, sdksExclude: [String]) {
         self.mode = mode
         self.repoRoot = repoRoot
+        self.fakeSrcRoot = fakeSrcRoot
+        self.sdksExclude = sdksExclude
     }
 
     func appendToBuildSettings(buildSettings: BuildSettings, wrappers: XCRCBinariesPaths) -> BuildSettings {
         var result = buildSettings
-        result["SWIFT_EXEC"] = wrappers.swiftc.path
+        setBuildSetting(buildSettings: &result, key: "SWIFT_EXEC", value: wrappers.swiftc.path )
+        setBuildSetting(buildSettings: &result, key: "SWIFT_USE_INTEGRATED_DRIVER", value: "NO" )
         // When generating artifacts, no need to shell-out all compilation commands to our wrappers
         if case .consumer = mode {
-            result["CC"] = wrappers.cc.path
-            result["LD"] = wrappers.ld.path
-            result["LIBTOOL"] = wrappers.libtool.path
+            setBuildSetting(buildSettings: &result, key: "CC", value: wrappers.cc.path )
+            setBuildSetting(buildSettings: &result, key: "LD", value: wrappers.ld.path )
+            // Setting LIBTOOL to '' breaks SwiftDriver intengration so resetting it to the original value
+            // 'libtool' for all excluded configurations
+            setBuildSetting(
+                buildSettings: &result,
+                key: "LIBTOOL",
+                value: wrappers.libtool.path,
+                excludedValue: "libtool"
+            )
+            setBuildSetting(buildSettings: &result, key: "LIPO", value: wrappers.lipo.path )
+            setBuildSetting(buildSettings: &result, key: "LDPLUSPLUS", value: wrappers.ldplusplus.path )
         }
 
         let existingSwiftFlags = result["OTHER_SWIFT_FLAGS"] as? String
@@ -58,10 +72,36 @@ class XcodeProjBuildSettingsIntegrateAppender: BuildSettingsIntegrateAppender {
         swiftFlags.assignFlag(key: "debug-prefix-map", value: "\(repoRoot.path)=$(XCRC_FAKE_SRCROOT)")
         clangFlags.assignFlag(key: "debug-prefix-map", value: "\(repoRoot.path)=$(XCRC_FAKE_SRCROOT)")
 
-        result["OTHER_SWIFT_FLAGS"] = swiftFlags.settingValue
-        result["OTHER_CFLAGS"] = clangFlags.settingValue
+        setBuildSetting(buildSettings: &result, key: "OTHER_SWIFT_FLAGS", value: swiftFlags.settingValue )
+        setBuildSetting(buildSettings: &result, key: "OTHER_CFLAGS", value: clangFlags.settingValue )
 
-        result["XCRC_FAKE_SRCROOT"] = "/\(String(repeating: "x", count: 10))"
+        setBuildSetting(buildSettings: &result, key: "XCRC_FAKE_SRCROOT", value: "\(fakeSrcRoot.path)" )
+        setBuildSetting(buildSettings: &result, key: "XCRC_PLATFORM_PREFERRED_ARCH", value:
+        """
+        $(LINK_FILE_LIST_$(CURRENT_VARIANT)_$(PLATFORM_PREFERRED_ARCH):dir:standardizepath:file:default=arm64)
+        """
+        )
+
+        explicitlyDisableSDKs(buildSettings: &result)
         return result
     }
+
+    private func setBuildSetting(buildSettings: inout BuildSettings, key: String, value: String?, excludedValue: String = "") {
+        buildSettings[key] = value
+        guard value != nil else {
+            // no need to exclude as the value will
+            return
+        }
+        // Erase all overrides for a given sdk so a default toolchain is used
+        for skippedSDK in sdksExclude {
+            buildSettings["\(key)[sdk=\(skippedSDK)]"] = excludedValue
+        }
+    }
+
+    // For all exlcuded SDKs passes XCRC_DISABLED=TRUE, which will cut-off early the pre_build phase
+    private func explicitlyDisableSDKs(buildSettings: inout BuildSettings) {
+        for skippedSDK in sdksExclude {
+            buildSettings["XCRC_DISABLED[sdk=\(skippedSDK)]"] = "YES"
+        }
+    }
 }

Sources/XCRemoteCache/Commands/Prepare/Integrate/IncludeOracle.swift

@@ -35,7 +35,7 @@ struct IncludeExcludeOracle: IncludeOracle {
 
 
     func shouldInclude(identifier: OracleIdentifierType) -> Bool {
-        // exclude array has precedence.
+        // exclude array has precedence
         if excludes.contains(identifier) {
             return false
         }

Sources/XCRemoteCache/Commands/Prepare/Integrate/IntegrateContext.swift

@@ -52,7 +52,9 @@ extension IntegrateContext {
             cc: binariesDir.appendingPathComponent("xccc"),
             swiftc: binariesDir.appendingPathComponent("xcswiftc"),
             libtool: binariesDir.appendingPathComponent("xclibtool"),
+            lipo: binariesDir.appendingPathComponent("xclipo"),
             ld: binariesDir.appendingPathComponent("xcld"),
+            ldplusplus: binariesDir.appendingPathComponent("xcldplusplus"),
             prebuild: binariesDir.appendingPathComponent("xcprebuild"),
             postbuild: binariesDir.appendingPathComponent("xcpostbuild")
         )

Sources/XCRemoteCache/Commands/Prepare/Integrate/LLDBInitPatcher.swift

@@ -56,7 +56,7 @@ class FileLLDBInitPatcher: LLDBInitPatcher {
     }
 
     private func findIndices(in collection: [String], value: String) -> [Int] {
-        collection.enumerated().reduce([]) { (result, line) -> [Int] in
+        collection.enumerated().reduce([]) { result, line -> [Int] in
             if line.element == Self.preambleString {
                 return result + [line.offset]
             }
@@ -75,7 +75,7 @@ class FileLLDBInitPatcher: LLDBInitPatcher {
             var contentLines = originalContentLines
             let preambleIndices = findIndices(in: contentLines, value: Self.preambleString)
 
-            if preambleIndices.count > 0 {
+            if !preambleIndices.isEmpty {
                 let firstLLDBCommandIndex = preambleIndices[0] + 1
                 if firstLLDBCommandIndex >= contentLines.count {
                     // corrupted file, append the script line at the bottom

Sources/XCRemoteCache/Commands/Prepare/Integrate/XCIntegrate.swift

@@ -34,6 +34,7 @@ public class XCIntegrate {
     private let consumerEligiblePlatforms: String
     private let lldbMode: LLDBInitMode
     private let fakeSrcRoot: String
+    private let sdksExclude: String
     private let output: String?
 
     public init(
@@ -48,6 +49,7 @@ public class XCIntegrate {
         consumerEligiblePlatforms: String,
         lldbMode: LLDBInitMode,
         fakeSrcRoot: String,
+        sdksExclude: String,
         output: String?
     ) {
         projectPath = input
@@ -61,9 +63,11 @@ public class XCIntegrate {
         self.consumerEligiblePlatforms = consumerEligiblePlatforms
         self.lldbMode = lldbMode
         self.fakeSrcRoot = fakeSrcRoot
+        self.sdksExclude = sdksExclude
         self.output = output
     }
 
+    // swiftlint:disable:next function_body_length
     public func main() {
         do {
             let env = ProcessInfo.processInfo.environment
@@ -73,7 +77,7 @@ public class XCIntegrate {
             let binariesDir = commandURL.deletingLastPathComponent()
 
             let srcRoot: URL = URL(fileURLWithPath: projectPath).deletingLastPathComponent()
-            let config = try XCRemoteCacheConfigReader(srcRootPath: srcRoot.path, fileManager: fileManager)
+            let config = try XCRemoteCacheConfigReader(srcRootPath: srcRoot.path, fileReader: fileManager)
                 .readConfiguration()
 
             let context = try IntegrateContext(
@@ -96,7 +100,9 @@ public class XCIntegrate {
             )
             let buildSettingsAppender = XcodeProjBuildSettingsIntegrateAppender(
                 mode: context.mode,
-                repoRoot: context.repoRoot
+                repoRoot: context.repoRoot,
+                fakeSrcRoot: context.fakeSrcRoot,
+                sdksExclude: sdksExclude.integrateArrayArguments
             )
             let lldbPatcher: LLDBInitPatcher
             switch lldbMode {
@@ -114,7 +120,7 @@ public class XCIntegrate {
 
             let integrator = XcodeProjIntegrate(
                 project: context.projectPath,
-                mode:context.mode,
+                mode: context.mode,
                 binaries: context.binaries,
                 configurationIncludeOracle: configurationOracle,
                 targetIncludeOracle: targetOracle,

Sources/XCRemoteCache/Commands/Prepare/Integrate/XCRCBinariesPaths.swift

@@ -25,7 +25,9 @@ struct XCRCBinariesPaths {
     let cc: URL
     let swiftc: URL
     let libtool: URL
+    let lipo: URL
     let ld: URL
+    let ldplusplus: URL
     let prebuild: URL
     let postbuild: URL
 }

Sources/XCRemoteCache/Commands/Prepare/Integrate/XcodeProjIntegrate.swift

@@ -100,11 +100,11 @@ struct XcodeProjIntegrate: Integrate {
             outputPaths: [
                 """
                 $(TARGET_BUILD_DIR)/$(MODULES_FOLDER_PATH)/$(PRODUCT_MODULE_NAME).swiftmodule/\
-                $(PLATFORM_PREFERRED_ARCH).swiftmodule.md5
+                $(XCRC_PLATFORM_PREFERRED_ARCH).swiftmodule.md5
                 """,
                 """
                 $(TARGET_BUILD_DIR)/$(MODULES_FOLDER_PATH)/$(PRODUCT_MODULE_NAME).swiftmodule/\
-                $(PLATFORM_PREFERRED_ARCH)-$(LLVM_TARGET_TRIPLE_VENDOR)-$(SWIFT_PLATFORM_TARGET_PREFIX)\
+                $(XCRC_PLATFORM_PREFERRED_ARCH)-$(LLVM_TARGET_TRIPLE_VENDOR)-$(SWIFT_PLATFORM_TARGET_PREFIX)\
                 $(LLVM_TARGET_TRIPLE_SUFFIX).swiftmodule.md5
                 """,
             ],
@@ -114,7 +114,9 @@ struct XcodeProjIntegrate: Integrate {
         markPhase = PBXShellScriptBuildPhase(
             name: "\(Self.BuildStepPrefix)RemoteCache_mark",
             inputPaths: [binaries.prepare.path],
-            shellScript: "\"$SCRIPT_INPUT_FILE_0\" mark --configuration \"$CONFIGURATION\" --platform \"$PLATFORM_NAME\""
+            shellScript:
+                "\"$SCRIPT_INPUT_FILE_0\" mark " +
+                "--configuration \"$CONFIGURATION\" --platform \"$PLATFORM_NAME\""
         )
     }
 
@@ -129,6 +131,7 @@ struct XcodeProjIntegrate: Integrate {
         try encodedYAML.write(to: configOverrideLocation, atomically: false, encoding: .utf8)
     }
 
+    // swiftlint:disable:next function_body_length
     func run() throws {
         let outputFile = output ?? projectURL
         let projectRoot = projectURL.deletingLastPathComponent()
@@ -224,13 +227,15 @@ struct XcodeProjIntegrate: Integrate {
         let previousRCPhases = target.buildPhases.filter(isRCPhase)
         target.buildPhases.removeAll(where: previousRCPhases.contains)
 
-        if target.buildPhases.map(\.buildPhase).contains(.sources) {
+        if let sourceIndex = target.buildPhases.map(\.buildPhase).firstIndex(of: .sources) {
             // add (pre|post)build phases only when a target has some compilation steps
             // otherwise they make no sense (nothing to store in an artifact)
-            pbxproj.add(object: prebuildPhase)
-            target.buildPhases.insert(prebuildPhase, at: 0)
+            // add postbuild right after compilation as custom build steps may depend on files generated by
+            // the xcpostbuild (e.g. to copy -Swift.h.md5)
             pbxproj.add(object: postbuildPhase)
-            target.buildPhases.append(postbuildPhase)
+            target.buildPhases.insert(postbuildPhase, at: sourceIndex + 1)
+            pbxproj.add(object: prebuildPhase)
+            target.buildPhases.insert(prebuildPhase, at: sourceIndex)
         }
     }
 

Sources/XCRemoteCache/Commands/Prepare/Integrate/XcodeSettingsFlags.swift

@@ -100,7 +100,7 @@ struct XcodeSettingsCFlags: XcodeSettingsFlags {
         case (.some(let existing), _):
             var flagsComponents: [String] = existing.split(separator: " ").map(String.init)
             // remove (if exists)
-            let existingFlagIndex = flagsComponents.firstIndex { (component) -> Bool in
+            let existingFlagIndex = flagsComponents.firstIndex { component -> Bool in
                 component.hasPrefix("\(Self.prefix)\(key)=")
             }
             if let index = existingFlagIndex {

Sources/XCRemoteCache/Commands/Prepare/Prepare.swift

@@ -77,7 +77,18 @@ class Prepare: PrepareLogic {
             guard fileAccessor.fileExists(atPath: PhaseCacheModeController.xcodeSelectLink.path) else {
                 throw PrepareError.missingXcodeSelectDirectory
             }
-            let commonSha = try gitClient.getCommonPrimarySha()
+
+            let commonSha: String
+            do {
+                commonSha = try gitClient.getCommonPrimarySha()
+            } catch let GitClientError.noCommonShaWithPrimaryRepo(remoteName, error) {
+                guard context.gracefullyHandleMissingCommonSha else {
+                    throw GitClientError.noCommonShaWithPrimaryRepo(remoteName: remoteName, error: error)
+                }
+                infoLog("Cannot find a common sha with the primary branch: \(error). Gracefully disabling remote cache")
+                try disable()
+                return .failed
+            }
 
             if context.offline {
                 // Optimistically take first common sha

Sources/XCRemoteCache/Commands/Prepare/PrepareContext.swift

@@ -52,6 +52,8 @@ public struct PrepareContext {
     let cacheHealthPathProbeCount: Int
     /// clang wrapper output file
     let xcccCommand: URL
+    /// gracefully disable remote cache for missing common sha with the primary branch
+    let gracefullyHandleMissingCommonSha: Bool
 }
 
 extension PrepareContext {
@@ -77,5 +79,6 @@ extension PrepareContext {
         cacheAddresses = try config.cacheAddresses.map(URL.build)
         cacheHealthPath = config.cacheHealthPath
         cacheHealthPathProbeCount = config.cacheHealthPathProbeCount
+        gracefullyHandleMissingCommonSha = config.gracefullyHandleMissingCommonSha
     }
 }

Sources/XCRemoteCache/Commands/Prepare/PrepareMarkContext.swift

@@ -31,10 +31,12 @@ public struct PrepareMarkContext {
     let recommendedCacheAddress: URL
     /// All remote servers to mark
     let cacheAddresses: [URL]
+    /// XCRemoteCache is explicitly disabled
+    let disabled: Bool
 }
 
 extension PrepareMarkContext {
-    init(_ config: XCRemoteCacheConfig) throws {
+    init(_ config: XCRemoteCacheConfig, env: [String: String]) throws {
         let sourceRoot = URL(fileURLWithPath: config.sourceRoot, isDirectory: true)
         repoRoot = URL(fileURLWithPath: config.repoRoot, relativeTo: sourceRoot)
         guard let address = URL(string: config.recommendedCacheAddress) else {
@@ -43,5 +45,6 @@ extension PrepareMarkContext {
         }
         recommendedCacheAddress = address
         cacheAddresses = try config.cacheAddresses.map(URL.build)
+        disabled = try env.readEnv(key: "XCRC_DISABLED") ?? false
     }
 }

Sources/XCRemoteCache/Commands/Prepare/XCConfig.swift

@@ -18,7 +18,6 @@
 // under the License.
 
 import Foundation
-import Yams
 
 /// Print current configuration to the console
 public class XCConfig {
@@ -33,7 +32,7 @@ public class XCConfig {
         let fileManager = FileManager.default
         let config: XCRemoteCacheConfig
         do {
-            config = try XCRemoteCacheConfigReader(env: env, fileManager: fileManager).readConfiguration()
+            config = try XCRemoteCacheConfigReader(env: env, fileReader: fileManager).readConfiguration()
         } catch {
             exit(1, "FATAL: Prepare initialization failed with error: \(error)")
         }

Sources/XCRemoteCache/Commands/Prepare/XCPrepare.swift

@@ -18,7 +18,6 @@
 // under the License.
 
 import Foundation
-import Yams
 
 /// Switch between Online/Offline modes
 public enum XCPrepareMode {
@@ -62,7 +61,7 @@ public class XCPrepare {
         var context: PrepareContext
         let xcodeVersion: String
         do {
-            config = try XCRemoteCacheConfigReader(env: env, fileManager: fileManager).readConfiguration()
+            config = try XCRemoteCacheConfigReader(env: env, fileReader: fileManager).readConfiguration()
             context = try PrepareContext(config, offline: offline)
             xcodeVersion = try customXcodeBuildNumber ?? XcodeProbeImpl(shell: shellGetStdout).read().buildVersion
         } catch {
@@ -79,6 +78,7 @@ public class XCPrepare {
                 awsV4Signature = AWSV4Signature(
                     secretKey: config.AWSSecretKey,
                     accessKey: config.AWSAccessKey,
+                    securityToken: config.AWSSecurityToken,
                     region: config.AWSRegion,
                     service: config.AWSService,
                     date: Date(timeIntervalSinceNow: 0)
@@ -87,6 +87,7 @@ public class XCPrepare {
             let networkClient = NetworkClientImpl(
                 session: sessionFactory.build(),
                 retries: config.downloadRetries,
+                retryDelay: config.retryDelay,
                 fileManager: fileManager,
                 awsV4Signature: awsV4Signature
             )

Sources/XCRemoteCache/Commands/Prepare/XCPrepareMark.swift

@@ -38,6 +38,7 @@ public class XCPrepareMark {
         self.commit = commit
     }
 
+    // swiftlint:disable:next function_body_length
     public func main() {
         let env = ProcessInfo.processInfo.environment
         let fileManager = FileManager.default
@@ -45,13 +46,18 @@ public class XCPrepareMark {
         let context: PrepareMarkContext
         let xcodeVersion: String
         do {
-            config = try XCRemoteCacheConfigReader(env: env, fileManager: fileManager).readConfiguration()
-            context = try PrepareMarkContext(config)
+            config = try XCRemoteCacheConfigReader(env: env, fileReader: fileManager).readConfiguration()
+            context = try PrepareMarkContext(config, env: env)
             xcodeVersion = try xcode ?? XcodeProbeImpl(shell: shellGetStdout).read().buildVersion
         } catch {
             exit(1, "FATAL: Prepare initialization failed with error: \(error)")
         }
 
+        guard !context.disabled else {
+            infoLog("XCRemoteCache explicitly disabled for marking.")
+            return
+        }
+
         do {
             let sessionFactory = DefaultURLSessionFactory(config: config)
             var awsV4Signature: AWSV4Signature?
@@ -59,6 +65,7 @@ public class XCPrepareMark {
                 awsV4Signature = AWSV4Signature(
                     secretKey: config.AWSSecretKey,
                     accessKey: config.AWSAccessKey,
+                    securityToken: config.AWSSecurityToken,
                     region: config.AWSRegion,
                     service: config.AWSService,
                     date: Date(timeIntervalSinceNow: 0)
@@ -67,6 +74,7 @@ public class XCPrepareMark {
             let networkClient = NetworkClientImpl(
                 session: sessionFactory.build(),
                 retries: config.uploadRetries,
+                retryDelay: config.retryDelay,
                 fileManager: fileManager,
                 awsV4Signature: awsV4Signature
             )
@@ -74,6 +82,7 @@ public class XCPrepareMark {
                 mode: .producer,
                 downloadStreamURL: context.recommendedCacheAddress,
                 upstreamStreamURL: context.cacheAddresses,
+                uploadBatchSize: config.uploadBatchSize,
                 networkClient: networkClient
             ) { [configuration, platform] cacheAddress in
                 // Prepare URLs don't include target name or envFingperint, which are valid only for a target level

Sources/XCRemoteCache/Commands/Prepare/XCStats.swift

@@ -18,7 +18,6 @@
 // under the License.
 
 import Foundation
-import Yams
 
 /// Manages XCRemoteCache statistics: rests, print to the standard output etc
 public class XCStats {
@@ -37,7 +36,7 @@ public class XCStats {
         let config: XCRemoteCacheConfig
         let context: XCStatsContext
         do {
-            config = try XCRemoteCacheConfigReader(env: env, fileManager: fileManager).readConfiguration()
+            config = try XCRemoteCacheConfigReader(env: env, fileReader: fileManager).readConfiguration()
             try context = XCStatsContext(config, fileManager: fileManager)
         } catch {
             exit(1, "FATAL: Prepare initialization failed with error: \(error)")

Sources/XCRemoteCache/Commands/ProductBinaryCreator/XCCreateBinary.swift

@@ -70,7 +70,7 @@ public class XCCreateBinary {
         let config: XCRemoteCacheConfig
         do {
             let srcRoot: URL = URL(fileURLWithPath: fileManager.currentDirectoryPath)
-            config = try XCRemoteCacheConfigReader(srcRootPath: srcRoot.path, fileManager: fileManager)
+            config = try XCRemoteCacheConfigReader(srcRootPath: srcRoot.path, fileReader: fileManager)
                 .readConfiguration()
         } catch {
             errorLog("\(stepDescription) initialization failed with error: \(error). Fallbacking to \(fallbackCommand)")
@@ -78,7 +78,12 @@ public class XCCreateBinary {
         }
         let markerURL = tempDir.appendingPathComponent(config.modeMarkerPath)
         do {
-            let organizer = ZipArtifactOrganizer(targetTempDir: tempDir, fileManager: fileManager)
+            let organizer = ZipArtifactOrganizer(
+                targetTempDir: tempDir,
+                // Creation binary doesn't call artifact preprocessing
+                artifactProcessors: [],
+                fileManager: fileManager
+            )
             let dependenciesWriter = FileDatWriter(dependencyInfo, fileManager: fileManager)
             let markerReader = FileMarkerReader(markerURL, fileManager: fileManager)
             guard fileManager.fileExists(atPath: markerURL.path) else {

Sources/XCRemoteCache/Commands/Swiftc/MirroredLinkingSwiftcProductsGenerator.swift

@@ -55,7 +55,7 @@ class MirroredLinkingSwiftcProductsGenerator: SwiftcProductsGenerator {
     func generateFrom(
         artifactSwiftModuleFiles: [SwiftmoduleFileExtension: URL],
         artifactSwiftModuleObjCFile: URL
-    ) throws -> URL {
+    ) throws -> SwiftcProductsGeneratorOutput {
         /// Predict moduleName from the `*.swiftmodule` artifact
         let foundSwiftmoduleFile = artifactSwiftModuleFiles[.swiftmodule]
         guard let mainSwiftmoduleFile = foundSwiftmoduleFile else {

Sources/XCRemoteCache/Commands/Swiftc/Swiftc.swift

@@ -120,9 +120,9 @@ class Swiftc: SwiftcProtocol {
                 let prebuildDiscoveryURL = context.tempDir.appendingPathComponent(context.prebuildDependenciesPath)
                 let prebuildDiscoverWriter = dependenciesWriterFactory(prebuildDiscoveryURL, fileManager)
                 try prebuildDiscoverWriter.write(skipForSha: remoteCommit)
-            case .consumer, .producer:
-                // Never skips prebuild phase and fallbacks to the swiftc compilation for:
-                // 1) Not enabled remote cache or 2) producer
+            case .consumer, .producer, .producerFast:
+                // Never skip prebuild phase and fallback to the swiftc compilation for:
+                // 1) Not enabled remote cache, 2) producer(s)
                 break
             }
             return .forceFallback

Sources/XCRemoteCache/Commands/Swiftc/SwiftcContext.swift

@@ -24,6 +24,8 @@ public struct SwiftcContext {
         case producer
         /// Commit sha of the commit to use during remote cache
         case consumer(commit: RemoteCommitInfo)
+        /// Remote artifact exists and can be optimistically used in place of a local compilation
+        case producerFast
     }
 
     let objcHeaderOutput: URL
@@ -74,6 +76,14 @@ public struct SwiftcContext {
             mode = .consumer(commit: remoteCommit)
         case .producer:
             mode = .producer
+        case .producerFast:
+            let remoteCommit = RemoteCommitInfo(try? String(contentsOf: remoteCommitLocation).trim())
+            switch remoteCommit {
+            case .unavailable:
+                mode = .producer
+            case .available:
+                mode = .producerFast
+            }
         }
         invocationHistoryFile = URL(fileURLWithPath: config.compilationHistoryFile, relativeTo: tempDir)
     }

Sources/XCRemoteCache/Commands/Swiftc/SwiftcOrchestrator.swift

@@ -110,12 +110,18 @@ class SwiftcOrchestrator {
                     try invocationStorage.store(args: invocationArgs)
                 }
             } catch {
-                // The critical section is protected by a lock. Some other process already called compilation history.
-                // We only need to call our current step then.
+                // The critical section is protected by a lock. Some other process already called compilation history
+                // We only need to call our current step then
                 fallbackToDefault(command: swiftcCommand)
             }
         case .consumer:
             fallbackToDefault(command: swiftcCommand)
+        case .producerFast:
+            let compileStepResult = try swiftc.mockCompilation()
+            if case .forceFallback = compileStepResult {
+                // cannot reuse cached artifact. Build it locally and upload to the server just as for the producer
+                fallthrough
+            }
         case .producer:
             var swiftcArgs = ProcessInfo().arguments
             swiftcArgs = try producerFallbackCommandProcessors.reduce(swiftcArgs) { args, processor in

Sources/XCRemoteCache/Commands/Swiftc/SwiftcProductsGenerator.swift

@@ -20,20 +20,25 @@
 import Foundation
 
 enum DiskSwiftcProductsGeneratorError: Error {
-    /// When a generator was asked to generate unknown swiftmodule extension file.
+    /// When a generator was asked to generate unknown swiftmodule extension file
     /// Probably a programmer error: asking to generate excessive extensions, not listed in
     /// `SwiftmoduleFileExtension.SwiftmoduleExtensions`
     case unknownSwiftmoduleFile
 }
 
+struct SwiftcProductsGeneratorOutput {
+    let swiftmoduleDir: URL
+    let objcHeaderFile: URL
+}
+
 /// Generates swiftc product to the expected location
 protocol SwiftcProductsGenerator {
     /// Generates products from given files
-    /// - Returns: location dir where .swiftmodule files have been placed
+    /// - Returns: location dir where .swiftmodule and ObjC header files have been placed
     func generateFrom(
         artifactSwiftModuleFiles: [SwiftmoduleFileExtension: URL],
         artifactSwiftModuleObjCFile: URL
-    ) throws -> URL
+    ) throws -> SwiftcProductsGeneratorOutput
 }
 
 /// Generator that produces all products in the locations where Xcode expects it, using provided disk copier
@@ -64,7 +69,7 @@ class DiskSwiftcProductsGenerator: SwiftcProductsGenerator {
     func generateFrom(
         artifactSwiftModuleFiles sourceAtifactSwiftModuleFiles: [SwiftmoduleFileExtension: URL],
         artifactSwiftModuleObjCFile: URL
-    ) throws -> URL {
+    ) throws -> SwiftcProductsGeneratorOutput {
         // Move cached -Swift.h file to the expected location
         try diskCopier.copy(file: artifactSwiftModuleObjCFile, destination: objcHeaderOutput)
         for (ext, url) in sourceAtifactSwiftModuleFiles {
@@ -85,6 +90,9 @@ class DiskSwiftcProductsGenerator: SwiftcProductsGenerator {
         }
 
         // Build parent dir of the .swiftmodule file that contains a module
-        return modulePathOutput.deletingLastPathComponent()
+        return SwiftcProductsGeneratorOutput(
+            swiftmoduleDir: modulePathOutput.deletingLastPathComponent(),
+            objcHeaderFile: objcHeaderOutput
+        )
     }
 }

Sources/XCRemoteCache/Commands/Swiftc/XCSwiftc.swift

@@ -70,7 +70,7 @@ public class XCSwiftc {
         let context: SwiftcContext
         do {
             let srcRoot: URL = URL(fileURLWithPath: fileManager.currentDirectoryPath)
-            config = try XCRemoteCacheConfigReader(srcRootPath: srcRoot.path, fileManager: fileManager)
+            config = try XCRemoteCacheConfigReader(srcRootPath: srcRoot.path, fileReader: fileManager)
                 .readConfiguration()
             context = try SwiftcContext(config: config, input: inputArgs)
         } catch {
@@ -83,7 +83,12 @@ public class XCSwiftc {
 
         let inputReader = SwiftcFilemapInputEditor(context.filemap, fileManager: fileManager)
         let fileListEditor = FileListEditor(context.fileList, fileManager: fileManager)
-        let artifactOrganizer = ZipArtifactOrganizer(targetTempDir: context.tempDir, fileManager: fileManager)
+        let artifactOrganizer = ZipArtifactOrganizer(
+            targetTempDir: context.tempDir,
+            // xcswiftc  doesn't call artifact preprocessing
+            artifactProcessors: [],
+            fileManager: fileManager
+        )
         // TODO: check for allowedFile comparing a list of all inputfiles, not dependencies from a marker
         let makerReferencedFilesListScanner = FileListScannerImpl(markerReader, caseSensitive: false)
         let allowedFilesListScanner = ExceptionsFilteredFileListScanner(
@@ -101,7 +106,10 @@ public class XCSwiftc {
             objcHeaderOutput: context.objcHeaderOutput,
             diskCopier: HardLinkDiskCopier(fileManager: fileManager)
         )
-        let allInvocationsStorage = ExistingFileStorage(storageFile: context.invocationHistoryFile, command: swiftcCommand)
+        let allInvocationsStorage = ExistingFileStorage(
+            storageFile: context.invocationHistoryFile,
+            command: swiftcCommand
+        )
         // When fallbacking to local compilation do not call historical `swiftc` invocations
         // The current fallback invocation already compiles all files in a target
         let invocationStorage = FilteredInvocationStorage(

Sources/XCRemoteCache/Config/Mode.swift

@@ -20,4 +20,5 @@
 public enum Mode: String, Codable, CaseIterable {
     case consumer
     case producer
+    case producerFast = "producer-fast"
 }

Sources/XCRemoteCache/Config/XCRemoteCacheConfig.swift

@@ -83,6 +83,10 @@ public struct XCRemoteCacheConfig: Encodable {
     var downloadRetries: Int = 0
     /// Number of retries for upload requests
     var uploadRetries: Int = 3
+    /// Delay between retries in seconds
+    var retryDelay: Double = 10.0
+    /// Maximum number of simultaneous requests. 0 means no limits
+    var uploadBatchSize: Int = 0
     /// Extra headers appended to all remote HTTP(S) requests
     var requestCustomHeaders: [String: String] = [:]
     /// Filename (without an extension) of the compilation input file that is used
@@ -94,10 +98,11 @@ public struct XCRemoteCacheConfig: Encodable {
     var focusedTargets: [String] = []
     ///  Disable cache for http requests to fecth metadata and download artifacts
     var disableHttpCache: Bool = false
-    /// Path, relative to $TARGET_TEMP_DIR which gathers all compilation commands that should be executed if a target
-    /// switches to local compilation. Example: A new `.swift` file invalidates remote artifact and triggers local compilation
-    /// When that happens, all previously skipped clang build steps need to be eventually called locally - this file lists
-    /// all these commands.
+    /// Path, relative to $TARGET_TEMP_DIR which gathers all compilation commands that should be e
+    /// xecuted if a target switches to local compilation
+    /// Example: A new `.swift` file invalidates remote arXcodeProjIntegrate.swifttifact and triggers local compilation
+    /// When that happens, all previously skipped clang build steps
+    /// need to be eventually called locally - this file lists all these commands
     var compilationHistoryFile: String = "history.compile"
     /// Timeout for remote response data interval (in seconds). If an interval between data chunks is
     /// longer than a timeout, a request fails
@@ -106,24 +111,52 @@ public struct XCRemoteCacheConfig: Encodable {
     var turnOffRemoteCacheOnFirstTimeout: Bool = false
     /// List of all extensions that should carry over source fingerprints. Extensions of all product files that
     /// contain non-deterministic content (absolute paths, timestamp, etc) should be included
-    var productFilesExtensionsWithContentOverride = ["swiftmodule"]
+    /// .h files may contain absolute paths if NS_ENUM is used in a public API from Swift code
+    var productFilesExtensionsWithContentOverride = ["swiftmodule", "h"]
     /// If true, plugins for thinning support should be enabled
     var thinningEnabled: Bool = false
     /// Module name of a target that works as a helper for thinned targets
     var thinningTargetModuleName: String = "ThinningRemoteCacheModule"
+    /// Opt-in pretty json formatting for meta files
+    var prettifyMetaFiles: Bool = false
     /// Secret key for AWS V4 Signature, if this is set the Authentication Header will be added
     var AWSSecretKey: String = ""
     /// Access key for AWS V4 Signature
     var AWSAccessKey: String = ""
+    /// Temporary security token provided by the AWS Security Token Service
+    var AWSSecurityToken: String?
     /// Region for AWS V4 Signature (e.g. `eu`)
     var AWSRegion: String = ""
     /// Service for AWS V4 Signature (e.g. `storage`)
     var AWSService: String = ""
+    /// A dictionary of files path remapping that should be applied to make it absolute path agnostic on a list of
+    /// dependencies. Useful if a project refers files out of repo root, either compilation files or precompiled
+    /// dependencies. Keys represent generic replacement and values are substrings that should be replaced
+    /// Example: for mapping `["COOL_LIBRARY": "/CoolLibrary"]`
+    /// `/CoolLibrary/main.swift`will be represented as `$(COOL_LIBRARY)/main.swift`)
+    /// Warning: remapping order is not-deterministic so avoid remappings with multiple matchings
+    var outOfBandMappings: [String: String] = [:]
+    /// If true, SSL certificate validation is disabled
+    var disableCertificateVerification: Bool = false
+    /// A feature flag to disable virtual file system overlay support (temporary)
+    var disableVFSOverlay: Bool = false
+    /// A list of extra ENVs that should be used as placeholders in the dependency list
+    /// ENV rewrite process is optimistic - does nothing if an ENV is not defined in the pre/postbuild process
+    var customRewriteEnvs: [String] = []
+    /// Regexes of files that should not be included in a list of dependencies. Warning! Add entries here
+    /// with caution - excluding dependencies that are relevant might lead to a target overcaching
+    /// Note: The regex can match either partially or fully the filepath, e.g. `\\.modulemap$` will exclude
+    /// all `.modulemap` files
+    var irrelevantDependenciesPaths: [String] = []
+    /// If true, do not fail `prepare` if cannot find the most recent common commits with the primary branch
+    /// That might useful on CI, where a shallow clone is used
+    var gracefullyHandleMissingCommonSha: Bool = false
 }
 
 extension XCRemoteCacheConfig {
     /// Merges existing config with the other config and returns a final result
     /// `other` scheme overrides existing configuration
+    // swiftlint:disable:next function_body_length
     func merged(with scheme: ConfigFileScheme) -> XCRemoteCacheConfig {
         var merge = self
         merge.mode = scheme.mode ?? mode
@@ -150,6 +183,8 @@ extension XCRemoteCacheConfig {
         merge.statsDir = scheme.statsDir ?? statsDir
         merge.downloadRetries = scheme.downloadRetries ?? downloadRetries
         merge.uploadRetries = scheme.uploadRetries ?? uploadRetries
+        merge.retryDelay = scheme.retryDelay ?? retryDelay
+        merge.uploadBatchSize = scheme.uploadBatchSize ?? uploadBatchSize
         merge.requestCustomHeaders = scheme.requestCustomHeaders ?? requestCustomHeaders
         merge.thinTargetMockFilename = scheme.thinTargetMockFilename ?? thinTargetMockFilename
         merge.focusedTargets = scheme.focusedTargets ?? focusedTargets
@@ -163,10 +198,19 @@ extension XCRemoteCacheConfig {
             scheme.productFilesExtensionsWithContentOverride ?? productFilesExtensionsWithContentOverride
         merge.thinningEnabled = scheme.thinningEnabled ?? thinningEnabled
         merge.thinningTargetModuleName = scheme.thinningTargetModuleName ?? thinningTargetModuleName
+        merge.prettifyMetaFiles = scheme.prettifyMetaFiles ?? prettifyMetaFiles
         merge.AWSAccessKey = scheme.AWSAccessKey ?? AWSAccessKey
         merge.AWSSecretKey = scheme.AWSSecretKey ?? AWSSecretKey
+        merge.AWSSecurityToken = scheme.AWSSecurityToken ?? AWSSecurityToken
         merge.AWSRegion = scheme.AWSRegion ?? AWSRegion
         merge.AWSService = scheme.AWSService ?? AWSService
+        merge.outOfBandMappings = scheme.outOfBandMappings ?? outOfBandMappings
+        merge.disableCertificateVerification = scheme.disableCertificateVerification ?? disableCertificateVerification
+        merge.disableVFSOverlay = scheme.disableVFSOverlay ?? disableVFSOverlay
+        merge.customRewriteEnvs = scheme.customRewriteEnvs ?? customRewriteEnvs
+        merge.irrelevantDependenciesPaths = scheme.irrelevantDependenciesPaths ?? irrelevantDependenciesPaths
+        merge.gracefullyHandleMissingCommonSha =
+            scheme.gracefullyHandleMissingCommonSha ?? gracefullyHandleMissingCommonSha
         return merge
     }
 
@@ -211,6 +255,8 @@ struct ConfigFileScheme: Decodable {
     let statsDir: String?
     let downloadRetries: Int?
     let uploadRetries: Int?
+    let retryDelay: Double?
+    let uploadBatchSize: Int?
     let requestCustomHeaders: [String: String]?
     let thinTargetMockFilename: String?
     let focusedTargets: [String]?
@@ -221,10 +267,18 @@ struct ConfigFileScheme: Decodable {
     let productFilesExtensionsWithContentOverride: [String]?
     let thinningEnabled: Bool?
     let thinningTargetModuleName: String?
+    let prettifyMetaFiles: Bool?
     let AWSSecretKey: String?
     let AWSAccessKey: String?
+    let AWSSecurityToken: String?
     let AWSRegion: String?
     let AWSService: String?
+    let outOfBandMappings: [String: String]?
+    let disableCertificateVerification: Bool?
+    let disableVFSOverlay: Bool?
+    let customRewriteEnvs: [String]?
+    let irrelevantDependenciesPaths: [String]?
+    let gracefullyHandleMissingCommonSha: Bool?
 
     // Yams library doesn't support encoding strategy, see https://github.com/jpsim/Yams/issues/84
     enum CodingKeys: String, CodingKey {
@@ -252,6 +306,8 @@ struct ConfigFileScheme: Decodable {
         case statsDir = "stats_dir"
         case downloadRetries = "download_retries"
         case uploadRetries = "upload_retries"
+        case retryDelay = "retry_delay"
+        case uploadBatchSize = "upload_batch_size"
         case requestCustomHeaders = "request_custom_headers"
         case thinTargetMockFilename = "thin_target_mock_filename"
         case focusedTargets = "focused_targets"
@@ -262,10 +318,18 @@ struct ConfigFileScheme: Decodable {
         case productFilesExtensionsWithContentOverride = "product_files_extensions_with_content_override"
         case thinningEnabled = "thinning_enabled"
         case thinningTargetModuleName = "thinning_target_module_name"
+        case prettifyMetaFiles = "prettify_meta_files"
         case AWSSecretKey = "aws_secret_key"
         case AWSAccessKey = "aws_access_key"
+        case AWSSecurityToken = "aws_security_token"
         case AWSRegion = "aws_region"
         case AWSService = "aws_service"
+        case outOfBandMappings = "out_of_band_mappings"
+        case disableCertificateVerification = "disable_certificate_verification"
+        case disableVFSOverlay = "disable_vfs_overlay"
+        case customRewriteEnvs = "custom_rewrite_envs"
+        case irrelevantDependenciesPaths = "irrelevant_dependencies_paths"
+        case gracefullyHandleMissingCommonSha = "gracefully_handle_missing_common_sha"
     }
 }
 
@@ -278,31 +342,42 @@ class XCRemoteCacheConfigReader {
     /// Name of the configuration file, required in $(SRCROOT) location
     private static let configurationFile = ".rcinfo"
     private let srcRoot: String
-    private let fileManager: FileManager
+    private let fileReader: FileReader
     private lazy var yamlDecorer = YAMLDecoder(encoding: .utf8)
 
-    init(env: [String: String], fileManager: FileManager) throws {
+    init(env: [String: String], fileReader: FileReader) throws {
         let explicitSrcRoot: String? = env.readEnv(key: "SRCROOT")
-        srcRoot = explicitSrcRoot ?? fileManager.currentDirectoryPath
-        self.fileManager = fileManager
+        srcRoot = explicitSrcRoot ?? FileManager.default.currentDirectoryPath
+        self.fileReader = fileReader
     }
 
-    init(srcRootPath srcRoot: String, fileManager: FileManager) {
+    init(srcRootPath srcRoot: String, fileReader: FileReader) {
         self.srcRoot = srcRoot
-        self.fileManager = fileManager
+        self.fileReader = fileReader
     }
 
+    // Reads the final configuration by loading all extra configs
+    // until reaching a config that doesn't override `extraConfigurationFile`
     func readConfiguration() throws -> XCRemoteCacheConfig {
         let rootURL = URL(fileURLWithPath: srcRoot)
         let configURL = URL(fileURLWithPath: Self.configurationFile, relativeTo: rootURL)
         let userConfigs = try readUserConfig(configURL)
         var config = XCRemoteCacheConfig(sourceRoot: srcRoot).merged(with: userConfigs)
-        let extraConfURL = URL(fileURLWithPath: config.extraConfigurationFile, relativeTo: rootURL)
-        do {
-            let extraConfig = try readUserConfig(extraConfURL)
-            config = config.merged(with: extraConfig)
-        } catch {
-            infoLog("Extra config override failed with \(error). Skipping extra configuration")
+        var extraConfURL = URL(fileURLWithPath: config.extraConfigurationFile, relativeTo: rootURL)
+        var visitedFiles = Set([configURL])
+        while !visitedFiles.contains(extraConfURL) {
+            do {
+                let extraConfig = try readUserConfig(extraConfURL)
+                debugLog("Reading extra configuration from \(extraConfURL)")
+                config = config.merged(with: extraConfig)
+                visitedFiles.insert(extraConfURL)
+                // Advance extra configuration
+                extraConfURL = URL(fileURLWithPath: config.extraConfigurationFile, relativeTo: rootURL)
+            } catch {
+                infoLog("Extra config override failed with \(error). Skipping extra configuration")
+                // swiftlint:disable:next unneeded_break_in_switch
+                break
+            }
         }
 
         return try config.verifyAndApplyDefaults()
@@ -310,7 +385,7 @@ class XCRemoteCacheConfigReader {
 
     /// Reads user configuration from a file
     private func readUserConfig(_ file: URL) throws -> ConfigFileScheme {
-        let configurationContent = fileManager.contents(atPath: file.path)
+        let configurationContent = try fileReader.contents(atPath: file.path)
         guard let configurationData = configurationContent else {
             throw XCRemoteCacheConfigReaderError.missingConfigurationFile(file)
         }

Sources/XCRemoteCache/Dependencies/DependenciesReader.swift

@@ -51,34 +51,22 @@ public class FileDependenciesReader: DependenciesReader {
     public func findDependencies() throws -> [String] {
         let yaml = try readRaw()
 
-        struct ParseState {
-            var buffer: String = ""
-            var prevChar: Character?
-            var result: [String] = []
-            func with(buffer: String? = nil, prevChar: Character? = nil, result: [String]? = nil) -> ParseState {
-                var new = self
-                new.buffer = buffer ?? new.buffer
-                new.prevChar = prevChar ?? new.prevChar
-                new.result = result ?? new.result
-                return new
-            }
-        }
-
         let dependencies = yaml.reduce(Set<String>()) { prev, arg1 -> Set<String> in
             let (key, value) = arg1
             switch key {
             case "dependencies":
                 // 'clang' output formatting
-                return Set(splitDependencyFileList(value))
+                return Set(parseDependencyFileList(value))
             case let s where s.hasSuffix(".o") || s.hasSuffix(".bc"):
                 // 'swiftc' output formatting
-                // take dependencies from any .o or .bc file.
+                // take dependencies from any .o or .bc file
                 // Note: For WMO, all .{o|bc} files have the same dependencies
-                return Set(splitDependencyFileList(value))
+                return Set(parseDependencyFileList(value))
             default:
                 return prev
             }
         }
+
         return Array(dependencies)
     }
 
@@ -92,56 +80,92 @@ public class FileDependenciesReader: DependenciesReader {
         return yaml.mapValues { $0.components(separatedBy: .whitespaces) }
     }
 
-    private func readRaw() throws -> [String: String] {
+    func readRaw() throws -> [String: String] {
+        let fileData = try getFileData()
+        let fileString = try getFileStringFromData(fileData: fileData)
+        let yaml = try getYaml(fileString: fileString)
+        return yaml
+    }
+
+    func getFileData() throws -> Data {
         guard let fileData = fileManager.contents(atPath: file.path) else {
             throw DependenciesReaderError.readingError
         }
+        return fileData
+    }
+
+    func getFileStringFromData(fileData: Data) throws -> String {
         guard let fileString = String(data: fileData, encoding: .utf8) else {
             throw DependenciesReaderError.invalidFile
         }
-        // .d matches the .yaml format
+        return fileString
+    }
+
+    func getYaml(fileString: String) throws -> [String: String] {
         guard let yaml = try Yams.load(yaml: fileString) as? [String: String] else {
             throw DependenciesReaderError.invalidFile
         }
         return yaml
     }
 
-    /// Splits space or new line separated files into a set of files
+    /// Parses the String to get the list of files
+    /// It iterates over the String using its UTF8View since it is more performant (String type operates
+    /// in a higher abstraction level and supports features that have a negative impact in the performance)
     /// It supports escaping whitespace charaters, prefixed with "\\"
     /// - Parameter string: string of whitespace charaters separated file paths
     /// - Returns: Array of all file paths
-    private func splitDependencyFileList(_ string: String) -> [String] {
-        struct ParseState {
-            var buffer: String = ""
-            var prevChar: Character?
-            var result: [String] = []
-            func with(buffer: String? = nil, prevChar: Character? = nil, result: [String]? = nil) -> ParseState {
-                var new = self
-                new.buffer = buffer ?? new.buffer
-                new.prevChar = prevChar ?? new.prevChar
-                new.result = result ?? new.result
-                return new
-            }
+    func parseDependencyFileList(_ string: String) -> [String] {
+        var result: [String] = []
+        var prevChar: UTF8.CodeUnit?
+
+        // These index are used to move over the UTF8View of the string
+        // The goal is to optimize the memory used, since UTF8View uses
+        // the same memory as the original String without copying it
+        var startIndex = string.utf8.startIndex
+        var endIndex = startIndex
+
+        // This buffer is only used to save the part of the path that has been already parsed when finding a backslash
+        var buffer: String = ""
+
+        for c in string.utf8 {
+          switch c {
+          case UTF8.CodeUnit(ascii: "\n") where prevChar == UTF8.CodeUnit(ascii: "\\"):
+              startIndex = string.utf8.index(after: startIndex)
+              endIndex = startIndex
+          case UTF8.CodeUnit(ascii: " ") where startIndex == endIndex && buffer.isEmpty:
+              startIndex = string.utf8.index(after: startIndex)
+              endIndex = startIndex
+          case UTF8.CodeUnit(ascii: " ") where prevChar != UTF8.CodeUnit(ascii: "\\"):
+              // If a space is found and it is not escaped, then that's the end of the file path
+              buffer += String(Substring(string.utf8[startIndex ..< endIndex]))
+              result.append(buffer)
+              buffer = ""
+              prevChar = nil
+              startIndex = string.utf8.index(after: endIndex)
+              endIndex = startIndex
+          case UTF8.CodeUnit(ascii: "\\"):
+              // If a backslash is found it is not included in the file path
+              // The current parsed range of the UTF8View is saved in the buffer as a String
+              buffer += String(Substring(string.utf8[startIndex ..< endIndex]))
+              // The backslash is assigned as the previous char
+              prevChar = c
+              // The indexes are moved to the next char so we continue parsing the String
+              startIndex = string.utf8.index(after: endIndex)
+              endIndex = startIndex
+          default:
+              // As long as it is possible the indexes are used to track the range of the string that
+              // will be included in the file path (until it ends or until a backslash is found)
+              endIndex = string.utf8.index(after: endIndex)
+              // The char is assigned as the previous char
+              prevChar = c
+          }
         }
-        let parseResult = string.reduce(ParseState()) { total, char in
-            switch char {
-            case "\n" where total.prevChar == "\\":
-                return total
-            case " " where total.buffer.isEmpty:
-                return total
-            case " " where total.prevChar == "\\":
-                return total.with(buffer: "\(total.buffer) ")
-            case " ":
-                return total.with(buffer: "", prevChar: nil, result: total.result + [total.buffer])
-            case "\\":
-                return total.with(prevChar: "\\")
-            default:
-                return total.with(buffer: "\(total.buffer)\(char)", prevChar: char, result: total.result)
-            }
+
+        if startIndex != endIndex {
+            buffer += String(Substring(string.utf8[startIndex ..< endIndex]))
+            result.append(buffer)
         }
-        if !parseResult.buffer.isEmpty {
-            return parseResult.result + [parseResult.buffer]
-        }
-        return parseResult.result
+
+        return result
     }
 }

Sources/XCRemoteCache/Dependencies/DependenciesRemapper.swift

@@ -22,9 +22,9 @@ import Foundation
 /// Replaces paths formats between generic (placeholders-based) and local
 protocol DependenciesRemapper {
     /// Replaces all generic paths (with placeholders) to a local paths
-    func replace(genericPaths: [String]) -> [String]
+    func replace(genericPaths: [String]) throws -> [String]
     /// Replaces all local paths to the generic dependencies paths
-    func replace(localPaths: [String]) -> [String]
+    func replace(localPaths: [String]) throws -> [String]
 }
 
 class DependenciesRemapperComposite: DependenciesRemapper {
@@ -34,15 +34,15 @@ class DependenciesRemapperComposite: DependenciesRemapper {
         self.remappers = remappers
     }
 
-    func replace(genericPaths: [String]) -> [String] {
-        remappers.reduce(genericPaths) { prev, mapper in
-            mapper.replace(genericPaths: prev)
+    func replace(genericPaths: [String]) throws -> [String] {
+        try remappers.reversed().reduce(genericPaths) { prev, mapper in
+            try mapper.replace(genericPaths: prev)
         }
     }
 
-    func replace(localPaths: [String]) -> [String] {
-        remappers.reduce(localPaths) { prev, mapper in
-            mapper.replace(localPaths: prev)
+    func replace(localPaths: [String]) throws -> [String] {
+        try remappers.reduce(localPaths) { prev, mapper in
+            try mapper.replace(localPaths: prev)
         }
     }
 }
@@ -59,16 +59,16 @@ final class StringDependenciesRemapper: DependenciesRemapper {
         self.mappings = mappings
     }
 
-    func replace(genericPaths: [String]) -> [String] {
+    func replace(genericPaths: [String]) throws -> [String] {
         return genericPaths.map { path in
-            let localPath = mappings.reduce(path) { prevPath, mapping in
+            let localPath = mappings.reversed().reduce(path) { prevPath, mapping in
                 prevPath.replacingOccurrences(of: mapping.generic, with: mapping.local)
             }
             return localPath
         }
     }
 
-    func replace(localPaths: [String]) -> [String] {
+    func replace(localPaths: [String]) throws -> [String] {
         return localPaths.map { path in
             let result = mappings.reduce(path) { prevPath, mapping in
                 prevPath.replacingOccurrences(of: mapping.local, with: mapping.generic)
@@ -77,14 +77,3 @@ final class StringDependenciesRemapper: DependenciesRemapper {
         }
     }
 }
-
-
-extension StringDependenciesRemapper {
-    static func buildFromEnvs(keys: [String], envs: [String: String]) throws -> Self {
-        let mappings: [Mapping] = try keys.map { key in
-            let localValue: String = try envs.readEnv(key: key)
-            return Mapping(generic: "$(\(key))", local: localValue)
-        }
-        return Self(mappings: mappings)
-    }
-}

Sources/XCRemoteCache/Dependencies/DependenciesWriter.swift

@@ -50,7 +50,7 @@ public class FileDependenciesWriter: DependenciesWriter {
         var content = ""
         for (file, deps) in dependencies {
             content.append(file + ": ")
-            content.append(deps.joined(separator: " "))
+            content.append(deps.map { $0.replacingOccurrences(of: " ", with: "\\ ") }.joined(separator: " "))
             content.append("\n")
         }
         try content.write(to: file, atomically: true, encoding: .utf8)

Sources/XCRemoteCache/Dependencies/DependencyProcessor.swift

@@ -27,8 +27,11 @@ public struct Dependency: Equatable {
         case source
         case fingerprint
         case intermediate
+        case derivedFile
         // Product of the target itself
         case ownProduct
+        // User-excluded path
+        case userExcluded
         case unknown
     }
 
@@ -55,14 +58,18 @@ class DependencyProcessorImpl: DependencyProcessor {
     private let productPath: String
     private let sourcePath: String
     private let intermediatePath: String
+    private let derivedFilesPath: String
     private let bundlePath: String?
+    private let skippedRegexes: [String]
 
-    init(xcode: URL, product: URL, source: URL, intermediate: URL, bundle: URL?) {
-        xcodePath = xcode.path
-        productPath = product.path
-        sourcePath = source.path
-        intermediatePath = intermediate.path
-        bundlePath = bundle?.path
+    init(xcode: URL, product: URL, source: URL, intermediate: URL, derivedFiles: URL, bundle: URL?, skippedRegexes: [String]) {
+        xcodePath = xcode.path.dirPath()
+        productPath = product.path.dirPath()
+        sourcePath = source.path.dirPath()
+        intermediatePath = intermediate.path.dirPath()
+        derivedFilesPath = derivedFiles.path.dirPath()
+        bundlePath = bundle?.path.dirPath()
+        self.skippedRegexes = skippedRegexes
     }
 
     func process(_ files: [URL]) -> [Dependency] {
@@ -72,11 +79,15 @@ class DependencyProcessorImpl: DependencyProcessor {
 
     private func classify(_ files: [URL]) -> [Dependency] {
         return files.map { file -> Dependency in
-            let filePath = file.path
-            if filePath.hasPrefix(xcodePath) {
+            let filePath = file.resolvingSymlinksInPath().path
+            if skippedRegexes.contains(where: { filePath.range(of: $0, options: .regularExpression) != nil }) {
+                return Dependency(url: file, type: .userExcluded)
+            } else if filePath.hasPrefix(xcodePath) {
                 return Dependency(url: file, type: .xcode)
             } else if filePath.hasPrefix(intermediatePath) {
                 return Dependency(url: file, type: .intermediate)
+            } else if filePath.hasPrefix(derivedFilesPath) {
+                return Dependency(url: file, type: .derivedFile)
             } else if let bundle = bundlePath, filePath.hasPrefix(bundle) {
                 // If a target produces a bundle, explicitly classify all
                 // of products to distinguish from other targets products
@@ -107,7 +118,18 @@ class DependencyProcessorImpl: DependencyProcessor {
         // - All files in `*/Interemediates/*` - this file are created on-fly for a given target
         // - Some files may depend on its own product (e.g. .m may #include *-Swift.h) - we know products will match
         //   because in case of a hit, these will be taken from the artifact
-        let irrelevantDependenciesType: [Dependency.Kind] = [.xcode, .intermediate, .ownProduct]
+        // - Customized DERIVED_FILE_DIR may change a directory of
+        //   derived files, which by default is under `*/Interemediates`
+        // - User-specified (in .rcinfo) files to exclude
+        let irrelevantDependenciesType: [Dependency.Kind] = [
+            .xcode, .intermediate, .ownProduct, .derivedFile, .userExcluded,
+        ]
         return !irrelevantDependenciesType.contains(dependency.type)
     }
 }
+
+fileprivate extension String {
+    func dirPath() -> String {
+        hasSuffix("/") ? self : appending("/")
+    }
+}

Sources/XCRemoteCache/Dependencies/FingerprintSyncer.swift

@@ -30,6 +30,10 @@ protocol FingerprintSyncer {
     func decorate(sourceDir: URL, fingerprint: String) throws
     /// Deletes fingerprint overrides in the dir (if already created)
     func delete(sourceDir: URL) throws
+    /// Sets a fingerprint override for a singe file placed
+    func decorate(file: URL, fingerprint: String) throws
+    /// Deletes fingerprint override for a file  (if already created)
+    func delete(file: URL) throws
 }
 
 class FileFingerprintSyncer: FingerprintSyncer {
@@ -78,4 +82,25 @@ class FileFingerprintSyncer: FingerprintSyncer {
             try dirAccessor.removeItem(atPath: file.path)
         }
     }
+
+    func decorate(file: URL, fingerprint: String) throws {
+        guard let fingerprintData = fingerprint.data(using: .utf8) else {
+            throw FingerprintSyncerError.invalidFingerprint
+        }
+        let fingerprintFile = file.appendingPathExtension(fingerprintExtension)
+        try dirAccessor.write(toPath: fingerprintFile.path, contents: fingerprintData)
+    }
+
+    func delete(file: URL) throws {
+        guard case .file = try dirAccessor.itemType(atPath: file.path) else {
+            // no file to decorate (no module was generated)
+            return
+        }
+        let overrideURL = file.appendingPathExtension(fingerprintExtension)
+        guard case .file = try dirAccessor.itemType(atPath: overrideURL.path) else {
+            // no override
+            return
+        }
+        try dirAccessor.removeItem(atPath: overrideURL.path)
+    }
 }

Sources/XCRemoteCache/Dependencies/OverlayDependenciesRemapper.swift

@@ -0,0 +1,67 @@
+// Copyright (c) 2021 Spotify AB.
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import Foundation
+
+/// File paths remapper according the virtual file system mappings
+/// - Warning: this class is not thread safe
+class OverlayDependenciesRemapper: DependenciesRemapper {
+    private let overlayReader: OverlayReader
+    private var mappings: [OverlayMapping]?
+
+    init(overlayReader: OverlayReader) {
+        self.overlayReader = overlayReader
+    }
+
+    /// Lazily Reads mappings from a file
+    /// - Warning: this function is not thread safe
+    private func getMappings() throws -> [OverlayMapping] {
+        guard let mappings = mappings else {
+            let mappings = try overlayReader.provideMappings()
+            self.mappings = mappings
+            return mappings
+        }
+        return mappings
+    }
+
+    private func mapPath(
+        _ path: String,
+        source: KeyPath<OverlayMapping, URL>,
+        destination: KeyPath<OverlayMapping, URL>
+    ) throws -> String {
+        guard let mapping = try getMappings().first(where: { $0[keyPath: source].path == path }) else {
+            // TODO: support partial mappings, where a directory path can be replaced with some other directory
+            // no direct mapping found
+            return path
+        }
+        return mapping[keyPath: destination].path
+    }
+
+    func replace(genericPaths: [String]) throws -> [String] {
+        try genericPaths.map {
+            try mapPath($0, source: \.virtual, destination: \.local)
+        }
+    }
+
+    func replace(localPaths: [String]) throws -> [String] {
+        try localPaths.map {
+            try mapPath($0, source: \.local, destination: \.virtual)
+        }
+    }
+}

Sources/XCRemoteCache/Dependencies/OverlayReader.swift

@@ -0,0 +1,133 @@
+// Copyright (c) 2021 Spotify AB.
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import Foundation
+
+/// Maps overlay's virtual URL with an actual (local) location
+struct OverlayMapping: Hashable {
+    let virtual: URL
+    let local: URL
+}
+
+enum JsonOverlayReaderError: Error {
+    /// The source file is missing
+    case missingSourceFile(URL)
+    /// The file exists but its content is invalid
+    case invalidSourceContent(URL)
+    /// the overlay format is not supported - either contains a nested directory or a single file 
+    case unsupportedFormat
+}
+/// Provides virtual file system overlay mappings
+protocol OverlayReader {
+    func provideMappings() throws -> [OverlayMapping]
+}
+
+class JsonOverlayReader: OverlayReader {
+
+    enum Mode {
+        /// Interrupts the operation if the representation file is missing
+        case strict
+        /// Assume empty overlay mapping if the file doesn't exist
+        case bestEffort
+    }
+
+    private struct Overlay: Decodable {
+        enum OverlayType: String, Decodable {
+            case file
+            case directory
+        }
+
+        struct Content: Decodable {
+            let externalContents: String
+            let name: String
+            let type: OverlayType
+
+            enum CodingKeys: String, CodingKey {
+                case externalContents = "external-contents"
+                case name
+                case type
+            }
+        }
+
+        struct RootContent: Decodable {
+            let contents: [Content]
+            let name: String
+            let type: OverlayType
+        }
+        let roots: [RootContent]
+    }
+
+    private lazy var jsonDecoder = JSONDecoder()
+    private let json: URL
+    private let mode: Mode
+    private let fileReader: FileReader
+
+
+    init(_ json: URL, mode: Mode, fileReader: FileReader) {
+        self.json = json
+        self.mode = mode
+        self.fileReader = fileReader
+    }
+
+    func provideMappings() throws -> [OverlayMapping] {
+        guard let jsonContent = try fileReader.contents(atPath: json.path) else {
+            switch mode {
+            case .strict:
+                throw JsonOverlayReaderError.missingSourceFile(json)
+            case .bestEffort:
+                debugLog("overlay mapping file \(json) doesn't exist. Skipping overlay for the best-effort mode.")
+                return []
+            }
+        }
+        do {
+        let overlay: Overlay = try jsonDecoder.decode(Overlay.self, from: jsonContent)
+            let mappings: [OverlayMapping] = try overlay.roots.reduce([]) { prev, root in
+                switch root.type {
+                case .directory:
+                    // iterate all contents
+                    let dir = URL(fileURLWithPath: root.name)
+                    let mappings: [OverlayMapping] = try root.contents.map { content in
+                        switch content.type {
+                        case .file:
+                            let virtual = dir.appendingPathComponent(content.name)
+                            let local = URL(fileURLWithPath: content.externalContents)
+                            return .init(virtual: virtual, local: local)
+                        case .directory:
+                            throw JsonOverlayReaderError.unsupportedFormat
+                        }
+
+                    }
+                    return prev + mappings
+                case .file:
+                    throw JsonOverlayReaderError.unsupportedFormat
+                }
+            }
+            return mappings
+        } catch {
+            switch mode {
+            case .strict:
+                throw error
+            case .bestEffort:
+                errorLog("Overlay reader has failed with an error \(error). Best-effort mode - skipping an overlay.")
+                return []
+            }
+        }
+    }
+
+}

Sources/XCRemoteCache/Dependencies/PathDependenciesRemapperFactory.swift

@@ -0,0 +1,47 @@
+// Copyright (c) 2021 Spotify AB.
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import Foundation
+
+enum PathDependenciesRemapperFactoryError: Error {
+    /// Remapping keys are duplicated and can lead to undetermined results
+    case mappingKeyDuplication
+}
+
+class PathDependenciesRemapperFactory {
+    func build(
+        orderKeys: [String],
+        envs: [String: String],
+        customMappings: [String: String]
+    ) throws -> StringDependenciesRemapper {
+        let mappingMap = try envs.merging(customMappings) { _, _ in
+            throw PathDependenciesRemapperFactoryError.mappingKeyDuplication
+        }
+        let mappingOrderKeys = orderKeys + customMappings.keys
+        let mappings: [StringDependenciesRemapper.Mapping] = mappingOrderKeys.compactMap { key in
+            guard let localURL: URL = mappingMap.readEnv(key: key) else {
+                debugLog("\(key) ENV to map a dependency is not defined")
+                return nil
+            }
+            infoLog("Found url to remapp: \(localURL). Remapping: \(localURL.standardized.path)")
+            return StringDependenciesRemapper.Mapping(generic: "$(\(key))", local: localURL.standardized.path)
+        }
+        return StringDependenciesRemapper(mappings: mappings)
+    }
+}

Sources/XCRemoteCache/Dependencies/TargetDepdenciesReader.swift

@@ -42,7 +42,7 @@ class TargetDependenciesReader: DependenciesReader {
         let allURLs = try dirScanner.items(at: directory)
         let mergedDependencies = try allURLs.reduce(Set<String>()) { (prev: Set<String>, file) in
             // include only these .d files that either have corresponding .o file (incremental) or end
-            // with '-master' (whole-module).
+            // with '-master' (whole-module)
             // Otherwise .d is probably just a leftover from previous builds
             let correspondingOutputURL = file.deletingPathExtension().appendingPathExtension("o")
             let isDependencyFile = file.pathExtension == "d"

Sources/XCRemoteCache/FileAccess/DirAccessor.swift

@@ -33,8 +33,13 @@ protocol DirScanner {
 
     /// Returns all items in a directory (shallow search)
     /// - Parameter at: url of an existing directory to search
-    /// - Throws: an error if dir doesn't exist or I/O error
+    /// - Throws: an error if a dir doesn't exist or on I/O error
     func items(at dir: URL) throws -> [URL]
+
+    /// Returns all items in a directory (recursive search)
+    /// - Parameter at: url of an existing directory to search
+    /// - Throws: an error if a dir doesn't exist or on I/O error
+    func recursiveItems(at dir: URL) throws -> [URL]
 }
 
 typealias DirAccessor = FileAccessor & DirScanner
@@ -54,4 +59,18 @@ extension FileManager: DirScanner {
         let resolvedDir = dir.resolvingSymlinksInPath()
         return try contentsOfDirectory(at: resolvedDir, includingPropertiesForKeys: nil, options: [])
     }
+
+    func recursiveItems(at dir: URL) throws -> [URL] {
+        // Iterating DFS
+        var queue: [URL] = [dir]
+        var results: [URL] = []
+        while let item = queue.popLast() {
+            if try itemType(atPath: item.path) == .dir {
+                try queue.append(contentsOf: items(at: item))
+            } else {
+                results.append(item)
+            }
+        }
+        return results
+    }
 }

Sources/XCRemoteCache/FileAccess/DirAccessorComposer.swift

@@ -52,4 +52,8 @@ class DirAccessorComposer: DirAccessor {
     func items(at dir: URL) throws -> [URL] {
         try dirScanner.items(at: dir)
     }
+
+    func recursiveItems(at dir: URL) throws -> [URL] {
+        try dirScanner.recursiveItems(at: dir)
+    }
 }

Sources/XCRemoteCache/Fingerprint/EnvironmentFingerprint.swift

@@ -19,10 +19,10 @@
 
 /// Generates a fingerprint string of the environment (compilation context)
 class EnvironmentFingerprintGenerator {
-    /// Default ENV variables constituing the environment fingerprint
+    /// Default ENV variables constituting the environment fingerprint
     private static let defaultEnvFingerprintKeys = [
         "GCC_PREPROCESSOR_DEFINITIONS",
-        "CLANG_PROFILE_DATA_DIRECTORY",
+        "CLANG_COVERAGE_MAPPING",
         "TARGET_NAME",
         "CONFIGURATION",
         "PLATFORM_NAME",
@@ -31,6 +31,7 @@ class EnvironmentFingerprintGenerator {
         "DYLIB_COMPATIBILITY_VERSION",
         "DYLIB_CURRENT_VERSION",
         "PRODUCT_MODULE_NAME",
+        "ARCHS",
     ]
     private let version: String
     private let customFingerprintEnvs: [String]
@@ -51,7 +52,9 @@ class EnvironmentFingerprintGenerator {
         }
         try fill(envKeys: Self.defaultEnvFingerprintKeys + customFingerprintEnvs)
         try generator.append(version)
-        return try generator.generate()
+        let result = try generator.generate()
+        generatedFingerprint = result
+        return result
     }
 
     /// Creates a fingerprint of the environemtn, by hashing all ENVs specified in keys

Sources/XCRemoteCache/Logger/Logger.swift

@@ -22,34 +22,36 @@ import Foundation
 import os.log
 
 
+private var processTag: String = ""
+
 public func exit(_ exitCode: Int32, _ message: String) -> Never {
-    os_log("%{public}@", log: OSLog.default, type: .error, message)
+    os_log("%{public}@%{public}@", log: OSLog.default, type: .error, processTag, message)
     printError(errorMessage: message)
     exit(exitCode)
 }
 
 func defaultLog(_ message: String) {
-    os_log("%{public}@", log: OSLog.default, type: .default, message)
+    os_log("%{public}@%{public}@", log: OSLog.default, type: .default, processTag, message)
 }
 
 func errorLog(_ message: String) {
-    os_log("%{public}@", log: OSLog.default, type: .error, message)
+    os_log("%{public}@%{public}@", log: OSLog.default, type: .error, processTag, message)
 }
 
 func infoLog(_ message: String) {
-    os_log("%{public}@", log: OSLog.default, type: .info, message)
+    os_log("%{public}@%{public}@", log: OSLog.default, type: .info, processTag, message)
 }
 
 func debugLog(_ message: String) {
-    os_log("%{public}@", log: OSLog.default, type: .debug, message)
+    os_log("%{public}@%{public}@", log: OSLog.default, type: .debug, processTag, message)
 }
 
 func printError(errorMessage: String) {
-    fputs("error: \(errorMessage)\n", stderr)
+    fputs("error: \(processTag)\(errorMessage)\n", stderr)
 }
 
 func printWarning(_ message: String) {
-    print("warning: \(message)")
+    print("warning: \(processTag)\(message)")
 }
 
 /// Prints a message to the user. It shows in Xcode (if applies) or console output
@@ -57,3 +59,7 @@ func printWarning(_ message: String) {
 func printToUser(_ message: String) {
     print("[RC] \(message)")
 }
+
+func updateProcessTag(_ tag: String) {
+    processTag = "(\(tag)) "
+}

Sources/XCRemoteCache/Models/MetaWriter.swift

@@ -0,0 +1,45 @@
+// Copyright (c) 2021 Spotify AB.
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import Foundation
+
+protocol MetaWriter {
+    func write<T>(_ meta: T, locationDir: URL) throws -> URL where T: Meta
+}
+
+class JsonMetaWriter: MetaWriter {
+    private let metaEncoder: JSONEncoder
+    private let fileWriter: FileWriter
+
+    init(fileWriter: FileWriter, pretty: Bool) {
+        self.fileWriter = fileWriter
+        let encoder = JSONEncoder()
+        if pretty {
+            encoder.outputFormatting = .prettyPrinted
+        }
+        self.metaEncoder = encoder
+    }
+
+    func write<T>(_ meta: T, locationDir: URL) throws -> URL where T: Meta {
+        let metaURL = locationDir.appendingPathComponent(meta.fileKey).appendingPathExtension("json")
+        let metaData = try metaEncoder.encode(meta)
+        try fileWriter.write(toPath: metaURL.path, contents: metaData)
+        return metaURL
+    }
+}

Sources/XCRemoteCache/Network/Authentication/AWSV4Signature.swift

@@ -23,21 +23,38 @@ struct AWSV4Signature {
 
     let secretKey: String
     let accessKey: String
+    let securityToken: String?
     let region: String
     let service: String
     let date: Date
 
-
     func addSignatureHeaderTo(request: inout URLRequest) {
 
         request.setValue(request.url?.host, forHTTPHeaderField: "host")
         request.setValue(StringToSign.ISO8601BasicFormatter.string(from: date), forHTTPHeaderField: "x-amz-date")
         request.setValue((request.httpBody ?? Data()).sha256(), forHTTPHeaderField: "x-amz-content-sha256")
 
+        if let securityToken = securityToken {
+            request.setValue(securityToken, forHTTPHeaderField: "x-amz-security-token")
+        }
+
         let canonicalRequest = CanonicalRequest(request: request)
-        let stringToSign = StringToSign(region: region, service: service, canonicalRequestHash: canonicalRequest.hash, date: date)
-        let awsV4SigningKey = AWSV4SigningKey(secretAccessKey: secretKey, region: region, service: service, date: date)
-        let signature = HMAC.calcHMAC(keyArray: awsV4SigningKey.value, value: stringToSign.value).map { String(format: "%02hhx", $0) }.joined()
+        let stringToSign = StringToSign(
+            region: region,
+            service: service,
+            canonicalRequestHash: canonicalRequest.hash,
+            date: date
+        )
+        let awsV4SigningKey = AWSV4SigningKey(
+            secretAccessKey: secretKey,
+            region: region,
+            service: service,
+            date: date
+        )
+        let signature = HMAC.calcHMAC(
+            keyArray: awsV4SigningKey.value,
+            value: stringToSign.value
+        ).map { String(format: "%02hhx", $0) }.joined()
 
         let authValue =
             "AWS4-HMAC-SHA256 " +

Sources/XCRemoteCache/Network/Authentication/CanonicalRequest.swift

@@ -34,7 +34,15 @@ struct CanonicalRequest {
         if url.path.isEmpty {
             path = "/"
         } else {
-            path = url.path
+            if let escapedPath = url.path.addingPercentEncoding(withAllowedCharacters: .urlPathAllowed) {
+                path = escapedPath
+            } else {
+                path = "/"
+                printWarning("""
+Escaping the path \(url.path) failed and a placeholder is used instead. \
+Make sure the path doesn't contain invalid characters.
+""")
+            }
         }
         return
             "\(httpMethod)\n" +

Sources/XCRemoteCache/Network/Authentication/HMAC.swift

@@ -52,7 +52,14 @@ struct HMAC {
 
     private static func calcHMAC(keyUnsafeBytes: UnsafeRawBufferPointer, value: String, out: UnsafeMutableRawPointer!) {
         value.data(using: .utf8)!.withUnsafeBytes { value in
-            CCHmac(CCHmacAlgorithm(kCCHmacAlgSHA256), keyUnsafeBytes.baseAddress, Int(keyUnsafeBytes.count), value.baseAddress, Int(value.count), out)
+            CCHmac(
+                CCHmacAlgorithm(kCCHmacAlgSHA256),
+                keyUnsafeBytes.baseAddress,
+                Int(keyUnsafeBytes.count),
+                value.baseAddress,
+                Int(value.count),
+                out
+            )
         }
     }
 }

Sources/XCRemoteCache/Network/IgnoringCertificatesTrustManager.swift

@@ -0,0 +1,32 @@
+// Copyright (c) 2021 Spotify AB.
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import Foundation
+
+final class IgnoringCertificatesTrustManager: NSObject, URLSessionDelegate {
+    func urlSession(_ session: URLSession, didReceive challenge: URLAuthenticationChallenge, completionHandler: @escaping (URLSession.AuthChallengeDisposition, URLCredential?) -> Void) {
+        guard let serverTrust = challenge.protectionSpace.serverTrust else {
+            completionHandler(.performDefaultHandling, nil)
+            return
+        }
+
+        let urlCredential = URLCredential(trust: serverTrust)
+        completionHandler(.useCredential, urlCredential)
+    }
+}

Sources/XCRemoteCache/Network/NetworkClientImpl.swift

@@ -29,12 +29,14 @@ class NetworkClientImpl: NetworkClient {
     private let session: URLSession
     private let fileManager: FileManager
     private let maxRetries: Int
+    private let retryDelay: TimeInterval
     private let awsV4Signature: AWSV4Signature?
 
-    init(session: URLSession, retries: Int, fileManager: FileManager, awsV4Signature: AWSV4Signature?) {
+    init(session: URLSession, retries: Int, retryDelay: TimeInterval, fileManager: FileManager, awsV4Signature: AWSV4Signature?) {
         self.session = session
         self.fileManager = fileManager
-        maxRetries = retries
+        self.maxRetries = retries
+        self.retryDelay = retryDelay
         self.awsV4Signature = awsV4Signature
     }
 
@@ -75,7 +77,12 @@ class NetworkClientImpl: NetworkClient {
     func download(_ url: URL, to location: URL, completion: @escaping (Result<Void, NetworkClientError>) -> Void) {
         var request = URLRequest(url: url)
         setupAuthenticationSignatureIfPresent(&request)
-        makeDownloadRequest(request, output: location, completion: completion)
+        makeDownloadRequest(
+            request,
+            output: location,
+            retries: maxRetries,
+            completion: completion
+        )
     }
 
     func upload(_ file: URL, as url: URL, completion: @escaping (Result<Void, NetworkClientError>) -> Void) {
@@ -123,7 +130,7 @@ class NetworkClientImpl: NetworkClient {
         dataTask.resume()
     }
 
-    private func makeDownloadRequest(_ request: URLRequest, output: URL, completion: @escaping (Result<Void, NetworkClientError>) -> Void) {
+    private func makeDownloadRequest(_ request: URLRequest, output: URL, retries: Int, completion: @escaping (Result<Void, NetworkClientError>) -> Void) {
         guard fileManager.fileExists(atPath: output.path) == false else {
             infoLog("Download file found in the destination, skipping download.")
             completion(.success(()))
@@ -133,6 +140,17 @@ class NetworkClientImpl: NetworkClient {
         let dataTask = session.downloadTask(with: request) { [fileManager] fileURL, _, error in
             guard let fileURL = fileURL else {
                 let networkError = error.map(NetworkClientError.build) ?? .inconsistentSession
+                if retries > 0 {
+                    infoLog("Download request failed with \(networkError). Left retries: \(retries).")
+                    self.retryDownload(
+                        request,
+                        output: output,
+                        retries: retries,
+                        completion: completion,
+                        after: self.retryDelay
+                    )
+                    return
+                }
                 errorLog("Download request failed: \(networkError)")
                 completion(.failure(networkError))
                 return
@@ -173,7 +191,13 @@ class NetworkClientImpl: NetworkClient {
             if let error = responseError {
                 if retries > 0 {
                     infoLog("Upload request failed with \(error). Left retries: \(retries).")
-                    self.makeUploadRequest(request, input: input, retries: retries - 1, completion: completion)
+                    self.retryUpload(
+                        request,
+                        input: input,
+                        retries: retries,
+                        completion: completion,
+                        after: self.retryDelay
+                    )
                     return
                 }
                 errorLog("Upload request failed: \(error)")
@@ -184,6 +208,30 @@ class NetworkClientImpl: NetworkClient {
         }
         dataTask.resume()
     }
+
+    private func retryUpload(_ request: URLRequest, input: URL, retries: Int, completion: @escaping (Result<Void, NetworkClientError>) -> Void, after: TimeInterval) {
+        DispatchQueue.global().asyncAfter(deadline: .now() + after) { [weak self] in
+            guard let self = self else { return }
+            self.makeUploadRequest(
+                request,
+                input: input,
+                retries: retries - 1,
+                completion: completion
+            )
+        }
+    }
+
+    private func retryDownload(_ request: URLRequest, output: URL, retries: Int, completion: @escaping (Result<Void, NetworkClientError>) -> Void, after: TimeInterval) {
+        DispatchQueue.global().asyncAfter(deadline: .now() + after) { [weak self] in
+            guard let self = self else { return }
+            self.makeDownloadRequest(
+                request,
+                output: output,
+                retries: retries - 1,
+                completion: completion
+            )
+        }
+    }
 }
 
 private extension NetworkClientError {

Sources/XCRemoteCache/Network/RemoteNetworkClientAbstractFactory.swift

@@ -27,11 +27,20 @@ class RemoteNetworkClientAbstractFactory {
     private let upstreamStreamURL: [URL]
     private let networkClient: NetworkClient
     private let urlBuilderFactory: (URL) throws -> URLBuilder
+    private let uploadBatchSize: Int
 
-    init(mode: Mode, downloadStreamURL: URL, upstreamStreamURL: [URL], networkClient: NetworkClient, urlBuilderFactory: @escaping (URL) throws -> URLBuilder) {
+    init(
+        mode: Mode,
+        downloadStreamURL: URL,
+        upstreamStreamURL: [URL],
+        uploadBatchSize: Int,
+        networkClient: NetworkClient,
+        urlBuilderFactory: @escaping (URL) throws -> URLBuilder
+    ) {
         self.mode = mode
         self.downloadStreamURL = downloadStreamURL
         self.upstreamStreamURL = upstreamStreamURL
+        self.uploadBatchSize = uploadBatchSize
         self.networkClient = networkClient
         self.urlBuilderFactory = urlBuilderFactory
     }
@@ -44,12 +53,13 @@ class RemoteNetworkClientAbstractFactory {
             return RemoteNetworkClientImpl(networkClient, downloadURLBuilder)
         }
         switch mode {
-        case .producer:
+        case .producer, .producerFast:
             let upstreamBuilders = try upstreamStreamURL.map(urlBuilderFactory)
             return ReplicatedRemotesNetworkClient(
                 networkClient,
                 download: downloadURLBuilder,
-                uploads: upstreamBuilders
+                uploads: upstreamBuilders,
+                uploadBatchSize: uploadBatchSize
             )
         case .consumer:
             return RemoteNetworkClientImpl(networkClient, downloadURLBuilder)

Sources/XCRemoteCache/Network/ReplicatedRemotesNetworkClient.swift

@@ -23,10 +23,12 @@ import Foundation
 class ReplicatedRemotesNetworkClient: RemoteNetworkClientImpl {
     private let networkClient: NetworkClient
     private let uploadURLBuilders: [URLBuilder]
+    private let uploadBatchSize: Int
 
-    init(_ networkClient: NetworkClient, download: URLBuilder, uploads uploadURLBuilders: [URLBuilder]) {
+    init(_ networkClient: NetworkClient, download: URLBuilder, uploads uploadURLBuilders: [URLBuilder], uploadBatchSize: Int) {
         self.networkClient = networkClient
         self.uploadURLBuilders = uploadURLBuilders
+        self.uploadBatchSize = uploadBatchSize
         super.init(networkClient, download)
     }
 
@@ -39,6 +41,9 @@ class ReplicatedRemotesNetworkClient: RemoteNetworkClientImpl {
         let group = DispatchGroup()
         var results: [Result<Void, NetworkClientError>] = Array(repeating: .failure(.noResponse), count: urls.count)
         urls.enumerated().forEach { index, url in
+            if uploadBatchSize > 0 && index > 0 && index % uploadBatchSize == 0 {
+                group.wait()
+            }
             group.enter()
             networkClient.upload(file, as: url) { receivedResult in
                 results[index] = receivedResult
@@ -58,6 +63,9 @@ class ReplicatedRemotesNetworkClient: RemoteNetworkClientImpl {
         let group = DispatchGroup()
         var results: [Result<Void, NetworkClientError>] = Array(repeating: .failure(.noResponse), count: urls.count)
         urls.enumerated().forEach { index, url in
+            if uploadBatchSize > 0 && index > 0 && index % uploadBatchSize == 0 {
+                group.wait()
+            }
             group.enter()
             networkClient.create(url) { receivedResult in
                 results[index] = receivedResult

Sources/XCRemoteCache/Network/URLSessionFactory.swift

@@ -36,6 +36,17 @@ class DefaultURLSessionFactory: URLSessionFactory {
         let configuration = URLSessionConfiguration.default
         configuration.httpAdditionalHeaders = config.requestCustomHeaders
         configuration.timeoutIntervalForRequest = config.timeoutResponseDataChunksInterval
-        return URLSession(configuration: configuration)
+        configuration.urlCache?.memoryCapacity = 0
+        configuration.urlCache?.diskCapacity = 0
+        switch config.disableCertificateVerification {
+        case true:
+            return URLSession(
+                configuration: configuration,
+                delegate: IgnoringCertificatesTrustManager(),
+                delegateQueue: nil
+            )
+        case false:
+            return URLSession(configuration: configuration)
+        }
     }
 }

Sources/XCRemoteCache/Stats/CacheHitLogger.swift

@@ -28,8 +28,8 @@ protocol CacheHitLogger {
 /// Logs target hit or miss, based on an action of a build
 class ActionSpecificCacheHitLogger: CacheHitLogger {
     private let statsLogger: StatsLogger
-    private let hitCounter: XCRemoteCacheStatistics.Counter
-    private let missCounter: XCRemoteCacheStatistics.Counter
+    private let hitCounter: XCRemoteCacheStatistics.Counter?
+    private let missCounter: XCRemoteCacheStatistics.Counter?
 
     init(action: BuildActionType, statsLogger: StatsLogger) {
         self.statsLogger = statsLogger
@@ -37,19 +37,24 @@ class ActionSpecificCacheHitLogger: CacheHitLogger {
         case .index:
             hitCounter = .indexingTargetHitCount
             missCounter = .indexingTargetMissCount
-        case .unknown:
-            fallthrough
         case .build:
             hitCounter = .targetCacheHit
             missCounter = .targetCacheMiss
+        case .unknown:
+            hitCounter = nil
+            missCounter = nil
         }
     }
 
     func logHit() throws {
-        try statsLogger.log(hitCounter)
+        if let hitCounter = hitCounter {
+            try statsLogger.log(hitCounter)
+        }
     }
 
     func logMiss() throws {
-        try statsLogger.log(missCounter)
+        if let missCounter = missCounter {
+            try statsLogger.log(missCounter)
+        }
     }
 }

Sources/XCRemoteCache/Stats/ExclusiveFileAccessor.swift

@@ -76,8 +76,8 @@ class ExclusiveFile: ExclusiveFileAccessor {
         guard flock(fd, LOCK_EX) == 0 else {
             throw FileAccessorError.lockingFailure
         }
-        // While having a lock, make sure the file still exists.
-        // It might delete it while we were waiting for a lock.
+        // While having a lock, make sure the file still exists
+        // It might delete it while we were waiting for a lock
         guard access(fileURL.path, F_OK) == 0 else {
             throw FileAccessorError.lockingFailure
         }

Sources/XCRemoteCache/Utils/ENVReader.swift

@@ -24,8 +24,15 @@ enum EnvironmentError: Error {
 }
 
 extension Dictionary where Key == String, Value == String {
-    func readEnv(key: String) throws -> URL {
+    func readEnv(key: String) -> URL? {
         guard let value = self[key].map(URL.init(fileURLWithPath:)) else {
+            return nil
+        }
+        return value
+    }
+
+    func readEnv(key: String) throws -> URL {
+        guard let value: URL = readEnv(key: key) else {
             throw EnvironmentError.missingEnv(key)
         }
         return value
@@ -48,4 +55,11 @@ extension Dictionary where Key == String, Value == String {
         }
         return value == "YES"
     }
+
+    func readEnv(key: String) throws -> Bool? {
+        guard let value = self[key] else {
+            return nil
+        }
+        return value == "YES"
+    }
 }

Sources/xcld/XCLDMain.swift

@@ -21,7 +21,7 @@ import Foundation
 import XCRemoteCache
 
 /// Wrapper for a `LD` program that copies the dynamic executable from a cached-downloaded location
-/// Fallbacks to a standard `clang` when the Ramote cache is not applicable (e.g. modified sources)
+/// Fallbacks to a standard `clang` when the Remote cache is not applicable (e.g. modified sources)
 public class XCLDMain {
     public func main() {
         let args = ProcessInfo().arguments
@@ -48,7 +48,16 @@ public class XCLDMain {
             i += 1
         }
         guard let outputInput = output, let filelistInput = filelist, let dependencyInfoInput = dependencyInfo else {
-            exit(1, "Missing 'output' argument. Args: \(args)")
+            let ldCommand = "clang"
+            print("Fallbacking to compilation using \(ldCommand).")
+
+            let args = ProcessInfo().arguments
+            let paramList = [ldCommand] + args.dropFirst()
+            let cargs = paramList.map { strdup($0) } + [nil]
+            execvp(ldCommand, cargs)
+
+            /// C-function `execv` returns only when the command fails
+            exit(1)
         }
 
 

Sources/xcldplusplus/XCLDPlusPlus.swift

@@ -0,0 +1,75 @@
+// Copyright (c) 2021 Spotify AB.
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import Foundation
+import XCRemoteCache
+
+/// Wrapper for a `LDPLUSPLUS` program that copies the dynamic executable from a cached-downloaded location
+/// Fallbacks to a standard `clang++` when the Remote cache is not applicable (e.g. modified sources)
+public class XCLDPlusPlusMain {
+    public func main() {
+        let args = ProcessInfo().arguments
+        var output: String?
+        var filelist: String?
+        var dependencyInfo: String?
+        var i = 0
+        while i < args.count {
+            switch args[i] {
+            case "-o":
+                output = args[i + 1]
+                i += 1
+            case "-filelist":
+                filelist = args[i + 1]
+                i += 1
+            case "-dependency_info":
+                // Skip following `-Xlinker` argument. Sample call:
+                // `clang -dynamiclib  ... -Xlinker -dependency_info -Xlinker /path/Target_dependency_info.dat`
+                dependencyInfo = args[i + 2]
+                i += 2
+            default:
+                break
+            }
+            i += 1
+        }
+        guard let outputInput = output, let filelistInput = filelist, let dependencyInfoInput = dependencyInfo else {
+            let ldCommand = "clang++"
+            print("Fallbacking to compilation using \(ldCommand).")
+
+            let args = ProcessInfo().arguments
+            let paramList = [ldCommand] + args.dropFirst()
+            let cargs = paramList.map { strdup($0) } + [nil]
+            execvp(ldCommand, cargs)
+
+            /// C-function `execv` returns only when the command fails
+            exit(1)
+        }
+
+
+        // TODO: consider using `clang_command` from .rcinfo
+        /// concrete clang path should be taken from the current toolchain
+        let fallbackCommand = "clang++"
+        XCCreateBinary(
+            output: outputInput,
+            filelist: filelistInput,
+            dependencyInfo: dependencyInfoInput,
+            fallbackCommand: fallbackCommand,
+            stepDescription: "xcldplusplus"
+        ).run()
+    }
+}

Sources/xcldplusplus/main.swift

@@ -0,0 +1,22 @@
+// Copyright (c) 2021 Spotify AB.
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import XCRemoteCache
+
+XCLDPlusPlusMain().main()

Sources/xclibtool/XCLibtoolMain.swift

@@ -18,53 +18,24 @@
 // under the License.
 
 import Foundation
+import xclibtoolSupport
 import XCRemoteCache
 
+public enum XCLibtoolMainError: Error {
+    case missingOutput
+    case unsupportedMode
+}
+
 /// Wrapper for a `libtool` program that copies the build executable (e.g. .a) from a cached-downloaded location
 /// Fallbacks to a standard `libtool` when the Ramote cache is not applicable (e.g. modified sources)
 public class XCLibtoolMain {
+    public init() { }
+
     public func main() {
         let args = ProcessInfo().arguments
-        var output: String?
-        // all input arguments library '.a'. Used to create an universal binary
-        var inputLibraries: [String] = []
-        var filelist: String?
-        var dependencyInfo: String?
-        var i = 0
-        while i < args.count {
-            switch args[i] {
-            case "-o":
-                output = args[i + 1]
-                i += 1
-            case "-filelist":
-                filelist = args[i + 1]
-                i += 1
-            case "-dependency_info":
-                dependencyInfo = args[i + 1]
-                i += 1
-            case let input where input.hasSuffix(".a"):
-                inputLibraries.append(input)
-            default:
-                break
-            }
-            i += 1
-        }
-        guard let outputInput = output else {
-            exit(1, "Missing 'output' argument. Args: \(args)")
-        }
 
-        let mode: XCLibtoolMode
-        if let filelistInput = filelist, let dependencyInfoInput = dependencyInfo {
-            // libtool is creating a library
-            mode = .createLibrary(output: outputInput, filelist: filelistInput, dependencyInfo: dependencyInfoInput)
-        } else if !inputLibraries.isEmpty {
-            // multiple input libraries suggest creating an universal binary
-            mode = .createUniversalBinary(output: outputInput, inputs: inputLibraries)
-        } else {
-            // unknown mode
-            exit(1, "Unsupported mode. Args: \(args)")
-        }
         do {
+            let mode = try XCLibtoolHelper.buildMode(args: args)
             try XCLibtool(mode).run()
         } catch {
             exit(1, "Failed with: \(error). Args: \(args)")

Sources/xclibtoolSupport/XCLibtoolHelper.swift

@@ -0,0 +1,72 @@
+// Copyright (c) 2023 Spotify AB.
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import Foundation
+import XCRemoteCache
+
+public enum XCLibtoolHelperError: Error {
+    case missingOutput
+    case unsupportedMode
+}
+
+public class XCLibtoolHelper {
+    public static func buildMode(args: [String]) throws -> XCLibtoolMode {
+        var output: String?
+        // all input arguments are '*.a' or no path extension. Used to create an universal binary
+        var inputLibraries: [String] = []
+        var filelist: String?
+        var dependencyInfo: String?
+        var i = 0
+        while i < args.count {
+            switch args[i] {
+            case "-o":
+                output = args[i + 1]
+                i += 1
+            case "-filelist":
+                filelist = args[i + 1]
+                i += 1
+            case "-dependency_info":
+                dependencyInfo = args[i + 1]
+                i += 1
+            case let input where ["", "a"].contains(URL(string: args[i])?.pathExtension):
+                // Support for static frameworks (no extension) and static libraries (.a)
+                inputLibraries.append(input)
+            default:
+                break
+            }
+            i += 1
+        }
+        guard let outputInput = output else {
+            throw XCLibtoolHelperError.missingOutput
+        }
+
+        let mode: XCLibtoolMode
+        if let filelistInput = filelist, let dependencyInfoInput = dependencyInfo {
+            // libtool is creating a library
+            mode = .createLibrary(output: outputInput, filelist: filelistInput, dependencyInfo: dependencyInfoInput)
+        } else if !inputLibraries.isEmpty {
+            // multiple input libraries suggest creating an universal binary
+            mode = .createUniversalBinary(output: outputInput, inputs: inputLibraries)
+        } else {
+            // unknown mode
+            throw XCLibtoolHelperError.unsupportedMode
+        }
+        return mode
+    }
+}

Sources/xclipo/XCLipoMain.swift

@@ -0,0 +1,71 @@
+// Copyright (c) 2023 Spotify AB.
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import Foundation
+import XCRemoteCache
+
+/// Wrapper for a `lipo` program that links any of input binaries to the destination paths
+/// Fallbacks to a standard `lipo` when the Ramote cache is not applicable (e.g. modified sources)
+public class XCLipoMain {
+    public init() { }
+
+    public func main() {
+        let args = ProcessInfo().arguments
+        var output: String?
+        var create = false
+        var inputs: [String] = []
+
+        var i = 1
+        while i < args.count {
+            switch args[i] {
+            case "-output":
+                output = args[i + 1]
+                i += 1
+            case "-create":
+                create = true
+            default:
+                inputs.append(args[i])
+            }
+            i += 1
+        }
+        let lipoCommand = "lipo"
+        guard let output = output, !inputs.isEmpty, create else {
+            print("Fallbacking to compilation using \(lipoCommand).")
+
+            let args = ProcessInfo().arguments
+            let paramList = [lipoCommand] + args.dropFirst()
+            let cargs = paramList.map { strdup($0) } + [nil]
+            execvp(lipoCommand, cargs)
+
+            /// C-function `execv` returns only when the command fails
+            exit(1)
+        }
+
+        do {
+            try XCLipo(
+                output: output,
+                inputs: inputs,
+                fallbackCommand: lipoCommand,
+                stepDescription: "xclipo"
+            ).run()
+        } catch {
+            exit(1, "Failed with: \(error). Args: \(args)")
+        }
+    }
+}

Sources/xclipo/main.swift

@@ -0,0 +1,22 @@
+// Copyright (c) 2023 Spotify AB.
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import XCRemoteCache
+
+XCLipoMain().main()

Sources/xcprepare/XCPrepareMain.swift

@@ -229,6 +229,11 @@ struct XCPrepareMain: ParsableCommand {
         )
         var fakeSrcRoot: String
 
+        @Option(name: .customLong("sdks-exclude"), default: "", help: """
+        comma separated list of sdks to not integrate XCRemoteCache (e.g. "watchos*, watchsimulator*")
+        """, transform: nonEmptyString)
+        var sdksExclude: String
+
 
         func run() throws {
             XCIntegrate(
@@ -243,6 +248,7 @@ struct XCPrepareMain: ParsableCommand {
                 consumerEligiblePlatforms: consumerEligiblePlatforms,
                 lldbMode: lldbInit,
                 fakeSrcRoot: fakeSrcRoot,
+                sdksExclude: sdksExclude,
                 output: output
             ).main()
         }

Sources/xcswiftc/XCSwiftcMain.swift

@@ -60,8 +60,16 @@ public class XCSwiftcMain {
             let targetInputInput = target,
             let swiftFileListInput = swiftFileList
             else {
-                print("Missing argument. Args: \(args)")
-                exit(1)
+            let swiftcCommand = "swiftc"
+            print("Fallbacking to compilation using \(swiftcCommand).")
+
+            let args = ProcessInfo().arguments
+            let paramList = [swiftcCommand] + args.dropFirst()
+            let cargs = paramList.map { strdup($0) } + [nil]
+            execvp(swiftcCommand, cargs)
+
+            /// C-function `execv` returns only when the command fails
+            exit(1)
         }
         let swiftcArgsInput = SwiftcArgInput(
             objcHeaderOutput: objcHeaderOutputInput,

Tests/XCRemoteCacheTests/Artifacts/ArtifactSwiftProductsBuilderImplTests.swift

@@ -27,15 +27,23 @@ class ArtifactSwiftProductsBuilderImplTests: FileXCTestCase {
     private var swiftmoduleFile: URL!
     private var swiftmoduleDocFile: URL!
     private var swiftmoduleSourceInfoFile: URL!
+    private var swiftmoduleInterfaceFile: URL!
+    private var privateSwiftmoduleInterfaceFile: URL!
+    private var abiJsonFile: URL!
     private var workingDir: URL!
     private var builder: ArtifactSwiftProductsBuilderImpl!
 
     override func setUpWithError() throws {
+        try super.setUpWithError()
         let rootDir = try prepareTempDir()
         moduleDir = rootDir.appendingPathComponent("Products")
         swiftmoduleFile = moduleDir.appendingPathComponent("MyModule.swiftmodule")
         swiftmoduleDocFile = moduleDir.appendingPathComponent("MyModule.swiftdoc")
         swiftmoduleSourceInfoFile = moduleDir.appendingPathComponent("MyModule.swiftsourceinfo")
+        swiftmoduleInterfaceFile = moduleDir.appendingPathComponent("MyModule.swiftinterface")
+        privateSwiftmoduleInterfaceFile = moduleDir.appendingPathComponent("MyModule.private.swiftinterface")
+        abiJsonFile = moduleDir.appendingPathComponent("MyModule.abi.json")
+
         workingDir = rootDir.appendingPathComponent("working")
         builder = ArtifactSwiftProductsBuilderImpl(
             workingDir: workingDir,
@@ -47,24 +55,41 @@ class ArtifactSwiftProductsBuilderImplTests: FileXCTestCase {
     func testIncludesRequiredSwiftmoduleFiles() throws {
         try fileManager.spt_createFile(swiftmoduleFile, content: "swiftmodule")
         try fileManager.spt_createFile(swiftmoduleDocFile, content: "swiftdoc")
-        let builderSwiftmoduleDir = builder.buildingArtifactSwiftModulesLocation().appendingPathComponent("arm64")
-        let expectedBuildedSwiftmoduleFile = builderSwiftmoduleDir.appendingPathComponent("MyModule.swiftmodule")
-        let expectedBuildedSwiftmoduledocFile = builderSwiftmoduleDir.appendingPathComponent("MyModule.swiftdoc")
+        let builderSwiftmoduleDir =
+            builder
+                .buildingArtifactSwiftModulesLocation()
+                .appendingPathComponent("arm64")
+        let expectedBuildedSwiftmoduleFile =
+            builderSwiftmoduleDir.appendingPathComponent("MyModule.swiftmodule")
+        let expectedBuildedSwiftmoduledocFile =
+            builderSwiftmoduleDir.appendingPathComponent("MyModule.swiftdoc")
 
         try builder.includeModuleDefinitionsToTheArtifact(arch: "arm64", moduleURL: swiftmoduleFile)
 
-        XCTAssertEqual(fileManager.contents(atPath: expectedBuildedSwiftmoduleFile.path), "swiftmodule".data(using: .utf8))
-        XCTAssertEqual(fileManager.contents(atPath: expectedBuildedSwiftmoduledocFile.path), "swiftdoc".data(using: .utf8))
+        XCTAssertEqual(
+            fileManager.contents(atPath: expectedBuildedSwiftmoduleFile.path),
+            "swiftmodule".data(using: .utf8)
+        )
+        XCTAssertEqual(
+            fileManager.contents(atPath: expectedBuildedSwiftmoduledocFile.path),
+            "swiftdoc".data(using: .utf8)
+        )
     }
 
-    func testIncludesAllSwiftmoduleFiles() throws {
+    func testIncludesAllBasicSwiftmoduleFiles() throws {
         try fileManager.spt_createEmptyFile(swiftmoduleFile)
         try fileManager.spt_createEmptyFile(swiftmoduleDocFile)
         try fileManager.spt_createEmptyFile(swiftmoduleSourceInfoFile)
-        let builderSwiftmoduleDir = builder.buildingArtifactSwiftModulesLocation().appendingPathComponent("arm64")
-        let expectedBuildedSwiftmoduleFile = builderSwiftmoduleDir.appendingPathComponent("MyModule.swiftmodule")
-        let expectedBuildedSwiftmoduledocFile = builderSwiftmoduleDir.appendingPathComponent("MyModule.swiftdoc")
-        let expectedBuildedSwiftSourceInfoFile = builderSwiftmoduleDir.appendingPathComponent("MyModule.swiftsourceinfo")
+        let builderSwiftmoduleDir =
+            builder
+                .buildingArtifactSwiftModulesLocation()
+                .appendingPathComponent("arm64")
+        let expectedBuildedSwiftmoduleFile =
+            builderSwiftmoduleDir.appendingPathComponent("MyModule.swiftmodule")
+        let expectedBuildedSwiftmoduledocFile =
+            builderSwiftmoduleDir.appendingPathComponent("MyModule.swiftdoc")
+        let expectedBuildedSwiftSourceInfoFile =
+            builderSwiftmoduleDir.appendingPathComponent("MyModule.swiftsourceinfo")
 
         try builder.includeModuleDefinitionsToTheArtifact(arch: "arm64", moduleURL: swiftmoduleFile)
 
@@ -73,7 +98,46 @@ class ArtifactSwiftProductsBuilderImplTests: FileXCTestCase {
         XCTAssertTrue(fileManager.fileExists(atPath: expectedBuildedSwiftSourceInfoFile.path))
     }
 
+    func testIncludesAllEvolutionEnabledSwiftmoduleFiles() throws {
+        try fileManager.spt_createEmptyFile(swiftmoduleFile)
+        try fileManager.spt_createEmptyFile(swiftmoduleDocFile)
+        try fileManager.spt_createEmptyFile(swiftmoduleSourceInfoFile)
+        try fileManager.spt_createEmptyFile(swiftmoduleInterfaceFile)
+        try fileManager.spt_createEmptyFile(privateSwiftmoduleInterfaceFile)
+        try fileManager.spt_createEmptyFile(abiJsonFile)
+        let builderSwiftmoduleDir =
+            builder
+                .buildingArtifactSwiftModulesLocation()
+                .appendingPathComponent("arm64")
+        let expectedBuildedSwiftmoduleFile =
+            builderSwiftmoduleDir.appendingPathComponent("MyModule.swiftmodule")
+        let expectedBuildedSwiftmoduledocFile =
+            builderSwiftmoduleDir.appendingPathComponent("MyModule.swiftdoc")
+        let expectedBuildedSwiftSourceInfoFile =
+            builderSwiftmoduleDir.appendingPathComponent("MyModule.swiftsourceinfo")
+        let expectedBuildedSwiftInterfaceFile =
+            builderSwiftmoduleDir.appendingPathComponent("MyModule.swiftinterface")
+        let expectedPrivateSwiftmoduleInterfaceFile =
+            builderSwiftmoduleDir.appendingPathComponent("MyModule.private.swiftinterface")
+        let expectedAbiJsonFile =
+            builderSwiftmoduleDir.appendingPathComponent("MyModule.abi.json")
+
+        try builder.includeModuleDefinitionsToTheArtifact(arch: "arm64", moduleURL: swiftmoduleFile)
+
+        XCTAssertTrue(fileManager.fileExists(atPath: expectedBuildedSwiftmoduleFile.path))
+        XCTAssertTrue(fileManager.fileExists(atPath: expectedBuildedSwiftmoduledocFile.path))
+        XCTAssertTrue(fileManager.fileExists(atPath: expectedBuildedSwiftSourceInfoFile.path))
+        XCTAssertTrue(fileManager.fileExists(atPath: expectedBuildedSwiftInterfaceFile.path))
+        XCTAssertTrue(fileManager.fileExists(atPath: expectedPrivateSwiftmoduleInterfaceFile.path))
+        XCTAssertTrue(fileManager.fileExists(atPath: expectedAbiJsonFile.path))
+    }
+
     func testFailsIncludingWhenMissingRequiredSwiftmoduleFiles() throws {
-        XCTAssertThrowsError(try builder.includeModuleDefinitionsToTheArtifact(arch: "arm64", moduleURL: swiftmoduleFile))
+        XCTAssertThrowsError(
+            try builder.includeModuleDefinitionsToTheArtifact(
+                arch: "arm64",
+                moduleURL: swiftmoduleFile
+            )
+        )
     }
 }

Tests/XCRemoteCacheTests/Artifacts/BuildArtifactCreatorTests.swift

@@ -31,6 +31,9 @@ class BuildArtifactCreatorTests: FileXCTestCase {
     private var swiftmoduleURL: URL!
     private var swiftdocURL: URL!
     private var swiftSourceInfoURL: URL!
+    private var swiftInterfaceURL: URL!
+    private var privateSwiftInterfaceURL: URL!
+    private var abiJsonURL: URL!
     private var executablePath: String!
     private var executableURL: URL!
     private var creator: BuildArtifactCreator!
@@ -50,11 +53,18 @@ class BuildArtifactCreatorTests: FileXCTestCase {
             .appendingPathComponent("Target.swiftdoc")
         swiftSourceInfoURL = workDirectory.appendingPathComponent("Objects-normal")
             .appendingPathComponent("Target.swiftsourceinfo")
+        swiftInterfaceURL = workDirectory.appendingPathComponent("Objects-normal")
+            .appendingPathComponent("Target.swiftinterface")
+        privateSwiftInterfaceURL = workDirectory.appendingPathComponent("Objects-normal")
+            .appendingPathComponent("Target.private.swiftinterface")
+        abiJsonURL = workDirectory.appendingPathComponent("Objects-normal")
+            .appendingPathComponent("Target.abi.json")
         executablePath = "libTarget.a"
         executableURL = buildDir.appendingPathComponent(executablePath)
         dSYM = executableURL.deletingPathExtension().appendingPathExtension(".dSYM")
         try fileManager.spt_createEmptyFile(executableURL)
         try fileManager.spt_createEmptyFile(headerURL)
+        let artifactProcessor = NoopArtifactProcessor()
 
         creator = BuildArtifactCreator(
             buildDir: buildDir,
@@ -63,6 +73,8 @@ class BuildArtifactCreatorTests: FileXCTestCase {
             moduleName: "Target",
             modulesFolderPath: "",
             dSYMPath: dSYM,
+            metaWriter: JsonMetaWriter(fileWriter: fileManager, pretty: false),
+            artifactProcessor: artifactProcessor,
             fileManager: fileManager
         )
     }
@@ -113,6 +125,32 @@ class BuildArtifactCreatorTests: FileXCTestCase {
         ])
     }
 
+    func testPackagesEvolutionEnabledSwiftmoduleFiles() throws {
+        try fileManager.spt_createEmptyFile(swiftmoduleURL)
+        try fileManager.spt_createEmptyFile(swiftdocURL)
+        try fileManager.spt_createEmptyFile(swiftSourceInfoURL)
+        try fileManager.spt_createEmptyFile(swiftInterfaceURL)
+        try fileManager.spt_createEmptyFile(privateSwiftInterfaceURL)
+        try fileManager.spt_createEmptyFile(abiJsonURL)
+
+        try creator.includeModuleDefinitionsToTheArtifact(arch: "arch", moduleURL: swiftmoduleURL)
+        let artifact = try creator.createArtifact(artifactKey: "key", meta: sampleMeta)
+
+        let unzippedURL = workDirectory.appendingPathComponent(UUID().uuidString)
+        try Zip.unzipFile(artifact.package, destination: unzippedURL, overwrite: true, password: nil, progress: nil)
+        let allFiles = try fileManager.spt_allFilesRecusively(unzippedURL)
+        XCTAssertEqual(Set(allFiles), [
+            unzippedURL.appendingPathComponent("libTarget.a"),
+            unzippedURL.appendingPathComponent("fileKey.json"),
+            unzippedURL.appendingPathComponent("swiftmodule/arch/Target.swiftmodule"),
+            unzippedURL.appendingPathComponent("swiftmodule/arch/Target.swiftdoc"),
+            unzippedURL.appendingPathComponent("swiftmodule/arch/Target.swiftsourceinfo"),
+            unzippedURL.appendingPathComponent("swiftmodule/arch/Target.swiftinterface"),
+            unzippedURL.appendingPathComponent("swiftmodule/arch/Target.private.swiftinterface"),
+            unzippedURL.appendingPathComponent("swiftmodule/arch/Target.abi.json"),
+        ])
+    }
+
     func testFailsPackageWhenSwiftmoduleRelatedFilesAreMissing() throws {
         // Creating only `Target.swiftmodule`, without `.swiftdoc`
         try fileManager.spt_createEmptyFile(swiftmoduleURL)

Tests/XCRemoteCacheTests/Artifacts/TextFileDependenciesRemapperTests.swift

@@ -0,0 +1,101 @@
+// Copyright (c) 2022 Spotify AB.
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+@testable import XCRemoteCache
+
+import XCTest
+
+class TextFileDependenciesRemapperTests: FileXCTestCase {
+
+    let stringsRemapper = StringDependenciesRemapper(
+        mappings: [
+            .init(generic: "$(SRCROOT)", local: "/example"),
+        ])
+    let fileAccessor = FileAccessorFake(mode: .strict)
+    var remapper: TextFileDependenciesRemapper!
+
+    override func setUp() {
+        super.setUp()
+        remapper = TextFileDependenciesRemapper(
+            remapper: stringsRemapper,
+            fileAccessor: fileAccessor
+        )
+    }
+
+    func testRemapsGenericPlaceholders() throws {
+        try fileAccessor.write(toPath: "/file", contents: "Some $(SRCROOT).")
+
+        try remapper.remap(fromGeneric: "/file")
+
+        try XCTAssertEqual(fileAccessor.contents(atPath: "/file"), "Some /example.")
+    }
+
+    func testRemapsLocalPathsToPlaceholders() throws {
+        try fileAccessor.write(toPath: "/file", contents: "Some /example.")
+
+        try remapper.remap(fromLocal: "/file")
+
+        try XCTAssertEqual(fileAccessor.contents(atPath: "/file"), "Some $(SRCROOT).")
+    }
+
+    func testPersistsEmptyLines() throws {
+        let multilineData = """
+        Line1
+
+        Line 2
+        """.data(using: .utf8)
+        try fileAccessor.write(toPath: "/file", contents: multilineData)
+
+        try remapper.remap(fromGeneric: "/file")
+
+        try XCTAssertEqual(fileAccessor.contents(atPath: "/file"), multilineData)
+    }
+
+    func testPersistsEmptyLineAtTheEnd() throws {
+        // swiftlint:disable trailing_whitespace
+        let multilineData = """
+        Line1
+
+        Line 2
+
+        """.data(using: .utf8)
+        // swiftlint:enable trailing_whitespace
+        try fileAccessor.write(toPath: "/file", contents: multilineData)
+
+        try remapper.remap(fromGeneric: "/file")
+
+        try XCTAssertEqual(fileAccessor.contents(atPath: "/file"), multilineData)
+    }
+
+    func testReplacesMultipletimesInLine() throws {
+        try fileAccessor.write(toPath: "/file", contents: "$(SRCROOT) and $(SRCROOT)")
+
+        try remapper.remap(fromGeneric: "/file")
+
+        try XCTAssertEqual(fileAccessor.contents(atPath: "/file"), "/example and /example")
+    }
+
+    func testReplacesInMultipleLine() throws {
+        try fileAccessor.write(toPath: "/file", contents: "$(SRCROOT)\n$(SRCROOT)")
+
+        try remapper.remap(fromGeneric: "/file")
+
+        try XCTAssertEqual(fileAccessor.contents(atPath: "/file"), "/example\n/example")
+    }
+}

Tests/XCRemoteCacheTests/Artifacts/UnzippedArtifactProcessorTests.swift

@@ -0,0 +1,71 @@
+// Copyright (c) 2022 Spotify AB.
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+@testable import XCRemoteCache
+
+import XCTest
+
+class UnzippedArtifactProcessorTests: FileXCTestCase {
+
+    private let fileAccessor = FileAccessorFake(mode: .strict)
+    private let remapper = StringDependenciesRemapper(mappings: [.init(generic: "$(SRCROOT)", local: "/local")])
+    private var fileRemapper: FileDependenciesRemapper!
+    private var processor: UnzippedArtifactProcessor!
+
+    override func setUp() {
+        super.setUp()
+        fileRemapper = TextFileDependenciesRemapper(remapper: remapper, fileAccessor: fileAccessor)
+        processor = UnzippedArtifactProcessor(
+            fileRemapper: fileRemapper,
+            dirScanner: fileAccessor
+        )
+    }
+
+    func testProcessingRawArtifactReplacesPlaceholders() throws {
+        try fileAccessor.write(toPath: "/artifact/include/file", contents: "Some $(SRCROOT)")
+
+        try processor.process(rawArtifact: "/artifact")
+
+        XCTAssertEqual(try fileAccessor.contents(atPath: "/artifact/include/file"), "Some /local")
+    }
+
+    func testProcessingRawArtifactReplacesInNestedInclude() throws {
+        try fileAccessor.write(toPath: "/artifact/include/nested/file", contents: "Some $(SRCROOT)")
+
+        try processor.process(rawArtifact: "/artifact")
+
+        XCTAssertEqual(try fileAccessor.contents(atPath: "/artifact/include/nested/file"), "Some /local")
+    }
+
+    func testProcessingRawArtifactDoesntReplacesInNonIncludeDir() throws {
+        try fileAccessor.write(toPath: "/artifact/some/file", contents: "Some $(SRCROOT)")
+
+        try processor.process(rawArtifact: "/artifact")
+
+        XCTAssertEqual(try fileAccessor.contents(atPath: "/artifact/some/file"), "Some $(SRCROOT)")
+    }
+
+    func testDoesntProcessEmptyFiles() throws {
+        try fileAccessor.write(toPath: "/artifact/include/.hidden", contents: "Some $(SRCROOT)")
+
+        try processor.process(rawArtifact: "/artifact")
+
+        XCTAssertEqual(try fileAccessor.contents(atPath: "/artifact/include/.hidden"), "Some $(SRCROOT)")
+    }
+}

Tests/XCRemoteCacheTests/Artifacts/ZipArtifactCreatorTests.swift

@@ -35,7 +35,11 @@ class ZipArtifactCreatorTests: FileXCTestCase {
     override func setUpWithError() throws {
         try super.setUpWithError()
         workingDir = try prepareTempDir().appendingPathComponent("creator")
-        creator = ZipArtifactCreator(workingDir: workingDir, fileManager: fileManager)
+        creator = ZipArtifactCreator(
+            workingDir: workingDir,
+            metaWriter: JsonMetaWriter(fileWriter: fileManager, pretty: false),
+            fileManager: fileManager
+        )
     }
 
     func testCreatingArtifactGeneratesValidArtifactId() throws {

Tests/XCRemoteCacheTests/Artifacts/ZipArtifactOrganizerTests.swift

@@ -54,7 +54,11 @@ class ZipArtifactOrganizerTests: XCTestCase {
 
     func testPreparePlacesArtifactInTheActiveLocation() throws {
         let zipURL = try prepareZipFile(content: "Magic", fileName: "content.txt")
-        let organizer = ZipArtifactOrganizer(targetTempDir: workingDirectory, fileManager: fileManager)
+        let organizer = ZipArtifactOrganizer(
+            targetTempDir: workingDirectory,
+            artifactProcessors: [],
+            fileManager: fileManager
+        )
 
         let preparedArtifact = try organizer.prepare(artifact: zipURL)
 
@@ -64,7 +68,11 @@ class ZipArtifactOrganizerTests: XCTestCase {
 
     func testPreparingExistingArtifact() throws {
         let zipURL = try prepareZipFile(content: "Magic", fileName: "content.txt")
-        let organizer = ZipArtifactOrganizer(targetTempDir: workingDirectory, fileManager: fileManager)
+        let organizer = ZipArtifactOrganizer(
+            targetTempDir: workingDirectory,
+            artifactProcessors: [],
+            fileManager: fileManager
+        )
 
         _ = try organizer.prepare(artifact: zipURL)
         let preparedArtifact = try organizer.prepare(artifact: zipURL)
@@ -75,7 +83,11 @@ class ZipArtifactOrganizerTests: XCTestCase {
 
     func testPreparePlacesArtifactInTheFileKeyRelatedLocation() throws {
         let zipURL = try prepareZipFile(content: "Magic", fileName: "content.txt", zipFileName: "abb32_fileKey")
-        let organizer = ZipArtifactOrganizer(targetTempDir: workingDirectory, fileManager: fileManager)
+        let organizer = ZipArtifactOrganizer(
+            targetTempDir: workingDirectory,
+            artifactProcessors: [],
+            fileManager: fileManager
+        )
         let expectedArtifactLocation = workingDirectory.appendingPathComponent("abb32_fileKey")
 
         let preparedArtifact = try organizer.prepare(artifact: zipURL)
@@ -89,7 +101,11 @@ class ZipArtifactOrganizerTests: XCTestCase {
         let artifactLocation = workingDirectory.appendingPathComponent("xccache")
             .appendingPathComponent(sampleFileKey, isDirectory: true)
         try fileManager.createDirectory(at: artifactLocation, withIntermediateDirectories: true, attributes: nil)
-        let organizer = ZipArtifactOrganizer(targetTempDir: workingDirectory, fileManager: fileManager)
+        let organizer = ZipArtifactOrganizer(
+            targetTempDir: workingDirectory,
+            artifactProcessors: [],
+            fileManager: fileManager
+        )
 
         let result = try organizer.prepareArtifactLocationFor(fileKey: sampleFileKey)
         if case .artifactExists(artifactDir: let u) = result {
@@ -105,7 +121,11 @@ class ZipArtifactOrganizerTests: XCTestCase {
             .appendingPathComponent("xccache")
             .appendingPathComponent(sampleFileKey)
             .appendingPathExtension("zip")
-        let organizer = ZipArtifactOrganizer(targetTempDir: workingDirectory, fileManager: fileManager)
+        let organizer = ZipArtifactOrganizer(
+            targetTempDir: workingDirectory,
+            artifactProcessors: [],
+            fileManager: fileManager
+        )
 
         let result = try organizer.prepareArtifactLocationFor(fileKey: sampleFileKey)
 
@@ -126,7 +146,11 @@ class ZipArtifactOrganizerTests: XCTestCase {
         try fileManager.createDirectory(at: activeArtifact, withIntermediateDirectories: true, attributes: nil)
         try fileManager.spt_forceSymbolicLink(at: activeLink, withDestinationURL: activeArtifact)
 
-        let organizer = ZipArtifactOrganizer(targetTempDir: workingDirectory, fileManager: fileManager)
+        let organizer = ZipArtifactOrganizer(
+            targetTempDir: workingDirectory,
+            artifactProcessors: [],
+            fileManager: fileManager
+        )
 
         let fileKey = try organizer.getActiveArtifactFilekey()
 

Tests/XCRemoteCacheTests/Commands/Plugins/Thinning/ThinningCreatorPluginTests.swift

@@ -33,7 +33,10 @@ class ThinningCreatorPluginTests: FileXCTestCase {
         targetTempDirRoot = workingDir.appendingPathComponent("Root")
         currentTargetTempDir = targetTempDirRoot.appendingPathComponent("Current.build")
         try fileManager.spt_createEmptyDir(currentTargetTempDir)
-        plugin = ThinningCreatorPlugin(targetTempDir: currentTargetTempDir, dirScanner: FileManager.default)
+        plugin = ThinningCreatorPlugin(
+            targetTempDir: currentTargetTempDir,
+            modeMarkerPath: "rc_marker.enabled",
+            dirScanner: FileManager.default)
     }
 
     func testReturnsEmptyExtraKeysForNoArtifacts() throws {
@@ -73,4 +76,62 @@ class ThinningCreatorPluginTests: FileXCTestCase {
 
         XCTAssertThrowsError(try plugin.extraMetaKeys(Self.sampleMeta))
     }
+
+    func testDefinesExtraMetaKeysForTargetsThatReusedArtifact() throws {
+        let otherTargetTempDir = targetTempDirRoot.appendingPathComponent("Other.build")
+        let marker = otherTargetTempDir.appendingPathComponent("rc_marker.enabled")
+        let reusedArtifact = otherTargetTempDir
+            .appendingPathComponent("xccache")
+            .appendingPathComponent("123")
+            .appendingPathExtension("zip")
+        try fileManager.spt_createEmptyFile(marker)
+        try fileManager.spt_createEmptyFile(reusedArtifact)
+
+        let extraKeys = try plugin.extraMetaKeys(Self.sampleMeta)
+
+        XCTAssertEqual(extraKeys, ["thinning_Other": "123"])
+    }
+
+    func testFailsGeneratingExtraMetaKeysForTwoArtifactsInTargetTempDir() throws {
+        let otherTargetTempDir = targetTempDirRoot.appendingPathComponent("Other.build")
+        let marker = otherTargetTempDir.appendingPathComponent("rc_marker.enabled")
+        let reusedArtifact1 = otherTargetTempDir
+            .appendingPathComponent("xccache")
+            .appendingPathComponent("001")
+            .appendingPathExtension("zip")
+        let reusedArtifact2 = otherTargetTempDir
+            .appendingPathComponent("xccache")
+            .appendingPathComponent("002")
+            .appendingPathExtension("zip")
+        try fileManager.spt_createEmptyFile(marker)
+        try fileManager.spt_createEmptyFile(reusedArtifact1)
+        try fileManager.spt_createEmptyFile(reusedArtifact2)
+
+        XCTAssertThrowsError(try plugin.extraMetaKeys(Self.sampleMeta))
+    }
+
+    func testDefinesExtraMetaKeysForGeneratedAndReusedArtifact() throws {
+        let otherTargetTempDir = targetTempDirRoot.appendingPathComponent("Generated.build")
+        let generatedArtifact = otherTargetTempDir
+            .appendingPathComponent("xccache")
+            .appendingPathComponent("produced")
+            .appendingPathComponent("000")
+            .appendingPathExtension("zip")
+        try fileManager.spt_createEmptyFile(generatedArtifact)
+        let reusedTargetTempDir = targetTempDirRoot.appendingPathComponent("Reused.build")
+        let marker = reusedTargetTempDir.appendingPathComponent("rc_marker.enabled")
+        let reusedArtifact = reusedTargetTempDir
+            .appendingPathComponent("xccache")
+            .appendingPathComponent("999")
+            .appendingPathExtension("zip")
+        try fileManager.spt_createEmptyFile(marker)
+        try fileManager.spt_createEmptyFile(reusedArtifact)
+
+        let extraKeys = try plugin.extraMetaKeys(Self.sampleMeta)
+
+        XCTAssertEqual(extraKeys, [
+            "thinning_Generated": "000",
+            "thinning_Reused": "999",
+        ])
+    }
 }

Tests/XCRemoteCacheTests/Commands/Plugins/Thinning/ThinningDiskSwiftcProductsGeneratorTests.swift

@@ -67,7 +67,8 @@ class ThinningDiskSwiftcProductsGeneratorTests: FileXCTestCase {
             artifactSwiftModuleObjCFile: headerFile
         )
 
-        XCTAssertEqual(generatedModulePath, destinationSwiftModuleDir)
+        XCTAssertEqual(generatedModulePath.swiftmoduleDir, destinationSwiftModuleDir)
+        XCTAssertEqual(generatedModulePath.objcHeaderFile, objCHeader)
         XCTAssertEqual(fileManager.contents(atPath: expectedSwiftSourceInfoFile.path), "sourceInfo".data(using: .utf8))
         XCTAssertEqual(fileManager.contents(atPath: objCHeader.path), "header".data(using: .utf8))
     }