Merge pull request #225 from polac24/fix-spaces-libtool

Support spaces in libtools paths

.github/ISSUE_TEMPLATE/BUG_REPORT.md

@@ -0,0 +1,65 @@
+---
+name: ⚠️ Bug Report
+about: Something isn't working as expected
+
+---
+
+<!--
+PLEASE HELP US PROCESS GITHUB ISSUES FASTER BY PROVIDING THE FOLLOWING INFORMATION.
+-->
+
+**My integration setup**
+
+[ ] CocoaPods cocoapods-xcremotecache plugin
+[ ] Automatic integration using `xcprepare integrate ...`
+[ ] Manual integration
+[ ] Carthage
+
+**Expected/desired behavior**
+<!-- Describe what the desired behavior would be. -->
+
+**Minimal reproduction of the problem with instructions**
+<!-- Please provide the *STEPS TO REPRODUCE*. -->
+
+**Producer Logs**
+<!-- Capture logs from 10 minutes: `log show --predicate 'sender BEGINSWITH "xc"' --style compact --info --debug -last 10m` -->
+
+<details>
+  <pre> [REPLACE THIS WITH YOUR INFORMATION] </pre>
+</details>
+
+**Consumer Logs**
+<!-- Capture logs from 10 minutes: `log show --predicate 'sender BEGINSWITH "xc"' --style compact --info --debug -last 10m` -->
+
+<details>
+  <pre> [REPLACE THIS WITH YOUR INFORMATION] </pre>
+</details>
+
+**Pods/Carthage file**
+<!-- Delete if you don't use CocoaPods or Carthage -->
+
+<details>
+  <pre> [REPLACE THIS WITH YOUR INFORMATION] </pre>
+</details>
+
+**Environment**
+
+* **XCRemoteCache:** X.Y.Z
+* **cocoapods-xcremotecache:** X.Y.Z <!-- check with `gem list cocoapods-xcremotecache` >
+* **HTTP cache server:** ... <!-- e.g. demo docker, nginx, AWS etc. >
+* **Xcode:** X.Y.Z
+
+**Post build stats**
+<!-- 
+To capture build statistics: 
+* call `xcprepare stats --reset` (or `XCRC/xcprepare stats --reset` for CocoaPods)
+* Build a project in Xcode
+* `xcprepare stats` (or `XCRC/xcprepare stats` for CocoaPods) 
+-->
+
+<details>
+  <pre> [REPLACE THIS WITH YOUR INFORMATION] </pre>
+</details>
+
+**Others**
+<!-- Anything else relevant?  Operating system version, , ... -->

.github/ISSUE_TEMPLATE/DOCUMENTATION_ISSUE.md

@@ -0,0 +1,16 @@
+---
+name: 📕 Documentation Issue
+about: Suggestion for a change in a documentation
+
+---
+
+<!--
+PLEASE HELP US PROCESS GITHUB ISSUES FASTER BY PROVIDING THE FOLLOWING INFORMATION.
+-->
+
+**A suggestion**
+<!-- Describe how could the documentation be improved. -->
+
+**Which file, section, line**
+<!-- Provide a section it relates (if exist). -->
+

.github/ISSUE_TEMPLATE/FEATURE_REQUEST.md

@@ -0,0 +1,20 @@
+---
+name: 🙏 Future Request
+about: Suggestion for an improvement, either behaviour or implementation
+
+---
+
+<!--
+PLEASE HELP US PROCESS GITHUB ISSUES FASTER BY PROVIDING THE FOLLOWING INFORMATION.
+-->
+
+
+**Expected/desired behavior**
+<!-- Describe what the desired behavior would be. -->
+
+**Relevant integration setup**
+
+[ ] CocoaPods cocoapods-xcremotecache plugin
+[ ] Automatic integration using `xcprepare integrate ...`
+[ ] Manual integration
+[ ] Carthage

.github/workflows/ci.yaml

@@ -9,11 +9,13 @@ jobs:
       - uses: actions/checkout@v1
       - name: SwiftLint
         uses: norio-nomura/action-swiftlint@3.1.0
+        with:
+          args: --strict
 
   macOS:
-    runs-on: macOS-latest
+    runs-on: macos-12
     env:
-      XCODE_VERSION: ${{ '13.1' }}
+      XCODE_VERSION: ${{ '14.2' }}
     steps:
       - name: Select Xcode
         run: "sudo xcode-select -s /Applications/Xcode_$XCODE_VERSION.app"
@@ -23,3 +25,5 @@ jobs:
         run: rake build[release]
       - name: Test
         run: rake test
+      - name: E2ETests
+        run: rake e2e_only

.github/workflows/docs.yaml

@@ -0,0 +1,24 @@
+name: Docs
+
+on:
+  push:
+    branches:
+      - master
+
+jobs:
+  docs:
+    runs-on: macos-12
+    env:
+      XCODE_VERSION: ${{ '14.2' }}
+    steps:
+      - uses: actions/checkout@v2
+      - name: Select Xcode
+        run: "sudo xcode-select -s /Applications/Xcode_$XCODE_VERSION.app"
+      - name: "Generate documentation"
+        run: "swift package --allow-writing-to-directory ./docs  generate-documentation --target XCRemoteCache --disable-indexing --transform-for-static-hosting --output-path ./docs  --hosting-base-path XCRemoteCache/"
+      - name: Deploy GH-pages
+        uses: peaceiris/actions-gh-pages@v3
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          publish_dir: ./docs
+          keep_files: false

.github/workflows/release.yaml

@@ -6,9 +6,9 @@ on:
 jobs:
   macOS:
     name: Add macOS binaries to release
-    runs-on: macOS-latest
+    runs-on: macos-12
     env:
-      XCODE_VERSION: ${{ '13.1' }}
+      XCODE_VERSION: ${{ '14.2' }}
     steps:
       - name: Select Xcode
         run: "sudo xcode-select -s /Applications/Xcode_$XCODE_VERSION.app"

.gitignore

@@ -1,9 +1,13 @@
 .DS_Store
 /.build
 /Packages
-/*.xcodeproj
+*.xcodeproj/
+*.xcworkspace/
 DerivedData
 /.swiftpm/
 releases
 tmp/
 .idea/
+xcuserdata
+*.gem
+Pods/

.swiftlint.yml

@@ -5,7 +5,6 @@ disabled_rules:
   - superfluous_disable_command # Disabled since we disable some rules pre-emptively to avoid issues in the future
   - todo                        # Temporarily disabled. We have too many right now hiding real issues :(
   - nesting                     # Does not make sense anymore since Swift 4 uses nested `CodingKeys` enums for example
-  - trailing_dot_in_comments    # Triggers warnings for generated file headers
 
 opt_in_rules:
   - anyobject_protocol
@@ -64,6 +63,8 @@ excluded:
   - docs/
   - fastlane/
   - DerivedData/
+  - e2eTests/XCRemoteCacheSample/Pods
+  - e2eTests/StandaloneSampleApp
 
 attributes:
   always_on_same_line:
@@ -88,6 +89,25 @@ file_header:
                      \/\/  Created by .*? on .*\.
                      \/\/  Copyright © \d{4} .*\. All rights reserved\.
                      \/\/
+  required_pattern: |
+                     \/\/ Copyright \(c\) \d{4} Spotify AB\.
+                     \/\/
+                     \/\/ Licensed to the Apache Software Foundation \(ASF\) under one
+                     \/\/ or more contributor license agreements\.  See the NOTICE file
+                     \/\/ distributed with this work for additional information
+                     \/\/ regarding copyright ownership\.  The ASF licenses this file
+                     \/\/ to you under the Apache License, Version 2.0 \(the
+                     \/\/ "License"\); you may not use this file except in compliance
+                     \/\/ with the License\.  You may obtain a copy of the License at
+                     \/\/
+                     \/\/ http:\/\/www.apache.org\/licenses\/LICENSE-2\.0
+                     \/\/
+                     \/\/ Unless required by applicable law or agreed to in writing,
+                     \/\/ software distributed under the License is distributed on an
+                     \/\/ \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+                     \/\/ KIND, either express or implied\.  See the License for the
+                     \/\/ specific language governing permissions and limitations
+                     \/\/ under the License\.
 force_cast: warning
 force_try: warning
 implicit_getter: warning
@@ -124,6 +144,6 @@ custom_rules:
     severity: warning
   trailing_dot_in_comments:
     name: "Trailing dot in comments"
-    regex: '^[ ]*///?[^\n]*\.\n'
+    regex: '^(?!\/\/\ Copyright\ \(c\)\ \d{4}\ Spotify AB\.|\/\/\ under\ the\ License\.)[ ]*///?[^\n]*\.\n'
     message: "There shouldn't be trailing dot in comments"
     severity: warning

Package.resolved

@@ -3,16 +3,16 @@
     "pins": [
       {
         "package": "AEXML",
-        "repositoryURL": "https://github.com/tadija/AEXML",
+        "repositoryURL": "https://github.com/tadija/AEXML.git",
         "state": {
           "branch": null,
-          "revision": "8623e73b193386909566a9ca20203e33a09af142",
-          "version": "4.5.0"
+          "revision": "38f7d00b23ecd891e1ee656fa6aeebd6ba04ecc3",
+          "version": "4.6.1"
         }
       },
       {
         "package": "PathKit",
-        "repositoryURL": "https://github.com/kylef/PathKit",
+        "repositoryURL": "https://github.com/kylef/PathKit.git",
         "state": {
           "branch": null,
           "revision": "73f8e9dca9b7a3078cb79128217dc8f2e585a511",
@@ -37,13 +37,22 @@
           "version": "0.0.5"
         }
       },
+      {
+        "package": "SwiftDocCPlugin",
+        "repositoryURL": "https://github.com/apple/swift-docc-plugin",
+        "state": {
+          "branch": null,
+          "revision": "3303b164430d9a7055ba484c8ead67a52f7b74f6",
+          "version": "1.0.0"
+        }
+      },
       {
         "package": "XcodeProj",
         "repositoryURL": "https://github.com/tuist/XcodeProj.git",
         "state": {
           "branch": null,
-          "revision": "0b18c3e7a10c241323397a80cb445051f4494971",
-          "version": "8.0.0"
+          "revision": "fae27b48bc14ff3fd9b02902e48c4665ce5a0793",
+          "version": "8.9.0"
         }
       },
       {
@@ -51,8 +60,8 @@
         "repositoryURL": "https://github.com/jpsim/Yams.git",
         "state": {
           "branch": null,
-          "revision": "53741ba55ecca5c7149d8c9f810913ec80845c69",
-          "version": "3.0.0"
+          "revision": "00c403debcd0a007b854bb35e598466207a2d58c",
+          "version": "5.0.0"
         }
       },
       {
@@ -60,8 +69,8 @@
         "repositoryURL": "https://github.com/marmelroy/Zip.git",
         "state": {
           "branch": null,
-          "revision": "80b1c3005ee25b4c7ce46c4029ac3347e8d5e37e",
-          "version": "2.0.0"
+          "revision": "67fa55813b9e7b3b9acee9c0ae501def28746d76",
+          "version": "2.1.2"
         }
       }
     ]

Package.swift

@@ -1,4 +1,5 @@
-// swift-tools-version:5.1
+// swift-tools-version:5.3
+// swiftlint:disable:previous file_header
 // The swift-tools-version declares the minimum version of Swift required to build this package
 
 import PackageDescription
@@ -12,10 +13,11 @@ let package = Package(
         .executable(name: "xcprebuild", targets: ["xcprebuild"]),
     ],
     dependencies: [
-        .package(url: "https://github.com/marmelroy/Zip.git", from: "2.0.0"),
-        .package(url: "https://github.com/jpsim/Yams.git", from: "3.0.0"),
+        .package(url: "https://github.com/marmelroy/Zip.git", from: "2.1.2"),
+        .package(url: "https://github.com/jpsim/Yams.git", from: "5.0.0"),
         .package(url: "https://github.com/apple/swift-argument-parser", from: "0.0.1"),
-        .package(url: "https://github.com/tuist/XcodeProj.git", from: "8.0.0"),
+        .package(url: "https://github.com/tuist/XcodeProj.git", from: "8.9.0"),
+        .package(url: "https://github.com/apple/swift-docc-plugin", from: "1.0.0"),
     ],
     targets: [
         .target(
@@ -30,8 +32,20 @@ let package = Package(
             name: "xcswiftc",
             dependencies: ["XCRemoteCache"]
         ),
+        .target(
+            name: "xcswift-frontend",
+            dependencies: ["XCRemoteCache"]
+        ),
+        .target(
+            name: "xclibtoolSupport",
+            dependencies: ["XCRemoteCache"]
+        ),
         .target(
             name: "xclibtool",
+            dependencies: ["XCRemoteCache", "xclibtoolSupport"]
+        ),
+        .target(
+            name: "xclipo",
             dependencies: ["XCRemoteCache"]
         ),
         .target(
@@ -49,14 +63,33 @@ let package = Package(
             name: "xcld",
             dependencies: ["XCRemoteCache"]
         ),
+        .target(
+            name: "xcldplusplus",
+            dependencies: ["XCRemoteCache"]
+        ),
         .target(
             // Wrapper target that builds all binaries but does nothing in runtime
             name: "Aggregator",
-            dependencies: ["xcprebuild", "xcswiftc", "xclibtool", "xcpostbuild", "xcprepare", "xcld"]
+            dependencies: [
+                "xcprebuild",
+                "xcswiftc",
+                "xcswift-frontend",
+                "xclibtool",
+                "xcpostbuild",
+                "xcprepare",
+                "xcld",
+                "xcldplusplus",
+                "xclipo",
+            ]
         ),
         .testTarget(
             name: "XCRemoteCacheTests",
-            dependencies: ["XCRemoteCache"]
+            dependencies: ["XCRemoteCache"],
+            resources: [.copy("TestData")]
+        ),
+        .testTarget(
+            name: "xclibtoolSupportTests",
+            dependencies: ["xclibtoolSupport"]
         ),
     ]
 )

README.md

@@ -8,6 +8,7 @@ _XCRemoteCache is a remote cache tool for Xcode projects. It reuses target artif
 [![Build Status](https://github.com/spotify/XCRemoteCache/workflows/CI/badge.svg)](https://github.com/spotify/XCRemoteCache/workflows/CI/badge.svg)
 [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](LICENSE)
 [![Slack](https://slackin.spotify.com/badge.svg)](https://slackin.spotify.com)
+[![Docs](https://github.com/spotify/XCRemoteCache/workflows/Docs/badge.svg)](https://spotify.github.io/XCRemoteCache/documentation/xcremotecache/)
 
 - [How and Why?](#how-and-why)
   * [Accurate target input files](#accurate-target-input-files)
@@ -44,6 +45,7 @@ _XCRemoteCache is a remote cache tool for Xcode projects. It reuses target artif
 - [Limitations](#limitations)
 - [FAQ](#faq)
 - [Development](#development)
+- [Architectural Designs](#architectural-designs)
 - [Release](#release)
   * [Releasing CocoaPods plugin](#releasing-cocoapods-plugin)
   * [Building release package](#building-release-package)
@@ -151,6 +153,7 @@ xcremotecache/xcprepare integrate --input <yourProject.xcodeproj> --mode consume
 | `--lldb-init` | LLDBInit mode. Appends to .lldbinit a command required for debugging. Supported values: 'none' (do not append to .lldbinit), 'user' (append to ~/.lldbinit) | `user` | ⬜️ |
 | `--fake-src-root` | An arbitrary source location shared between producers and consumers. Should be unique for a project. | `/xxxxxxxxxx` | ⬜️ |
 | `--output` | Save the project with integrated XCRemoteCache to a separate location.  | N/A | ⬜️ |
+| `--sdks-exclude` | comma separated list of sdks to not integrate XCRemoteCache (e.g. "watchos*, watchsimulator*"). (Experimental)  | `""` | ⬜️ |
 
 </details>
 
@@ -194,8 +197,11 @@ Configure Xcode targets that **should use** XCRemoteCache:
 * `CC` - `xccc_file` from your `.rcinfo` configuration (e.g. `xcremotecache/xccc`)
 * `SWIFT_EXEC` - location of `xcprepare` (e.g. `xcremotecache/xcswiftc`)
 * `LIBTOOL` - location of `xclibtool` (e.g. `xcremotecache/xclibtool`)
+* `LIPO` - location of `xclipo` (e.g. `xcremotecache/xclipo`)
 * `LD` - location of `xcld` (e.g. `xcremotecache/xcld`)
+* `LDPLUSPLUS` - location of `xcldplusplus` (e.g. `xcremotecache/xcldplusplus`)
 * `XCRC_PLATFORM_PREFERRED_ARCH` - `$(LINK_FILE_LIST_$(CURRENT_VARIANT)_$(PLATFORM_PREFERRED_ARCH):dir:standardizepath:file:default=arm64)`
+* `SWIFT_USE_INTEGRATED_DRIVER` - `NO` (required in Xcode 14.0+)
 
 <details>
   <summary>Screenshot</summary>
@@ -264,7 +270,42 @@ $ xcremotecache/xcprepare mark --configuration Debug --platform iphonesimulator
 
 That command creates an empty file on a remote server which informs that for given sha, configuration, platform, Xcode versions etc. all artifacts are available.
 
-_Note that for the `producer` mode, the prebuild build phase and `xccc`, `xcld`, `xclibtool` wrappers become no-op, so it is recommended to not add them for the `producer` mode._
+_Note that for the `producer` mode, the prebuild build phase and `xccc`, `xcld`, `xcldplusplus`, `xclibtool`, `xclipo` wrappers become no-op, so it is recommended to not add them for the `producer` mode._
+
+##### 7. Generalize `-Swift.h` (Optional only if using static library with a bridging header with public `NS_ENUM` exposed from ObjC)
+
+If a static library target contains a mixed target with a bridging header exposing an enum from ObjC in a public Swift API, your custom script that moves `*-Swift.h` to the shared location, it should also move `*-Swift.h.md5` next to it.
+
+Example:
+
+##### Existing script (Before):
+
+```shell
+ditto "${SCRIPT_INPUT_FILE_0}" "${SCRIPT_OUTPUT_FILE_0}"
+```
+
+where
+* `SCRIPT_INPUT_FILE_0="$(DERIVED_SOURCES_DIR)/$(SWIFT_OBJC_INTERFACE_HEADER_NAME)"`
+* `SCRIPT_OUTPUT_FILE_0="$(BUILT_PRODUCTS_DIR)/include/$(PRODUCT_MODULE_NAME)/$(SWIFT_OBJC_INTERFACE_HEADER_NAME)"`
+
+##### Correct script (After):
+
+```shell
+ditto "${SCRIPT_INPUT_FILE_0}" "${SCRIPT_OUTPUT_FILE_0}"
+[ -f "${SCRIPT_INPUT_FILE_1}" ] && ditto "${SCRIPT_INPUT_FILE_1}" "${SCRIPT_OUTPUT_FILE_1}" || rm -f "${SCRIPT_OUTPUT_FILE_1}"
+```
+
+where
+* `SCRIPT_INPUT_FILE_0="$(DERIVED_SOURCES_DIR)/$(SWIFT_OBJC_INTERFACE_HEADER_NAME)"`
+* `SCRIPT_INPUT_FILE_1="$(DERIVED_SOURCES_DIR)/$(SWIFT_OBJC_INTERFACE_HEADER_NAME).md5"`
+* `SCRIPT_OUTPUT_FILE_0="$(BUILT_PRODUCTS_DIR)/include/$(PRODUCT_MODULE_NAME)/$(SWIFT_OBJC_INTERFACE_HEADER_NAME)"`
+* `SCRIPT_OUTPUT_FILE_1="$(BUILT_PRODUCTS_DIR)/include/$(PRODUCT_MODULE_NAME)/$(SWIFT_OBJC_INTERFACE_HEADER_NAME).md5"`
+
+Note: This step is not required if at least one of these is true:
+
+* you build a framework (not a static library)
+* you don't expose `NS_ENUM` type from ObjC to Swift via a bridging header
+
 
 ## A full list of configuration parameters:
 
@@ -288,13 +329,15 @@ _Note that for the `producer` mode, the prebuild build phase and `xccc`, `xcld`,
 | `cache_commit_history` | Number of historical git commits to look for cache artifacts | `10` | ⬜️ |
 | `source_root` | Source root of the Xcode project | `""` | ⬜️ |
 | `fingerprint_override_extension` | Fingerprint override extension (sample override `Module.swiftmodule/x86_64.swiftmodule.md5`) | `md5` | ⬜️ |
-| `extra_configuration_file` | Configuration file that overrides project configuration | `user.rcinfo` | ⬜️ |
+| `extra_configuration_file` | Configuration file that overrides project configuration (this property can be overriden multiple times in different files to chain extra configuration files) | `user.rcinfo` | ⬜️ |
 | `publishing_sha` | Custom commit sha to publish artifact (producer only) | `nil` | ⬜️ |
 | `artifact_maximum_age` | Maximum age in days HTTP response should be locally cached before being evicted | `30` | ⬜️ |
 | `custom_fingerprint_envs` | Extra ENV keys that should be convoluted into the environment fingerprint | `[]` | ⬜️ |
 | `stats_dir` | Directory where all XCRemoteCache statistics (e.g. counters) are stored | `~/.xccache` | ⬜️ |
 | `download_retries` | Number of retries for download requests | `0` | ⬜️ |
 | `upload_retries` | Number of retries for upload requests | `3` | ⬜️ |
+| `retry_delay` | Delay between retries in seconds | `10` | ⬜️ |
+| `upload_batch_size` | Maximum number of simultaneous requests. 0 means no limits | `0` | ⬜️ |
 | `request_custom_headers` | Dictionary of extra HTTP headers for all remote server requests | `[]` | ⬜️ |
 | `thin_target_mock_filename` | Filename (without an extension) of the compilation input file that is used as a fake compilation for the forced-cached target (aka thin target) | `standin` | ⬜️ |
 | `focused_targets` | A list of all targets that are not thinned. If empty, all targets are meant to be non-thin | `[]` | ⬜️ |
@@ -308,10 +351,16 @@ _Note that for the `producer` mode, the prebuild build phase and `xccc`, `xcld`,
 | `prettify_meta_files` | A Boolean value that opts-in pretty JSON formatting for meta files | `false` | ⬜️ |
 | `aws_secret_key` | Secret key for AWS V4 Signature Authorization. If this is set to a non-empty String - an Authentication Header will be added based on this and the other `aws_*` parameters.| `""` | ⬜️ |
 | `aws_access_key` | Access key for AWS V4 Signature Authorization. | `""` | ⬜️ |
+| `aws_security_token` | Temporary security token provided by the AWS Security Token Service. | `nil` | ⬜️ |
 | `aws_region` | Region for AWS V4 Signature Authorization. E.g. `eu`.  | `""` | ⬜️ |
 | `aws_service` | Service for AWS V4 Signature Authorization. E.g. `storage`. | `""` | ⬜️ |
 | `out_of_band_mappings` | A dictionary of files path remapping that should be applied to make it absolute path agnostic on a list of dependencies. Useful if a project refers files out of repo root, either compilation files or precompiled dependencies. Keys represent generic replacement and values are substrings that should be replaced. Example: for mapping `["COOL_LIBRARY": "/CoolLibrary"]` `/CoolLibrary/main.swift`will be represented as `$(COOL_LIBRARY)/main.swift`). Warning: remapping order is not-deterministic so avoid remappings with multiple matchings. | `[:]` | ⬜️ |
 | `disable_certificate_verification` | A Boolean value that opts-in SSL certificate validation is disabled | `false` | ⬜️ |
+| `disable_vfs_overlay` | A feature flag to disable virtual file system overlay support (temporary) | `false` | ⬜️ |
+| `custom_rewrite_envs` | A list of extra ENVs that should be used as placeholders in the dependency list. ENV rewrite process is optimistic - does nothing if an ENV is not defined in the pre/postbuild process. | `[]` | ⬜️ |
+| `irrelevant_dependencies_paths` | Regexes of files that should not be included in a list of dependencies. Warning! Add entries here with caution - excluding dependencies that are relevant might lead to a target overcaching. The regex can match either partially or fully the filepath, e.g. `\\.modulemap$` will exclude all `.modulemap` files. | `[]` | ⬜️ |
+| `gracefully_handle_missing_common_sha` | If true, do not fail `prepare` if cannot find the most recent common commits with the primary branch. That might be useful on CI, where a shallow clone is used and cloning depth is not big enough to fetch a commit from a primary branch | `false` | ⬜️ |
+| `enable_swift_driver_integration` | Enable experimental integration with swift driver, added in Xcode 14 | `false` | ⬜️ |
 
 ## Backend cache server
 
@@ -349,6 +398,8 @@ XCRemoteCache supports Amazon S3 and Google Cloud Storage buckets to be used as
 
 To set it up use the configuration parameters `aws_secret_key`, `aws_access_key`, `aws_region`, and `aws_service` in the `.rcinfo` file. Specify the URL to the bucket in cache-addresses field in the same file.
 
+XCRemoteCache also supports [AWS Temporary Access Keys](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#temporary-access-keys). Use additional `aws_security_token` parameter combined with `aws_secret_key`, `aws_access_key` to set it up. [This page](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/configuring-instance-metadata-service.html) describes how to receive a security token. 
+
 Example
 ```yaml
 ...
@@ -402,12 +453,14 @@ Note: This setup is not recommended and may not be supported in future XCRemoteC
 * Recommended: multi-targets Xcode project
 * Recommended: do not use fast-forward PR strategy (use merge or squash instead)
 * Recommended: avoid `DWARF with dSYM File` "Debug Information Format" build setting. Use `DWARF` instead
+* Recommended: avoid having a symbolic link in the source root (e.g. placing a project in `/tmp`) 
 
 ## Limitations
 
 * Swift Package Manager (SPM) dependencies are not supported. _Because SPM does not allow customizing Build Settings, XCRemoteCache cannot specify `clang` and `swiftc` wrappers that control if the local compilation should be skipped (cache hit) or not (cache miss)_
 * Filenames with `_vers.c` suffix are reserved and cannot be used as a source file
 * All compilation files should be referenced via the git repo root. Referencing `/AbsolutePath/someOther.swift` or `../../someOther.swift` that resolve to the location outside of the git repo root is prohibited.
+* The new Swift driver (introduced by default in Xcode 14.0) is not supported and has to be disabled when using XCRemoteCache
 
 ## FAQ
 
@@ -417,6 +470,10 @@ Follow the [FAQ](docs/FAQ.md) page.
 
 Follow the [Development](docs/Development.md) guide. It has all the information on how to get started.
 
+## Architectural designs
+
+Follow the [Architectural designs](docs/design/ArchitecturalDesigns.md) document that describes and documents XCRemoteCache designs and implementation details.
+
 ## Release 
 
 To release a version, in [Releases](https://github.com/spotify/XCRemoteCache/releases) draft a new release with `v0.3.0{-rc0}` tag format. 

Rakefile

@@ -1,4 +1,5 @@
 # encoding: utf-8
+require_relative 'tasks/e2e'
 
 ################################
 # Rake configuration
@@ -9,7 +10,7 @@ DERIVED_DATA_DIR = File.join('.build').freeze
 RELEASES_ROOT_DIR = File.join('releases').freeze
 
 EXECUTABLE_NAME = 'XCRemoteCache'
-EXECUTABLE_NAMES = ['xclibtool', 'xcpostbuild', 'xcprebuild', 'xcprepare', 'xcswiftc', 'xcld'] 
+EXECUTABLE_NAMES = ['xclibtool', 'xcpostbuild', 'xcprebuild', 'xcprepare', 'xcswiftc', 'swiftc', 'xcswift-frontend', 'swift-frontend', 'xcld', 'xcldplusplus', 'xclipo']
 PROJECT_NAME = 'XCRemoteCache'
 
 SWIFTLINT_ENABLED = true
@@ -58,6 +59,10 @@ task :build, [:configuration, :arch, :sdks, :is_archive] do |task, args|
 
     # Path of the executable looks like: `.build/(debug|release)/XCRemoteCache`
     build_path_base = File.join(DERIVED_DATA_DIR, args.configuration)
+    # swift-frontent integration requires that the SWIFT_EXEC is `swiftc` so create
+    # a symbolic link between swiftc->xcswiftc and swift-frontend->xcswift-frontend
+    system("cd #{build_path_base} && ln -s xcswiftc swiftc")
+    system("cd #{build_path_base} && ln -s xcswift-frontend swift-frontend")
     sdk_build_paths = EXECUTABLE_NAMES.map {|e| File.join(build_path_base, e)}
 
     build_paths.push(sdk_build_paths)
@@ -71,12 +76,34 @@ task :build, [:configuration, :arch, :sdks, :is_archive] do |task, args|
   end
 end
 
+desc 'Build release artifacts'
+task :prepare_release do
+  system("rm -rf releases && rm -rf tmp")
+  Rake::Task['build'].invoke("release", "x86_64-apple-macosx")
+  system("mkdir -p tmp && unzip releases/XCRemoteCache.zip -d tmp/xcremotecache-x86_64")
+  system("rm -rf releases")
+  Rake::Task['build'].invoke("release", "arm64-apple-macosx")
+  system("rake 'build[release, arm64-apple-macosx]'")
+  system("mkdir -p tmp && unzip releases/XCRemoteCache.zip -d tmp/xcremotecache-arm64")
+  system("rm -rf releases")
+  system("mkdir -p releases && zip -jr releases/XCRemoteCache-macOS-x86_64.zip LICENSE README.md tmp/xcremotecache-x86_64")
+  system("zip -jr releases/XCRemoteCache-macOS-arm64.zip LICENSE README.md tmp/xcremotecache-arm64")
+  system("mkdir -p tmp/xcremotecache && ls tmp/xcremotecache-x86_64 | xargs -I {} lipo -create -output tmp/xcremotecache/{} tmp/xcremotecache-x86_64/{} tmp/xcremotecache-arm64/{}")
+  system("zip -jr releases/XCRemoteCache-macOS-arm64-x86_64.zip LICENSE README.md tmp/xcremotecache")
+end
+
 desc 'run tests with SPM'
 task :test do
   # Running tests
   spm_test()
 end
 
+desc 'build and run E2E tests'
+task :e2e => [:build, :e2e_only]
+
+desc 'run E2E tests without building the XCRemoteCache binary'
+task :e2e_only => ['e2e:run']
+
 ################################
 # Helper functions
 ################################
@@ -107,7 +134,9 @@ def create_release_zip(build_paths)
   # Create and move files into the release directory
   mkdir_p release_dir
   build_paths.each {|p|
-    cp_r p, release_dir
+    # -r for recursive
+    # -P for copying symbolic link as is
+    system("cp -rP #{p} #{release_dir}")
   }
   
   output_artifact_basename = "#{PROJECT_NAME}.zip"
@@ -116,7 +145,8 @@ def create_release_zip(build_paths)
     # -X: no extras (uid, gid, file times, ...)
     # -x: exclude .DS_Store
     # -r: recursive
-    system("zip -X -x '*.DS_Store' -r #{output_artifact_basename} .") or abort "zip failure"
+    # -y: to store symbolic links (used for swiftc -> xcswiftc)
+    system("zip -X -x '*.DS_Store' -r -y #{output_artifact_basename} .") or abort "zip failure"
     # List contents of zip file
     system("unzip -l #{output_artifact_basename}") or abort "unzip failure"
   end

Sources/XCRemoteCache/Artifacts/ArtifactCreator.swift

@@ -42,6 +42,7 @@ class BuildArtifactCreator: ArtifactSwiftProductsBuilderImpl, ArtifactCreator {
     private let modulesFolderPath: String
     private let dSYMPath: URL
     private let metaWriter: MetaWriter
+    private let artifactProcessor: ArtifactProcessor
     private let fileManager: FileManager
 
     init(
@@ -52,6 +53,7 @@ class BuildArtifactCreator: ArtifactSwiftProductsBuilderImpl, ArtifactCreator {
         modulesFolderPath: String,
         dSYMPath: URL,
         metaWriter: MetaWriter,
+        artifactProcessor: ArtifactProcessor,
         fileManager: FileManager
     ) {
         self.buildDir = buildDir
@@ -62,6 +64,7 @@ class BuildArtifactCreator: ArtifactSwiftProductsBuilderImpl, ArtifactCreator {
         self.fileManager = fileManager
         self.dSYMPath = dSYMPath
         self.metaWriter = metaWriter
+        self.artifactProcessor = artifactProcessor
         super.init(workingDir: tempDir, moduleName: moduleName, fileManager: fileManager)
     }
 
@@ -87,6 +90,7 @@ class BuildArtifactCreator: ArtifactSwiftProductsBuilderImpl, ArtifactCreator {
     /// - Parameter tempDir: Temp location to organize file hierarchy in the artifact
     /// - returns: URLs to include into the artifact package
     fileprivate func prepareSwiftArtifacts(tempDir: URL) throws -> [URL] {
+        try artifactProcessor.process(localArtifact: tempDir)
         var artifacts: [URL] = []
 
         // Add optional directory with generated ObjC headers

Sources/XCRemoteCache/Artifacts/ArtifactMetaUpdater.swift

@@ -0,0 +1,72 @@
+// Copyright (c) 2023 Spotify AB.
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import Foundation
+
+enum ArtifactMetaUpdaterError: Error {
+    /// The prebuild plugin execution was called but the local
+    /// path to the artifact directory is still unknown
+    /// Might happen that the artifact processor didn't invoke the updater's
+    /// .process() after downloading/activating an artifact
+    case artifactLocationIsUnknown
+}
+
+/// Updates the meta file in an unzipped artifact directory, by placing an up-to-date
+/// and remapped meta file. Updating the meta in the artifact allows reusing existing
+/// artifacts it a new meta.json schema has been released to the meta format, while
+/// artifacts are still backward-compatible
+class ArtifactMetaUpdater: ArtifactProcessor {
+    private var artifactLocation: URL?
+    private let metaWriter: MetaWriter
+    private let fileRemapper: FileDependenciesRemapper
+
+    init(
+        fileRemapper: FileDependenciesRemapper,
+        metaWriter: MetaWriter
+    ) {
+        self.metaWriter = metaWriter
+        self.fileRemapper = fileRemapper
+    }
+
+    /// Remembers the artifact location, used later in the plugin
+    /// - Parameter url: artifact's root directory
+    func process(rawArtifact url: URL) throws {
+        // Storing the location of the just downloaded/activated artifact
+        // Note, the `url` location already includes a meta (generated by producer
+        // while compiling and building an artifact)
+        artifactLocation = url
+    }
+
+    func process(localArtifact url: URL) throws {
+        // No need to do anything in the postbuild
+    }
+}
+
+extension ArtifactMetaUpdater: ArtifactConsumerPrebuildPlugin {
+
+    /// Updates the meta json file in a local, unzipped, artifact location. It also remaps
+    /// all paths so other steps (like actool or postbuild) don't have to do it again
+    func run(meta: MainArtifactMeta) throws {
+        guard let artifactLocation = artifactLocation else {
+            throw ArtifactMetaUpdaterError.artifactLocationIsUnknown
+        }
+        let metaURL = try metaWriter.write(meta, locationDir: artifactLocation)
+        try fileRemapper.remap(fromGeneric: metaURL)
+    }
+}

Sources/XCRemoteCache/Artifacts/ArtifactOrganizer.swift

@@ -31,7 +31,7 @@ enum ArtifactOrganizerLocationPreparationResult: Equatable {
     case preparedForArtifact(artifact: URL)
 }
 
-/// Prepares .zip artifact for the local operations
+/// Prepares existing .zip artifact for the local operations
 protocol ArtifactOrganizer {
     /// Prepares the location for the artifact unzipping
     /// - Parameter fileKey: artifact fileKey that corresponds to the zip filename on the remote cache server
@@ -47,11 +47,16 @@ protocol ArtifactOrganizer {
 }
 
 class ZipArtifactOrganizer: ArtifactOrganizer {
+    static let activeArtifactLocation = "active"
+
     private let cacheDir: URL
+    // all processors that should "prepare" the unzipped raw artifact
+    private let artifactProcessors: [ArtifactProcessor]
     private let fileManager: FileManager
 
-    init(targetTempDir: URL, fileManager: FileManager) {
+    init(targetTempDir: URL, artifactProcessors: [ArtifactProcessor], fileManager: FileManager) {
         cacheDir = targetTempDir.appendingPathComponent("xccache")
+        self.artifactProcessors = artifactProcessors
         self.fileManager = fileManager
     }
 
@@ -60,7 +65,7 @@ class ZipArtifactOrganizer: ArtifactOrganizer {
     }
 
     func getActiveArtifactLocation() -> URL {
-        return cacheDir.appendingPathComponent("active")
+        return cacheDir.appendingPathComponent(Self.self.activeArtifactLocation)
     }
 
     func getActiveArtifactFilekey() throws -> String {
@@ -87,16 +92,27 @@ class ZipArtifactOrganizer: ArtifactOrganizer {
         let destinationURL = artifact.deletingPathExtension()
         guard fileManager.fileExists(atPath: destinationURL.path) == false else {
             infoLog("Skipping artifact, already existing at \(destinationURL)")
+            try runArtifactProcessors(artifactLocation: destinationURL)
             return destinationURL
         }
-        // Uzipping to a temp file first to never leave the uncompleted zip in the final location
+        // Unzipping to a temp file first to never leave the uncompleted zip in the final location
         // when the command was interrupted (internal crash or `kill -9` signal)
         let tempDestination = destinationURL.appendingPathExtension("tmp")
         try Zip.unzipFile(artifact, destination: tempDestination, overwrite: true, password: nil)
+
         try fileManager.moveItem(at: tempDestination, to: destinationURL)
+        try runArtifactProcessors(artifactLocation: destinationURL)
         return destinationURL
     }
 
+    /// Iterates all processor when an artifact has been just downloaded or reused from already downloaded
+    /// and previously processed location
+    private func runArtifactProcessors(artifactLocation: URL) throws {
+        try artifactProcessors.forEach { processor in
+            try processor.process(rawArtifact: artifactLocation)
+        }
+    }
+
     func activate(extractedArtifact: URL) throws {
         let activeLocationURL = getActiveArtifactLocation()
         try fileManager.spt_forceSymbolicLink(at: activeLocationURL, withDestinationURL: extractedArtifact)

Sources/XCRemoteCache/Artifacts/ArtifactProcessor.swift

@@ -0,0 +1,80 @@
+// Copyright (c) 2022 Spotify AB.
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import Foundation
+
+
+/// Performs a pre/postprocessing on an artifact package
+/// Could be a place for file reorganization (to support legacy package formats) and/or
+/// remapp absolute paths in some package files
+protocol ArtifactProcessor {
+    /// Processes a raw artifact in a directory. Raw artifact is a format of an artifact
+    /// that is stored in a remote cache server (generic)
+    /// - Parameter rawArtifact: directory that contains raw artifact content
+    func process(rawArtifact: URL) throws
+
+    /// Processes a local artifact in a directory
+    /// - Parameter localArtifact: directory that contains local (machine-specific) artifact content
+    func process(localArtifact: URL) throws
+}
+
+/// Processes downloaded artifact by replacing generic paths in generated ObjC headers placed in ./include
+class UnzippedArtifactProcessor: ArtifactProcessor {
+    /// All directories in an artifact that should be processed by path remapping
+    private static let remappingDirs = ["include"]
+    private let fileRemapper: FileDependenciesRemapper
+    private let dirScanner: DirScanner
+
+    init(fileRemapper: FileDependenciesRemapper, dirScanner: DirScanner) {
+        self.fileRemapper = fileRemapper
+        self.dirScanner = dirScanner
+    }
+
+    private func findProcessingEligableFiles(path: String) throws -> [URL] {
+        let remappingURL = URL(fileURLWithPath: path)
+        let allFiles = try dirScanner.recursiveItems(at: remappingURL)
+        return allFiles.filter({ !$0.isHidden })
+    }
+
+    /// Replaces all generic paths in a raw artifact's `include` dir with
+    /// absolute paths, specific for a given machine and configuration
+    /// - Parameter rawArtifact: raw artifact location
+    func process(rawArtifact url: URL) throws {
+        for remappingDir in Self.remappingDirs {
+            let remappingPath = url.appendingPathComponent(remappingDir).path
+            let allFiles = try findProcessingEligableFiles(path: remappingPath)
+            try allFiles.forEach(fileRemapper.remap(fromGeneric:))
+        }
+    }
+
+    func process(localArtifact url: URL) throws {
+        for remappingDir in Self.remappingDirs {
+            let remappingPath = url.appendingPathComponent(remappingDir).path
+            let allFiles = try findProcessingEligableFiles(path: remappingPath)
+            try allFiles.forEach(fileRemapper.remap(fromLocal:))
+        }
+    }
+}
+
+fileprivate extension URL {
+    // Recognize hidden files starting with a dot
+    var isHidden: Bool {
+        lastPathComponent.hasPrefix(".")
+    }
+}

Sources/XCRemoteCache/Artifacts/ArtifactSwiftProductsBuilder.swift

@@ -51,8 +51,6 @@ protocol ArtifactSwiftProductsBuilder {
 /// # {workingDir}/xccache/produced/include/#{moduleName} (if `moduleName` is defined)
 class ArtifactSwiftProductsBuilderImpl: ArtifactSwiftProductsBuilder {
 
-    /// List of all required swiftmodule related extensions that should be copied to the artifact
-    private static let swiftmoduleExtensionsToInclude = ["swiftmodule", "swiftdoc", "swiftsourceinfo"]
     private let workingDir: URL
     private let moduleName: String?
     private let fileManager: FileManager
@@ -90,7 +88,7 @@ class ArtifactSwiftProductsBuilderImpl: ArtifactSwiftProductsBuilder {
             throw ArtifactSwiftProductsBuilderError.populatingNonExistingObjCHeader
         }
         try fileManager.createDirectory(at: moduleObjCURL, withIntermediateDirectories: true, attributes: nil)
-        try fileManager.spt_forceLinkItem(at: headerURL, to: headerArtifactURL)
+        try fileManager.spt_forceCopyItem(at: headerURL, to: headerArtifactURL)
     }
 
     func includeModuleDefinitionsToTheArtifact(arch: String, moduleURL: URL) throws {

Sources/XCRemoteCache/Artifacts/FileDependenciesRemapper.swift

@@ -0,0 +1,81 @@
+// Copyright (c) 2022 Spotify AB.
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import Foundation
+
+
+enum FileDependenciesRemapperError: Error {
+    /// Thrown when the file to remap is invalid (e.g. doesn't exist or has unexpected format)
+    case invalidRemappingFile(URL)
+}
+
+/// Replaces paths in a file content between generic (placeholder-based)
+/// and local formats
+protocol FileDependenciesRemapper {
+    /// Replaces all generic paths (with placeholders) to a local, machine
+    /// specific absolute paths
+    /// - Parameter url: location of a file that should be remapped in-place
+    func remap(fromGeneric url: URL) throws
+    /// Replaces all local, machine specific absolute paths to
+    /// generic ones
+    /// - Parameter url: location of a file that should be remapped in-place
+    func remap(fromLocal url: URL) throws
+}
+
+/// Remaps absolute paths in a text files stored on a disk
+/// Note: That class can be used only for text-based files, not binaries
+class TextFileDependenciesRemapper: FileDependenciesRemapper {
+    private static let linesSeparator = "\n"
+    private let remapper: DependenciesRemapper
+    private let fileAccessor: FileAccessor
+
+    init(remapper: DependenciesRemapper, fileAccessor: FileAccessor) {
+        self.remapper = remapper
+        self.fileAccessor = fileAccessor
+    }
+
+    private func readFileLines(_ url: URL) throws -> [String] {
+        guard let content = try fileAccessor.contents(atPath: url.path) else {
+            // the file is empty
+            return []
+        }
+        guard let contentString = String(data: content, encoding: .utf8) else {
+            throw FileDependenciesRemapperError.invalidRemappingFile(url)
+        }
+        return contentString.components(separatedBy: .newlines)
+    }
+
+    private func storeFileLines(lines: [String], url: URL) throws {
+        let contentString = lines.joined(separator: "\n")
+        let contentData = contentString.data(using: String.Encoding.utf8)
+        try fileAccessor.write(toPath: url.path, contents: contentData)
+    }
+
+    func remap(fromGeneric url: URL) throws {
+        let contentLines = try readFileLines(url)
+        let remappedContent = try remapper.replace(genericPaths: contentLines)
+        try storeFileLines(lines: remappedContent, url: url)
+    }
+
+    func remap(fromLocal url: URL) throws {
+        let contentLines = try readFileLines(url)
+        let remappedContent = try remapper.replace(localPaths: contentLines)
+        try storeFileLines(lines: remappedContent, url: url)
+    }
+}

Sources/XCRemoteCache/Artifacts/SwiftmoduleFileExtension.swift

@@ -30,6 +30,9 @@ enum SwiftmoduleFileExtension: String {
     case swiftmodule
     case swiftdoc
     case swiftsourceinfo
+    case swiftinterface
+    case privateSwiftinterface = "private.swiftinterface"
+    case abiJson = "abi.json"
 }
 
 extension SwiftmoduleFileExtension {
@@ -38,5 +41,8 @@ extension SwiftmoduleFileExtension {
         .swiftmodule: .required,
         .swiftdoc: .required,
         .swiftsourceinfo: .optional,
+        .swiftinterface: .optional,
+        .privateSwiftinterface: .optional,
+        .abiJson: .optional,
     ]
 }

Sources/XCRemoteCache/Commands/Libtool/FallbackXCLibtoolLogic.swift

@@ -0,0 +1,37 @@
+// Copyright (c) 2023 Spotify AB.
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import Foundation
+
+class FallbackXCLibtoolLogic: XCLibtoolLogic {
+    private let fallbackCommand: String
+
+    init(fallbackCommand: String) {
+        self.fallbackCommand = fallbackCommand
+    }
+
+    func run() {
+        let args = ProcessInfo().arguments
+        let paramList = [fallbackCommand] + args.dropFirst()
+        let cargs = paramList.map { strdup($0) } + [nil]
+        execvp(fallbackCommand, cargs)
+
+        exit(1)
+    }
+}

Sources/XCRemoteCache/Commands/Libtool/XCCreateUniversalBinary.swift

@@ -19,31 +19,42 @@
 
 import Foundation
 
-enum XCLibtoolCreateUniversalBinaryError: Error {
+enum XCCreateUniversalBinaryError: Error {
     ///  Missing ar libraries that should constitute an universal build
     case missingInputLibrary
 }
 
-/// Wrapper for `libtool` call for creating an universal binary
-class XCLibtoolCreateUniversalBinary: XCLibtoolLogic {
+/// Wrapper for `libtool`/`lipo` call for creating an universal binary
+class XCCreateUniversalBinary: XCLibtoolLogic {
     private let output: URL
     private let tempDir: URL
     private let firstInputURL: URL
+    private let toolName: String
+    private let fallbackCommand: String
 
-    init(output: String, inputs: [String]) throws {
+    init(
+        output: String,
+        inputs: [String],
+        toolName: String,
+        fallbackCommand: String
+    ) throws {
         self.output = URL(fileURLWithPath: output)
         guard let firstInput = inputs.first else {
-            throw XCLibtoolCreateUniversalBinaryError.missingInputLibrary
+            throw XCCreateUniversalBinaryError.missingInputLibrary
         }
         let firstInputURL = URL(fileURLWithPath: firstInput)
         // inputs are place in $TARGET_TEMP_DIR/Objects-normal/$ARCH/Binary/$TARGET_NAME.a
         // TODO: find better (stable) technique to determine `$TARGET_TEMP_DIR`
+        errorLog("\(firstInputURL.absoluteString)")
+
         tempDir = firstInputURL
             .deletingLastPathComponent()
             .deletingLastPathComponent()
             .deletingLastPathComponent()
             .deletingLastPathComponent()
         self.firstInputURL = firstInputURL
+        self.toolName = toolName
+        self.fallbackCommand = fallbackCommand
     }
 
     func run() {
@@ -52,10 +63,10 @@ class XCLibtoolCreateUniversalBinary: XCLibtoolLogic {
         let config: XCRemoteCacheConfig
         do {
             let srcRoot: URL = URL(fileURLWithPath: fileManager.currentDirectoryPath)
-            config = try XCRemoteCacheConfigReader(srcRootPath: srcRoot.path, fileManager: fileManager)
+            config = try XCRemoteCacheConfigReader(srcRootPath: srcRoot.path, fileReader: fileManager)
                 .readConfiguration()
         } catch {
-            errorLog("Libtool initialization failed with error: \(error). Fallbacking to libtool")
+            errorLog("\(toolName) initialization failed with error: \(error). Fallbacking to \(fallbackCommand)")
             fallbackToDefault()
         }
 
@@ -74,22 +85,21 @@ class XCLibtoolCreateUniversalBinary: XCLibtoolLogic {
             // that these are already an universal binary
             try fileManager.spt_forceLinkItem(at: firstInputURL, to: output)
         } catch {
-            errorLog("Libtool failed with error: \(error). Fallbacking to libtool")
+            errorLog("\(toolName) failed with error: \(error). Fallbacking to \(fallbackCommand)")
             do {
                 try fileManager.removeItem(at: markerURL)
                 fallbackToDefault()
             } catch {
-                exit(1, "FATAL: libtool failed with error: \(error)")
+                exit(1, "FATAL: \(fallbackCommand) failed with error: \(error)")
             }
         }
     }
 
     private func fallbackToDefault() -> Never {
         let args = ProcessInfo().arguments
-        let command = "libtool"
-        let paramList = [command] + args.dropFirst()
+        let paramList = [fallbackCommand] + args.dropFirst()
         let cargs = paramList.map { strdup($0) } + [nil]
-        execvp(command, cargs)
+        execvp(fallbackCommand, cargs)
 
         /// C-function `execv` returns only when the command fails
         exit(1)

Sources/XCRemoteCache/Commands/Libtool/XCLibtool.swift

@@ -20,11 +20,13 @@
 import Foundation
 
 /// Represents a mode that libtool was called
-public enum XCLibtoolMode {
+public enum XCLibtoolMode: Equatable {
     /// Creating a static library (ar format) from a set of .o input files
     case createLibrary(output: String, filelist: String, dependencyInfo: String)
     /// Creating a universal library (multiple-architectures) from a set of input .a static libraries
     case createUniversalBinary(output: String, inputs: [String])
+    /// print the toolchain version
+    case version
 }
 
 public class XCLibtool {
@@ -44,7 +46,14 @@ public class XCLibtool {
                 stepDescription: "Libtool"
             )
         case .createUniversalBinary(let output, let inputs):
-            logic = try XCLibtoolCreateUniversalBinary(output: output, inputs: inputs)
+            logic = try XCCreateUniversalBinary(
+                output: output,
+                inputs: inputs,
+                toolName: "Libtool",
+                fallbackCommand: "libtool"
+            )
+        case .version:
+            logic = FallbackXCLibtoolLogic(fallbackCommand: "libtool")
         }
     }
 

Sources/XCRemoteCache/Commands/Libtool/XCLipo.swift

@@ -0,0 +1,50 @@
+// Copyright (c) 2023 Spotify AB.
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import Foundation
+
+/// Wrapper for a `lipo` tool that creates a fat archive
+public class XCLipo {
+    private let logic: XCLibtoolLogic
+
+    public init(
+        output: String,
+        inputs: [String],
+        fallbackCommand: String,
+        stepDescription: String
+    ) throws {
+        errorLog("\(output)")
+        errorLog("\(inputs.joined(separator: ","))")
+        logic = try XCCreateUniversalBinary(
+            output: output,
+            inputs: inputs,
+            toolName: stepDescription,
+            fallbackCommand: fallbackCommand
+        )
+    }
+
+    /// Handles a `-create` action which is responsible to create a fat archive
+    /// If remote cache can reuse artifacts from a remote cache, it just links any of input
+    /// files to the destination (output) location because the binary in XCRC artifact already
+    /// contains a fat library
+    /// If a remote artifact cannot be reused, a fallback to the `lipo` command is performed
+    public func run() {
+        logic.run()
+    }
+}

Sources/XCRemoteCache/Commands/Plugins/Thinning/Factories/ThinningConsumerArtifactsOrganizerFactory.swift

@@ -27,13 +27,19 @@ protocol ThinningConsumerArtifactsOrganizerFactory {
 }
 
 class ThinningConsumerZipArtifactsOrganizerFactory: ThinningConsumerArtifactsOrganizerFactory {
+    private let processors: [ArtifactProcessor]
     private let fileManager: FileManager
 
-    init(fileManager: FileManager) {
+    init(processors: [ArtifactProcessor], fileManager: FileManager) {
+        self.processors = processors
         self.fileManager = fileManager
     }
 
     func build(targetTempDir: URL) -> ArtifactOrganizer {
-        ZipArtifactOrganizer(targetTempDir: targetTempDir, fileManager: fileManager)
+        ZipArtifactOrganizer(
+            targetTempDir: targetTempDir,
+            artifactProcessors: processors,
+            fileManager: fileManager
+        )
     }
 }

Sources/XCRemoteCache/Commands/Plugins/Thinning/ThinningCreatorPlugin.swift

@@ -38,7 +38,7 @@ class ThinningCreatorPlugin: ArtifactCreatorPlugin {
 
     /// Default Initializer
     /// - Parameter targetTempDir: Location of current target-specific temp dir (TARGET_TEMP_DIR)
-    /// - Parameter modeMarkerPath: path of maker file that informs if a given target can reuse remote artifacts.
+    /// - Parameter modeMarkerPath: path of maker file that informs if a given target can reuse remote artifacts
     /// - Parameter dirScanner: scanner to access disk and read files and directories hierarchy
     init(targetTempDir: URL, modeMarkerPath: String, dirScanner: DirScanner) {
         self.targetTempDir = targetTempDir
@@ -94,7 +94,7 @@ class ThinningCreatorPlugin: ArtifactCreatorPlugin {
         // ProducerFast mode:
         // If a target reused already existing artifact, it still has `$(TARGET_TEMP_DIR)/rc.enabled` marker file
         // and the reused zip is placed in:
-        // `$(TARGET_TEMP_DIR)/xccache/{{fileKey}}.zip` location.
+        // `$(TARGET_TEMP_DIR)/xccache/{{fileKey}}.zip` location
 
         let targetEnabledMarker = tempDir.appendingPathComponent(modeMarkerPath)
         let targetReusedArtifactRootDir = tempDir.appendingPathComponent("xccache")

Sources/XCRemoteCache/Commands/Plugins/Thinning/ThinningDiskSwiftcProductsGenerator.swift

@@ -58,7 +58,7 @@ class ThinningDiskSwiftcProductsGenerator: SwiftcProductsGenerator {
     func generateFrom(
         artifactSwiftModuleFiles sourceAtifactSwiftModuleFiles: [SwiftmoduleFileExtension: URL],
         artifactSwiftModuleObjCFile: URL
-    ) throws -> URL {
+    ) throws -> SwiftcProductsGeneratorOutput {
         // Move cached -Swift.h file to the expected location
         try diskCopier.copy(file: artifactSwiftModuleObjCFile, destination: objcHeaderOutput)
         for (ext, url) in sourceAtifactSwiftModuleFiles {
@@ -79,6 +79,9 @@ class ThinningDiskSwiftcProductsGenerator: SwiftcProductsGenerator {
         }
 
         // Build parent dir of the .swiftmodule file that contains a module
-        return modulePathOutput.deletingLastPathComponent()
+        return SwiftcProductsGeneratorOutput(
+            swiftmoduleDir: modulePathOutput.deletingLastPathComponent(),
+            objcHeaderFile: objcHeaderOutput
+        )
     }
 }

Sources/XCRemoteCache/Commands/Plugins/Thinning/ThinningPrebuildPlugin.swift

@@ -25,7 +25,7 @@ class ThinningConsumerPlugin {
 
     deinit {
         // initialised but never run plugin suggests that standard target fallbacks to the local development
-        // and DerivedData still misses build artifacts.
+        // and DerivedData still misses build artifacts
         guard wasRun else {
             let errorMessage = """
             \(type(of: self)) plugin has never been run, thinning cannot be supported. Verify you \

Sources/XCRemoteCache/Commands/Plugins/Thinning/UnzippedArtifactSwiftProductsOrganizer.swift

@@ -73,11 +73,12 @@ class UnzippedArtifactSwiftProductsOrganizer: SwiftProductsOrganizer {
             .appendingPathComponent(moduleName)
             .appendingPathComponent("\(moduleName)-Swift.h")
 
-        let generatedModuleDir = try productsGenerator.generateFrom(
+        let generatedModule = try productsGenerator.generateFrom(
             artifactSwiftModuleFiles: artifactSwiftmoduleFiles,
             artifactSwiftModuleObjCFile: artifactSwiftModuleObjCFile
         )
 
-        try fingerprintSyncer.decorate(sourceDir: generatedModuleDir, fingerprint: fingerprint)
+        try fingerprintSyncer.decorate(sourceDir: generatedModule.swiftmoduleDir, fingerprint: fingerprint)
+        try fingerprintSyncer.decorate(file: generatedModule.objcHeaderFile, fingerprint: fingerprint)
     }
 }

Sources/XCRemoteCache/Commands/Postbuild/Postbuild.swift

@@ -153,7 +153,7 @@ class Postbuild {
         // Replace all local paths to the generic ones (e.g. $SRCROOT)
         let remappers = [remapper] + creatorPlugins.compactMap(\.customPathsRemapper)
         let remapper = DependenciesRemapperComposite(remappers)
-        let abstractFingerprintFiles = remapper.replace(localPaths: dependencies.map(\.path))
+        let abstractFingerprintFiles = try remapper.replace(localPaths: dependencies.map(\.path))
         // TODO: use `inputs` read by dependenciesReader
         var meta = MainArtifactMeta(
             dependencies: abstractFingerprintFiles,
@@ -238,13 +238,32 @@ class Postbuild {
         let moduleSwiftProductURL = context.productsDir
             .appendingPathComponent(context.modulesFolderPath)
             .appendingPathComponent("\(modulename).swiftmodule")
+        let objcHeaderSwiftProductURL = context.derivedSourcesDir
+            .appendingPathComponent("\(modulename)-Swift.h")
+        // This header is obly valid if building a frameworks
+        let objcHeaderSwiftPublicPathURL = context.publicHeadersFolderPath?
+            .appendingPathComponent("\(modulename)-Swift.h")
         if let fingerprint = contextSpecificFingerprint {
             try fingerprintSyncer.decorate(
                 sourceDir: moduleSwiftProductURL,
                 fingerprint: fingerprint
             )
+            try fingerprintSyncer.decorate(
+                file: objcHeaderSwiftProductURL,
+                fingerprint: fingerprint
+            )
+            if let objcPublic = objcHeaderSwiftPublicPathURL {
+                try fingerprintSyncer.decorate(
+                    file: objcPublic,
+                    fingerprint: fingerprint
+                )
+            }
         } else {
             try fingerprintSyncer.delete(sourceDir: moduleSwiftProductURL)
+            try fingerprintSyncer.delete(sourceDir: objcHeaderSwiftProductURL)
+            if let objcPublic = objcHeaderSwiftPublicPathURL {
+                try fingerprintSyncer.delete(file: objcPublic)
+            }
         }
     }
 }

Sources/XCRemoteCache/Commands/Postbuild/PostbuildContext.swift

@@ -40,6 +40,7 @@ public struct PostbuildContext {
     var mode: Mode
     var targetName: String
     var targetTempDir: URL
+    var derivedFilesDir: URL
     /// Location where all compilation outputs (.o) are placed
     var compilationTempDir: URL
     var configuration: String
@@ -73,20 +74,34 @@ public struct PostbuildContext {
     let builtProductsDir: URL
     /// Location to the product bundle. Can be nil for libraries
     let bundleDir: URL?
-    let derivedSourcesDir: URL
+    var derivedSourcesDir: URL
     /// List of all targets to downloaded from the thinning aggregation target
     var thinnedTargets: [String]
-    /// Action type: build, indexbuild etc.
+    /// Action type: build, indexbuild etc
     var action: BuildActionType
     let modeMarkerPath: String
+    /// location of the json file that define virtual files system overlay
+    /// (mappings of the virtual location file -> local file path)
+    let overlayHeadersPath: URL
+    /// Regexes of files that should not be included in the dependency list
+    let irrelevantDependenciesPaths: [String]
+    /// Location of public headers. Not always available (e.g. static libraries)
+    var publicHeadersFolderPath: URL?
+    /// XCRemoteCache is explicitly disabled
+    let disabled: Bool
+    /// The LLBUILD_BUILD_ID ENV that describes the compilation identifier
+    /// it is used in the swift-frontend flow
+    let llbuildIdLockFile: URL
 }
 
 extension PostbuildContext {
+    // swiftlint:disable:next function_body_length
     init(_ config: XCRemoteCacheConfig, env: [String: String]) throws {
         mode = config.mode
         let targetNameValue: String = try env.readEnv(key: "TARGET_NAME")
         targetName = targetNameValue
         targetTempDir = try env.readEnv(key: "TARGET_TEMP_DIR")
+        derivedFilesDir = try env.readEnv(key: "DERIVED_FILE_DIR")
         let archs: [String] = try env.readEnv(key: "ARCHS").split(separator: " ").map(String.init)
         guard let firstArch = archs.first, !firstArch.isEmpty else {
             throw PostbuildContextError.missingArchitecture
@@ -117,7 +132,7 @@ extension PostbuildContext {
         dSYMPath = try env.readEnv(key: "DWARF_DSYM_FOLDER_PATH")
             .appendingPathComponent(env.readEnv(key: "DWARF_DSYM_FILE_NAME"))
         builtProductsDir = try env.readEnv(key: "BUILT_PRODUCTS_DIR")
-        if let contentsFolderPath = env.readEnv(key: "CONTENTS_FOLDER_PATH") {
+        if let contentsFolderPath: String = env.readEnv(key: "CONTENTS_FOLDER_PATH") {
             bundleDir = productsDir.appendingPathComponent(contentsFolderPath)
         } else {
             bundleDir = nil
@@ -127,5 +142,20 @@ extension PostbuildContext {
         thinnedTargets = thinFocusedTargetsString.split(separator: ",").map(String.init)
         action = (try? BuildActionType(rawValue: env.readEnv(key: "ACTION"))) ?? .unknown
         modeMarkerPath = config.modeMarkerPath
+        /// Note: The file has yaml extension, even it is in the json format
+        overlayHeadersPath = targetTempDir.appendingPathComponent("all-product-headers.yaml")
+        irrelevantDependenciesPaths = config.irrelevantDependenciesPaths
+        let publicHeadersPathEnv: String? = env.readEnv(key: "PUBLIC_HEADERS_FOLDER_PATH")
+        if let publicHeadersPath = publicHeadersPathEnv, publicHeadersPathEnv != "/usr/local/include" {
+            // '/usr/local/include' is a value of PUBLIC_HEADERS_FOLDER_PATH when no public headers are automatically
+            // generated and it is up to a project configuration to place it in a common location (e.g. static library)
+            publicHeadersFolderPath = builtProductsDir.appendingPathComponent(publicHeadersPath)
+        }
+        disabled = try env.readEnv(key: "XCRC_DISABLED") ?? false
+        let llbuildId: String = try env.readEnv(key: "LLBUILD_BUILD_ID")
+        llbuildIdLockFile = SwiftFrontendContext.buildLlbuildIdSharedLockUrl(
+            llbuildId: llbuildId,
+            tmpDir: targetTempDir
+        )
     }
 }

Sources/XCRemoteCache/Commands/Postbuild/XCPostbuild.swift

@@ -19,6 +19,7 @@
 
 import Foundation
 
+// swiftlint:disable type_body_length
 /// Checks current mode from a configuration and based on that:
 /// * triggers build completion
 /// * triggers uploading artifacts to the server for a 'producer' mode
@@ -33,8 +34,9 @@ public class XCPostbuild {
         let context: PostbuildContext
         let cacheHitLogger: CacheHitLogger
         do {
-            config = try XCRemoteCacheConfigReader(env: env, fileManager: fileManager).readConfiguration()
+            config = try XCRemoteCacheConfigReader(env: env, fileReader: fileManager).readConfiguration()
             context = try PostbuildContext(config, env: env)
+            updateProcessTag(context.targetName)
             let counterFactory: FileStatsCoordinator.CountersFactory = { file, count in
                 ExclusiveFileCounter(ExclusiveFile(file, mode: .override), countersCount: count)
             }
@@ -58,6 +60,7 @@ public class XCPostbuild {
             dependenciesWriter: FileDependenciesWriter.init,
             dependenciesReader: FileDependenciesReader.init,
             markerWriter: NoopMarkerWriter.init,
+            llbuildLockFile: context.llbuildIdLockFile,
             fileManager: fileManager
         )
 
@@ -66,8 +69,8 @@ public class XCPostbuild {
             // Initialize dependencies
             let primaryGitBranch = GitBranch(repoLocation: config.primaryRepo, branch: config.primaryBranch)
             let gitClient = GitClientImpl(repoRoot: config.repoRoot, primary: primaryGitBranch, shell: shellGetStdout)
-            let pathRemapper = try StringDependenciesRemapperFactory().build(
-                orderKeys: DependenciesMapping.rewrittenEnvs,
+            let envsRemapper = try PathDependenciesRemapperFactory().build(
+                orderKeys: DependenciesMapping.rewrittenEnvs + config.customRewriteEnvs,
                 envs: env,
                 customMappings: config.outOfBandMappings
             )
@@ -85,8 +88,14 @@ public class XCPostbuild {
                 fingerprintFilesGenerator,
                 algorithm: MD5Algorithm()
             )
-            let organizer = ZipArtifactOrganizer(targetTempDir: context.targetTempDir, fileManager: fileManager)
+            let organizer = ZipArtifactOrganizer(
+                targetTempDir: context.targetTempDir,
+                // In postbuild we don't preprocess artifacts (no need to replace path placeholders)
+                artifactProcessors: [],
+                fileManager: fileManager
+            )
             let metaWriter = JsonMetaWriter(fileWriter: fileManager, pretty: config.prettifyMetaFiles)
+            let fileRemapper = TextFileDependenciesRemapper(remapper: envsRemapper, fileAccessor: fileManager)
             let artifactCreator = BuildArtifactCreator(
                 buildDir: context.productsDir,
                 tempDir: context.targetTempDir,
@@ -95,6 +104,7 @@ public class XCPostbuild {
                 modulesFolderPath: context.modulesFolderPath,
                 dSYMPath: context.dSYMPath,
                 metaWriter: metaWriter,
+                artifactProcessor: UnzippedArtifactProcessor(fileRemapper: fileRemapper, dirScanner: fileManager),
                 fileManager: fileManager
             )
             let dirAccessor = DirAccessorComposer(
@@ -112,6 +122,7 @@ public class XCPostbuild {
                 awsV4Signature = AWSV4Signature(
                     secretKey: config.AWSSecretKey,
                     accessKey: config.AWSAccessKey,
+                    securityToken: config.AWSSecurityToken,
                     region: config.AWSRegion,
                     service: config.AWSService,
                     date: Date(timeIntervalSinceNow: 0)
@@ -120,6 +131,7 @@ public class XCPostbuild {
             let networkClient = NetworkClientImpl(
                 session: sessionFactory.build(),
                 retries: config.uploadRetries,
+                retryDelay: config.retryDelay,
                 fileManager: fileManager,
                 awsV4Signature: awsV4Signature
             )
@@ -127,6 +139,7 @@ public class XCPostbuild {
                 mode: context.mode,
                 downloadStreamURL: context.recommendedCacheAddress,
                 upstreamStreamURL: context.cacheAddresses,
+                uploadBatchSize: config.uploadBatchSize,
                 networkClient: networkClient,
                 urlBuilderFactory: {
                     try URLBuilderImpl(
@@ -140,17 +153,42 @@ public class XCPostbuild {
             let fileReaderFactory: (URL) -> DependenciesReader = {
                 FileDependenciesReader($0, accessor: fileManager)
             }
+            let assetsFileDependenciesFactory: (URL) -> DependenciesReader = {
+                AssetsFileDependenciesReader($0, dirAccessor: fileManager)
+            }
             let dependenciesReader = TargetDependenciesReader(
-                context.compilationTempDir,
-                fileDependeciesReaderFactory: fileReaderFactory,
+                compilationOutputDir: context.compilationTempDir,
+                assetsCatalogOutputDir: context.targetTempDir,
+                fileDependenciesReaderFactory: fileReaderFactory,
+                assetsDependenciesReaderFactory: assetsFileDependenciesFactory,
                 dirScanner: fileManager
             )
+            var remappers: [DependenciesRemapper] = []
+            if !config.disableVFSOverlay {
+                // As the PostbuildContext assumes file location and filename (`all-product-headers.yaml`)
+                // do not fail in case of a missing headers overlay file. In the future, all overlay files could be
+                // captured from the swiftc invocation similarly is stored in the `history.compile`
+                // for the consumer mode
+                let overlayReader = JsonOverlayReader(
+                    context.overlayHeadersPath,
+                    mode: .bestEffort,
+                    fileReader: fileManager
+                )
+                let overlayRemapper = OverlayDependenciesRemapper(
+                    overlayReader: overlayReader
+                )
+                remappers.append(overlayRemapper)
+            }
+            remappers.append(envsRemapper)
+            let pathRemapper = DependenciesRemapperComposite(remappers)
             let dependencyProcessor = DependencyProcessorImpl(
                 xcode: context.xcodeDir,
                 product: context.productsDir,
                 source: context.srcRoot,
                 intermediate: context.targetTempDir,
-                bundle: context.bundleDir
+                derivedFiles: context.derivedFilesDir,
+                bundle: context.bundleDir,
+                skippedRegexes: context.irrelevantDependenciesPaths
             )
             // Override fingerprints for all produced '.swiftmodule' files
             let fingerprintOverrideManager = FingerprintOverrideManagerImpl(
@@ -177,7 +215,10 @@ public class XCPostbuild {
                 if context.moduleName == config.thinningTargetModuleName {
                     switch context.mode {
                     case .consumer:
+                        // no need to process artifacts in postbuild. Prebuild has already
+                        // run a processor on a downloaded artifact
                         let artifactOrganizerFactory = ThinningConsumerZipArtifactsOrganizerFactory(
+                            processors: [],
                             fileManager: fileManager
                         )
                         let swiftProductsLocationProvider =
@@ -239,7 +280,12 @@ public class XCPostbuild {
             )
 
             // Trigger build completion
-            if try modeController.isEnabled() {
+            if context.disabled {
+                infoLog("XCRC fully disabled for \(context.targetName), \(context.platform), \(context.configuration)")
+                // Cutoff the process is disabled, but generate an "empty" list of dependencies
+                try? modeController.disable()
+                return
+            } else if try modeController.isEnabled() {
                 // Decorate .swiftmodule in the product dir with fingerprint(s) overrides from a cache artifact
                 try postbuildAction.performBuildCompletion()
             } else if context.mode == .consumer {
@@ -301,3 +347,4 @@ public class XCPostbuild {
         }
     }
 }
+// swiftlint:enable type_body_length

Sources/XCRemoteCache/Commands/Prebuild/Prebuild.swift

@@ -20,6 +20,7 @@
 import Foundation
 
 enum PrebuildResult: Equatable {
+    case disabled
     case incompatible
     case compatible(localDependencies: [URL])
 }
@@ -57,13 +58,18 @@ class Prebuild {
 
     // swiftlint:disable:next function_body_length
     public func perform() throws -> PrebuildResult {
+        guard !context.disabled else {
+            return .disabled
+        }
         guard case .available(let commit) = context.remoteCommit else {
             return .incompatible
         }
         do {
             let metaData = try networkClient.fetch(.meta(commit: commit))
             let meta = try metaReader.read(data: metaData)
-            let localDependencies = remapper.replace(genericPaths: meta.dependencies).map(URL.init(fileURLWithPath:))
+            let localDependencies = try remapper.replace(
+                genericPaths: meta.dependencies
+            ).map(URL.init(fileURLWithPath:))
             let localFingerprint = try generateFingerprint(for: localDependencies)
             if localFingerprint.raw != meta.rawFingerprint {
                 if context.forceCached {

Sources/XCRemoteCache/Commands/Prebuild/PrebuildContext.swift

@@ -43,6 +43,14 @@ public struct PrebuildContext {
     let targetName: String
     /// List of all targets to downloaded from the thinning aggregation target
     var thinnedTargets: [String]?
+    /// location of the json file that define virtual files system overlay
+    /// (mappings of the virtual location file -> local file path)
+    let overlayHeadersPath: URL
+    /// XCRemoteCache is explicitly disabled
+    let disabled: Bool
+    /// The LLBUILD_BUILD_ID ENV that describes the compilation identifier
+    /// it is used in the swift-frontend flow
+    let llbuildIdLockFile: URL
 }
 
 extension PrebuildContext {
@@ -64,5 +72,13 @@ extension PrebuildContext {
         self.targetName = targetName
         let thinFocusedTargetsString: String? = env.readEnv(key: "SPT_XCREMOTE_CACHE_THINNED_TARGETS")
         thinnedTargets = thinFocusedTargetsString?.split(separator: ",").map(String.init)
+        /// Note: The file has yaml extension, even it is in the json format
+        overlayHeadersPath = targetTempDir.appendingPathComponent("all-product-headers.yaml")
+        disabled = try env.readEnv(key: "XCRC_DISABLED") ?? false
+        let llbuildId: String = try env.readEnv(key: "LLBUILD_BUILD_ID")
+        llbuildIdLockFile = SwiftFrontendContext.buildLlbuildIdSharedLockUrl(
+            llbuildId: llbuildId,
+            tmpDir: targetTempDir
+        )
     }
 }

Sources/XCRemoteCache/Commands/Prebuild/XCPrebuild.swift

@@ -29,8 +29,9 @@ public class XCPrebuild {
         let config: XCRemoteCacheConfig
         let context: PrebuildContext
         do {
-            config = try XCRemoteCacheConfigReader(env: env, fileManager: fileManager).readConfiguration()
+            config = try XCRemoteCacheConfigReader(env: env, fileReader: fileManager).readConfiguration()
             context = try PrebuildContext(config, env: env)
+            updateProcessTag(context.targetName)
         } catch {
             // Fatal error:
             exit(1, "FATAL: Prebuild initialization failed with error: \(error)")
@@ -54,6 +55,7 @@ public class XCPrebuild {
             dependenciesWriter: FileDependenciesWriter.init,
             dependenciesReader: FileDependenciesReader.init,
             markerWriter: lazyMarkerWriterFactory,
+            llbuildLockFile: context.llbuildIdLockFile,
             fileManager: fileManager
         )
 
@@ -77,6 +79,10 @@ public class XCPrebuild {
             exit(0)
         }
 
+        let compilationHistoryOrganizer = CompilationHistoryFileOrganizer(
+            context.compilationHistoryFile,
+            fileManager: fileManager
+        )
         do {
             let envFingerprint = try EnvironmentFingerprintGenerator(
                 configuration: config,
@@ -95,6 +101,7 @@ public class XCPrebuild {
                 awsV4Signature = AWSV4Signature(
                     secretKey: config.AWSSecretKey,
                     accessKey: config.AWSAccessKey,
+                    securityToken: config.AWSSecurityToken,
                     region: config.AWSRegion,
                     service: config.AWSService,
                     date: Date(timeIntervalSinceNow: 0)
@@ -103,6 +110,7 @@ public class XCPrebuild {
             let networkClient = NetworkClientImpl(
                 session: sessionFactory.build(),
                 retries: config.downloadRetries,
+                retryDelay: config.retryDelay,
                 fileManager: fileManager,
                 awsV4Signature: awsV4Signature
             )
@@ -115,11 +123,27 @@ public class XCPrebuild {
             )
             let client: NetworkClient = config.disableHttpCache ? networkClient : cacheNetworkClient
             let remoteNetworkClient = RemoteNetworkClientImpl(client, urlBuilder)
-            let pathRemapper = try StringDependenciesRemapperFactory().build(
-                orderKeys: DependenciesMapping.rewrittenEnvs,
+            let envsRemapper = try PathDependenciesRemapperFactory().build(
+                orderKeys: DependenciesMapping.rewrittenEnvs + config.customRewriteEnvs,
                 envs: env,
                 customMappings: config.outOfBandMappings
             )
+            var remappers: [DependenciesRemapper] = []
+            if !config.disableVFSOverlay {
+                // As PrebuildContext assumes file location and its filename (`all-product-headers.yaml`)
+                // do not fail in case of a missing headers overlay file
+                let overlayReader = JsonOverlayReader(
+                    context.overlayHeadersPath,
+                    mode: .bestEffort,
+                    fileReader: fileManager
+                )
+                let overlayRemapper = OverlayDependenciesRemapper(
+                    overlayReader: overlayReader
+                )
+                remappers.append(overlayRemapper)
+            }
+            remappers.append(envsRemapper)
+            let pathRemapper = DependenciesRemapperComposite(remappers)
             let filesFingerprintGenerator = FingerprintAccumulatorImpl(
                 algorithm: MD5Algorithm(),
                 fileManager: fileManager
@@ -129,17 +153,29 @@ public class XCPrebuild {
                 filesFingerprintGenerator,
                 algorithm: MD5Algorithm()
             )
-            let organizer = ZipArtifactOrganizer(targetTempDir: context.targetTempDir, fileManager: fileManager)
-            let compilationHistoryOrganizer = CompilationHistoryFileOrganizer(
-                context.compilationHistoryFile,
+            let fileRemapper = TextFileDependenciesRemapper(
+                remapper: envsRemapper,
+                fileAccessor: fileManager
+            )
+            let artifactProcessor = UnzippedArtifactProcessor(fileRemapper: fileRemapper, dirScanner: fileManager)
+            let metaUpdater = ArtifactMetaUpdater(
+                fileRemapper: fileRemapper,
+                metaWriter: JsonMetaWriter(fileWriter: fileManager, pretty: true)
+            )
+            let organizer = ZipArtifactOrganizer(
+                targetTempDir: context.targetTempDir,
+                artifactProcessors: [artifactProcessor, metaUpdater],
                 fileManager: fileManager
             )
             let metaReader = JsonMetaReader(fileAccessor: fileManager)
-            var consumerPlugins: [ArtifactConsumerPrebuildPlugin] = []
+            var consumerPlugins: [ArtifactConsumerPrebuildPlugin] = [metaUpdater]
 
             if config.thinningEnabled {
                 if context.moduleName == config.thinningTargetModuleName, let thinnedTarget = context.thinnedTargets {
-                    let organizerFactory = ThinningConsumerZipArtifactsOrganizerFactory(fileManager: .default)
+                    let organizerFactory = ThinningConsumerZipArtifactsOrganizerFactory(
+                        processors: [artifactProcessor],
+                        fileManager: fileManager
+                    )
                     let aggregationPlugin = ThinningConsumerPrebuildPlugin(
                         targetName: context.targetName,
                         tempDir: context.targetTempDir,
@@ -171,7 +207,9 @@ public class XCPrebuild {
             case .compatible(localDependencies: let dependencies):
                 // TODO: pass `allowedInputFiles` observed in the build time
                 try modeController.enable(allowedInputFiles: dependencies, dependencies: dependencies)
-                compilationHistoryOrganizer.reset()
+            case .disabled:
+                infoLog("XCRemoteCache is explicitly disabled")
+                try modeController.disable()
             }
         } catch {
             disableRemoteCache(
@@ -179,6 +217,7 @@ public class XCPrebuild {
                 errorMessage: "Prebuild step failed with error: \(error)"
             )
         }
+        compilationHistoryOrganizer.reset()
     }
 
     private func disableRemoteCache(modeController: PhaseCacheModeController, errorMessage: String?) {

Sources/XCRemoteCache/Commands/Prepare/CCWrapperBuilder.swift

@@ -72,7 +72,17 @@ class TemplateBasedCCWrapperBuilder: CCWrapperBuilder {
         )
         infoLog("ClangWrapperBuilder compiles file at \(compilationFile).")
         // -O3: optimize for faster execution
-        let args = [clangCommand, "-arch", "arm64", "-arch", "x86_64", "-O3", compilationFile.path, "-o", destination.path]
+        let args = [
+            clangCommand,
+            "-arch",
+            "arm64",
+            "-arch",
+            "x86_64",
+            "-O3",
+            compilationFile.path,
+            "-o",
+            destination.path,
+        ]
         let compilationOutput = try shell("xcrun", args, URL(fileURLWithPath: "").path, nil)
         infoLog("Clang compilation output: \(compilationOutput)")
     }
@@ -385,6 +395,9 @@ class TemplateBasedCCWrapperBuilder: CCWrapperBuilder {
             const char *dependency_arg_name = "-MF";
             const char *output_arg_name = "-o";
             const char *serialize_diagnostics_arg_name = "--serialize-diagnostics";
+            const char *language_mode_arg_name = "-x";
+            const char *precompile_objc_header_arg_value = "objective-c-header";
+            const char *precompile_c_header_arg_value = "c-header";
             const char *clang_cmd = "\(clangCommand)";
             const char *markerFile = "\(markerFilename)";
             const char *compilationHistoryFile = "\(compilationHistoryFilename)";
@@ -399,6 +412,7 @@ class TemplateBasedCCWrapperBuilder: CCWrapperBuilder {
             const char *output_file= NULL;
             const char *input_file = NULL;
             const char *diagnostics_file = NULL;
+            const char *language_mode = NULL;
 
             for (int i = 1; i < argc; i++) {
                 if (strcmp(argv[i], dependency_arg_name) == 0 && i < (argc - 1) ) {
@@ -419,6 +433,12 @@ class TemplateBasedCCWrapperBuilder: CCWrapperBuilder {
                     i += 1;
                     clang_args[i] = argv[i];
                     diagnostics_file = argv[i];
+                } if (strcmp(argv[i], language_mode_arg_name) == 0 && i < (argc - 1) ) {
+                    // called with "-x path" pattern and not the last argument
+                    clang_args[i] = argv[i];
+                    i += 1;
+                    clang_args[i] = argv[i];
+                    language_mode = argv[i];
                 } else if (
                     isSuffixed(argv[i],".m") ||
                     isSuffixed(argv[i],".mm") ||
@@ -426,10 +446,13 @@ class TemplateBasedCCWrapperBuilder: CCWrapperBuilder {
                     isSuffixed(argv[i],".cc") ||
                     isSuffixed(argv[i],".cpp") ||
                     isSuffixed(argv[i],".c++") ||
-                    isSuffixed(argv[i],".cxx")
+                    isSuffixed(argv[i],".cxx") ||
+                    isSuffixed(argv[i],".S") ||
+                    isSuffixed(argv[i],".s")
                 ) {
                     // a full list of extensions is taken from https://clang.llvm.org/docs/ClangFormatStyleOptions.html
                     // support for .m,.mm,.c,.cc,.cpp,.c++,.cxx input files
+                    // .s and .S are assembly files
                     clang_args[i] = argv[i];
                     input_file = argv[i];
                 } else {
@@ -438,6 +461,14 @@ class TemplateBasedCCWrapperBuilder: CCWrapperBuilder {
                 }
             }
 
+           // null-terminating the args array needed for local compilation fallback
+           clang_args[argc] = NULL;
+
+           // Verify mode. Even a target is cached, pch mode is not supported. Fallback to the local compilation
+           if (language_mode != NULL && (strcmp(language_mode, precompile_objc_header_arg_value) == 0 || strcmp(language_mode, precompile_c_header_arg_value) == 0)) {
+               return execvp(clang_cmd, (char *const*) clang_args);
+           }
+
            // Verify all input arguments
            if (dependency_file == NULL) {
                fprintf(stderr, "error: missing %s input\\n", dependency_arg_name);
@@ -508,8 +539,6 @@ class TemplateBasedCCWrapperBuilder: CCWrapperBuilder {
                }
            }
 
-           // null-terminating the args array
-           clang_args[argc] = NULL;
            #pragma GCC diagnostic push
            #pragma GCC diagnostic ignored "-Wincompatible-pointer-types-discards-qualifiers"
            /// execvp takes $PATH to consideration

Sources/XCRemoteCache/Commands/Prepare/Integrate/BuildSettingsIntegrateAppender.swift

@@ -21,6 +21,11 @@ import Foundation
 
 typealias BuildSettings = [String: Any]
 
+struct BuildSettingsIntegrateAppenderOption: OptionSet {
+    let rawValue: Int
+
+    static let disableSwiftDriverIntegration = BuildSettingsIntegrateAppenderOption(rawValue: 1 << 0)
+}
 // Manages Xcode build settings
 protocol BuildSettingsIntegrateAppender {
     /// Appends XCRemoteCache-specific build settings
@@ -33,20 +38,44 @@ protocol BuildSettingsIntegrateAppender {
 class XcodeProjBuildSettingsIntegrateAppender: BuildSettingsIntegrateAppender {
     private let mode: Mode
     private let repoRoot: URL
+    private let fakeSrcRoot: URL
+    private let sdksExclude: [String]
+    private let options: BuildSettingsIntegrateAppenderOption
 
-    init(mode: Mode, repoRoot: URL) {
+    init(
+        mode: Mode,
+        repoRoot: URL,
+        fakeSrcRoot: URL,
+        sdksExclude: [String],
+        options: BuildSettingsIntegrateAppenderOption
+    ) {
         self.mode = mode
         self.repoRoot = repoRoot
+        self.fakeSrcRoot = fakeSrcRoot
+        self.sdksExclude = sdksExclude
+        self.options = options
     }
 
     func appendToBuildSettings(buildSettings: BuildSettings, wrappers: XCRCBinariesPaths) -> BuildSettings {
         var result = buildSettings
-        result["SWIFT_EXEC"] = wrappers.swiftc.path
+        setBuildSetting(buildSettings: &result, key: "SWIFT_EXEC", value: wrappers.swiftc.path )
+        if options.contains(.disableSwiftDriverIntegration) {
+            setBuildSetting(buildSettings: &result, key: "SWIFT_USE_INTEGRATED_DRIVER", value: "NO" )
+        }
         // When generating artifacts, no need to shell-out all compilation commands to our wrappers
         if case .consumer = mode {
-            result["CC"] = wrappers.cc.path
-            result["LD"] = wrappers.ld.path
-            result["LIBTOOL"] = wrappers.libtool.path
+            setBuildSetting(buildSettings: &result, key: "CC", value: wrappers.cc.path )
+            setBuildSetting(buildSettings: &result, key: "LD", value: wrappers.ld.path )
+            // Setting LIBTOOL to '' breaks SwiftDriver intengration so resetting it to the original value
+            // 'libtool' for all excluded configurations
+            setBuildSetting(
+                buildSettings: &result,
+                key: "LIBTOOL",
+                value: wrappers.libtool.path,
+                excludedValue: "libtool"
+            )
+            setBuildSetting(buildSettings: &result, key: "LIPO", value: wrappers.lipo.path )
+            setBuildSetting(buildSettings: &result, key: "LDPLUSPLUS", value: wrappers.ldplusplus.path )
         }
 
         let existingSwiftFlags = result["OTHER_SWIFT_FLAGS"] as? String
@@ -58,11 +87,36 @@ class XcodeProjBuildSettingsIntegrateAppender: BuildSettingsIntegrateAppender {
         swiftFlags.assignFlag(key: "debug-prefix-map", value: "\(repoRoot.path)=$(XCRC_FAKE_SRCROOT)")
         clangFlags.assignFlag(key: "debug-prefix-map", value: "\(repoRoot.path)=$(XCRC_FAKE_SRCROOT)")
 
-        result["OTHER_SWIFT_FLAGS"] = swiftFlags.settingValue
-        result["OTHER_CFLAGS"] = clangFlags.settingValue
+        setBuildSetting(buildSettings: &result, key: "OTHER_SWIFT_FLAGS", value: swiftFlags.settingValue )
+        setBuildSetting(buildSettings: &result, key: "OTHER_CFLAGS", value: clangFlags.settingValue )
 
-        result["XCRC_FAKE_SRCROOT"] = "/\(String(repeating: "x", count: 10))"
-        result["XCRC_PLATFORM_PREFERRED_ARCH"] = "$(LINK_FILE_LIST_$(CURRENT_VARIANT)_$(PLATFORM_PREFERRED_ARCH):dir:standardizepath:file:default=arm64)"
+        setBuildSetting(buildSettings: &result, key: "XCRC_FAKE_SRCROOT", value: "\(fakeSrcRoot.path)" )
+        setBuildSetting(buildSettings: &result, key: "XCRC_PLATFORM_PREFERRED_ARCH", value:
+        """
+        $(LINK_FILE_LIST_$(CURRENT_VARIANT)_$(PLATFORM_PREFERRED_ARCH):dir:standardizepath:file:default=arm64)
+        """
+        )
+
+        explicitlyDisableSDKs(buildSettings: &result)
         return result
     }
+
+    private func setBuildSetting(buildSettings: inout BuildSettings, key: String, value: String?, excludedValue: String = "") {
+        buildSettings[key] = value
+        guard value != nil else {
+            // no need to exclude as the value will
+            return
+        }
+        // Erase all overrides for a given sdk so a default toolchain is used
+        for skippedSDK in sdksExclude {
+            buildSettings["\(key)[sdk=\(skippedSDK)]"] = excludedValue
+        }
+    }
+
+    // For all exlcuded SDKs passes XCRC_DISABLED=TRUE, which will cut-off early the pre_build phase
+    private func explicitlyDisableSDKs(buildSettings: inout BuildSettings) {
+        for skippedSDK in sdksExclude {
+            buildSettings["XCRC_DISABLED[sdk=\(skippedSDK)]"] = "YES"
+        }
+    }
 }

Sources/XCRemoteCache/Commands/Prepare/Integrate/IncludeOracle.swift

@@ -35,7 +35,7 @@ struct IncludeExcludeOracle: IncludeOracle {
 
 
     func shouldInclude(identifier: OracleIdentifierType) -> Bool {
-        // exclude array has precedence.
+        // exclude array has precedence
         if excludes.contains(identifier) {
             return false
         }

Sources/XCRemoteCache/Commands/Prepare/Integrate/IntegrateContext.swift

@@ -27,14 +27,14 @@ struct IntegrateContext {
     let configOverride: URL
     let fakeSrcRoot: URL
     let output: URL?
+    let buildSettingsAppenderOptions: BuildSettingsIntegrateAppenderOption
 }
 
 extension IntegrateContext {
     init(
         input: String,
-        repoRootPath: String,
+        config: XCRemoteCacheConfig,
         mode: Mode,
-        configOverridePath: String,
         env: [String: String],
         binariesDir: URL,
         fakeSrcRoot: String,
@@ -42,19 +42,29 @@ extension IntegrateContext {
     ) throws {
         projectPath = URL(fileURLWithPath: input)
         let srcRoot = projectPath.deletingLastPathComponent()
-        repoRoot = URL(fileURLWithPath: repoRootPath, relativeTo: srcRoot)
+        repoRoot = URL(fileURLWithPath: config.repoRoot, relativeTo: srcRoot)
         self.mode = mode
-        configOverride = URL(fileURLWithPath: configOverridePath, relativeTo: srcRoot)
+        configOverride = URL(fileURLWithPath: config.extraConfigurationFile, relativeTo: srcRoot)
         output = outputPath.flatMap(URL.init(fileURLWithPath:))
         self.fakeSrcRoot = URL(fileURLWithPath: fakeSrcRoot)
+        var swiftcBinaryName = "swiftc"
+        var buildSettingsAppenderOptions: BuildSettingsIntegrateAppenderOption = []
+        // Keep the legacy behaviour (supported in Xcode 14 and lower)
+        if !config.enableSwiftDriverIntegration {
+            buildSettingsAppenderOptions.insert(.disableSwiftDriverIntegration)
+            swiftcBinaryName = "xcswiftc"
+        }
         binaries = XCRCBinariesPaths(
             prepare: binariesDir.appendingPathComponent("xcprepare"),
             cc: binariesDir.appendingPathComponent("xccc"),
-            swiftc: binariesDir.appendingPathComponent("xcswiftc"),
+            swiftc: binariesDir.appendingPathComponent(swiftcBinaryName),
             libtool: binariesDir.appendingPathComponent("xclibtool"),
+            lipo: binariesDir.appendingPathComponent("xclipo"),
             ld: binariesDir.appendingPathComponent("xcld"),
+            ldplusplus: binariesDir.appendingPathComponent("xcldplusplus"),
             prebuild: binariesDir.appendingPathComponent("xcprebuild"),
             postbuild: binariesDir.appendingPathComponent("xcpostbuild")
         )
+        self.buildSettingsAppenderOptions = buildSettingsAppenderOptions
     }
 }

Sources/XCRemoteCache/Commands/Prepare/Integrate/XCIntegrate.swift

@@ -34,6 +34,7 @@ public class XCIntegrate {
     private let consumerEligiblePlatforms: String
     private let lldbMode: LLDBInitMode
     private let fakeSrcRoot: String
+    private let sdksExclude: String
     private let output: String?
 
     public init(
@@ -48,6 +49,7 @@ public class XCIntegrate {
         consumerEligiblePlatforms: String,
         lldbMode: LLDBInitMode,
         fakeSrcRoot: String,
+        sdksExclude: String,
         output: String?
     ) {
         projectPath = input
@@ -61,6 +63,7 @@ public class XCIntegrate {
         self.consumerEligiblePlatforms = consumerEligiblePlatforms
         self.lldbMode = lldbMode
         self.fakeSrcRoot = fakeSrcRoot
+        self.sdksExclude = sdksExclude
         self.output = output
     }
 
@@ -74,14 +77,13 @@ public class XCIntegrate {
             let binariesDir = commandURL.deletingLastPathComponent()
 
             let srcRoot: URL = URL(fileURLWithPath: projectPath).deletingLastPathComponent()
-            let config = try XCRemoteCacheConfigReader(srcRootPath: srcRoot.path, fileManager: fileManager)
+            let config = try XCRemoteCacheConfigReader(srcRootPath: srcRoot.path, fileReader: fileManager)
                 .readConfiguration()
 
             let context = try IntegrateContext(
                 input: projectPath,
-                repoRootPath: config.repoRoot,
+                config: config,
                 mode: mode,
-                configOverridePath: config.extraConfigurationFile,
                 env: env,
                 binariesDir: binariesDir,
                 fakeSrcRoot: fakeSrcRoot,
@@ -97,7 +99,10 @@ public class XCIntegrate {
             )
             let buildSettingsAppender = XcodeProjBuildSettingsIntegrateAppender(
                 mode: context.mode,
-                repoRoot: context.repoRoot
+                repoRoot: context.repoRoot,
+                fakeSrcRoot: context.fakeSrcRoot,
+                sdksExclude: sdksExclude.integrateArrayArguments,
+                options: context.buildSettingsAppenderOptions
             )
             let lldbPatcher: LLDBInitPatcher
             switch lldbMode {

Sources/XCRemoteCache/Commands/Prepare/Integrate/XCRCBinariesPaths.swift

@@ -25,7 +25,9 @@ struct XCRCBinariesPaths {
     let cc: URL
     let swiftc: URL
     let libtool: URL
+    let lipo: URL
     let ld: URL
+    let ldplusplus: URL
     let prebuild: URL
     let postbuild: URL
 }

Sources/XCRemoteCache/Commands/Prepare/Integrate/XcodeProjIntegrate.swift

@@ -227,13 +227,15 @@ struct XcodeProjIntegrate: Integrate {
         let previousRCPhases = target.buildPhases.filter(isRCPhase)
         target.buildPhases.removeAll(where: previousRCPhases.contains)
 
-        if target.buildPhases.map(\.buildPhase).contains(.sources) {
+        if let sourceIndex = target.buildPhases.map(\.buildPhase).firstIndex(of: .sources) {
             // add (pre|post)build phases only when a target has some compilation steps
             // otherwise they make no sense (nothing to store in an artifact)
-            pbxproj.add(object: prebuildPhase)
-            target.buildPhases.insert(prebuildPhase, at: 0)
+            // add postbuild right after compilation as custom build steps may depend on files generated by
+            // the xcpostbuild (e.g. to copy -Swift.h.md5)
             pbxproj.add(object: postbuildPhase)
-            target.buildPhases.append(postbuildPhase)
+            target.buildPhases.insert(postbuildPhase, at: sourceIndex + 1)
+            pbxproj.add(object: prebuildPhase)
+            target.buildPhases.insert(prebuildPhase, at: sourceIndex)
         }
     }
 

Sources/XCRemoteCache/Commands/Prepare/Prepare.swift

@@ -77,7 +77,18 @@ class Prepare: PrepareLogic {
             guard fileAccessor.fileExists(atPath: PhaseCacheModeController.xcodeSelectLink.path) else {
                 throw PrepareError.missingXcodeSelectDirectory
             }
-            let commonSha = try gitClient.getCommonPrimarySha()
+
+            let commonSha: String
+            do {
+                commonSha = try gitClient.getCommonPrimarySha()
+            } catch let GitClientError.noCommonShaWithPrimaryRepo(remoteName, error) {
+                guard context.gracefullyHandleMissingCommonSha else {
+                    throw GitClientError.noCommonShaWithPrimaryRepo(remoteName: remoteName, error: error)
+                }
+                infoLog("Cannot find a common sha with the primary branch: \(error). Gracefully disabling remote cache")
+                try disable()
+                return .failed
+            }
 
             if context.offline {
                 // Optimistically take first common sha

Sources/XCRemoteCache/Commands/Prepare/PrepareContext.swift

@@ -52,6 +52,8 @@ public struct PrepareContext {
     let cacheHealthPathProbeCount: Int
     /// clang wrapper output file
     let xcccCommand: URL
+    /// gracefully disable remote cache for missing common sha with the primary branch
+    let gracefullyHandleMissingCommonSha: Bool
 }
 
 extension PrepareContext {
@@ -77,5 +79,6 @@ extension PrepareContext {
         cacheAddresses = try config.cacheAddresses.map(URL.build)
         cacheHealthPath = config.cacheHealthPath
         cacheHealthPathProbeCount = config.cacheHealthPathProbeCount
+        gracefullyHandleMissingCommonSha = config.gracefullyHandleMissingCommonSha
     }
 }

Sources/XCRemoteCache/Commands/Prepare/PrepareMarkContext.swift

@@ -31,10 +31,12 @@ public struct PrepareMarkContext {
     let recommendedCacheAddress: URL
     /// All remote servers to mark
     let cacheAddresses: [URL]
+    /// XCRemoteCache is explicitly disabled
+    let disabled: Bool
 }
 
 extension PrepareMarkContext {
-    init(_ config: XCRemoteCacheConfig) throws {
+    init(_ config: XCRemoteCacheConfig, env: [String: String]) throws {
         let sourceRoot = URL(fileURLWithPath: config.sourceRoot, isDirectory: true)
         repoRoot = URL(fileURLWithPath: config.repoRoot, relativeTo: sourceRoot)
         guard let address = URL(string: config.recommendedCacheAddress) else {
@@ -43,5 +45,6 @@ extension PrepareMarkContext {
         }
         recommendedCacheAddress = address
         cacheAddresses = try config.cacheAddresses.map(URL.build)
+        disabled = try env.readEnv(key: "XCRC_DISABLED") ?? false
     }
 }

Sources/XCRemoteCache/Commands/Prepare/XCConfig.swift

@@ -18,7 +18,6 @@
 // under the License.
 
 import Foundation
-import Yams
 
 /// Print current configuration to the console
 public class XCConfig {
@@ -33,7 +32,7 @@ public class XCConfig {
         let fileManager = FileManager.default
         let config: XCRemoteCacheConfig
         do {
-            config = try XCRemoteCacheConfigReader(env: env, fileManager: fileManager).readConfiguration()
+            config = try XCRemoteCacheConfigReader(env: env, fileReader: fileManager).readConfiguration()
         } catch {
             exit(1, "FATAL: Prepare initialization failed with error: \(error)")
         }

Sources/XCRemoteCache/Commands/Prepare/XCPrepare.swift

@@ -18,7 +18,6 @@
 // under the License.
 
 import Foundation
-import Yams
 
 /// Switch between Online/Offline modes
 public enum XCPrepareMode {
@@ -62,7 +61,7 @@ public class XCPrepare {
         var context: PrepareContext
         let xcodeVersion: String
         do {
-            config = try XCRemoteCacheConfigReader(env: env, fileManager: fileManager).readConfiguration()
+            config = try XCRemoteCacheConfigReader(env: env, fileReader: fileManager).readConfiguration()
             context = try PrepareContext(config, offline: offline)
             xcodeVersion = try customXcodeBuildNumber ?? XcodeProbeImpl(shell: shellGetStdout).read().buildVersion
         } catch {
@@ -79,6 +78,7 @@ public class XCPrepare {
                 awsV4Signature = AWSV4Signature(
                     secretKey: config.AWSSecretKey,
                     accessKey: config.AWSAccessKey,
+                    securityToken: config.AWSSecurityToken,
                     region: config.AWSRegion,
                     service: config.AWSService,
                     date: Date(timeIntervalSinceNow: 0)
@@ -87,6 +87,7 @@ public class XCPrepare {
             let networkClient = NetworkClientImpl(
                 session: sessionFactory.build(),
                 retries: config.downloadRetries,
+                retryDelay: config.retryDelay,
                 fileManager: fileManager,
                 awsV4Signature: awsV4Signature
             )

Sources/XCRemoteCache/Commands/Prepare/XCPrepareMark.swift

@@ -46,13 +46,18 @@ public class XCPrepareMark {
         let context: PrepareMarkContext
         let xcodeVersion: String
         do {
-            config = try XCRemoteCacheConfigReader(env: env, fileManager: fileManager).readConfiguration()
-            context = try PrepareMarkContext(config)
+            config = try XCRemoteCacheConfigReader(env: env, fileReader: fileManager).readConfiguration()
+            context = try PrepareMarkContext(config, env: env)
             xcodeVersion = try xcode ?? XcodeProbeImpl(shell: shellGetStdout).read().buildVersion
         } catch {
             exit(1, "FATAL: Prepare initialization failed with error: \(error)")
         }
 
+        guard !context.disabled else {
+            infoLog("XCRemoteCache explicitly disabled for marking.")
+            return
+        }
+
         do {
             let sessionFactory = DefaultURLSessionFactory(config: config)
             var awsV4Signature: AWSV4Signature?
@@ -60,6 +65,7 @@ public class XCPrepareMark {
                 awsV4Signature = AWSV4Signature(
                     secretKey: config.AWSSecretKey,
                     accessKey: config.AWSAccessKey,
+                    securityToken: config.AWSSecurityToken,
                     region: config.AWSRegion,
                     service: config.AWSService,
                     date: Date(timeIntervalSinceNow: 0)
@@ -68,6 +74,7 @@ public class XCPrepareMark {
             let networkClient = NetworkClientImpl(
                 session: sessionFactory.build(),
                 retries: config.uploadRetries,
+                retryDelay: config.retryDelay,
                 fileManager: fileManager,
                 awsV4Signature: awsV4Signature
             )
@@ -75,6 +82,7 @@ public class XCPrepareMark {
                 mode: .producer,
                 downloadStreamURL: context.recommendedCacheAddress,
                 upstreamStreamURL: context.cacheAddresses,
+                uploadBatchSize: config.uploadBatchSize,
                 networkClient: networkClient
             ) { [configuration, platform] cacheAddress in
                 // Prepare URLs don't include target name or envFingperint, which are valid only for a target level

Sources/XCRemoteCache/Commands/Prepare/XCStats.swift

@@ -18,7 +18,6 @@
 // under the License.
 
 import Foundation
-import Yams
 
 /// Manages XCRemoteCache statistics: rests, print to the standard output etc
 public class XCStats {
@@ -37,7 +36,7 @@ public class XCStats {
         let config: XCRemoteCacheConfig
         let context: XCStatsContext
         do {
-            config = try XCRemoteCacheConfigReader(env: env, fileManager: fileManager).readConfiguration()
+            config = try XCRemoteCacheConfigReader(env: env, fileReader: fileManager).readConfiguration()
             try context = XCStatsContext(config, fileManager: fileManager)
         } catch {
             exit(1, "FATAL: Prepare initialization failed with error: \(error)")

Sources/XCRemoteCache/Commands/ProductBinaryCreator/XCCreateBinary.swift

@@ -70,7 +70,7 @@ public class XCCreateBinary {
         let config: XCRemoteCacheConfig
         do {
             let srcRoot: URL = URL(fileURLWithPath: fileManager.currentDirectoryPath)
-            config = try XCRemoteCacheConfigReader(srcRootPath: srcRoot.path, fileManager: fileManager)
+            config = try XCRemoteCacheConfigReader(srcRootPath: srcRoot.path, fileReader: fileManager)
                 .readConfiguration()
         } catch {
             errorLog("\(stepDescription) initialization failed with error: \(error). Fallbacking to \(fallbackCommand)")
@@ -78,7 +78,12 @@ public class XCCreateBinary {
         }
         let markerURL = tempDir.appendingPathComponent(config.modeMarkerPath)
         do {
-            let organizer = ZipArtifactOrganizer(targetTempDir: tempDir, fileManager: fileManager)
+            let organizer = ZipArtifactOrganizer(
+                targetTempDir: tempDir,
+                // Creation binary doesn't call artifact preprocessing
+                artifactProcessors: [],
+                fileManager: fileManager
+            )
             let dependenciesWriter = FileDatWriter(dependencyInfo, fileManager: fileManager)
             let markerReader = FileMarkerReader(markerURL, fileManager: fileManager)
             guard fileManager.fileExists(atPath: markerURL.path) else {

Sources/XCRemoteCache/Commands/SwiftFrontend/SwiftFrontendArgInput.swift

@@ -0,0 +1,244 @@
+// Copyright (c) 2023 Spotify AB.
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import Foundation
+
+enum SwiftFrontendArgInputError: Error, Equatable {
+    // swift-frontend should either be compling or emiting a module
+    case bothCompilationAndEmitAction
+    // no .swift files have been passed as input files
+    case noCompilationInputs
+    // no -primary-file .swift files have been passed as input files
+    case noPrimaryFileCompilationInputs
+    // number of -emit-dependencies-path doesn't match compilation inputs
+    case dependenciesOuputCountDoesntMatch(expected: Int, parsed: Int)
+    // number of -serialize-diagnostics-path doesn't match compilation inputs
+    case diagnosticsOuputCountDoesntMatch(expected: Int, parsed: Int)
+    // number of -o doesn't match compilation inputs
+    case outputsOuputCountDoesntMatch(expected: Int, parsed: Int)
+    // number of -o for emit-module can be only 1
+    case emitModulOuputCountIsNot1(parsed: Int)
+    // number of -emit-dependencies-path for emit-module can be 0 or 1 (generate or not)
+    case emitModuleDependenciesOuputCountIsHigherThan1(parsed: Int)
+    // number of -serialize-diagnostics-path for emit-module can be 0 or 1 (generate or not)
+    case emitModuleDiagnosticsOuputCountIsHigherThan1(parsed: Int)
+    // emit-module requires -emit-objc-header-path
+    case emitModuleMissingObjcHeaderPath
+    // -target is required
+    case emitMissingTarget
+    // -moduleName is required
+    case emitMissingModuleName
+}
+
+public struct SwiftFrontendArgInput {
+    let compile: Bool
+    let emitModule: Bool
+    let objcHeaderOutput: String?
+    let moduleName: String?
+    let target: String?
+    let primaryInputPaths: [String]
+    let inputPaths: [String]
+    var outputPaths: [String]
+    var dependenciesPaths: [String]
+    // Extra params
+    // Diagnostics are not supported yet in the XCRemoteCache (cached artifacts assumes no warnings)
+    var diagnosticsPaths: [String]
+    // Unsed for now:
+    // .swiftsourceinfo and .swiftdoc will be placed next to the .swiftmodule
+    let sourceInfoPath: String?
+    let docPath: String?
+    // Passed as -supplementary-output-file-map
+    let supplementaryOutputFileMap: String?
+
+    /// Manual initializer implementation required to be public
+    public init(
+        compile: Bool,
+        emitModule: Bool,
+        objcHeaderOutput: String?,
+        moduleName: String?,
+        target: String?,
+        primaryInputPaths: [String],
+        inputPaths: [String],
+        outputPaths: [String],
+        dependenciesPaths: [String],
+        diagnosticsPaths: [String],
+        sourceInfoPath: String?,
+        docPath: String?,
+        supplementaryOutputFileMap: String?
+    ) {
+        self.compile = compile
+        self.emitModule = emitModule
+        self.objcHeaderOutput = objcHeaderOutput
+        self.moduleName = moduleName
+        self.target = target
+        self.primaryInputPaths = primaryInputPaths
+        self.inputPaths = inputPaths
+        self.outputPaths = outputPaths
+        self.dependenciesPaths = dependenciesPaths
+        self.diagnosticsPaths = diagnosticsPaths
+        self.sourceInfoPath = sourceInfoPath
+        self.docPath = docPath
+        self.supplementaryOutputFileMap = supplementaryOutputFileMap
+    }
+
+    private func generateForCompilation(
+        config: XCRemoteCacheConfig,
+        target: String,
+        moduleName: String
+    ) throws -> SwiftcContext {
+        let primaryInputsCount = primaryInputPaths.count
+
+        guard primaryInputsCount > 0 else {
+            throw SwiftFrontendArgInputError.noPrimaryFileCompilationInputs
+        }
+        guard [primaryInputsCount, 0].contains(dependenciesPaths.count) else {
+            throw SwiftFrontendArgInputError.dependenciesOuputCountDoesntMatch(
+                expected: primaryInputsCount,
+                parsed: dependenciesPaths.count
+            )
+        }
+        guard [primaryInputsCount, 0].contains(diagnosticsPaths.count) else {
+            throw SwiftFrontendArgInputError.diagnosticsOuputCountDoesntMatch(
+                expected: primaryInputsCount,
+                parsed: diagnosticsPaths.count
+            )
+        }
+        guard outputPaths.count == primaryInputsCount else {
+            throw SwiftFrontendArgInputError.outputsOuputCountDoesntMatch(
+                expected: primaryInputsCount,
+                parsed: outputPaths.count
+            )
+        }
+        let primaryInputFilesURLs: [URL] = primaryInputPaths.map(URL.init(fileURLWithPath:))
+
+        let steps: SwiftcContext.SwiftcSteps = SwiftcContext.SwiftcSteps(
+            compileFilesScope: .subset(primaryInputFilesURLs),
+            emitModule: nil
+        )
+
+        let compilationFilesInputs = buildCompilationFilesInputs(
+            primaryInputsCount: primaryInputsCount,
+            primaryInputFilesURLs: primaryInputFilesURLs
+        )
+
+        return try .init(
+            config: config,
+            moduleName: moduleName,
+            steps: steps,
+            inputs: compilationFilesInputs,
+            target: target,
+            compilationFiles: .list(inputPaths),
+            exampleWorkspaceFilePath: outputPaths[0]
+        )
+    }
+
+    private func buildCompilationFilesInputs(
+        primaryInputsCount: Int,
+        primaryInputFilesURLs: [URL]
+    ) -> SwiftcContext.CompilationFilesInputs {
+        if let compimentaryFileMa = supplementaryOutputFileMap {
+            return .supplementaryFileMap(compimentaryFileMa)
+        } else {
+            return .map((0..<primaryInputsCount).reduce(
+                [String: SwiftFileCompilationInfo]()
+            ) { prev, i in
+                var new = prev
+                new[primaryInputPaths[i]] = SwiftFileCompilationInfo(
+                    file: primaryInputFilesURLs[i],
+                    dependencies: dependenciesPaths.get(i).map(URL.init(fileURLWithPath:)),
+                    object: outputPaths.get(i).map(URL.init(fileURLWithPath:)),
+                    // for now - swift-dependencies are not requested in the driver compilation mode
+                    swiftDependencies: nil
+                )
+                return new
+            })
+        }
+    }
+
+    private func generateForEmitModule(
+        config: XCRemoteCacheConfig,
+        target: String,
+        moduleName: String
+    ) throws -> SwiftcContext {
+        guard outputPaths.count == 1 else {
+            throw SwiftFrontendArgInputError.emitModulOuputCountIsNot1(parsed: outputPaths.count)
+        }
+        guard let objcHeaderOutput = objcHeaderOutput else {
+            throw SwiftFrontendArgInputError.emitModuleMissingObjcHeaderPath
+        }
+        guard diagnosticsPaths.count <= 1 else {
+            throw SwiftFrontendArgInputError.emitModuleDiagnosticsOuputCountIsHigherThan1(
+                parsed: diagnosticsPaths.count
+            )
+        }
+        guard dependenciesPaths.count <= 1 else {
+            throw SwiftFrontendArgInputError.emitModuleDependenciesOuputCountIsHigherThan1(
+                parsed: dependenciesPaths.count
+            )
+        }
+
+        let steps: SwiftcContext.SwiftcSteps = SwiftcContext.SwiftcSteps(
+            compileFilesScope: .none,
+            emitModule: SwiftcContext.SwiftcStepEmitModule(
+                objcHeaderOutput: URL(fileURLWithPath: objcHeaderOutput),
+                modulePathOutput: URL(fileURLWithPath: outputPaths[0]),
+                dependencies: dependenciesPaths.first.map(URL.init(fileURLWithPath:))
+            )
+        )
+        return try .init(
+            config: config,
+            moduleName: moduleName,
+            steps: steps,
+            inputs: .map([:]),
+            target: target,
+            compilationFiles: .list(inputPaths),
+            exampleWorkspaceFilePath: objcHeaderOutput
+        )
+    }
+
+    func generateSwiftcContext(config: XCRemoteCacheConfig) throws -> SwiftcContext {
+        guard compile != emitModule else {
+            throw SwiftFrontendArgInputError.bothCompilationAndEmitAction
+        }
+        let inputPathsCount = inputPaths.count
+        guard inputPathsCount > 0 else {
+            throw SwiftFrontendArgInputError.noCompilationInputs
+        }
+        guard let target = target else {
+            throw SwiftFrontendArgInputError.emitMissingTarget
+        }
+        guard let moduleName = moduleName else {
+            throw SwiftFrontendArgInputError.emitMissingModuleName
+        }
+
+        if compile {
+            return try generateForCompilation(
+                config: config,
+                target: target,
+                moduleName: moduleName
+            )
+        } else {
+            return try generateForEmitModule(
+                config: config,
+                target: target,
+                moduleName: moduleName
+            )
+        }
+    }
+}

Sources/XCRemoteCache/Commands/SwiftFrontend/SwiftFrontendContext.swift

@@ -0,0 +1,42 @@
+// Copyright (c) 2023 Spotify AB.
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import Foundation
+
+struct SwiftFrontendContext {
+    /// File lock used for synchronizing multiple invocations
+    let invocationLockFile: URL
+}
+
+extension SwiftFrontendContext {
+    init(_ swiftcContext: SwiftcContext, env: [String: String]) throws {
+        /// The LLBUILD_BUILD_ID ENV that describes the entire (incl. parent's swiftc) bui;d
+        let llbuildId: String = try env.readEnv(key: "LLBUILD_BUILD_ID")
+        invocationLockFile = Self.self.buildLlbuildIdSharedLockUrl(
+            llbuildId: llbuildId,
+            tmpDir: swiftcContext.tempDir
+        )
+    }
+
+    /// Generate the filename to be used to synchronize multiple swift-frontend invocations
+    /// The same file is used in prebuild, xcswift-frontend and postbuild (to clean it up)
+    static func buildLlbuildIdSharedLockUrl(llbuildId: String, tmpDir: URL) -> URL {
+        return tmpDir.appendingPathComponent(llbuildId).appendingPathExtension("lock")
+    }
+}

Sources/XCRemoteCache/Commands/SwiftFrontend/SwiftFrontendOrchestrator.swift

@@ -0,0 +1,136 @@
+// Copyright (c) 2023 Spotify AB.
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import Foundation
+
+/// Manages  the `swift-frontend` logic
+protocol SwiftFrontendOrchestrator {
+    /// Executes the critical section according to the required order
+    /// - Parameter criticalSection: the block that should be synchronized
+    func run(criticalSection: () -> Void ) throws
+}
+
+/// The default orchestrator that manages the order or swift-frontend invocations
+/// For emit-module (the "first" process) action, it locks a shared file between all swift-frontend invocations,
+/// verifies that the mocking can be done and continues the mocking/fall-backing along the lock release
+/// For the compilation action, tries to acquire a lock and waits until the "emit-module" makes a decision
+/// if the compilation should be skipped and a "mocking" should used instead
+class CommonSwiftFrontendOrchestrator {
+    /// Content saved to the shared file
+    /// Safe to use forced unwrapping
+    private static let emitModuleContent = "done".data(using: .utf8)!
+
+    enum Action {
+        case emitModule
+        case compile
+    }
+    private let mode: SwiftcContext.SwiftcMode
+    private let action: Action
+    private let lockAccessor: ExclusiveFileAccessor
+    private let maxLockTimeout: TimeInterval
+
+    init(
+        mode: SwiftcContext.SwiftcMode,
+        action: Action,
+        lockAccessor: ExclusiveFileAccessor,
+        maxLockTimeout: TimeInterval
+    ) {
+        self.mode = mode
+        self.action = action
+        self.lockAccessor = lockAccessor
+        self.maxLockTimeout = maxLockTimeout
+    }
+
+    func run(criticalSection: () throws -> Void) throws {
+        guard case .consumer(commit: .available) = mode else {
+            // no need to lock anything - just allow fallbacking to the `swiftc or swift-frontend`
+            // for a producer or a consumer where RC is disabled (we have already caught the
+            // cache miss)
+            try criticalSection()
+            return
+        }
+        try executeMockAttemp(criticalSection: criticalSection)
+    }
+
+    private func executeMockAttemp(criticalSection: () throws -> Void) throws {
+        switch action {
+        case .emitModule:
+            try validateEmitModuleStep(criticalSection: criticalSection)
+        case .compile:
+            try waitForEmitModuleLock(criticalSection: criticalSection)
+        }
+    }
+
+
+    /// For emit-module, wrap the critical section with the shared lock so other processes (compilation)
+    /// have to wait until emit-module finishes
+    /// Once the emit-module is done, the "magical" string is saved to the file and the lock is released
+    ///
+    /// Note: The design of wrapping the entire "emit-module" has a small performance downside if inside
+    /// the critical section, the code realizes that remote cache cannot be used
+    /// (in practice - a new file has been added)
+    /// None of compilation process (so with '-c' args) can continue until the entire emit-module logic finishes
+    /// Because it is expected to happen not that often and emit-module is usually quite fast, this makes the
+    /// implementation way simpler. If we ever want to optimize it, we should release the lock as early
+    /// as we know, the remote cache cannot be used. Then all other compilation process (-c) can run
+    /// in parallel with emit-module
+    private func validateEmitModuleStep(criticalSection: () throws -> Void) throws {
+        debugLog("starting the emit-module step: locking")
+        try lockAccessor.exclusiveAccess { handle in
+            debugLog("starting the emit-module step: locked")
+            // writing to the file content proactively - incase the critical section never returns
+            // (in case of a fallback to the local compilation), all awaiting swift-frontend processes
+            // will be immediately unblocked
+            handle.write(Self.self.emitModuleContent)
+            try criticalSection()
+            debugLog("lock file emit-module criticial end")
+        }
+    }
+
+    /// Locks a shared file in a loop until its content is non-empty - meaning the "parent" emit-module
+    /// has already finished
+    private func waitForEmitModuleLock(criticalSection: () throws -> Void) throws {
+        // emit-module process should really quickly obtain a lock (it is always invoked
+        // by Xcode as a first process)
+        var executed = false
+        let startingDate = Date()
+        while !executed {
+            debugLog("lock file compilation trying to acquire a lock ....")
+            try lockAccessor.exclusiveAccess { handle in
+                if !handle.availableData.isEmpty {
+                    // the file is not empty so the emit-module process is done with the "check"
+                    debugLog("swift-frontend lock file is unlocked for compilation")
+                    try criticalSection()
+                    executed = true
+                } else {
+                    debugLog("swift-frontend lock file is not ready for compilation")
+                }
+            }
+            // When a max locking time is achieved, execute anyway
+            if !executed && Date().timeIntervalSince(startingDate) > self.maxLockTimeout {
+                errorLog("""
+                Executing command \(action) without lock synchronization. That may be cause by the\
+                crashed or extremely long emit-module. Contact XCRemoteCache authors about this error.
+                """)
+                try criticalSection()
+                executed = true
+            }
+        }
+    }
+}

Sources/XCRemoteCache/Commands/SwiftFrontend/XCSwiftFrontend.swift

@@ -0,0 +1,80 @@
+// Copyright (c) 2023 Spotify AB.
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import Foundation
+
+public class XCSwiftFrontend: XCSwiftAbstract<SwiftFrontendArgInput> {
+    // don't lock individual compilation invocations for more than 10s
+    private static let MaxLockingTimeout: TimeInterval = 10
+    private let env: [String: String]
+
+    public init(
+        command: String,
+        inputArgs: SwiftFrontendArgInput,
+        env: [String: String],
+        dependenciesWriter: @escaping (URL, FileManager) -> DependenciesWriter,
+        touchFactory: @escaping (URL, FileManager) -> Touch
+    ) throws {
+        self.env = env
+        super.init(
+            command: command,
+            inputArgs: inputArgs,
+            dependenciesWriter: dependenciesWriter,
+            touchFactory: touchFactory
+        )
+    }
+
+    override func buildContext() throws -> (XCRemoteCacheConfig, SwiftcContext) {
+        let fileManager = FileManager.default
+        let config: XCRemoteCacheConfig
+        let context: SwiftcContext
+
+        let srcRoot: URL = URL(fileURLWithPath: fileManager.currentDirectoryPath)
+        config = try XCRemoteCacheConfigReader(srcRootPath: srcRoot.path, fileReader: fileManager)
+            .readConfiguration()
+        context = try SwiftcContext(config: config, input: inputArgs)
+        // do not cache this context, as it is subject to change when
+        // the emit-module finds that the cached artifact cannot be used
+        return (config, context)
+    }
+
+    override public func run() throws {
+        do {
+            let (_, context) = try buildContext()
+
+            let frontendContext = try SwiftFrontendContext(context, env: env)
+            let sharedLock = ExclusiveFile(frontendContext.invocationLockFile, mode: .override)
+
+            let action: CommonSwiftFrontendOrchestrator.Action = inputArgs.emitModule ? .emitModule : .compile
+            let swiftFrontendOrchestrator = CommonSwiftFrontendOrchestrator(
+                mode: context.mode,
+                action: action,
+                lockAccessor: sharedLock,
+                maxLockTimeout: Self.self.MaxLockingTimeout
+            )
+
+            try swiftFrontendOrchestrator.run(criticalSection: super.run)
+        } catch {
+            // Splitting into 2 invocations as os_log truncates a massage
+            defaultLog("Cannot correctly orchestrate the \(command) with params \(inputArgs)")
+            defaultLog("Cannot correctly orchestrate error: \(error)")
+            throw error
+        }
+    }
+}

Sources/XCRemoteCache/Commands/Swiftc/FilenameBasedAllowedInputDeterminer.swift

@@ -0,0 +1,38 @@
+// Copyright (c) 2023 Spotify AB.
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import Foundation
+
+/// Decides if an input to the compilation step should allow reusing the cached artifact
+protocol AllowedInputDeterminer {
+    /// Decides if the input file is allowed to be compiled, even not specified in the dependency list
+    func allowedNonDependencyInput(file: URL) -> Bool
+}
+
+class FilenameBasedAllowedInputDeterminer: AllowedInputDeterminer {
+    private let filenames: [String]
+
+    init(_ filenames: [String]) {
+        self.filenames = filenames
+    }
+
+    func allowedNonDependencyInput(file: URL) -> Bool {
+        return filenames.contains(file.lastPathComponent)
+    }
+}

Sources/XCRemoteCache/Commands/Swiftc/MirroredLinkingSwiftcProductsGenerator.swift

@@ -55,7 +55,7 @@ class MirroredLinkingSwiftcProductsGenerator: SwiftcProductsGenerator {
     func generateFrom(
         artifactSwiftModuleFiles: [SwiftmoduleFileExtension: URL],
         artifactSwiftModuleObjCFile: URL
-    ) throws -> URL {
+    ) throws -> SwiftcProductsGeneratorOutput {
         /// Predict moduleName from the `*.swiftmodule` artifact
         let foundSwiftmoduleFile = artifactSwiftModuleFiles[.swiftmodule]
         guard let mainSwiftmoduleFile = foundSwiftmoduleFile else {

Sources/XCRemoteCache/Commands/Swiftc/NoopSwiftcProductsGenerator.swift

@@ -0,0 +1,38 @@
+// Copyright (c) 2023 Spotify AB.
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import Foundation
+
+/// Products generator that doesn't create any swiftmodule. It is used in the compilation swift-frontend mocking, where
+/// only individual .o files are created and not .swiftmodule of -Swift.h
+/// (which is part of swift-frontend -emit-module invocation)
+class NoopSwiftcProductsGenerator: SwiftcProductsGenerator {
+    func generateFrom(
+        artifactSwiftModuleFiles: [SwiftmoduleFileExtension: URL],
+        artifactSwiftModuleObjCFile: URL
+    ) throws -> SwiftcProductsGeneratorOutput {
+        infoLog("""
+        Invoking module generation from NoopSwiftcProductsGenerator does nothing. \
+        It might be a side-effect of a plugin asking to generate a module.
+        """)
+        // NoopSwiftcProductsGenerator is intended only for the swift-frontend
+        let trivialURL = URL(fileURLWithPath: "/non-existing")
+        return SwiftcProductsGeneratorOutput(swiftmoduleDir: trivialURL, objcHeaderFile: trivialURL)
+    }
+}

Sources/XCRemoteCache/Commands/Swiftc/StaticSwiftcInputReader.swift

@@ -0,0 +1,46 @@
+// Copyright (c) 2023 Spotify AB.
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import Foundation
+
+class StaticSwiftcInputReader: SwiftcInputReader {
+    private let moduleDependencies: URL?
+    private let swiftDependencies: URL?
+    private let compilationFiles: [SwiftFileCompilationInfo]
+
+    init(
+        moduleDependencies: URL?,
+        swiftDependencies: URL?,
+        compilationFiles: [SwiftFileCompilationInfo]
+    ) {
+        self.moduleDependencies = moduleDependencies
+        self.swiftDependencies = swiftDependencies
+        self.compilationFiles = compilationFiles
+    }
+
+    func read() throws -> SwiftCompilationInfo {
+        return .init(
+            info: .init(
+                dependencies: moduleDependencies,
+                swiftDependencies: swiftDependencies
+            ),
+            files: compilationFiles
+        )
+    }
+}

Sources/XCRemoteCache/Commands/Swiftc/Swiftc.swift

@@ -57,6 +57,7 @@ class Swiftc: SwiftcProtocol {
     private let dependenciesWriterFactory: (URL, FileManager) -> DependenciesWriter
     private let touchFactory: (URL, FileManager) -> Touch
     private let plugins: [SwiftcProductGenerationPlugin]
+    private let allowedInputDeterminer: AllowedInputDeterminer
 
     init(
         inputFileListReader: ListReader,
@@ -70,7 +71,8 @@ class Swiftc: SwiftcProtocol {
         fileManager: FileManager,
         dependenciesWriterFactory: @escaping (URL, FileManager) -> DependenciesWriter,
         touchFactory: @escaping (URL, FileManager) -> Touch,
-        plugins: [SwiftcProductGenerationPlugin]
+        plugins: [SwiftcProductGenerationPlugin],
+        allowedInputDeterminer: AllowedInputDeterminer
     ) {
         self.inputFileListReader = inputFileListReader
         self.markerReader = markerReader
@@ -84,6 +86,7 @@ class Swiftc: SwiftcProtocol {
         self.dependenciesWriterFactory = dependenciesWriterFactory
         self.touchFactory = touchFactory
         self.plugins = plugins
+        self.allowedInputDeterminer = allowedInputDeterminer
     }
 
     // swiftlint:disable:next function_body_length
@@ -96,13 +99,17 @@ class Swiftc: SwiftcProtocol {
 
         let inputFilesInputs = try inputFileListReader.listFilesURLs()
         let markerAllowedFiles = try markerReader.listFilesURLs()
+        let allDependencies = Set(markerAllowedFiles + inputFilesInputs)
         let cachedDependenciesWriterFactory = CachedFileDependenciesWriterFactory(
-            dependencies: markerAllowedFiles,
+            dependencies: Array(allDependencies),
             fileManager: fileManager,
             writerFactory: dependenciesWriterFactory
         )
         // Verify all input files to be present in a marker fileList
-        let disallowedInputs = try inputFilesInputs.filter { try !allowedFilesListScanner.contains($0) }
+        let disallowedInputs = try inputFilesInputs.filter { file in
+            try !allowedFilesListScanner.contains(file) &&
+            !allowedInputDeterminer.allowedNonDependencyInput(file: file)
+        }
 
         if !disallowedInputs.isEmpty {
             // New file (disallowedFile) added without modifying the rest of the feature. Fallback to swiftc and
@@ -132,7 +139,7 @@ class Swiftc: SwiftcProtocol {
 
         // Read swiftmodule location from XCRemoteCache
         // arbitrary format swiftmodule/${arch}/${moduleName}.swift{module|doc|sourceinfo}
-        let moduleName = context.modulePathOutput.deletingPathExtension().lastPathComponent
+        let moduleName = context.moduleName
         let allCompilations = try inputFilesReader.read()
         let artifactSwiftmoduleDir = artifactLocation
             .appendingPathComponent("swiftmodule")
@@ -145,20 +152,24 @@ class Swiftc: SwiftcProtocol {
                 }
         )
 
-        // Build -Swift.h location from XCRemoteCache arbitrary format include/${arch}/${target}-Swift.h
-        let artifactSwiftModuleObjCDir = artifactLocation
-            .appendingPathComponent("include")
-            .appendingPathComponent(context.arch)
-            .appendingPathComponent(context.moduleName)
-        // Move cached xxxx-Swift.h to the location passed in arglist
-        // Alternatively, artifactSwiftModuleObjCFile could be built as a first .h file in artifactSwiftModuleObjCDir
-        let artifactSwiftModuleObjCFile = artifactSwiftModuleObjCDir
-            .appendingPathComponent(context.objcHeaderOutput.lastPathComponent)
+        // emit module (if requested)
+        if let emitModule = context.steps.emitModule {
+            // Build -Swift.h location from XCRemoteCache arbitrary format include/${arch}/${target}-Swift.h
+            let artifactSwiftModuleObjCDir = artifactLocation
+                .appendingPathComponent("include")
+                .appendingPathComponent(context.arch)
+                .appendingPathComponent(context.moduleName)
+            // Move cached xxxx-Swift.h to the location passed in arglist
+            // Alternatively, artifactSwiftModuleObjCFile could be built as a first .h
+            // file in artifactSwiftModuleObjCDir
+            let artifactSwiftModuleObjCFile = artifactSwiftModuleObjCDir
+                .appendingPathComponent(emitModule.objcHeaderOutput.lastPathComponent)
 
-        _ = try productsGenerator.generateFrom(
-            artifactSwiftModuleFiles: artifactSwiftmoduleFiles,
-            artifactSwiftModuleObjCFile: artifactSwiftModuleObjCFile
-        )
+            _ = try productsGenerator.generateFrom(
+                artifactSwiftModuleFiles: artifactSwiftmoduleFiles,
+                artifactSwiftModuleObjCFile: artifactSwiftModuleObjCFile
+            )
+        }
 
         try plugins.forEach {
             try $0.generate(for: allCompilations)
@@ -176,8 +187,10 @@ class Swiftc: SwiftcProtocol {
                 try cachedDependenciesWriterFactory.generate(output: individualDeps)
             }
         }
-        // Save .d for the entire module
-        try cachedDependenciesWriterFactory.generate(output: allCompilations.info.swiftDependencies)
+        // Save .d for the entire module (might not be required in the `swift-frontend -c` mode)
+        if let swiftDependencies = allCompilations.info.swiftDependencies {
+            try cachedDependenciesWriterFactory.generate(output: swiftDependencies)
+        }
         // Generate .d file with all deps in the "-master.d" (e.g. for WMO)
         if let wmoDeps = allCompilations.info.dependencies {
             try cachedDependenciesWriterFactory.generate(output: wmoDeps)

Sources/XCRemoteCache/Commands/Swiftc/SwiftcContext.swift

@@ -20,6 +20,53 @@
 import Foundation
 
 public struct SwiftcContext {
+    /// Describes the action if the module emit should happen
+    /// that generates .swiftmodule and/or -Swift.h
+    public struct SwiftcStepEmitModule: Equatable {
+        // where the -Swift.h should be placed
+        let objcHeaderOutput: URL
+        // where should the .swiftmodule be placed
+        let modulePathOutput: URL
+        // might be passed as an explicit argument in the swiftc
+        // -emit-dependencies-path
+        let dependencies: URL?
+    }
+
+    /// Which files (from the list of all files in the module)
+    /// should be compiled in this process
+    public enum SwiftcStepCompileFilesScope: Equatable {
+        /// used if only emit module should be done
+        case none
+        case all
+        case subset([URL])
+    }
+
+    /// Describes which steps should be done as a part of this process
+    public struct SwiftcSteps: Equatable {
+        /// which files should be compiled
+        let compileFilesScope: SwiftcStepCompileFilesScope
+        /// if a module should be generated
+        let emitModule: SwiftcStepEmitModule?
+    }
+
+    /// Defines how a list of input files (*.swift) is passed to the invocation
+    public enum CompilationFilesSource: Equatable {
+        /// defined in a separate file (via @/.../*.SwiftFileList)
+        case fileList(String)
+        /// explicitly passed a list of files
+        case list([String])
+    }
+
+    /// Defines how a list of output files (*.d, *.o etc.) is passed to the invocation
+    public enum CompilationFilesInputs: Equatable {
+        /// defined in a separate file (via -output-file-map)
+        case fileMap(String)
+        /// defined in a separate file (via -supplementary-output-file-map)
+        case supplementaryFileMap(String)
+        /// explicitly passed in the invocation
+        case map([String: SwiftFileCompilationInfo])
+    }
+
     enum SwiftcMode: Equatable {
         case producer
         /// Commit sha of the commit to use during remote cache
@@ -28,14 +75,13 @@ public struct SwiftcContext {
         case producerFast
     }
 
-    let objcHeaderOutput: URL
+    let steps: SwiftcSteps
     let moduleName: String
-    let modulePathOutput: URL
-    /// File that defines output files locations (.d, .swiftmodule etc.)
-    let filemap: URL
+    /// A source that defines output files locations (.d, .swiftmodule etc.)
+    let inputs: CompilationFilesInputs
     let target: String
-    /// File that contains input files for the swift module compilation
-    let fileList: URL
+    /// A source that contains all input files for the swift module compilation
+    let compilationFiles: CompilationFilesSource
     let tempDir: URL
     let arch: String
     let prebuildDependenciesPath: String
@@ -43,29 +89,29 @@ public struct SwiftcContext {
     /// File that stores all compilation invocation arguments
     let invocationHistoryFile: URL
 
-
     public init(
         config: XCRemoteCacheConfig,
-        objcHeaderOutput: String,
         moduleName: String,
-        modulePathOutput: String,
-        filemap: String,
+        steps: SwiftcSteps,
+        inputs: CompilationFilesInputs,
         target: String,
-        fileList: String
+        compilationFiles: CompilationFilesSource,
+        /// any workspace file path - all other intermediate files for this compilation
+        /// are placed next to it. This path is used to infer the arch and TARGET_TEMP_DIR
+        exampleWorkspaceFilePath: String
     ) throws {
-        self.objcHeaderOutput = URL(fileURLWithPath: objcHeaderOutput)
         self.moduleName = moduleName
-        self.modulePathOutput = URL(fileURLWithPath: modulePathOutput)
-        self.filemap = URL(fileURLWithPath: filemap)
+        self.steps = steps
+        self.inputs = inputs
         self.target = target
-        self.fileList = URL(fileURLWithPath: fileList)
-        // modulePathOutput is place in $TARGET_TEMP_DIR/Objects-normal/$ARCH/$TARGET_NAME.swiftmodule
+        self.compilationFiles = compilationFiles
+        // exampleWorkspaceFilePath has a format $TARGET_TEMP_DIR/Objects-normal/$ARCH/some.file
         // That may be subject to change for other Xcode versions
-        tempDir = URL(fileURLWithPath: modulePathOutput)
+        tempDir = URL(fileURLWithPath: exampleWorkspaceFilePath)
             .deletingLastPathComponent()
             .deletingLastPathComponent()
             .deletingLastPathComponent()
-        arch = URL(fileURLWithPath: modulePathOutput).deletingLastPathComponent().lastPathComponent
+        arch = URL(fileURLWithPath: exampleWorkspaceFilePath).deletingLastPathComponent().lastPathComponent
 
         let srcRoot: URL = URL(fileURLWithPath: config.sourceRoot)
         let remoteCommitLocation = URL(fileURLWithPath: config.remoteCommitFile, relativeTo: srcRoot)
@@ -92,14 +138,32 @@ public struct SwiftcContext {
         config: XCRemoteCacheConfig,
         input: SwiftcArgInput
     ) throws {
+        let steps = SwiftcSteps(
+            compileFilesScope: .all,
+            emitModule: SwiftcStepEmitModule(
+                objcHeaderOutput: URL(fileURLWithPath: (input.objcHeaderOutput)),
+                modulePathOutput: URL(fileURLWithPath: input.modulePathOutput),
+                // in `swiftc`, .d dependencies are pass in the output filemap
+                dependencies: nil
+            )
+        )
+        let inputs = CompilationFilesInputs.fileMap(input.filemap)
+        let compilationFiles = CompilationFilesSource.fileList(input.fileList)
         try self.init(
             config: config,
-            objcHeaderOutput: input.objcHeaderOutput,
             moduleName: input.moduleName,
-            modulePathOutput: input.modulePathOutput,
-            filemap: input.filemap,
+            steps: steps,
+            inputs: inputs,
             target: input.target,
-            fileList: input.fileList
+            compilationFiles: compilationFiles,
+            exampleWorkspaceFilePath: input.modulePathOutput
         )
     }
+
+    init(
+        config: XCRemoteCacheConfig,
+        input: SwiftFrontendArgInput
+    ) throws {
+        self = try input.generateSwiftcContext(config: config)
+    }
 }

Sources/XCRemoteCache/Commands/Swiftc/SwiftcFilemapInputEditor.swift

@@ -18,11 +18,16 @@
 // under the License.
 
 import Foundation
+import Yams
 
 /// Errors with reading swiftc inputs
 enum SwiftcInputReaderError: Error {
     case readingFailed
     case invalidFormat
+    /// The file is not in the yaml format
+    case invalidYamlFormat
+    /// The yaml string contains illegal characters
+    case invalidYamlString
     case missingField(String)
 }
 
@@ -45,10 +50,11 @@ struct SwiftCompilationInfo: Encodable, Equatable {
 struct SwiftModuleCompilationInfo: Encodable, Equatable {
     // not present for incremental builds
     let dependencies: URL?
-    let swiftDependencies: URL
+    // might be nil for the swift-frontend '-c' invocation
+    let swiftDependencies: URL?
 }
 
-struct SwiftFileCompilationInfo: Encodable, Equatable {
+public struct SwiftFileCompilationInfo: Encodable, Hashable {
     let file: URL
     // not present for WMO builds
     let dependencies: URL?
@@ -60,11 +66,18 @@ struct SwiftFileCompilationInfo: Encodable, Equatable {
 
 class SwiftcFilemapInputEditor: SwiftcInputReader, SwiftcInputWriter {
 
+    enum Format {
+        case json
+        case yaml
+    }
+
     private let file: URL
+    private let fileFormat: Format
     private let fileManager: FileManager
 
-    init(_ file: URL, fileManager: FileManager) {
+    init(_ file: URL, fileFormat: Format, fileManager: FileManager) {
         self.file = file
+        self.fileFormat = fileFormat
         self.fileManager = fileManager
     }
 
@@ -72,7 +85,7 @@ class SwiftcFilemapInputEditor: SwiftcInputReader, SwiftcInputWriter {
         guard let content = fileManager.contents(atPath: file.path) else {
             throw SwiftcInputReaderError.readingFailed
         }
-        guard let representation = try JSONSerialization.jsonObject(with: content, options: []) as? [String: Any] else {
+        guard let representation = try decodeFile(content: content) else {
             throw SwiftcInputReaderError.invalidFormat
         }
         return try SwiftCompilationInfo(from: representation)
@@ -82,11 +95,23 @@ class SwiftcFilemapInputEditor: SwiftcInputReader, SwiftcInputWriter {
         let data = try JSONSerialization.data(withJSONObject: info.dump(), options: [.prettyPrinted])
         fileManager.createFile(atPath: file.path, contents: data, attributes: nil)
     }
+
+    private func decodeFile(content: Data) throws -> [String: Any]? {
+        switch fileFormat {
+        case .json:
+            return try JSONSerialization.jsonObject(with: content, options: []) as? [String: Any]
+        case .yaml:
+            guard let stringContent = String(data: content, encoding: .utf8) else {
+                throw SwiftcInputReaderError.invalidYamlString
+            }
+            return try Yams.load(yaml: stringContent) as? [String: Any]
+        }
+    }
 }
 
 extension SwiftCompilationInfo {
     init(from object: [String: Any]) throws {
-        info = try SwiftModuleCompilationInfo(from: object[""])
+        info = try SwiftModuleCompilationInfo(from: object["", default: [:]])
         files = try object.reduce([]) { prev, new in
             let (key, value) = new
             if key.isEmpty {
@@ -111,14 +136,14 @@ extension SwiftModuleCompilationInfo {
         guard let dict = object as? [String: String] else {
             throw SwiftcInputReaderError.invalidFormat
         }
-        swiftDependencies = try dict.readURL(key: "swift-dependencies")
+        swiftDependencies = dict.readURL(key: "swift-dependencies")
         dependencies = dict.readURL(key: "dependencies")
     }
 
     func dump() -> [String: String] {
         return [
             "dependencies": dependencies?.path,
-            "swift-dependencies": swiftDependencies.path,
+            "swift-dependencies": swiftDependencies?.path,
         ].compactMapValues { $0 }
     }
 }

Sources/XCRemoteCache/Commands/Swiftc/SwiftcOrchestrator.swift

@@ -27,8 +27,10 @@ class SwiftcOrchestrator {
     private let mode: SwiftcContext.SwiftcMode
     // swiftc command that should be called to generate artifacts
     private let swiftcCommand: String
-    private let objcHeaderOutput: URL
-    private let moduleOutput: URL
+    // Might be nil if invoking from frontend compilation: `swift-frontend -c`
+    private let objcHeaderOutput: URL?
+    // Might be nil if invoking from frontend compilation: `swift-frontend -c`
+    private let moduleOutput: URL?
     private let arch: String
     private let artifactBuilder: ArtifactSwiftProductsBuilder
     private let shellOut: ShellOut
@@ -39,8 +41,8 @@ class SwiftcOrchestrator {
         mode: SwiftcContext.SwiftcMode,
         swiftc: SwiftcProtocol,
         swiftcCommand: String,
-        objcHeaderOutput: URL,
-        moduleOutput: URL,
+        objcHeaderOutput: URL?,
+        moduleOutput: URL?,
         arch: String,
         artifactBuilder: ArtifactSwiftProductsBuilder,
         producerFallbackCommandProcessors: [ShellCommandsProcessor],
@@ -110,8 +112,8 @@ class SwiftcOrchestrator {
                     try invocationStorage.store(args: invocationArgs)
                 }
             } catch {
-                // The critical section is protected by a lock. Some other process already called compilation history.
-                // We only need to call our current step then.
+                // The critical section is protected by a lock. Some other process already called compilation history
+                // We only need to call our current step then
                 fallbackToDefault(command: swiftcCommand)
             }
         case .consumer:
@@ -128,10 +130,14 @@ class SwiftcOrchestrator {
                 try processor.applyArgsRewrite(args)
             }
             try fallbackToDefaultAndWait(command: swiftcCommand, args: swiftcArgs)
-            // move generated .h to the location where artifact creator expects it
-            try artifactBuilder.includeObjCHeaderToTheArtifact(arch: arch, headerURL: objcHeaderOutput)
-            // move generated .swiftmodule to the location where artifact creator expects it
-            try artifactBuilder.includeModuleDefinitionsToTheArtifact(arch: arch, moduleURL: moduleOutput)
+            if let objcHeaderOutput = objcHeaderOutput {
+                // move generated .h to the location where artifact creator expects it
+                try artifactBuilder.includeObjCHeaderToTheArtifact(arch: arch, headerURL: objcHeaderOutput)
+            }
+            if let moduleOutput = moduleOutput {
+                // move generated .swiftmodule to the location where artifact creator expects it
+                try artifactBuilder.includeModuleDefinitionsToTheArtifact(arch: arch, moduleURL: moduleOutput)
+            }
 
             try producerFallbackCommandProcessors.forEach {
                 try $0.postCommandProcessing()

Sources/XCRemoteCache/Commands/Swiftc/SwiftcProductsGenerator.swift

@@ -20,20 +20,25 @@
 import Foundation
 
 enum DiskSwiftcProductsGeneratorError: Error {
-    /// When a generator was asked to generate unknown swiftmodule extension file.
+    /// When a generator was asked to generate unknown swiftmodule extension file
     /// Probably a programmer error: asking to generate excessive extensions, not listed in
     /// `SwiftmoduleFileExtension.SwiftmoduleExtensions`
     case unknownSwiftmoduleFile
 }
 
+struct SwiftcProductsGeneratorOutput {
+    let swiftmoduleDir: URL
+    let objcHeaderFile: URL
+}
+
 /// Generates swiftc product to the expected location
 protocol SwiftcProductsGenerator {
     /// Generates products from given files
-    /// - Returns: location dir where .swiftmodule files have been placed
+    /// - Returns: location dir where .swiftmodule and ObjC header files have been placed
     func generateFrom(
         artifactSwiftModuleFiles: [SwiftmoduleFileExtension: URL],
         artifactSwiftModuleObjCFile: URL
-    ) throws -> URL
+    ) throws -> SwiftcProductsGeneratorOutput
 }
 
 /// Generator that produces all products in the locations where Xcode expects it, using provided disk copier
@@ -64,7 +69,7 @@ class DiskSwiftcProductsGenerator: SwiftcProductsGenerator {
     func generateFrom(
         artifactSwiftModuleFiles sourceAtifactSwiftModuleFiles: [SwiftmoduleFileExtension: URL],
         artifactSwiftModuleObjCFile: URL
-    ) throws -> URL {
+    ) throws -> SwiftcProductsGeneratorOutput {
         // Move cached -Swift.h file to the expected location
         try diskCopier.copy(file: artifactSwiftModuleObjCFile, destination: objcHeaderOutput)
         for (ext, url) in sourceAtifactSwiftModuleFiles {
@@ -85,6 +90,9 @@ class DiskSwiftcProductsGenerator: SwiftcProductsGenerator {
         }
 
         // Build parent dir of the .swiftmodule file that contains a module
-        return modulePathOutput.deletingLastPathComponent()
+        return SwiftcProductsGeneratorOutput(
+            swiftmoduleDir: modulePathOutput.deletingLastPathComponent(),
+            objcHeaderFile: objcHeaderOutput
+        )
     }
 }

Sources/XCRemoteCache/Commands/Swiftc/XCSwiftc.swift

@@ -45,15 +45,15 @@ public struct SwiftcArgInput {
     }
 }
 
-public class XCSwiftc {
-    private let command: String
-    private let inputArgs: SwiftcArgInput
+public class XCSwiftAbstract<InputArgs> {
+    let command: String
+    let inputArgs: InputArgs
     private let dependenciesWriterFactory: (URL, FileManager) -> DependenciesWriter
     private let touchFactory: (URL, FileManager) -> Touch
 
     public init(
         command: String,
-        inputArgs: SwiftcArgInput,
+        inputArgs: InputArgs,
         dependenciesWriter: @escaping (URL, FileManager) -> DependenciesWriter,
         touchFactory: @escaping (URL, FileManager) -> Touch
     ) {
@@ -63,27 +63,58 @@ public class XCSwiftc {
         self.touchFactory = touchFactory
     }
 
+    func buildContext() throws -> (XCRemoteCacheConfig, SwiftcContext) {
+        fatalError("Need to override in \(Self.self)")
+    }
+
     // swiftlint:disable:next function_body_length
-    public func run() {
+    public func run() throws {
         let fileManager = FileManager.default
-        let config: XCRemoteCacheConfig
-        let context: SwiftcContext
-        do {
-            let srcRoot: URL = URL(fileURLWithPath: fileManager.currentDirectoryPath)
-            config = try XCRemoteCacheConfigReader(srcRootPath: srcRoot.path, fileManager: fileManager)
-                .readConfiguration()
-            context = try SwiftcContext(config: config, input: inputArgs)
-        } catch {
-            exit(1, "FATAL: Swiftc initialization failed with error: \(error)")
-        }
+        let (config, context) = try buildContext()
+
         let swiftcCommand = config.swiftcCommand
         let markerURL = context.tempDir.appendingPathComponent(config.modeMarkerPath)
         let markerReader = FileMarkerReader(markerURL, fileManager: fileManager)
         let markerWriter = FileMarkerWriter(markerURL, fileAccessor: fileManager)
 
-        let inputReader = SwiftcFilemapInputEditor(context.filemap, fileManager: fileManager)
-        let fileListEditor = FileListEditor(context.fileList, fileManager: fileManager)
-        let artifactOrganizer = ZipArtifactOrganizer(targetTempDir: context.tempDir, fileManager: fileManager)
+        let inputReader: SwiftcInputReader
+        switch context.inputs {
+        case .fileMap(let path):
+            inputReader = SwiftcFilemapInputEditor(
+                URL(fileURLWithPath: path),
+                fileFormat: .json,
+                fileManager: fileManager
+            )
+        case .supplementaryFileMap(let path):
+            // Supplementary file map is endoded in the yaml file (contraty to
+            // the standard filemap, which is in json)
+            inputReader = SwiftcFilemapInputEditor(
+                URL(fileURLWithPath: path),
+                fileFormat: .yaml,
+                fileManager: fileManager
+            )
+        case .map(let map):
+            // static - passed via the arguments list
+            inputReader = StaticSwiftcInputReader(
+                moduleDependencies: context.steps.emitModule?.dependencies,
+                // with Xcode 14, inputs via cmd are only used for compilations
+                swiftDependencies: nil,
+                compilationFiles: Array(map.values)
+            )
+        }
+        let fileListReader: ListReader
+        switch context.compilationFiles {
+        case .fileList(let path):
+            fileListReader = FileListEditor(URL(fileURLWithPath: path), fileManager: fileManager)
+        case .list(let paths):
+            fileListReader = StaticFileListReader(list: paths.map(URL.init(fileURLWithPath:)))
+        }
+        let artifactOrganizer = ZipArtifactOrganizer(
+            targetTempDir: context.tempDir,
+            // xcswiftc  doesn't call artifact preprocessing
+            artifactProcessors: [],
+            fileManager: fileManager
+        )
         // TODO: check for allowedFile comparing a list of all inputfiles, not dependencies from a marker
         let makerReferencedFilesListScanner = FileListScannerImpl(markerReader, caseSensitive: false)
         let allowedFilesListScanner = ExceptionsFilteredFileListScanner(
@@ -96,11 +127,20 @@ public class XCSwiftc {
             moduleName: context.moduleName,
             fileManager: fileManager
         )
-        let productsGenerator = DiskSwiftcProductsGenerator(
-            modulePathOutput: context.modulePathOutput,
-            objcHeaderOutput: context.objcHeaderOutput,
-            diskCopier: HardLinkDiskCopier(fileManager: fileManager)
-        )
+        let productsGenerator: SwiftcProductsGenerator
+        if let emitModule = context.steps.emitModule {
+            productsGenerator = DiskSwiftcProductsGenerator(
+                modulePathOutput: emitModule.modulePathOutput,
+                objcHeaderOutput: emitModule.objcHeaderOutput,
+                diskCopier: HardLinkDiskCopier(fileManager: fileManager)
+            )
+        } else {
+            // If the module was not requested for this proces (compiling files only)
+            // do nothing, when someone (e.g. a plugin) asks for the products generation
+            // This generation will happend in a separate process, where the module
+            // generation is requested
+            productsGenerator = NoopSwiftcProductsGenerator()
+        }
         let allInvocationsStorage = ExistingFileStorage(
             storageFile: context.invocationHistoryFile,
             command: swiftcCommand
@@ -112,9 +152,12 @@ public class XCSwiftc {
             retrieveIgnoredCommands: [swiftcCommand]
         )
         let shellOut = ProcessShellOut()
+        // Always allow an input file from the actool generation step
+        // As of Xcode15, the filename is confirmed to be static
+        let allowedInputDeterminer = FilenameBasedAllowedInputDeterminer(["GeneratedAssetSymbols.swift"])
 
         let swiftc = Swiftc(
-            inputFileListReader: fileListEditor,
+            inputFileListReader: fileListReader,
             markerReader: markerReader,
             allowedFilesListScanner: allowedFilesListScanner,
             artifactOrganizer: artifactOrganizer,
@@ -125,24 +168,35 @@ public class XCSwiftc {
             fileManager: fileManager,
             dependenciesWriterFactory: dependenciesWriterFactory,
             touchFactory: touchFactory,
-            plugins: []
+            plugins: [],
+            allowedInputDeterminer: allowedInputDeterminer
         )
         let orchestrator = SwiftcOrchestrator(
             mode: context.mode,
             swiftc: swiftc,
             swiftcCommand: swiftcCommand,
-            objcHeaderOutput: context.objcHeaderOutput,
-            moduleOutput: context.modulePathOutput,
+            objcHeaderOutput: context.steps.emitModule?.objcHeaderOutput,
+            moduleOutput: context.steps.emitModule?.modulePathOutput,
             arch: context.arch,
             artifactBuilder: artifactBuilder,
             producerFallbackCommandProcessors: [],
             invocationStorage: invocationStorage,
             shellOut: shellOut
         )
-        do {
-            try orchestrator.run()
-        } catch {
-            exit(1, "Swiftc failed with error: \(error)")
-        }
+        try orchestrator.run()
+    }
+}
+
+public class XCSwiftc: XCSwiftAbstract<SwiftcArgInput> {
+    override func buildContext() throws -> (XCRemoteCacheConfig, SwiftcContext) {
+        let fileReader = FileManager.default
+        let config: XCRemoteCacheConfig
+        let context: SwiftcContext
+        let srcRoot: URL = URL(fileURLWithPath: fileReader.currentDirectoryPath)
+        config = try XCRemoteCacheConfigReader(srcRootPath: srcRoot.path, fileReader: fileReader)
+            .readConfiguration()
+        context = try SwiftcContext(config: config, input: inputArgs)
+
+        return (config, context)
     }
 }

Sources/XCRemoteCache/Config/XCRemoteCacheConfig.swift

@@ -17,6 +17,8 @@
 // specific language governing permissions and limitations
 // under the License.
 
+// swiftlint:disable file_length
+
 import Foundation
 import Yams
 
@@ -57,6 +59,8 @@ public struct XCRemoteCacheConfig: Encodable {
     var clangCommand: String = "clang"
     /// Command for a standard Swift compilation (swiftc)
     var swiftcCommand: String = "swiftc"
+    /// Command for a standard Swift frontend compilation (swift-frontend)
+    var swiftFrontendCommand: String = "swift-frontend"
     /// Path of the primary repository that produces cache artifacts
     var primaryRepo: String = ""
     /// Main (primary) branch that produces cache artifacts (default to 'master')
@@ -83,6 +87,10 @@ public struct XCRemoteCacheConfig: Encodable {
     var downloadRetries: Int = 0
     /// Number of retries for upload requests
     var uploadRetries: Int = 3
+    /// Delay between retries in seconds
+    var retryDelay: Double = 10.0
+    /// Maximum number of simultaneous requests. 0 means no limits
+    var uploadBatchSize: Int = 0
     /// Extra headers appended to all remote HTTP(S) requests
     var requestCustomHeaders: [String: String] = [:]
     /// Filename (without an extension) of the compilation input file that is used
@@ -95,10 +103,10 @@ public struct XCRemoteCacheConfig: Encodable {
     ///  Disable cache for http requests to fecth metadata and download artifacts
     var disableHttpCache: Bool = false
     /// Path, relative to $TARGET_TEMP_DIR which gathers all compilation commands that should be e
-    /// xecuted if a target switches to local compilation.
+    /// xecuted if a target switches to local compilation
     /// Example: A new `.swift` file invalidates remote arXcodeProjIntegrate.swifttifact and triggers local compilation
     /// When that happens, all previously skipped clang build steps
-    /// need to be eventually called locally - this file lists all these commands.
+    /// need to be eventually called locally - this file lists all these commands
     var compilationHistoryFile: String = "history.compile"
     /// Timeout for remote response data interval (in seconds). If an interval between data chunks is
     /// longer than a timeout, a request fails
@@ -107,7 +115,8 @@ public struct XCRemoteCacheConfig: Encodable {
     var turnOffRemoteCacheOnFirstTimeout: Bool = false
     /// List of all extensions that should carry over source fingerprints. Extensions of all product files that
     /// contain non-deterministic content (absolute paths, timestamp, etc) should be included
-    var productFilesExtensionsWithContentOverride = ["swiftmodule"]
+    /// .h files may contain absolute paths if NS_ENUM is used in a public API from Swift code
+    var productFilesExtensionsWithContentOverride = ["swiftmodule", "h"]
     /// If true, plugins for thinning support should be enabled
     var thinningEnabled: Bool = false
     /// Module name of a target that works as a helper for thinned targets
@@ -118,19 +127,36 @@ public struct XCRemoteCacheConfig: Encodable {
     var AWSSecretKey: String = ""
     /// Access key for AWS V4 Signature
     var AWSAccessKey: String = ""
+    /// Temporary security token provided by the AWS Security Token Service
+    var AWSSecurityToken: String?
     /// Region for AWS V4 Signature (e.g. `eu`)
     var AWSRegion: String = ""
     /// Service for AWS V4 Signature (e.g. `storage`)
     var AWSService: String = ""
-    /// A dictionary of files path remapping that should be applied to make it absolute path agnostic on a list of dependencies.
-    /// Useful if a project refers files out of repo root, either compilation files or precompiled dependencies.
-    /// Keys represent generic replacement and values are substrings that should be replaced.
+    /// A dictionary of files path remapping that should be applied to make it absolute path agnostic on a list of
+    /// dependencies. Useful if a project refers files out of repo root, either compilation files or precompiled
+    /// dependencies. Keys represent generic replacement and values are substrings that should be replaced
     /// Example: for mapping `["COOL_LIBRARY": "/CoolLibrary"]`
-    /// `/CoolLibrary/main.swift`will be represented as `$(COOL_LIBRARY)/main.swift`).
-    /// Warning: remapping order is not-deterministic so avoid remappings with multiple matchings.
+    /// `/CoolLibrary/main.swift`will be represented as `$(COOL_LIBRARY)/main.swift`)
+    /// Warning: remapping order is not-deterministic so avoid remappings with multiple matchings
     var outOfBandMappings: [String: String] = [:]
     /// If true, SSL certificate validation is disabled
     var disableCertificateVerification: Bool = false
+    /// A feature flag to disable virtual file system overlay support (temporary)
+    var disableVFSOverlay: Bool = false
+    /// A list of extra ENVs that should be used as placeholders in the dependency list
+    /// ENV rewrite process is optimistic - does nothing if an ENV is not defined in the pre/postbuild process
+    var customRewriteEnvs: [String] = []
+    /// Regexes of files that should not be included in a list of dependencies. Warning! Add entries here
+    /// with caution - excluding dependencies that are relevant might lead to a target overcaching
+    /// Note: The regex can match either partially or fully the filepath, e.g. `\\.modulemap$` will exclude
+    /// all `.modulemap` files
+    var irrelevantDependenciesPaths: [String] = []
+    /// If true, do not fail `prepare` if cannot find the most recent common commits with the primary branch
+    /// That might useful on CI, where a shallow clone is used
+    var gracefullyHandleMissingCommonSha: Bool = false
+    /// Enable experimental integration with swift driver, added in Xcode 14
+    var enableSwiftDriverIntegration: Bool = false
 }
 
 extension XCRemoteCacheConfig {
@@ -163,6 +189,8 @@ extension XCRemoteCacheConfig {
         merge.statsDir = scheme.statsDir ?? statsDir
         merge.downloadRetries = scheme.downloadRetries ?? downloadRetries
         merge.uploadRetries = scheme.uploadRetries ?? uploadRetries
+        merge.retryDelay = scheme.retryDelay ?? retryDelay
+        merge.uploadBatchSize = scheme.uploadBatchSize ?? uploadBatchSize
         merge.requestCustomHeaders = scheme.requestCustomHeaders ?? requestCustomHeaders
         merge.thinTargetMockFilename = scheme.thinTargetMockFilename ?? thinTargetMockFilename
         merge.focusedTargets = scheme.focusedTargets ?? focusedTargets
@@ -179,10 +207,17 @@ extension XCRemoteCacheConfig {
         merge.prettifyMetaFiles = scheme.prettifyMetaFiles ?? prettifyMetaFiles
         merge.AWSAccessKey = scheme.AWSAccessKey ?? AWSAccessKey
         merge.AWSSecretKey = scheme.AWSSecretKey ?? AWSSecretKey
+        merge.AWSSecurityToken = scheme.AWSSecurityToken ?? AWSSecurityToken
         merge.AWSRegion = scheme.AWSRegion ?? AWSRegion
         merge.AWSService = scheme.AWSService ?? AWSService
         merge.outOfBandMappings = scheme.outOfBandMappings ?? outOfBandMappings
         merge.disableCertificateVerification = scheme.disableCertificateVerification ?? disableCertificateVerification
+        merge.disableVFSOverlay = scheme.disableVFSOverlay ?? disableVFSOverlay
+        merge.customRewriteEnvs = scheme.customRewriteEnvs ?? customRewriteEnvs
+        merge.irrelevantDependenciesPaths = scheme.irrelevantDependenciesPaths ?? irrelevantDependenciesPaths
+        merge.gracefullyHandleMissingCommonSha =
+            scheme.gracefullyHandleMissingCommonSha ?? gracefullyHandleMissingCommonSha
+        merge.enableSwiftDriverIntegration = scheme.enableSwiftDriverIntegration ?? enableSwiftDriverIntegration
         return merge
     }
 
@@ -227,6 +262,8 @@ struct ConfigFileScheme: Decodable {
     let statsDir: String?
     let downloadRetries: Int?
     let uploadRetries: Int?
+    let retryDelay: Double?
+    let uploadBatchSize: Int?
     let requestCustomHeaders: [String: String]?
     let thinTargetMockFilename: String?
     let focusedTargets: [String]?
@@ -240,10 +277,16 @@ struct ConfigFileScheme: Decodable {
     let prettifyMetaFiles: Bool?
     let AWSSecretKey: String?
     let AWSAccessKey: String?
+    let AWSSecurityToken: String?
     let AWSRegion: String?
     let AWSService: String?
     let outOfBandMappings: [String: String]?
     let disableCertificateVerification: Bool?
+    let disableVFSOverlay: Bool?
+    let customRewriteEnvs: [String]?
+    let irrelevantDependenciesPaths: [String]?
+    let gracefullyHandleMissingCommonSha: Bool?
+    let enableSwiftDriverIntegration: Bool?
 
     // Yams library doesn't support encoding strategy, see https://github.com/jpsim/Yams/issues/84
     enum CodingKeys: String, CodingKey {
@@ -271,6 +314,8 @@ struct ConfigFileScheme: Decodable {
         case statsDir = "stats_dir"
         case downloadRetries = "download_retries"
         case uploadRetries = "upload_retries"
+        case retryDelay = "retry_delay"
+        case uploadBatchSize = "upload_batch_size"
         case requestCustomHeaders = "request_custom_headers"
         case thinTargetMockFilename = "thin_target_mock_filename"
         case focusedTargets = "focused_targets"
@@ -284,10 +329,16 @@ struct ConfigFileScheme: Decodable {
         case prettifyMetaFiles = "prettify_meta_files"
         case AWSSecretKey = "aws_secret_key"
         case AWSAccessKey = "aws_access_key"
+        case AWSSecurityToken = "aws_security_token"
         case AWSRegion = "aws_region"
         case AWSService = "aws_service"
         case outOfBandMappings = "out_of_band_mappings"
         case disableCertificateVerification = "disable_certificate_verification"
+        case disableVFSOverlay = "disable_vfs_overlay"
+        case customRewriteEnvs = "custom_rewrite_envs"
+        case irrelevantDependenciesPaths = "irrelevant_dependencies_paths"
+        case gracefullyHandleMissingCommonSha = "gracefully_handle_missing_common_sha"
+        case enableSwiftDriverIntegration = "enable_swift_driver_integration"
     }
 }
 
@@ -300,31 +351,42 @@ class XCRemoteCacheConfigReader {
     /// Name of the configuration file, required in $(SRCROOT) location
     private static let configurationFile = ".rcinfo"
     private let srcRoot: String
-    private let fileManager: FileManager
+    private let fileReader: FileReader
     private lazy var yamlDecorer = YAMLDecoder(encoding: .utf8)
 
-    init(env: [String: String], fileManager: FileManager) throws {
+    init(env: [String: String], fileReader: FileReader) throws {
         let explicitSrcRoot: String? = env.readEnv(key: "SRCROOT")
-        srcRoot = explicitSrcRoot ?? fileManager.currentDirectoryPath
-        self.fileManager = fileManager
+        srcRoot = explicitSrcRoot ?? FileManager.default.currentDirectoryPath
+        self.fileReader = fileReader
     }
 
-    init(srcRootPath srcRoot: String, fileManager: FileManager) {
+    init(srcRootPath srcRoot: String, fileReader: FileReader) {
         self.srcRoot = srcRoot
-        self.fileManager = fileManager
+        self.fileReader = fileReader
     }
 
+    // Reads the final configuration by loading all extra configs
+    // until reaching a config that doesn't override `extraConfigurationFile`
     func readConfiguration() throws -> XCRemoteCacheConfig {
         let rootURL = URL(fileURLWithPath: srcRoot)
         let configURL = URL(fileURLWithPath: Self.configurationFile, relativeTo: rootURL)
         let userConfigs = try readUserConfig(configURL)
         var config = XCRemoteCacheConfig(sourceRoot: srcRoot).merged(with: userConfigs)
-        let extraConfURL = URL(fileURLWithPath: config.extraConfigurationFile, relativeTo: rootURL)
-        do {
-            let extraConfig = try readUserConfig(extraConfURL)
-            config = config.merged(with: extraConfig)
-        } catch {
-            infoLog("Extra config override failed with \(error). Skipping extra configuration")
+        var extraConfURL = URL(fileURLWithPath: config.extraConfigurationFile, relativeTo: rootURL)
+        var visitedFiles = Set([configURL])
+        while !visitedFiles.contains(extraConfURL) {
+            do {
+                let extraConfig = try readUserConfig(extraConfURL)
+                debugLog("Reading extra configuration from \(extraConfURL)")
+                config = config.merged(with: extraConfig)
+                visitedFiles.insert(extraConfURL)
+                // Advance extra configuration
+                extraConfURL = URL(fileURLWithPath: config.extraConfigurationFile, relativeTo: rootURL)
+            } catch {
+                infoLog("Extra config override failed with \(error). Skipping extra configuration")
+                // swiftlint:disable:next unneeded_break_in_switch
+                break
+            }
         }
 
         return try config.verifyAndApplyDefaults()
@@ -332,7 +394,7 @@ class XCRemoteCacheConfigReader {
 
     /// Reads user configuration from a file
     private func readUserConfig(_ file: URL) throws -> ConfigFileScheme {
-        let configurationContent = fileManager.contents(atPath: file.path)
+        let configurationContent = try fileReader.contents(atPath: file.path)
         guard let configurationData = configurationContent else {
             throw XCRemoteCacheConfigReaderError.missingConfigurationFile(file)
         }

Sources/XCRemoteCache/Dependencies/AssetsFileDependenciesReader.swift

@@ -0,0 +1,77 @@
+// Copyright (c) 2023 Spotify AB.
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import Foundation
+
+/// Parser for `assetcatalog_dependencies` file: an output of the `actool`
+/// that lists all dependencies of this command
+class AssetsFileDependenciesReader: DependenciesReader {
+    private let file: URL
+    private let dirAccessor: DirAccessor
+
+    public init(_ file: URL, dirAccessor: DirAccessor) {
+        self.file = file
+        self.dirAccessor = dirAccessor
+    }
+
+    public func findDependencies() throws -> [String] {
+        return try Array(findAllDependencies())
+    }
+
+    public func findInputs() throws -> [String] {
+        // XCRemoteCache doesn't use it yet
+        exit(1, "TODO: implement")
+    }
+
+    public func readFilesAndDependencies() throws -> [String: [String]] {
+        return try ["": findAllDependencies()]
+    }
+
+    private func findAllDependencies() throws -> [String] {
+        let fileData = try getFileData()
+        // all dependency files are separated by the \0 byte
+        // each path has a file type prefix:
+        // 0x10 - directory
+        // 0x40 - file
+        // We only care about dirs, as *.xcassets is a folder
+        let pathDatas = fileData.split(separator: 0x0)
+        let paths = pathDatas
+            .filter { !$0.isEmpty && $0.first == 0x10 }
+            .map { String(data: $0.dropFirst(), encoding: .utf8)! }
+            .map(URL.init(fileURLWithPath:))
+        let xcassetsPaths = paths.filter { path in
+            path.pathExtension == "xcassets"
+        }
+        return try xcassetsPaths.flatMap { try findAssetsContentJsons(xcasset: $0) }
+    }
+
+    private func findAssetsContentJsons(xcasset: URL) throws -> [String] {
+        return try dirAccessor.recursiveItems(at: xcasset).filter { url in
+            url.lastPathComponent == "Contents.json"
+        }.map(\.path)
+    }
+
+    private func getFileData() throws -> Data {
+        guard let fileData = try dirAccessor.contents(atPath: file.path) else {
+            throw DependenciesReaderError.readingError
+        }
+        return fileData
+    }
+
+}

Sources/XCRemoteCache/Dependencies/CacheModeController.swift

@@ -48,6 +48,7 @@ class PhaseCacheModeController: CacheModeController {
     private let dependenciesWriter: DependenciesWriter
     private let dependenciesReader: DependenciesReader
     private let markerWriter: MarkerWriter
+    private let llbuildLockFile: URL
     private let fileManager: FileManager
 
     init(
@@ -59,6 +60,7 @@ class PhaseCacheModeController: CacheModeController {
         dependenciesWriter: (URL, FileManager) -> DependenciesWriter,
         dependenciesReader: (URL, FileManager) -> DependenciesReader,
         markerWriter: (URL, FileManager) -> MarkerWriter,
+        llbuildLockFile: URL,
         fileManager: FileManager
     ) {
 
@@ -69,10 +71,12 @@ class PhaseCacheModeController: CacheModeController {
         let discoveryURL = tempDir.appendingPathComponent(phaseDependencyPath)
         self.dependenciesWriter = dependenciesWriter(discoveryURL, fileManager)
         self.dependenciesReader = dependenciesReader(discoveryURL, fileManager)
+        self.llbuildLockFile = llbuildLockFile
         self.markerWriter = markerWriter(modeMarker, fileManager)
     }
 
     func enable(allowedInputFiles: [URL], dependencies: [URL]) throws {
+        try cleanupLlBuildLock()
         // marker file contains filepaths that contribute to the build products
         // and should invalidate all other target steps (swiftc,libtool etc.)
         let targetSensitiveFiles = dependencies + [modeMarker, Self.xcodeSelectLink]
@@ -84,6 +88,7 @@ class PhaseCacheModeController: CacheModeController {
     }
 
     func disable() throws {
+        try cleanupLlBuildLock()
         guard !forceCached else {
             throw PhaseCacheModeControllerError.cannotUseRemoteCacheForForcedCacheMode
         }
@@ -110,8 +115,24 @@ class PhaseCacheModeController: CacheModeController {
         } catch {
             // Gracefully don't disable a cache
             // That may happen if building a target for the first time
-            errorLog("Couldn't verify if should disable RC for \(commitValue).")
+            debugLog("Couldn't verify if should disable RC for \(commitValue).")
         }
         return false
     }
+
+    // cleanup the build lock file (if exists) as the very last step of this controller
+    // this is just a non-critical cleanup step to not leave {{LLBUILD_BUILD_ID}}.lock
+    // files in $TARGET_TEMP_DIR. It is expected that both prebuild and postbuild will
+    // invoke it, to ensure:
+    // - swift-frontend synchronization is done per-target build
+    // - no .lock leftover files
+    private func cleanupLlBuildLock() throws {
+        if fileManager.fileExists(atPath: llbuildLockFile.path) {
+            do {
+                try fileManager.removeItem(at: llbuildLockFile)
+            } catch {
+                printWarning("Removing llbuild lock at \(llbuildLockFile.path) failed. Error: \(error)")
+            }
+        }
+    }
 }

Sources/XCRemoteCache/Dependencies/DependenciesReader.swift

@@ -51,34 +51,22 @@ public class FileDependenciesReader: DependenciesReader {
     public func findDependencies() throws -> [String] {
         let yaml = try readRaw()
 
-        struct ParseState {
-            var buffer: String = ""
-            var prevChar: Character?
-            var result: [String] = []
-            func with(buffer: String? = nil, prevChar: Character? = nil, result: [String]? = nil) -> ParseState {
-                var new = self
-                new.buffer = buffer ?? new.buffer
-                new.prevChar = prevChar ?? new.prevChar
-                new.result = result ?? new.result
-                return new
-            }
-        }
-
         let dependencies = yaml.reduce(Set<String>()) { prev, arg1 -> Set<String> in
             let (key, value) = arg1
             switch key {
             case "dependencies":
                 // 'clang' output formatting
-                return Set(splitDependencyFileList(value))
+                return Set(parseDependencyFileList(value))
             case let s where s.hasSuffix(".o") || s.hasSuffix(".bc"):
                 // 'swiftc' output formatting
-                // take dependencies from any .o or .bc file.
+                // take dependencies from any .o or .bc file
                 // Note: For WMO, all .{o|bc} files have the same dependencies
-                return Set(splitDependencyFileList(value))
+                return Set(parseDependencyFileList(value))
             default:
                 return prev
             }
         }
+
         return Array(dependencies)
     }
 
@@ -92,56 +80,92 @@ public class FileDependenciesReader: DependenciesReader {
         return yaml.mapValues { $0.components(separatedBy: .whitespaces) }
     }
 
-    private func readRaw() throws -> [String: String] {
+    func readRaw() throws -> [String: String] {
+        let fileData = try getFileData()
+        let fileString = try getFileStringFromData(fileData: fileData)
+        let yaml = try getYaml(fileString: fileString)
+        return yaml
+    }
+
+    func getFileData() throws -> Data {
         guard let fileData = fileManager.contents(atPath: file.path) else {
             throw DependenciesReaderError.readingError
         }
+        return fileData
+    }
+
+    func getFileStringFromData(fileData: Data) throws -> String {
         guard let fileString = String(data: fileData, encoding: .utf8) else {
             throw DependenciesReaderError.invalidFile
         }
-        // .d matches the .yaml format
+        return fileString
+    }
+
+    func getYaml(fileString: String) throws -> [String: String] {
         guard let yaml = try Yams.load(yaml: fileString) as? [String: String] else {
             throw DependenciesReaderError.invalidFile
         }
         return yaml
     }
 
-    /// Splits space or new line separated files into a set of files
+    /// Parses the String to get the list of files
+    /// It iterates over the String using its UTF8View since it is more performant (String type operates
+    /// in a higher abstraction level and supports features that have a negative impact in the performance)
     /// It supports escaping whitespace charaters, prefixed with "\\"
     /// - Parameter string: string of whitespace charaters separated file paths
     /// - Returns: Array of all file paths
-    private func splitDependencyFileList(_ string: String) -> [String] {
-        struct ParseState {
-            var buffer: String = ""
-            var prevChar: Character?
-            var result: [String] = []
-            func with(buffer: String? = nil, prevChar: Character? = nil, result: [String]? = nil) -> ParseState {
-                var new = self
-                new.buffer = buffer ?? new.buffer
-                new.prevChar = prevChar ?? new.prevChar
-                new.result = result ?? new.result
-                return new
-            }
+    func parseDependencyFileList(_ string: String) -> [String] {
+        var result: [String] = []
+        var prevChar: UTF8.CodeUnit?
+
+        // These index are used to move over the UTF8View of the string
+        // The goal is to optimize the memory used, since UTF8View uses
+        // the same memory as the original String without copying it
+        var startIndex = string.utf8.startIndex
+        var endIndex = startIndex
+
+        // This buffer is only used to save the part of the path that has been already parsed when finding a backslash
+        var buffer: String = ""
+
+        for c in string.utf8 {
+          switch c {
+          case UTF8.CodeUnit(ascii: "\n") where prevChar == UTF8.CodeUnit(ascii: "\\"):
+              startIndex = string.utf8.index(after: startIndex)
+              endIndex = startIndex
+          case UTF8.CodeUnit(ascii: " ") where startIndex == endIndex && buffer.isEmpty:
+              startIndex = string.utf8.index(after: startIndex)
+              endIndex = startIndex
+          case UTF8.CodeUnit(ascii: " ") where prevChar != UTF8.CodeUnit(ascii: "\\"):
+              // If a space is found and it is not escaped, then that's the end of the file path
+              buffer += String(Substring(string.utf8[startIndex ..< endIndex]))
+              result.append(buffer)
+              buffer = ""
+              prevChar = nil
+              startIndex = string.utf8.index(after: endIndex)
+              endIndex = startIndex
+          case UTF8.CodeUnit(ascii: "\\"):
+              // If a backslash is found it is not included in the file path
+              // The current parsed range of the UTF8View is saved in the buffer as a String
+              buffer += String(Substring(string.utf8[startIndex ..< endIndex]))
+              // The backslash is assigned as the previous char
+              prevChar = c
+              // The indexes are moved to the next char so we continue parsing the String
+              startIndex = string.utf8.index(after: endIndex)
+              endIndex = startIndex
+          default:
+              // As long as it is possible the indexes are used to track the range of the string that
+              // will be included in the file path (until it ends or until a backslash is found)
+              endIndex = string.utf8.index(after: endIndex)
+              // The char is assigned as the previous char
+              prevChar = c
+          }
         }
-        let parseResult = string.reduce(ParseState()) { total, char in
-            switch char {
-            case "\n" where total.prevChar == "\\":
-                return total
-            case " " where total.buffer.isEmpty:
-                return total
-            case " " where total.prevChar == "\\":
-                return total.with(buffer: "\(total.buffer) ")
-            case " ":
-                return total.with(buffer: "", prevChar: nil, result: total.result + [total.buffer])
-            case "\\":
-                return total.with(prevChar: "\\")
-            default:
-                return total.with(buffer: "\(total.buffer)\(char)", prevChar: char, result: total.result)
-            }
+
+        if startIndex != endIndex {
+            buffer += String(Substring(string.utf8[startIndex ..< endIndex]))
+            result.append(buffer)
         }
-        if !parseResult.buffer.isEmpty {
-            return parseResult.result + [parseResult.buffer]
-        }
-        return parseResult.result
+
+        return result
     }
 }

Sources/XCRemoteCache/Dependencies/DependenciesRemapper.swift

@@ -22,9 +22,9 @@ import Foundation
 /// Replaces paths formats between generic (placeholders-based) and local
 protocol DependenciesRemapper {
     /// Replaces all generic paths (with placeholders) to a local paths
-    func replace(genericPaths: [String]) -> [String]
+    func replace(genericPaths: [String]) throws -> [String]
     /// Replaces all local paths to the generic dependencies paths
-    func replace(localPaths: [String]) -> [String]
+    func replace(localPaths: [String]) throws -> [String]
 }
 
 class DependenciesRemapperComposite: DependenciesRemapper {
@@ -34,15 +34,15 @@ class DependenciesRemapperComposite: DependenciesRemapper {
         self.remappers = remappers
     }
 
-    func replace(genericPaths: [String]) -> [String] {
-        remappers.reversed().reduce(genericPaths) { prev, mapper in
-            mapper.replace(genericPaths: prev)
+    func replace(genericPaths: [String]) throws -> [String] {
+        try remappers.reversed().reduce(genericPaths) { prev, mapper in
+            try mapper.replace(genericPaths: prev)
         }
     }
 
-    func replace(localPaths: [String]) -> [String] {
-        remappers.reduce(localPaths) { prev, mapper in
-            mapper.replace(localPaths: prev)
+    func replace(localPaths: [String]) throws -> [String] {
+        try remappers.reduce(localPaths) { prev, mapper in
+            try mapper.replace(localPaths: prev)
         }
     }
 }
@@ -59,7 +59,7 @@ final class StringDependenciesRemapper: DependenciesRemapper {
         self.mappings = mappings
     }
 
-    func replace(genericPaths: [String]) -> [String] {
+    func replace(genericPaths: [String]) throws -> [String] {
         return genericPaths.map { path in
             let localPath = mappings.reversed().reduce(path) { prevPath, mapping in
                 prevPath.replacingOccurrences(of: mapping.generic, with: mapping.local)
@@ -68,7 +68,7 @@ final class StringDependenciesRemapper: DependenciesRemapper {
         }
     }
 
-    func replace(localPaths: [String]) -> [String] {
+    func replace(localPaths: [String]) throws -> [String] {
         return localPaths.map { path in
             let result = mappings.reduce(path) { prevPath, mapping in
                 prevPath.replacingOccurrences(of: mapping.local, with: mapping.generic)

Sources/XCRemoteCache/Dependencies/DependenciesWriter.swift

@@ -50,7 +50,7 @@ public class FileDependenciesWriter: DependenciesWriter {
         var content = ""
         for (file, deps) in dependencies {
             content.append(file + ": ")
-            content.append(deps.joined(separator: " "))
+            content.append(deps.map { $0.replacingOccurrences(of: " ", with: "\\ ") }.joined(separator: " "))
             content.append("\n")
         }
         try content.write(to: file, atomically: true, encoding: .utf8)

Sources/XCRemoteCache/Dependencies/DependencyProcessor.swift

@@ -27,8 +27,11 @@ public struct Dependency: Equatable {
         case source
         case fingerprint
         case intermediate
+        case derivedFile
         // Product of the target itself
         case ownProduct
+        // User-excluded path
+        case userExcluded
         case unknown
     }
 
@@ -55,14 +58,18 @@ class DependencyProcessorImpl: DependencyProcessor {
     private let productPath: String
     private let sourcePath: String
     private let intermediatePath: String
+    private let derivedFilesPath: String
     private let bundlePath: String?
+    private let skippedRegexes: [String]
 
-    init(xcode: URL, product: URL, source: URL, intermediate: URL, bundle: URL?) {
-        xcodePath = xcode.path
-        productPath = product.path
-        sourcePath = source.path
-        intermediatePath = intermediate.path
-        bundlePath = bundle?.path
+    init(xcode: URL, product: URL, source: URL, intermediate: URL, derivedFiles: URL, bundle: URL?, skippedRegexes: [String]) {
+        xcodePath = xcode.path.dirPath()
+        productPath = product.path.dirPath()
+        sourcePath = source.path.dirPath()
+        intermediatePath = intermediate.path.dirPath()
+        derivedFilesPath = derivedFiles.path.dirPath()
+        bundlePath = bundle?.path.dirPath()
+        self.skippedRegexes = skippedRegexes
     }
 
     func process(_ files: [URL]) -> [Dependency] {
@@ -72,11 +79,15 @@ class DependencyProcessorImpl: DependencyProcessor {
 
     private func classify(_ files: [URL]) -> [Dependency] {
         return files.map { file -> Dependency in
-            let filePath = file.path
-            if filePath.hasPrefix(xcodePath) {
+            let filePath = file.resolvingSymlinksInPath().path
+            if skippedRegexes.contains(where: { filePath.range(of: $0, options: .regularExpression) != nil }) {
+                return Dependency(url: file, type: .userExcluded)
+            } else if filePath.hasPrefix(xcodePath) {
                 return Dependency(url: file, type: .xcode)
             } else if filePath.hasPrefix(intermediatePath) {
                 return Dependency(url: file, type: .intermediate)
+            } else if filePath.hasPrefix(derivedFilesPath) {
+                return Dependency(url: file, type: .derivedFile)
             } else if let bundle = bundlePath, filePath.hasPrefix(bundle) {
                 // If a target produces a bundle, explicitly classify all
                 // of products to distinguish from other targets products
@@ -107,7 +118,18 @@ class DependencyProcessorImpl: DependencyProcessor {
         // - All files in `*/Interemediates/*` - this file are created on-fly for a given target
         // - Some files may depend on its own product (e.g. .m may #include *-Swift.h) - we know products will match
         //   because in case of a hit, these will be taken from the artifact
-        let irrelevantDependenciesType: [Dependency.Kind] = [.xcode, .intermediate, .ownProduct]
+        // - Customized DERIVED_FILE_DIR may change a directory of
+        //   derived files, which by default is under `*/Interemediates`
+        // - User-specified (in .rcinfo) files to exclude
+        let irrelevantDependenciesType: [Dependency.Kind] = [
+            .xcode, .intermediate, .ownProduct, .derivedFile, .userExcluded,
+        ]
         return !irrelevantDependenciesType.contains(dependency.type)
     }
 }
+
+fileprivate extension String {
+    func dirPath() -> String {
+        hasSuffix("/") ? self : appending("/")
+    }
+}

Sources/XCRemoteCache/Dependencies/FingerprintSyncer.swift

@@ -30,6 +30,10 @@ protocol FingerprintSyncer {
     func decorate(sourceDir: URL, fingerprint: String) throws
     /// Deletes fingerprint overrides in the dir (if already created)
     func delete(sourceDir: URL) throws
+    /// Sets a fingerprint override for a singe file placed
+    func decorate(file: URL, fingerprint: String) throws
+    /// Deletes fingerprint override for a file  (if already created)
+    func delete(file: URL) throws
 }
 
 class FileFingerprintSyncer: FingerprintSyncer {
@@ -78,4 +82,25 @@ class FileFingerprintSyncer: FingerprintSyncer {
             try dirAccessor.removeItem(atPath: file.path)
         }
     }
+
+    func decorate(file: URL, fingerprint: String) throws {
+        guard let fingerprintData = fingerprint.data(using: .utf8) else {
+            throw FingerprintSyncerError.invalidFingerprint
+        }
+        let fingerprintFile = file.appendingPathExtension(fingerprintExtension)
+        try dirAccessor.write(toPath: fingerprintFile.path, contents: fingerprintData)
+    }
+
+    func delete(file: URL) throws {
+        guard case .file = try dirAccessor.itemType(atPath: file.path) else {
+            // no file to decorate (no module was generated)
+            return
+        }
+        let overrideURL = file.appendingPathExtension(fingerprintExtension)
+        guard case .file = try dirAccessor.itemType(atPath: overrideURL.path) else {
+            // no override
+            return
+        }
+        try dirAccessor.removeItem(atPath: overrideURL.path)
+    }
 }

Sources/XCRemoteCache/Dependencies/MarkerReader.swift

@@ -22,12 +22,12 @@ import Foundation
 /// Reads a list of files from a marker file
 class FileMarkerReader: ListReader {
     private let file: URL
-    private let fileManager: FileManager
+    private let fileReader: FileReader
     private var cachedFiles: [URL]?
 
-    init(_ file: URL, fileManager: FileManager) {
+    init(_ file: URL, fileManager: FileReader) {
         self.file = file
-        self.fileManager = fileManager
+        self.fileReader = fileManager
     }
 
     func listFilesURLs() throws -> [URL] {
@@ -45,6 +45,6 @@ class FileMarkerReader: ListReader {
     }
 
     func canRead() -> Bool {
-        return fileManager.fileExists(atPath: file.path)
+        return fileReader.fileExists(atPath: file.path)
     }
 }

Sources/XCRemoteCache/Dependencies/OverlayDependenciesRemapper.swift

@@ -0,0 +1,67 @@
+// Copyright (c) 2021 Spotify AB.
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import Foundation
+
+/// File paths remapper according the virtual file system mappings
+/// - Warning: this class is not thread safe
+class OverlayDependenciesRemapper: DependenciesRemapper {
+    private let overlayReader: OverlayReader
+    private var mappings: [OverlayMapping]?
+
+    init(overlayReader: OverlayReader) {
+        self.overlayReader = overlayReader
+    }
+
+    /// Lazily Reads mappings from a file
+    /// - Warning: this function is not thread safe
+    private func getMappings() throws -> [OverlayMapping] {
+        guard let mappings = mappings else {
+            let mappings = try overlayReader.provideMappings()
+            self.mappings = mappings
+            return mappings
+        }
+        return mappings
+    }
+
+    private func mapPath(
+        _ path: String,
+        source: KeyPath<OverlayMapping, URL>,
+        destination: KeyPath<OverlayMapping, URL>
+    ) throws -> String {
+        guard let mapping = try getMappings().first(where: { $0[keyPath: source].path == path }) else {
+            // TODO: support partial mappings, where a directory path can be replaced with some other directory
+            // no direct mapping found
+            return path
+        }
+        return mapping[keyPath: destination].path
+    }
+
+    func replace(genericPaths: [String]) throws -> [String] {
+        try genericPaths.map {
+            try mapPath($0, source: \.virtual, destination: \.local)
+        }
+    }
+
+    func replace(localPaths: [String]) throws -> [String] {
+        try localPaths.map {
+            try mapPath($0, source: \.local, destination: \.virtual)
+        }
+    }
+}

Sources/XCRemoteCache/Dependencies/OverlayReader.swift

@@ -0,0 +1,133 @@
+// Copyright (c) 2021 Spotify AB.
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import Foundation
+
+/// Maps overlay's virtual URL with an actual (local) location
+struct OverlayMapping: Hashable {
+    let virtual: URL
+    let local: URL
+}
+
+enum JsonOverlayReaderError: Error {
+    /// The source file is missing
+    case missingSourceFile(URL)
+    /// The file exists but its content is invalid
+    case invalidSourceContent(URL)
+    /// the overlay format is not supported - either contains a nested directory or a single file 
+    case unsupportedFormat
+}
+/// Provides virtual file system overlay mappings
+protocol OverlayReader {
+    func provideMappings() throws -> [OverlayMapping]
+}
+
+class JsonOverlayReader: OverlayReader {
+
+    enum Mode {
+        /// Interrupts the operation if the representation file is missing
+        case strict
+        /// Assume empty overlay mapping if the file doesn't exist
+        case bestEffort
+    }
+
+    private struct Overlay: Decodable {
+        enum OverlayType: String, Decodable {
+            case file
+            case directory
+        }
+
+        struct Content: Decodable {
+            let externalContents: String
+            let name: String
+            let type: OverlayType
+
+            enum CodingKeys: String, CodingKey {
+                case externalContents = "external-contents"
+                case name
+                case type
+            }
+        }
+
+        struct RootContent: Decodable {
+            let contents: [Content]
+            let name: String
+            let type: OverlayType
+        }
+        let roots: [RootContent]
+    }
+
+    private lazy var jsonDecoder = JSONDecoder()
+    private let json: URL
+    private let mode: Mode
+    private let fileReader: FileReader
+
+
+    init(_ json: URL, mode: Mode, fileReader: FileReader) {
+        self.json = json
+        self.mode = mode
+        self.fileReader = fileReader
+    }
+
+    func provideMappings() throws -> [OverlayMapping] {
+        guard let jsonContent = try fileReader.contents(atPath: json.path) else {
+            switch mode {
+            case .strict:
+                throw JsonOverlayReaderError.missingSourceFile(json)
+            case .bestEffort:
+                debugLog("overlay mapping file \(json) doesn't exist. Skipping overlay for the best-effort mode.")
+                return []
+            }
+        }
+        do {
+        let overlay: Overlay = try jsonDecoder.decode(Overlay.self, from: jsonContent)
+            let mappings: [OverlayMapping] = try overlay.roots.reduce([]) { prev, root in
+                switch root.type {
+                case .directory:
+                    // iterate all contents
+                    let dir = URL(fileURLWithPath: root.name)
+                    let mappings: [OverlayMapping] = try root.contents.map { content in
+                        switch content.type {
+                        case .file:
+                            let virtual = dir.appendingPathComponent(content.name)
+                            let local = URL(fileURLWithPath: content.externalContents)
+                            return .init(virtual: virtual, local: local)
+                        case .directory:
+                            throw JsonOverlayReaderError.unsupportedFormat
+                        }
+
+                    }
+                    return prev + mappings
+                case .file:
+                    throw JsonOverlayReaderError.unsupportedFormat
+                }
+            }
+            return mappings
+        } catch {
+            switch mode {
+            case .strict:
+                throw error
+            case .bestEffort:
+                errorLog("Overlay reader has failed with an error \(error). Best-effort mode - skipping an overlay.")
+                return []
+            }
+        }
+    }
+
+}

Sources/XCRemoteCache/Dependencies/PathDependenciesRemapperFactory.swift

@@ -19,24 +19,28 @@
 
 import Foundation
 
-enum StringDependenciesRemapperFactoryError: Error {
+enum PathDependenciesRemapperFactoryError: Error {
     /// Remapping keys are duplicated and can lead to undetermined results
     case mappingKeyDuplication
 }
 
-class StringDependenciesRemapperFactory {
+class PathDependenciesRemapperFactory {
     func build(
         orderKeys: [String],
         envs: [String: String],
         customMappings: [String: String]
     ) throws -> StringDependenciesRemapper {
-        let mappingMap = try envs.merging(customMappings) { envValue, outOfBandMapping in
-            throw StringDependenciesRemapperFactoryError.mappingKeyDuplication
+        let mappingMap = try envs.merging(customMappings) { _, _ in
+            throw PathDependenciesRemapperFactoryError.mappingKeyDuplication
         }
-        let mappingOrderKeys =  orderKeys + customMappings.keys
-        let mappings: [StringDependenciesRemapper.Mapping] = try mappingOrderKeys.map { key in
-            let localValue: String = try mappingMap.readEnv(key: key)
-            return StringDependenciesRemapper.Mapping(generic: "$(\(key))", local: localValue)
+        let mappingOrderKeys = orderKeys + customMappings.keys
+        let mappings: [StringDependenciesRemapper.Mapping] = mappingOrderKeys.compactMap { key in
+            guard let localURL: URL = mappingMap.readEnv(key: key) else {
+                debugLog("\(key) ENV to map a dependency is not defined")
+                return nil
+            }
+            infoLog("Found url to remapp: \(localURL). Remapping: \(localURL.standardized.path)")
+            return StringDependenciesRemapper.Mapping(generic: "$(\(key))", local: localURL.standardized.path)
         }
         return StringDependenciesRemapper(mappings: mappings)
     }

Sources/XCRemoteCache/Dependencies/StaticFileListReader.swift

@@ -0,0 +1,36 @@
+// Copyright (c) 2023 Spotify AB.
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import Foundation
+
+class StaticFileListReader: ListReader {
+    private let list: [URL]
+
+    init(list: [URL]) {
+        self.list = list
+    }
+
+    func listFilesURLs() throws -> [URL] {
+        list
+    }
+
+    func canRead() -> Bool {
+        true
+    }
+}

Sources/XCRemoteCache/Dependencies/TargetDepdenciesReader.swift

@@ -21,28 +21,37 @@ import Foundation
 
 /// Reads and aggregates all compilation dependencies from a single directory
 class TargetDependenciesReader: DependenciesReader {
-    private let directory: URL
+    // As of Xcode15, the filename is static
+    private static let assetsDependenciesFilename = "assetcatalog_dependencies"
+    private let compilationDirectory: URL
+    private let assetsCatalogOutputDir: URL
     private let dirScanner: DirScanner
-    private let fileDependeciesReaderFactory: (URL) -> DependenciesReader
+    private let fileDependenciesReaderFactory: (URL) -> DependenciesReader
+    private let assetsDependenciesReaderFactory: (URL) -> DependenciesReader
 
     public init(
-        _ directory: URL,
-        fileDependeciesReaderFactory: @escaping (URL) -> DependenciesReader,
+        compilationOutputDir: URL,
+        assetsCatalogOutputDir: URL,
+        fileDependenciesReaderFactory: @escaping (URL) -> DependenciesReader,
+        assetsDependenciesReaderFactory: @escaping (URL) -> DependenciesReader,
         dirScanner: DirScanner
     ) {
-        self.directory = directory
+        self.compilationDirectory = compilationOutputDir
+        self.assetsCatalogOutputDir = assetsCatalogOutputDir
         self.dirScanner = dirScanner
-        self.fileDependeciesReaderFactory = fileDependeciesReaderFactory
+        self.fileDependenciesReaderFactory = fileDependenciesReaderFactory
+        self.assetsDependenciesReaderFactory = assetsDependenciesReaderFactory
     }
 
     // Optimized way of finding dependencies only for files that have corresponding .o file on a disk
+    // includes also inputs to the `actool` assets generator
     public func findDependencies() throws -> [String] {
         // Not calling `readFilesAndDependencies` as it may unnecessary call expensive `findDependencies()` for
         // files that eventually will not be considered
-        let allURLs = try dirScanner.items(at: directory)
-        let mergedDependencies = try allURLs.reduce(Set<String>()) { (prev: Set<String>, file) in
+        let allCompilationOutputURLs = try dirScanner.items(at: compilationDirectory)
+        var mergedDependencies = try allCompilationOutputURLs.reduce(Set<String>()) { (prev: Set<String>, file) in
             // include only these .d files that either have corresponding .o file (incremental) or end
-            // with '-master' (whole-module).
+            // with '-master' (whole-module)
             // Otherwise .d is probably just a leftover from previous builds
             let correspondingOutputURL = file.deletingPathExtension().appendingPathExtension("o")
             let isDependencyFile = file.pathExtension == "d"
@@ -53,20 +62,33 @@ class TargetDependenciesReader: DependenciesReader {
                 return prev
             }
 
-            return try prev.union(fileDependeciesReaderFactory(file).findDependencies())
+            return try prev.union(fileDependenciesReaderFactory(file).findDependencies())
         }
+        // include also dependencies from optional assets compilation (`actool`)
+        try mergedDependencies.formUnion(findAssetsCatalogDependencies())
         return Array(mergedDependencies).sorted()
     }
 
+    // finds all assets compilation's dependencies, which are always appended to the list of
+    // files to compare on the consumer side (in the fingerprint comparison)
+    private func findAssetsCatalogDependencies() throws -> Set<String> {
+        let expectedAssetsDepsFile = assetsCatalogOutputDir
+            .appendingPathComponent(Self.assetsDependenciesFilename)
+        guard try dirScanner.itemType(atPath: expectedAssetsDepsFile.path) == .file else {
+            return []
+        }
+        return try Set(assetsDependenciesReaderFactory(expectedAssetsDepsFile).findDependencies())
+    }
+
     public func findInputs() throws -> [String] {
         fatalError("TODO: implement")
     }
 
     public func readFilesAndDependencies() throws -> [String: [String]] {
-        let allURLs = try dirScanner.items(at: directory)
+        let allURLs = try dirScanner.items(at: compilationDirectory)
         return try allURLs.reduce([String: [String]]()) { prev, file in
             var new = prev
-            new[file.path] = try fileDependeciesReaderFactory(file).findDependencies()
+            new[file.path] = try fileDependenciesReaderFactory(file).findDependencies()
             return new
         }
     }

Sources/XCRemoteCache/FileAccess/DirAccessor.swift

@@ -33,8 +33,13 @@ protocol DirScanner {
 
     /// Returns all items in a directory (shallow search)
     /// - Parameter at: url of an existing directory to search
-    /// - Throws: an error if dir doesn't exist or I/O error
+    /// - Throws: an error if a dir doesn't exist or on I/O error
     func items(at dir: URL) throws -> [URL]
+
+    /// Returns all items in a directory (recursive search)
+    /// - Parameter at: url of an existing directory to search
+    /// - Throws: an error if a dir doesn't exist or on I/O error
+    func recursiveItems(at dir: URL) throws -> [URL]
 }
 
 typealias DirAccessor = FileAccessor & DirScanner
@@ -54,4 +59,18 @@ extension FileManager: DirScanner {
         let resolvedDir = dir.resolvingSymlinksInPath()
         return try contentsOfDirectory(at: resolvedDir, includingPropertiesForKeys: nil, options: [])
     }
+
+    func recursiveItems(at dir: URL) throws -> [URL] {
+        // Iterating DFS
+        var queue: [URL] = [dir]
+        var results: [URL] = []
+        while let item = queue.popLast() {
+            if try itemType(atPath: item.path) == .dir {
+                try queue.append(contentsOf: items(at: item))
+            } else {
+                results.append(item)
+            }
+        }
+        return results
+    }
 }

Sources/XCRemoteCache/FileAccess/DirAccessorComposer.swift

@@ -52,4 +52,8 @@ class DirAccessorComposer: DirAccessor {
     func items(at dir: URL) throws -> [URL] {
         try dirScanner.items(at: dir)
     }
+
+    func recursiveItems(at dir: URL) throws -> [URL] {
+        try dirScanner.recursiveItems(at: dir)
+    }
 }

Sources/XCRemoteCache/Fingerprint/EnvironmentFingerprint.swift

@@ -31,7 +31,7 @@ class EnvironmentFingerprintGenerator {
         "DYLIB_COMPATIBILITY_VERSION",
         "DYLIB_CURRENT_VERSION",
         "PRODUCT_MODULE_NAME",
-        "ARCHS"
+        "ARCHS",
     ]
     private let version: String
     private let customFingerprintEnvs: [String]
@@ -52,7 +52,9 @@ class EnvironmentFingerprintGenerator {
         }
         try fill(envKeys: Self.defaultEnvFingerprintKeys + customFingerprintEnvs)
         try generator.append(version)
-        return try generator.generate()
+        let result = try generator.generate()
+        generatedFingerprint = result
+        return result
     }
 
     /// Creates a fingerprint of the environemtn, by hashing all ENVs specified in keys

Sources/XCRemoteCache/FlowControl/RemoteCommitInfo.swift

@@ -27,7 +27,7 @@ enum RemoteCommitInfo: Equatable {
 extension RemoteCommitInfo {
     init(_ commit: String?) {
         switch commit {
-        case .some(let value) where !value.isEmpty :
+        case .some(let value) where !value.isEmpty:
             self = .available(commit: value)
         default:
             self = .unavailable

Sources/XCRemoteCache/Logger/Logger.swift

@@ -22,34 +22,36 @@ import Foundation
 import os.log
 
 
+private var processTag: String = ""
+
 public func exit(_ exitCode: Int32, _ message: String) -> Never {
-    os_log("%{public}@", log: OSLog.default, type: .error, message)
+    os_log("%{public}@%{public}@", log: OSLog.default, type: .error, processTag, message)
     printError(errorMessage: message)
     exit(exitCode)
 }
 
 func defaultLog(_ message: String) {
-    os_log("%{public}@", log: OSLog.default, type: .default, message)
+    os_log("%{public}@%{public}@", log: OSLog.default, type: .default, processTag, message)
 }
 
 func errorLog(_ message: String) {
-    os_log("%{public}@", log: OSLog.default, type: .error, message)
+    os_log("%{public}@%{public}@", log: OSLog.default, type: .error, processTag, message)
 }
 
 func infoLog(_ message: String) {
-    os_log("%{public}@", log: OSLog.default, type: .info, message)
+    os_log("%{public}@%{public}@", log: OSLog.default, type: .info, processTag, message)
 }
 
 func debugLog(_ message: String) {
-    os_log("%{public}@", log: OSLog.default, type: .debug, message)
+    os_log("%{public}@%{public}@", log: OSLog.default, type: .debug, processTag, message)
 }
 
 func printError(errorMessage: String) {
-    fputs("error: \(errorMessage)\n", stderr)
+    fputs("error: \(processTag)\(errorMessage)\n", stderr)
 }
 
 func printWarning(_ message: String) {
-    print("warning: \(message)")
+    print("warning: \(processTag)\(message)")
 }
 
 /// Prints a message to the user. It shows in Xcode (if applies) or console output
@@ -57,3 +59,7 @@ func printWarning(_ message: String) {
 func printToUser(_ message: String) {
     print("[RC] \(message)")
 }
+
+func updateProcessTag(_ tag: String) {
+    processTag = "(\(tag)) "
+}

Sources/XCRemoteCache/Models/MetaWriter.swift

@@ -20,7 +20,7 @@
 import Foundation
 
 protocol MetaWriter {
-    func write<T>(_ meta: T, locationDir : URL) throws -> URL where T : Meta
+    func write<T>(_ meta: T, locationDir: URL) throws -> URL where T: Meta
 }
 
 class JsonMetaWriter: MetaWriter {
@@ -36,7 +36,7 @@ class JsonMetaWriter: MetaWriter {
         self.metaEncoder = encoder
     }
 
-    func write<T>(_ meta: T, locationDir : URL) throws -> URL where T : Meta {
+    func write<T>(_ meta: T, locationDir: URL) throws -> URL where T: Meta {
         let metaURL = locationDir.appendingPathComponent(meta.fileKey).appendingPathExtension("json")
         let metaData = try metaEncoder.encode(meta)
         try fileWriter.write(toPath: metaURL.path, contents: metaData)

Sources/XCRemoteCache/Network/Authentication/AWSV4Signature.swift

@@ -23,17 +23,21 @@ struct AWSV4Signature {
 
     let secretKey: String
     let accessKey: String
+    let securityToken: String?
     let region: String
     let service: String
     let date: Date
 
-
     func addSignatureHeaderTo(request: inout URLRequest) {
 
         request.setValue(request.url?.host, forHTTPHeaderField: "host")
         request.setValue(StringToSign.ISO8601BasicFormatter.string(from: date), forHTTPHeaderField: "x-amz-date")
         request.setValue((request.httpBody ?? Data()).sha256(), forHTTPHeaderField: "x-amz-content-sha256")
 
+        if let securityToken = securityToken {
+            request.setValue(securityToken, forHTTPHeaderField: "x-amz-security-token")
+        }
+
         let canonicalRequest = CanonicalRequest(request: request)
         let stringToSign = StringToSign(
             region: region,

Sources/XCRemoteCache/Network/Authentication/CanonicalRequest.swift

@@ -34,7 +34,15 @@ struct CanonicalRequest {
         if url.path.isEmpty {
             path = "/"
         } else {
-            path = url.path
+            if let escapedPath = url.path.addingPercentEncoding(withAllowedCharacters: .urlPathAllowed) {
+                path = escapedPath
+            } else {
+                path = "/"
+                printWarning("""
+Escaping the path \(url.path) failed and a placeholder is used instead. \
+Make sure the path doesn't contain invalid characters.
+""")
+            }
         }
         return
             "\(httpMethod)\n" +

Sources/XCRemoteCache/Network/IgnoringCertificatesTrustManager.swift

@@ -25,7 +25,7 @@ final class IgnoringCertificatesTrustManager: NSObject, URLSessionDelegate {
             completionHandler(.performDefaultHandling, nil)
             return
         }
-        
+
         let urlCredential = URLCredential(trust: serverTrust)
         completionHandler(.useCredential, urlCredential)
     }

Sources/XCRemoteCache/Network/NetworkClientImpl.swift

@@ -29,12 +29,14 @@ class NetworkClientImpl: NetworkClient {
     private let session: URLSession
     private let fileManager: FileManager
     private let maxRetries: Int
+    private let retryDelay: TimeInterval
     private let awsV4Signature: AWSV4Signature?
 
-    init(session: URLSession, retries: Int, fileManager: FileManager, awsV4Signature: AWSV4Signature?) {
+    init(session: URLSession, retries: Int, retryDelay: TimeInterval, fileManager: FileManager, awsV4Signature: AWSV4Signature?) {
         self.session = session
         self.fileManager = fileManager
-        maxRetries = retries
+        self.maxRetries = retries
+        self.retryDelay = retryDelay
         self.awsV4Signature = awsV4Signature
     }
 
@@ -75,7 +77,12 @@ class NetworkClientImpl: NetworkClient {
     func download(_ url: URL, to location: URL, completion: @escaping (Result<Void, NetworkClientError>) -> Void) {
         var request = URLRequest(url: url)
         setupAuthenticationSignatureIfPresent(&request)
-        makeDownloadRequest(request, output: location, completion: completion)
+        makeDownloadRequest(
+            request,
+            output: location,
+            retries: maxRetries,
+            completion: completion
+        )
     }
 
     func upload(_ file: URL, as url: URL, completion: @escaping (Result<Void, NetworkClientError>) -> Void) {
@@ -123,7 +130,7 @@ class NetworkClientImpl: NetworkClient {
         dataTask.resume()
     }
 
-    private func makeDownloadRequest(_ request: URLRequest, output: URL, completion: @escaping (Result<Void, NetworkClientError>) -> Void) {
+    private func makeDownloadRequest(_ request: URLRequest, output: URL, retries: Int, completion: @escaping (Result<Void, NetworkClientError>) -> Void) {
         guard fileManager.fileExists(atPath: output.path) == false else {
             infoLog("Download file found in the destination, skipping download.")
             completion(.success(()))
@@ -133,6 +140,17 @@ class NetworkClientImpl: NetworkClient {
         let dataTask = session.downloadTask(with: request) { [fileManager] fileURL, _, error in
             guard let fileURL = fileURL else {
                 let networkError = error.map(NetworkClientError.build) ?? .inconsistentSession
+                if retries > 0 {
+                    infoLog("Download request failed with \(networkError). Left retries: \(retries).")
+                    self.retryDownload(
+                        request,
+                        output: output,
+                        retries: retries,
+                        completion: completion,
+                        after: self.retryDelay
+                    )
+                    return
+                }
                 errorLog("Download request failed: \(networkError)")
                 completion(.failure(networkError))
                 return
@@ -173,7 +191,13 @@ class NetworkClientImpl: NetworkClient {
             if let error = responseError {
                 if retries > 0 {
                     infoLog("Upload request failed with \(error). Left retries: \(retries).")
-                    self.makeUploadRequest(request, input: input, retries: retries - 1, completion: completion)
+                    self.retryUpload(
+                        request,
+                        input: input,
+                        retries: retries,
+                        completion: completion,
+                        after: self.retryDelay
+                    )
                     return
                 }
                 errorLog("Upload request failed: \(error)")
@@ -184,6 +208,30 @@ class NetworkClientImpl: NetworkClient {
         }
         dataTask.resume()
     }
+
+    private func retryUpload(_ request: URLRequest, input: URL, retries: Int, completion: @escaping (Result<Void, NetworkClientError>) -> Void, after: TimeInterval) {
+        DispatchQueue.global().asyncAfter(deadline: .now() + after) { [weak self] in
+            guard let self = self else { return }
+            self.makeUploadRequest(
+                request,
+                input: input,
+                retries: retries - 1,
+                completion: completion
+            )
+        }
+    }
+
+    private func retryDownload(_ request: URLRequest, output: URL, retries: Int, completion: @escaping (Result<Void, NetworkClientError>) -> Void, after: TimeInterval) {
+        DispatchQueue.global().asyncAfter(deadline: .now() + after) { [weak self] in
+            guard let self = self else { return }
+            self.makeDownloadRequest(
+                request,
+                output: output,
+                retries: retries - 1,
+                completion: completion
+            )
+        }
+    }
 }
 
 private extension NetworkClientError {

Sources/XCRemoteCache/Network/RemoteNetworkClientAbstractFactory.swift

@@ -27,11 +27,20 @@ class RemoteNetworkClientAbstractFactory {
     private let upstreamStreamURL: [URL]
     private let networkClient: NetworkClient
     private let urlBuilderFactory: (URL) throws -> URLBuilder
+    private let uploadBatchSize: Int
 
-    init(mode: Mode, downloadStreamURL: URL, upstreamStreamURL: [URL], networkClient: NetworkClient, urlBuilderFactory: @escaping (URL) throws -> URLBuilder) {
+    init(
+        mode: Mode,
+        downloadStreamURL: URL,
+        upstreamStreamURL: [URL],
+        uploadBatchSize: Int,
+        networkClient: NetworkClient,
+        urlBuilderFactory: @escaping (URL) throws -> URLBuilder
+    ) {
         self.mode = mode
         self.downloadStreamURL = downloadStreamURL
         self.upstreamStreamURL = upstreamStreamURL
+        self.uploadBatchSize = uploadBatchSize
         self.networkClient = networkClient
         self.urlBuilderFactory = urlBuilderFactory
     }
@@ -49,7 +58,8 @@ class RemoteNetworkClientAbstractFactory {
             return ReplicatedRemotesNetworkClient(
                 networkClient,
                 download: downloadURLBuilder,
-                uploads: upstreamBuilders
+                uploads: upstreamBuilders,
+                uploadBatchSize: uploadBatchSize
             )
         case .consumer:
             return RemoteNetworkClientImpl(networkClient, downloadURLBuilder)

Sources/XCRemoteCache/Network/ReplicatedRemotesNetworkClient.swift

@@ -23,10 +23,12 @@ import Foundation
 class ReplicatedRemotesNetworkClient: RemoteNetworkClientImpl {
     private let networkClient: NetworkClient
     private let uploadURLBuilders: [URLBuilder]
+    private let uploadBatchSize: Int
 
-    init(_ networkClient: NetworkClient, download: URLBuilder, uploads uploadURLBuilders: [URLBuilder]) {
+    init(_ networkClient: NetworkClient, download: URLBuilder, uploads uploadURLBuilders: [URLBuilder], uploadBatchSize: Int) {
         self.networkClient = networkClient
         self.uploadURLBuilders = uploadURLBuilders
+        self.uploadBatchSize = uploadBatchSize
         super.init(networkClient, download)
     }
 
@@ -39,6 +41,9 @@ class ReplicatedRemotesNetworkClient: RemoteNetworkClientImpl {
         let group = DispatchGroup()
         var results: [Result<Void, NetworkClientError>] = Array(repeating: .failure(.noResponse), count: urls.count)
         urls.enumerated().forEach { index, url in
+            if uploadBatchSize > 0 && index > 0 && index % uploadBatchSize == 0 {
+                group.wait()
+            }
             group.enter()
             networkClient.upload(file, as: url) { receivedResult in
                 results[index] = receivedResult
@@ -58,6 +63,9 @@ class ReplicatedRemotesNetworkClient: RemoteNetworkClientImpl {
         let group = DispatchGroup()
         var results: [Result<Void, NetworkClientError>] = Array(repeating: .failure(.noResponse), count: urls.count)
         urls.enumerated().forEach { index, url in
+            if uploadBatchSize > 0 && index > 0 && index % uploadBatchSize == 0 {
+                group.wait()
+            }
             group.enter()
             networkClient.create(url) { receivedResult in
                 results[index] = receivedResult

Sources/XCRemoteCache/Network/URLSessionFactory.swift

@@ -36,6 +36,8 @@ class DefaultURLSessionFactory: URLSessionFactory {
         let configuration = URLSessionConfiguration.default
         configuration.httpAdditionalHeaders = config.requestCustomHeaders
         configuration.timeoutIntervalForRequest = config.timeoutResponseDataChunksInterval
+        configuration.urlCache?.memoryCapacity = 0
+        configuration.urlCache?.diskCapacity = 0
         switch config.disableCertificateVerification {
         case true:
             return URLSession(

Sources/XCRemoteCache/Stats/ExclusiveFileAccessor.swift

@@ -76,8 +76,8 @@ class ExclusiveFile: ExclusiveFileAccessor {
         guard flock(fd, LOCK_EX) == 0 else {
             throw FileAccessorError.lockingFailure
         }
-        // While having a lock, make sure the file still exists.
-        // It might delete it while we were waiting for a lock.
+        // While having a lock, make sure the file still exists
+        // It might delete it while we were waiting for a lock
         guard access(fileURL.path, F_OK) == 0 else {
             throw FileAccessorError.lockingFailure
         }

Sources/XCRemoteCache/Utils/Array+Utils.swift

@@ -0,0 +1,29 @@
+// Copyright (c) 2023 Spotify AB.
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import Foundation
+
+public extension Array {
+    func get(_ i: Index) -> Element? {
+        guard count > i else {
+            return nil
+        }
+        return self[i]
+    }
+}

Sources/XCRemoteCache/Utils/ENVReader.swift

@@ -24,8 +24,15 @@ enum EnvironmentError: Error {
 }
 
 extension Dictionary where Key == String, Value == String {
-    func readEnv(key: String) throws -> URL {
+    func readEnv(key: String) -> URL? {
         guard let value = self[key].map(URL.init(fileURLWithPath:)) else {
+            return nil
+        }
+        return value
+    }
+
+    func readEnv(key: String) throws -> URL {
+        guard let value: URL = readEnv(key: key) else {
             throw EnvironmentError.missingEnv(key)
         }
         return value
@@ -48,4 +55,11 @@ extension Dictionary where Key == String, Value == String {
         }
         return value == "YES"
     }
+
+    func readEnv(key: String) throws -> Bool? {
+        guard let value = self[key] else {
+            return nil
+        }
+        return value == "YES"
+    }
 }

Sources/xcld/XCLDMain.swift

@@ -21,7 +21,7 @@ import Foundation
 import XCRemoteCache
 
 /// Wrapper for a `LD` program that copies the dynamic executable from a cached-downloaded location
-/// Fallbacks to a standard `clang` when the Ramote cache is not applicable (e.g. modified sources)
+/// Fallbacks to a standard `clang` when the Remote cache is not applicable (e.g. modified sources)
 public class XCLDMain {
     public func main() {
         let args = ProcessInfo().arguments
@@ -48,7 +48,16 @@ public class XCLDMain {
             i += 1
         }
         guard let outputInput = output, let filelistInput = filelist, let dependencyInfoInput = dependencyInfo else {
-            exit(1, "Missing 'output' argument. Args: \(args)")
+            let ldCommand = "clang"
+            print("Fallbacking to compilation using \(ldCommand).")
+
+            let args = ProcessInfo().arguments
+            let paramList = [ldCommand] + args.dropFirst()
+            let cargs = paramList.map { strdup($0) } + [nil]
+            execvp(ldCommand, cargs)
+
+            /// C-function `execv` returns only when the command fails
+            exit(1)
         }
 
 

Sources/xcldplusplus/XCLDPlusPlus.swift

@@ -0,0 +1,75 @@
+// Copyright (c) 2021 Spotify AB.
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import Foundation
+import XCRemoteCache
+
+/// Wrapper for a `LDPLUSPLUS` program that copies the dynamic executable from a cached-downloaded location
+/// Fallbacks to a standard `clang++` when the Remote cache is not applicable (e.g. modified sources)
+public class XCLDPlusPlusMain {
+    public func main() {
+        let args = ProcessInfo().arguments
+        var output: String?
+        var filelist: String?
+        var dependencyInfo: String?
+        var i = 0
+        while i < args.count {
+            switch args[i] {
+            case "-o":
+                output = args[i + 1]
+                i += 1
+            case "-filelist":
+                filelist = args[i + 1]
+                i += 1
+            case "-dependency_info":
+                // Skip following `-Xlinker` argument. Sample call:
+                // `clang -dynamiclib  ... -Xlinker -dependency_info -Xlinker /path/Target_dependency_info.dat`
+                dependencyInfo = args[i + 2]
+                i += 2
+            default:
+                break
+            }
+            i += 1
+        }
+        guard let outputInput = output, let filelistInput = filelist, let dependencyInfoInput = dependencyInfo else {
+            let ldCommand = "clang++"
+            print("Fallbacking to compilation using \(ldCommand).")
+
+            let args = ProcessInfo().arguments
+            let paramList = [ldCommand] + args.dropFirst()
+            let cargs = paramList.map { strdup($0) } + [nil]
+            execvp(ldCommand, cargs)
+
+            /// C-function `execv` returns only when the command fails
+            exit(1)
+        }
+
+
+        // TODO: consider using `clang_command` from .rcinfo
+        /// concrete clang path should be taken from the current toolchain
+        let fallbackCommand = "clang++"
+        XCCreateBinary(
+            output: outputInput,
+            filelist: filelistInput,
+            dependencyInfo: dependencyInfoInput,
+            fallbackCommand: fallbackCommand,
+            stepDescription: "xcldplusplus"
+        ).run()
+    }
+}

Sources/xcldplusplus/main.swift

@@ -0,0 +1,22 @@
+// Copyright (c) 2021 Spotify AB.
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import XCRemoteCache
+
+XCLDPlusPlusMain().main()

Sources/xclibtool/XCLibtoolMain.swift

@@ -18,53 +18,24 @@
 // under the License.
 
 import Foundation
+import xclibtoolSupport
 import XCRemoteCache
 
+public enum XCLibtoolMainError: Error {
+    case missingOutput
+    case unsupportedMode
+}
+
 /// Wrapper for a `libtool` program that copies the build executable (e.g. .a) from a cached-downloaded location
 /// Fallbacks to a standard `libtool` when the Ramote cache is not applicable (e.g. modified sources)
 public class XCLibtoolMain {
+    public init() { }
+
     public func main() {
         let args = ProcessInfo().arguments
-        var output: String?
-        // all input arguments library '.a'. Used to create an universal binary
-        var inputLibraries: [String] = []
-        var filelist: String?
-        var dependencyInfo: String?
-        var i = 0
-        while i < args.count {
-            switch args[i] {
-            case "-o":
-                output = args[i + 1]
-                i += 1
-            case "-filelist":
-                filelist = args[i + 1]
-                i += 1
-            case "-dependency_info":
-                dependencyInfo = args[i + 1]
-                i += 1
-            case let input where input.hasSuffix(".a"):
-                inputLibraries.append(input)
-            default:
-                break
-            }
-            i += 1
-        }
-        guard let outputInput = output else {
-            exit(1, "Missing 'output' argument. Args: \(args)")
-        }
 
-        let mode: XCLibtoolMode
-        if let filelistInput = filelist, let dependencyInfoInput = dependencyInfo {
-            // libtool is creating a library
-            mode = .createLibrary(output: outputInput, filelist: filelistInput, dependencyInfo: dependencyInfoInput)
-        } else if !inputLibraries.isEmpty {
-            // multiple input libraries suggest creating an universal binary
-            mode = .createUniversalBinary(output: outputInput, inputs: inputLibraries)
-        } else {
-            // unknown mode
-            exit(1, "Unsupported mode. Args: \(args)")
-        }
         do {
+            let mode = try XCLibtoolHelper.buildMode(args: Array(args.dropFirst()))
             try XCLibtool(mode).run()
         } catch {
             exit(1, "Failed with: \(error). Args: \(args)")

Sources/xclibtoolSupport/XCLibtoolHelper.swift

@@ -0,0 +1,80 @@
+// Copyright (c) 2023 Spotify AB.
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import Foundation
+import XCRemoteCache
+
+public enum XCLibtoolHelperError: Error {
+    case missingOutput
+    case unsupportedMode
+}
+
+public class XCLibtoolHelper {
+    public static func buildMode(args: [String]) throws -> XCLibtoolMode {
+        var output: String?
+        // all input arguments are '*.a' or no path extension. Used to create an universal binary
+        var inputLibraries: [String] = []
+        var filelist: String?
+        var dependencyInfo: String?
+        var asksForVersion = false
+        var i = 0
+        while i < args.count {
+            switch args[i] {
+            case "-V":
+                asksForVersion = true
+            case "-o":
+                output = args[i + 1]
+                i += 1
+            case "-filelist":
+                filelist = args[i + 1]
+                i += 1
+            case "-dependency_info":
+                dependencyInfo = args[i + 1]
+                i += 1
+            case let input where input.starts(with: "/") &&
+                ["", "a"].contains(URL(fileURLWithPath: input).pathExtension):
+                // Assume always absolute paths to the library
+                // Support for static frameworks (no extension) and static libraries (.a)
+                inputLibraries.append(input)
+            default:
+                break
+            }
+            i += 1
+        }
+        if asksForVersion {
+            return .version
+        }
+        guard let outputInput = output else {
+            throw XCLibtoolHelperError.missingOutput
+        }
+
+        let mode: XCLibtoolMode
+        if let filelistInput = filelist, let dependencyInfoInput = dependencyInfo {
+            // libtool is creating a library
+            mode = .createLibrary(output: outputInput, filelist: filelistInput, dependencyInfo: dependencyInfoInput)
+        } else if !inputLibraries.isEmpty {
+            // multiple input libraries suggest creating an universal binary
+            mode = .createUniversalBinary(output: outputInput, inputs: inputLibraries)
+        } else {
+            // unknown mode
+            throw XCLibtoolHelperError.unsupportedMode
+        }
+        return mode
+    }
+}

Sources/xclipo/XCLipoMain.swift

@@ -0,0 +1,71 @@
+// Copyright (c) 2023 Spotify AB.
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import Foundation
+import XCRemoteCache
+
+/// Wrapper for a `lipo` program that links any of input binaries to the destination paths
+/// Fallbacks to a standard `lipo` when the Ramote cache is not applicable (e.g. modified sources)
+public class XCLipoMain {
+    public init() { }
+
+    public func main() {
+        let args = ProcessInfo().arguments
+        var output: String?
+        var create = false
+        var inputs: [String] = []
+
+        var i = 1
+        while i < args.count {
+            switch args[i] {
+            case "-output":
+                output = args[i + 1]
+                i += 1
+            case "-create":
+                create = true
+            default:
+                inputs.append(args[i])
+            }
+            i += 1
+        }
+        let lipoCommand = "lipo"
+        guard let output = output, !inputs.isEmpty, create else {
+            print("Fallbacking to compilation using \(lipoCommand).")
+
+            let args = ProcessInfo().arguments
+            let paramList = [lipoCommand] + args.dropFirst()
+            let cargs = paramList.map { strdup($0) } + [nil]
+            execvp(lipoCommand, cargs)
+
+            /// C-function `execv` returns only when the command fails
+            exit(1)
+        }
+
+        do {
+            try XCLipo(
+                output: output,
+                inputs: inputs,
+                fallbackCommand: lipoCommand,
+                stepDescription: "xclipo"
+            ).run()
+        } catch {
+            exit(1, "Failed with: \(error). Args: \(args)")
+        }
+    }
+}

Sources/xclipo/main.swift

@@ -0,0 +1,22 @@
+// Copyright (c) 2023 Spotify AB.
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import XCRemoteCache
+
+XCLipoMain().main()