Merge pull request #85 from polac24/20220214-remapper-stable

Do not change dependencies order when remapping overlay

.github/ISSUE_TEMPLATE/BUG_REPORT.md

@@ -0,0 +1,65 @@
+---
+name: ⚠️ Bug Report
+about: Something isn't working as expected
+
+---
+
+<!--
+PLEASE HELP US PROCESS GITHUB ISSUES FASTER BY PROVIDING THE FOLLOWING INFORMATION.
+-->
+
+**My integration setup**
+
+[ ] CocoaPods cocoapods-xcremotecache plugin
+[ ] Automatic integration using `xcprepare integrate ...`
+[ ] Manual integration
+[ ] Carthage
+
+**Expected/desired behavior**
+<!-- Describe what the desired behavior would be. -->
+
+**Minimal reproduction of the problem with instructions**
+<!-- Please provide the *STEPS TO REPRODUCE*. -->
+
+**Producer Logs**
+<!-- Capture logs from 10 minutes: `log show --predicate 'sender BEGINSWITH "xc"' --style compact --info --debug -last 10m` -->
+
+<details>
+  <pre> [REPLACE THIS WITH YOUR INFORMATION] </pre>
+</details>
+
+**Consumer Logs**
+<!-- Capture logs from 10 minutes: `log show --predicate 'sender BEGINSWITH "xc"' --style compact --info --debug -last 10m` -->
+
+<details>
+  <pre> [REPLACE THIS WITH YOUR INFORMATION] </pre>
+</details>
+
+**Pods/Carthage file**
+<!-- Delete if you don't use CocoaPods or Carthage -->
+
+<details>
+  <pre> [REPLACE THIS WITH YOUR INFORMATION] </pre>
+</details>
+
+**Environment**
+
+* **XCRemoteCache:** X.Y.Z
+* **cocoapods-xcremotecache:** X.Y.Z <!-- check with `gem list cocoapods-xcremotecache` >
+* **HTTP cache server:** ... <!-- e.g. demo docker, nginx, AWS etc. >
+* **Xcode:** X.Y.Z
+
+**Post build stats**
+<!-- 
+To capture build statistics: 
+* call `xcprepare stats --reset` (or `XCRC/xcprepare stats --reset` for CocoaPods)
+* Build a project in Xcode
+* `xcprepare stats` (or `XCRC/xcprepare stats` for CocoaPods) 
+-->
+
+<details>
+  <pre> [REPLACE THIS WITH YOUR INFORMATION] </pre>
+</details>
+
+**Others**
+<!-- Anything else relevant?  Operating system version, , ... -->

.github/ISSUE_TEMPLATE/DOCUMENTATION_ISSUE.md

@@ -0,0 +1,16 @@
+---
+name: 📕 Documentation Issue
+about: Suggestion for a change in a documentation
+
+---
+
+<!--
+PLEASE HELP US PROCESS GITHUB ISSUES FASTER BY PROVIDING THE FOLLOWING INFORMATION.
+-->
+
+**A suggestion**
+<!-- Describe how could the documentation be improved. -->
+
+**Which file, section, line**
+<!-- Provide a section it relates (if exist). -->
+

.github/ISSUE_TEMPLATE/FEATURE_REQUEST.md

@@ -0,0 +1,20 @@
+---
+name: 🙏 Future Request
+about: Suggestion for an improvement, either behaviour or implementation
+
+---
+
+<!--
+PLEASE HELP US PROCESS GITHUB ISSUES FASTER BY PROVIDING THE FOLLOWING INFORMATION.
+-->
+
+
+**Expected/desired behavior**
+<!-- Describe what the desired behavior would be. -->
+
+**Relevant integration setup**
+
+[ ] CocoaPods cocoapods-xcremotecache plugin
+[ ] Automatic integration using `xcprepare integrate ...`
+[ ] Manual integration
+[ ] Carthage

.github/workflows/ci.yaml

@@ -13,7 +13,7 @@ jobs:
   macOS:
     runs-on: macOS-latest
     env:
-      XCODE_VERSION: ${{ '12.4' }}
+      XCODE_VERSION: ${{ '13.1' }}
     steps:
       - name: Select Xcode
         run: "sudo xcode-select -s /Applications/Xcode_$XCODE_VERSION.app"
@@ -23,3 +23,5 @@ jobs:
         run: rake build[release]
       - name: Test
         run: rake test
+      - name: E2ETests
+        run: rake e2e_only

.github/workflows/release.yaml

@@ -8,7 +8,7 @@ jobs:
     name: Add macOS binaries to release
     runs-on: macOS-latest
     env:
-      XCODE_VERSION: ${{ '12.4' }}
+      XCODE_VERSION: ${{ '13.1' }}
     steps:
       - name: Select Xcode
         run: "sudo xcode-select -s /Applications/Xcode_$XCODE_VERSION.app"
@@ -44,3 +44,29 @@ jobs:
           file_glob: true
           tag: ${{ github.ref }}
           overwrite: true
+  cocoapods:
+    name: Publish CocoaPods plugin
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    defaults:
+      run:
+        working-directory: cocoapods-plugin
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: '3.0'
+      - run: bundle install
+      - name: Publish to RubyGems
+        run: |
+          mkdir -p $HOME/.gem
+          touch $HOME/.gem/credentials
+          chmod 0600 $HOME/.gem/credentials
+          printf -- "---\n:rubygems_api_key: ${GEM_HOST_API_KEY}\n" > $HOME/.gem/credentials
+          gem build *.gemspec
+          CURRENT_VERSION=$(gem list cocoapods-xcremotecache  --remote  -q | sed 's/[^0-9\.]//g')
+          [ -f cocoapods-xcremotecache-$CURRENT_VERSION.gem ] && echo "Version $CURRENT_VERSION already exists" || gem push *.gem
+        env:
+          GEM_HOST_API_KEY: "${{secrets.RUBYGEMS_AUTH_TOKEN}}"

.gitignore

@@ -1,8 +1,13 @@
 .DS_Store
 /.build
 /Packages
-/*.xcodeproj
+*.xcodeproj/
+*.xcworkspace/
 DerivedData
 /.swiftpm/
 releases
-tmp/
+tmp/
+.idea/
+xcuserdata
+*.gem
+Pods/

.swiftlint.yml

@@ -5,6 +5,7 @@ disabled_rules:
   - superfluous_disable_command # Disabled since we disable some rules pre-emptively to avoid issues in the future
   - todo                        # Temporarily disabled. We have too many right now hiding real issues :(
   - nesting                     # Does not make sense anymore since Swift 4 uses nested `CodingKeys` enums for example
+  - trailing_dot_in_comments    # Triggers warnings for generated file headers
 
 opt_in_rules:
   - anyobject_protocol

Package.swift

@@ -1,4 +1,4 @@
-// swift-tools-version:5.1
+// swift-tools-version:5.3
 // The swift-tools-version declares the minimum version of Swift required to build this package
 
 import PackageDescription
@@ -56,7 +56,8 @@ let package = Package(
         ),
         .testTarget(
             name: "XCRemoteCacheTests",
-            dependencies: ["XCRemoteCache"]
+            dependencies: ["XCRemoteCache"],
+            resources: [.copy("TestData")]
         ),
     ]
 )

README.md

@@ -1,13 +1,15 @@
 <p align="center">
-    <img src="docs/img/logo.png" width="75%">
+    <img src="docs/img/logo.png#gh-light-mode-only" width="75%">
+    <img src="docs/img/logo-dark.png#gh-dark-mode-only" width="75%">
 </p>
 
 _XCRemoteCache is a remote cache tool for Xcode projects. It reuses target artifacts generated on a remote machine, served from a simple REST server._
 
 [![Build Status](https://github.com/spotify/XCRemoteCache/workflows/CI/badge.svg)](https://github.com/spotify/XCRemoteCache/workflows/CI/badge.svg)
 [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](LICENSE)
+[![Slack](https://slackin.spotify.com/badge.svg)](https://slackin.spotify.com)
 
-- [How and Why?](#how-and-why-)
+- [How and Why?](#how-and-why)
   * [Accurate target input files](#accurate-target-input-files)
     + [New file added to the target](#new-file-added-to-the-target)
   * [Debug symbols](#debug-symbols)
@@ -36,10 +38,14 @@ _XCRemoteCache is a remote cache tool for Xcode projects. It reuses target artif
   * [Amazon S3 and Google Cloud Storage](#amazon-s3-and-google-cloud-storage)
 - [CocoaPods plugin](#cocoapods-plugin)
 - [Requirements](#requirements)
+- [Apple silicon support](#apple-silicon-support)
+  * [Artifacts per architecture (Recommended)](#artifacts-per-architecture-recommended)
+  * [Fat artifacts](#fat-artifacts)
 - [Limitations](#limitations)
 - [FAQ](#faq)
 - [Development](#development)
 - [Release](#release)
+  * [Releasing CocoaPods plugin](#releasing-cocoapods-plugin)
   * [Building release package](#building-release-package)
 - [Contributing](#contributing)
 - [Code of conduct](#code-of-conduct)
@@ -50,7 +56,7 @@ _XCRemoteCache is a remote cache tool for Xcode projects. It reuses target artif
 
 The caching mechanism is based on remote artifacts that should be generated and uploaded to the cache server for each commit on a `master` branch, preferably as a part of CI/CD step. Xcode products are not portable between different Xcode versions, each XCRemoteCache artifact is linked with a specific Xcode build number that generated it. To support multiple Xcode versions, artifacts generation should happen for each Xcode version.
 
-The artifact resue flow is as follows: XCRemoteCache performs a target precheck (aka prebuild) and if a fingerprint for local sources matches the one computed on a generation side, several compilation steps wrappers (e.g. `xcswiftc`, `xccc`, `xclibtool`) mock corresponding compilation step(s) and linking (or archiving) moves the cached build artifact to the expected location.
+The artifact reuse flow is as follows: XCRemoteCache performs a target precheck (aka prebuild) and if a fingerprint for local sources matches the one computed on a generation side, several compilation steps wrappers (e.g. `xcswiftc`, `xccc`, `xclibtool`) mock corresponding compilation step(s) and linking (or archiving) moves the cached build artifact to the expected location.
 
 > Multiple commits that have the same target sources reuse artifact package on a remote server.
 
@@ -134,7 +140,7 @@ xcremotecache/xcprepare integrate --input <yourProject.xcodeproj> --mode consume
 | Argument | Description | Default | Required |
 | ------------- | ------------- | ------------- | ------------- |
 | `--input` | .xcodeproj location  | N/A | ✅ |
-| `--mode` | mode. Supported values: `consumer`, `producer` | N/A | ✅ |
+| `--mode` | mode. Supported values: `consumer`, `producer`, `producer-fast`(experimental) | N/A | ✅ |
 | `--targets-include` | comma-separated list of targets to integrate XCRemoteCache. | `""` | ⬜️ |
 | `--targets-exclude` | comma-separated list of targets to not integrate XCRemoteCache. Takes priority over --targets-include.  | `""` | ⬜️ |
 | `--configurations-include` | comma-separated list of configurations to integrate XCRemoteCache.  | `""` | ⬜️ |
@@ -189,6 +195,7 @@ Configure Xcode targets that **should use** XCRemoteCache:
 * `SWIFT_EXEC` - location of `xcprepare` (e.g. `xcremotecache/xcswiftc`)
 * `LIBTOOL` - location of `xclibtool` (e.g. `xcremotecache/xclibtool`)
 * `LD` - location of `xcld` (e.g. `xcremotecache/xcld`)
+* `XCRC_PLATFORM_PREFERRED_ARCH` - `$(LINK_FILE_LIST_$(CURRENT_VARIANT)_$(PLATFORM_PREFERRED_ARCH):dir:standardizepath:file:default=arm64)`
 
 <details>
   <summary>Screenshot</summary>
@@ -208,8 +215,8 @@ Configure Xcode targets that **should use** XCRemoteCache:
 * command: `"$SCRIPT_INPUT_FILE_0"`
 * input files: location of `xcpostbuild` command (e.g. `xcremotecache/xcpostbuild`)
 * output files: 
-  * `$(TARGET_BUILD_DIR)/$(MODULES_FOLDER_PATH)/$(PRODUCT_MODULE_NAME).swiftmodule/$(PLATFORM_PREFERRED_ARCH).swiftmodule.md5`
-  * `$(TARGET_BUILD_DIR)/$(MODULES_FOLDER_PATH)/$(PRODUCT_MODULE_NAME).swiftmodule/$(PLATFORM_PREFERRED_ARCH)-$(LLVM_TARGET_TRIPLE_VENDOR)-$(SWIFT_PLATFORM_TARGET_PREFIX)$(LLVM_TARGET_TRIPLE_SUFFIX).swiftmodule.md5`
+  * `$(TARGET_BUILD_DIR)/$(MODULES_FOLDER_PATH)/$(PRODUCT_MODULE_NAME).swiftmodule/$(XCRC_PLATFORM_PREFERRED_ARCH).swiftmodule.md5`
+  * `$(TARGET_BUILD_DIR)/$(MODULES_FOLDER_PATH)/$(PRODUCT_MODULE_NAME).swiftmodule/$(XCRC_PLATFORM_PREFERRED_ARCH)-$(LLVM_TARGET_TRIPLE_VENDOR)-$(SWIFT_PLATFORM_TARGET_PREFIX)$(LLVM_TARGET_TRIPLE_SUFFIX).swiftmodule.md5`
 * discovery dependency file: `$(TARGET_TEMP_DIR)/postbuild.d`
 
 <details>
@@ -298,11 +305,14 @@ _Note that for the `producer` mode, the prebuild build phase and `xccc`, `xcld`,
 | `product_files_extensions_with_content_override ` | List of all extensions that should carry over source fingerprints. Extensions of all product files that contain non-deterministic content (absolute paths, timestamp, etc) should be included. | `["swiftmodule"]` | ⬜️ |
 | `thinning_enabled ` | If true, support for thin projects is enabled | `false` | ⬜️ |
 | `thinning_target_module_name ` | Module name of a target that works as a helper for thinned targets | `"ThinningRemoteCacheModule"` | ⬜️ |
+| `prettify_meta_files` | A Boolean value that opts-in pretty JSON formatting for meta files | `false` | ⬜️ |
 | `aws_secret_key` | Secret key for AWS V4 Signature Authorization. If this is set to a non-empty String - an Authentication Header will be added based on this and the other `aws_*` parameters.| `""` | ⬜️ |
 | `aws_access_key` | Access key for AWS V4 Signature Authorization. | `""` | ⬜️ |
 | `aws_region` | Region for AWS V4 Signature Authorization. E.g. `eu`.  | `""` | ⬜️ |
 | `aws_service` | Service for AWS V4 Signature Authorization. E.g. `storage`. | `""` | ⬜️ |
-
+| `out_of_band_mappings` | A dictionary of files path remapping that should be applied to make it absolute path agnostic on a list of dependencies. Useful if a project refers files out of repo root, either compilation files or precompiled dependencies. Keys represent generic replacement and values are substrings that should be replaced. Example: for mapping `["COOL_LIBRARY": "/CoolLibrary"]` `/CoolLibrary/main.swift`will be represented as `$(COOL_LIBRARY)/main.swift`). Warning: remapping order is not-deterministic so avoid remappings with multiple matchings. | `[:]` | ⬜️ |
+| `disable_certificate_verification` | A Boolean value that opts-in SSL certificate validation is disabled | `false` | ⬜️ |
+| `disable_vfs_overlay` | A feature flag to disable virtual file system overlay support (temporary) | `false` | ⬜️ |
 
 ## Backend cache server
 
@@ -358,6 +368,31 @@ Retention Policy: Buckets usually have a retention policy option which ensures o
 
 Head over to our [cocoapods-plugin](cocoapods-plugin/README.md) docs to see how to integrate XCRemoteCache in your CocoaPods project.
 
+## Apple silicon support
+
+### Artifacts per architecture (Recommended)
+
+_If all of your machines (both producer and all consumers have the same architecture, either Intel or Apple Silicon), you don't have to do anything._
+
+XCRemoteCache supports building artifacts for Apple silicon consumers. Is it recommended to build separately for `x86_64` and `arm64` architectures to have single-architecture artifacts that do not require downloading irrelevant binaries. Here are required steps if you want to support both Intel and Apple silicon consumers.
+
+* Building for a simulator on a producer: run a first build for `x86_64`, clean a build and build again for `arm64`, e.g.:
+```
+xcodebuild ARCHS=x86_64 ONLY_ACTIVE_ARCH=NO build ...
+xcodebuild clean
+xcodebuild ARCHS=arm64 ONLY_ACTIVE_ARCH=NO build ...
+```
+
+### Fat artifacts
+
+If you prefer to generate far artifacts (with both Intel and Apple silicon binaries), you can disable "Build Archive Architecture Only" on a producer side, e.g.
+
+```
+xcodebuild ONLY_ACTIVE_ARCH=NO build ...
+```
+
+Note: This setup is not recommended and may not be supported in future XCRemoteCache releases.
+
 ## Requirements
 
 * The repo under `git` version control
@@ -371,7 +406,7 @@ Head over to our [cocoapods-plugin](cocoapods-plugin/README.md) docs to see how
 
 ## Limitations
 
-* Swift Package Manager (SPM) projects are not supported
+* Swift Package Manager (SPM) dependencies are not supported. _Because SPM does not allow customizing Build Settings, XCRemoteCache cannot specify `clang` and `swiftc` wrappers that control if the local compilation should be skipped (cache hit) or not (cache miss)_
 * Filenames with `_vers.c` suffix are reserved and cannot be used as a source file
 * All compilation files should be referenced via the git repo root. Referencing `/AbsolutePath/someOther.swift` or `../../someOther.swift` that resolve to the location outside of the git repo root is prohibited.
 
@@ -388,6 +423,12 @@ Follow the [Development](docs/Development.md) guide. It has all the information
 To release a version, in [Releases](https://github.com/spotify/XCRemoteCache/releases) draft a new release with `v0.3.0{-rc0}` tag format. 
 Packages with binaries will be automatically uploaded to the GitHub [Releases](https://github.com/spotify/XCRemoteCache/releases) page.
 
+### Releasing CocoaPods plugin
+
+Bump a gem version defined in [gem_version.rb](cocoapods-plugin/lib/cocoapods-xcremotecache/gem_version.rb) and create a new release described above.
+
+A plugin is automatically uploaded to [RubyGems](https://rubygems.org/gems/cocoapods-xcremotecache) if a given version doesn't exist yet.
+
 ### Building release package
 
 To build a release zip package for a single platform (e.g. `x86_64-apple-macosx`, `arm64-apple-macosx`), call:
@@ -398,6 +439,12 @@ rake 'build[release, x86_64-apple-macosx]'
 
 The zip package will be generated at `releases/XCRemoteCache.zip`.
 
+## Support
+
+Create a [new issue](https://github.com/spotify/XCRemoteCache/issues/new) with as many details as possible.
+
+Reach us at the `#xcremotecache` channel in [Slack](https://slackin.spotify.com/).
+
 ## Contributing
 
 We feel that a welcoming community is important and we ask that you follow Spotify's 

Rakefile

@@ -1,4 +1,5 @@
 # encoding: utf-8
+require_relative 'tasks/e2e'
 
 ################################
 # Rake configuration
@@ -28,7 +29,7 @@ task :lint => [:prepare] do
   puts 'Run linting'
 
   system("swiftformat --lint --config .swiftformat --cache ignore .") or abort "swiftformat failure" if SWIFTFORMAT_ENABLED
-  system("swiftlint lint --config .swiftlint.yml") or abort "swiftlint failure" if SWIFTLINT_ENABLED
+  system("swiftlint lint --config .swiftlint.yml --strict") or abort "swiftlint failure" if SWIFTLINT_ENABLED
 end
 
 task :autocorrect => [:prepare]  do 
@@ -77,6 +78,12 @@ task :test do
   spm_test()
 end
 
+desc 'build and run E2E tests'
+task :e2e => [:build, :e2e_only]
+
+desc 'run E2E tests without building the XCRemoteCache binary'
+task :e2e_only => ['e2e:run']
+
 ################################
 # Helper functions
 ################################

Sources/XCRemoteCache/Artifacts/ArtifactCreator.swift

@@ -41,6 +41,7 @@ class BuildArtifactCreator: ArtifactSwiftProductsBuilderImpl, ArtifactCreator {
     private let moduleName: String?
     private let modulesFolderPath: String
     private let dSYMPath: URL
+    private let metaWriter: MetaWriter
     private let fileManager: FileManager
 
     init(
@@ -50,6 +51,7 @@ class BuildArtifactCreator: ArtifactSwiftProductsBuilderImpl, ArtifactCreator {
         moduleName: String?,
         modulesFolderPath: String,
         dSYMPath: URL,
+        metaWriter: MetaWriter,
         fileManager: FileManager
     ) {
         self.buildDir = buildDir
@@ -59,6 +61,7 @@ class BuildArtifactCreator: ArtifactSwiftProductsBuilderImpl, ArtifactCreator {
         self.moduleName = moduleName
         self.fileManager = fileManager
         self.dSYMPath = dSYMPath
+        self.metaWriter = metaWriter
         super.init(workingDir: tempDir, moduleName: moduleName, fileManager: fileManager)
     }
 
@@ -72,7 +75,11 @@ class BuildArtifactCreator: ArtifactSwiftProductsBuilderImpl, ArtifactCreator {
         let dynamicLibraryArtifacts = try prepareDynamicLibraryArtifacts()
         zipPaths.append(contentsOf: dynamicLibraryArtifacts)
 
-        let creator = ZipArtifactCreator(workingDir: zipWorkingDir, fileManager: fileManager)
+        let creator = ZipArtifactCreator(
+            workingDir: zipWorkingDir,
+            metaWriter: metaWriter,
+            fileManager: fileManager
+        )
         return try creator.createArtifact(zipContent: zipPaths, artifactKey: artifactKey, meta: meta)
     }
 

Sources/XCRemoteCache/Artifacts/ArtifactSwiftProductsBuilder.swift

@@ -51,8 +51,6 @@ protocol ArtifactSwiftProductsBuilder {
 /// # {workingDir}/xccache/produced/include/#{moduleName} (if `moduleName` is defined)
 class ArtifactSwiftProductsBuilderImpl: ArtifactSwiftProductsBuilder {
 
-    /// List of all required swiftmodule related extensions that should be copied to the artifact
-    private static let swiftmoduleExtensionsToInclude = ["swiftmodule", "swiftdoc", "swiftsourceinfo"]
     private let workingDir: URL
     private let moduleName: String?
     private let fileManager: FileManager

Sources/XCRemoteCache/Artifacts/SwiftmoduleFileExtension.swift

@@ -30,6 +30,7 @@ enum SwiftmoduleFileExtension: String {
     case swiftmodule
     case swiftdoc
     case swiftsourceinfo
+    case swiftinterface
 }
 
 extension SwiftmoduleFileExtension {
@@ -38,5 +39,6 @@ extension SwiftmoduleFileExtension {
         .swiftmodule: .required,
         .swiftdoc: .required,
         .swiftsourceinfo: .optional,
+        .swiftinterface: .optional,
     ]
 }

Sources/XCRemoteCache/Artifacts/ZipArtifactCreator.swift

@@ -23,11 +23,12 @@ import Zip
 class ZipArtifactCreator {
     /// Location where zip file should be generated
     private let workingDir: URL
+    private let metaWriter: MetaWriter
     private let fileManager: FileManager
-    private let metaEncoder = JSONEncoder()
 
-    init(workingDir: URL, fileManager: FileManager) {
+    init(workingDir: URL, metaWriter: MetaWriter, fileManager: FileManager) {
         self.workingDir = workingDir
+        self.metaWriter = metaWriter
         self.fileManager = fileManager
     }
 
@@ -35,18 +36,10 @@ class ZipArtifactCreator {
         let zipURL = workingDir.appendingPathComponent("\(artifactKey).zip")
         try fileManager.createDirectory(at: workingDir, withIntermediateDirectories: true, attributes: nil)
         // Include meta json to the artifact
-        let metaURL = try dumpMeta(meta)
+        let metaURL = try metaWriter.write(meta, locationDir: workingDir)
         let zipPaths = zipContent + [metaURL]
 
         try Zip.zipFiles(paths: zipPaths, zipFilePath: zipURL, password: nil, progress: nil)
         return Artifact(id: artifactKey, package: zipURL, meta: metaURL)
     }
-
-    // Save meta to a local file
-    private func dumpMeta<T: Meta>(_ meta: T) throws -> URL {
-        let metaURL = workingDir.appendingPathComponent(meta.fileKey).appendingPathExtension("json")
-        let metaData = try metaEncoder.encode(meta)
-        try fileManager.spt_writeToFile(atPath: metaURL.path, contents: metaData)
-        return metaURL
-    }
 }

Sources/XCRemoteCache/Commands/Plugins/Thinning/ThinningCreatorPlugin.swift

@@ -33,13 +33,16 @@ enum ThinningCreatorPluginError: Error {
 /// Warning! This plugin assumes that producer's DerivedData are always cleaned before a build
 class ThinningCreatorPlugin: ArtifactCreatorPlugin {
     private let targetTempDir: URL
+    private let modeMarkerPath: String
     private let dirScanner: DirScanner
 
     /// Default Initializer
     /// - Parameter targetTempDir: Location of current target-specific temp dir (TARGET_TEMP_DIR)
+    /// - Parameter modeMarkerPath: path of maker file that informs if a given target can reuse remote artifacts.
     /// - Parameter dirScanner: scanner to access disk and read files and directories hierarchy
-    init(targetTempDir: URL, dirScanner: DirScanner) {
+    init(targetTempDir: URL, modeMarkerPath: String, dirScanner: DirScanner) {
         self.targetTempDir = targetTempDir
+        self.modeMarkerPath = modeMarkerPath
         self.dirScanner = dirScanner
     }
 
@@ -57,31 +60,19 @@ class ThinningCreatorPlugin: ArtifactCreatorPlugin {
             let fileKey: String
         }
         let uploadedTargetArtifacts = try allURLs.compactMap { tempDir -> TargetTuple? in
-            // All targets that uploaded their artifacts, have it placed in the
-            // `$(TARGET_TEMP_DIR)/xccache/produced/{{fileKey}}.zip` location. Find all targets that have such a file
-
-            let targetGeneratedArtifactRootDir = tempDir
-                .appendingPathComponent("xccache")
-                .appendingPathComponent("produced")
-            guard try dirScanner.itemType(atPath: targetGeneratedArtifactRootDir.path) == ItemType.dir else {
-                // given target didn't generate any artifacts (e.g. it is never cached with XCRemoteCache)
-                return nil
-            }
-
-            let allFilesProduced = try dirScanner.items(at: targetGeneratedArtifactRootDir)
-            let allArtifacts = allFilesProduced.filter { $0.pathExtension == "zip" }
-            guard !allArtifacts.isEmpty else {
+            let potentialArtifacts = try findTargetPackageZip(tempDir: tempDir)
+            guard !potentialArtifacts.isEmpty else {
                 // there is no generated *.zip file, so given target didn't create an artifact - it could be
                 // just a helper target (like the target we integrate this plugin with)
                 return nil
             }
             // Find {{fileKey}} based on the .zip file basename
-            guard allArtifacts.count == 1 else {
+            guard potentialArtifacts.count == 1 else {
                 throw ThinningCreatorPluginError.noSingleTargetArtifactsGenerated(
-                    rootDir: targetGeneratedArtifactRootDir
+                    rootDir: tempDir
                 )
             }
-            let fileKey = allArtifacts[0].deletingPathExtension().lastPathComponent
+            let fileKey = potentialArtifacts[0].deletingPathExtension().lastPathComponent
             // Taking target name from tempDir, which has a structures "*.build"
             let targetName = tempDir.deletingPathExtension().lastPathComponent
             return TargetTuple(targetName: targetName, fileKey: fileKey)
@@ -96,6 +87,41 @@ class ThinningCreatorPlugin: ArtifactCreatorPlugin {
         return Dictionary(uniqueKeysWithValues: extraKeysTuples)
     }
 
+    private func findTargetPackageZip(tempDir: URL) throws -> [URL] {
+        // Producer mode:
+        // All targets that uploaded their artifacts, have it placed in the
+        // `$(TARGET_TEMP_DIR)/xccache/produced/{{fileKey}}.zip` location. Find all targets that have such a file
+        // ProducerFast mode:
+        // If a target reused already existing artifact, it still has `$(TARGET_TEMP_DIR)/rc.enabled` marker file
+        // and the reused zip is placed in:
+        // `$(TARGET_TEMP_DIR)/xccache/{{fileKey}}.zip` location.
+
+        let targetEnabledMarker = tempDir.appendingPathComponent(modeMarkerPath)
+        let targetReusedArtifactRootDir = tempDir.appendingPathComponent("xccache")
+        let targetGeneratedArtifactRootDir = tempDir
+            .appendingPathComponent("xccache")
+            .appendingPathComponent("produced")
+
+        let pathToDirWithZipArtifacts: URL
+        // try the `.producerFast` scenario first (the artifact was not locally
+        // generated but just reused from the remote cache)
+        if try dirScanner.itemType(atPath: targetEnabledMarker.path) == ItemType.file {
+            pathToDirWithZipArtifacts = targetReusedArtifactRootDir
+        } else {
+            // cover a case when a target was build locally and an artifact
+            // has just been created (locally)
+            guard try dirScanner.itemType(atPath: targetGeneratedArtifactRootDir.path) == ItemType.dir else {
+                // given target didn't generate any artifacts (e.g. it is never cached with XCRemoteCache)
+                return []
+            }
+            pathToDirWithZipArtifacts = targetGeneratedArtifactRootDir
+        }
+
+        let allFilesProduced = try dirScanner.items(at: pathToDirWithZipArtifacts)
+        let allArtifacts = allFilesProduced.filter { $0.pathExtension == "zip" }
+        return allArtifacts
+    }
+
     func artifactToUpload(main: MainArtifactMeta) throws -> [Artifact] {
         return []
     }

Sources/XCRemoteCache/Commands/Plugins/Thinning/ThinningDiskSwiftcProductsGenerator.swift

@@ -40,7 +40,7 @@ class ThinningDiskSwiftcProductsGenerator: SwiftcProductsGenerator {
         destinationSwiftmodulePaths = Dictionary(
             uniqueKeysWithValues: SwiftmoduleFileExtension.SwiftmoduleExtensions
                 .map { ext, _ in
-                    switch (ext) {
+                    switch ext {
                     case .swiftsourceinfo:
                         let dest = modulePathDir.appendingPathComponent("Project")
                             .appendingPathComponent(moduleName)

Sources/XCRemoteCache/Commands/Postbuild/Postbuild.swift

@@ -41,6 +41,7 @@ class Postbuild {
     private let dSYMOrganizer: DSYMOrganizer
     private let modeController: CacheModeController
     private let metaReader: MetaReader
+    private let metaWriter: MetaWriter
     private let creatorPlugins: [ArtifactCreatorPlugin]
     private let consumerPlugins: [ArtifactConsumerPostbuildPlugin]
 
@@ -58,6 +59,7 @@ class Postbuild {
         dSYMOrganizer: DSYMOrganizer,
         modeController: CacheModeController,
         metaReader: MetaReader,
+        metaWriter: MetaWriter,
         creatorPlugins: [ArtifactCreatorPlugin],
         consumerPlugins: [ArtifactConsumerPostbuildPlugin]
     ) {
@@ -74,6 +76,7 @@ class Postbuild {
         self.dSYMOrganizer = dSYMOrganizer
         self.modeController = modeController
         self.metaReader = metaReader
+        self.metaWriter = metaWriter
         self.creatorPlugins = creatorPlugins
         self.consumerPlugins = consumerPlugins
     }
@@ -125,6 +128,22 @@ class Postbuild {
         try generateFingerprintOverrides(contextSpecificFingerprint: fingerprint.contextSpecific)
     }
 
+    /// Uploads only a meta to the remote server - useful when the file artifact (.zip) already exists on a remote
+    /// server and only a meta for a current commit sha has to be uploaded
+    public func performMetaUpload(meta: MainArtifactMeta, for commit: String) throws {
+        // Reset plugins keys as these are unique to each
+        var meta = meta
+        meta.pluginsKeys = [:]
+        meta = try creatorPlugins.reduce(meta) { prevMeta, plugin in
+            var meta = prevMeta
+            // add extra keys from the plugin. A plugin overrides previously defined keys in case of duplication
+            meta.pluginsKeys = try meta.pluginsKeys.merging(plugin.extraMetaKeys(prevMeta), uniquingKeysWith: { $1 })
+            return meta
+        }
+        let metaPath = try metaWriter.write(meta, locationDir: context.targetTempDir)
+        try networkClient.uploadSynchronously(metaPath, as: .meta(commit: commit))
+    }
+
     /// Builds an artifact package and uploads it to the remote server
     public func performBuildUpload(for commit: String) throws {
         let dependencies = try generateDependencies()
@@ -134,7 +153,7 @@ class Postbuild {
         // Replace all local paths to the generic ones (e.g. $SRCROOT)
         let remappers = [remapper] + creatorPlugins.compactMap(\.customPathsRemapper)
         let remapper = DependenciesRemapperComposite(remappers)
-        let abstractFingerprintFiles = remapper.replace(localPaths: dependencies.map(\.path))
+        let abstractFingerprintFiles = try remapper.replace(localPaths: dependencies.map(\.path))
         // TODO: use `inputs` read by dependenciesReader
         var meta = MainArtifactMeta(
             dependencies: abstractFingerprintFiles,

Sources/XCRemoteCache/Commands/Postbuild/PostbuildContext.swift

@@ -32,6 +32,8 @@ enum MachOType: String, Codable {
 enum PostbuildContextError: Error {
     /// URL address is not a valid URL
     case invalidAddress(String)
+    /// ARCHS env does not contain any architecture to build
+    case missingArchitecture
 }
 
 public struct PostbuildContext {
@@ -64,6 +66,9 @@ public struct PostbuildContext {
     var machOType: MachOType
     var wasDsymGenerated: Bool
     var dSYMPath: URL
+    // building architecture. Used to find all dependencies from *.d files
+    // Warning: if two architectures are built (e.g. for disabled "Build Archive
+    // Architecture Only"), a first architecture one is picked
     let arch: String
     let builtProductsDir: URL
     /// Location to the product bundle. Can be nil for libraries
@@ -73,6 +78,9 @@ public struct PostbuildContext {
     var thinnedTargets: [String]
     /// Action type: build, indexbuild etc.
     var action: BuildActionType
+    let modeMarkerPath: String
+    /// location of the json file that define virtual files system overlay (mappings of the virtual location file -> local file path)
+    let overlayHeadersPath: URL
 }
 
 extension PostbuildContext {
@@ -81,8 +89,13 @@ extension PostbuildContext {
         let targetNameValue: String = try env.readEnv(key: "TARGET_NAME")
         targetName = targetNameValue
         targetTempDir = try env.readEnv(key: "TARGET_TEMP_DIR")
-        arch = try env.readEnv(key: "PLATFORM_PREFERRED_ARCH")
-        compilationTempDir = try env.readEnv(key: "OBJECT_FILE_DIR_normal").appendingPathComponent(arch)
+        let archs: [String] = try env.readEnv(key: "ARCHS").split(separator: " ").map(String.init)
+        guard let firstArch = archs.first, !firstArch.isEmpty else {
+            throw PostbuildContextError.missingArchitecture
+        }
+        arch = firstArch
+        let variant: String = try env.readEnv(key: "CURRENT_VARIANT")
+        compilationTempDir = try env.readEnv(key: "OBJECT_FILE_DIR_\(variant)").appendingPathComponent(arch)
         configuration = try env.readEnv(key: "CONFIGURATION")
         platform = try env.readEnv(key: "PLATFORM_NAME")
         xcodeBuildNumber = try env.readEnv(key: "XCODE_PRODUCT_BUILD_VERSION")
@@ -115,5 +128,8 @@ extension PostbuildContext {
         let thinFocusedTargetsString: String = env.readEnv(key: "SPT_XCREMOTE_CACHE_THINNED_TARGETS") ?? ""
         thinnedTargets = thinFocusedTargetsString.split(separator: ",").map(String.init)
         action = (try? BuildActionType(rawValue: env.readEnv(key: "ACTION"))) ?? .unknown
+        modeMarkerPath = config.modeMarkerPath
+        /// Note: The file has yaml extension, even it is in the json format
+        overlayHeadersPath = targetTempDir.appendingPathComponent("all-product-headers.yaml")
     }
 }

Sources/XCRemoteCache/Commands/Postbuild/XCPostbuild.swift

@@ -66,9 +66,10 @@ public class XCPostbuild {
             // Initialize dependencies
             let primaryGitBranch = GitBranch(repoLocation: config.primaryRepo, branch: config.primaryBranch)
             let gitClient = GitClientImpl(repoRoot: config.repoRoot, primary: primaryGitBranch, shell: shellGetStdout)
-            let pathRemapper = try StringDependenciesRemapper.buildFromEnvs(
-                keys: DependenciesMapping.rewrittenEnvs,
-                envs: env
+            let envsRemapper = try StringDependenciesRemapperFactory().build(
+                orderKeys: DependenciesMapping.rewrittenEnvs,
+                envs: env,
+                customMappings: config.outOfBandMappings
             )
             let envFingerprint = try EnvironmentFingerprintGenerator(
                 configuration: config,
@@ -85,6 +86,7 @@ public class XCPostbuild {
                 algorithm: MD5Algorithm()
             )
             let organizer = ZipArtifactOrganizer(targetTempDir: context.targetTempDir, fileManager: fileManager)
+            let metaWriter = JsonMetaWriter(fileWriter: fileManager, pretty: config.prettifyMetaFiles)
             let artifactCreator = BuildArtifactCreator(
                 buildDir: context.productsDir,
                 tempDir: context.targetTempDir,
@@ -92,6 +94,7 @@ public class XCPostbuild {
                 moduleName: context.moduleName,
                 modulesFolderPath: context.modulesFolderPath,
                 dSYMPath: context.dSYMPath,
+                metaWriter: metaWriter,
                 fileManager: fileManager
             )
             let dirAccessor = DirAccessorComposer(
@@ -142,6 +145,23 @@ public class XCPostbuild {
                 fileDependeciesReaderFactory: fileReaderFactory,
                 dirScanner: fileManager
             )
+            var remappers: [DependenciesRemapper] = []
+            if !config.disableVFSOverlay {
+                // As the PostbuildContext assumes file location and filename (`all-product-headers.yaml`)
+                // do not fail in case of a missing headers overlay file. In the future, all overlay files could be
+                // captured from the swiftc invocation similarly is stored in the `history.compile` for the consumer mode.
+                let overlayReader = JsonOverlayReader(
+                    context.overlayHeadersPath,
+                    mode: .bestEffort,
+                    fileReader: fileManager
+                )
+                let overlayRemapper = OverlayDependenciesRemapper(
+                    overlayReader: overlayReader
+                )
+                remappers.append(overlayRemapper)
+            }
+            remappers.append(envsRemapper)
+            let pathRemapper = DependenciesRemapperComposite(remappers)
             let dependencyProcessor = DependencyProcessorImpl(
                 xcode: context.xcodeDir,
                 product: context.productsDir,
@@ -205,9 +225,10 @@ public class XCPostbuild {
                             worker: DispatchGroupParallelizationWorker(qos: .userInitiated)
                         )
                         consumerPlugins.append(thinningPlugin)
-                    case .producer:
+                    case .producer, .producerFast:
                         let thinningPlugin = ThinningCreatorPlugin(
                             targetTempDir: context.targetTempDir,
+                            modeMarkerPath: context.modeMarkerPath,
                             dirScanner: fileManager
                         )
                         creatorPlugins.append(thinningPlugin)
@@ -229,6 +250,7 @@ public class XCPostbuild {
                 dSYMOrganizer: dSYMOrganizer,
                 modeController: modeController,
                 metaReader: metaReader,
+                metaWriter: metaWriter,
                 creatorPlugins: creatorPlugins,
                 consumerPlugins: consumerPlugins
             )
@@ -243,11 +265,22 @@ public class XCPostbuild {
                 try postbuildAction.deleteFingerprintOverrides()
             }
 
+
             // Trigger uploading the artifact
-            if context.mode == .producer {
+            switch (context.mode, try modeController.isEnabled(), context.remoteCommit) {
+            case (.producerFast, true, .available(commit: let commitToReuse)):
+                // Upload only updated meta. Artifact zip is already on a remote server
+                let referenceCommit = try config.publishingSha ?? gitClient.getCurrentSha()
+                let metaData = try remoteNetworkClient.fetch(.meta(commit: commitToReuse))
+                let meta = try metaReader.read(data: metaData)
+                try postbuildAction.performMetaUpload(meta: meta, for: referenceCommit)
+            case (.producer, _, _), (.producerFast, _, _):
                 // Generate artifacts and upload to the remote server for a reference sha
                 let referenceCommit = try config.publishingSha ?? gitClient.getCurrentSha()
                 try postbuildAction.performBuildUpload(for: referenceCommit)
+            default:
+                // Consumer does not upload anything
+                break
             }
 
             let executableURL = context.productsDir.appendingPathComponent(context.executablePath)
@@ -261,9 +294,8 @@ public class XCPostbuild {
             } else {
                 try postbuildAction.performBuildCleanup()
                 try cacheHitLogger.logMiss()
-                // Producer mode doesn't use cached artifacts so modeController is not enabled. If producer
-                // reaches this point, there were no issues with publishing
-                let actionName = context.mode == .producer ? "Published" : "Disabled"
+                // If producers reach this point, there were no issues with publishing
+                let actionName = context.mode == .consumer ? "Disabled" : "Published"
                 printToUser("\(actionName) remote cache for \(context.targetName)")
             }
         } catch PluginError.unrecoverableError(let error) {

Sources/XCRemoteCache/Commands/Prebuild/Prebuild.swift

@@ -55,6 +55,7 @@ class Prebuild {
         self.artifactConsumerPrebuildPlugins = artifactConsumerPrebuildPlugins
     }
 
+    // swiftlint:disable:next function_body_length
     public func perform() throws -> PrebuildResult {
         guard case .available(let commit) = context.remoteCommit else {
             return .incompatible
@@ -62,7 +63,7 @@ class Prebuild {
         do {
             let metaData = try networkClient.fetch(.meta(commit: commit))
             let meta = try metaReader.read(data: metaData)
-            let localDependencies = remapper.replace(genericPaths: meta.dependencies).map(URL.init(fileURLWithPath:))
+            let localDependencies = try remapper.replace(genericPaths: meta.dependencies).map(URL.init(fileURLWithPath:))
             let localFingerprint = try generateFingerprint(for: localDependencies)
             if localFingerprint.raw != meta.rawFingerprint {
                 if context.forceCached {

Sources/XCRemoteCache/Commands/Prebuild/PrebuildContext.swift

@@ -43,6 +43,8 @@ public struct PrebuildContext {
     let targetName: String
     /// List of all targets to downloaded from the thinning aggregation target
     var thinnedTargets: [String]?
+    /// location of the json file that define virtual files system overlay (mappings of the virtual location file -> local file path)
+    let overlayHeadersPath: URL
 }
 
 extension PrebuildContext {
@@ -64,5 +66,7 @@ extension PrebuildContext {
         self.targetName = targetName
         let thinFocusedTargetsString: String? = env.readEnv(key: "SPT_XCREMOTE_CACHE_THINNED_TARGETS")
         thinnedTargets = thinFocusedTargetsString?.split(separator: ",").map(String.init)
+        /// Note: The file has yaml extension, even it is in the json format
+        overlayHeadersPath = targetTempDir.appendingPathComponent("all-product-headers.yaml")
     }
 }

Sources/XCRemoteCache/Commands/Prebuild/XCPrebuild.swift

@@ -115,10 +115,27 @@ public class XCPrebuild {
             )
             let client: NetworkClient = config.disableHttpCache ? networkClient : cacheNetworkClient
             let remoteNetworkClient = RemoteNetworkClientImpl(client, urlBuilder)
-            let pathRemapper = try StringDependenciesRemapper.buildFromEnvs(
-                keys: DependenciesMapping.rewrittenEnvs,
-                envs: env
+            let envsRemapper = try StringDependenciesRemapperFactory().build(
+                orderKeys: DependenciesMapping.rewrittenEnvs,
+                envs: env,
+                customMappings: config.outOfBandMappings
             )
+            var remappers: [DependenciesRemapper] = []
+            if !config.disableVFSOverlay {
+                // As PrebuildContext assumes file location and its filename (`all-product-headers.yaml`)
+                // do not fail in case of a missing headers overlay file.
+                let overlayReader = JsonOverlayReader(
+                    context.overlayHeadersPath,
+                    mode: .bestEffort,
+                    fileReader: fileManager
+                )
+                let overlayRemapper = OverlayDependenciesRemapper(
+                    overlayReader: overlayReader
+                )
+                remappers.append(overlayRemapper)
+            }
+            remappers.append(envsRemapper)
+            let pathRemapper = DependenciesRemapperComposite(remappers)
             let filesFingerprintGenerator = FingerprintAccumulatorImpl(
                 algorithm: MD5Algorithm(),
                 fileManager: fileManager
@@ -129,7 +146,10 @@ public class XCPrebuild {
                 algorithm: MD5Algorithm()
             )
             let organizer = ZipArtifactOrganizer(targetTempDir: context.targetTempDir, fileManager: fileManager)
-            let compilationHistoryOrganizer = CompilationHistoryFileOrganizer(context.compilationHistoryFile, fileManager: fileManager)
+            let compilationHistoryOrganizer = CompilationHistoryFileOrganizer(
+                context.compilationHistoryFile,
+                fileManager: fileManager
+            )
             let metaReader = JsonMetaReader(fileAccessor: fileManager)
             var consumerPlugins: [ArtifactConsumerPrebuildPlugin] = []
 

Sources/XCRemoteCache/Commands/Prepare/CCWrapperBuilder.swift

@@ -72,13 +72,14 @@ class TemplateBasedCCWrapperBuilder: CCWrapperBuilder {
         )
         infoLog("ClangWrapperBuilder compiles file at \(compilationFile).")
         // -O3: optimize for faster execution
-        let args = [clangCommand, "-O3", compilationFile.path, "-o", destination.path]
+        let args = [clangCommand, "-arch", "arm64", "-arch", "x86_64", "-O3", compilationFile.path, "-o", destination.path]
         let compilationOutput = try shell("xcrun", args, URL(fileURLWithPath: "").path, nil)
         infoLog("Clang compilation output: \(compilationOutput)")
     }
 
 
     /// Generates source of the cc wrapper
+    // swiftlint:disable line_length
     // swiftlint:disable:next function_body_length
     private func buildWrapperSource(clangCommand: String, markerFilename: String, commitSha: String) -> String {
         return """
@@ -516,5 +517,5 @@ class TemplateBasedCCWrapperBuilder: CCWrapperBuilder {
            #pragma GCC diagnostic pop
         }
         """
-    } // swiftlint:disable:next file_length
-}
+    } // swiftlint:disable:next file_length line_length
+} // swiftlint:enable line_length

Sources/XCRemoteCache/Commands/Prepare/Integrate/BuildSettingsIntegrateAppender.swift

@@ -62,6 +62,7 @@ class XcodeProjBuildSettingsIntegrateAppender: BuildSettingsIntegrateAppender {
         result["OTHER_CFLAGS"] = clangFlags.settingValue
 
         result["XCRC_FAKE_SRCROOT"] = "/\(String(repeating: "x", count: 10))"
+        result["XCRC_PLATFORM_PREFERRED_ARCH"] = "$(LINK_FILE_LIST_$(CURRENT_VARIANT)_$(PLATFORM_PREFERRED_ARCH):dir:standardizepath:file:default=arm64)"
         return result
     }
 }

Sources/XCRemoteCache/Commands/Prepare/Integrate/LLDBInitPatcher.swift

@@ -56,7 +56,7 @@ class FileLLDBInitPatcher: LLDBInitPatcher {
     }
 
     private func findIndices(in collection: [String], value: String) -> [Int] {
-        collection.enumerated().reduce([]) { (result, line) -> [Int] in
+        collection.enumerated().reduce([]) { result, line -> [Int] in
             if line.element == Self.preambleString {
                 return result + [line.offset]
             }
@@ -75,7 +75,7 @@ class FileLLDBInitPatcher: LLDBInitPatcher {
             var contentLines = originalContentLines
             let preambleIndices = findIndices(in: contentLines, value: Self.preambleString)
 
-            if preambleIndices.count > 0 {
+            if !preambleIndices.isEmpty {
                 let firstLLDBCommandIndex = preambleIndices[0] + 1
                 if firstLLDBCommandIndex >= contentLines.count {
                     // corrupted file, append the script line at the bottom

Sources/XCRemoteCache/Commands/Prepare/Integrate/XCIntegrate.swift

@@ -64,6 +64,7 @@ public class XCIntegrate {
         self.output = output
     }
 
+    // swiftlint:disable:next function_body_length
     public func main() {
         do {
             let env = ProcessInfo.processInfo.environment
@@ -114,7 +115,7 @@ public class XCIntegrate {
 
             let integrator = XcodeProjIntegrate(
                 project: context.projectPath,
-                mode:context.mode,
+                mode: context.mode,
                 binaries: context.binaries,
                 configurationIncludeOracle: configurationOracle,
                 targetIncludeOracle: targetOracle,

Sources/XCRemoteCache/Commands/Prepare/Integrate/XcodeProjIntegrate.swift

@@ -100,11 +100,11 @@ struct XcodeProjIntegrate: Integrate {
             outputPaths: [
                 """
                 $(TARGET_BUILD_DIR)/$(MODULES_FOLDER_PATH)/$(PRODUCT_MODULE_NAME).swiftmodule/\
-                $(PLATFORM_PREFERRED_ARCH).swiftmodule.md5
+                $(XCRC_PLATFORM_PREFERRED_ARCH).swiftmodule.md5
                 """,
                 """
                 $(TARGET_BUILD_DIR)/$(MODULES_FOLDER_PATH)/$(PRODUCT_MODULE_NAME).swiftmodule/\
-                $(PLATFORM_PREFERRED_ARCH)-$(LLVM_TARGET_TRIPLE_VENDOR)-$(SWIFT_PLATFORM_TARGET_PREFIX)\
+                $(XCRC_PLATFORM_PREFERRED_ARCH)-$(LLVM_TARGET_TRIPLE_VENDOR)-$(SWIFT_PLATFORM_TARGET_PREFIX)\
                 $(LLVM_TARGET_TRIPLE_SUFFIX).swiftmodule.md5
                 """,
             ],
@@ -114,7 +114,9 @@ struct XcodeProjIntegrate: Integrate {
         markPhase = PBXShellScriptBuildPhase(
             name: "\(Self.BuildStepPrefix)RemoteCache_mark",
             inputPaths: [binaries.prepare.path],
-            shellScript: "\"$SCRIPT_INPUT_FILE_0\" mark --configuration \"$CONFIGURATION\" --platform \"$PLATFORM_NAME\""
+            shellScript:
+                "\"$SCRIPT_INPUT_FILE_0\" mark " +
+                "--configuration \"$CONFIGURATION\" --platform \"$PLATFORM_NAME\""
         )
     }
 
@@ -129,6 +131,7 @@ struct XcodeProjIntegrate: Integrate {
         try encodedYAML.write(to: configOverrideLocation, atomically: false, encoding: .utf8)
     }
 
+    // swiftlint:disable:next function_body_length
     func run() throws {
         let outputFile = output ?? projectURL
         let projectRoot = projectURL.deletingLastPathComponent()
@@ -224,11 +227,11 @@ struct XcodeProjIntegrate: Integrate {
         let previousRCPhases = target.buildPhases.filter(isRCPhase)
         target.buildPhases.removeAll(where: previousRCPhases.contains)
 
-        if target.buildPhases.map(\.buildPhase).contains(.sources) {
+        if let sourceIndex = target.buildPhases.map(\.buildPhase).firstIndex(of: .sources) {
             // add (pre|post)build phases only when a target has some compilation steps
             // otherwise they make no sense (nothing to store in an artifact)
             pbxproj.add(object: prebuildPhase)
-            target.buildPhases.insert(prebuildPhase, at: 0)
+            target.buildPhases.insert(prebuildPhase, at: sourceIndex)
             pbxproj.add(object: postbuildPhase)
             target.buildPhases.append(postbuildPhase)
         }

Sources/XCRemoteCache/Commands/Prepare/Integrate/XcodeSettingsFlags.swift

@@ -100,7 +100,7 @@ struct XcodeSettingsCFlags: XcodeSettingsFlags {
         case (.some(let existing), _):
             var flagsComponents: [String] = existing.split(separator: " ").map(String.init)
             // remove (if exists)
-            let existingFlagIndex = flagsComponents.firstIndex { (component) -> Bool in
+            let existingFlagIndex = flagsComponents.firstIndex { component -> Bool in
                 component.hasPrefix("\(Self.prefix)\(key)=")
             }
             if let index = existingFlagIndex {

Sources/XCRemoteCache/Commands/Prepare/XCConfig.swift

@@ -18,7 +18,6 @@
 // under the License.
 
 import Foundation
-import Yams
 
 /// Print current configuration to the console
 public class XCConfig {

Sources/XCRemoteCache/Commands/Prepare/XCPrepare.swift

@@ -18,7 +18,6 @@
 // under the License.
 
 import Foundation
-import Yams
 
 /// Switch between Online/Offline modes
 public enum XCPrepareMode {

Sources/XCRemoteCache/Commands/Prepare/XCPrepareMark.swift

@@ -38,6 +38,7 @@ public class XCPrepareMark {
         self.commit = commit
     }
 
+    // swiftlint:disable:next function_body_length
     public func main() {
         let env = ProcessInfo.processInfo.environment
         let fileManager = FileManager.default

Sources/XCRemoteCache/Commands/Prepare/XCStats.swift

@@ -18,7 +18,6 @@
 // under the License.
 
 import Foundation
-import Yams
 
 /// Manages XCRemoteCache statistics: rests, print to the standard output etc
 public class XCStats {

Sources/XCRemoteCache/Commands/Swiftc/Swiftc.swift

@@ -120,9 +120,9 @@ class Swiftc: SwiftcProtocol {
                 let prebuildDiscoveryURL = context.tempDir.appendingPathComponent(context.prebuildDependenciesPath)
                 let prebuildDiscoverWriter = dependenciesWriterFactory(prebuildDiscoveryURL, fileManager)
                 try prebuildDiscoverWriter.write(skipForSha: remoteCommit)
-            case .consumer, .producer:
-                // Never skips prebuild phase and fallbacks to the swiftc compilation for:
-                // 1) Not enabled remote cache or 2) producer
+            case .consumer, .producer, .producerFast:
+                // Never skip prebuild phase and fallback to the swiftc compilation for:
+                // 1) Not enabled remote cache, 2) producer(s)
                 break
             }
             return .forceFallback

Sources/XCRemoteCache/Commands/Swiftc/SwiftcContext.swift

@@ -24,6 +24,8 @@ public struct SwiftcContext {
         case producer
         /// Commit sha of the commit to use during remote cache
         case consumer(commit: RemoteCommitInfo)
+        /// Remote artifact exists and can be optimistically used in place of a local compilation
+        case producerFast
     }
 
     let objcHeaderOutput: URL
@@ -74,6 +76,14 @@ public struct SwiftcContext {
             mode = .consumer(commit: remoteCommit)
         case .producer:
             mode = .producer
+        case .producerFast:
+            let remoteCommit = RemoteCommitInfo(try? String(contentsOf: remoteCommitLocation).trim())
+            switch remoteCommit {
+            case .unavailable:
+                mode = .producer
+            case .available:
+                mode = .producerFast
+            }
         }
         invocationHistoryFile = URL(fileURLWithPath: config.compilationHistoryFile, relativeTo: tempDir)
     }

Sources/XCRemoteCache/Commands/Swiftc/SwiftcOrchestrator.swift

@@ -116,6 +116,12 @@ class SwiftcOrchestrator {
             }
         case .consumer:
             fallbackToDefault(command: swiftcCommand)
+        case .producerFast:
+            let compileStepResult = try swiftc.mockCompilation()
+            if case .forceFallback = compileStepResult {
+                // cannot reuse cached artifact. Build it locally and upload to the server just as for the producer
+                fallthrough
+            }
         case .producer:
             var swiftcArgs = ProcessInfo().arguments
             swiftcArgs = try producerFallbackCommandProcessors.reduce(swiftcArgs) { args, processor in

Sources/XCRemoteCache/Commands/Swiftc/XCSwiftc.swift

@@ -101,7 +101,10 @@ public class XCSwiftc {
             objcHeaderOutput: context.objcHeaderOutput,
             diskCopier: HardLinkDiskCopier(fileManager: fileManager)
         )
-        let allInvocationsStorage = ExistingFileStorage(storageFile: context.invocationHistoryFile, command: swiftcCommand)
+        let allInvocationsStorage = ExistingFileStorage(
+            storageFile: context.invocationHistoryFile,
+            command: swiftcCommand
+        )
         // When fallbacking to local compilation do not call historical `swiftc` invocations
         // The current fallback invocation already compiles all files in a target
         let invocationStorage = FilteredInvocationStorage(

Sources/XCRemoteCache/Config/Mode.swift

@@ -20,4 +20,5 @@
 public enum Mode: String, Codable, CaseIterable {
     case consumer
     case producer
+    case producerFast = "producer-fast"
 }

Sources/XCRemoteCache/Config/XCRemoteCacheConfig.swift

@@ -94,10 +94,11 @@ public struct XCRemoteCacheConfig: Encodable {
     var focusedTargets: [String] = []
     ///  Disable cache for http requests to fecth metadata and download artifacts
     var disableHttpCache: Bool = false
-    /// Path, relative to $TARGET_TEMP_DIR which gathers all compilation commands that should be executed if a target
-    /// switches to local compilation. Example: A new `.swift` file invalidates remote artifact and triggers local compilation
-    /// When that happens, all previously skipped clang build steps need to be eventually called locally - this file lists
-    /// all these commands.
+    /// Path, relative to $TARGET_TEMP_DIR which gathers all compilation commands that should be e
+    /// xecuted if a target switches to local compilation.
+    /// Example: A new `.swift` file invalidates remote arXcodeProjIntegrate.swifttifact and triggers local compilation
+    /// When that happens, all previously skipped clang build steps
+    /// need to be eventually called locally - this file lists all these commands.
     var compilationHistoryFile: String = "history.compile"
     /// Timeout for remote response data interval (in seconds). If an interval between data chunks is
     /// longer than a timeout, a request fails
@@ -111,6 +112,8 @@ public struct XCRemoteCacheConfig: Encodable {
     var thinningEnabled: Bool = false
     /// Module name of a target that works as a helper for thinned targets
     var thinningTargetModuleName: String = "ThinningRemoteCacheModule"
+    /// Opt-in pretty json formatting for meta files
+    var prettifyMetaFiles: Bool = false
     /// Secret key for AWS V4 Signature, if this is set the Authentication Header will be added
     var AWSSecretKey: String = ""
     /// Access key for AWS V4 Signature
@@ -119,11 +122,23 @@ public struct XCRemoteCacheConfig: Encodable {
     var AWSRegion: String = ""
     /// Service for AWS V4 Signature (e.g. `storage`)
     var AWSService: String = ""
+    /// A dictionary of files path remapping that should be applied to make it absolute path agnostic on a list of dependencies.
+    /// Useful if a project refers files out of repo root, either compilation files or precompiled dependencies.
+    /// Keys represent generic replacement and values are substrings that should be replaced.
+    /// Example: for mapping `["COOL_LIBRARY": "/CoolLibrary"]`
+    /// `/CoolLibrary/main.swift`will be represented as `$(COOL_LIBRARY)/main.swift`).
+    /// Warning: remapping order is not-deterministic so avoid remappings with multiple matchings.
+    var outOfBandMappings: [String: String] = [:]
+    /// If true, SSL certificate validation is disabled
+    var disableCertificateVerification: Bool = false
+    /// A feature flag to disable virtual file system overlay support (temporary)
+    var disableVFSOverlay: Bool = false
 }
 
 extension XCRemoteCacheConfig {
     /// Merges existing config with the other config and returns a final result
     /// `other` scheme overrides existing configuration
+    // swiftlint:disable:next function_body_length
     func merged(with scheme: ConfigFileScheme) -> XCRemoteCacheConfig {
         var merge = self
         merge.mode = scheme.mode ?? mode
@@ -163,10 +178,14 @@ extension XCRemoteCacheConfig {
             scheme.productFilesExtensionsWithContentOverride ?? productFilesExtensionsWithContentOverride
         merge.thinningEnabled = scheme.thinningEnabled ?? thinningEnabled
         merge.thinningTargetModuleName = scheme.thinningTargetModuleName ?? thinningTargetModuleName
+        merge.prettifyMetaFiles = scheme.prettifyMetaFiles ?? prettifyMetaFiles
         merge.AWSAccessKey = scheme.AWSAccessKey ?? AWSAccessKey
         merge.AWSSecretKey = scheme.AWSSecretKey ?? AWSSecretKey
         merge.AWSRegion = scheme.AWSRegion ?? AWSRegion
         merge.AWSService = scheme.AWSService ?? AWSService
+        merge.outOfBandMappings = scheme.outOfBandMappings ?? outOfBandMappings
+        merge.disableCertificateVerification = scheme.disableCertificateVerification ?? disableCertificateVerification
+        merge.disableVFSOverlay = scheme.disableVFSOverlay ?? disableVFSOverlay
         return merge
     }
 
@@ -221,10 +240,14 @@ struct ConfigFileScheme: Decodable {
     let productFilesExtensionsWithContentOverride: [String]?
     let thinningEnabled: Bool?
     let thinningTargetModuleName: String?
+    let prettifyMetaFiles: Bool?
     let AWSSecretKey: String?
     let AWSAccessKey: String?
     let AWSRegion: String?
     let AWSService: String?
+    let outOfBandMappings: [String: String]?
+    let disableCertificateVerification: Bool?
+    let disableVFSOverlay: Bool?
 
     // Yams library doesn't support encoding strategy, see https://github.com/jpsim/Yams/issues/84
     enum CodingKeys: String, CodingKey {
@@ -262,10 +285,14 @@ struct ConfigFileScheme: Decodable {
         case productFilesExtensionsWithContentOverride = "product_files_extensions_with_content_override"
         case thinningEnabled = "thinning_enabled"
         case thinningTargetModuleName = "thinning_target_module_name"
+        case prettifyMetaFiles = "prettify_meta_files"
         case AWSSecretKey = "aws_secret_key"
         case AWSAccessKey = "aws_access_key"
         case AWSRegion = "aws_region"
         case AWSService = "aws_service"
+        case outOfBandMappings = "out_of_band_mappings"
+        case disableCertificateVerification = "disable_certificate_verification"
+        case disableVFSOverlay = "disable_vfs_overlay"
     }
 }
 

Sources/XCRemoteCache/Dependencies/DependenciesRemapper.swift

@@ -22,9 +22,9 @@ import Foundation
 /// Replaces paths formats between generic (placeholders-based) and local
 protocol DependenciesRemapper {
     /// Replaces all generic paths (with placeholders) to a local paths
-    func replace(genericPaths: [String]) -> [String]
+    func replace(genericPaths: [String]) throws -> [String]
     /// Replaces all local paths to the generic dependencies paths
-    func replace(localPaths: [String]) -> [String]
+    func replace(localPaths: [String]) throws -> [String]
 }
 
 class DependenciesRemapperComposite: DependenciesRemapper {
@@ -34,15 +34,15 @@ class DependenciesRemapperComposite: DependenciesRemapper {
         self.remappers = remappers
     }
 
-    func replace(genericPaths: [String]) -> [String] {
-        remappers.reduce(genericPaths) { prev, mapper in
-            mapper.replace(genericPaths: prev)
+    func replace(genericPaths: [String]) throws -> [String] {
+        try remappers.reversed().reduce(genericPaths) { prev, mapper in
+            try mapper.replace(genericPaths: prev)
         }
     }
 
-    func replace(localPaths: [String]) -> [String] {
-        remappers.reduce(localPaths) { prev, mapper in
-            mapper.replace(localPaths: prev)
+    func replace(localPaths: [String]) throws -> [String] {
+        try remappers.reduce(localPaths) { prev, mapper in
+            try mapper.replace(localPaths: prev)
         }
     }
 }
@@ -59,16 +59,16 @@ final class StringDependenciesRemapper: DependenciesRemapper {
         self.mappings = mappings
     }
 
-    func replace(genericPaths: [String]) -> [String] {
+    func replace(genericPaths: [String]) throws -> [String] {
         return genericPaths.map { path in
-            let localPath = mappings.reduce(path) { prevPath, mapping in
+            let localPath = mappings.reversed().reduce(path) { prevPath, mapping in
                 prevPath.replacingOccurrences(of: mapping.generic, with: mapping.local)
             }
             return localPath
         }
     }
 
-    func replace(localPaths: [String]) -> [String] {
+    func replace(localPaths: [String]) throws -> [String] {
         return localPaths.map { path in
             let result = mappings.reduce(path) { prevPath, mapping in
                 prevPath.replacingOccurrences(of: mapping.local, with: mapping.generic)
@@ -77,14 +77,3 @@ final class StringDependenciesRemapper: DependenciesRemapper {
         }
     }
 }
-
-
-extension StringDependenciesRemapper {
-    static func buildFromEnvs(keys: [String], envs: [String: String]) throws -> Self {
-        let mappings: [Mapping] = try keys.map { key in
-            let localValue: String = try envs.readEnv(key: key)
-            return Mapping(generic: "$(\(key))", local: localValue)
-        }
-        return Self(mappings: mappings)
-    }
-}

Sources/XCRemoteCache/Dependencies/DependencyProcessor.swift

@@ -58,11 +58,11 @@ class DependencyProcessorImpl: DependencyProcessor {
     private let bundlePath: String?
 
     init(xcode: URL, product: URL, source: URL, intermediate: URL, bundle: URL?) {
-        xcodePath = xcode.path
-        productPath = product.path
-        sourcePath = source.path
-        intermediatePath = intermediate.path
-        bundlePath = bundle?.path
+        xcodePath = xcode.path.dirPath()
+        productPath = product.path.dirPath()
+        sourcePath = source.path.dirPath()
+        intermediatePath = intermediate.path.dirPath()
+        bundlePath = bundle?.path.dirPath()
     }
 
     func process(_ files: [URL]) -> [Dependency] {
@@ -72,7 +72,7 @@ class DependencyProcessorImpl: DependencyProcessor {
 
     private func classify(_ files: [URL]) -> [Dependency] {
         return files.map { file -> Dependency in
-            let filePath = file.path
+            let filePath = file.resolvingSymlinksInPath().path
             if filePath.hasPrefix(xcodePath) {
                 return Dependency(url: file, type: .xcode)
             } else if filePath.hasPrefix(intermediatePath) {
@@ -111,3 +111,9 @@ class DependencyProcessorImpl: DependencyProcessor {
         return !irrelevantDependenciesType.contains(dependency.type)
     }
 }
+
+fileprivate extension String {
+    func dirPath() -> String {
+        hasSuffix("/") ? self : appending("/")
+    }
+}

Sources/XCRemoteCache/Dependencies/OverlayDependenciesRemapper.swift

@@ -0,0 +1,67 @@
+// Copyright (c) 2021 Spotify AB.
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import Foundation
+
+/// File paths remapper according the virtual file system mappings
+/// - Warning: this class is not thread safe
+class OverlayDependenciesRemapper: DependenciesRemapper {
+    private let overlayReader: OverlayReader
+    private var mappings: [OverlayMapping]?
+
+    init(overlayReader: OverlayReader) {
+        self.overlayReader = overlayReader
+    }
+
+    /// Lazily Reads mappings from a file
+    /// - Warning: this function is not thread safe
+    private func getMappings() throws -> [OverlayMapping] {
+        guard let mappings = mappings else {
+            let mappings = try overlayReader.provideMappings()
+            self.mappings = mappings
+            return mappings
+        }
+        return mappings
+    }
+
+    private func mapPath(
+        _ path: String,
+        source: KeyPath<OverlayMapping,URL>,
+        destination: KeyPath<OverlayMapping,URL>
+    ) throws -> String {
+        guard let mapping = try getMappings().first(where: { $0[keyPath: source].path == path }) else {
+            // TODO: support partial mappings, where a directory path can be replaced with some other directory
+            // no direct mapping found
+            return path
+        }
+        return mapping[keyPath: destination].path
+    }
+
+    func replace(genericPaths: [String]) throws -> [String] {
+        try genericPaths.map {
+            try mapPath($0, source: \.virtual, destination: \.local)
+        }
+    }
+
+    func replace(localPaths: [String]) throws -> [String] {
+        try localPaths.map {
+            try mapPath($0, source: \.local, destination: \.virtual)
+        }
+    }
+}

Sources/XCRemoteCache/Dependencies/OverlayReader.swift

@@ -0,0 +1,133 @@
+// Copyright (c) 2021 Spotify AB.
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import Foundation
+
+/// Maps overlay's virtual URL with an actual (local) location
+struct OverlayMapping: Hashable {
+    let virtual: URL
+    let local: URL
+}
+
+enum JsonOverlayReaderError: Error {
+    /// The source file is missing
+    case missingSourceFile(URL)
+    /// The file exists but its content is invalid
+    case invalidSourceContent(URL)
+    /// the overlay format is not supported - either contains a nested directory or a single file 
+    case unsupportedFormat
+}
+/// Provides virtual file system overlay mappings
+protocol OverlayReader {
+    func provideMappings() throws -> [OverlayMapping]
+}
+
+class JsonOverlayReader: OverlayReader {
+
+    enum Mode {
+        /// Interrupts the operation if the representation file is missing
+        case strict
+        /// Assume empty overlay mapping if the file doesn't exist
+        case bestEffort
+    }
+
+    private struct Overlay: Decodable {
+        enum OverlayType: String, Decodable {
+            case file
+            case directory
+        }
+
+        struct Content: Decodable {
+            let externalContents: String
+            let name: String
+            let type: OverlayType
+
+            enum CodingKeys: String, CodingKey {
+                case externalContents = "external-contents"
+                case name
+                case type
+            }
+        }
+
+        struct RootContent: Decodable {
+            let contents: [Content]
+            let name: String
+            let type: OverlayType
+        }
+        let roots: [RootContent]
+    }
+
+    private lazy var jsonDecoder = JSONDecoder()
+    private let json: URL
+    private let mode: Mode
+    private let fileReader: FileReader
+
+
+    init(_ json: URL, mode: Mode, fileReader: FileReader) {
+        self.json = json
+        self.mode = mode
+        self.fileReader = fileReader
+    }
+
+    func provideMappings() throws -> [OverlayMapping] {
+        guard let jsonContent = try fileReader.contents(atPath: json.path) else {
+            switch mode {
+            case .strict:
+                throw JsonOverlayReaderError.missingSourceFile(json)
+            case .bestEffort:
+                printWarning("overlay mapping file \(json) doesn't exist. Skipping overlay for the best-effort mode.")
+                return []
+            }
+        }
+        do {
+        let overlay: Overlay = try jsonDecoder.decode(Overlay.self, from: jsonContent)
+            let mappings: [OverlayMapping] = try overlay.roots.reduce([]) { prev, root in
+                switch root.type {
+                case .directory:
+                    //iterate all contents
+                    let dir = URL(fileURLWithPath: root.name)
+                    let mappings: [OverlayMapping] = try root.contents.map { content in
+                        switch content.type {
+                        case .file:
+                            let virtual = dir.appendingPathComponent(content.name)
+                            let local = URL(fileURLWithPath: content.externalContents)
+                            return .init(virtual: virtual, local: local)
+                        case .directory:
+                            throw JsonOverlayReaderError.unsupportedFormat
+                        }
+
+                    }
+                    return prev + mappings
+                case .file:
+                    throw JsonOverlayReaderError.unsupportedFormat
+                }
+            }
+            return mappings
+        } catch {
+            switch mode {
+            case .strict:
+                throw error
+            case .bestEffort:
+                printWarning("Overlay reader has failed with an error \(error). Best-effort mode - skipping an overlay.")
+                return []
+            }
+        }
+    }
+
+}

Sources/XCRemoteCache/Dependencies/StringDependenciesRemapperFactory.swift

@@ -0,0 +1,43 @@
+// Copyright (c) 2021 Spotify AB.
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import Foundation
+
+enum StringDependenciesRemapperFactoryError: Error {
+    /// Remapping keys are duplicated and can lead to undetermined results
+    case mappingKeyDuplication
+}
+
+class StringDependenciesRemapperFactory {
+    func build(
+        orderKeys: [String],
+        envs: [String: String],
+        customMappings: [String: String]
+    ) throws -> StringDependenciesRemapper {
+        let mappingMap = try envs.merging(customMappings) { envValue, outOfBandMapping in
+            throw StringDependenciesRemapperFactoryError.mappingKeyDuplication
+        }
+        let mappingOrderKeys =  orderKeys + customMappings.keys
+        let mappings: [StringDependenciesRemapper.Mapping] = try mappingOrderKeys.map { key in
+            let localValue: String = try mappingMap.readEnv(key: key)
+            return StringDependenciesRemapper.Mapping(generic: "$(\(key))", local: localValue)
+        }
+        return StringDependenciesRemapper(mappings: mappings)
+    }
+}

Sources/XCRemoteCache/Fingerprint/EnvironmentFingerprint.swift

@@ -19,10 +19,10 @@
 
 /// Generates a fingerprint string of the environment (compilation context)
 class EnvironmentFingerprintGenerator {
-    /// Default ENV variables constituing the environment fingerprint
+    /// Default ENV variables constituting the environment fingerprint
     private static let defaultEnvFingerprintKeys = [
         "GCC_PREPROCESSOR_DEFINITIONS",
-        "CLANG_PROFILE_DATA_DIRECTORY",
+        "CLANG_COVERAGE_MAPPING",
         "TARGET_NAME",
         "CONFIGURATION",
         "PLATFORM_NAME",
@@ -31,6 +31,7 @@ class EnvironmentFingerprintGenerator {
         "DYLIB_COMPATIBILITY_VERSION",
         "DYLIB_CURRENT_VERSION",
         "PRODUCT_MODULE_NAME",
+        "ARCHS"
     ]
     private let version: String
     private let customFingerprintEnvs: [String]

Sources/XCRemoteCache/Models/MetaWriter.swift

@@ -0,0 +1,45 @@
+// Copyright (c) 2021 Spotify AB.
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import Foundation
+
+protocol MetaWriter {
+    func write<T>(_ meta: T, locationDir : URL) throws -> URL where T : Meta
+}
+
+class JsonMetaWriter: MetaWriter {
+    private let metaEncoder: JSONEncoder
+    private let fileWriter: FileWriter
+
+    init(fileWriter: FileWriter, pretty: Bool) {
+        self.fileWriter = fileWriter
+        let encoder = JSONEncoder()
+        if pretty {
+            encoder.outputFormatting = .prettyPrinted
+        }
+        self.metaEncoder = encoder
+    }
+
+    func write<T>(_ meta: T, locationDir : URL) throws -> URL where T : Meta {
+        let metaURL = locationDir.appendingPathComponent(meta.fileKey).appendingPathExtension("json")
+        let metaData = try metaEncoder.encode(meta)
+        try fileWriter.write(toPath: metaURL.path, contents: metaData)
+        return metaURL
+    }
+}

Sources/XCRemoteCache/Network/Authentication/AWSV4Signature.swift

@@ -35,9 +35,22 @@ struct AWSV4Signature {
         request.setValue((request.httpBody ?? Data()).sha256(), forHTTPHeaderField: "x-amz-content-sha256")
 
         let canonicalRequest = CanonicalRequest(request: request)
-        let stringToSign = StringToSign(region: region, service: service, canonicalRequestHash: canonicalRequest.hash, date: date)
-        let awsV4SigningKey = AWSV4SigningKey(secretAccessKey: secretKey, region: region, service: service, date: date)
-        let signature = HMAC.calcHMAC(keyArray: awsV4SigningKey.value, value: stringToSign.value).map { String(format: "%02hhx", $0) }.joined()
+        let stringToSign = StringToSign(
+            region: region,
+            service: service,
+            canonicalRequestHash: canonicalRequest.hash,
+            date: date
+        )
+        let awsV4SigningKey = AWSV4SigningKey(
+            secretAccessKey: secretKey,
+            region: region,
+            service: service,
+            date: date
+        )
+        let signature = HMAC.calcHMAC(
+            keyArray: awsV4SigningKey.value,
+            value: stringToSign.value
+        ).map { String(format: "%02hhx", $0) }.joined()
 
         let authValue =
             "AWS4-HMAC-SHA256 " +

Sources/XCRemoteCache/Network/Authentication/HMAC.swift

@@ -52,7 +52,14 @@ struct HMAC {
 
     private static func calcHMAC(keyUnsafeBytes: UnsafeRawBufferPointer, value: String, out: UnsafeMutableRawPointer!) {
         value.data(using: .utf8)!.withUnsafeBytes { value in
-            CCHmac(CCHmacAlgorithm(kCCHmacAlgSHA256), keyUnsafeBytes.baseAddress, Int(keyUnsafeBytes.count), value.baseAddress, Int(value.count), out)
+            CCHmac(
+                CCHmacAlgorithm(kCCHmacAlgSHA256),
+                keyUnsafeBytes.baseAddress,
+                Int(keyUnsafeBytes.count),
+                value.baseAddress,
+                Int(value.count),
+                out
+            )
         }
     }
 }

Sources/XCRemoteCache/Network/IgnoringCertificatesTrustManager.swift

@@ -0,0 +1,32 @@
+// Copyright (c) 2021 Spotify AB.
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import Foundation
+
+final class IgnoringCertificatesTrustManager: NSObject, URLSessionDelegate {
+    func urlSession(_ session: URLSession, didReceive challenge: URLAuthenticationChallenge, completionHandler: @escaping (URLSession.AuthChallengeDisposition, URLCredential?) -> Void) {
+        guard let serverTrust = challenge.protectionSpace.serverTrust else {
+            completionHandler(.performDefaultHandling, nil)
+            return
+        }
+        
+        let urlCredential = URLCredential(trust: serverTrust)
+        completionHandler(.useCredential, urlCredential)
+    }
+}

Sources/XCRemoteCache/Network/RemoteNetworkClientAbstractFactory.swift

@@ -44,7 +44,7 @@ class RemoteNetworkClientAbstractFactory {
             return RemoteNetworkClientImpl(networkClient, downloadURLBuilder)
         }
         switch mode {
-        case .producer:
+        case .producer, .producerFast:
             let upstreamBuilders = try upstreamStreamURL.map(urlBuilderFactory)
             return ReplicatedRemotesNetworkClient(
                 networkClient,

Sources/XCRemoteCache/Network/URLSessionFactory.swift

@@ -36,6 +36,15 @@ class DefaultURLSessionFactory: URLSessionFactory {
         let configuration = URLSessionConfiguration.default
         configuration.httpAdditionalHeaders = config.requestCustomHeaders
         configuration.timeoutIntervalForRequest = config.timeoutResponseDataChunksInterval
-        return URLSession(configuration: configuration)
+        switch config.disableCertificateVerification {
+        case true:
+            return URLSession(
+                configuration: configuration,
+                delegate: IgnoringCertificatesTrustManager(),
+                delegateQueue: nil
+            )
+        case false:
+            return URLSession(configuration: configuration)
+        }
     }
 }

Sources/XCRemoteCache/Stats/CacheHitLogger.swift

@@ -28,8 +28,8 @@ protocol CacheHitLogger {
 /// Logs target hit or miss, based on an action of a build
 class ActionSpecificCacheHitLogger: CacheHitLogger {
     private let statsLogger: StatsLogger
-    private let hitCounter: XCRemoteCacheStatistics.Counter
-    private let missCounter: XCRemoteCacheStatistics.Counter
+    private let hitCounter: XCRemoteCacheStatistics.Counter?
+    private let missCounter: XCRemoteCacheStatistics.Counter?
 
     init(action: BuildActionType, statsLogger: StatsLogger) {
         self.statsLogger = statsLogger
@@ -37,19 +37,24 @@ class ActionSpecificCacheHitLogger: CacheHitLogger {
         case .index:
             hitCounter = .indexingTargetHitCount
             missCounter = .indexingTargetMissCount
-        case .unknown:
-            fallthrough
         case .build:
             hitCounter = .targetCacheHit
             missCounter = .targetCacheMiss
+        case .unknown:
+            hitCounter = nil
+            missCounter = nil
         }
     }
 
     func logHit() throws {
-        try statsLogger.log(hitCounter)
+        if let hitCounter = hitCounter {
+            try statsLogger.log(hitCounter)
+        }
     }
 
     func logMiss() throws {
-        try statsLogger.log(missCounter)
+        if let missCounter = missCounter {
+            try statsLogger.log(missCounter)
+        }
     }
 }

Tests/XCRemoteCacheTests/Artifacts/ArtifactSwiftProductsBuilderImplTests.swift

@@ -27,15 +27,18 @@ class ArtifactSwiftProductsBuilderImplTests: FileXCTestCase {
     private var swiftmoduleFile: URL!
     private var swiftmoduleDocFile: URL!
     private var swiftmoduleSourceInfoFile: URL!
+    private var swiftmoduleInterfaceFile: URL!
     private var workingDir: URL!
     private var builder: ArtifactSwiftProductsBuilderImpl!
 
     override func setUpWithError() throws {
+        try super.setUpWithError()
         let rootDir = try prepareTempDir()
         moduleDir = rootDir.appendingPathComponent("Products")
         swiftmoduleFile = moduleDir.appendingPathComponent("MyModule.swiftmodule")
         swiftmoduleDocFile = moduleDir.appendingPathComponent("MyModule.swiftdoc")
         swiftmoduleSourceInfoFile = moduleDir.appendingPathComponent("MyModule.swiftsourceinfo")
+        swiftmoduleInterfaceFile = moduleDir.appendingPathComponent("MyModule.swiftinterface")
         workingDir = rootDir.appendingPathComponent("working")
         builder = ArtifactSwiftProductsBuilderImpl(
             workingDir: workingDir,
@@ -47,24 +50,41 @@ class ArtifactSwiftProductsBuilderImplTests: FileXCTestCase {
     func testIncludesRequiredSwiftmoduleFiles() throws {
         try fileManager.spt_createFile(swiftmoduleFile, content: "swiftmodule")
         try fileManager.spt_createFile(swiftmoduleDocFile, content: "swiftdoc")
-        let builderSwiftmoduleDir = builder.buildingArtifactSwiftModulesLocation().appendingPathComponent("arm64")
-        let expectedBuildedSwiftmoduleFile = builderSwiftmoduleDir.appendingPathComponent("MyModule.swiftmodule")
-        let expectedBuildedSwiftmoduledocFile = builderSwiftmoduleDir.appendingPathComponent("MyModule.swiftdoc")
+        let builderSwiftmoduleDir =
+            builder
+                .buildingArtifactSwiftModulesLocation()
+                .appendingPathComponent("arm64")
+        let expectedBuildedSwiftmoduleFile =
+            builderSwiftmoduleDir.appendingPathComponent("MyModule.swiftmodule")
+        let expectedBuildedSwiftmoduledocFile =
+            builderSwiftmoduleDir.appendingPathComponent("MyModule.swiftdoc")
 
         try builder.includeModuleDefinitionsToTheArtifact(arch: "arm64", moduleURL: swiftmoduleFile)
 
-        XCTAssertEqual(fileManager.contents(atPath: expectedBuildedSwiftmoduleFile.path), "swiftmodule".data(using: .utf8))
-        XCTAssertEqual(fileManager.contents(atPath: expectedBuildedSwiftmoduledocFile.path), "swiftdoc".data(using: .utf8))
+        XCTAssertEqual(
+            fileManager.contents(atPath: expectedBuildedSwiftmoduleFile.path),
+            "swiftmodule".data(using: .utf8)
+        )
+        XCTAssertEqual(
+            fileManager.contents(atPath: expectedBuildedSwiftmoduledocFile.path),
+            "swiftdoc".data(using: .utf8)
+        )
     }
 
-    func testIncludesAllSwiftmoduleFiles() throws {
+    func testIncludesAllBasicSwiftmoduleFiles() throws {
         try fileManager.spt_createEmptyFile(swiftmoduleFile)
         try fileManager.spt_createEmptyFile(swiftmoduleDocFile)
         try fileManager.spt_createEmptyFile(swiftmoduleSourceInfoFile)
-        let builderSwiftmoduleDir = builder.buildingArtifactSwiftModulesLocation().appendingPathComponent("arm64")
-        let expectedBuildedSwiftmoduleFile = builderSwiftmoduleDir.appendingPathComponent("MyModule.swiftmodule")
-        let expectedBuildedSwiftmoduledocFile = builderSwiftmoduleDir.appendingPathComponent("MyModule.swiftdoc")
-        let expectedBuildedSwiftSourceInfoFile = builderSwiftmoduleDir.appendingPathComponent("MyModule.swiftsourceinfo")
+        let builderSwiftmoduleDir =
+            builder
+                .buildingArtifactSwiftModulesLocation()
+                .appendingPathComponent("arm64")
+        let expectedBuildedSwiftmoduleFile =
+            builderSwiftmoduleDir.appendingPathComponent("MyModule.swiftmodule")
+        let expectedBuildedSwiftmoduledocFile =
+            builderSwiftmoduleDir.appendingPathComponent("MyModule.swiftdoc")
+        let expectedBuildedSwiftSourceInfoFile =
+            builderSwiftmoduleDir.appendingPathComponent("MyModule.swiftsourceinfo")
 
         try builder.includeModuleDefinitionsToTheArtifact(arch: "arm64", moduleURL: swiftmoduleFile)
 
@@ -73,7 +93,38 @@ class ArtifactSwiftProductsBuilderImplTests: FileXCTestCase {
         XCTAssertTrue(fileManager.fileExists(atPath: expectedBuildedSwiftSourceInfoFile.path))
     }
 
+    func testIncludesAllEvolutionEnabledSwiftmoduleFiles() throws {
+        try fileManager.spt_createEmptyFile(swiftmoduleFile)
+        try fileManager.spt_createEmptyFile(swiftmoduleDocFile)
+        try fileManager.spt_createEmptyFile(swiftmoduleSourceInfoFile)
+        try fileManager.spt_createEmptyFile(swiftmoduleInterfaceFile)
+        let builderSwiftmoduleDir =
+            builder
+                .buildingArtifactSwiftModulesLocation()
+                .appendingPathComponent("arm64")
+        let expectedBuildedSwiftmoduleFile =
+            builderSwiftmoduleDir.appendingPathComponent("MyModule.swiftmodule")
+        let expectedBuildedSwiftmoduledocFile =
+            builderSwiftmoduleDir.appendingPathComponent("MyModule.swiftdoc")
+        let expectedBuildedSwiftSourceInfoFile =
+            builderSwiftmoduleDir.appendingPathComponent("MyModule.swiftsourceinfo")
+        let expectedBuildedSwiftInterfaceFile =
+            builderSwiftmoduleDir.appendingPathComponent("MyModule.swiftinterface")
+
+        try builder.includeModuleDefinitionsToTheArtifact(arch: "arm64", moduleURL: swiftmoduleFile)
+
+        XCTAssertTrue(fileManager.fileExists(atPath: expectedBuildedSwiftmoduleFile.path))
+        XCTAssertTrue(fileManager.fileExists(atPath: expectedBuildedSwiftmoduledocFile.path))
+        XCTAssertTrue(fileManager.fileExists(atPath: expectedBuildedSwiftSourceInfoFile.path))
+        XCTAssertTrue(fileManager.fileExists(atPath: expectedBuildedSwiftInterfaceFile.path))
+    }
+
     func testFailsIncludingWhenMissingRequiredSwiftmoduleFiles() throws {
-        XCTAssertThrowsError(try builder.includeModuleDefinitionsToTheArtifact(arch: "arm64", moduleURL: swiftmoduleFile))
+        XCTAssertThrowsError(
+            try builder.includeModuleDefinitionsToTheArtifact(
+                arch: "arm64",
+                moduleURL: swiftmoduleFile
+            )
+        )
     }
 }

Tests/XCRemoteCacheTests/Artifacts/BuildArtifactCreatorTests.swift

@@ -31,6 +31,7 @@ class BuildArtifactCreatorTests: FileXCTestCase {
     private var swiftmoduleURL: URL!
     private var swiftdocURL: URL!
     private var swiftSourceInfoURL: URL!
+    private var swiftInterfaceURL: URL!
     private var executablePath: String!
     private var executableURL: URL!
     private var creator: BuildArtifactCreator!
@@ -50,6 +51,8 @@ class BuildArtifactCreatorTests: FileXCTestCase {
             .appendingPathComponent("Target.swiftdoc")
         swiftSourceInfoURL = workDirectory.appendingPathComponent("Objects-normal")
             .appendingPathComponent("Target.swiftsourceinfo")
+        swiftInterfaceURL = workDirectory.appendingPathComponent("Objects-normal")
+            .appendingPathComponent("Target.swiftinterface")
         executablePath = "libTarget.a"
         executableURL = buildDir.appendingPathComponent(executablePath)
         dSYM = executableURL.deletingPathExtension().appendingPathExtension(".dSYM")
@@ -63,6 +66,7 @@ class BuildArtifactCreatorTests: FileXCTestCase {
             moduleName: "Target",
             modulesFolderPath: "",
             dSYMPath: dSYM,
+            metaWriter: JsonMetaWriter(fileWriter: fileManager, pretty: false),
             fileManager: fileManager
         )
     }
@@ -113,6 +117,28 @@ class BuildArtifactCreatorTests: FileXCTestCase {
         ])
     }
 
+    func testPackagesEvolutionEnabledSwiftmoduleFiles() throws {
+        try fileManager.spt_createEmptyFile(swiftmoduleURL)
+        try fileManager.spt_createEmptyFile(swiftdocURL)
+        try fileManager.spt_createEmptyFile(swiftSourceInfoURL)
+        try fileManager.spt_createEmptyFile(swiftInterfaceURL)
+
+        try creator.includeModuleDefinitionsToTheArtifact(arch: "arch", moduleURL: swiftmoduleURL)
+        let artifact = try creator.createArtifact(artifactKey: "key", meta: sampleMeta)
+
+        let unzippedURL = workDirectory.appendingPathComponent(UUID().uuidString)
+        try Zip.unzipFile(artifact.package, destination: unzippedURL, overwrite: true, password: nil, progress: nil)
+        let allFiles = try fileManager.spt_allFilesRecusively(unzippedURL)
+        XCTAssertEqual(Set(allFiles), [
+            unzippedURL.appendingPathComponent("libTarget.a"),
+            unzippedURL.appendingPathComponent("fileKey.json"),
+            unzippedURL.appendingPathComponent("swiftmodule/arch/Target.swiftmodule"),
+            unzippedURL.appendingPathComponent("swiftmodule/arch/Target.swiftdoc"),
+            unzippedURL.appendingPathComponent("swiftmodule/arch/Target.swiftsourceinfo"),
+            unzippedURL.appendingPathComponent("swiftmodule/arch/Target.swiftinterface"),
+        ])
+    }
+
     func testFailsPackageWhenSwiftmoduleRelatedFilesAreMissing() throws {
         // Creating only `Target.swiftmodule`, without `.swiftdoc`
         try fileManager.spt_createEmptyFile(swiftmoduleURL)

Tests/XCRemoteCacheTests/Artifacts/ZipArtifactCreatorTests.swift

@@ -35,7 +35,11 @@ class ZipArtifactCreatorTests: FileXCTestCase {
     override func setUpWithError() throws {
         try super.setUpWithError()
         workingDir = try prepareTempDir().appendingPathComponent("creator")
-        creator = ZipArtifactCreator(workingDir: workingDir, fileManager: fileManager)
+        creator = ZipArtifactCreator(
+            workingDir: workingDir,
+            metaWriter: JsonMetaWriter(fileWriter: fileManager, pretty: false),
+            fileManager: fileManager
+        )
     }
 
     func testCreatingArtifactGeneratesValidArtifactId() throws {

Tests/XCRemoteCacheTests/Commands/Plugins/Thinning/ThinningCreatorPluginTests.swift

@@ -33,7 +33,10 @@ class ThinningCreatorPluginTests: FileXCTestCase {
         targetTempDirRoot = workingDir.appendingPathComponent("Root")
         currentTargetTempDir = targetTempDirRoot.appendingPathComponent("Current.build")
         try fileManager.spt_createEmptyDir(currentTargetTempDir)
-        plugin = ThinningCreatorPlugin(targetTempDir: currentTargetTempDir, dirScanner: FileManager.default)
+        plugin = ThinningCreatorPlugin(
+            targetTempDir: currentTargetTempDir,
+            modeMarkerPath: "rc_marker.enabled",
+            dirScanner: FileManager.default)
     }
 
     func testReturnsEmptyExtraKeysForNoArtifacts() throws {
@@ -73,4 +76,62 @@ class ThinningCreatorPluginTests: FileXCTestCase {
 
         XCTAssertThrowsError(try plugin.extraMetaKeys(Self.sampleMeta))
     }
+
+    func testDefinesExtraMetaKeysForTargetsThatReusedArtifact() throws {
+        let otherTargetTempDir = targetTempDirRoot.appendingPathComponent("Other.build")
+        let marker = otherTargetTempDir.appendingPathComponent("rc_marker.enabled")
+        let reusedArtifact = otherTargetTempDir
+            .appendingPathComponent("xccache")
+            .appendingPathComponent("123")
+            .appendingPathExtension("zip")
+        try fileManager.spt_createEmptyFile(marker)
+        try fileManager.spt_createEmptyFile(reusedArtifact)
+
+        let extraKeys = try plugin.extraMetaKeys(Self.sampleMeta)
+
+        XCTAssertEqual(extraKeys, ["thinning_Other": "123"])
+    }
+
+    func testFailsGeneratingExtraMetaKeysForTwoArtifactsInTargetTempDir() throws {
+        let otherTargetTempDir = targetTempDirRoot.appendingPathComponent("Other.build")
+        let marker = otherTargetTempDir.appendingPathComponent("rc_marker.enabled")
+        let reusedArtifact1 = otherTargetTempDir
+            .appendingPathComponent("xccache")
+            .appendingPathComponent("001")
+            .appendingPathExtension("zip")
+        let reusedArtifact2 = otherTargetTempDir
+            .appendingPathComponent("xccache")
+            .appendingPathComponent("002")
+            .appendingPathExtension("zip")
+        try fileManager.spt_createEmptyFile(marker)
+        try fileManager.spt_createEmptyFile(reusedArtifact1)
+        try fileManager.spt_createEmptyFile(reusedArtifact2)
+
+        XCTAssertThrowsError(try plugin.extraMetaKeys(Self.sampleMeta))
+    }
+
+    func testDefinesExtraMetaKeysForGeneratedAndReusedArtifact() throws {
+        let otherTargetTempDir = targetTempDirRoot.appendingPathComponent("Generated.build")
+        let generatedArtifact = otherTargetTempDir
+            .appendingPathComponent("xccache")
+            .appendingPathComponent("produced")
+            .appendingPathComponent("000")
+            .appendingPathExtension("zip")
+        try fileManager.spt_createEmptyFile(generatedArtifact)
+        let reusedTargetTempDir = targetTempDirRoot.appendingPathComponent("Reused.build")
+        let marker = reusedTargetTempDir.appendingPathComponent("rc_marker.enabled")
+        let reusedArtifact = reusedTargetTempDir
+            .appendingPathComponent("xccache")
+            .appendingPathComponent("999")
+            .appendingPathExtension("zip")
+        try fileManager.spt_createEmptyFile(marker)
+        try fileManager.spt_createEmptyFile(reusedArtifact)
+
+        let extraKeys = try plugin.extraMetaKeys(Self.sampleMeta)
+
+        XCTAssertEqual(extraKeys, [
+            "thinning_Generated": "000",
+            "thinning_Reused": "999"
+        ])
+    }
 }

Tests/XCRemoteCacheTests/Commands/PostbuildContextTests.swift

@@ -26,8 +26,8 @@ class PostbuildContextTests: FileXCTestCase {
     private static let SampleEnvs = [
         "TARGET_NAME": "TARGET_NAME",
         "TARGET_TEMP_DIR": "TARGET_TEMP_DIR",
-        "PLATFORM_PREFERRED_ARCH": "PLATFORM_PREFERRED_ARCH",
-        "OBJECT_FILE_DIR_normal": "OBJECT_FILE_DIR_normal" ,
+        "ARCHS": "x86_64",
+        "OBJECT_FILE_DIR_normal": "/OBJECT_FILE_DIR_normal" ,
         "CONFIGURATION": "CONFIGURATION",
         "PLATFORM_NAME": "PLATFORM_NAME",
         "XCODE_PRODUCT_BUILD_VERSION": "XCODE_PRODUCT_BUILD_VERSION",
@@ -42,6 +42,7 @@ class PostbuildContextTests: FileXCTestCase {
         "DWARF_DSYM_FILE_NAME": "DWARF_DSYM_FILE_NAME",
         "BUILT_PRODUCTS_DIR": "BUILT_PRODUCTS_DIR",
         "DERIVED_SOURCES_DIR": "DERIVED_SOURCES_DIR",
+        "CURRENT_VARIANT": "normal"
     ]
 
     override func setUpWithError() throws {
@@ -87,4 +88,45 @@ class PostbuildContextTests: FileXCTestCase {
 
         XCTAssertEqual(context.action, .index)
     }
+
+    func testReadsSingleArch() throws {
+        var envs = Self.SampleEnvs
+        envs["ARCHS"] = "x86_64"
+        let context = try PostbuildContext(config, env: envs)
+
+        XCTAssertEqual(context.arch, "x86_64")
+    }
+
+    func testReadsFirstArch() throws {
+        var envs = Self.SampleEnvs
+        envs["ARCHS"] = "x86_64 arm64"
+        let context = try PostbuildContext(config, env: envs)
+
+        XCTAssertEqual(context.arch, "x86_64")
+    }
+
+    func testFailsForEmptyArch() throws {
+        var envs = Self.SampleEnvs
+        envs["ARCHS"] = ""
+
+        XCTAssertThrowsError(try PostbuildContext(config, env: envs))
+    }
+
+    func testFailsForMissingArch() throws {
+        var envs = Self.SampleEnvs
+        envs["ARCHS"] = nil
+
+        XCTAssertThrowsError(try PostbuildContext(config, env: envs))
+    }
+
+    func testFindTempDirForCustomVariant() throws {
+        var envs = Self.SampleEnvs
+        envs["ARCHS"] = "x86_64"
+        envs["CURRENT_VARIANT"] = "custom"
+        envs["OBJECT_FILE_DIR_custom"] = "/OBJECT_FILE_DIR_custom"
+
+        let context = try PostbuildContext(config, env: envs)
+
+        XCTAssertEqual(context.compilationTempDir, "/OBJECT_FILE_DIR_custom/x86_64")
+    }
 }

Tests/XCRemoteCacheTests/Commands/PostbuildTests.swift

@@ -50,7 +50,9 @@ class PostbuildTests: FileXCTestCase {
         bundleDir: nil,
         derivedSourcesDir: "",
         thinnedTargets: [],
-        action: .build
+        action: .build,
+        modeMarkerPath: "",
+        overlayHeadersPath: ""
     )
     private var network = RemoteNetworkClientImpl(
         NetworkClientFake(fileManager: .default),
@@ -84,6 +86,7 @@ class PostbuildTests: FileXCTestCase {
     )
     private var modeController = CacheModeControllerFake()
     private var metaReader = JsonMetaReader(fileAccessor: FileManager.default)
+    private var metaWriter = JsonMetaWriter(fileWriter: FileManager.default, pretty: false)
     private static let SampleMeta = MainArtifactSampleMeta.defaults
     private var sampleMetaFile: URL!
 
@@ -122,6 +125,7 @@ class PostbuildTests: FileXCTestCase {
             dSYMOrganizer: DSYMOrganizerFake(dSYMFile: nil),
             modeController: modeController,
             metaReader: metaReader,
+            metaWriter: metaWriter,
             creatorPlugins: [],
             consumerPlugins: []
         )
@@ -151,6 +155,7 @@ class PostbuildTests: FileXCTestCase {
             dSYMOrganizer: DSYMOrganizerFake(dSYMFile: nil),
             modeController: modeController,
             metaReader: metaReader,
+            metaWriter: metaWriter,
             creatorPlugins: [],
             consumerPlugins: []
         )
@@ -178,6 +183,7 @@ class PostbuildTests: FileXCTestCase {
             dSYMOrganizer: dsymOrganizer,
             modeController: modeController,
             metaReader: metaReader,
+            metaWriter: metaWriter,
             creatorPlugins: [],
             consumerPlugins: []
         )
@@ -205,6 +211,7 @@ class PostbuildTests: FileXCTestCase {
             dSYMOrganizer: dsymOrganizer,
             modeController: modeController,
             metaReader: metaReader,
+            metaWriter: metaWriter,
             creatorPlugins: [],
             consumerPlugins: []
         )
@@ -242,6 +249,7 @@ class PostbuildTests: FileXCTestCase {
             dSYMOrganizer: dsymOrganizer,
             modeController: modeController,
             metaReader: metaReader,
+            metaWriter: metaWriter,
             creatorPlugins: [],
             consumerPlugins: []
         )
@@ -282,6 +290,7 @@ class PostbuildTests: FileXCTestCase {
             dSYMOrganizer: dsymOrganizer,
             modeController: modeController,
             metaReader: metaReader,
+            metaWriter: metaWriter,
             creatorPlugins: [],
             consumerPlugins: []
         )
@@ -307,6 +316,7 @@ class PostbuildTests: FileXCTestCase {
             dSYMOrganizer: DSYMOrganizerFake(dSYMFile: nil),
             modeController: fakeModeController,
             metaReader: metaReader,
+            metaWriter: metaWriter,
             creatorPlugins: [],
             consumerPlugins: []
         )
@@ -354,6 +364,7 @@ class PostbuildTests: FileXCTestCase {
             dSYMOrganizer: DSYMOrganizerFake(dSYMFile: nil),
             modeController: modeController,
             metaReader: metaReader,
+            metaWriter: metaWriter,
             creatorPlugins: [plugin],
             consumerPlugins: []
         )
@@ -385,6 +396,7 @@ class PostbuildTests: FileXCTestCase {
             dSYMOrganizer: DSYMOrganizerFake(dSYMFile: nil),
             modeController: modeController,
             metaReader: metaReader,
+            metaWriter: metaWriter,
             creatorPlugins: [plugin],
             consumerPlugins: []
         )
@@ -417,6 +429,7 @@ class PostbuildTests: FileXCTestCase {
             dSYMOrganizer: DSYMOrganizerFake(dSYMFile: nil),
             modeController: modeController,
             metaReader: metaReader,
+            metaWriter: metaWriter,
             creatorPlugins: [],
             consumerPlugins: [consumerPlugin]
         )
@@ -454,6 +467,7 @@ class PostbuildTests: FileXCTestCase {
             dSYMOrganizer: DSYMOrganizerFake(dSYMFile: nil),
             modeController: modeController,
             metaReader: metaReader,
+            metaWriter: metaWriter,
             creatorPlugins: [],
             consumerPlugins: [consumerPlugin]
         )
@@ -485,6 +499,7 @@ class PostbuildTests: FileXCTestCase {
             dSYMOrganizer: DSYMOrganizerFake(dSYMFile: nil),
             modeController: modeController,
             metaReader: metaReader,
+            metaWriter: metaWriter,
             creatorPlugins: [],
             consumerPlugins: [consumerPlugin]
         )
@@ -518,6 +533,7 @@ class PostbuildTests: FileXCTestCase {
             dSYMOrganizer: DSYMOrganizerFake(dSYMFile: nil),
             modeController: modeController,
             metaReader: metaReader,
+            metaWriter: metaWriter,
             creatorPlugins: [],
             consumerPlugins: []
         )
@@ -550,6 +566,7 @@ class PostbuildTests: FileXCTestCase {
             dSYMOrganizer: DSYMOrganizerFake(dSYMFile: nil),
             modeController: modeController,
             metaReader: metaReader,
+            metaWriter: metaWriter,
             creatorPlugins: [],
             consumerPlugins: []
         )
@@ -558,5 +575,68 @@ class PostbuildTests: FileXCTestCase {
         try postbuild.deleteFingerprintOverrides()
 
         XCTAssertFalse(fileManager.fileExists(atPath: previousFingerprintOverride.path))
-    } // swiftlint:disable:next file_length
+    }
+
+    func testUploadingMeta() throws {
+        let postbuild = Postbuild(
+            context: postbuildContext,
+            networkClient: network,
+            remapper: remapper,
+            fingerprintAccumulator: fingerprintGenerator,
+            artifactsOrganizer: organizer,
+            artifactCreator: artifactCreator,
+            fingerprintSyncer: syncer,
+            dependenciesReader: dependenciesReader,
+            dependencyProcessor: processor,
+            fingerprintOverrideManager: overrideManager,
+            dSYMOrganizer: DSYMOrganizerFake(dSYMFile: nil),
+            modeController: modeController,
+            metaReader: metaReader,
+            metaWriter: metaWriter,
+            creatorPlugins: [],
+            consumerPlugins: []
+        )
+
+        try postbuild.performMetaUpload(meta: Self.SampleMeta, for: "33")
+
+
+        let data = try network.fetch(.meta(commit: "33"))
+        let downloadedMeta = try metaReader.read(data: data)
+
+        XCTAssertEqual(downloadedMeta, Self.SampleMeta)
+    }
+
+    func testUploadingMetaWithNewPluginKeys() throws {
+        let plugin = MetaAppenderArtifactCreatorPlugin(["New": "Value"])
+        let postbuild = Postbuild(
+            context: postbuildContext,
+            networkClient: network,
+            remapper: remapper,
+            fingerprintAccumulator: fingerprintGenerator,
+            artifactsOrganizer: organizer,
+            artifactCreator: artifactCreator,
+            fingerprintSyncer: syncer,
+            dependenciesReader: dependenciesReader,
+            dependencyProcessor: processor,
+            fingerprintOverrideManager: overrideManager,
+            dSYMOrganizer: DSYMOrganizerFake(dSYMFile: nil),
+            modeController: modeController,
+            metaReader: metaReader,
+            metaWriter: metaWriter,
+            creatorPlugins: [plugin],
+            consumerPlugins: []
+        )
+        var meta = Self.SampleMeta
+        meta.pluginsKeys = ["Previous": "Value"]
+        var expectedMeta = meta
+        expectedMeta.pluginsKeys = ["New": "Value"]
+
+        try postbuild.performMetaUpload(meta: meta, for: "33")
+
+        let data = try network.fetch(.meta(commit: "33"))
+        let downloadedMeta = try metaReader.read(data: data)
+
+        XCTAssertEqual(downloadedMeta, expectedMeta)
+    }
 }
+// swiftlint:disable:next file_length

Tests/XCRemoteCacheTests/Commands/PrebuildTests.swift

@@ -62,7 +62,8 @@ class PrebuildTests: FileXCTestCase {
             forceCached: false,
             compilationHistoryFile: compilationHistory,
             turnOffRemoteCacheOnFirstTimeout: true,
-            targetName: ""
+            targetName: "",
+            overlayHeadersPath: ""
         )
         contextCached = PrebuildContext(
             targetTempDir: sampleURL,
@@ -74,7 +75,8 @@ class PrebuildTests: FileXCTestCase {
             forceCached: true,
             compilationHistoryFile: compilationHistory,
             turnOffRemoteCacheOnFirstTimeout: true,
-            targetName: ""
+            targetName: "",
+            overlayHeadersPath: ""
         )
         organizer = ArtifactOrganizerFake(artifactRoot: artifactsRoot, unzippedExtension: "unzip")
         globalCacheSwitcher = InMemoryGlobalCacheSwitcher()
@@ -238,7 +240,8 @@ class PrebuildTests: FileXCTestCase {
             forceCached: false,
             compilationHistoryFile: compilationHistory,
             turnOffRemoteCacheOnFirstTimeout: true,
-            targetName: ""
+            targetName: "",
+            overlayHeadersPath: ""
         )
 
         let prebuild = Prebuild(
@@ -268,7 +271,8 @@ class PrebuildTests: FileXCTestCase {
             forceCached: false,
             compilationHistoryFile: compilationHistory,
             turnOffRemoteCacheOnFirstTimeout: true,
-            targetName: ""
+            targetName: "",
+            overlayHeadersPath: ""
         )
         metaContent = try generateMeta(fingerprint: generator.generate(), filekey: "1")
         let downloadedArtifactPackage = artifactsRoot.appendingPathComponent("1")
@@ -330,7 +334,8 @@ class PrebuildTests: FileXCTestCase {
             forceCached: false,
             compilationHistoryFile: compilationHistory,
             turnOffRemoteCacheOnFirstTimeout: false,
-            targetName: ""
+            targetName: "",
+            overlayHeadersPath: ""
         )
         try globalCacheSwitcher.enable(sha: "1")
         let prebuild = Prebuild(

Tests/XCRemoteCacheTests/Commands/Prepare/Integrate/FileLLDBInitPatcherTests.swift

@@ -28,6 +28,7 @@ class FileLLDBInitPatcherTests: XCTestCase {
     private var patcher: FileLLDBInitPatcher!
 
     override func setUp() {
+        super.setUp()
         accessor = FileAccessorFake(mode: .normal)
         patcher = FileLLDBInitPatcher(
             file: lldbInitPath,

Tests/XCRemoteCacheTests/Commands/SwiftcContextTests.swift

@@ -62,4 +62,19 @@ class SwiftcContextTests: FileXCTestCase {
 
         XCTAssertEqual(context.mode, .consumer(commit: .unavailable))
     }
+
+    func testProducerModeWhenFileWithCommitShaExistsIsResolvedToProducerFast() throws {
+        config.mode = .producerFast
+        let context = try SwiftcContext(config: config, input: input)
+
+        XCTAssertEqual(context.mode, .producerFast)
+    }
+
+    func testProducerModeWhenFileWithCommitShaDoesntExxistIsResolvedToProducer() throws {
+        config.mode = .producerFast
+        try fileManager.spt_deleteItem(at: remoteCommitFile)
+        let context = try SwiftcContext(config: config, input: input)
+
+        XCTAssertEqual(context.mode, .producer)
+    }
 }

Tests/XCRemoteCacheTests/Commands/SwiftcFilemapInputEditorTests.swift

@@ -21,8 +21,8 @@
 import XCTest
 
 class SwiftcFilemapInputEditorTests: FileXCTestCase {
-
-    private let sampleInfo = SwiftCompilationInfo(info: SwiftModuleCompilationInfo(
+    private let sampleInfo = SwiftCompilationInfo(
+        info: SwiftModuleCompilationInfo(
         dependencies: nil,
         swiftDependencies: "/"
     ), files: [])
@@ -65,17 +65,19 @@ class SwiftcFilemapInputEditorTests: FileXCTestCase {
            }
         }
         """#.data(using: .utf8)!
-        let expectedInfo = SwiftCompilationInfo(info: SwiftModuleCompilationInfo(
-            dependencies: "/master.d",
-            swiftDependencies: "/master.swiftdeps"
-        ), files: [
-            SwiftFileCompilationInfo(
-                file: "/file1.swift",
-                dependencies: "/file1.d",
-                object: "/file1.o",
-                swiftDependencies: "/file1.swiftdeps"
+        let expectedInfo = SwiftCompilationInfo(
+            info: SwiftModuleCompilationInfo(
+                dependencies: "/master.d",
+                swiftDependencies: "/master.swiftdeps"
             ),
-        ])
+            files: [
+                SwiftFileCompilationInfo(
+                    file: "/file1.swift",
+                    dependencies: "/file1.d",
+                    object: "/file1.o",
+                    swiftDependencies: "/file1.swiftdeps"
+                ),
+            ])
         try fileManager.spt_writeToFile(atPath: inputFile.path, contents: infoContentData)
 
         let readInfo = try editor.read()
@@ -93,17 +95,18 @@ class SwiftcFilemapInputEditorTests: FileXCTestCase {
     }
 
     func testWritingSavesContentWithOptionalParameters() throws {
-        let extendedInfo = SwiftCompilationInfo(info: SwiftModuleCompilationInfo(
-            dependencies: "/master.d",
-            swiftDependencies: "/master.swiftdeps"
-        ), files: [
-            SwiftFileCompilationInfo(
-                file: "/file1.swift",
-                dependencies: "/file1.d",
-                object: "/file1.o",
-                swiftDependencies: "/file1.swiftdeps"
-            ),
-        ])
+        let extendedInfo = SwiftCompilationInfo(
+            info: SwiftModuleCompilationInfo(
+                dependencies: "/master.d",
+                swiftDependencies: "/master.swiftdeps"
+            ), files: [
+                SwiftFileCompilationInfo(
+                    file: "/file1.swift",
+                    dependencies: "/file1.d",
+                    object: "/file1.o",
+                    swiftDependencies: "/file1.swiftdeps"
+                ),
+            ])
 
         try editor.write(extendedInfo)
 

Tests/XCRemoteCacheTests/Commands/SwiftcOrchestratorTests.swift

@@ -190,4 +190,45 @@ class SwiftcOrchestratorTests: XCTestCase {
 
         XCTAssertNotNil(shellOutSpy.switchedProcess)
     }
+
+    func testForFailedCompilationMockInProducerFastModeBuildsArtifactObjCHeader() throws {
+        let swiftc = SwiftcMock(mockingResult: .forceFallback)
+        let orchestrator = SwiftcOrchestrator(
+            mode: .producerFast,
+            swiftc: swiftc,
+            swiftcCommand: "",
+            objcHeaderOutput: objcHeaderURL,
+            moduleOutput: moduleOutputURL,
+            arch: "archTest",
+            artifactBuilder: artifactBuilder,
+            producerFallbackCommandProcessors: [],
+            invocationStorage: invocationStorage,
+            shellOut: shellOutSpy
+        )
+
+        try orchestrator.run()
+
+        XCTAssertEqual(artifactBuilder.addedObjCHeaders, ["archTest": [objcHeaderURL]])
+    }
+
+    func testSuccessedMockInProducerFastModeDoesntFillObjCHeader() throws {
+        let swiftc = SwiftcMock(mockingResult: .success)
+        let orchestrator = SwiftcOrchestrator(
+            mode: .producerFast,
+            swiftc: swiftc,
+            swiftcCommand: "",
+            objcHeaderOutput: objcHeaderURL,
+            moduleOutput: moduleOutputURL,
+            arch: "arch",
+            artifactBuilder: artifactBuilder,
+            producerFallbackCommandProcessors: [],
+            invocationStorage: invocationStorage,
+            shellOut: shellOutSpy
+        )
+
+        try orchestrator.run()
+
+        XCTAssertEqual(artifactBuilder.addedObjCHeaders, [:])
+    }
+
 }

Tests/XCRemoteCacheTests/Commands/SwiftcTests.swift

@@ -277,7 +277,12 @@ class SwiftcTests: FileXCTestCase {
         let artifactObjCHeader = URL(fileURLWithPath: "/cachedArtifact/include/archTest/Target-Swift.h")
         let artifactSwiftmodule = URL(fileURLWithPath: "/cachedArtifact/swiftmodule/archTest/Target.swiftmodule")
         let artifactSwiftdoc = URL(fileURLWithPath: "/cachedArtifact/swiftmodule/archTest/Target.swiftdoc")
-        let artifactSwiftSourceInfo = URL(fileURLWithPath: "/cachedArtifact/swiftmodule/archTest/Target.swiftsourceinfo")
+        let artifactSwiftSourceInfo = URL(
+            fileURLWithPath: "/cachedArtifact/swiftmodule/archTest/Target.swiftsourceinfo"
+        )
+        let artifactSwiftInterfaceInfo = URL(
+            fileURLWithPath: "/cachedArtifact/swiftmodule/archTest/Target.swiftinterface"
+        )
 
         artifactOrganizer = ArtifactOrganizerFake(artifactRoot: artifactRoot)
         let swiftc = Swiftc(
@@ -301,12 +306,14 @@ class SwiftcTests: FileXCTestCase {
         let swiftModuleURL = swiftModuleFiles.0[.swiftmodule]
         let swiftDocURL = swiftModuleFiles.0[.swiftdoc]
         let swiftSourceInfoURL = swiftModuleFiles.0[.swiftsourceinfo]
+        let swiftInterfaceURL = swiftModuleFiles.0[.swiftinterface]
         let swiftHeaderURL = swiftModuleFiles.1
 
         XCTAssertEqual(swiftModuleURL, artifactSwiftmodule)
         XCTAssertEqual(swiftDocURL, artifactSwiftdoc)
         XCTAssertEqual(swiftSourceInfoURL, artifactSwiftSourceInfo)
         XCTAssertEqual(swiftHeaderURL, artifactObjCHeader)
+        XCTAssertEqual(swiftInterfaceURL, artifactSwiftInterfaceInfo)
     }
 
 
@@ -457,5 +464,5 @@ class SwiftcTests: FileXCTestCase {
         )
 
         XCTAssertNoThrow(try swiftc.mockCompilation())
-    }
+    } // swiftlint:disable:next file_length
 }

Tests/XCRemoteCacheTests/Commands/XCCCTests.swift

@@ -28,7 +28,7 @@ class TemplateBasedCCWrapperBuilderTests: FileXCTestCase {
     private static let history = "history.compile"
     private static let prebuild = "prebuild.d"
     private static let commitSha = "321"
-    private static let timeout = 5.0
+    private static let timeout = 10.0
 
     static let xccc: URL = {
         let fileManager = FileManager.default

Tests/XCRemoteCacheTests/Config/ModeTests.swift

@@ -0,0 +1,33 @@
+// Copyright (c) 2021 Spotify AB.
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+@testable import XCRemoteCache
+import XCTest
+import Yams
+
+class ModeTests: XCTestCase {
+
+    func testProducerFast() throws {
+        let yaml = "producer-fast"
+        let decoder = YAMLDecoder(encoding: .utf8)
+        let mode: Mode = try decoder.decode(from: yaml)
+
+        XCTAssertEqual(mode, .producerFast)
+    }
+}

Tests/XCRemoteCacheTests/Dependencies/DependenciesRemapperCompositeTests.swift

@@ -29,53 +29,90 @@ class DependenciesRemapperCompositeTests: XCTestCase {
         StringDependenciesRemapper.Mapping(generic: "$(PWD)", local: "/pwd"),
     ]
 
-    func testNoRemappersIsTransparent() {
+    func testNoRemappersIsTransparent() throws {
         let remapper = DependenciesRemapperComposite([])
 
-        let genericPath = remapper.replace(localPaths: ["/tmp/root/some.swift"])
+        let genericPath = try remapper.replace(localPaths: ["/tmp/root/some.swift"])
 
         XCTAssertEqual(genericPath, ["/tmp/root/some.swift"])
     }
 
-    func testOneRemapperReplacesLocalPaths() {
+    func testOneRemapperReplacesLocalPaths() throws {
         let remapper = DependenciesRemapperComposite([
             StringDependenciesRemapper(mappings: mappings1),
         ])
 
-        let genericPath = remapper.replace(localPaths: ["/tmp/root/some.swift"])
+        let genericPath = try remapper.replace(localPaths: ["/tmp/root/some.swift"])
 
         XCTAssertEqual(genericPath, ["$(SRC_ROOT)/some.swift"])
     }
 
-    func testOneRemapperReplacesGenericPaths() {
+    func testOneRemapperReplacesGenericPaths() throws {
         let remapper = DependenciesRemapperComposite([
             StringDependenciesRemapper(mappings: mappings1),
         ])
 
-        let localPath = remapper.replace(genericPaths: ["$(SRC_ROOT)/some.swift"])
+        let localPath = try remapper.replace(genericPaths: ["$(SRC_ROOT)/some.swift"])
 
         XCTAssertEqual(localPath, ["/tmp/root/some.swift"])
     }
 
-    func testTwoRemappersReplacesLocalPaths() {
+    func testTwoRemappersReplacesLocalPaths() throws {
         let remapper = DependenciesRemapperComposite([
             StringDependenciesRemapper(mappings: mappings1),
             StringDependenciesRemapper(mappings: mappings2),
         ])
 
-        let genericPath = remapper.replace(localPaths: ["/tmp/root/some.swift", "/pwd/other.swift"])
+        let genericPath = try remapper.replace(localPaths: ["/tmp/root/some.swift", "/pwd/other.swift"])
 
         XCTAssertEqual(genericPath, ["$(SRC_ROOT)/some.swift", "$(PWD)/other.swift"])
     }
 
-    func testOneRemappersReplacesGenericPaths() {
+    func testOneRemappersReplacesGenericPaths() throws {
         let remapper = DependenciesRemapperComposite([
             StringDependenciesRemapper(mappings: mappings1),
             StringDependenciesRemapper(mappings: mappings2),
         ])
 
-        let localPath = remapper.replace(genericPaths: ["$(SRC_ROOT)/some.swift", "$(PWD)/other.swift"])
+        let localPath = try remapper.replace(genericPaths: ["$(SRC_ROOT)/some.swift", "$(PWD)/other.swift"])
 
         XCTAssertEqual(localPath, ["/tmp/root/some.swift", "/pwd/other.swift"])
     }
+
+    func testRemapsMultipleMatchingMappers() throws {
+        let remapper = DependenciesRemapperComposite([
+            StringDependenciesRemapper(mappings: [StringDependenciesRemapper.Mapping(generic: "$(ROOT)", local: "/root")]),
+            StringDependenciesRemapper(mappings: [StringDependenciesRemapper.Mapping(generic: "$(SPECIFIC)", local: "$(ROOT)/specific")])
+        ])
+        let localPaths = ["/root/specific/file"]
+
+        let genericPaths = try remapper.replace(localPaths: localPaths)
+
+        XCTAssertEqual(genericPaths, ["$(SPECIFIC)/file"])
+    }
+
+    func testRemapsBackToLocalWithRevertedRemappersOrder() throws {
+        let remapper = DependenciesRemapperComposite([
+            StringDependenciesRemapper(mappings: [StringDependenciesRemapper.Mapping(generic: "$(ROOT)", local: "/root")]),
+            StringDependenciesRemapper(mappings: [StringDependenciesRemapper.Mapping(generic: "$(SPECIFIC)", local: "$(ROOT)/specific")])
+        ])
+        let genericPaths = ["$(SPECIFIC)/file"]
+
+        let localPaths = try remapper.replace(genericPaths: genericPaths)
+
+        XCTAssertEqual(localPaths, ["/root/specific/file"])
+    }
+
+    func testRemappingTwoMappingsBackAndForthIsIdentical() throws {
+        let remapper = DependenciesRemapperComposite([
+            StringDependenciesRemapper(mappings: [StringDependenciesRemapper.Mapping(generic: "$(ROOT)", local: "/root")]),
+            StringDependenciesRemapper(mappings: [StringDependenciesRemapper.Mapping(generic: "$(SPECIFIC)", local: "$(ROOT)/specific")])
+        ])
+        let localPaths = ["/root/specific/file"]
+
+        let genericPaths = try remapper.replace(localPaths: localPaths)
+        let remappedLocalPaths = try remapper.replace(genericPaths: genericPaths)
+
+        XCTAssertEqual(localPaths, remappedLocalPaths)
+    }
 }

Tests/XCRemoteCacheTests/Dependencies/DependencyProcessorImplTests.swift

@@ -20,7 +20,7 @@
 @testable import XCRemoteCache
 import XCTest
 
-class DependencyProcessorImplTests: XCTestCase {
+class DependencyProcessorImplTests: FileXCTestCase {
 
     let processor = DependencyProcessorImpl(
         xcode: "/Xcode",
@@ -70,6 +70,14 @@ class DependencyProcessorImplTests: XCTestCase {
         XCTAssertEqual(dependencies, [])
     }
 
+    func testDoesNotFilterOutOtherProductModulemap() throws {
+        let dependencies = processor.process([
+            "/ProductOther/some.modulemap",
+        ])
+
+        XCTAssertEqual(dependencies, [.init(url: "/ProductOther/some.modulemap", type: .unknown)])
+    }
+
     func testDoesNotFilterOutNonProductModulemap() throws {
         let dependencies = processor.process([
             "/Source/some.modulemap",
@@ -109,4 +117,66 @@ class DependencyProcessorImplTests: XCTestCase {
 
         XCTAssertEqual(dependencies, [.init(url: "/xxx/some", type: .unknown)])
     }
+
+    func testFiltersOutIntermediateBySymlink() throws {
+        let sampleDir = try prepareTempDir()
+
+        let intermediateDirReal = sampleDir.appendingPathComponent("Intermediate")
+        let symlink = sampleDir.appendingPathComponent("symlink")
+        let someFilename = "some"
+
+        let processor = DependencyProcessorImpl(
+            xcode: "/Xcode",
+            product: "/Product",
+            source: "/Source",
+            intermediate: intermediateDirReal,
+            bundle: "/Bundle"
+        )
+
+        let intermediateFileSymlink = createSymlink(filename: someFilename, sourceDir: symlink, destinationDir: intermediateDirReal)
+
+        let dependencies = processor.process([
+            intermediateFileSymlink
+        ])
+
+        XCTAssertEqual(dependencies, [])
+    }
+
+    func testDoesNotFilterOutSourceBySymlink() throws {
+        let sampleDir = try prepareTempDir()
+
+        let sourceDirReal = sampleDir.appendingPathComponent("Source")
+        let symlink = sampleDir.appendingPathComponent("symlink")
+        let someFilename = "some"
+
+        let processor = DependencyProcessorImpl(
+            xcode: "/Xcode",
+            product: "/Product",
+            source: sourceDirReal,
+            intermediate: "/Intermediate",
+            bundle: "/Bundle"
+        )
+
+        let sourceFileSymlink = createSymlink(filename: someFilename, sourceDir: symlink, destinationDir: sourceDirReal)
+
+        let dependencies = processor.process([
+            sourceFileSymlink
+        ])
+
+        XCTAssertEqual(dependencies, [.init(url: sourceFileSymlink, type: .source)])
+    }
+
+    /**
+     * Creates Symlink from sourceDir to destinationDir and creates empty file inside it
+     * return URL with symbolic link from sourceDir to destinationDir
+     */
+    fileprivate func createSymlink(filename: String, sourceDir: URL, destinationDir: URL) -> URL {
+        let fileMng = FileManager.default
+
+        XCTAssertNoThrow(try fileMng.spt_forceSymbolicLink(at: sourceDir,
+                                                           withDestinationURL: destinationDir))
+        XCTAssertNoThrow(try fileMng.spt_createEmptyFile(destinationDir.appendingPathComponent(filename)))
+
+        return sourceDir.appendingPathComponent(filename)
+    }
 }

Tests/XCRemoteCacheTests/Dependencies/FileFingerprintSyncerTests.swift

@@ -27,6 +27,7 @@ class FileFingerprintSyncerTests: FileXCTestCase {
     private var swiftmoduleDir: URL!
 
     override func setUpWithError() throws {
+        try super.setUpWithError()
         syncer = FileFingerprintSyncer(
             fingerprintOverrideExtension: "md5",
             dirAccessor: fileManager,

Tests/XCRemoteCacheTests/Dependencies/OverlayDependenciesRemapperTests.swift

@@ -0,0 +1,71 @@
+// Copyright (c) 2021 Spotify AB.
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+@testable import XCRemoteCache
+import XCTest
+
+class OverlayDependenciesRemapperTests: XCTestCase {
+    private let overlayReader = OverlayReaderFake(
+        mappings: [.init(virtual: "/file.h", local: "/Intermediate/Some/file.h")]
+    )
+
+    func testMappingFromLocalToGeneric() throws {
+        let remapper = OverlayDependenciesRemapper(overlayReader: overlayReader)
+
+        let dependencies = try remapper.replace(localPaths: ["/Intermediate/Some/file.h"])
+        XCTAssertEqual(dependencies, ["/file.h"])
+    }
+
+    func testMappingFromGenericToLocal() throws {
+        let remapper = OverlayDependenciesRemapper(overlayReader: overlayReader)
+
+
+        let dependencies = try remapper.replace(genericPaths: ["/file.h"])
+        XCTAssertEqual(dependencies, ["/Intermediate/Some/file.h"])
+    }
+
+    func testGenericDependenciesAreNotMerged() throws {
+        let remapper = OverlayDependenciesRemapper(overlayReader: overlayReader)
+
+
+        let dependencies = try remapper.replace(localPaths: ["/Intermediate/Some/file.h", "/file.h"])
+        XCTAssertEqual(dependencies, ["/file.h", "/file.h"])
+    }
+
+    func testLocalDependenciesAreNotMerged() throws {
+        let remapper = OverlayDependenciesRemapper(overlayReader: overlayReader)
+
+
+        let dependencies = try remapper.replace(genericPaths: ["/Intermediate/Some/file.h", "/file.h"])
+        XCTAssertEqual(dependencies, ["/Intermediate/Some/file.h", "/Intermediate/Some/file.h"])
+    }
+
+    func testMappingsAreReadOnce() throws {
+        let remapper = OverlayDependenciesRemapper(overlayReader: overlayReader)
+
+        let firstDependencies = try remapper.replace(localPaths: ["/Intermediate/Some/file.h"])
+        // Update mappings in-fly to verify the previous value is cached in a remapper
+        overlayReader.mappings = []
+        let secondDependencies = try remapper.replace(localPaths: ["/Intermediate/Some/file.h"])
+
+        XCTAssertEqual(firstDependencies, ["/file.h"])
+        XCTAssertEqual(secondDependencies, ["/file.h"])
+    }
+}
+

Tests/XCRemoteCacheTests/Dependencies/OverlayReaderTests.swift

@@ -0,0 +1,84 @@
+// Copyright (c) 2021 Spotify AB.
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+@testable import XCRemoteCache
+import XCTest
+
+class JsonOverlayReaderTests: FileXCTestCase {
+    private static let resourcesSubdirectory = "TestData/Dependencies/JsonOverlayReaderTests"
+
+    func testParsingWithSuccess() throws {
+        let file = try pathForTestData(name: "overlayReaderDefault")
+        let reader = JsonOverlayReader(file, mode: .strict, fileReader: FileManager.default)
+        let mappings = try reader.provideMappings()
+
+        let expectedMappings = [
+            OverlayMapping(virtual: "/DerivedDataProducts/Target1.framework/Headers/Target1.h", local: "/Path/Target1/Target1.h"),
+            OverlayMapping(virtual: "/DerivedDataProducts/Target2.framework/Modules/module.modulemap", local: "/DerivedDataIntermediate/Target2.build/module.modulemap")
+        ]
+        XCTAssertEqual(Set(mappings), Set(expectedMappings))
+    }
+
+    func testFailsWithMissingFileForStrictMode() throws {
+        let file: URL = "nonExiting"
+        let reader = JsonOverlayReader(file, mode: .strict, fileReader: FileManager.default)
+
+        XCTAssertThrowsError(try reader.provideMappings())
+    }
+
+    func testReturnsEmpptyMappingForMissingFileForBestEffortMode() throws {
+        let file: URL = "nonExiting"
+        let reader = JsonOverlayReader(file, mode: .bestEffort, fileReader: FileManager.default)
+
+        let mappings = try reader.provideMappings()
+
+        XCTAssertEqual(mappings, [])
+    }
+
+    func testParsingEmptyOverlay() throws {
+        let file = try pathForTestData(name: "overlayReaderEmpty")
+        let reader = JsonOverlayReader(file, mode: .strict, fileReader: FileManager.default)
+        let mappings = try reader.provideMappings()
+
+        XCTAssertEqual(mappings, [])
+    }
+
+    func testInvalidJsonDoesntThrowForBestEffortMode() throws {
+        let workingDir = try prepareTempDir()
+        let file = workingDir.appendingPathExtension("overlay.json")
+        try fileManager.spt_createEmptyFile(file)
+        let reader = JsonOverlayReader(file, mode: .bestEffort, fileReader: fileManager)
+        let mappings = try reader.provideMappings()
+
+        XCTAssertEqual(mappings, [])
+    }
+
+    func testInvalidJsonThrowsForStrictMode() throws {
+        let workingDir = try prepareTempDir()
+        let file = workingDir.appendingPathExtension("overlay.json")
+        try fileManager.spt_createEmptyFile(file)
+        let reader = JsonOverlayReader(file, mode: .strict, fileReader: fileManager)
+
+        XCTAssertThrowsError(try reader.provideMappings())
+    }
+
+    private func pathForTestData(name: String) throws -> URL {
+        return try XCTUnwrap(Bundle.module.url(forResource: name, withExtension: "json", subdirectory: JsonOverlayReaderTests.resourcesSubdirectory))
+    }
+}

Tests/XCRemoteCacheTests/Dependencies/StringDependenciesRemapperFactoryTests.swift

@@ -0,0 +1,71 @@
+// Copyright (c) 2021 Spotify AB.
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+@testable import XCRemoteCache
+import XCTest
+
+class StringDependenciesRemapperFactoryTests: XCTestCase {
+    private var factory: StringDependenciesRemapperFactory!
+
+    override func setUp() {
+        factory = StringDependenciesRemapperFactory()
+    }
+
+    func testMappingsFromEnvMaps() throws {
+        let remapper = try factory.build(
+            orderKeys: ["SRC_ROOT"],
+            envs: ["SRC_ROOT": "/tmp/root"],
+            customMappings: [:]
+        )
+
+        let localPaths = try remapper.replace(genericPaths: ["$(SRC_ROOT)/some.swift"])
+        XCTAssertEqual(localPaths, ["/tmp/root/some.swift"])
+    }
+
+    func testInvalidMappingsFromEnvFails() throws {
+        XCTAssertThrowsError(
+            try factory.build(
+                orderKeys: ["SRC_ROOT"],
+                envs: ["NO_SRC_ROOT": ""],
+                customMappings: [:]
+            )
+        )
+    }
+
+    func testBuildingRemapperWithMergedCustomMappings() throws {
+        let remapper = try factory.build(
+            orderKeys: ["PWD"],
+            envs: ["PWD": "/some"],
+            customMappings: ["TMP": "/tmp"]
+        )
+
+        let genericPaths = try remapper.replace(localPaths: ["/some/repoFile.swift", "/tmp/externalFile.swift"])
+        XCTAssertEqual(genericPaths, ["$(PWD)/repoFile.swift", "$(TMP)/externalFile.swift"])
+    }
+
+    func testFailsBuildingRemapperWithConflictedMappings() throws {
+        XCTAssertThrowsError(
+            try factory.build(
+            orderKeys: ["PWD"],
+            envs: ["PWD": "/some"],
+            customMappings: ["PWD": "/other"]
+            )
+        )
+    }
+}

Tests/XCRemoteCacheTests/Dependencies/StringDependenciesRemapperTests.swift

@@ -30,34 +30,34 @@ class StringDependenciesRemapperTests: XCTestCase {
         remapper = StringDependenciesRemapper(mappings: mappings)
     }
 
-    func testMappingSingleGenericPathReplacesWithLocalPath() {
-        let localPaths = remapper.replace(genericPaths: ["$(SRC_ROOT)/some.swift"])
+    func testMappingSingleGenericPathReplacesWithLocalPath() throws {
+        let localPaths = try remapper.replace(genericPaths: ["$(SRC_ROOT)/some.swift"])
 
         XCTAssertEqual(localPaths, ["/tmp/root/some.swift"])
     }
 
-    func testRewritingSingleLocalPathReplacesWithGenericPath() {
-        let genericPaths = remapper.replace(localPaths: ["/tmp/root/some.swift"])
+    func testRewritingSingleLocalPathReplacesWithGenericPath() throws {
+        let genericPaths = try remapper.replace(localPaths: ["/tmp/root/some.swift"])
 
         XCTAssertEqual(genericPaths, ["$(SRC_ROOT)/some.swift"])
     }
 
-    func testRewritingLocalToGenericAndLocalIsIdentical() {
+    func testRewritingLocalToGenericAndLocalIsIdentical() throws {
         let inputLocalPaths = ["/tmp/root/some.swift"]
 
-        let genericPaths = remapper.replace(localPaths: inputLocalPaths)
-        let localPaths = remapper.replace(genericPaths: genericPaths)
+        let genericPaths = try remapper.replace(localPaths: inputLocalPaths)
+        let localPaths = try remapper.replace(genericPaths: genericPaths)
 
         XCTAssertEqual(localPaths, inputLocalPaths)
     }
 
-    func testRewritingUnrelatedDirReturnsInputPath() {
-        let genericPaths = remapper.replace(localPaths: ["/other/some.swift"])
+    func testRewritingUnrelatedDirReturnsInputPath() throws {
+        let genericPaths = try remapper.replace(localPaths: ["/other/some.swift"])
 
         XCTAssertEqual(genericPaths, ["/other/some.swift"])
     }
 
-    func testMultipleMatchesTakeTheFirstMapping() {
+    func testMultipleMatchesTakeTheFirstMapping() throws {
         let mappings: [StringDependenciesRemapper.Mapping] = [
             .init(generic: "$(SRC_ROOT)", local: "/tmp/root"),
             .init(generic: "$(PWD)", local: "/tmp"),
@@ -65,22 +65,34 @@ class StringDependenciesRemapperTests: XCTestCase {
         remapper = StringDependenciesRemapper(mappings: mappings)
 
 
-        let genericPaths = remapper.replace(localPaths: ["/tmp/root/some.swift", "/tmp/extra.swift"])
+        let genericPaths = try remapper.replace(localPaths: ["/tmp/root/some.swift", "/tmp/extra.swift"])
 
         XCTAssertEqual(genericPaths, ["$(SRC_ROOT)/some.swift", "$(PWD)/extra.swift"])
     }
 
-    func testMappingsFromEnvMaps() throws {
-        remapper = try StringDependenciesRemapper.buildFromEnvs(keys: ["SRC_ROOT"], envs: ["SRC_ROOT": "/tmp/root"])
+    func testMappingsLocalPathsIsDoneInOrder() throws {
+        let mappings: [StringDependenciesRemapper.Mapping] = [
+            .init(generic: "$(TMP)", local: "/tmp"),
+            .init(generic: "$(ROOT)", local: "$(TMP)/root"),
+        ]
+        remapper = StringDependenciesRemapper(mappings: mappings)
 
-        let localPaths = remapper.replace(genericPaths: ["$(SRC_ROOT)/some.swift"])
+
+        let genericPaths = try remapper.replace(localPaths: ["/tmp/root/some.swift"])
+
+        XCTAssertEqual(genericPaths, ["$(ROOT)/some.swift"])
+    }
+
+    func testMappingsGenericPathsIsDoneInReversedOrder() throws {
+        let mappings: [StringDependenciesRemapper.Mapping] = [
+            .init(generic: "$(TMP)", local: "/tmp"),
+            .init(generic: "$(ROOT)", local: "$(TMP)/root"),
+        ]
+        remapper = StringDependenciesRemapper(mappings: mappings)
+
+
+        let localPaths = try remapper.replace(genericPaths: ["$(ROOT)/some.swift"])
 
         XCTAssertEqual(localPaths, ["/tmp/root/some.swift"])
     }
-
-    func testInvalidMappingsFromEnvFAils() throws {
-        XCTAssertThrowsError(
-            try StringDependenciesRemapper.buildFromEnvs(keys: ["SRC_ROOT"], envs: ["NO_SRC_ROOT": ""])
-        )
-    }
 }

Tests/XCRemoteCacheTests/Dependencies/TargetDependenciesReaderTests.swift

@@ -27,6 +27,7 @@ class TargetDependenciesReaderTests: XCTestCase {
     private var reader: TargetDependenciesReader!
 
     override func setUp() {
+        super.setUp()
         dirAccessor = DirAccessorFake()
         /// A Factory that builds a faked dependency reader that returns a single dependency,
         /// a basename of the input .d file and the ".swift" extension

Tests/XCRemoteCacheTests/FileAccess/Copier/CopyDiskCopierTests.swift

@@ -27,6 +27,7 @@ class CopyDiskCopierTests: FileXCTestCase {
     private var emptySourceFile: URL!
 
     override func setUpWithError() throws {
+        try super.setUpWithError()
         workingDir = try prepareTempDir()
         emptySourceFile = workingDir.appendingPathComponent("source")
         try fileManager.spt_writeToFile(atPath: emptySourceFile.path, contents: Data())

Tests/XCRemoteCacheTests/Fingerprint/EnvironmentFingerprintGeneratorTests.swift

@@ -24,7 +24,7 @@ class EnvironmentFingerprintGeneratorTests: XCTestCase {
 
     private static let defaultENV = [
         "GCC_PREPROCESSOR_DEFINITIONS": "GCC",
-        "CLANG_PROFILE_DATA_DIRECTORY": "CLANG",
+        "CLANG_COVERAGE_MAPPING": "YES",
         "TARGET_NAME": "TARGET",
         "CONFIGURATION": "CONG",
         "PLATFORM_NAME": "PLAT",
@@ -33,6 +33,7 @@ class EnvironmentFingerprintGeneratorTests: XCTestCase {
         "DYLIB_COMPATIBILITY_VERSION": "2",
         "DYLIB_CURRENT_VERSION": "3",
         "PRODUCT_MODULE_NAME": "4",
+        "ARCHS": "AR"
     ]
     /// Corresponds to EnvironmentFingerprintGenerator.version
     private static let currentVersion = "5"
@@ -55,7 +56,7 @@ class EnvironmentFingerprintGeneratorTests: XCTestCase {
     func testConsidersDefaultEnvs() throws {
         let fingerprint = try fingerprintGenerator.generateFingerprint()
 
-        XCTAssertEqual(fingerprint, "GCC,CLANG,TARGET,CONG,PLAT,XC,1,2,3,4,\(Self.currentVersion)")
+        XCTAssertEqual(fingerprint, "GCC,YES,TARGET,CONG,PLAT,XC,1,2,3,4,AR,\(Self.currentVersion)")
     }
 
     func testFingerprintIncludesVersionAsLastComponent() throws {
@@ -73,7 +74,7 @@ class EnvironmentFingerprintGeneratorTests: XCTestCase {
 
         let fingerprint = try fingerprintGenerator.generateFingerprint()
 
-        XCTAssertEqual(fingerprint, ",,,,,,,,,,\(Self.currentVersion)")
+        XCTAssertEqual(fingerprint, ",,,,,,,,,,,\(Self.currentVersion)")
     }
 
     func testConsidersCustomEnvs() throws {
@@ -89,6 +90,6 @@ class EnvironmentFingerprintGeneratorTests: XCTestCase {
 
         let fingerprint = try fingerprintGenerator.generateFingerprint()
 
-        XCTAssertEqual(fingerprint, "GCC,CLANG,TARGET,CONG,PLAT,XC,1,2,3,4,CUSTOM_VALUE,\(Self.currentVersion)")
+        XCTAssertEqual(fingerprint, "GCC,YES,TARGET,CONG,PLAT,XC,1,2,3,4,AR,CUSTOM_VALUE,\(Self.currentVersion)")
     }
 }

Tests/XCRemoteCacheTests/Models/JsonMetaWriterTests.swift

@@ -0,0 +1,61 @@
+// Copyright (c) 2021 Spotify AB.
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+@testable import XCRemoteCache
+import XCTest
+
+class JsonMetaWriterTests: XCTestCase {
+
+    func testWritesToFileWithFilekeyFilename() throws {
+        let fileAccessor = FileAccessorFake(mode: .normal)
+        let writer = JsonMetaWriter(fileWriter: fileAccessor, pretty: false)
+        let workingDir: URL = "/"
+        let meta = MainArtifactSampleMeta.defaults
+
+        let url = try writer.write(MainArtifactSampleMeta.defaults, locationDir: workingDir)
+
+        XCTAssertEqual(url, workingDir.appendingPathComponent(meta.fileKey).appendingPathExtension("json"))
+    }
+
+    func testWritesMetaInValidFormat() throws {
+        let fileAccessor = FileAccessorFake(mode: .normal)
+        let writer = JsonMetaWriter(fileWriter: fileAccessor, pretty: false)
+        let reader = JsonMetaReader(fileAccessor: fileAccessor)
+        let workingDir: URL = "/"
+        let meta = MainArtifactSampleMeta.defaults
+
+        let url = try writer.write(MainArtifactSampleMeta.defaults, locationDir: workingDir)
+
+        let readMeta = try reader.read(localFile: url)
+        XCTAssertEqual(readMeta, meta)
+    }
+
+    func testWritesPrettyMetaInValidFormat() throws {
+        let fileAccessor = FileAccessorFake(mode: .normal)
+        let writer = JsonMetaWriter(fileWriter: fileAccessor, pretty: true)
+        let reader = JsonMetaReader(fileAccessor: fileAccessor)
+        let workingDir: URL = "/"
+        let meta = MainArtifactSampleMeta.defaults
+
+        let url = try writer.write(MainArtifactSampleMeta.defaults, locationDir: workingDir)
+
+        let readMeta = try reader.read(localFile: url)
+        XCTAssertEqual(readMeta, meta)
+    }
+}

Tests/XCRemoteCacheTests/Output/FilteredInvocationStorageTests.swift

@@ -26,6 +26,7 @@ class FilteredInvocationStorageTests: XCTestCase {
     var storage: FilteredInvocationStorage!
 
     override func setUp() {
+        super.setUp()
         storage = FilteredInvocationStorage(storage: underlyingStorage, retrieveIgnoredCommands: ["to_ignore"])
     }
 

Tests/XCRemoteCacheTests/Output/InvocationFileStorageTests.swift

@@ -28,6 +28,7 @@ class InvocationFileStorageTests: FileXCTestCase {
     private var storage: ExistingFileStorage!
 
     override func setUpWithError() throws {
+        try super.setUpWithError()
         file = try prepareTempDir().appendingPathComponent("file.history")
         try fileManager.spt_createEmptyFile(file)
         storage = ExistingFileStorage(storageFile: file, command: command)

Tests/XCRemoteCacheTests/Stats/ActionSpecificCacheHitLoggerTests.swift

@@ -25,6 +25,7 @@ class ActionSpecificCacheHitLoggerTests: FileXCTestCase {
     private var coordinator: StatsCoordinator!
 
     override func setUp() {
+        super.setUp()
         coordinator = InMemoryStatsCoordinator()
     }
 

Tests/XCRemoteCacheTests/TestData/Dependencies/JsonOverlayReaderTests/overlayReaderDefault.json

@@ -0,0 +1,31 @@
+{
+  "case-sensitive": "false",
+  "roots":
+    [
+      {
+        "contents":
+          [
+            {
+              "external-contents": "/Path/Target1/Target1.h",
+              "name": "Target1.h",
+              "type": "file",
+            },
+          ],
+        "name": "/DerivedDataProducts/Target1.framework/Headers",
+        "type": "directory",
+      },
+      {
+        "contents":
+          [
+            {
+              "external-contents": "/DerivedDataIntermediate/Target2.build/module.modulemap",
+              "name": "module.modulemap",
+              "type": "file",
+            },
+          ],
+        "name": "/DerivedDataProducts/Target2.framework/Modules",
+        "type": "directory",
+      },
+    ],
+  "version": 0,
+}

Tests/XCRemoteCacheTests/TestData/Dependencies/JsonOverlayReaderTests/overlayReaderEmpty.json

@@ -0,0 +1 @@
+{"case-sensitive":"false","roots":[],"version":0}

Tests/XCRemoteCacheTests/TestDoubles/OverlayReaderFake.swift

@@ -0,0 +1,33 @@
+// Copyright (c) 2021 Spotify AB.
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import Foundation
+@testable import XCRemoteCache
+
+class OverlayReaderFake: OverlayReader {
+    var mappings: [OverlayMapping]
+    
+    init(mappings: [OverlayMapping]) {
+        self.mappings = mappings
+    }
+
+    func provideMappings() throws -> [OverlayMapping] {
+        return mappings
+    }
+}

cocoapods-plugin/README.md

@@ -4,6 +4,14 @@ The CocoaPods plugin that integrates XCRemoteCache with the project.
 
 ## Installation
 
+### Using RubyGems
+
+```bash
+gem install cocoapods-xcremotecache
+```
+
+### From sources
+
 Build & install the plugin
 
 ```bash
@@ -44,6 +52,8 @@ An object that is passed to the `xcremotecache` can contain all properties suppo
 | `modify_lldb_init` | Controls if the pod integration should modify `~/.lldbinit` | `true` | ⬜️ |
 | `xccc_file` | The path where should be placed the `xccc` binary (in the pod installation phase) | `{podfile_dir}/.rc/xccc` | ⬜️ |
 | `remote_commit_file` | The path of the file with the remote commit sha (in the pod installation phase) | `{podfile_dir}/.rc/arc.rc`| ⬜️ |
+| `prettify_meta_files` | A Boolean value that opts-in pretty JSON formatting for meta files | `false` | ⬜️ |
+| `disable_certificate_verification` | A Boolean value that opts-in SSL certificate validation is disabled | `false` | ⬜️ |
 
 ## Uninstalling
 
@@ -52,3 +62,7 @@ To fully uninstall the plugin, call:
 ```bash
 gem uninstall cocoapods-xcremotecache
 ```
+
+## Limitations
+
+* When `generate_multiple_pod_projects` mode is enabled, only first-party targets are cached by XCRemoteCache (all dependencies are compiled locally).

cocoapods-plugin/lib/cocoapods-xcremotecache/command/hooks.rb

@@ -17,6 +17,7 @@ require 'cocoapods/resolver'
 require 'open-uri'
 require 'yaml'
 require 'json'
+require 'pathname'
 
 
 module CocoapodsXCRemoteCacheModifier
@@ -28,6 +29,7 @@ module CocoapodsXCRemoteCacheModifier
     LLDB_INIT_PATH = "#{ENV['HOME']}/.lldbinit"
     FAT_ARCHIVE_NAME_INFIX = 'arm64-x86_64'
 
+    # List of plugins' user properties that should be copied to .rcinfo 
     CUSTOM_CONFIGURATION_KEYS = [
       'enabled', 
       'xcrc_location',
@@ -36,7 +38,9 @@ module CocoapodsXCRemoteCacheModifier
       'final_target',
       'check_build_configuration', 
       'check_platform', 
-      'modify_lldb_init'
+      'modify_lldb_init',
+      'prettify_meta_files',
+      'disable_certificate_verification'
     ]
 
     class XCRemoteCache
@@ -46,18 +50,20 @@ module CocoapodsXCRemoteCacheModifier
         @@configuration = c
       end
 
-      def self.set_configuration_default_values(user_proj_directory)
+      def self.set_configuration_default_values
         default_values = {
           'mode' => 'consumer',
           'enabled' => true, 
-          'xcrc_location' => "#{user_proj_directory}/XCRC", 
+          'xcrc_location' => "XCRC",
           'exclude_build_configurations' => [],
           'check_build_configuration' => 'Debug',
           'check_platform' => 'iphonesimulator', 
           'modify_lldb_init' => true, 
-          'xccc_file' => "#{user_proj_directory}/#{BIN_DIR}/xccc",
-          'remote_commit_file' => "#{user_proj_directory}/#{BIN_DIR}/arc.rc",
+          'xccc_file' => "#{BIN_DIR}/xccc",
+          'remote_commit_file' => "#{BIN_DIR}/arc.rc",
           'exclude_targets' => [],
+          'prettify_meta_files' => false,
+          'disable_certificate_verification' => false
         }
         @@configuration.merge! default_values.select { |k, v| !@@configuration.key?(k) }
       end
@@ -76,8 +82,8 @@ module CocoapodsXCRemoteCacheModifier
         end
 
         mode = @@configuration['mode']
-        unless mode == 'consumer' || mode == 'producer'
-          throw "Incorrect 'mode' value. Allowed values are ['consumer', 'producer'], but you provided '#{mode}'. A typo?"
+        unless mode == 'consumer' || mode == 'producer' || mode == 'producer-fast'
+          throw "Incorrect 'mode' value. Allowed values are ['consumer', 'producer', 'producer-fast'], but you provided '#{mode}'. A typo?"
         end
 
         unless mode == 'consumer' || @@configuration.key?('final_target')
@@ -89,61 +95,100 @@ module CocoapodsXCRemoteCacheModifier
         @@configuration.select { |key, value| !CUSTOM_CONFIGURATION_KEYS.include?(key) }
       end
 
-      def self.enable_xcremotecache(target, user_proj_directory, xc_location, xc_cc_path, mode, exclude_build_configurations, check_build_configuration, check_platform, final_target)
-          target.build_configurations.each do |config| 
-            # apply only for relevant Configurations
-            next if exclude_build_configurations.include?(config.name)
-            if mode == 'consumer'
-              config.build_settings['CC'] = [xc_cc_path]
-            end
-            config.build_settings['SWIFT_EXEC'] = ["#{xc_location}/xcswiftc"]
-            config.build_settings['LIBTOOL'] = ["#{xc_location}/xclibtool"]
-            config.build_settings['LD'] = ["#{xc_location}/xcld"]
+      def self.parent_dir(path, parent_count)
+        "../" * parent_count + path
+      end
 
-            config.build_settings['XCREMOTE_CACHE_FAKE_SRCROOT'] = FAKE_SRCROOT
-            add_cflags!(config.build_settings, '-fdebug-prefix-map', "#{user_proj_directory}=$(XCREMOTE_CACHE_FAKE_SRCROOT)")
-            add_swiftflags!(config.build_settings, '-debug-prefix-map', "#{user_proj_directory}=$(XCREMOTE_CACHE_FAKE_SRCROOT)")
-          end
-        
+      # @param target [Target] target to apply XCRemoteCache
+      # @param repo_distance [Integer] distance from the git repo root to the target's $SRCROOT 
+      # @param xc_location [String] path to the dir with all XCRemoteCache binaries, relative to the repo root
+      # @param xc_cc_path [String] path to the XCRemoteCache clang wrapper, relative to the repo root
+      # @param mode [String] mode name ('consumer', 'producer', 'producer-fast' etc.)
+      # @param exclude_build_configurations [String[]] list of targets that should have disabled remote cache
+      # @param final_target [String] name of target that should trigger marking
+      def self.enable_xcremotecache(target, repo_distance, xc_location, xc_cc_path, mode, exclude_build_configurations, final_target)
+        srcroot_relative_xc_location = parent_dir(xc_location, repo_distance)
 
-        # User project is not generated from scratch (contrary to `Pods`), delete all previous XCRemoteCache phases
-        target.build_phases.delete_if {|phase| 
-          # Some phases (e.g. PBXSourcesBuildPhase) don't have strict name check respond_to?
-          if phase.respond_to?(:name) 
-              phase.name != nil && phase.name.start_with?("[XCRC]")
+        target.build_configurations.each do |config|
+          # apply only for relevant Configurations
+          next if exclude_build_configurations.include?(config.name)
+          if mode == 'consumer'
+            config.build_settings['CC'] = ["$SRCROOT/#{parent_dir(xc_cc_path, repo_distance)}"]
+          elsif mode == 'producer' || mode == 'producer-fast'
+            config.build_settings.delete('CC') if config.build_settings.key?('CC')
           end
-        }
+          config.build_settings['SWIFT_EXEC'] = ["$SRCROOT/#{srcroot_relative_xc_location}/xcswiftc"]
+          config.build_settings['LIBTOOL'] = ["$SRCROOT/#{srcroot_relative_xc_location}/xclibtool"]
+          config.build_settings['LD'] = ["$SRCROOT/#{srcroot_relative_xc_location}/xcld"]
+
+          config.build_settings['XCREMOTE_CACHE_FAKE_SRCROOT'] = FAKE_SRCROOT
+          config.build_settings['XCRC_PLATFORM_PREFERRED_ARCH'] = ["$(LINK_FILE_LIST_$(CURRENT_VARIANT)_$(PLATFORM_PREFERRED_ARCH):dir:standardizepath:file:default=arm64)"]
+          debug_prefix_map_replacement = '$(SRCROOT' + ':dir:standardizepath' * repo_distance + ')'
+          add_cflags!(config.build_settings, '-fdebug-prefix-map', "#{debug_prefix_map_replacement}=$(XCREMOTE_CACHE_FAKE_SRCROOT)")
+          add_swiftflags!(config.build_settings, '-debug-prefix-map', "#{debug_prefix_map_replacement}=$(XCREMOTE_CACHE_FAKE_SRCROOT)")
+        end
 
         # Prebuild
         if mode == 'consumer'
-          prebuild_script = target.new_shell_script_build_phase("[XCRC] Prebuild")
+          existing_prebuild_script = target.build_phases.detect do |phase|
+            if phase.respond_to?(:name)
+              phase.name != nil && phase.name.start_with?("[XCRC] Prebuild")
+            end
+          end
+
+          prebuild_script = existing_prebuild_script || target.new_shell_script_build_phase("[XCRC] Prebuild #{target.name}")
           prebuild_script.shell_script = "\"$SCRIPT_INPUT_FILE_0\""
-          prebuild_script.input_paths = ["#{xc_location}/xcprebuild"]
+          prebuild_script.input_paths = ["$SRCROOT/#{srcroot_relative_xc_location}/xcprebuild"]
           prebuild_script.output_paths = [
-            "$(TARGET_TEMP_DIR)/rc.enabled", 
+            "$(TARGET_TEMP_DIR)/rc.enabled",
             "$(DWARF_DSYM_FOLDER_PATH)/$(DWARF_DSYM_FILE_NAME)"
           ]
           prebuild_script.dependency_file = "$(TARGET_TEMP_DIR)/prebuild.d"
 
-          # Move prebuild (last element) to the first position (to make it real 'prebuild')
-          target.build_phases.rotate!(-1)
+          # Move prebuild (last element) to the position before compile sources phase (to make it real 'prebuild')
+          compile_phase_index = target.build_phases.index(target.source_build_phase)
+          target.build_phases.insert(compile_phase_index, target.build_phases.delete(prebuild_script))
+        elsif mode == 'producer' || mode == 'producer-fast'
+          # Delete existing prebuild build phase (to support switching between modes)
+          target.build_phases.delete_if do |phase|
+            if phase.respond_to?(:name)
+              phase.name != nil && phase.name.start_with?("[XCRC] Prebuild")
+            end
+          end
         end
 
         # Postbuild
-        postbuild_script = target.new_shell_script_build_phase("[XCRC] Postbuild")
+        existing_postbuild_script = target.build_phases.detect do |phase|
+          if phase.respond_to?(:name)
+            phase.name != nil && phase.name.start_with?("[XCRC] Postbuild")
+          end
+        end
+        postbuild_script = existing_postbuild_script || target.new_shell_script_build_phase("[XCRC] Postbuild #{target.name}")
         postbuild_script.shell_script = "\"$SCRIPT_INPUT_FILE_0\""
-        postbuild_script.input_paths = ["#{xc_location}/xcpostbuild"]
+        postbuild_script.input_paths = ["$SRCROOT/#{srcroot_relative_xc_location}/xcpostbuild"]
         postbuild_script.output_paths = [
-          "$(TARGET_BUILD_DIR)/$(MODULES_FOLDER_PATH)/$(PRODUCT_MODULE_NAME).swiftmodule/$(PLATFORM_PREFERRED_ARCH).swiftmodule.md5", 
-          "$(TARGET_BUILD_DIR)/$(MODULES_FOLDER_PATH)/$(PRODUCT_MODULE_NAME).swiftmodule/$(PLATFORM_PREFERRED_ARCH)-$(LLVM_TARGET_TRIPLE_VENDOR)-$(SWIFT_PLATFORM_TARGET_PREFIX)$(LLVM_TARGET_TRIPLE_SUFFIX).swiftmodule.md5"
+          "$(TARGET_BUILD_DIR)/$(MODULES_FOLDER_PATH)/$(PRODUCT_MODULE_NAME).swiftmodule/$(XCRC_PLATFORM_PREFERRED_ARCH).swiftmodule.md5",
+          "$(TARGET_BUILD_DIR)/$(MODULES_FOLDER_PATH)/$(PRODUCT_MODULE_NAME).swiftmodule/$(XCRC_PLATFORM_PREFERRED_ARCH)-$(LLVM_TARGET_TRIPLE_VENDOR)-$(SWIFT_PLATFORM_TARGET_PREFIX)$(LLVM_TARGET_TRIPLE_SUFFIX).swiftmodule.md5"
         ]
         postbuild_script.dependency_file = "$(TARGET_TEMP_DIR)/postbuild.d"
 
         # Mark a sha as ready for a given platform and configuration when building the final_target
-        if mode == 'producer' && target.name == final_target
-          mark_script = target.new_shell_script_build_phase("[XCRC] Mark")
+        if (mode == 'producer' || mode == 'producer-fast') && target.name == final_target
+          existing_mark_script = target.build_phases.detect do |phase|
+            if phase.respond_to?(:name)
+              phase.name != nil && phase.name.start_with?("[XCRC] Mark")
+            end
+          end
+          mark_script = existing_mark_script || target.new_shell_script_build_phase("[XCRC] Mark")
           mark_script.shell_script = "\"$SCRIPT_INPUT_FILE_0\" mark --configuration $CONFIGURATION --platform $PLATFORM_NAME"
-          mark_script.input_paths = ["#{xc_location}/xcprepare"]
+          mark_script.input_paths = ["$SRCROOT/#{srcroot_relative_xc_location}/xcprepare"]
+        else
+          # Delete existing mark build phase (to support switching between modes or changing the final target)
+          target.build_phases.delete_if do |phase|
+            if phase.respond_to?(:name)
+              phase.name != nil && phase.name.start_with?("[XCRC] Mark")
+            end
+          end
         end
       end
 
@@ -153,8 +198,9 @@ module CocoapodsXCRemoteCacheModifier
           config.build_settings.delete('SWIFT_EXEC') if config.build_settings.key?('SWIFT_EXEC')
           config.build_settings.delete('LIBTOOL') if config.build_settings.key?('LIBTOOL')
           config.build_settings.delete('LD') if config.build_settings.key?('LD')
-          # Add Fake src root for ObjC & Swift
+          # Remove Fake src root for ObjC & Swift
           config.build_settings.delete('XCREMOTE_CACHE_FAKE_SRCROOT')
+          config.build_settings.delete('XCRC_PLATFORM_PREFERRED_ARCH')
           remove_cflags!(config.build_settings, '-fdebug-prefix-map')
           remove_swiftflags!(config.build_settings, '-debug-prefix-map')
         end
@@ -216,14 +262,14 @@ module CocoapodsXCRemoteCacheModifier
       end
 
       def self.add_cflags!(options, key, value)
-        return if options.fetch('OTHER_CFLAGS',[]).include?(' ' + value)
-        options['OTHER_CFLAGS'] = remove_cflags!(options, key) << " #{key}=#{value}" 
+        return if options.fetch('OTHER_CFLAGS',[]).include?(value)
+        options['OTHER_CFLAGS'] = remove_cflags!(options, key) << "#{key}=#{value}" 
       end
 
       def self.remove_cflags!(options, key)
         cflags_arr = options.fetch('OTHER_CFLAGS', ['$(inherited)'])
         cflags_arr = [cflags_arr] if cflags_arr.kind_of? String
-        options['OTHER_CFLAGS'] = cflags_arr.delete_if {|flag| flag.start_with?(" #{key}=") }
+        options['OTHER_CFLAGS'] = cflags_arr.delete_if {|flag| flag.include?("#{key}=") }
         options['OTHER_CFLAGS']
       end
 
@@ -238,12 +284,27 @@ module CocoapodsXCRemoteCacheModifier
       end
 
       # Uninstall the XCRemoteCache
-      def self.disable_xcremotecache(user_project)
+      def self.disable_xcremotecache(user_project, pods_project = nil)
         user_project.targets.each do |target|
           disable_xcremotecache_for_target(target)
         end
         user_project.save()
 
+        unless pods_project.nil?
+          pods_project.native_targets.each do |target|
+            disable_xcremotecache_for_target(target)
+          end
+          pods_proj_directory = pods_project.project_dir
+          pods_project.root_object.project_references.each do |subproj_ref|
+            generated_project = Xcodeproj::Project.open("#{pods_proj_directory}/#{subproj_ref[:project_ref].path}")
+            generated_project.native_targets.each do |target|
+              disable_xcremotecache_for_target(target)
+            end
+            generated_project.save()
+          end
+          pods_project.save()
+        end
+
         # Remove .lldbinit rewrite
         save_lldbinit_rewrite(nil) unless !@@configuration['modify_lldb_init']
       end
@@ -251,6 +312,7 @@ module CocoapodsXCRemoteCacheModifier
       # Returns the content (array of lines) of the lldbinit with stripped XCRemoteCache rewrite
       def self.clean_lldbinit_content(lldbinit_path)
         all_lines = []
+        return all_lines unless File.exist?(lldbinit_path)
         File.open(lldbinit_path) { |file|
           while(line = file.gets) != nil
             line = line.strip
@@ -290,7 +352,7 @@ module CocoapodsXCRemoteCacheModifier
 
         begin    
           user_proj_directory = File.dirname(user_project.path)
-          set_configuration_default_values(user_proj_directory)
+          set_configuration_default_values
 
           unless @@configuration['enabled']
             Pod::UI.puts "[XCRC] XCRemoteCache disabled"
@@ -299,7 +361,6 @@ module CocoapodsXCRemoteCacheModifier
           end
 
           validate_configuration()
-
           mode = @@configuration['mode']
           xccc_location = @@configuration['xccc_file']
           remote_commit_file = @@configuration['remote_commit_file']
@@ -310,54 +371,90 @@ module CocoapodsXCRemoteCacheModifier
           check_build_configuration = @@configuration['check_build_configuration']
           check_platform = @@configuration['check_platform']
 
+          xccc_location_absolute = "#{user_proj_directory}/#{xccc_location}"
+          xcrc_location_absolute = "#{user_proj_directory}/#{xcrc_location}"
+          remote_commit_file_absolute = "#{user_proj_directory}/#{remote_commit_file}"
+
           # Download XCRC
-          download_xcrc_if_needed(xcrc_location)
+          download_xcrc_if_needed(xcrc_location_absolute)
 
           # Save .rcinfo
-          save_rcinfo(generate_rcinfo(), user_proj_directory)
+          root_rcinfo = generate_rcinfo()
+          save_rcinfo(root_rcinfo, user_proj_directory)
 
           # Create directory for xccc & arc.rc location
           Dir.mkdir(BIN_DIR) unless File.exist?(BIN_DIR)
 
           # Remove previous xccc & arc.rc
-          File.delete(remote_commit_file) if File.exist?(remote_commit_file)
-          File.delete(xccc_location) if File.exist?(xccc_location)
+          File.delete(remote_commit_file_absolute) if File.exist?(remote_commit_file_absolute)
+          File.delete(xccc_location_absolute) if File.exist?(xccc_location_absolute)
 
           # Prepare XCRC
+
+          # Pods projects can be generated only once (if incremental_installation is enabled)
+          # Always integrate XCRemoteCache to all Pods, in case it will be needed later
+          unless installer_context.pods_project.nil?
+            # Attach XCRemoteCache to Pods targets
+            # Enable only for native targets which can have compilation steps
+            installer_context.pods_project.native_targets.each do |target|
+                next if target.source_build_phase.files_references.empty?
+                next if target.name.start_with?("Pods-")
+                next if target.name.end_with?("Tests")
+                next if exclude_targets.include?(target.name)
+                enable_xcremotecache(target, 1, xcrc_location, xccc_location, mode, exclude_build_configurations, final_target)
+            end
+
+            # Create .rcinfo into `Pods` directory as that .xcodeproj reads configuration from .xcodeproj location
+            pods_proj_directory = installer_context.sandbox_root
+
+            # Attach XCRemoteCache to Generated Pods projects
+            installer_context.pods_project.root_object.project_references.each do |subproj_ref|
+                generated_project = Xcodeproj::Project.open("#{pods_proj_directory}/#{subproj_ref[:project_ref].path}")
+                generated_project.native_targets.each do |target|
+                    next if target.source_build_phase.files_references.empty?
+                    next if target.name.end_with?("Tests")
+                    next if exclude_targets.include?(target.name)
+                    enable_xcremotecache(target, 1, xcrc_location, xccc_location, mode, exclude_build_configurations, final_target)
+                end
+                generated_project.save()
+            end
+
+            # Manual Pods/.rcinfo generation
+            
+            # all paths in .rcinfo are relative to the root so paths used in Pods.xcodeproj need to be aligned
+            pods_path = Pathname.new(pods_proj_directory)
+            root_path = Pathname.new(user_proj_directory)
+            root_path_to_pods = root_path.relative_path_from(pods_path)
+            
+            pods_rcinfo = root_rcinfo.merge({
+              'remote_commit_file' => "#{root_path_to_pods}/#{remote_commit_file}",
+              'xccc_file' => "#{root_path_to_pods}/#{xccc_location}"
+            })
+            save_rcinfo(pods_rcinfo, pods_proj_directory)
+
+            installer_context.pods_project.save()
+          end
+
+          # Enabled/disable XCRemoteCache for the main (user) project
           begin
-            prepare_result = YAML.load`#{xcrc_location}/xcprepare --configuration #{check_build_configuration} --platform #{check_platform}`
+            prepare_result = YAML.load`#{xcrc_location_absolute}/xcprepare --configuration #{check_build_configuration} --platform #{check_platform}`
             unless prepare_result['result'] || mode != 'consumer'
               # Uninstall the XCRemoteCache for the consumer mode
-              disable_xcremotecache(user_project)
+              disable_xcremotecache(user_project, installer_context.pods_project)
               Pod::UI.puts "[XCRC] XCRemoteCache disabled - no artifacts available"
               next
             end
           rescue => error
-            disable_xcremotecache(user_project)
+            disable_xcremotecache(user_project, installer_context.pods_project)
             Pod::UI.puts "[XCRC] XCRemoteCache failed with an error: #{error}."
             next
           end
 
-          # Attach XCRemoteCache to Pods targets
-          installer_context.pods_project.targets.each do |target|
-              next if target.name.start_with?("Pods-")
-              next if target.name.end_with?("Tests")
-              next if exclude_targets.include?(target.name)
-              enable_xcremotecache(target, user_proj_directory, xcrc_location, xccc_location, mode, exclude_build_configurations, check_build_configuration, check_platform, final_target)
-          end
-
-          # Create .rcinfo into `Pods` directory as that .xcodeproj reads configuration from .xcodeproj location
-          pods_proj_directory = installer_context.sandbox_root
-          
-          # Manual .rcinfo generation (in YAML format)
-          save_rcinfo({'extra_configuration_file' => "#{user_proj_directory}/.rcinfo"}, pods_proj_directory)
-
-          installer_context.pods_project.save()
 
           # Attach XCRC to the app targets
           user_project.targets.each do |target|
               next if exclude_targets.include?(target.name)
-              enable_xcremotecache(target, user_proj_directory, xcrc_location, xccc_location, mode, exclude_build_configurations, check_build_configuration, check_platform, final_target)
+              enable_xcremotecache(target, 0, xcrc_location, xccc_location, mode, exclude_build_configurations, final_target)
           end
 
           # Set Target sourcemap
@@ -373,7 +470,7 @@ module CocoapodsXCRemoteCacheModifier
         rescue Exception => e
           Pod::UI.puts "[XCRC] XCRemoteCache disabled with error: #{e}"
           puts e.full_message(highlight: true, order: :top)
-          disable_xcremotecache(user_project)
+          disable_xcremotecache(user_project, installer_context.pods_project)
         end
       end
     end

cocoapods-plugin/lib/cocoapods-xcremotecache/gem_version.rb

@@ -13,5 +13,5 @@
 # limitations under the License.
 
 module CocoapodsXcremotecache
-  VERSION = "0.0.1"
+  VERSION = "0.0.6"
 end

docs/Development.md

@@ -25,7 +25,18 @@ The generated Xcode project contains schemes for each output application (like `
 
 #### Running tests in Xcode
 
-All unit tests are placed in `XCRemoteCacheTests`. To run them from Xcode, just pick any application scheme and run tests (⌘+U).   
+All unit tests are placed in `XCRemoteCacheTests`. To run them from Xcode, just pick any application scheme and run tests (⌘+U).
+
+#### Running E2E tests
+
+E2E tests build a CocoaPods plugin, locally build both `producer` and `consumer` modes and verify 100% hit rate. All `Podfile` templates are placed in [e2eTests/tests](../e2eTests/tests). As a backend server, nginx server with a sample [nginx.conf](../e2eTests/nginx/nginx.conf) configuration is used. The sample server exposes local location `/tmp/cache` under http://localhost:8080.
+
+To run tests locally, install `nginx` (e.g. `brew install nginx`) and call: 
+
+```bash
+rake 'build[release]'
+rake e2e_only
+```
 
 ## Project organization
 
@@ -71,3 +82,5 @@ To enable thinning target on the consumer side:
 * Prefer using fakes instead of spies or mocks. Place testing doubles in [Tests/XCRemoteCacheTests/TestDoubles](../Tests/XCRemoteCacheTests/TestDoubles) so other tests can reuse them
 
 * If you test a scenario that accesses a file on a disk, consider using the `DiskUsageSizeProviderTests` class that isolates a working directory and eliminates potential file leaks between testcases
+
+* For dependency injection arguments, avoid passing default values (e.g. `init(dep: SomeDependency = SomeDependency())` and require passing explicit values (e.g. `init(dep: SomeDependency))`. It will be clear from a call site which dependencies is used and suggests adding a testcase when a new dependency is added.

e2eTests/XCRemoteCacheSample/XCRemoteCacheSample.xcodeproj/project.pbxproj

@@ -0,0 +1,363 @@
+// !$*UTF8*$!
+{
+	archiveVersion = 1;
+	classes = {
+	};
+	objectVersion = 55;
+	objects = {
+
+/* Begin PBXBuildFile section */
+		3695D9CC27A3218C007F3792 /* AppDelegate.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3695D9CB27A3218C007F3792 /* AppDelegate.swift */; };
+		3695D9CE27A3218C007F3792 /* SceneDelegate.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3695D9CD27A3218C007F3792 /* SceneDelegate.swift */; };
+		3695D9D027A3218C007F3792 /* ViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3695D9CF27A3218C007F3792 /* ViewController.swift */; };
+		3695D9D327A3218C007F3792 /* Main.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 3695D9D127A3218C007F3792 /* Main.storyboard */; };
+		3695D9D527A3218D007F3792 /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = 3695D9D427A3218D007F3792 /* Assets.xcassets */; };
+		3695D9D827A3218D007F3792 /* LaunchScreen.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 3695D9D627A3218D007F3792 /* LaunchScreen.storyboard */; };
+/* End PBXBuildFile section */
+
+/* Begin PBXFileReference section */
+		3695D9C827A3218C007F3792 /* XCRemoteCacheSample.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = XCRemoteCacheSample.app; sourceTree = BUILT_PRODUCTS_DIR; };
+		3695D9CB27A3218C007F3792 /* AppDelegate.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AppDelegate.swift; sourceTree = "<group>"; };
+		3695D9CD27A3218C007F3792 /* SceneDelegate.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SceneDelegate.swift; sourceTree = "<group>"; };
+		3695D9CF27A3218C007F3792 /* ViewController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ViewController.swift; sourceTree = "<group>"; };
+		3695D9D227A3218C007F3792 /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; name = Base; path = Base.lproj/Main.storyboard; sourceTree = "<group>"; };
+		3695D9D427A3218D007F3792 /* Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; path = Assets.xcassets; sourceTree = "<group>"; };
+		3695D9D727A3218D007F3792 /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; name = Base; path = Base.lproj/LaunchScreen.storyboard; sourceTree = "<group>"; };
+		3695D9D927A3218D007F3792 /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = "<group>"; };
+/* End PBXFileReference section */
+
+/* Begin PBXFrameworksBuildPhase section */
+		3695D9C527A3218C007F3792 /* Frameworks */ = {
+			isa = PBXFrameworksBuildPhase;
+			buildActionMask = 2147483647;
+			files = (
+			);
+			runOnlyForDeploymentPostprocessing = 0;
+		};
+/* End PBXFrameworksBuildPhase section */
+
+/* Begin PBXGroup section */
+		3695D9BF27A3218C007F3792 = {
+			isa = PBXGroup;
+			children = (
+				3695D9CA27A3218C007F3792 /* XCRemoteCacheSample */,
+				3695D9C927A3218C007F3792 /* Products */,
+			);
+			sourceTree = "<group>";
+		};
+		3695D9C927A3218C007F3792 /* Products */ = {
+			isa = PBXGroup;
+			children = (
+				3695D9C827A3218C007F3792 /* XCRemoteCacheSample.app */,
+			);
+			name = Products;
+			sourceTree = "<group>";
+		};
+		3695D9CA27A3218C007F3792 /* XCRemoteCacheSample */ = {
+			isa = PBXGroup;
+			children = (
+				3695D9CB27A3218C007F3792 /* AppDelegate.swift */,
+				3695D9CD27A3218C007F3792 /* SceneDelegate.swift */,
+				3695D9CF27A3218C007F3792 /* ViewController.swift */,
+				3695D9D127A3218C007F3792 /* Main.storyboard */,
+				3695D9D427A3218D007F3792 /* Assets.xcassets */,
+				3695D9D627A3218D007F3792 /* LaunchScreen.storyboard */,
+				3695D9D927A3218D007F3792 /* Info.plist */,
+			);
+			path = XCRemoteCacheSample;
+			sourceTree = "<group>";
+		};
+/* End PBXGroup section */
+
+/* Begin PBXNativeTarget section */
+		3695D9C727A3218C007F3792 /* XCRemoteCacheSample */ = {
+			isa = PBXNativeTarget;
+			buildConfigurationList = 3695D9DC27A3218D007F3792 /* Build configuration list for PBXNativeTarget "XCRemoteCacheSample" */;
+			buildPhases = (
+				3695D9C427A3218C007F3792 /* Sources */,
+				3695D9C527A3218C007F3792 /* Frameworks */,
+				3695D9C627A3218C007F3792 /* Resources */,
+			);
+			buildRules = (
+			);
+			dependencies = (
+			);
+			name = XCRemoteCacheSample;
+			productName = XCRemoteCacheSample;
+			productReference = 3695D9C827A3218C007F3792 /* XCRemoteCacheSample.app */;
+			productType = "com.apple.product-type.application";
+		};
+/* End PBXNativeTarget section */
+
+/* Begin PBXProject section */
+		3695D9C027A3218C007F3792 /* Project object */ = {
+			isa = PBXProject;
+			attributes = {
+				BuildIndependentTargetsInParallel = 1;
+				LastSwiftUpdateCheck = 1320;
+				LastUpgradeCheck = 1320;
+				TargetAttributes = {
+					3695D9C727A3218C007F3792 = {
+						CreatedOnToolsVersion = 13.2.1;
+					};
+				};
+			};
+			buildConfigurationList = 3695D9C327A3218C007F3792 /* Build configuration list for PBXProject "XCRemoteCacheSample" */;
+			compatibilityVersion = "Xcode 13.0";
+			developmentRegion = en;
+			hasScannedForEncodings = 0;
+			knownRegions = (
+				en,
+				Base,
+			);
+			mainGroup = 3695D9BF27A3218C007F3792;
+			productRefGroup = 3695D9C927A3218C007F3792 /* Products */;
+			projectDirPath = "";
+			projectRoot = "";
+			targets = (
+				3695D9C727A3218C007F3792 /* XCRemoteCacheSample */,
+			);
+		};
+/* End PBXProject section */
+
+/* Begin PBXResourcesBuildPhase section */
+		3695D9C627A3218C007F3792 /* Resources */ = {
+			isa = PBXResourcesBuildPhase;
+			buildActionMask = 2147483647;
+			files = (
+				3695D9D827A3218D007F3792 /* LaunchScreen.storyboard in Resources */,
+				3695D9D527A3218D007F3792 /* Assets.xcassets in Resources */,
+				3695D9D327A3218C007F3792 /* Main.storyboard in Resources */,
+			);
+			runOnlyForDeploymentPostprocessing = 0;
+		};
+/* End PBXResourcesBuildPhase section */
+
+/* Begin PBXSourcesBuildPhase section */
+		3695D9C427A3218C007F3792 /* Sources */ = {
+			isa = PBXSourcesBuildPhase;
+			buildActionMask = 2147483647;
+			files = (
+				3695D9D027A3218C007F3792 /* ViewController.swift in Sources */,
+				3695D9CC27A3218C007F3792 /* AppDelegate.swift in Sources */,
+				3695D9CE27A3218C007F3792 /* SceneDelegate.swift in Sources */,
+			);
+			runOnlyForDeploymentPostprocessing = 0;
+		};
+/* End PBXSourcesBuildPhase section */
+
+/* Begin PBXVariantGroup section */
+		3695D9D127A3218C007F3792 /* Main.storyboard */ = {
+			isa = PBXVariantGroup;
+			children = (
+				3695D9D227A3218C007F3792 /* Base */,
+			);
+			name = Main.storyboard;
+			sourceTree = "<group>";
+		};
+		3695D9D627A3218D007F3792 /* LaunchScreen.storyboard */ = {
+			isa = PBXVariantGroup;
+			children = (
+				3695D9D727A3218D007F3792 /* Base */,
+			);
+			name = LaunchScreen.storyboard;
+			sourceTree = "<group>";
+		};
+/* End PBXVariantGroup section */
+
+/* Begin XCBuildConfiguration section */
+		3695D9DA27A3218D007F3792 /* Debug */ = {
+			isa = XCBuildConfiguration;
+			buildSettings = {
+				ALWAYS_SEARCH_USER_PATHS = NO;
+				CLANG_ANALYZER_NONNULL = YES;
+				CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE;
+				CLANG_CXX_LANGUAGE_STANDARD = "gnu++17";
+				CLANG_CXX_LIBRARY = "libc++";
+				CLANG_ENABLE_MODULES = YES;
+				CLANG_ENABLE_OBJC_ARC = YES;
+				CLANG_ENABLE_OBJC_WEAK = YES;
+				CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES;
+				CLANG_WARN_BOOL_CONVERSION = YES;
+				CLANG_WARN_COMMA = YES;
+				CLANG_WARN_CONSTANT_CONVERSION = YES;
+				CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES;
+				CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR;
+				CLANG_WARN_DOCUMENTATION_COMMENTS = YES;
+				CLANG_WARN_EMPTY_BODY = YES;
+				CLANG_WARN_ENUM_CONVERSION = YES;
+				CLANG_WARN_INFINITE_RECURSION = YES;
+				CLANG_WARN_INT_CONVERSION = YES;
+				CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES;
+				CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES;
+				CLANG_WARN_OBJC_LITERAL_CONVERSION = YES;
+				CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR;
+				CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES;
+				CLANG_WARN_RANGE_LOOP_ANALYSIS = YES;
+				CLANG_WARN_STRICT_PROTOTYPES = YES;
+				CLANG_WARN_SUSPICIOUS_MOVE = YES;
+				CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE;
+				CLANG_WARN_UNREACHABLE_CODE = YES;
+				CLANG_WARN__DUPLICATE_METHOD_MATCH = YES;
+				COPY_PHASE_STRIP = NO;
+				DEBUG_INFORMATION_FORMAT = dwarf;
+				ENABLE_STRICT_OBJC_MSGSEND = YES;
+				ENABLE_TESTABILITY = YES;
+				GCC_C_LANGUAGE_STANDARD = gnu11;
+				GCC_DYNAMIC_NO_PIC = NO;
+				GCC_NO_COMMON_BLOCKS = YES;
+				GCC_OPTIMIZATION_LEVEL = 0;
+				GCC_PREPROCESSOR_DEFINITIONS = (
+					"DEBUG=1",
+					"$(inherited)",
+				);
+				GCC_WARN_64_TO_32_BIT_CONVERSION = YES;
+				GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR;
+				GCC_WARN_UNDECLARED_SELECTOR = YES;
+				GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
+				GCC_WARN_UNUSED_FUNCTION = YES;
+				GCC_WARN_UNUSED_VARIABLE = YES;
+				IPHONEOS_DEPLOYMENT_TARGET = 15.2;
+				MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE;
+				MTL_FAST_MATH = YES;
+				ONLY_ACTIVE_ARCH = YES;
+				SDKROOT = iphoneos;
+				SWIFT_ACTIVE_COMPILATION_CONDITIONS = DEBUG;
+				SWIFT_OPTIMIZATION_LEVEL = "-Onone";
+			};
+			name = Debug;
+		};
+		3695D9DB27A3218D007F3792 /* Release */ = {
+			isa = XCBuildConfiguration;
+			buildSettings = {
+				ALWAYS_SEARCH_USER_PATHS = NO;
+				CLANG_ANALYZER_NONNULL = YES;
+				CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE;
+				CLANG_CXX_LANGUAGE_STANDARD = "gnu++17";
+				CLANG_CXX_LIBRARY = "libc++";
+				CLANG_ENABLE_MODULES = YES;
+				CLANG_ENABLE_OBJC_ARC = YES;
+				CLANG_ENABLE_OBJC_WEAK = YES;
+				CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES;
+				CLANG_WARN_BOOL_CONVERSION = YES;
+				CLANG_WARN_COMMA = YES;
+				CLANG_WARN_CONSTANT_CONVERSION = YES;
+				CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES;
+				CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR;
+				CLANG_WARN_DOCUMENTATION_COMMENTS = YES;
+				CLANG_WARN_EMPTY_BODY = YES;
+				CLANG_WARN_ENUM_CONVERSION = YES;
+				CLANG_WARN_INFINITE_RECURSION = YES;
+				CLANG_WARN_INT_CONVERSION = YES;
+				CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES;
+				CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES;
+				CLANG_WARN_OBJC_LITERAL_CONVERSION = YES;
+				CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR;
+				CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES;
+				CLANG_WARN_RANGE_LOOP_ANALYSIS = YES;
+				CLANG_WARN_STRICT_PROTOTYPES = YES;
+				CLANG_WARN_SUSPICIOUS_MOVE = YES;
+				CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE;
+				CLANG_WARN_UNREACHABLE_CODE = YES;
+				CLANG_WARN__DUPLICATE_METHOD_MATCH = YES;
+				COPY_PHASE_STRIP = NO;
+				DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym";
+				ENABLE_NS_ASSERTIONS = NO;
+				ENABLE_STRICT_OBJC_MSGSEND = YES;
+				GCC_C_LANGUAGE_STANDARD = gnu11;
+				GCC_NO_COMMON_BLOCKS = YES;
+				GCC_WARN_64_TO_32_BIT_CONVERSION = YES;
+				GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR;
+				GCC_WARN_UNDECLARED_SELECTOR = YES;
+				GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
+				GCC_WARN_UNUSED_FUNCTION = YES;
+				GCC_WARN_UNUSED_VARIABLE = YES;
+				IPHONEOS_DEPLOYMENT_TARGET = 15.2;
+				MTL_ENABLE_DEBUG_INFO = NO;
+				MTL_FAST_MATH = YES;
+				SDKROOT = iphoneos;
+				SWIFT_COMPILATION_MODE = wholemodule;
+				SWIFT_OPTIMIZATION_LEVEL = "-O";
+				VALIDATE_PRODUCT = YES;
+			};
+			name = Release;
+		};
+		3695D9DD27A3218D007F3792 /* Debug */ = {
+			isa = XCBuildConfiguration;
+			buildSettings = {
+				ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
+				ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor;
+				CODE_SIGN_STYLE = Automatic;
+				CURRENT_PROJECT_VERSION = 1;
+				GENERATE_INFOPLIST_FILE = YES;
+				INFOPLIST_FILE = XCRemoteCacheSample/Info.plist;
+				INFOPLIST_KEY_UIApplicationSupportsIndirectInputEvents = YES;
+				INFOPLIST_KEY_UILaunchStoryboardName = LaunchScreen;
+				INFOPLIST_KEY_UIMainStoryboardFile = Main;
+				INFOPLIST_KEY_UISupportedInterfaceOrientations_iPad = "UIInterfaceOrientationPortrait UIInterfaceOrientationPortraitUpsideDown UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight";
+				INFOPLIST_KEY_UISupportedInterfaceOrientations_iPhone = "UIInterfaceOrientationPortrait UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight";
+				LD_RUNPATH_SEARCH_PATHS = (
+					"$(inherited)",
+					"@executable_path/Frameworks",
+				);
+				MARKETING_VERSION = 1.0;
+				PRODUCT_BUNDLE_IDENTIFIER = com.xcremotecache.XCRemoteCacheSample;
+				PRODUCT_NAME = "$(TARGET_NAME)";
+				SWIFT_EMIT_LOC_STRINGS = YES;
+				SWIFT_VERSION = 5.0;
+				TARGETED_DEVICE_FAMILY = "1,2";
+			};
+			name = Debug;
+		};
+		3695D9DE27A3218D007F3792 /* Release */ = {
+			isa = XCBuildConfiguration;
+			buildSettings = {
+				ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
+				ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor;
+				CODE_SIGN_STYLE = Automatic;
+				CURRENT_PROJECT_VERSION = 1;
+				GENERATE_INFOPLIST_FILE = YES;
+				INFOPLIST_FILE = XCRemoteCacheSample/Info.plist;
+				INFOPLIST_KEY_UIApplicationSupportsIndirectInputEvents = YES;
+				INFOPLIST_KEY_UILaunchStoryboardName = LaunchScreen;
+				INFOPLIST_KEY_UIMainStoryboardFile = Main;
+				INFOPLIST_KEY_UISupportedInterfaceOrientations_iPad = "UIInterfaceOrientationPortrait UIInterfaceOrientationPortraitUpsideDown UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight";
+				INFOPLIST_KEY_UISupportedInterfaceOrientations_iPhone = "UIInterfaceOrientationPortrait UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight";
+				LD_RUNPATH_SEARCH_PATHS = (
+					"$(inherited)",
+					"@executable_path/Frameworks",
+				);
+				MARKETING_VERSION = 1.0;
+				PRODUCT_BUNDLE_IDENTIFIER = com.xcremotecache.XCRemoteCacheSample;
+				PRODUCT_NAME = "$(TARGET_NAME)";
+				SWIFT_EMIT_LOC_STRINGS = YES;
+				SWIFT_VERSION = 5.0;
+				TARGETED_DEVICE_FAMILY = "1,2";
+			};
+			name = Release;
+		};
+/* End XCBuildConfiguration section */
+
+/* Begin XCConfigurationList section */
+		3695D9C327A3218C007F3792 /* Build configuration list for PBXProject "XCRemoteCacheSample" */ = {
+			isa = XCConfigurationList;
+			buildConfigurations = (
+				3695D9DA27A3218D007F3792 /* Debug */,
+				3695D9DB27A3218D007F3792 /* Release */,
+			);
+			defaultConfigurationIsVisible = 0;
+			defaultConfigurationName = Release;
+		};
+		3695D9DC27A3218D007F3792 /* Build configuration list for PBXNativeTarget "XCRemoteCacheSample" */ = {
+			isa = XCConfigurationList;
+			buildConfigurations = (
+				3695D9DD27A3218D007F3792 /* Debug */,
+				3695D9DE27A3218D007F3792 /* Release */,
+			);
+			defaultConfigurationIsVisible = 0;
+			defaultConfigurationName = Release;
+		};
+/* End XCConfigurationList section */
+	};
+	rootObject = 3695D9C027A3218C007F3792 /* Project object */;
+}

e2eTests/XCRemoteCacheSample/XCRemoteCacheSample.xcodeproj/project.xcworkspace/contents.xcworkspacedata

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<Workspace
+   version = "1.0">
+   <FileRef
+      location = "self:">
+   </FileRef>
+</Workspace>

e2eTests/XCRemoteCacheSample/XCRemoteCacheSample.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<key>IDEDidComputeMac32BitWarning</key>
+	<true/>
+</dict>
+</plist>

e2eTests/XCRemoteCacheSample/XCRemoteCacheSample/AppDelegate.swift

@@ -0,0 +1,41 @@
+// Copyright (c) 2021 Spotify AB.
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import UIKit
+
+@main
+class AppDelegate: UIResponder, UIApplicationDelegate {
+
+
+
+    func application(_ application: UIApplication, didFinishLaunchingWithOptions launchOptions: [UIApplication.LaunchOptionsKey: Any]?) -> Bool {
+        return true
+    }
+
+    // MARK: UISceneSession Lifecycle
+
+    func application(_ application: UIApplication, configurationForConnecting connectingSceneSession: UISceneSession, options: UIScene.ConnectionOptions) -> UISceneConfiguration {
+        return UISceneConfiguration(name: "Default Configuration", sessionRole: connectingSceneSession.role)
+    }
+
+    func application(_ application: UIApplication, didDiscardSceneSessions sceneSessions: Set<UISceneSession>) {
+    }
+
+}
+

e2eTests/XCRemoteCacheSample/XCRemoteCacheSample/Assets.xcassets/AccentColor.colorset/Contents.json

@@ -0,0 +1,11 @@
+{
+  "colors" : [
+    {
+      "idiom" : "universal"
+    }
+  ],
+  "info" : {
+    "author" : "xcode",
+    "version" : 1
+  }
+}

e2eTests/XCRemoteCacheSample/XCRemoteCacheSample/Assets.xcassets/AppIcon.appiconset/Contents.json

@@ -0,0 +1,98 @@
+{
+  "images" : [
+    {
+      "idiom" : "iphone",
+      "scale" : "2x",
+      "size" : "20x20"
+    },
+    {
+      "idiom" : "iphone",
+      "scale" : "3x",
+      "size" : "20x20"
+    },
+    {
+      "idiom" : "iphone",
+      "scale" : "2x",
+      "size" : "29x29"
+    },
+    {
+      "idiom" : "iphone",
+      "scale" : "3x",
+      "size" : "29x29"
+    },
+    {
+      "idiom" : "iphone",
+      "scale" : "2x",
+      "size" : "40x40"
+    },
+    {
+      "idiom" : "iphone",
+      "scale" : "3x",
+      "size" : "40x40"
+    },
+    {
+      "idiom" : "iphone",
+      "scale" : "2x",
+      "size" : "60x60"
+    },
+    {
+      "idiom" : "iphone",
+      "scale" : "3x",
+      "size" : "60x60"
+    },
+    {
+      "idiom" : "ipad",
+      "scale" : "1x",
+      "size" : "20x20"
+    },
+    {
+      "idiom" : "ipad",
+      "scale" : "2x",
+      "size" : "20x20"
+    },
+    {
+      "idiom" : "ipad",
+      "scale" : "1x",
+      "size" : "29x29"
+    },
+    {
+      "idiom" : "ipad",
+      "scale" : "2x",
+      "size" : "29x29"
+    },
+    {
+      "idiom" : "ipad",
+      "scale" : "1x",
+      "size" : "40x40"
+    },
+    {
+      "idiom" : "ipad",
+      "scale" : "2x",
+      "size" : "40x40"
+    },
+    {
+      "idiom" : "ipad",
+      "scale" : "1x",
+      "size" : "76x76"
+    },
+    {
+      "idiom" : "ipad",
+      "scale" : "2x",
+      "size" : "76x76"
+    },
+    {
+      "idiom" : "ipad",
+      "scale" : "2x",
+      "size" : "83.5x83.5"
+    },
+    {
+      "idiom" : "ios-marketing",
+      "scale" : "1x",
+      "size" : "1024x1024"
+    }
+  ],
+  "info" : {
+    "author" : "xcode",
+    "version" : 1
+  }
+}

e2eTests/XCRemoteCacheSample/XCRemoteCacheSample/Assets.xcassets/Contents.json

@@ -0,0 +1,6 @@
+{
+  "info" : {
+    "author" : "xcode",
+    "version" : 1
+  }
+}

e2eTests/XCRemoteCacheSample/XCRemoteCacheSample/Base.lproj/LaunchScreen.storyboard

@@ -0,0 +1,25 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<document type="com.apple.InterfaceBuilder3.CocoaTouch.Storyboard.XIB" version="3.0" toolsVersion="13122.16" targetRuntime="iOS.CocoaTouch" propertyAccessControl="none" useAutolayout="YES" launchScreen="YES" useTraitCollections="YES" useSafeAreas="YES" colorMatched="YES" initialViewController="01J-lp-oVM">
+    <dependencies>
+        <plugIn identifier="com.apple.InterfaceBuilder.IBCocoaTouchPlugin" version="13104.12"/>
+        <capability name="Safe area layout guides" minToolsVersion="9.0"/>
+        <capability name="documents saved in the Xcode 8 format" minToolsVersion="8.0"/>
+    </dependencies>
+    <scenes>
+        <!--View Controller-->
+        <scene sceneID="EHf-IW-A2E">
+            <objects>
+                <viewController id="01J-lp-oVM" sceneMemberID="viewController">
+                    <view key="view" contentMode="scaleToFill" id="Ze5-6b-2t3">
+                        <rect key="frame" x="0.0" y="0.0" width="375" height="667"/>
+                        <autoresizingMask key="autoresizingMask" widthSizable="YES" heightSizable="YES"/>
+                        <color key="backgroundColor" xcode11CocoaTouchSystemColor="systemBackgroundColor" cocoaTouchSystemColor="whiteColor"/>
+                        <viewLayoutGuide key="safeArea" id="6Tk-OE-BBY"/>
+                    </view>
+                </viewController>
+                <placeholder placeholderIdentifier="IBFirstResponder" id="iYj-Kq-Ea1" userLabel="First Responder" sceneMemberID="firstResponder"/>
+            </objects>
+            <point key="canvasLocation" x="53" y="375"/>
+        </scene>
+    </scenes>
+</document>

e2eTests/XCRemoteCacheSample/XCRemoteCacheSample/Base.lproj/Main.storyboard

@@ -0,0 +1,24 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<document type="com.apple.InterfaceBuilder3.CocoaTouch.Storyboard.XIB" version="3.0" toolsVersion="13122.16" targetRuntime="iOS.CocoaTouch" propertyAccessControl="none" useAutolayout="YES" useTraitCollections="YES" useSafeAreas="YES" colorMatched="YES" initialViewController="BYZ-38-t0r">
+    <dependencies>
+        <plugIn identifier="com.apple.InterfaceBuilder.IBCocoaTouchPlugin" version="13104.12"/>
+        <capability name="Safe area layout guides" minToolsVersion="9.0"/>
+        <capability name="documents saved in the Xcode 8 format" minToolsVersion="8.0"/>
+    </dependencies>
+    <scenes>
+        <!--View Controller-->
+        <scene sceneID="tne-QT-ifu">
+            <objects>
+                <viewController id="BYZ-38-t0r" customClass="ViewController" customModuleProvider="target" sceneMemberID="viewController">
+                    <view key="view" contentMode="scaleToFill" id="8bC-Xf-vdC">
+                        <rect key="frame" x="0.0" y="0.0" width="375" height="667"/>
+                        <autoresizingMask key="autoresizingMask" widthSizable="YES" heightSizable="YES"/>
+                        <color key="backgroundColor" xcode11CocoaTouchSystemColor="systemBackgroundColor" cocoaTouchSystemColor="whiteColor"/>
+                        <viewLayoutGuide key="safeArea" id="6Tk-OE-BBY"/>
+                    </view>
+                </viewController>
+                <placeholder placeholderIdentifier="IBFirstResponder" id="dkx-z0-nzr" sceneMemberID="firstResponder"/>
+            </objects>
+        </scene>
+    </scenes>
+</document>

e2eTests/XCRemoteCacheSample/XCRemoteCacheSample/Info.plist

@@ -0,0 +1,25 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<key>UIApplicationSceneManifest</key>
+	<dict>
+		<key>UIApplicationSupportsMultipleScenes</key>
+		<false/>
+		<key>UISceneConfigurations</key>
+		<dict>
+			<key>UIWindowSceneSessionRoleApplication</key>
+			<array>
+				<dict>
+					<key>UISceneConfigurationName</key>
+					<string>Default Configuration</string>
+					<key>UISceneDelegateClassName</key>
+					<string>$(PRODUCT_MODULE_NAME).SceneDelegate</string>
+					<key>UISceneStoryboardFile</key>
+					<string>Main</string>
+				</dict>
+			</array>
+		</dict>
+	</dict>
+</dict>
+</plist>

e2eTests/XCRemoteCacheSample/XCRemoteCacheSample/SceneDelegate.swift

@@ -0,0 +1,44 @@
+// Copyright (c) 2021 Spotify AB.
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import UIKit
+
+class SceneDelegate: UIResponder, UIWindowSceneDelegate {
+    var window: UIWindow?
+
+
+    func scene(_ scene: UIScene, willConnectTo session: UISceneSession, options connectionOptions: UIScene.ConnectionOptions) {
+    }
+
+    func sceneDidDisconnect(_ scene: UIScene) {
+    }
+
+    func sceneDidBecomeActive(_ scene: UIScene) {
+    }
+
+    func sceneWillResignActive(_ scene: UIScene) {
+    }
+
+    func sceneWillEnterForeground(_ scene: UIScene) {
+    }
+
+    func sceneDidEnterBackground(_ scene: UIScene) {
+    }
+}
+

e2eTests/XCRemoteCacheSample/XCRemoteCacheSample/ViewController.swift

@@ -0,0 +1,30 @@
+// Copyright (c) 2021 Spotify AB.
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import UIKit
+
+class ViewController: UIViewController {
+
+    override func viewDidLoad() {
+        super.viewDidLoad()
+    }
+
+
+}
+

e2eTests/nginx/nginx.conf

@@ -0,0 +1,43 @@
+# Nginx configuration used for E2E tests cache server
+# Listens HTTP on port 8080
+
+
+worker_processes  1;
+
+events {
+    worker_connections  1024;
+}
+
+
+http {
+    default_type  application/octet-stream;
+
+    server {
+        listen       8080;
+        server_name  localhost;
+
+        location / {
+            root   html;
+            index  index.html index.htm;
+        }
+
+        location /cache/ {
+            # The path to the directory where nginx should store the cache contents.
+            root /tmp/cache;
+            # Allow PUT
+            dav_methods PUT;
+            create_full_put_path on;
+            # The maximum size of a single file.
+            client_max_body_size 1G;
+            allow all;
+        }
+        sendfile        on;
+
+        keepalive_timeout  65;
+
+        error_page   500 502 503 504  /50x.html;
+        location = /50x.html {
+            root   html;
+        }
+    }
+}