Merge pull request #205 from polac24/bartosz/20230516-support-xcode143

Support Xcode14.3

README.md

@@ -358,6 +358,7 @@ Note: This step is not required if at least one of these is true:
 | `disable_vfs_overlay` | A feature flag to disable virtual file system overlay support (temporary) | `false` | ⬜️ |
 | `custom_rewrite_envs` | A list of extra ENVs that should be used as placeholders in the dependency list. ENV rewrite process is optimistic - does nothing if an ENV is not defined in the pre/postbuild process. | `[]` | ⬜️ |
 | `irrelevant_dependencies_paths` | Regexes of files that should not be included in a list of dependencies. Warning! Add entries here with caution - excluding dependencies that are relevant might lead to a target overcaching. The regex can match either partially or fully the filepath, e.g. `\\.modulemap$` will exclude all `.modulemap` files. | `[]` | ⬜️ |
+| `gracefully_handle_missing_common_sha` | If true, do not fail `prepare` if cannot find the most recent common commits with the primary branch. That might be useful on CI, where a shallow clone is used and cloning depth is not big enough to fetch a commit from a primary branch | `false` | ⬜️ |
 
 ## Backend cache server
 

Sources/XCRemoteCache/Commands/Libtool/FallbackXCLibtoolLogic.swift

@@ -0,0 +1,37 @@
+// Copyright (c) 2023 Spotify AB.
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import Foundation
+
+class FallbackXCLibtoolLogic: XCLibtoolLogic {
+    private let fallbackCommand: String
+
+    init(fallbackCommand: String) {
+        self.fallbackCommand = fallbackCommand
+    }
+
+    func run() {
+        let args = ProcessInfo().arguments
+        let paramList = [fallbackCommand] + args.dropFirst()
+        let cargs = paramList.map { strdup($0) } + [nil]
+        execvp(fallbackCommand, cargs)
+
+        exit(1)
+    }
+}

Sources/XCRemoteCache/Commands/Libtool/XCLibtool.swift

@@ -25,6 +25,8 @@ public enum XCLibtoolMode: Equatable {
     case createLibrary(output: String, filelist: String, dependencyInfo: String)
     /// Creating a universal library (multiple-architectures) from a set of input .a static libraries
     case createUniversalBinary(output: String, inputs: [String])
+    /// print the toolchain version
+    case version
 }
 
 public class XCLibtool {
@@ -50,6 +52,8 @@ public class XCLibtool {
                 toolName: "Libtool",
                 fallbackCommand: "libtool"
             )
+        case .version:
+            logic = FallbackXCLibtoolLogic(fallbackCommand: "libtool")
         }
     }
 

Sources/XCRemoteCache/Commands/Prepare/Prepare.swift

@@ -77,7 +77,18 @@ class Prepare: PrepareLogic {
             guard fileAccessor.fileExists(atPath: PhaseCacheModeController.xcodeSelectLink.path) else {
                 throw PrepareError.missingXcodeSelectDirectory
             }
-            let commonSha = try gitClient.getCommonPrimarySha()
+
+            let commonSha: String
+            do {
+                commonSha = try gitClient.getCommonPrimarySha()
+            } catch let GitClientError.noCommonShaWithPrimaryRepo(remoteName, error) {
+                guard context.gracefullyHandleMissingCommonSha else {
+                    throw GitClientError.noCommonShaWithPrimaryRepo(remoteName: remoteName, error: error)
+                }
+                infoLog("Cannot find a common sha with the primary branch: \(error). Gracefully disabling remote cache")
+                try disable()
+                return .failed
+            }
 
             if context.offline {
                 // Optimistically take first common sha

Sources/XCRemoteCache/Commands/Prepare/PrepareContext.swift

@@ -52,6 +52,8 @@ public struct PrepareContext {
     let cacheHealthPathProbeCount: Int
     /// clang wrapper output file
     let xcccCommand: URL
+    /// gracefully disable remote cache for missing common sha with the primary branch
+    let gracefullyHandleMissingCommonSha: Bool
 }
 
 extension PrepareContext {
@@ -77,5 +79,6 @@ extension PrepareContext {
         cacheAddresses = try config.cacheAddresses.map(URL.build)
         cacheHealthPath = config.cacheHealthPath
         cacheHealthPathProbeCount = config.cacheHealthPathProbeCount
+        gracefullyHandleMissingCommonSha = config.gracefullyHandleMissingCommonSha
     }
 }

Sources/XCRemoteCache/Config/XCRemoteCacheConfig.swift

@@ -148,6 +148,9 @@ public struct XCRemoteCacheConfig: Encodable {
     /// Note: The regex can match either partially or fully the filepath, e.g. `\\.modulemap$` will exclude
     /// all `.modulemap` files
     var irrelevantDependenciesPaths: [String] = []
+    /// If true, do not fail `prepare` if cannot find the most recent common commits with the primary branch
+    /// That might useful on CI, where a shallow clone is used
+    var gracefullyHandleMissingCommonSha: Bool = false
 }
 
 extension XCRemoteCacheConfig {
@@ -206,6 +209,8 @@ extension XCRemoteCacheConfig {
         merge.disableVFSOverlay = scheme.disableVFSOverlay ?? disableVFSOverlay
         merge.customRewriteEnvs = scheme.customRewriteEnvs ?? customRewriteEnvs
         merge.irrelevantDependenciesPaths = scheme.irrelevantDependenciesPaths ?? irrelevantDependenciesPaths
+        merge.gracefullyHandleMissingCommonSha =
+            scheme.gracefullyHandleMissingCommonSha ?? gracefullyHandleMissingCommonSha
         return merge
     }
 
@@ -273,6 +278,7 @@ struct ConfigFileScheme: Decodable {
     let disableVFSOverlay: Bool?
     let customRewriteEnvs: [String]?
     let irrelevantDependenciesPaths: [String]?
+    let gracefullyHandleMissingCommonSha: Bool?
 
     // Yams library doesn't support encoding strategy, see https://github.com/jpsim/Yams/issues/84
     enum CodingKeys: String, CodingKey {
@@ -323,6 +329,7 @@ struct ConfigFileScheme: Decodable {
         case disableVFSOverlay = "disable_vfs_overlay"
         case customRewriteEnvs = "custom_rewrite_envs"
         case irrelevantDependenciesPaths = "irrelevant_dependencies_paths"
+        case gracefullyHandleMissingCommonSha = "gracefully_handle_missing_common_sha"
     }
 }
 

Sources/XCRemoteCache/Dependencies/CacheModeController.swift

@@ -110,7 +110,7 @@ class PhaseCacheModeController: CacheModeController {
         } catch {
             // Gracefully don't disable a cache
             // That may happen if building a target for the first time
-            errorLog("Couldn't verify if should disable RC for \(commitValue).")
+            debugLog("Couldn't verify if should disable RC for \(commitValue).")
         }
         return false
     }

Sources/xclibtoolSupport/XCLibtoolHelper.swift

@@ -32,9 +32,12 @@ public class XCLibtoolHelper {
         var inputLibraries: [String] = []
         var filelist: String?
         var dependencyInfo: String?
+        var asksForVersion = false
         var i = 0
         while i < args.count {
             switch args[i] {
+            case "-V":
+                asksForVersion = true
             case "-o":
                 output = args[i + 1]
                 i += 1
@@ -52,6 +55,9 @@ public class XCLibtoolHelper {
             }
             i += 1
         }
+        if asksForVersion {
+            return .version
+        }
         guard let outputInput = output else {
             throw XCLibtoolHelperError.missingOutput
         }

Tests/XCRemoteCacheTests/Commands/PrepareTests.swift

@@ -89,4 +89,66 @@ class PrepareTests: XCTestCase {
 
         XCTAssertEqual(result, .preparedFor(sha: .init(sha: "2", age: 0), recommendedCacheAddress: remoteURL))
     }
+
+    func testFailsForMissingCommonShaWhenConfiguredToGrefullyDisable() throws {
+        git = GitClientFake(shaHistory: [], primaryBranchIndex: 0)
+        config.gracefullyHandleMissingCommonSha = true
+
+        let prepare = Prepare(
+            context: try PrepareContext(config, offline: false),
+            gitClient: git,
+            networkClients: [],
+            ccBuilder: CCWrapperBuilderFake(),
+            fileAccessor: FileManager.default,
+            globalCacheSwitcher: globalCacheSwitcher,
+            cacheInvalidator: CacheInvalidatorFake()
+        )
+
+        let result = try prepare.prepare()
+
+        XCTAssertEqual(result, .failed)
+    }
+
+    func testDisablesGlobalCacheForMissingCommonShaWhenConfiguredToGrefullyDisable() throws {
+        git = GitClientFake(shaHistory: [], primaryBranchIndex: 0)
+        config.gracefullyHandleMissingCommonSha = true
+        try globalCacheSwitcher.enable(sha: "starting_state")
+
+        let prepare = Prepare(
+            context: try PrepareContext(config, offline: false),
+            gitClient: git,
+            networkClients: [],
+            ccBuilder: CCWrapperBuilderFake(),
+            fileAccessor: FileManager.default,
+            globalCacheSwitcher: globalCacheSwitcher,
+            cacheInvalidator: CacheInvalidatorFake()
+        )
+
+        _ = try prepare.prepare()
+
+        XCTAssertEqual(globalCacheSwitcher.state, .disabled)
+    }
+
+    func testThrowsForMissingCommonSha() throws {
+        git = GitClientFake(shaHistory: [], primaryBranchIndex: 0)
+        config.gracefullyHandleMissingCommonSha = false
+
+        let prepare = Prepare(
+            context: try PrepareContext(config, offline: false),
+            gitClient: git,
+            networkClients: [],
+            ccBuilder: CCWrapperBuilderFake(),
+            fileAccessor: FileManager.default,
+            globalCacheSwitcher: globalCacheSwitcher,
+            cacheInvalidator: CacheInvalidatorFake()
+        )
+
+        XCTAssertThrowsError(try prepare.prepare()) { error in
+            switch error {
+            case GitClientError.noCommonShaWithPrimaryRepo: break
+            default:
+                XCTFail("Not expected error")
+            }
+        }
+    }
 }

Tests/XCRemoteCacheTests/TestDoubles/GitClientFake.swift

@@ -34,7 +34,10 @@ class GitClientFake: GitClient {
     }
 
     func getCommonPrimarySha() throws -> String {
-        shaHistory[primaryBranchIndex].sha
+        guard shaHistory.count > primaryBranchIndex else {
+            throw GitClientError.noCommonShaWithPrimaryRepo(remoteName: "testing", error: "SampleError")
+        }
+        return shaHistory[primaryBranchIndex].sha
     }
 
     func getShaDate(sha: String) throws -> Date {

tasks/e2e.rb

@@ -3,6 +3,10 @@ require "ostruct"
 
 desc 'Support for E2E tests: building XCRemoteCache-enabled xcodeproj using xcodebuild'
 namespace :e2e do
+    # Name of the configuration used in both standalone and CocoaPods tests
+    CONFIGURATION = 'Debug'
+    # Supported only in standalone
+    CONFIGURATIONS_EXCLUDE = 'Release'
     COCOAPODS_DIR = 'cocoapods-plugin'
     COCOAPODS_GEMSPEC_FILENAME = "cocoapods-xcremotecache.gemspec"
     E2E_COCOAPODS_SAMPLE_DIR = 'e2eTests/XCRemoteCacheSample'
@@ -61,10 +65,10 @@ namespace :e2e do
             clean_git
             # Run integrate the project
             system("pwd")
-            system("#{XCRC_BINARIES}/xcprepare integrate --input StandaloneApp.xcodeproj --mode producer --final-producer-target StandaloneApp")
+            system("#{XCRC_BINARIES}/xcprepare integrate --input StandaloneApp.xcodeproj --mode producer --final-producer-target StandaloneApp --configurations-exclude #{CONFIGURATIONS_EXCLUDE}")
             # Build the project to fill in the cache
-            build_project(nil, "StandaloneApp.xcodeproj", 'WatchExtension', 'watch', 'watchOS')
-            build_project(nil, "StandaloneApp.xcodeproj", 'StandaloneApp')
+            build_project(nil, "StandaloneApp.xcodeproj", 'WatchExtension', 'watch', 'watchOS', CONFIGURATION)
+            build_project(nil, "StandaloneApp.xcodeproj", 'StandaloneApp', 'iphone', 'iOS', CONFIGURATION)
             system("#{XCRC_BINARIES}/xcprepare stats --reset --format json")
         end
 
@@ -78,16 +82,16 @@ namespace :e2e do
 
         prepare_for_standalone(consumer_srcroot)
         Dir.chdir(consumer_srcroot) do
-            system("#{XCRC_BINARIES}/xcprepare integrate --input StandaloneApp.xcodeproj --mode consumer")
-            build_project(nil, "StandaloneApp.xcodeproj", 'WatchExtension', 'watch', 'watchOS', {'derivedDataPath' => "#{DERIVED_DATA_PATH}_consumer"})
-            build_project(nil, "StandaloneApp.xcodeproj", 'StandaloneApp', 'iphone', 'iOS', {'derivedDataPath' => "#{DERIVED_DATA_PATH}_consumer"})
+            system("#{XCRC_BINARIES}/xcprepare integrate --input StandaloneApp.xcodeproj --mode consumer --final-producer-target StandaloneApp --consumer-eligible-configurations #{CONFIGURATION} --configurations-exclude #{CONFIGURATIONS_EXCLUDE}")
+            build_project(nil, "StandaloneApp.xcodeproj", 'WatchExtension', 'watch', 'watchOS', CONFIGURATION, {'derivedDataPath' => "#{DERIVED_DATA_PATH}_consumer"})
+            build_project(nil, "StandaloneApp.xcodeproj", 'StandaloneApp', 'iphone', 'iOS', CONFIGURATION, {'derivedDataPath' => "#{DERIVED_DATA_PATH}_consumer"})
             valide_hit_rate(OpenStruct.new(DEFAULT_EXPECTATIONS))
 
             puts 'Building standalone consumer with local change...'
             # Extra: validate local compilation of the Standalone ObjC code
             system("echo '' >> StandaloneApp/StandaloneObjc.m")
-            build_project(nil, "StandaloneApp.xcodeproj", 'WatchExtension', 'watch', 'watchOS', {'derivedDataPath' => "#{DERIVED_DATA_PATH}_consumer_local"})
-            build_project(nil, "StandaloneApp.xcodeproj", 'StandaloneApp', 'iphone', 'iOS', {'derivedDataPath' => "#{DERIVED_DATA_PATH}_consumer_local"})
+            build_project(nil, "StandaloneApp.xcodeproj", 'WatchExtension', 'watch', 'watchOS', CONFIGURATION, {'derivedDataPath' => "#{DERIVED_DATA_PATH}_consumer_local"})
+            build_project(nil, "StandaloneApp.xcodeproj", 'StandaloneApp', 'iphone', 'iOS', CONFIGURATION, {'derivedDataPath' => "#{DERIVED_DATA_PATH}_consumer_local"})
         end
 
         # Revert all side effects
@@ -167,12 +171,12 @@ namespace :e2e do
         end
     end
 
-    def self.build_project(workspace, project, scheme, sdk = 'iphone', platform = 'iOS', extra_args = {})
+    def self.build_project(workspace, project, scheme, sdk = 'iphone', platform = 'iOS', configuration = 'Debug', extra_args = {})
         xcodebuild_args = {
             'workspace' => workspace,
             'project' => project,
             'scheme' => scheme,
-            'configuration' => 'Debug',
+            'configuration' => configuration,
             'sdk' => "#{sdk}simulator",
             'destination' => "generic/platform=#{platform} Simulator",
             'derivedDataPath' => DERIVED_DATA_PATH,
@@ -193,9 +197,9 @@ namespace :e2e do
         end
     end
 
-    def self.build_project_cocoapods(sdk = 'iphone', platform = 'iOS', extra_args = {})
+    def self.build_project_cocoapods(sdk = 'iphone', platform = 'iOS', configuration = 'Debug', extra_args = {})
         system('pod install')
-        build_project('XCRemoteCacheSample.xcworkspace', nil, 'XCRemoteCacheSample', sdk, platform, extra_args)
+        build_project('XCRemoteCacheSample.xcworkspace', nil, 'XCRemoteCacheSample', sdk, platform, configuration, extra_args)
     end
 
     def self.read_stats
@@ -238,7 +242,7 @@ namespace :e2e do
         dump_podfile(producer_configuration, template_path)
         puts('Building producer ...')
         Dir.chdir(E2E_COCOAPODS_SAMPLE_DIR) do
-            build_project_cocoapods
+            build_project_cocoapods('iphone', 'iOS', CONFIGURATION)
             # reset XCRemoteCache stats
             system("#{XCRC_BINARIES}/xcprepare stats --reset --format json")
         end
@@ -248,7 +252,7 @@ namespace :e2e do
         dump_podfile(consumer_configuration, template_path)
         puts('Building consumer ...')
         Dir.chdir(E2E_COCOAPODS_SAMPLE_DIR) do
-            build_project_cocoapods('iphone', 'iOS', {'derivedDataPath' => "#{DERIVED_DATA_PATH}_consumer"})
+            build_project_cocoapods('iphone', 'iOS', CONFIGURATION, {'derivedDataPath' => "#{DERIVED_DATA_PATH}_consumer"})
             valide_hit_rate(expectations)
         end
     end