Merge pull request #412 from yonaskolb/lockfile

Add caching
2026-03-18 20:02:25 +00:00 · 2018-12-18 22:24:07 +11:00
parent f87642c938 c8db304024
commit 4cc4906ddf
12 changed files with 471 additions and 9 deletions
@@ -2,6 +2,9 @@

 ## Master

+#### Added
+- Added an experiment new caching feature. Pass `--use-cache` to opt in. This will read and write from a cache file to prevent unnecessarily generating the project. Give it a try as it may become the default in a future release [#412](https://github.com/yonaskolb/XcodeGen/pull/412) @yonaskolb
+
 #### Changed
 - Changed spelling of build phases to **preBuildPhase** and **postBuildPhase**. The older names are deprecated but still work [402](https://github.com/yonaskolb/XcodeGen/pull/402) @brentleyjones
 - Moved generation to a specific subcommand `xcodegen generate`. Simple `xcodegen` will continue to work for now [#437](https://github.com/yonaskolb/XcodeGen/pull/437) @yonaskolb
@@ -121,9 +121,12 @@ xcodegen generate
 This will look for a project spec in the current directory called `project.yml` and generate an Xcode project with the name defined in the spec.

 Options:
+
 - **--spec**: An optional path to a `.yml` or `.json` project spec. Defaults to `project.yml`
 - **--project**: An optional path to a directory where the project will be generated. By default this is the directory the spec lives in.
 - **--quiet**: Suppress informational and success messages.
+- **--use-cache**: Used to prevent unnecessarily generating the project. If this is set, then a cache file will be written to when a project is generated. If `xcodegen` is later run but the spec and all the files it contains are the same, the project won't be generated.
+- **--cache-path**: A custom path to use for your cache file. This defaults to `~/.xcodegen/cache/{PROJECT_SPEC_PATH_HASH}`

 Use `xcodegen help` to see more detailed usage information.

@@ -165,3 +165,40 @@ extension Project {
        return jsonDictionary
    }
 }
+
+extension Project {
+
+    public var allFiles: [Path] {
+        var files: [Path] = []
+        files.append(contentsOf: configFilePaths)
+        for fileGroup in fileGroups {
+            let fileGroupPath = basePath + fileGroup
+            let fileGroupChildren = (try? fileGroupPath.recursiveChildren()) ?? []
+            files.append(contentsOf: fileGroupChildren)
+            files.append(fileGroupPath)
+        }
+
+        for target in aggregateTargets {
+            files.append(contentsOf: target.configFilePaths)
+        }
+        
+        for target in targets {
+            files.append(contentsOf: target.configFilePaths)
+            for source in target.sources {
+                let sourcePath = basePath + source.path
+                let sourceChildren = (try? sourcePath.recursiveChildren()) ?? []
+                files.append(contentsOf: sourceChildren)
+                files.append(sourcePath)
+            }
+        }
+        return files
+    }
+}
+
+extension BuildSettingsContainer {
+
+    fileprivate var configFilePaths: [Path] {
+        return configFiles.values.map{ Path($0) }
+    }
+}
+
@@ -10,7 +10,7 @@ extension Project {
        try self.init(basePath: path.parent(), jsonDictionary: dictionary)
    }

-    private static func loadDictionary(path: Path) throws -> JSONDictionary {
+    public static func loadDictionary(path: Path) throws -> JSONDictionary {

        // Depending on the extension we will either load the file as YAML or JSON
        var json: [String: Any]
@@ -1,11 +1,15 @@
 import Foundation

-public struct Version: CustomStringConvertible, Equatable, Comparable {
+public struct Version: CustomStringConvertible, Equatable, Comparable, ExpressibleByStringLiteral {

    public var major: UInt
    public var minor: UInt
    public var patch: UInt

+    public init(stringLiteral value: String) {
+        try! self.init(value)
+    }
+
    public init(_ string: String) throws {
        let components = try string.split(separator: ".").map { (componentString) -> UInt in
            guard let uint = UInt(componentString) else {
@@ -10,9 +10,19 @@ class GenerateCommand: Command {
    let name: String = "generate"
    let shortDescription: String = "Generate an Xcode project from a spec"

-    let quiet = Flag("-q", "--quiet", description: "Suppress all informational and success output", defaultValue: false)
+    let quiet = Flag("-q", "--quiet",
+                     description: "Suppress all informational and success output",
+                     defaultValue: false)

-    let spec = Key<Path>("-s", "--spec", description: "The path to the project spec file. Defaults to project.yml")
+    let useCache = Flag("-c", "--use-cache",
+                        description: "Use a cache for the xcodegen spec. This will prevent unnecessarily generating the project if nothing has changed",
+                        defaultValue: false)
+
+    let cacheFilePath = Key<Path>("--cache-path",
+                                  description: "Where the cache file will be loaded from and save to. Defaults to ~/.xcodegen/cache/{SPEC_PATH_HASH}")
+
+    let spec = Key<Path>("-s", "--spec",
+                         description: "The path to the project spec file. Defaults to project.yml")

    let projectDirectory = Key<Path>("-p", "--project", description: "The path to the directory where the project should be generated. Defaults to the directory the spec is in. The filename is defined in the project spec")

@@ -32,15 +42,50 @@ class GenerateCommand: Command {
            throw GenerationError.missingProjectSpec(projectSpecPath)
        }

+        let specLoader = SpecLoader(version: version)
        let project: Project
+
+        // load project spec
        do {
-            project = try Project(path: projectSpecPath)
+            project = try specLoader.loadProject(path: projectSpecPath)
+            info("Loaded project:\n  \(project.debugDescription.replacingOccurrences(of: "\n", with: "\n  "))")
        } catch {
            throw GenerationError.projectSpecParsingError(error)
        }

-        info("Loaded project:\n  \(project.debugDescription.replacingOccurrences(of: "\n", with: "\n  "))")
+        let projectPath = projectDirectory + "\(project.name).xcodeproj"

+        let cacheFilePath = self.cacheFilePath.value ??
+            Path("~/.xcodegen/cache/\(projectSpecPath.absolute().string.md5)").absolute()
+        var cacheFile: CacheFile?
+
+        // read cache
+        if useCache.value || self.cacheFilePath.value != nil {
+            do {
+                cacheFile = try specLoader.generateCacheFile()
+            } catch {
+                throw GenerationError.projectSpecParsingError(error)
+            }
+        }
+
+        let projectExists = XcodeProj.pbxprojPath(projectPath).exists
+
+        // check cache
+        if let cacheFile = cacheFile,
+            projectExists,
+            cacheFilePath.exists {
+            do {
+                let existingCacheFile: String = try cacheFilePath.read()
+                if cacheFile.string == existingCacheFile {
+                    info("Project has not changed since cache was written")
+                    return
+                }
+            } catch {
+                info("Couldn't load cache at \(cacheFile)")
+            }
+        }
+
+        // validate project
        do {
            try project.validateMinimumXcodeGenVersion(version)
            try project.validate()
@@ -48,6 +93,7 @@ class GenerateCommand: Command {
            throw GenerationError.validationError(error)
        }

+        // generate project
        info("⚙️  Generating project...")
        let xcodeProject: XcodeProj
        do {
@@ -57,18 +103,27 @@ class GenerateCommand: Command {
            throw GenerationError.generationError(error)
        }

+        // write project
        info("⚙️  Writing project...")
-        let projectPath = projectDirectory + "\(project.name).xcodeproj"
        do {

            let fileWriter = FileWriter(project: project)
            try fileWriter.writeXcodeProject(xcodeProject, to: projectPath)
            try fileWriter.writePlists()
+            success("Created project at \(projectPath)")
        } catch {
            throw GenerationError.writingError(error)
        }

-        success("Created project at \(projectPath)")
+        // write cache
+        if let cacheFile = cacheFile {
+            do {
+                try cacheFilePath.parent().mkpath()
+                try cacheFilePath.write(cacheFile.string)
+            } catch {
+                info("Failed to write cache: \(error.localizedDescription)")
+            }
+        }
    }

    func info(_ string: String) {
@@ -7,6 +7,7 @@ import Rainbow
 enum GenerationError: Error, CustomStringConvertible, ProcessError {
    case missingProjectSpec(Path)
    case projectSpecParsingError(Error)
+    case cacheGenerationError(Error)
    case validationError(SpecValidationError)
    case generationError(Error)
    case writingError(Error)
@@ -17,6 +18,8 @@ enum GenerationError: Error, CustomStringConvertible, ProcessError {
            return "No project spec found at \(path.absolute())"
        case .projectSpecParsingError(let error):
            return "Parsing project spec failed: \(error)"
+        case .cacheGenerationError(let error):
+            return "Couldn't generate cache file: \(error)"
        case .validationError(let error):
            return error.description
        case .generationError(let error):
@@ -0,0 +1,33 @@
+import Foundation
+import ProjectSpec
+
+
+public class CacheFile {
+
+    public let string: String
+
+    init?(version: Version, projectDictionary: [String: Any], project: Project) throws {
+
+        guard #available(OSX 10.13, *) else { return nil }
+
+        let files = Array(Set(project.allFiles))
+            .map { $0.byRemovingBase(path: project.basePath).string }
+            .sorted { $0.localizedStandardCompare($1) == .orderedAscending }
+            .joined(separator: "\n")
+
+        let data = try JSONSerialization.data(withJSONObject: projectDictionary, options: [.sortedKeys, .prettyPrinted])
+        let spec = String(data: data, encoding: .utf8)!
+
+        string = """
+        # XCODEGEN VERSION
+        \(version)
+
+        # SPEC
+        \(spec)
+
+        # FILES
+        \(files)"
+
+        """
+    }
+}
@@ -0,0 +1,277 @@
+// To date, adding CommonCrypto to a Swift framework is problematic. See:
+// http://stackoverflow.com/questions/25248598/importing-commoncrypto-in-a-swift-framework
+// We're using a subset and modified version of CryptoSwift as an alternative.
+// The following is an altered source version that only includes MD5. The original software can be found at:
+// https://github.com/krzyzanowskim/CryptoSwift
+// This is the original copyright notice:
+
+/*
+ Copyright (C) 2014 Marcin Krzyżanowski <marcin.krzyzanowski@gmail.com>
+ This software is provided 'as-is', without any express or implied warranty.
+ In no event will the authors be held liable for any damages arising from the use of this software.
+ Permission is granted to anyone to use this software for any purpose,including commercial applications, and to alter it and redistribute it freely, subject to the following restrictions:
+ - The origin of this software must not be misrepresented; you must not claim that you wrote the original software. If you use this software in a product, an acknowledgment in the product documentation is required.
+ - Altered source versions must be plainly marked as such, and must not be misrepresented as being the original software.
+ - This notice may not be removed or altered from any source or binary distribution.
+ */
+
+import Foundation
+
+extension String {
+    public var md5: String {
+        if let data = data(using: .utf8, allowLossyConversion: true) {
+            let message = data.withUnsafeBytes { bytes -> [UInt8] in
+                return Array(UnsafeBufferPointer(start: bytes, count: data.count))
+            }
+
+            let MD5Calculator = MD5(message)
+            let MD5Data = MD5Calculator.calculate()
+
+            var MD5String = String()
+            for c in MD5Data {
+                MD5String += String(format: "%02x", c)
+            }
+            return MD5String
+
+        } else {
+            return self
+        }
+    }
+}
+
+/** array of bytes, little-endian representation */
+func arrayOfBytes<T>(_ value: T, length: Int? = nil) -> [UInt8] {
+    let totalBytes = length ?? (MemoryLayout<T>.size * 8)
+
+    let valuePointer = UnsafeMutablePointer<T>.allocate(capacity: 1)
+    valuePointer.pointee = value
+
+    let bytes = valuePointer.withMemoryRebound(to: UInt8.self, capacity: totalBytes) { (bytesPointer) -> [UInt8] in
+        var bytes = [UInt8](repeating: 0, count: totalBytes)
+        for j in 0 ..< min(MemoryLayout<T>.size, totalBytes) {
+            bytes[totalBytes - 1 - j] = (bytesPointer + j).pointee
+        }
+        return bytes
+    }
+
+    #if swift(>=4.1)
+    valuePointer.deinitialize(count: 1)
+    valuePointer.deallocate()
+    #else
+    valuePointer.deinitialize()
+    valuePointer.deallocate(capacity: 1)
+    #endif
+
+    return bytes
+}
+
+extension Int {
+    /** Array of bytes with optional padding (little-endian) */
+    func bytes(_ totalBytes: Int = MemoryLayout<Int>.size) -> [UInt8] {
+        return arrayOfBytes(self, length: totalBytes)
+    }
+}
+
+extension NSMutableData {
+    /** Convenient way to append bytes */
+    func appendBytes(_ arrayOfBytes: [UInt8]) {
+        append(arrayOfBytes, length: arrayOfBytes.count)
+    }
+}
+
+protocol HashProtocol {
+    var message: Array<UInt8> { get }
+
+    /** Common part for hash calculation. Prepare header data. */
+    func prepare(_ len: Int) -> Array<UInt8>
+}
+
+extension HashProtocol {
+    func prepare(_ len: Int) -> Array<UInt8> {
+        var tmpMessage = message
+
+        // Step 1. Append Padding Bits
+        tmpMessage.append(0x80) // append one bit (UInt8 with one bit) to message
+
+        // append "0" bit until message length in bits ≡ 448 (mod 512)
+        var msgLength = tmpMessage.count
+        var counter = 0
+
+        while msgLength % len != (len - 8) {
+            counter += 1
+            msgLength += 1
+        }
+
+        tmpMessage += Array<UInt8>(repeating: 0, count: counter)
+        return tmpMessage
+    }
+}
+
+func toUInt32Array(_ slice: ArraySlice<UInt8>) -> Array<UInt32> {
+    var result = Array<UInt32>()
+    result.reserveCapacity(16)
+
+    for idx in stride(from: slice.startIndex, to: slice.endIndex, by: MemoryLayout<UInt32>.size) {
+        let d0 = UInt32(slice[idx.advanced(by: 3)]) << 24
+        let d1 = UInt32(slice[idx.advanced(by: 2)]) << 16
+        let d2 = UInt32(slice[idx.advanced(by: 1)]) << 8
+        let d3 = UInt32(slice[idx])
+        let val: UInt32 = d0 | d1 | d2 | d3
+
+        result.append(val)
+    }
+    return result
+}
+
+struct BytesIterator: IteratorProtocol {
+    let chunkSize: Int
+    let data: [UInt8]
+
+    init(chunkSize: Int, data: [UInt8]) {
+        self.chunkSize = chunkSize
+        self.data = data
+    }
+
+    var offset = 0
+
+    mutating func next() -> ArraySlice<UInt8>? {
+        let end = min(chunkSize, data.count - offset)
+        let result = data[offset ..< offset + end]
+        offset += result.count
+        return result.count > 0 ? result : nil
+    }
+}
+
+struct BytesSequence: Sequence {
+    let chunkSize: Int
+    let data: [UInt8]
+
+    func makeIterator() -> BytesIterator {
+        return BytesIterator(chunkSize: chunkSize, data: data)
+    }
+}
+
+func rotateLeft(_ value: UInt32, bits: UInt32) -> UInt32 {
+    return ((value << bits) & 0xFFFF_FFFF) | (value >> (32 - bits))
+}
+
+class MD5: HashProtocol {
+    static let size = 16 // 128 / 8
+    let message: [UInt8]
+
+    init(_ message: [UInt8]) {
+        self.message = message
+    }
+
+    /** specifies the per-round shift amounts */
+    private let shifts: [UInt32] = [
+        7, 12, 17, 22, 7, 12, 17, 22, 7, 12, 17, 22, 7, 12, 17, 22,
+        5, 9, 14, 20, 5, 9, 14, 20, 5, 9, 14, 20, 5, 9, 14, 20,
+        4, 11, 16, 23, 4, 11, 16, 23, 4, 11, 16, 23, 4, 11, 16, 23,
+        6, 10, 15, 21, 6, 10, 15, 21, 6, 10, 15, 21, 6, 10, 15, 21,
+        ]
+
+    /** binary integer part of the sines of integers (Radians) */
+    private let sines: [UInt32] = [
+        0xD76A_A478, 0xE8C7_B756, 0x2420_70DB, 0xC1BD_CEEE,
+        0xF57C_0FAF, 0x4787_C62A, 0xA830_4613, 0xFD46_9501,
+        0x6980_98D8, 0x8B44_F7AF, 0xFFFF_5BB1, 0x895C_D7BE,
+        0x6B90_1122, 0xFD98_7193, 0xA679_438E, 0x49B4_0821,
+        0xF61E_2562, 0xC040_B340, 0x265E_5A51, 0xE9B6_C7AA,
+        0xD62F_105D, 0x0244_1453, 0xD8A1_E681, 0xE7D3_FBC8,
+        0x21E1_CDE6, 0xC337_07D6, 0xF4D5_0D87, 0x455A_14ED,
+        0xA9E3_E905, 0xFCEF_A3F8, 0x676F_02D9, 0x8D2A_4C8A,
+        0xFFFA_3942, 0x8771_F681, 0x6D9D_6122, 0xFDE5_380C,
+        0xA4BE_EA44, 0x4BDE_CFA9, 0xF6BB_4B60, 0xBEBF_BC70,
+        0x289B_7EC6, 0xEAA1_27FA, 0xD4EF_3085, 0x4881D05,
+        0xD9D4_D039, 0xE6DB_99E5, 0x1FA2_7CF8, 0xC4AC_5665,
+        0xF429_2244, 0x432A_FF97, 0xAB94_23A7, 0xFC93_A039,
+        0x655B_59C3, 0x8F0C_CC92, 0xFFEF_F47D, 0x8584_5DD1,
+        0x6FA8_7E4F, 0xFE2C_E6E0, 0xA301_4314, 0x4E08_11A1,
+        0xF753_7E82, 0xBD3A_F235, 0x2AD7_D2BB, 0xEB86_D391,
+        ]
+
+    private let hashes: [UInt32] = [0x6745_2301, 0xEFCD_AB89, 0x98BA_DCFE, 0x1032_5476]
+
+    func calculate() -> [UInt8] {
+        var tmpMessage = prepare(64)
+        tmpMessage.reserveCapacity(tmpMessage.count + 4)
+
+        // hash values
+        var hh = hashes
+
+        // Step 2. Append Length a 64-bit representation of lengthInBits
+        let lengthInBits = (message.count * 8)
+        let lengthBytes = lengthInBits.bytes(64 / 8)
+        tmpMessage += lengthBytes.reversed()
+
+        // Process the message in successive 512-bit chunks:
+        let chunkSizeBytes = 512 / 8 // 64
+
+        for chunk in BytesSequence(chunkSize: chunkSizeBytes, data: tmpMessage) {
+            // break chunk into sixteen 32-bit words M[j], 0 ≤ j ≤ 15
+            var M = toUInt32Array(chunk)
+            assert(M.count == 16, "Invalid array")
+
+            // Initialize hash value for this chunk:
+            var A: UInt32 = hh[0]
+            var B: UInt32 = hh[1]
+            var C: UInt32 = hh[2]
+            var D: UInt32 = hh[3]
+
+            var dTemp: UInt32 = 0
+
+            // Main loop
+            for j in 0 ..< sines.count {
+                var g = 0
+                var F: UInt32 = 0
+
+                switch j {
+                case 0 ... 15:
+                    F = (B & C) | ((~B) & D)
+                    g = j
+                    break
+                case 16 ... 31:
+                    F = (D & B) | (~D & C)
+                    g = (5 * j + 1) % 16
+                    break
+                case 32 ... 47:
+                    F = B ^ C ^ D
+                    g = (3 * j + 5) % 16
+                    break
+                case 48 ... 63:
+                    F = C ^ (B | (~D))
+                    g = (7 * j) % 16
+                    break
+                default:
+                    break
+                }
+                dTemp = D
+                D = C
+                C = B
+                B = B &+ rotateLeft((A &+ F &+ sines[j] &+ M[g]), bits: shifts[j])
+                A = dTemp
+            }
+
+            hh[0] = hh[0] &+ A
+            hh[1] = hh[1] &+ B
+            hh[2] = hh[2] &+ C
+            hh[3] = hh[3] &+ D
+        }
+
+        var result = [UInt8]()
+        result.reserveCapacity(hh.count / 4)
+
+        hh.forEach {
+            let itemLE = $0.littleEndian
+            let r1 = UInt8(itemLE & 0xFF)
+            let r2 = UInt8((itemLE >> 8) & 0xFF)
+            let r3 = UInt8((itemLE >> 16) & 0xFF)
+            let r4 = UInt8((itemLE >> 24) & 0xFF)
+            result += [r1, r2, r3, r4]
+        }
+        return result
+    }
+}
+
+// swiftlint:enable all
@@ -3,7 +3,7 @@ import PathKit

 extension Path {

-    func byRemovingBase(path: Path) -> Path {
+    public func byRemovingBase(path: Path) -> Path {
        return Path(normalize().string.replacingOccurrences(of: "\(path.normalize().string)/", with: ""))
    }
 }
@@ -0,0 +1,38 @@
+import Foundation
+import JSONUtilities
+import PathKit
+import ProjectSpec
+import xcodeproj
+import Yams
+
+public class SpecLoader {
+
+    var project: Project!
+    private var projectDictionary: [String: Any]?
+    let version: Version
+
+    public init(version: Version) {
+        self.version = version
+    }
+
+    public func loadProject(path: Path) throws -> Project {
+        let dictionary = try Project.loadDictionary(path: path)
+        let project = try Project(basePath: path.parent(), jsonDictionary: dictionary)
+
+        self.project = project
+        self.projectDictionary = dictionary
+
+        return project
+    }
+
+    public func generateCacheFile() throws -> CacheFile? {
+        guard let projectDictionary = projectDictionary,
+            let project = project else {
+                return nil
+        }
+        return try CacheFile(version: version,
+                              projectDictionary: projectDictionary,
+                              project: project)
+    }
+
+}
@@ -40,6 +40,15 @@ class FixturePerformanceTests: XCTestCase {
        }
    }

+    func testCacheFileGeneration() throws {
+        let specLoader = SpecLoader(version: "1.2")
+        _ = try specLoader.loadProject(path: specPath)
+
+        self.measure {
+            _ = try! specLoader.generateCacheFile()
+        }
+    }
+
    func testFixtureGeneration() throws {
        let project = try Project(path: specPath)
        measure {