Files
XcodeGen/Sources/ProjectSpec/SpecFile.swift
T
Mathieu Olivari e7f753785e Fix includes related issues and improve their performances (#1275)
* Fix recursive include path when relativePath is not set

If relativePath is not set on a particular include, the first level of
include will currently work, but starting at the second level of
iteration, the computed include path will fail as relativePath will be
appended over and over onto the filePath. We're fixing that recursion
problem here and adding the corresponding tests to make sure it doesn't
happen again.

* Include projectRoot in include paths

The projectRoot setting (when specified) is currently ignored when
computing the include paths. We're fixing that in that commit.

* Use memoization during recursive SpecFiles creation

SpecFile objects are created by recursive through includes. On a large
project with programatically generated SpecFile, it is not rare to have
hundreds of SpecFiles, creating a large web of include dependencies.
In such a case, it is not rare either for a particular SpecFile to be
included by multiple other SpecFiles. When that happens, XcodeGen
currently creates a SpecFile object every time a SpecFile gets included,
which can lead to an exponential growth of includes.

I have seen hundreds of files being turned into hundred of thousands of
SpecFile object creations, which leads to an impractical XcodeGen run of
tens of minutes.

This change adds memoization during SpecFile recursion, in order to
reuse the previously created SpecFiles, if available, instead of
re-creating them.

* Update CHANGELOG.md

Add the following changes to the changelog:
* b97bdc4 - Use memoization during recursive SpecFiles creation
* a6b96ad - Include projectRoot in include paths
* 557b074 - Fix recursive include path when relativePath is not set
2022-11-03 19:05:46 +11:00

243 lines
9.6 KiB
Swift

import Foundation
import JSONUtilities
import PathKit
import Yams
public struct SpecFile {
public let basePath: Path
public let relativePath: Path
public let jsonDictionary: JSONDictionary
public let subSpecs: [SpecFile]
private let filePath: Path
fileprivate struct Include {
let path: Path
let relativePaths: Bool
let enable: Bool
static let defaultRelativePaths = true
static let defaultEnable = true
init?(any: Any) {
if let string = any as? String {
path = Path(string)
relativePaths = Include.defaultRelativePaths
enable = Include.defaultEnable
} else if let dictionary = any as? JSONDictionary, let path = dictionary["path"] as? String {
self.path = Path(path)
relativePaths = Self.resolveBoolean(dictionary, key: "relativePaths") ?? Include.defaultRelativePaths
enable = Self.resolveBoolean(dictionary, key: "enable") ?? Include.defaultEnable
} else {
return nil
}
}
static func parse(json: Any?) -> [Include] {
if let array = json as? [Any] {
return array.compactMap(Include.init)
} else if let object = json, let include = Include(any: object) {
return [include]
} else {
return []
}
}
private static func resolveBoolean(_ dictionary: [String: Any], key: String) -> Bool? {
dictionary[key] as? Bool ?? (dictionary[key] as? NSString)?.boolValue
}
}
public init(path: Path, cachedSpecFiles: inout [Path: SpecFile], variables: [String: String] = [:]) throws {
try self.init(filePath: path, basePath: path.parent(), cachedSpecFiles: &cachedSpecFiles, variables: variables)
}
public init(filePath: Path, jsonDictionary: JSONDictionary, basePath: Path = "", relativePath: Path = "", subSpecs: [SpecFile] = []) {
self.basePath = basePath
self.relativePath = relativePath
self.jsonDictionary = jsonDictionary
self.subSpecs = subSpecs
self.filePath = filePath
}
private init(include: Include, basePath: Path, relativePath: Path, cachedSpecFiles: inout [Path: SpecFile], variables: [String: String]) throws {
let basePath = include.relativePaths ? (basePath + relativePath) : basePath
let relativePath = include.relativePaths ? include.path.parent() : Path()
let includePath = include.relativePaths ? basePath + relativePath + include.path.lastComponent : basePath + include.path
try self.init(filePath: includePath, basePath: basePath, cachedSpecFiles: &cachedSpecFiles, variables: variables, relativePath: relativePath)
}
public init(filePath: Path, basePath: Path, cachedSpecFiles: inout [Path: SpecFile], variables: [String: String] = [:], relativePath: Path = "") throws {
let jsonDictionary = try SpecFile.loadDictionary(path: filePath).expand(variables: variables)
let includes = Include.parse(json: jsonDictionary["include"])
let subSpecs: [SpecFile] = try includes
.filter(\.enable)
.map { include in
if let specFile = cachedSpecFiles[filePath] {
return specFile
} else {
return try SpecFile(include: include, basePath: basePath, relativePath: relativePath, cachedSpecFiles: &cachedSpecFiles, variables: variables)
}
}
self.init(filePath: filePath, jsonDictionary: jsonDictionary, basePath: basePath, relativePath: relativePath, subSpecs: subSpecs)
cachedSpecFiles[filePath] = self
}
static func loadDictionary(path: Path) throws -> JSONDictionary {
// Depending on the extension we will either load the file as YAML or JSON
if path.extension?.lowercased() == "json" {
let data: Data = try path.read()
let jsonData = try JSONSerialization.jsonObject(with: data, options: .allowFragments)
guard let jsonDictionary = jsonData as? [String: Any] else {
fatalError("Invalid JSON at path \(path)")
}
return jsonDictionary
} else {
return try loadYamlDictionary(path: path)
}
}
public func resolvedDictionary() -> JSONDictionary {
resolvedDictionaryWithUniqueTargets()
}
private func resolvedDictionaryWithUniqueTargets() -> JSONDictionary {
var cachedSpecFiles: [Path: SpecFile] = [:]
let resolvedSpec = resolvingPaths(cachedSpecFiles: &cachedSpecFiles)
var value = Set<String>()
return resolvedSpec.mergedDictionary(set: &value)
}
func mergedDictionary(set mergedTargets: inout Set<String>) -> JSONDictionary {
let name = filePath.description
guard !mergedTargets.contains(name) else { return [:] }
mergedTargets.insert(name)
return jsonDictionary.merged(onto:
subSpecs
.map { $0.mergedDictionary(set: &mergedTargets) }
.reduce([:]) { $1.merged(onto: $0) })
}
func resolvingPaths(cachedSpecFiles: inout [Path: SpecFile], relativeTo basePath: Path = Path()) -> SpecFile {
if let cachedSpecFile = cachedSpecFiles[filePath] {
return cachedSpecFile
}
let relativePath = (basePath + self.relativePath).normalize()
guard relativePath != Path() else {
return self
}
let jsonDictionary = Project.pathProperties.resolvingPaths(in: self.jsonDictionary, relativeTo: relativePath)
let specFile = SpecFile(
filePath: filePath,
jsonDictionary: jsonDictionary,
relativePath: self.relativePath,
subSpecs: subSpecs.map { $0.resolvingPaths(cachedSpecFiles: &cachedSpecFiles, relativeTo: relativePath) }
)
cachedSpecFiles[filePath] = specFile
return specFile
}
}
extension Dictionary where Key == String, Value: Any {
func merged(onto other: [Key: Value]) -> [Key: Value] {
var merged = other
for (key, value) in self {
if key.hasSuffix(":REPLACE") {
let newKey = key[key.startIndex..<key.index(key.endIndex, offsetBy: -8)]
merged[Key(newKey)] = value
} else if let dictionary = value as? [Key: Value], let base = merged[key] as? [Key: Value] {
merged[key] = dictionary.merged(onto: base) as? Value
} else if let array = value as? [Any], let base = merged[key] as? [Any] {
merged[key] = (base + array) as? Value
} else {
merged[key] = value
}
}
return merged
}
func expand(variables: [String: String]) -> JSONDictionary {
var expanded: JSONDictionary = self
if !variables.isEmpty {
for (key, value) in self {
let newKey = expand(variables: variables, in: key)
if newKey != key {
expanded.removeValue(forKey: key)
}
expanded[newKey] = expand(variables: variables, in: value)
}
}
return expanded
}
private func expand(variables: [String: String], in value: Any) -> Any {
switch value {
case let dictionary as JSONDictionary:
return dictionary.expand(variables: variables)
case let string as String:
return expand(variables: variables, in: string)
case let array as [JSONDictionary]:
return array.map { $0.expand(variables: variables) }
case let array as [String]:
return array.map { self.expand(variables: variables, in: $0) }
case let anyArray as [Any]:
return anyArray.map { self.expand(variables: variables, in: $0) }
default:
return value
}
}
private func expand(variables: [String: String], in string: String) -> String {
var result = string
var index = result.startIndex
while index < result.endIndex {
let substring = result[index...]
if substring.count < 4 {
// We need at least 4 characters: ${x}
index = result.endIndex
} else if substring[index] == "$"
&& substring[substring.index(index, offsetBy: 1)] == "{"
&& substring[substring.index(index, offsetBy: 2)] != "}" {
// This is the start of a variable expansion...
let variableStart = index
if let variableEnd = substring.firstIndex(of: "}") {
// ...with an end
let nameStart = result.index(variableStart, offsetBy: 2) // Skipping ${
let nameEnd = result.index(variableEnd, offsetBy: -1) // Removing trailing }
let name = result[nameStart...nameEnd]
if let value = variables[String(name)] {
result.replaceSubrange(variableStart...variableEnd, with: value)
index = result.index(index, offsetBy: value.count)
} else {
// Skip this whole variable for which we don't have a value
index = result.index(after: variableEnd)
}
} else {
// Malformed variable, skip the whole string
index = result.endIndex
}
} else {
// Move on to the next $ and start again or finish early
index = result[result.index(after: index)...].firstIndex(of: "$") ?? result.endIndex
}
}
return result
}
}