diff --git a/CHANGELOG.md b/CHANGELOG.md index 393a4151..e11b34f4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,8 @@ ## Next Version #### Added + +- Added Bash 4 style recursive globbing (`**/*`) when excluding files from target sources. [#636](https://github.com/yonaskolb/XcodeGen/pull/636) @bclymer - Added ability to disable main thread checker on Run and Test schemes and TargetSchemes [#601](https://github.com/yonaskolb/XcodeGen/pull/601) @wag-miles #### Fixed diff --git a/Docs/ProjectSpec.md b/Docs/ProjectSpec.md index 01264599..938a8563 100644 --- a/Docs/ProjectSpec.md +++ b/Docs/ProjectSpec.md @@ -304,7 +304,7 @@ A source can be provided via a string (the path) or an object of the form: - [x] **path**: **String** - The path to the source file or directory. - [ ] **name**: **String** - Can be used to override the name of the source file or directory. By default the last component of the path is used for the name - [ ] **compilerFlags**: **[String]** or **String** - A list of compilerFlags to add to files under this specific path provided as a list or a space delimitted string. Defaults to empty. -- [ ] **excludes**: **[String]** - A list of [global patterns](https://en.wikipedia.org/wiki/Glob_(programming)) representing the files to exclude. These rules are relative to `path` and _not the directory where `project.yml` resides_. +- [ ] **excludes**: **[String]** - A list of [global patterns](https://en.wikipedia.org/wiki/Glob_(programming)) representing the files to exclude. These rules are relative to `path` and _not the directory where `project.yml` resides_. XcodeGen uses Bash 4's Glob behaviors where globstar (**) is enabled. - [ ] **createIntermediateGroups**: **Bool** - This overrides the value in [Options](#options) - [ ] **optional**: **Bool** - Disable missing path check. Defaults to false. - [ ] **buildPhase**: **String** - This manually sets the build phase this file or files in this directory will be added to, otherwise XcodeGen will guess based on the file extension. Note that `Info.plist` files will never be added to any build phases, no matter what this setting is. Possible values are: @@ -348,6 +348,8 @@ targets: - "ios/*.[mh]" - "configs/server[0-2].json" - "*-Private.h" + - "**/*.md" # excludes all files with the .md extension + - "ios/**/*Tests.[hm] # excludes all files with an h or m extension within the ios directory. compilerFlags: - "-Werror" - "-Wextra" diff --git a/Sources/XcodeGenKit/Glob.swift b/Sources/XcodeGenKit/Glob.swift new file mode 100644 index 00000000..f74a063f --- /dev/null +++ b/Sources/XcodeGenKit/Glob.swift @@ -0,0 +1,237 @@ +// +// Created by Eric Firestone on 3/22/16. +// Copyright © 2016 Square, Inc. All rights reserved. +// Released under the Apache v2 License. +// +// Adapted from https://gist.github.com/blakemerryman/76312e1cbf8aec248167 +// Adapted from https://gist.github.com/efirestone/ce01ae109e08772647eb061b3bb387c3 + + +import Foundation + + +public let GlobBehaviorBashV3 = Glob.Behavior( + supportsGlobstar: false, + includesFilesFromRootOfGlobstar: false, + includesDirectoriesInResults: true, + includesFilesInResultsIfTrailingSlash: false +) +public let GlobBehaviorBashV4 = Glob.Behavior( + supportsGlobstar: true, // Matches Bash v4 with "shopt -s globstar" option + includesFilesFromRootOfGlobstar: true, + includesDirectoriesInResults: true, + includesFilesInResultsIfTrailingSlash: false +) +public let GlobBehaviorGradle = Glob.Behavior( + supportsGlobstar: true, + includesFilesFromRootOfGlobstar: true, + includesDirectoriesInResults: false, + includesFilesInResultsIfTrailingSlash: true +) + + +/** + Finds files on the file system using pattern matching. + */ +public class Glob: Collection { + + /** + * Different glob implementations have different behaviors, so the behavior of this + * implementation is customizable. + */ + public struct Behavior { + // If true then a globstar ("**") causes matching to be done recursively in subdirectories. + // If false then "**" is treated the same as "*" + let supportsGlobstar: Bool + + // If true the results from the directory where the globstar is declared will be included as well. + // For example, with the pattern "dir/**/*.ext" the fie "dir/file.ext" would be included if this + // property is true, and would be omitted if it's false. + let includesFilesFromRootOfGlobstar: Bool + + // If false then the results will not include directory entries. This does not affect recursion depth. + let includesDirectoriesInResults: Bool + + // If false and the last characters of the pattern are "**/" then only directories are returned in the results. + let includesFilesInResultsIfTrailingSlash: Bool + } + + public static var defaultBehavior = GlobBehaviorBashV4 + + public static let defaultBlacklistedDirectories = ["node_modules", "Pods"] + + private var isDirectoryCache = [String: Bool]() + + public let behavior: Behavior + public let blacklistedDirectories: [String] + var paths = [String]() + public var startIndex: Int { return paths.startIndex } + public var endIndex: Int { return paths.endIndex } + + /// Initialize a glob + /// + /// - Parameters: + /// - pattern: The pattern to use when building the list of matching directories. + /// - behavior: See individual descriptions on `Glob.Behavior` values. + /// - blacklistedDirectories: An array of directories to ignore at the root level of the project. + public init(pattern: String, behavior: Behavior = Glob.defaultBehavior, blacklistedDirectories: [String] = defaultBlacklistedDirectories) { + + self.behavior = behavior + self.blacklistedDirectories = blacklistedDirectories + + var adjustedPattern = pattern + let hasTrailingGlobstarSlash = pattern.hasSuffix("**/") + var includeFiles = !hasTrailingGlobstarSlash + + if behavior.includesFilesInResultsIfTrailingSlash { + includeFiles = true + if hasTrailingGlobstarSlash { + // Grab the files too. + adjustedPattern += "*" + } + } + + let patterns = behavior.supportsGlobstar ? expandGlobstar(pattern: adjustedPattern) : [adjustedPattern] + + for pattern in patterns { + var gt = glob_t() + if executeGlob(pattern: pattern, gt: >) { + populateFiles(gt: gt, includeFiles: includeFiles) + } + + globfree(>) + } + + paths = Array(Set(paths)).sorted { lhs, rhs in + lhs.compare(rhs) != ComparisonResult.orderedDescending + } + + clearCaches() + } + + // MARK: Subscript Support + + public subscript(i: Int) -> String { + return paths[i] + } + + // MARK: Protocol of IndexableBase + + public func index(after i: Int) -> Int { + return i + 1 + } + + // MARK: Private + + private var globalFlags = GLOB_TILDE | GLOB_BRACE | GLOB_MARK + + private func executeGlob(pattern: UnsafePointer, gt: UnsafeMutablePointer) -> Bool { + return 0 == glob(pattern, globalFlags, nil, gt) + } + + private func expandGlobstar(pattern: String) -> [String] { + guard pattern.contains("**") else { + return [pattern] + } + + var results = [String]() + var parts = pattern.components(separatedBy: "**") + let firstPart = parts.removeFirst() + var lastPart = parts.joined(separator: "**") + + var directories: [String] + + if FileManager.default.fileExists(atPath: firstPart) { + do { + directories = try exploreDirectories(path: firstPart) + } catch { + directories = [] + print("Error parsing file system item: \(error)") + } + } else { + directories = [] + } + + if behavior.includesFilesFromRootOfGlobstar { + // Check the base directory for the glob star as well. + directories.insert(firstPart, at: 0) + + // Include the globstar root directory ("dir/") in a pattern like "dir/**" or "dir/**/" + if lastPart.isEmpty { + results.append(firstPart) + } + } + + if lastPart.isEmpty { + lastPart = "*" + } + for directory in directories { + let partiallyResolvedPattern = NSString(string: directory).appendingPathComponent(lastPart) + results.append(contentsOf: expandGlobstar(pattern: partiallyResolvedPattern)) + } + + return results + } + + private func exploreDirectories(path: String) throws -> [String] { + return try FileManager.default.contentsOfDirectory(atPath: path) + .compactMap { subpath -> [String]? in + if blacklistedDirectories.contains(subpath) { + return nil + } + let firstLevelPath = NSString(string: path).appendingPathComponent(subpath) + guard isDirectory(path: firstLevelPath) else { + return nil + } + var subDirs: [String] = try FileManager.default.subpathsOfDirectory(atPath: firstLevelPath) + .compactMap { subpath -> String? in + let fullPath = NSString(string: firstLevelPath).appendingPathComponent(subpath) + return isDirectory(path: fullPath) ? fullPath : nil + } + subDirs.append(firstLevelPath) + return subDirs + } + .joined() + .array() + } + + private func isDirectory(path: String) -> Bool { + if let isDirectory = isDirectoryCache[path] { + return isDirectory + } + + var isDirectoryBool = ObjCBool(false) + let isDirectory = FileManager.default.fileExists(atPath: path, isDirectory: &isDirectoryBool) && isDirectoryBool.boolValue + isDirectoryCache[path] = isDirectory + + return isDirectory + } + + private func clearCaches() { + isDirectoryCache.removeAll() + } + + private func populateFiles(gt: glob_t, includeFiles: Bool) { + let includeDirectories = behavior.includesDirectoriesInResults + + for i in 0.. [Element] { + return Array(self) + } +} + diff --git a/Sources/XcodeGenKit/SourceGenerator.swift b/Sources/XcodeGenKit/SourceGenerator.swift index 2ab9a4a4..5167a457 100644 --- a/Sources/XcodeGenKit/SourceGenerator.swift +++ b/Sources/XcodeGenKit/SourceGenerator.swift @@ -302,8 +302,10 @@ class SourceGenerator { let rootSourcePath = project.basePath + targetSource.path return Set( - targetSource.excludes.map { - Path.glob("\(rootSourcePath)/\($0)") + targetSource.excludes.map { pattern in + guard !pattern.isEmpty else { return [] } + return Glob(pattern: "\(rootSourcePath)/\(pattern)") + .map { Path($0) } .map { guard $0.isDirectory else { return [$0] diff --git a/Tests/XcodeGenKitTests/GlobTests.swift b/Tests/XcodeGenKitTests/GlobTests.swift new file mode 100644 index 00000000..4fb6a70d --- /dev/null +++ b/Tests/XcodeGenKitTests/GlobTests.swift @@ -0,0 +1,279 @@ +// +// Created by Eric Firestone on 3/22/16. +// Copyright © 2016 Square, Inc. All rights reserved. +// Released under the Apache v2 License. +// +// Adapted from https://gist.github.com/blakemerryman/76312e1cbf8aec248167 +// Adapted from https://gist.github.com/efirestone/ce01ae109e08772647eb061b3bb387c3 + +import XCTest +@testable import XcodeGenKit + +class GlobTests : XCTestCase { + + let tmpFiles = ["foo", "bar", "baz", "dir1/file1.ext", "dir1/dir2/dir3/file2.ext", "dir1/file1.extfoo"] + var tmpDir = "" + + override func setUp() { + super.setUp() + + self.tmpDir = newTmpDir() + + let flags = S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH + mkdir("\(tmpDir)/dir1/", flags) + mkdir("\(tmpDir)/dir1/dir2", flags) + mkdir("\(tmpDir)/dir1/dir2/dir3", flags) + + for file in tmpFiles { + close(open("\(tmpDir)/\(file)", O_CREAT)) + } + } + + override func tearDown() { + for file in tmpFiles { + unlink("\(tmpDir)/\(file)") + } + rmdir("\(tmpDir)/dir1/dir2/dir3") + rmdir("\(tmpDir)/dir1/dir2") + rmdir("\(tmpDir)/dir1") + rmdir(self.tmpDir) + + super.tearDown() + } + + private func newTmpDir() -> String { + var tmpDirTmpl = "/tmp/glob-test.XXXXX".cString(using: .utf8)! + return String(validatingUTF8: mkdtemp(&tmpDirTmpl))! + } + + func testBraces() { + let pattern = "\(tmpDir)/ba{r,y,z}" + let glob = Glob(pattern: pattern) + var contents = [String]() + for file in glob { + contents.append(file) + } + XCTAssertEqual(contents, ["\(tmpDir)/bar", "\(tmpDir)/baz"], "matching with braces failed") + } + + func testNothingMatches() { + let pattern = "\(tmpDir)/nothing" + let glob = Glob(pattern: pattern) + var contents = [String]() + for file in glob { + contents.append(file) + } + XCTAssertEqual(contents, [], "expected empty list of files") + } + + func testDirectAccess() { + let pattern = "\(tmpDir)/ba{r,y,z}" + let glob = Glob(pattern: pattern) + XCTAssertEqual(glob.paths, ["\(tmpDir)/bar", "\(tmpDir)/baz"], "matching with braces failed") + } + + func testIterateTwice() { + let pattern = "\(tmpDir)/ba{r,y,z}" + let glob = Glob(pattern: pattern) + var contents1 = [String]() + var contents2 = [String]() + for file in glob { + contents1.append(file) + } + let filesAfterOnce = glob.paths + for file in glob { + contents2.append(file) + } + XCTAssertEqual(contents1, contents2, "results for calling for-in twice are the same") + XCTAssertEqual(glob.paths, filesAfterOnce, "calling for-in twice doesn't only memoizes once") + } + + func testIndexing() { + let pattern = "\(tmpDir)/ba{r,y,z}" + let glob = Glob(pattern: pattern) + XCTAssertEqual(glob[0], "\(tmpDir)/bar", "indexing") + } + + // MARK: - Globstar - Bash v3 + + func testGlobstarBashV3NoSlash() { + // Should be the equivalent of "ls -d -1 /(tmpdir)/**" + let pattern = "\(tmpDir)/**" + let glob = Glob(pattern: pattern, behavior: GlobBehaviorBashV3) + XCTAssertEqual(glob.paths, ["\(tmpDir)/bar", "\(tmpDir)/baz", "\(tmpDir)/dir1/", "\(tmpDir)/foo"]) + } + + func testGlobstarBashV3WithSlash() { + // Should be the equivalent of "ls -d -1 /(tmpdir)/**/" + let pattern = "\(tmpDir)/**/" + let glob = Glob(pattern: pattern, behavior: GlobBehaviorBashV3) + XCTAssertEqual(glob.paths, ["\(tmpDir)/dir1/"]) + } + + func testGlobstarBashV3WithSlashAndWildcard() { + // Should be the equivalent of "ls -d -1 /(tmpdir)/**/*" + let pattern = "\(tmpDir)/**/*" + let glob = Glob(pattern: pattern, behavior: GlobBehaviorBashV3) + XCTAssertEqual(glob.paths, ["\(tmpDir)/dir1/dir2/", "\(tmpDir)/dir1/file1.ext", "\(tmpDir)/dir1/file1.extfoo"]) + } + + func testDoubleGlobstarBashV3() { + let pattern = "\(tmpDir)/**/dir2/**/*" + let glob = Glob(pattern: pattern, behavior: GlobBehaviorBashV3) + XCTAssertEqual(glob.paths, ["\(tmpDir)/dir1/dir2/dir3/file2.ext"]) + } + + // MARK: - Globstar - Bash v4 + + func testGlobstarBashV4NoSlash() { + // Should be the equivalent of "ls -d -1 /(tmpdir)/**" + let pattern = "\(tmpDir)/**" + let glob = Glob(pattern: pattern, behavior: GlobBehaviorBashV4) + XCTAssertEqual(glob.paths, [ + "\(tmpDir)/", + "\(tmpDir)/bar", + "\(tmpDir)/baz", + "\(tmpDir)/dir1/", + "\(tmpDir)/dir1/dir2/", + "\(tmpDir)/dir1/dir2/dir3/", + "\(tmpDir)/dir1/dir2/dir3/file2.ext", + "\(tmpDir)/dir1/file1.ext", + "\(tmpDir)/dir1/file1.extfoo", + "\(tmpDir)/foo" + ]) + } + + func testGlobstarBashV4WithSlash() { + // Should be the equivalent of "ls -d -1 /(tmpdir)/**/" + let pattern = "\(tmpDir)/**/" + let glob = Glob(pattern: pattern, behavior: GlobBehaviorBashV4) + XCTAssertEqual(glob.paths, [ + "\(tmpDir)/", + "\(tmpDir)/dir1/", + "\(tmpDir)/dir1/dir2/", + "\(tmpDir)/dir1/dir2/dir3/", + ]) + } + + func testGlobstarBashV4WithSlashAndWildcard() { + // Should be the equivalent of "ls -d -1 /(tmpdir)/**/*" + let pattern = "\(tmpDir)/**/*" + let glob = Glob(pattern: pattern, behavior: GlobBehaviorBashV4) + XCTAssertEqual(glob.paths, [ + "\(tmpDir)/bar", + "\(tmpDir)/baz", + "\(tmpDir)/dir1/", + "\(tmpDir)/dir1/dir2/", + "\(tmpDir)/dir1/dir2/dir3/", + "\(tmpDir)/dir1/dir2/dir3/file2.ext", + "\(tmpDir)/dir1/file1.ext", + "\(tmpDir)/dir1/file1.extfoo", + "\(tmpDir)/foo", + ]) + } + + func testDoubleGlobstarBashV4() { + let pattern = "\(tmpDir)/**/dir2/**/*" + let glob = Glob(pattern: pattern, behavior: GlobBehaviorBashV4) + XCTAssertEqual(glob.paths, [ + "\(tmpDir)/dir1/dir2/dir3/", + "\(tmpDir)/dir1/dir2/dir3/file2.ext", + ]) + } + + func testDoubleGlobstarBashV4WithFileExtension() { + // Should be the equivalent of "ls -d -1 /(tmpdir)/**/*.ext" + // Should not find "\(tmpDir)/dir1/file1.extfoo" which the file extension prefix is .ext + let pattern = "\(tmpDir)/**/*.ext" + let glob = Glob(pattern: pattern, behavior: GlobBehaviorBashV4) + XCTAssertEqual(glob.paths, [ + "\(tmpDir)/dir1/dir2/dir3/file2.ext", + "\(tmpDir)/dir1/file1.ext" + ]) + } + + // MARK: - Globstar - Gradle + + func testGlobstarGradleNoSlash() { + // Should be the equivalent of + // FileTree tree = project.fileTree((Object)'/tmp') { + // include 'glob-test.7m0Lp/**' + // } + // + // Note that the sort order currently matches Bash and not Gradle + let pattern = "\(tmpDir)/**" + let glob = Glob(pattern: pattern, behavior: GlobBehaviorGradle) + XCTAssertEqual(glob.paths, [ + "\(tmpDir)/bar", + "\(tmpDir)/baz", + "\(tmpDir)/dir1/dir2/dir3/file2.ext", + "\(tmpDir)/dir1/file1.ext", + "\(tmpDir)/dir1/file1.extfoo", + "\(tmpDir)/foo", + ]) + } + + func testGlobstarGradleWithSlash() { + // Should be the equivalent of + // FileTree tree = project.fileTree((Object)'/tmp') { + // include 'glob-test.7m0Lp/**/' + // } + // + // Note that the sort order currently matches Bash and not Gradle + let pattern = "\(tmpDir)/**/" + let glob = Glob(pattern: pattern, behavior: GlobBehaviorGradle) + XCTAssertEqual(glob.paths, [ + "\(tmpDir)/bar", + "\(tmpDir)/baz", + "\(tmpDir)/dir1/dir2/dir3/file2.ext", + "\(tmpDir)/dir1/file1.ext", + "\(tmpDir)/dir1/file1.extfoo", + "\(tmpDir)/foo", + ]) + } + + func testGlobstarGradleWithSlashAndWildcard() { + // Should be the equivalent of + // FileTree tree = project.fileTree((Object)'/tmp') { + // include 'glob-test.7m0Lp/**/*' + // } + // + // Note that the sort order currently matches Bash and not Gradle + let pattern = "\(tmpDir)/**/*" + let glob = Glob(pattern: pattern, behavior: GlobBehaviorGradle) + XCTAssertEqual(glob.paths, [ + "\(tmpDir)/bar", + "\(tmpDir)/baz", + "\(tmpDir)/dir1/dir2/dir3/file2.ext", + "\(tmpDir)/dir1/file1.ext", + "\(tmpDir)/dir1/file1.extfoo", + "\(tmpDir)/foo", + ]) + } + + func testDoubleGlobstarGradle() { + // Should be the equivalent of + // FileTree tree = project.fileTree((Object)'/tmp') { + // include 'glob-test.7m0Lp/**/dir2/**/*' + // } + // + // Note that the sort order currently matches Bash and not Gradle + let pattern = "\(tmpDir)/**/dir2/**/*" + let glob = Glob(pattern: pattern, behavior: GlobBehaviorGradle) + XCTAssertEqual(glob.paths, [ + "\(tmpDir)/dir1/dir2/dir3/file2.ext", + ]) + } + + func testBlacklistedDirectories() { + let pattern = "\(tmpDir)/**/*" + + let glob = Glob(pattern: pattern, behavior: GlobBehaviorGradle, blacklistedDirectories: ["dir1"]) + + XCTAssertEqual(glob.paths, [ + "\(tmpDir)/bar", + "\(tmpDir)/baz", + "\(tmpDir)/foo", + ]) + } +} diff --git a/Tests/XcodeGenKitTests/SourceGeneratorTests.swift b/Tests/XcodeGenKitTests/SourceGeneratorTests.swift index bac0e043..5faf8305 100644 --- a/Tests/XcodeGenKitTests/SourceGeneratorTests.swift +++ b/Tests/XcodeGenKitTests/SourceGeneratorTests.swift @@ -222,7 +222,9 @@ class SourceGeneratorTests: XCTestCase { - B: - b.swift - b.ignored + - b.alsoIgnored - a.ignored + - a.alsoIgnored - B: - b.swift - D: @@ -272,8 +274,8 @@ class SourceGeneratorTests: XCTestCase { "*.ignored", "*.xcodeproj", "*.playground", - // not supported - // "**/*.ignored", + "**/*.ignored", + "A/B/**/*.alsoIgnored", ] let target = Target(name: "Test", type: .application, platform: .iOS, sources: [TargetSource(path: "Sources", excludes: excludes)]) @@ -283,6 +285,7 @@ class SourceGeneratorTests: XCTestCase { let project = Project(basePath: directoryPath, name: "Test", targets: [target], options: options) let pbxProj = try project.generatePbxProj() try pbxProj.expectFile(paths: ["Sources", "A", "a.swift"]) + try pbxProj.expectFile(paths: ["Sources", "A", "a.alsoIgnored"]) try pbxProj.expectFile(paths: ["Sources", "D", "d.h"]) try pbxProj.expectFile(paths: ["Sources", "D", "d.m"]) try pbxProj.expectFile(paths: ["Sources", "E", "e.jpg"]) @@ -303,9 +306,9 @@ class SourceGeneratorTests: XCTestCase { try pbxProj.expectFileMissing(paths: ["Sources", "ignore.file"]) try pbxProj.expectFileMissing(paths: ["Sources", "project.xcodeproj"]) try pbxProj.expectFileMissing(paths: ["Sources", "a.playground"]) - // not supported: "**/*.ignored" - // try pbxProj.expectFileMissing(paths: ["Sources", "A", "a.ignored"]) - // try pbxProj.expectFileMissing(paths: ["Sources", "A", "B", "b.ignored"]) + try pbxProj.expectFileMissing(paths: ["Sources", "A", "a.ignored"]) + try pbxProj.expectFileMissing(paths: ["Sources", "A", "B", "b.ignored"]) + try pbxProj.expectFileMissing(paths: ["Sources", "A", "B", "b.alsoIgnored"]) } try test(generateEmptyDirectories: false)