Add recursive glob support

CHANGELOG.md

@@ -3,6 +3,8 @@
 ## Next Version
 
 #### Added
+
+- Added Bash 4 style recursive globbing (`**/*`) when excluding files from target sources. [#636](https://github.com/yonaskolb/XcodeGen/pull/636) @bclymer
 - Added ability to disable main thread checker on Run and Test schemes and TargetSchemes [#601](https://github.com/yonaskolb/XcodeGen/pull/601) @wag-miles
 
 #### Fixed

Docs/ProjectSpec.md

@@ -304,7 +304,7 @@ A source can be provided via a string (the path) or an object of the form:
 - [x] **path**: **String** - The path to the source file or directory.
 - [ ] **name**: **String** - Can be used to override the name of the source file or directory. By default the last component of the path is used for the name
 - [ ] **compilerFlags**: **[String]** or **String** - A list of compilerFlags to add to files under this specific path provided as a list or a space delimitted string. Defaults to empty.
-- [ ] **excludes**: **[String]** - A list of [global patterns](https://en.wikipedia.org/wiki/Glob_(programming)) representing the files to exclude. These rules are relative to `path` and _not the directory where `project.yml` resides_.
+- [ ] **excludes**: **[String]** - A list of [global patterns](https://en.wikipedia.org/wiki/Glob_(programming)) representing the files to exclude. These rules are relative to `path` and _not the directory where `project.yml` resides_. XcodeGen uses Bash 4's Glob behaviors where globstar (**) is enabled.
 - [ ] **createIntermediateGroups**: **Bool** - This overrides the value in [Options](#options)
 - [ ] **optional**: **Bool** - Disable missing path check. Defaults to false.
 - [ ] **buildPhase**: **String** - This manually sets the build phase this file or files in this directory will be added to, otherwise XcodeGen will guess based on the file extension. Note that `Info.plist` files will never be added to any build phases, no matter what this setting is. Possible values are:
@@ -348,6 +348,8 @@ targets:
           - "ios/*.[mh]"
           - "configs/server[0-2].json"
           - "*-Private.h"
+          - "**/*.md" # excludes all files with the .md extension
+          - "ios/**/*Tests.[hm] # excludes all files with an h or m extension within the ios directory.
         compilerFlags:
           - "-Werror"
           - "-Wextra"

Sources/XcodeGenKit/Glob.swift

@@ -0,0 +1,237 @@
+//
+//  Created by Eric Firestone on 3/22/16.
+//  Copyright © 2016 Square, Inc. All rights reserved.
+//  Released under the Apache v2 License.
+//
+//  Adapted from https://gist.github.com/blakemerryman/76312e1cbf8aec248167
+//  Adapted from https://gist.github.com/efirestone/ce01ae109e08772647eb061b3bb387c3
+
+
+import Foundation
+
+
+public let GlobBehaviorBashV3 = Glob.Behavior(
+    supportsGlobstar: false,
+    includesFilesFromRootOfGlobstar: false,
+    includesDirectoriesInResults: true,
+    includesFilesInResultsIfTrailingSlash: false
+)
+public let GlobBehaviorBashV4 = Glob.Behavior(
+    supportsGlobstar: true, // Matches Bash v4 with "shopt -s globstar" option
+    includesFilesFromRootOfGlobstar: true,
+    includesDirectoriesInResults: true,
+    includesFilesInResultsIfTrailingSlash: false
+)
+public let GlobBehaviorGradle = Glob.Behavior(
+    supportsGlobstar: true,
+    includesFilesFromRootOfGlobstar: true,
+    includesDirectoriesInResults: false,
+    includesFilesInResultsIfTrailingSlash: true
+)
+
+
+/**
+ Finds files on the file system using pattern matching.
+ */
+public class Glob: Collection {
+
+    /**
+     * Different glob implementations have different behaviors, so the behavior of this
+     * implementation is customizable.
+     */
+    public struct Behavior {
+        // If true then a globstar ("**") causes matching to be done recursively in subdirectories.
+        // If false then "**" is treated the same as "*"
+        let supportsGlobstar: Bool
+
+        // If true the results from the directory where the globstar is declared will be included as well.
+        // For example, with the pattern "dir/**/*.ext" the fie "dir/file.ext" would be included if this
+        // property is true, and would be omitted if it's false.
+        let includesFilesFromRootOfGlobstar: Bool
+
+        // If false then the results will not include directory entries. This does not affect recursion depth.
+        let includesDirectoriesInResults: Bool
+
+        // If false and the last characters of the pattern are "**/" then only directories are returned in the results.
+        let includesFilesInResultsIfTrailingSlash: Bool
+    }
+
+    public static var defaultBehavior = GlobBehaviorBashV4
+
+    public static let defaultBlacklistedDirectories = ["node_modules", "Pods"]
+
+    private var isDirectoryCache = [String: Bool]()
+
+    public let behavior: Behavior
+    public let blacklistedDirectories: [String]
+    var paths = [String]()
+    public var startIndex: Int { return paths.startIndex }
+    public var endIndex: Int   { return paths.endIndex   }
+
+    /// Initialize a glob
+    ///
+    /// - Parameters:
+    ///   - pattern: The pattern to use when building the list of matching directories.
+    ///   - behavior: See individual descriptions on `Glob.Behavior` values.
+    ///   - blacklistedDirectories: An array of directories to ignore at the root level of the project.
+    public init(pattern: String, behavior: Behavior = Glob.defaultBehavior, blacklistedDirectories: [String] = defaultBlacklistedDirectories) {
+
+        self.behavior = behavior
+        self.blacklistedDirectories = blacklistedDirectories
+
+        var adjustedPattern = pattern
+        let hasTrailingGlobstarSlash = pattern.hasSuffix("**/")
+        var includeFiles = !hasTrailingGlobstarSlash
+
+        if behavior.includesFilesInResultsIfTrailingSlash {
+            includeFiles = true
+            if hasTrailingGlobstarSlash {
+                // Grab the files too.
+                adjustedPattern += "*"
+            }
+        }
+
+        let patterns = behavior.supportsGlobstar ? expandGlobstar(pattern: adjustedPattern) : [adjustedPattern]
+
+        for pattern in patterns {
+            var gt = glob_t()
+            if executeGlob(pattern: pattern, gt: &gt) {
+                populateFiles(gt: gt, includeFiles: includeFiles)
+            }
+
+            globfree(&gt)
+        }
+
+        paths = Array(Set(paths)).sorted { lhs, rhs in
+            lhs.compare(rhs) != ComparisonResult.orderedDescending
+        }
+
+        clearCaches()
+    }
+
+    // MARK: Subscript Support
+
+    public subscript(i: Int) -> String {
+        return paths[i]
+    }
+
+    // MARK: Protocol of IndexableBase
+
+    public func index(after i: Int) -> Int {
+        return i + 1
+    }
+
+    // MARK: Private
+
+    private var globalFlags = GLOB_TILDE | GLOB_BRACE | GLOB_MARK
+
+    private func executeGlob(pattern: UnsafePointer<CChar>, gt: UnsafeMutablePointer<glob_t>) -> Bool {
+        return 0 == glob(pattern, globalFlags, nil, gt)
+    }
+
+    private func expandGlobstar(pattern: String) -> [String] {
+        guard pattern.contains("**") else {
+            return [pattern]
+        }
+
+        var results = [String]()
+        var parts = pattern.components(separatedBy: "**")
+        let firstPart = parts.removeFirst()
+        var lastPart = parts.joined(separator: "**")
+
+        var directories: [String]
+
+        if FileManager.default.fileExists(atPath: firstPart) {
+            do {
+                directories = try exploreDirectories(path: firstPart)
+            } catch {
+                directories = []
+                print("Error parsing file system item: \(error)")
+            }
+        } else {
+            directories = []
+        }
+
+        if behavior.includesFilesFromRootOfGlobstar {
+            // Check the base directory for the glob star as well.
+            directories.insert(firstPart, at: 0)
+
+            // Include the globstar root directory ("dir/") in a pattern like "dir/**" or "dir/**/"
+            if lastPart.isEmpty {
+                results.append(firstPart)
+            }
+        }
+
+        if lastPart.isEmpty {
+            lastPart = "*"
+        }
+        for directory in directories {
+            let partiallyResolvedPattern = NSString(string: directory).appendingPathComponent(lastPart)
+            results.append(contentsOf: expandGlobstar(pattern: partiallyResolvedPattern))
+        }
+
+        return results
+    }
+
+    private func exploreDirectories(path: String) throws -> [String] {
+        return try FileManager.default.contentsOfDirectory(atPath: path)
+            .compactMap { subpath -> [String]? in
+                if blacklistedDirectories.contains(subpath) {
+                    return nil
+                }
+                let firstLevelPath = NSString(string: path).appendingPathComponent(subpath)
+                guard isDirectory(path: firstLevelPath) else {
+                    return nil
+                }
+                var subDirs: [String] = try FileManager.default.subpathsOfDirectory(atPath: firstLevelPath)
+                    .compactMap { subpath -> String? in
+                        let fullPath = NSString(string: firstLevelPath).appendingPathComponent(subpath)
+                        return isDirectory(path: fullPath) ? fullPath : nil
+                    }
+                subDirs.append(firstLevelPath)
+                return subDirs
+            }
+            .joined()
+            .array()
+    }
+
+    private func isDirectory(path: String) -> Bool {
+        if let isDirectory = isDirectoryCache[path] {
+            return isDirectory
+        }
+
+        var isDirectoryBool = ObjCBool(false)
+        let isDirectory = FileManager.default.fileExists(atPath: path, isDirectory: &isDirectoryBool) && isDirectoryBool.boolValue
+        isDirectoryCache[path] = isDirectory
+
+        return isDirectory
+    }
+
+    private func clearCaches() {
+        isDirectoryCache.removeAll()
+    }
+
+    private func populateFiles(gt: glob_t, includeFiles: Bool) {
+        let includeDirectories = behavior.includesDirectoriesInResults
+
+        for i in 0..<Int(gt.gl_matchc) {
+            if let path = String(validatingUTF8: gt.gl_pathv[i]!) {
+                if !includeFiles || !includeDirectories {
+                    let isDirectory = self.isDirectory(path: path)
+                    if (!includeFiles && !isDirectory) || (!includeDirectories && isDirectory) {
+                        continue
+                    }
+                }
+
+                paths.append(path)
+            }
+        }
+    }
+}
+
+private extension Sequence {
+    func array() -> [Element] {
+        return Array(self)
+    }
+}
+

Sources/XcodeGenKit/SourceGenerator.swift

@@ -302,8 +302,10 @@ class SourceGenerator {
         let rootSourcePath = project.basePath + targetSource.path
 
         return Set(
-            targetSource.excludes.map {
-                Path.glob("\(rootSourcePath)/\($0)")
+            targetSource.excludes.map { pattern in
+                guard !pattern.isEmpty else { return [] }
+                return Glob(pattern: "\(rootSourcePath)/\(pattern)")
+                    .map { Path($0) }
                     .map {
                         guard $0.isDirectory else {
                             return [$0]