diff --git a/.gitignore b/.gitignore index 269ba187..5ef51459 100644 --- a/.gitignore +++ b/.gitignore @@ -30,4 +30,5 @@ examples/performance_tests/7 examples/performance_tests/8 # Ignore test -ignored.xml \ No newline at end of file +ignored.xml +gitignorefile.txt \ No newline at end of file diff --git a/.ignore b/.ignore index 48b8bf90..4f19eae5 100644 --- a/.ignore +++ b/.ignore @@ -1 +1,2 @@ vendor/ +ignorefile.txt \ No newline at end of file diff --git a/examples/ignore/README.md b/examples/ignore/README.md new file mode 100644 index 00000000..0abc7f0a --- /dev/null +++ b/examples/ignore/README.md @@ -0,0 +1 @@ +Files in here are to ensure that .ignore and .gitignore work recursively \ No newline at end of file diff --git a/processor/file.go b/processor/file.go index 608f5b9c..66c64481 100644 --- a/processor/file.go +++ b/processor/file.go @@ -1,7 +1,6 @@ package processor import ( - "errors" "fmt" "github.com/karrick/godirwalk" "github.com/monochromegane/go-gitignore" @@ -98,37 +97,15 @@ func walkDirectoryParallel(root string, output chan *FileJob) { all, _ = ioutil.ReadDir(root) } - var gitIgnore gitignore.IgnoreMatcher - gitIgnoreError := errors.New("") - + ignores := []gitignore.IgnoreMatcher{} if !GitIgnore { - // TODO the gitIgnore should check for further gitignores deeper in the tree - gitIgnore, gitIgnoreError = gitignore.NewGitIgnore(filepath.Join(root, ".gitignore")) - if Verbose { - if gitIgnoreError == nil { - printWarn(fmt.Sprintf("found and loaded gitignore file: %s", filepath.Join(root, ".gitignore"))) - } else { - printWarn(fmt.Sprintf("no gitignore found: %s", filepath.Join(root, ".gitignore"))) - } - } + ignores = loadIgnoreFile(root, ".gitignore", ignores) } - - var ignore gitignore.IgnoreMatcher - ignoreError := errors.New("") - if !Ignore { - ignore, ignoreError = gitignore.NewGitIgnore(filepath.Join(root, ".ignore")) - if Verbose { - if ignoreError == nil { - printWarn(fmt.Sprintf("found and loaded ignore file: %s", filepath.Join(root, ".ignore"))) - } else { - printWarn(fmt.Sprintf("no ignore found: %s", filepath.Join(root, ".ignore"))) - } - } + ignores = loadIgnoreFile(root, ".ignore", ignores) } resetGc := false - var excludes []*regexp.Regexp for _, exclude := range Exclude { @@ -162,24 +139,19 @@ func walkDirectoryParallel(root string, output chan *FileJob) { } } - if gitIgnoreError == nil && gitIgnore.Match(filepath.Join(root, f.Name()), true) { - if Verbose { - printWarn("skipping directory due to git ignore: " + filepath.Join(root, f.Name())) - } - shouldSkip = true - } - - if ignoreError == nil && ignore.Match(filepath.Join(root, f.Name()), true) { - if Verbose { - printWarn("skipping directory due to ignore: " + filepath.Join(root, f.Name())) + for _, i := range ignores { + if i.Match(filepath.Join(root, f.Name()), true) { + if Verbose { + printWarn("skipping directory due to ignore: " + filepath.Join(root, f.Name())) + } + shouldSkip = true } - shouldSkip = true } if !shouldSkip { wg.Add(1) go func(toWalk string) { - filejobs := walkDirectory(toWalk, PathBlacklist, extensionLookup) + filejobs := walkDirectory(toWalk, PathBlacklist, extensionLookup, ignores) for i := 0; i < len(filejobs); i++ { for _, lan := range filejobs[i].PossibleLanguages { LoadLanguageFeature(lan) @@ -206,18 +178,13 @@ func walkDirectoryParallel(root string, output chan *FileJob) { shouldSkip := false - if gitIgnoreError == nil && gitIgnore.Match(fpath, false) { - if Verbose { - printWarn("skipping file due to git ignore: " + f.Name()) - } - shouldSkip = true - } - - if ignoreError == nil && ignore.Match(fpath, false) { - if Verbose { - printWarn("skipping file due to ignore: " + f.Name()) + for _, i := range ignores { + if i.Match(filepath.Join(root, f.Name()), false) { + if Verbose { + printWarn("skipping file due to ignore: " + filepath.Join(root, f.Name())) + } + shouldSkip = true } - shouldSkip = true } for _, exclude := range excludes { @@ -268,7 +235,24 @@ func walkDirectoryParallel(root string, output chan *FileJob) { } } -func walkDirectory(toWalk string, blackList []string, extensionLookup map[string][]string) []FileJob { +func loadIgnoreFile(root string, filename string, ignores []gitignore.IgnoreMatcher) []gitignore.IgnoreMatcher { + ig, err := gitignore.NewGitIgnore(filepath.Join(root, filename)) + + if err == nil { + ignores = append(ignores, ig) + } + + if Verbose { + if err == nil { + printWarn(fmt.Sprintf("found and loaded ignore file: %s", filepath.Join(root, filename))) + } else { + printWarn(fmt.Sprintf("no ignore found: %s", filepath.Join(root, filename))) + } + } + return ignores +} + +func walkDirectory(toWalk string, blackList []string, extensionLookup map[string][]string, ignores []gitignore.IgnoreMatcher) []FileJob { extension := "" var filejobs []FileJob @@ -308,9 +292,25 @@ func walkDirectory(toWalk string, blackList []string, extensionLookup map[string return filepath.SkipDir } } - } - if !info.IsDir() { + for _, i := range ignores { + if i.Match(root, true) { + if Verbose { + printWarn("skipping directory due to ignore: " + root) + } + return filepath.SkipDir + } + } + } else { + for _, i := range ignores { + if i.Match(filepath.Join(root, info.Name()), false) { + if Verbose { + printWarn("skipping file due to ignore: " + filepath.Join(root, info.Name())) + } + return nil + } + } + // Lookup in case the full name matches language, ok := extensionLookup[strings.ToLower(info.Name())] diff --git a/processor/file_test.go b/processor/file_test.go index 9eb183d9..17ef3102 100644 --- a/processor/file_test.go +++ b/processor/file_test.go @@ -1,6 +1,7 @@ package processor import ( + "github.com/monochromegane/go-gitignore" "math/rand" "path/filepath" "strings" @@ -191,7 +192,7 @@ func TestWalkDirectory(t *testing.T) { Debug = true Exclude = []string{"test"} ProcessConstants() - files := walkDirectory(".", []string{}, ExtensionToLanguage) + files := walkDirectory(".", []string{}, ExtensionToLanguage, []gitignore.IgnoreMatcher{}) if len(files) == 0 { t.Error("Expected at least one file") diff --git a/test-all.sh b/test-all.sh index a7c8a54c..15829cc7 100755 --- a/test-all.sh +++ b/test-all.sh @@ -249,7 +249,7 @@ fi # Turn off gitignore https://github.com/boyter/scc/issues/53 touch ignored.xml -a=$(./scc | grep Total) +a=$(./scc | grep Total) b=$(./scc --no-gitignore | grep Total) if [ "$a" == "$b" ]; then echo -e "${RED}=======================================================" @@ -260,9 +260,11 @@ else echo -e "${GREEN}PASSED git ignore filter" fi -a=$(./scc | grep Total) +a=$(./scc | grep Total) b=$(./scc --no-ignore | grep Total) if [ "$a" == "$b" ]; then + echo "$a" + echo "$b" echo -e "${RED}=======================================================" echo -e "FAILED ignore filter" echo -e "=================================================${NC}" @@ -271,6 +273,34 @@ else echo -e "${GREEN}PASSED ignore filter" fi +touch ./examples/ignore/ignorefile.txt +a=$(./scc --by-file | grep ignorefile) +b=$(./scc --by-file --no-ignore | grep ignorefile) +if [ "$a" == "$b" ]; then + echo "$a" + echo "$b" + echo -e "${RED}=======================================================" + echo -e "FAILED ignore recursive filter" + echo -e "=================================================${NC}" + exit +else + echo -e "${GREEN}PASSED ignore recursive filter" +fi + +touch ./examples/ignore/gitignorefile.txt +a=$(./scc --by-file | grep gitignorefile) +b=$(./scc --by-file --no-gitignore | grep gitignorefile) +if [ "$a" == "$b" ]; then + echo "$a" + echo "$b" + echo -e "${RED}=======================================================" + echo -e "FAILED gitignore recursive filter" + echo -e "=================================================${NC}" + exit +else + echo -e "${GREEN}PASSED gitignore recursive filter" +fi + # Try out specific languages for i in 'Bosque ' 'Flow9 ' 'Bitbucket Pipeline ' 'Docker ignore ' 'Q# ' 'Futhark ' 'Alloy ' 'Wren ' 'Monkey C ' 'Alchemist ' 'Luna ' 'ignore ' do @@ -288,6 +318,8 @@ echo -e "${NC}Cleaning up..." rm ./scc rm ./ignored.xml rm .tmp_scc_yaml +rm ./examples/ignore/gitignorefile.txt +rm ./examples/ignore/ignorefile.txt echo -e "${GREEN}=================================================" echo -e "ALL TESTS PASSED"