Skip to content

Commit

Permalink
Merge 43d7ea8 into fc00bcd
Browse files Browse the repository at this point in the history
  • Loading branch information
@boyter
boyter committed Jul 29, 2019
2 parents fc00bcd + 43d7ea8 commit d9eeadd
Show file tree
Hide file tree
Showing 6 changed files with 92 additions and 56 deletions.
3 changes: 2 additions & 1 deletion .gitignore
View file
Expand Up @@ -30,4 +30,5 @@ examples/performance_tests/7
examples/performance_tests/8

# Ignore test
ignored.xml
ignored.xml
gitignorefile.txt
1 change: 1 addition & 0 deletions .ignore
View file
@@ -1 +1,2 @@
vendor/
ignorefile.txt
1 change: 1 addition & 0 deletions examples/ignore/README.md
View file
@@ -0,0 +1 @@
Files in here are to ensure that .ignore and .gitignore work recursively
104 changes: 52 additions & 52 deletions processor/file.go
View file
@@ -1,7 +1,6 @@
package processor

import (
"errors"
"fmt"
"github.com/karrick/godirwalk"
"github.com/monochromegane/go-gitignore"
Expand Down Expand Up @@ -98,37 +97,15 @@ func walkDirectoryParallel(root string, output chan *FileJob) {
all, _ = ioutil.ReadDir(root)
}

var gitIgnore gitignore.IgnoreMatcher
gitIgnoreError := errors.New("")

ignores := []gitignore.IgnoreMatcher{}
if !GitIgnore {
// TODO the gitIgnore should check for further gitignores deeper in the tree
gitIgnore, gitIgnoreError = gitignore.NewGitIgnore(filepath.Join(root, ".gitignore"))
if Verbose {
if gitIgnoreError == nil {
printWarn(fmt.Sprintf("found and loaded gitignore file: %s", filepath.Join(root, ".gitignore")))
} else {
printWarn(fmt.Sprintf("no gitignore found: %s", filepath.Join(root, ".gitignore")))
}
}
ignores = loadIgnoreFile(root, ".gitignore", ignores)
}

var ignore gitignore.IgnoreMatcher
ignoreError := errors.New("")

if !Ignore {
ignore, ignoreError = gitignore.NewGitIgnore(filepath.Join(root, ".ignore"))
if Verbose {
if ignoreError == nil {
printWarn(fmt.Sprintf("found and loaded ignore file: %s", filepath.Join(root, ".ignore")))
} else {
printWarn(fmt.Sprintf("no ignore found: %s", filepath.Join(root, ".ignore")))
}
}
ignores = loadIgnoreFile(root, ".ignore", ignores)
}

resetGc := false

var excludes []*regexp.Regexp

for _, exclude := range Exclude {
Expand Down Expand Up @@ -162,24 +139,19 @@ func walkDirectoryParallel(root string, output chan *FileJob) {
}
}

if gitIgnoreError == nil && gitIgnore.Match(filepath.Join(root, f.Name()), true) {
if Verbose {
printWarn("skipping directory due to git ignore: " + filepath.Join(root, f.Name()))
}
shouldSkip = true
}

if ignoreError == nil && ignore.Match(filepath.Join(root, f.Name()), true) {
if Verbose {
printWarn("skipping directory due to ignore: " + filepath.Join(root, f.Name()))
for _, i := range ignores {
if i.Match(filepath.Join(root, f.Name()), true) {
if Verbose {
printWarn("skipping directory due to ignore: " + filepath.Join(root, f.Name()))
}
shouldSkip = true
}
shouldSkip = true
}

if !shouldSkip {
wg.Add(1)
go func(toWalk string) {
filejobs := walkDirectory(toWalk, PathBlacklist, extensionLookup)
filejobs := walkDirectory(toWalk, PathBlacklist, extensionLookup, ignores)
for i := 0; i < len(filejobs); i++ {
for _, lan := range filejobs[i].PossibleLanguages {
LoadLanguageFeature(lan)
Expand All @@ -206,18 +178,13 @@ func walkDirectoryParallel(root string, output chan *FileJob) {

shouldSkip := false

if gitIgnoreError == nil && gitIgnore.Match(fpath, false) {
if Verbose {
printWarn("skipping file due to git ignore: " + f.Name())
}
shouldSkip = true
}

if ignoreError == nil && ignore.Match(fpath, false) {
if Verbose {
printWarn("skipping file due to ignore: " + f.Name())
for _, i := range ignores {
if i.Match(filepath.Join(root, f.Name()), false) {
if Verbose {
printWarn("skipping file due to ignore: " + filepath.Join(root, f.Name()))
}
shouldSkip = true
}
shouldSkip = true
}

for _, exclude := range excludes {
Expand Down Expand Up @@ -268,7 +235,24 @@ func walkDirectoryParallel(root string, output chan *FileJob) {
}
}

func walkDirectory(toWalk string, blackList []string, extensionLookup map[string][]string) []FileJob {
func loadIgnoreFile(root string, filename string, ignores []gitignore.IgnoreMatcher) []gitignore.IgnoreMatcher {
ig, err := gitignore.NewGitIgnore(filepath.Join(root, filename))

if err == nil {
ignores = append(ignores, ig)
}

if Verbose {
if err == nil {
printWarn(fmt.Sprintf("found and loaded ignore file: %s", filepath.Join(root, filename)))
} else {
printWarn(fmt.Sprintf("no ignore found: %s", filepath.Join(root, filename)))
}
}
return ignores
}

func walkDirectory(toWalk string, blackList []string, extensionLookup map[string][]string, ignores []gitignore.IgnoreMatcher) []FileJob {
extension := ""
var filejobs []FileJob

Expand Down Expand Up @@ -308,9 +292,25 @@ func walkDirectory(toWalk string, blackList []string, extensionLookup map[string
return filepath.SkipDir
}
}
}

if !info.IsDir() {
for _, i := range ignores {
if i.Match(root, true) {
if Verbose {
printWarn("skipping directory due to ignore: " + root)
}
return filepath.SkipDir
}
}
} else {
for _, i := range ignores {
if i.Match(filepath.Join(root, info.Name()), false) {
if Verbose {
printWarn("skipping file due to ignore: " + filepath.Join(root, info.Name()))
}
return nil
}
}

// Lookup in case the full name matches
language, ok := extensionLookup[strings.ToLower(info.Name())]

Expand Down
3 changes: 2 additions & 1 deletion processor/file_test.go
View file
@@ -1,6 +1,7 @@
package processor

import (
"github.com/monochromegane/go-gitignore"
"math/rand"
"path/filepath"
"strings"
Expand Down Expand Up @@ -191,7 +192,7 @@ func TestWalkDirectory(t *testing.T) {
Debug = true
Exclude = []string{"test"}
ProcessConstants()
files := walkDirectory(".", []string{}, ExtensionToLanguage)
files := walkDirectory(".", []string{}, ExtensionToLanguage, []gitignore.IgnoreMatcher{})

if len(files) == 0 {
t.Error("Expected at least one file")
Expand Down
36 changes: 34 additions & 2 deletions test-all.sh
View file
Expand Up @@ -249,7 +249,7 @@ fi

# Turn off gitignore https://github.com/boyter/scc/issues/53
touch ignored.xml
a=$(./scc | grep Total)
a=$(./scc | grep Total)
b=$(./scc --no-gitignore | grep Total)
if [ "$a" == "$b" ]; then
echo -e "${RED}======================================================="
Expand All @@ -260,9 +260,11 @@ else
echo -e "${GREEN}PASSED git ignore filter"
fi

a=$(./scc | grep Total)
a=$(./scc | grep Total)
b=$(./scc --no-ignore | grep Total)
if [ "$a" == "$b" ]; then
echo "$a"
echo "$b"
echo -e "${RED}======================================================="
echo -e "FAILED ignore filter"
echo -e "=================================================${NC}"
Expand All @@ -271,6 +273,34 @@ else
echo -e "${GREEN}PASSED ignore filter"
fi

touch ./examples/ignore/ignorefile.txt
a=$(./scc --by-file | grep ignorefile)
b=$(./scc --by-file --no-ignore | grep ignorefile)
if [ "$a" == "$b" ]; then
echo "$a"
echo "$b"
echo -e "${RED}======================================================="
echo -e "FAILED ignore recursive filter"
echo -e "=================================================${NC}"
exit
else
echo -e "${GREEN}PASSED ignore recursive filter"
fi

touch ./examples/ignore/gitignorefile.txt
a=$(./scc --by-file | grep gitignorefile)
b=$(./scc --by-file --no-gitignore | grep gitignorefile)
if [ "$a" == "$b" ]; then
echo "$a"
echo "$b"
echo -e "${RED}======================================================="
echo -e "FAILED gitignore recursive filter"
echo -e "=================================================${NC}"
exit
else
echo -e "${GREEN}PASSED gitignore recursive filter"
fi

# Try out specific languages
for i in 'Bosque ' 'Flow9 ' 'Bitbucket Pipeline ' 'Docker ignore ' 'Q# ' 'Futhark ' 'Alloy ' 'Wren ' 'Monkey C ' 'Alchemist ' 'Luna ' 'ignore '
do
Expand All @@ -288,6 +318,8 @@ echo -e "${NC}Cleaning up..."
rm ./scc
rm ./ignored.xml
rm .tmp_scc_yaml
rm ./examples/ignore/gitignorefile.txt
rm ./examples/ignore/ignorefile.txt

echo -e "${GREEN}================================================="
echo -e "ALL TESTS PASSED"
Expand Down

0 comments on commit d9eeadd

Please sign in to comment.