-
-
Notifications
You must be signed in to change notification settings - Fork 248
/
file.go
167 lines (143 loc) · 3.69 KB
/
file.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
// SPDX-License-Identifier: MIT OR Unlicense
package processor
import (
"fmt"
"os"
"path/filepath"
"strings"
"sync"
)
// Used as quick lookup for files with the same name to avoid some processing
// needs to be sync.Map as it potentially could be called by many GoRoutines
var extensionCache sync.Map
// A custom version of extracting extensions for a file
// which also has a case insensitive cache in order to save
// some needless processing
func getExtension(name string) string {
name = strings.ToLower(name)
extension, ok := extensionCache.Load(name)
if ok {
return extension.(string)
}
ext := filepath.Ext(name)
if ext == "" || strings.LastIndex(name, ".") == 0 {
extension = name
} else {
// Handling multiple dots or multiple extensions only needs to delete the last extension
// and then call filepath.Ext.
// If there are multiple extensions, it is the value of subExt,
// otherwise subExt is an empty string.
subExt := filepath.Ext(strings.TrimSuffix(name, ext))
extension = strings.TrimPrefix(subExt+ext, ".")
}
extensionCache.Store(name, extension)
return extension.(string)
}
func newFileJob(path, name string, fileInfo os.FileInfo) *FileJob {
if NoLarge {
if fileInfo.Size() >= LargeByteCount {
if Verbose {
printWarn(fmt.Sprintf("skipping large file due to byte size: %s", path))
}
return nil
}
}
var symPath = ""
// Check if the file is a symlink and if we want to count those then work out its path and rejig
// everything so we can count the real file to ensure the counts are correct
if fileInfo.Mode()&os.ModeSymlink == os.ModeSymlink {
if !IncludeSymLinks {
if Verbose {
printWarn(fmt.Sprintf("skipping symlink file: %s", name))
}
return nil
}
var err error
symPath, err = filepath.EvalSymlinks(path)
if err != nil {
printError(err.Error())
return nil
}
fileInfo, err = os.Lstat(symPath)
if err != nil {
printError(err.Error())
return nil
}
}
// Skip non-regular files. They are unlikely to be code and may hang if we
// try and read them.
if !fileInfo.Mode().IsRegular() {
if Verbose {
printWarn(fmt.Sprintf("skipping non-regular file: %s", path))
}
return nil
}
language, extension := DetectLanguage(name)
if len(language) != 0 {
// check if extensions in the allow list, which should limit to just those extensions
if len(AllowListExtensions) != 0 {
ok := false
for _, x := range AllowListExtensions {
if x == extension {
ok = true
}
}
if !ok {
if Verbose {
printWarn(fmt.Sprintf("skipping file as not in allow list: %s", name))
}
return nil
}
}
// check if we should exclude this type
if len(ExcludeListExtensions) != 0 {
ok := true
for _, x := range ExcludeListExtensions {
if x == extension {
ok = false
}
}
if !ok {
if Verbose {
printWarn(fmt.Sprintf("skipping file as in exclude list: %s", name))
}
return nil
}
}
if len(ExcludeFilename) != 0 {
ok := true
for _, x := range ExcludeFilename {
if strings.Contains(name, x) {
ok = false
}
}
if !ok {
if Verbose {
printWarn(fmt.Sprintf("skipping file as in exclude file list: %s", name))
}
return nil
}
}
for _, l := range language {
LoadLanguageFeature(l)
}
if !CountIgnore {
for _, l := range language {
if l == "ignore" || l == "gitignore" {
return nil
}
}
}
return &FileJob{
Location: path,
Symlocation: symPath,
Filename: name,
Extension: extension,
PossibleLanguages: language,
Bytes: fileInfo.Size(),
}
} else if Verbose {
printWarn(fmt.Sprintf("skipping file unknown extension: %s", name))
}
return nil
}