feat: support config ignore patterns

cmd/config/config.go

@@ -33,15 +33,16 @@ type OriginConfig struct {
 }
 
 type OptionalConfig struct {
-	UI          bool   `json:"ui"`
-	Dedup       bool   `json:"dedup"`
-	DirOnly     bool   `json:"dir"`
-	VulnOnly    bool   `json:"vuln"`
-	SaveDev     bool   `json:"dev"`
-	ProgressBar bool   `json:"progress"`
-	TLSVerify   bool   `json:"tls"`
-	Proxy       string `json:"proxy"`
-	Dynamic     bool   `json:"dynamic"`
+	UI          bool     `json:"ui"`
+	Dedup       bool     `json:"dedup"`
+	DirOnly     bool     `json:"dir"`
+	VulnOnly    bool     `json:"vuln"`
+	SaveDev     bool     `json:"dev"`
+	ProgressBar bool     `json:"progress"`
+	TLSVerify   bool     `json:"tls"`
+	Proxy       string   `json:"proxy"`
+	Dynamic     bool     `json:"dynamic"`
+	Ignore      []string `json:"ignore"`
 }
 
 type RepoConfig struct {

config.json

@@ -30,6 +30,10 @@
     // only detect directory (skip compress file)
     "dir": false,
 
+    // 忽略指定路径(兼容 .gitignore 语法)
+    // ignore paths (compatible with .gitignore syntax)
+    "ignore": [],
+
     // 仅保留漏洞组件
     // only save components with vulnerability
     "vuln": false,

go.mod

@@ -13,6 +13,7 @@ require (
 	github.com/nwaples/rardecode v1.1.3
 	github.com/pkg/errors v0.9.1
 	github.com/rivo/tview v0.0.0-20231126152417-33a1d271f2b6
+	github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06
 	github.com/titanous/json5 v1.0.0
 	github.com/veraison/swid v1.1.0
 	golang.org/x/term v0.14.0

go.sum

@@ -7,6 +7,7 @@ github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYr
 github.com/axgle/mahonia v0.0.0-20180208002826-3358181d7394 h1:OYA+5W64v3OgClL+IrOD63t4i/RW7RqrAVl9LTZ9UqQ=
 github.com/axgle/mahonia v0.0.0-20180208002826-3358181d7394/go.mod h1:Q8n74mJTIgjX4RBBcHnJ05h//6/k6foqmgE45jTQtxg=
 github.com/bradleyjkemp/cupaloy/v2 v2.8.0 h1:any4BmKE+jGIaMpnU8YgH/I2LPiLBufr6oMMlVBbn9M=
+github.com/bradleyjkemp/cupaloy/v2 v2.8.0/go.mod h1:bm7JXdkRd4BHJk9HpwqAI8BoAY1lps46Enkdqw6aRX0=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -25,6 +26,7 @@ github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9
 github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI=
 github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
 github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ=
+github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo=
 github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU=
 github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
@@ -40,6 +42,7 @@ github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/
 github.com/jlaffaye/ftp v0.2.0 h1:lXNvW7cBu7R/68bknOX3MrRIIqZ61zELs1P2RAiA3lg=
 github.com/jlaffaye/ftp v0.2.0/go.mod h1:is2Ds5qkhceAPy2xD6RLI6hmp/qysSoymZ+Z2uTnspI=
 github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
+github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY=
 github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
@@ -64,20 +67,27 @@ github.com/rivo/uniseg v0.4.3/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUc
 github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis=
 github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
 github.com/robertkrimen/otto v0.2.1 h1:FVP0PJ0AHIjC+N4pKCG9yCDz6LHNPCwi/GKID5pGGF0=
+github.com/robertkrimen/otto v0.2.1/go.mod h1:UPwtJ1Xu7JrLcZjNWN8orJaM5n5YEtqL//farB5FlRY=
+github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 h1:OkMGxebDjyw0ULyrTYWeN0UNCCkmCWfjPnIA2W6oviI=
+github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06/go.mod h1:+ePHsJ1keEjQtpvf9HHw0f4ZeJ0TLRsxhunSI2hYJSs=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/terminalstatic/go-xsd-validate v0.1.5 h1:RqpJnf6HGE2CB/lZB1A8BYguk8uRtcvYAPLCF15qguo=
+github.com/terminalstatic/go-xsd-validate v0.1.5/go.mod h1:18lsvYFofBflqCrvo1umpABZ99+GneNTw2kEEc8UPJw=
 github.com/titanous/json5 v1.0.0 h1:hJf8Su1d9NuI/ffpxgxQfxh/UiBFZX7bMPid0rIL/7s=
 github.com/titanous/json5 v1.0.0/go.mod h1:7JH1M8/LHKc6cyP5o5g3CSaRj+mBrIimTxzpvmckH8c=
 github.com/veraison/swid v1.1.0 h1:jEf/jobG6j7r9W9HSj2jDi1IGGs7aMKyDgfGEMxQ6is=
 github.com/veraison/swid v1.1.0/go.mod h1:d5jt76uMNbTfQ+f2qU4Lt8RvWOTsv6PFgstIM1QdMH0=
 github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
 github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
 github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c=
+github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
 github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0=
+github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
 github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74=
+github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
@@ -114,6 +124,7 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/sourcemap.v1 v1.0.5 h1:inv58fC9f9J3TK2Y2R1NPntXEn3/wjWHkonhIUODNTI=
+gopkg.in/sourcemap.v1 v1.0.5/go.mod h1:2RlvNNSMglmRrcvhfuzp4hQHwOtjxlbjX7UPY/GXb78=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

main.go

@@ -21,6 +21,7 @@ import (
 	"github.com/xmirrorsecurity/opensca-cli/v3/opensca/common"
 	"github.com/xmirrorsecurity/opensca-cli/v3/opensca/logs"
 	"github.com/xmirrorsecurity/opensca-cli/v3/opensca/model"
+	"github.com/xmirrorsecurity/opensca-cli/v3/opensca/sca/filter"
 	"github.com/xmirrorsecurity/opensca-cli/v3/opensca/sca/java"
 	"github.com/xmirrorsecurity/opensca-cli/v3/opensca/sca/javascript"
 	"github.com/xmirrorsecurity/opensca-cli/v3/opensca/sca/php"
@@ -53,6 +54,7 @@ func main() {
 	if config.Conf().Optional.DirOnly {
 		arg.ExtractFileFilter = func(relpath string) bool { return false }
 	}
+	arg.IgnoreFileFilter = filter.IgnorePatterns(config.Conf().Optional.Ignore)
 
 	// 开启进度条
 	var stopProgress func()
@@ -123,8 +125,6 @@ func args() {
 	flag.Parse()
 
 	if v {
-		fmt.Println(logo)
-		fmt.Println("Current version: ", version)
 		os.Exit(0)
 	}
 

opensca/run.go

@@ -27,6 +27,8 @@ type TaskArg struct {
 
 	// 额外的文件过滤函数 默认为压缩文件名过滤函数
 	ExtractFileFilter walk.ExtractFileFilter
+	// 忽略文件过滤函数 命中时跳过扫描
+	IgnoreFileFilter walk.ExtractFileFilter
 	// 额外的结果回调函数
 	ResCallFunc model.ResCallback
 }
@@ -94,7 +96,7 @@ func RunTask(ctx context.Context, arg *TaskArg) (result TaskResult) {
 
 		return false
 
-	}, func(parent *model.File, files []*model.File) {
+	}, arg.IgnoreFileFilter, func(parent *model.File, files []*model.File) {
 
 		for _, sca := range arg.Sca {
 

opensca/sca/filter/filter.go

@@ -3,6 +3,8 @@ package filter
 import (
 	"path/filepath"
 	"strings"
+
+	gitignore "github.com/sabhiram/go-gitignore"
 )
 
 func filterFunc(strFunc func(string, string) bool, args ...string) func(string) bool {
@@ -88,3 +90,31 @@ var (
 		".bz2",
 	)
 )
+
+func IgnorePatterns(patterns []string) func(string) bool {
+	if len(patterns) == 0 {
+		return nil
+	}
+	matcher := gitignore.CompileIgnoreLines(patterns...)
+	return func(filename string) bool {
+		for _, candidate := range ignoreCandidates(filename) {
+			if matcher.MatchesPath(candidate) {
+				return true
+			}
+		}
+		return false
+	}
+}
+
+func ignoreCandidates(filename string) []string {
+	filename = filepath.ToSlash(filepath.Clean(filename))
+	filename = strings.TrimPrefix(filename, "./")
+	if filename == "." || filename == "" {
+		return nil
+	}
+	candidates := []string{filename}
+	if i := strings.Index(filename, "/"); i != -1 && i+1 < len(filename) {
+		candidates = append(candidates, filename[i+1:])
+	}
+	return candidates
+}

opensca/walk/walk.go

@@ -23,7 +23,7 @@ type WalkFileFunc func(parent *model.File, files []*model.File)
 // filter: 过滤需要提取的文件
 // do: 对文件的操作
 // size: 检测文件大小
-func Walk(ctx context.Context, name, origin string, filter ExtractFileFilter, do WalkFileFunc) (size int64, err error) {
+func Walk(ctx context.Context, name, origin string, filter ExtractFileFilter, ignore ExtractFileFilter, do WalkFileFunc) (size int64, err error) {
 
 	delete, file, err := download(origin)
 	if err != nil {
@@ -52,12 +52,12 @@ func Walk(ctx context.Context, name, origin string, filter ExtractFileFilter, do
 
 	parent := model.NewFile(file, name)
 	wg := &sync.WaitGroup{}
-	err = walk(ctx, wg, parent, filter, do)
+	err = walk(ctx, wg, parent, filter, ignore, do)
 	wg.Wait()
 	return
 }
 
-func walk(ctx context.Context, wg *sync.WaitGroup, parent *model.File, filterFunc ExtractFileFilter, walkFunc WalkFileFunc) error {
+func walk(ctx context.Context, wg *sync.WaitGroup, parent *model.File, filterFunc, ignoreFunc ExtractFileFilter, walkFunc WalkFileFunc) error {
 
 	var files []*model.File
 
@@ -73,15 +73,22 @@ func walk(ctx context.Context, wg *sync.WaitGroup, parent *model.File, filterFun
 			logs.Warn(err)
 			return nil
 		}
+
+		rel := filepath.Join(parent.Relpath(), strings.TrimPrefix(path, parent.Abspath()))
+		if path != parent.Abspath() && ignoreFunc != nil && ignoreFunc(rel) {
+			if info.IsDir() {
+				return filepath.SkipDir
+			}
+			return nil
+		}
+
 		if info.IsDir() {
 			if strings.HasSuffix(path, ".git") || strings.HasSuffix(path, ".opensca-cache") || strings.HasSuffix(path, ".temp") {
 				return filepath.SkipDir
 			}
 			return nil
 		}
 
-		rel := filepath.Join(parent.Relpath(), strings.TrimPrefix(path, parent.Abspath()))
-
 		if filterFunc != nil && !filterFunc(rel) {
 			return nil
 		}
@@ -98,7 +105,7 @@ func walk(ctx context.Context, wg *sync.WaitGroup, parent *model.File, filterFun
 				defer wg.Done()
 				defer os.RemoveAll(dir)
 				parent := model.NewFile(dir, rel)
-				if err := walk(ctx, wg, parent, filterFunc, walkFunc); err != nil {
+				if err := walk(ctx, wg, parent, filterFunc, ignoreFunc, walkFunc); err != nil {
 					logs.Warn(err)
 				}
 			}()