feat: ignore minified JS files (#1025)

new/detector/composition/testhelper/testhelper.go

@@ -15,6 +15,7 @@ import (
 	"github.com/bearer/bearer/pkg/report/output"
 	"github.com/bearer/bearer/pkg/types"
 	"github.com/bradleyjkemp/cupaloy"
+	"github.com/hhatto/gocloc"
 	"gopkg.in/yaml.v3"
 )
 
@@ -76,7 +77,16 @@ func getRulesFromYaml(t *testing.T, ruleBytes []byte) map[string]*settings.Rule
 }
 
 func (runner *Runner) RunTest(t *testing.T, testdataPath string, snapshotPath string) {
-	files, err := filelist.Discover(testdataPath, runner.config)
+	dummyGoclocLanguage := gocloc.Language{}
+	dummyGoclocResult := gocloc.Result{
+		Total: &dummyGoclocLanguage,
+		Files: map[string]*gocloc.ClocFile{
+			testdataPath: {Code: 10},
+		},
+		Languages:     map[string]*gocloc.Language{},
+		MaxPathLength: 0,
+	}
+	files, err := filelist.Discover(testdataPath, &dummyGoclocResult, runner.config)
 	if err != nil {
 		t.Fatalf("failed to discover files: %s", err)
 	}

pkg/commands/artifact/run.go

@@ -65,12 +65,13 @@ type Runner interface {
 type runner struct {
 	reportPath     string
 	reuseDetection bool
+	goclocResult   *gocloc.Result
 	scanSettings   settings.Config
 }
 
 // NewRunner initializes Runner that provides scanning functionalities.
-func NewRunner(ctx context.Context, scanSettings settings.Config) Runner {
-	r := &runner{scanSettings: scanSettings}
+func NewRunner(ctx context.Context, scanSettings settings.Config, goclocResult *gocloc.Result) Runner {
+	r := &runner{scanSettings: scanSettings, goclocResult: goclocResult}
 
 	scanID, err := buildScanID(scanSettings)
 	if err != nil {
@@ -201,6 +202,7 @@ func (r *runner) scanArtifact(ctx context.Context, opts flag.Options) (types.Rep
 				CommitSHA:         "",
 			},
 			r.scanSettings,
+			r.goclocResult,
 			r.reportPath,
 		); err != nil {
 			return types.Report{}, err
@@ -240,7 +242,7 @@ func Run(ctx context.Context, opts flag.Options, targetKind TargetKind) (err err
 		}
 	}()
 
-	r := NewRunner(ctx, scanSettings)
+	r := NewRunner(ctx, scanSettings, inputgocloc)
 	defer r.Close(ctx)
 
 	if !r.CacheUsed() && scanSettings.CacheUsed {

pkg/commands/process/orchestrator/fileignore/fileignore.go

@@ -2,11 +2,13 @@ package fileignore
 
 import (
 	"bytes"
+	"fmt"
 	"io/fs"
 	"os"
 	"strings"
 
 	"github.com/bearer/bearer/pkg/commands/process/settings"
+	"github.com/hhatto/gocloc"
 	"github.com/monochromegane/go-gitignore"
 	"github.com/rs/zerolog/log"
 )
@@ -25,7 +27,7 @@ func New(projectPath string, config settings.Config) *FileIgnore {
 	}
 }
 
-func (fileignore *FileIgnore) Ignore(projectPath string, filePath string, d fs.DirEntry) bool {
+func (fileignore *FileIgnore) Ignore(projectPath string, filePath string, goclocResult *gocloc.Result, d fs.DirEntry) bool {
 	relativePath := strings.TrimPrefix(filePath, projectPath)
 	trimmedPath := strings.TrimPrefix(relativePath, "/")
 
@@ -51,6 +53,32 @@ func (fileignore *FileIgnore) Ignore(projectPath string, filePath string, d fs.D
 			log.Debug().Msgf("skipping file due to size: %s %s", projectPath, relativePath)
 			return true
 		}
+		if isMinified(fmt.Sprintf("%s%s", projectPath, filePath), fileInfo.Size(), goclocResult) {
+			log.Debug().Msgf("skipping file (suspected minified JS): %s%s", projectPath, filePath)
+			return true
+		}
+	}
+
+	return false
+}
+
+func isMinified(fullPath string, size int64, goclocResult *gocloc.Result) bool {
+	if strings.HasSuffix(fullPath, ".min.js") {
+		return true
+	}
+
+	if strings.HasSuffix(fullPath, ".js") {
+		goclocFileResult := goclocResult.Files[fullPath]
+
+		if goclocFileResult == nil {
+			// couldn't find file
+			return false
+		}
+
+		if goclocFileResult.Blanks == 0 && goclocFileResult.Comments == 0 && size > int64(5000) {
+			// > 5KB JS file with no blank lines or comments -> assume minified
+			return true
+		}
 	}
 
 	return false

pkg/commands/process/orchestrator/filelist/filelist.go

@@ -11,11 +11,12 @@ import (
 	"github.com/bearer/bearer/pkg/commands/process/settings"
 	"github.com/bearer/bearer/pkg/commands/process/worker/work"
 	"github.com/bearer/bearer/pkg/util/gitutil"
+	"github.com/hhatto/gocloc"
 	"github.com/rs/zerolog/log"
 )
 
 // Discover searches directory for files to scan, skipping the ones specified by skip config and assigning timeout speficfied by timeout config
-func Discover(projectPath string, config settings.Config) ([]work.File, error) {
+func Discover(projectPath string, goclocResult *gocloc.Result, config settings.Config) ([]work.File, error) {
 	var files []work.File
 
 	haveDir, statErr := isDir(projectPath)
@@ -52,7 +53,7 @@ func Discover(projectPath string, config settings.Config) ([]work.File, error) {
 				continue
 			}
 			fileEntry := fs.FileInfoToDirEntry(fileInfo)
-			if ignore.Ignore(projectPath, pathFromGit, fileEntry) {
+			if ignore.Ignore(projectPath, pathFromGit, goclocResult, fileEntry) {
 				log.Debug().Msgf("Skipping file: %s", pathFromGit)
 				continue
 			}
@@ -65,7 +66,7 @@ func Discover(projectPath string, config settings.Config) ([]work.File, error) {
 			}
 			dirEntry := fs.FileInfoToDirEntry(dirInfo)
 			dirRelativePath := filepath.Dir(pathFromGit)
-			if ignore.Ignore(projectPath, dirRelativePath, dirEntry) {
+			if ignore.Ignore(projectPath, dirRelativePath, goclocResult, dirEntry) {
 				log.Debug().Msgf("Skipping parent directory: %s", dirRelativePath)
 				continue
 			}
@@ -92,7 +93,7 @@ func Discover(projectPath string, config settings.Config) ([]work.File, error) {
 		}
 
 		if d.IsDir() {
-			if ignore.Ignore(projectPath, filePath, d) {
+			if ignore.Ignore(projectPath, filePath, goclocResult, d) {
 				return filepath.SkipDir
 			}
 
@@ -102,7 +103,7 @@ func Discover(projectPath string, config settings.Config) ([]work.File, error) {
 		relativePath := strings.TrimPrefix(filePath, projectPath)
 		relativePath = "/" + relativePath
 
-		if ignore.Ignore(projectPath, filePath, d) {
+		if ignore.Ignore(projectPath, filePath, goclocResult, d) {
 			log.Debug().Msgf("skipping file due to file skip rules: %s", relativePath)
 
 			return nil

pkg/commands/process/orchestrator/filelist/filelist_test.go

@@ -8,6 +8,7 @@ import (
 	"github.com/bearer/bearer/pkg/commands/process/settings"
 	"github.com/bearer/bearer/pkg/commands/process/worker/work"
 	"github.com/bearer/bearer/pkg/flag"
+	"github.com/hhatto/gocloc"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -80,11 +81,39 @@ func TestFileList(t *testing.T) {
 			},
 			Want: nil,
 		},
+		{
+			Name: "Find files - skip - dir - happy path",
+			Input: input{
+				projectPath: filepath.Join("testdata", "happy_path", "skip"),
+				config: settings.Config{
+					Scan: flag.ScanOptions{
+						SkipPath: []string{"users"},
+					},
+					Worker: settings.WorkerOptions{
+						FileSizeMaximum:           100000,
+						TimeoutFileBytesPerSecond: 1,
+					},
+				},
+			},
+			Want: nil,
+		},
 	}
 
 	for _, testCase := range tests {
+		dummyGoclocLanguage := gocloc.Language{}
+		dummyGoclocResult := gocloc.Result{
+			Total: &dummyGoclocLanguage,
+			Files: map[string]*gocloc.ClocFile{
+				"minifiedJs.min.js": {Code: 2},
+				"users.go":          {Code: 6},
+				"user.go":           {Code: 0},
+				"admin.go":          {Code: 6},
+			},
+			Languages:     map[string]*gocloc.Language{},
+			MaxPathLength: 0,
+		}
 		t.Run(testCase.Name, func(t *testing.T) {
-			output, err := filelist.Discover(testCase.Input.projectPath, testCase.Input.config)
+			output, err := filelist.Discover(testCase.Input.projectPath, &dummyGoclocResult, testCase.Input.config)
 
 			if testCase.WantError {
 				if err == nil {

pkg/commands/process/orchestrator/filelist/testdata/happy_path/skip/users/minifiedJs.min.js

@@ -0,0 +1 @@
+(()=>{var e=(()=>{})})

pkg/commands/process/orchestrator/orchestrator.go

@@ -7,6 +7,7 @@ import (
 	"path"
 	"sync"
 
+	"github.com/hhatto/gocloc"
 	"github.com/rs/zerolog/log"
 	"github.com/schollz/progressbar/v3"
 
@@ -40,14 +41,15 @@ type orchestrator struct {
 func newOrchestrator(
 	repository work.Repository,
 	config settings.Config,
+	goclocResult *gocloc.Result,
 	reportPath string,
 ) (*orchestrator, error) {
 	reportFile, err := os.Create(reportPath)
 	if err != nil {
 		return nil, err
 	}
 
-	files, err := filelist.Discover(config.Scan.Target, config)
+	files, err := filelist.Discover(config.Scan.Target, goclocResult, config)
 	if err != nil {
 		reportFile.Close()
 		return nil, err
@@ -171,12 +173,12 @@ func (orchestrator *orchestrator) writeFileError(file work.File, fileErr error)
 	orchestrator.reportMutex.Unlock()
 }
 
-func Scan(repository work.Repository, config settings.Config, reportPath string) error {
+func Scan(repository work.Repository, config settings.Config, goclogResult *gocloc.Result, reportPath string) error {
 	if !config.Scan.Quiet {
 		output.StdErrLogger().Msgf("Scanning target %s", config.Scan.Target)
 	}
 
-	orchestrator, err := newOrchestrator(repository, config, reportPath)
+	orchestrator, err := newOrchestrator(repository, config, goclogResult, reportPath)
 	if err != nil {
 		return err
 	}

pkg/report/output/output.go

@@ -22,6 +22,7 @@ import (
 	pointer "github.com/bearer/bearer/pkg/util/pointers"
 	"github.com/gitsight/go-vcsurl"
 	"github.com/google/uuid"
+	"github.com/hhatto/gocloc"
 	"github.com/rs/zerolog/log"
 
 	dataflowtypes "github.com/bearer/bearer/pkg/report/output/dataflow/types"
@@ -76,7 +77,7 @@ func GetOutput(report types.Report, config settings.Config) (any, *dataflow.Data
 			}
 		}
 
-		files := getDiscoveredFiles(config)
+		files := getDiscoveredFiles(config, report.Inputgocloc)
 
 		return BearerReport{
 			Findings:   securityResults,
@@ -94,8 +95,8 @@ func GetOutput(report types.Report, config settings.Config) (any, *dataflow.Data
 	return nil, nil, fmt.Errorf(`--report flag "%s" is not supported`, config.Report.Report)
 }
 
-func getDiscoveredFiles(config settings.Config) []string {
-	filesDiscovered, _ := filelist.Discover(config.Scan.Target, config)
+func getDiscoveredFiles(config settings.Config, goclocResult *gocloc.Result) []string {
+	filesDiscovered, _ := filelist.Discover(config.Scan.Target, goclocResult, config)
 	files := []string{}
 	for _, fileDiscovered := range filesDiscovered {
 		files = append(files, file.GetFullFilename(config.Scan.Target, fileDiscovered.FilePath))
@@ -176,7 +177,7 @@ func reportSecurity(
 			return
 		}
 
-		tmpDir, filename, err := createBearerGzipFileReport(config, meta, securityResults, dataflow)
+		tmpDir, filename, err := createBearerGzipFileReport(config, meta, securityResults, report.Inputgocloc, dataflow)
 		if err != nil {
 			config.Client.Error = pointer.String("Could not compress report.")
 			log.Debug().Msgf("error creating report %s", err)
@@ -220,6 +221,7 @@ func createBearerGzipFileReport(
 	config settings.Config,
 	meta *Meta,
 	securityResults *security.Results,
+	goclocResult *gocloc.Result,
 	dataflow *dataflow.DataFlow,
 ) (*string, *string, error) {
 	tempDir, err := os.MkdirTemp("", "reports")
@@ -232,7 +234,7 @@ func createBearerGzipFileReport(
 		return &tempDir, nil, err
 	}
 
-	files := getDiscoveredFiles(config)
+	files := getDiscoveredFiles(config, goclocResult)
 
 	content, _ := ReportJSON(&BearerReport{
 		Findings:   securityResults,