add table output for cli

Dockerfile

@@ -31,5 +31,5 @@ COPY --from=builder /home/deepfence/src/SecretScanner/SecretScanner .
 COPY --from=builder /home/deepfence/src/SecretScanner/config.yaml .
 WORKDIR /home/deepfence/output
 
-ENTRYPOINT ["/home/deepfence/usr/SecretScanner", "-config-path", "/home/deepfence/usr", "-quiet"]
+ENTRYPOINT ["/home/deepfence/usr/SecretScanner", "-config-path", "/home/deepfence/usr"]
 CMD ["-h"]

Makefile

@@ -12,3 +12,7 @@ SecretScanner: $(PWD)/**/*.go $(PWD)/agent-plugins-grpc/**/*.go
 	go build -ldflags="-extldflags=-static" -buildvcs=false -v .
 
 .PHONY: clean bootstrap
+
+.PHONY: docker
+docker:
+	docker build -t deepfenceio/deepfence_secret_scanner:latest .

core/options.go

@@ -9,6 +9,8 @@ import (
 const (
 	TempDirSuffix          = "SecretScanning"
 	ExtractedImageFilesDir = "ExtractedFiles"
+	JsonOutput             = "json"
+	TableOutput            = "table"
 )
 
 type Options struct {
@@ -20,17 +22,22 @@ type Options struct {
 	HostMountPath     *string
 	ConfigPath        *repeatableStringValue
 	MergeConfigs      *bool
-	OutputPath        *string
-	JsonFilename      *string
 	ImageName         *string
 	MultipleMatch     *bool
 	MaxMultiMatch     *uint
 	MaxSecrets        *uint
 	ContainerId       *string
 	ContainerNS       *string
-	Quiet             *bool
 	WorkersPerScan    *int
 	InactiveThreshold *int
+	OutFormat         *string
+	ConsoleUrl        *string
+	ConsolePort       *int
+	DeepfenceKey      *string
+	FailOnCount       *int
+	FailOnHighCount   *int
+	FailOnMediumCount *int
+	FailOnLowCount    *int
 }
 
 type repeatableStringValue struct {
@@ -60,17 +67,22 @@ func ParseOptions() (*Options, error) {
 		HostMountPath:     flag.String("host-mount-path", "", "If scanning the host, specify the host mount path for path exclusions to work correctly."),
 		ConfigPath:        &repeatableStringValue{},
 		MergeConfigs:      flag.Bool("merge-configs", false, "Merge config files specified by --config-path into the default config"),
-		OutputPath:        flag.String("output-path", ".", "Output directory where json file will be stored. If not set, it will output to current directory"),
-		JsonFilename:      flag.String("json-filename", "", "Output json file name. If not set, it will automatically create a filename based on image or dir name"),
 		ImageName:         flag.String("image-name", "", "Name of the image along with tag to scan for secrets"),
 		MultipleMatch:     flag.Bool("multi-match", false, "Output multiple matches of same pattern in one file. By default, only one match of a pattern is output for a file for better performance"),
 		MaxMultiMatch:     flag.Uint("max-multi-match", 3, "Maximum number of matches of same pattern in one file. This is used only when multi-match option is enabled."),
 		MaxSecrets:        flag.Uint("max-secrets", 1000, "Maximum number of secrets to find in one container image or file system."),
 		ContainerId:       flag.String("container-id", "", "Id of existing container ID"),
 		ContainerNS:       flag.String("container-ns", "", "Namespace of existing container to scan, empty for docker runtime"),
-		Quiet:             flag.Bool("quiet", false, "Don't display any output in stdout"),
-		WorkersPerScan:    flag.Int("workers-per-scan", 1, "Number of concrrent workers per scan"),
+		WorkersPerScan:    flag.Int("workers-per-scan", 1, "Number of concurrent workers per scan"),
 		InactiveThreshold: flag.Int("inactive-threshold", 600, "Threshold for Inactive scan in seconds"),
+		OutFormat:         flag.String("output", TableOutput, "Output format: json or table"),
+		ConsoleUrl:        flag.String("console-url", "", "Deepfence Management Console URL"),
+		ConsolePort:       flag.Int("console-port", 443, "Deepfence Management Console Port"),
+		DeepfenceKey:      flag.String("deepfence-key", "", "Deepfence key for auth"),
+		FailOnCount:       flag.Int("fail-on-count", -1, "Exit with status 1 if number of secrets found is >= this value (Default: -1)"),
+		FailOnHighCount:   flag.Int("fail-on-high-count", -1, "Exit with status 1 if number of high secrets found is >= this value (Default: -1)"),
+		FailOnMediumCount: flag.Int("fail-on-medium-count", -1, "Exit with status 1 if number of medium secrets found is >= this value (Default: -1)"),
+		FailOnLowCount:    flag.Int("fail-on-low-count", -1, "Exit with status 1 if number of low secrets found is >= this value (Default: -1)"),
 	}
 	flag.Var(options.ConfigPath, "config-path", "Searches for config.yaml from given directory. If not set, tries to find it from SecretScanner binary's and current directory.  Can be specified multiple times.")
 	flag.Parse()

core/util.go

@@ -48,30 +48,6 @@ func getSanitizedString(imageName string) string {
 	return sanitizedName
 }
 
-// GetJsonFilepath Return complete path and filename for json output file
-// @parameters
-// image - Name of the container image or dir, for which json filename and path will be created
-// @returns
-// string - Sanitized string which can used as path and filename of json output file
-// Error - Errors if path can't be created. Otherwise, returns nil
-func GetJsonFilepath(input string) (string, error) {
-	outputDir := *GetSession().Options.OutputPath
-	JsonFilename := *GetSession().Options.JsonFilename
-	if !PathExists(outputDir) {
-		err := CreateRecursiveDir(outputDir)
-		if err != nil {
-			GetSession().Log.Error("GetJsonFilepath: Could not create output dir: %s", err)
-			return "", err
-		}
-	}
-	if JsonFilename == "" {
-		JsonFilename = getSanitizedString(input) + "-secrets.json"
-	}
-	jsonFilePath := filepath.Join(outputDir, JsonFilename)
-	GetSession().Log.Info("Complete json file path and name: %s", jsonFilePath)
-	return jsonFilePath, nil
-}
-
 // GetTmpDir Create a temporrary directory to extract the conetents of container image
 // @parameters
 // imageName - Name of the container image
@@ -124,7 +100,6 @@ func DeleteTmpDir(outputDir string) error {
 // path - Directory whose contents need to be deleted
 // wildcard - patterns to match the filenames (e.g. '*')
 func DeleteFiles(path string, wildCard string) {
-
 	var val string
 	files, _ := filepath.Glob(path + wildCard)
 	for _, val = range files {

go.mod

@@ -7,9 +7,13 @@ replace github.com/deepfence/agent-plugins-grpc => ./agent-plugins-grpc
 require (
 	github.com/Jeffail/tunny v0.1.4
 	github.com/deepfence/agent-plugins-grpc v0.0.0-00010101000000-000000000000
+	github.com/deepfence/golang_deepfence_sdk/client v0.0.0-20230630084500-8fb0280d6010
+	github.com/deepfence/golang_deepfence_sdk/utils v0.0.0-20230630084500-8fb0280d6010
 	github.com/deepfence/vessel v0.11.1
 	github.com/fatih/color v1.15.0
 	github.com/flier/gohs v1.2.2
+	github.com/olekukonko/tablewriter v0.0.5
+	github.com/sirupsen/logrus v1.9.3
 	google.golang.org/grpc v1.56.1
 	gopkg.in/yaml.v3 v3.0.1
 )
@@ -39,9 +43,12 @@ require (
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/google/go-cmp v0.5.9 // indirect
 	github.com/google/uuid v1.3.0 // indirect
+	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
+	github.com/hashicorp/go-retryablehttp v0.7.4 // indirect
 	github.com/klauspost/compress v1.16.0 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.17 // indirect
+	github.com/mattn/go-runewidth v0.0.9 // indirect
 	github.com/moby/locker v1.0.1 // indirect
 	github.com/moby/sys/mountinfo v0.6.2 // indirect
 	github.com/moby/sys/sequential v0.5.0 // indirect
@@ -52,7 +59,6 @@ require (
 	github.com/opencontainers/runtime-spec v1.1.0-rc.1 // indirect
 	github.com/opencontainers/selinux v1.11.0 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
-	github.com/sirupsen/logrus v1.9.3 // indirect
 	go.opencensus.io v0.24.0 // indirect
 	go.opentelemetry.io/otel v1.14.0 // indirect
 	go.opentelemetry.io/otel/trace v1.14.0 // indirect

go.sum

@@ -37,6 +37,10 @@ github.com/cyphar/filepath-securejoin v0.2.3/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxG
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/deepfence/golang_deepfence_sdk/client v0.0.0-20230630084500-8fb0280d6010 h1:GyiH95PstGB/0rkxEI3qUi2XFG+IbSnXW5+fAu5f9lI=
+github.com/deepfence/golang_deepfence_sdk/client v0.0.0-20230630084500-8fb0280d6010/go.mod h1:+rchMc4YNjCoHo0YAwKsT+DRBNr1hdDG0WrvAOOCc5k=
+github.com/deepfence/golang_deepfence_sdk/utils v0.0.0-20230630084500-8fb0280d6010 h1:LVj2g3fEbS2JBwN6kDgM1+f24Cpnh3EQibKs/GDjtok=
+github.com/deepfence/golang_deepfence_sdk/utils v0.0.0-20230630084500-8fb0280d6010/go.mod h1:C3CqMr7oE9RmHZWXIVDWFLuGaNDDaoSBSlILLQJxlew=
 github.com/deepfence/vessel v0.11.1 h1:RSnPHv/HX9Vrcujxzp6l4cjzF7a/34lVvh+jr8Hq8YA=
 github.com/deepfence/vessel v0.11.1/go.mod h1:uSMZ7HZePuQzHH2kKdRJ/r8kYPz9ZgkffYhFiccmeHk=
 github.com/docker/distribution v2.8.1+incompatible h1:Q50tZOPR6T/hjNsyc9g8/syEs6bk8XXApsHjKukMl68=
@@ -100,6 +104,12 @@ github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+
 github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
 github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g=
+github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
+github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
+github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI=
+github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ=
+github.com/hashicorp/go-retryablehttp v0.7.4 h1:ZQgVdpTdAL7WpMIwLzCfbalOcSUdkDZnpUv3/+BxzFA=
+github.com/hashicorp/go-retryablehttp v0.7.4/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8=
 github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
@@ -113,6 +123,8 @@ github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovk
 github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
 github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng=
 github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
+github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0=
+github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
 github.com/moby/locker v1.0.1 h1:fOXqR41zeveg4fFODix+1Ch4mj/gT0NE1XJbp/epuBg=
 github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc=
 github.com/moby/sys/mountinfo v0.5.0/go.mod h1:3bMD3Rg+zkqx8MRYPi7Pyb0Ie97QEBmdxbhnCLlSvSU=
@@ -125,6 +137,8 @@ github.com/moby/sys/signal v0.7.0/go.mod h1:GQ6ObYZfqacOwTtlXvcmh9A26dVRul/hbOZn
 github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0=
 github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
 github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ=
+github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
+github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
 github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
 github.com/opencontainers/image-spec v1.1.0-rc2.0.20221005185240-3a7f492d3f1b h1:YWuSjZCQAPM8UUBLkYUk1e+rZcvWHJmFb6i6rM44Xs8=

main.go

@@ -27,6 +27,8 @@ package main
 import (
 	"flag"
 	"fmt"
+	"strconv"
+	"time"
 
 	"github.com/deepfence/SecretScanner/core"
 	"github.com/deepfence/SecretScanner/output"
@@ -64,8 +66,6 @@ func findSecretsInImage(image string) (*output.JsonImageSecretsOutput, error) {
 	jsonImageSecretsOutput := output.JsonImageSecretsOutput{ImageName: image}
 	jsonImageSecretsOutput.SetTime()
 	jsonImageSecretsOutput.SetImageId(res.ImageId)
-	jsonImageSecretsOutput.PrintJsonHeader()
-	jsonImageSecretsOutput.PrintJsonFooter()
 	jsonImageSecretsOutput.SetSecrets(res.Secrets)
 
 	return &jsonImageSecretsOutput, nil
@@ -87,8 +87,6 @@ func findSecretsInDir(dir string) (*output.JsonDirSecretsOutput, error) {
 
 	jsonDirSecretsOutput := output.JsonDirSecretsOutput{DirName: *session.Options.Local}
 	jsonDirSecretsOutput.SetTime()
-	jsonDirSecretsOutput.PrintJsonHeader()
-	jsonDirSecretsOutput.PrintJsonFooter()
 	jsonDirSecretsOutput.SetSecrets(secrets)
 
 	return &jsonDirSecretsOutput, nil
@@ -108,64 +106,105 @@ func findSecretsInContainer(containerId string, containerNS string) (*output.Jso
 	jsonImageSecretsOutput := output.JsonImageSecretsOutput{ContainerId: containerId}
 	jsonImageSecretsOutput.SetTime()
 	jsonImageSecretsOutput.SetImageId(res.ContainerId)
-	jsonImageSecretsOutput.PrintJsonHeader()
-	jsonImageSecretsOutput.PrintJsonFooter()
 	jsonImageSecretsOutput.SetSecrets(res.Secrets)
 
 	return &jsonImageSecretsOutput, nil
 }
 
 type SecretsWriter interface {
-	WriteSecrets(jsonFilename string) error
+	WriteJson() error
+	WriteTable() error
+	GetSecrets() []output.SecretFound
 }
 
-func runOnce() {
-	var output SecretsWriter
-	var input string
+func runOnce(format string) {
+	var result SecretsWriter
+	var err error
+	node_type := ""
+	node_id := ""
 
 	// Scan container image for secrets
 	if len(*session.Options.ImageName) > 0 {
+		node_type = "image"
+		node_id = *session.Options.ImageName
 		fmt.Printf("Scanning image %s for secrets...\n", *session.Options.ImageName)
-		jsonOutput, err := findSecretsInImage(*session.Options.ImageName)
+		result, err = findSecretsInImage(*session.Options.ImageName)
 		if err != nil {
 			core.GetSession().Log.Fatal("main: error while scanning image: %s", err)
 		}
-		output = jsonOutput
 	}
 
 	// Scan local directory for secrets
 	if len(*session.Options.Local) > 0 {
+		node_id = output.GetHostname()
 		fmt.Printf("[*] Scanning local directory: %s\n", color.BlueString(*session.Options.Local))
-		jsonOutput, err := findSecretsInDir(*session.Options.Local)
+		result, err = findSecretsInDir(*session.Options.Local)
 		if err != nil {
 			core.GetSession().Log.Fatal("main: error while scanning dir: %s", err)
 		}
-		output = jsonOutput
 	}
 
 	// Scan existing container for secrets
 	if len(*session.Options.ContainerId) > 0 {
+		node_type = "container_image"
+		node_id = *session.Options.ContainerId
 		fmt.Printf("Scanning container %s for secrets...\n", *session.Options.ContainerId)
-		jsonOutput, err := findSecretsInContainer(*session.Options.ContainerId, *session.Options.ContainerNS)
+		result, err = findSecretsInContainer(*session.Options.ContainerId, *session.Options.ContainerNS)
 		if err != nil {
 			core.GetSession().Log.Fatal("main: error while scanning container: %s", err)
 		}
-		output = jsonOutput
 	}
 
-	if output == nil {
+	if result == nil {
 		core.GetSession().Log.Error("set either -local or -image-name flag")
 		return
 	}
 
-	jsonFilename, err := core.GetJsonFilepath(input)
-	if err != nil {
-		core.GetSession().Log.Fatal("main: error while retrieving json output: %s", err)
+	if len(*core.GetSession().Options.ConsoleUrl) != 0 && len(*core.GetSession().Options.DeepfenceKey) != 0 {
+		pub, err := output.NewPublisher(
+			*core.GetSession().Options.ConsoleUrl,
+			strconv.Itoa(*core.GetSession().Options.ConsolePort),
+			*core.GetSession().Options.DeepfenceKey,
+		)
+		if err != nil {
+			core.GetSession().Log.Error(err.Error())
+		}
+
+		pub.SendReport(output.GetHostname(), *session.Options.ImageName, *session.Options.ContainerId, node_type)
+		scanId := pub.StartScan(node_id, node_type)
+		if len(scanId) == 0 {
+			scanId = fmt.Sprintf("%s-%d", node_id, time.Now().UnixMilli())
+		}
+		pub.IngestSecretScanResults(scanId, result.GetSecrets())
+		core.GetSession().Log.Info("scan id %s", scanId)
 	}
-	err = output.WriteSecrets(jsonFilename)
-	if err != nil {
-		core.GetSession().Log.Fatal("main: error whilewriting secrets: %s", err)
+
+	counts := output.CountBySeverity(result.GetSecrets())
+	core.GetSession().Log.Info("result severity counts: %+v", counts)
+
+	fmt.Println("summary:")
+	fmt.Printf("  total=%d high=%d medium=%d low=%d\n",
+		counts.Total, counts.High, counts.Medium, counts.Low)
+
+	if format == core.JsonOutput {
+		err = result.WriteJson()
+		if err != nil {
+			core.GetSession().Log.Fatal("main: error while writing secrets: %s", err)
+		}
+	} else {
+		err = result.WriteTable()
+		if err != nil {
+			core.GetSession().Log.Fatal("main: error while writing secrets: %s", err)
+		}
 	}
+
+	output.FailOn(
+		counts,
+		*core.GetSession().Options.FailOnHighCount,
+		*core.GetSession().Options.FailOnMediumCount,
+		*core.GetSession().Options.FailOnLowCount,
+		*core.GetSession().Options.FailOnCount,
+	)
 }
 
 func main() {
@@ -193,6 +232,6 @@ func main() {
 			core.GetSession().Log.Fatal("main: failed to serve through http: %v", err)
 		}
 	} else {
-		runOnce()
+		runOnce(*core.GetSession().Options.OutFormat)
 	}
 }