diff --git a/Dockerfile b/Dockerfile index 9713f12..f8cfd4b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -31,5 +31,5 @@ COPY --from=builder /home/deepfence/src/SecretScanner/SecretScanner . COPY --from=builder /home/deepfence/src/SecretScanner/config.yaml . WORKDIR /home/deepfence/output -ENTRYPOINT ["/home/deepfence/usr/SecretScanner", "-config-path", "/home/deepfence/usr", "-quiet"] +ENTRYPOINT ["/home/deepfence/usr/SecretScanner", "-config-path", "/home/deepfence/usr"] CMD ["-h"] diff --git a/Makefile b/Makefile index 165a088..b025049 100644 --- a/Makefile +++ b/Makefile @@ -12,3 +12,7 @@ SecretScanner: $(PWD)/**/*.go $(PWD)/agent-plugins-grpc/**/*.go go build -ldflags="-extldflags=-static" -buildvcs=false -v . .PHONY: clean bootstrap + +.PHONY: docker +docker: + docker build -t deepfenceio/deepfence_secret_scanner:latest . diff --git a/core/options.go b/core/options.go index 12c0440..e61035a 100644 --- a/core/options.go +++ b/core/options.go @@ -9,6 +9,8 @@ import ( const ( TempDirSuffix = "SecretScanning" ExtractedImageFilesDir = "ExtractedFiles" + JsonOutput = "json" + TableOutput = "table" ) type Options struct { @@ -20,17 +22,22 @@ type Options struct { HostMountPath *string ConfigPath *repeatableStringValue MergeConfigs *bool - OutputPath *string - JsonFilename *string ImageName *string MultipleMatch *bool MaxMultiMatch *uint MaxSecrets *uint ContainerId *string ContainerNS *string - Quiet *bool WorkersPerScan *int InactiveThreshold *int + OutFormat *string + ConsoleUrl *string + ConsolePort *int + DeepfenceKey *string + FailOnCount *int + FailOnHighCount *int + FailOnMediumCount *int + FailOnLowCount *int } type repeatableStringValue struct { @@ -60,17 +67,22 @@ func ParseOptions() (*Options, error) { HostMountPath: flag.String("host-mount-path", "", "If scanning the host, specify the host mount path for path exclusions to work correctly."), ConfigPath: &repeatableStringValue{}, MergeConfigs: flag.Bool("merge-configs", false, "Merge config files specified by --config-path into the default config"), - OutputPath: flag.String("output-path", ".", "Output directory where json file will be stored. If not set, it will output to current directory"), - JsonFilename: flag.String("json-filename", "", "Output json file name. If not set, it will automatically create a filename based on image or dir name"), ImageName: flag.String("image-name", "", "Name of the image along with tag to scan for secrets"), MultipleMatch: flag.Bool("multi-match", false, "Output multiple matches of same pattern in one file. By default, only one match of a pattern is output for a file for better performance"), MaxMultiMatch: flag.Uint("max-multi-match", 3, "Maximum number of matches of same pattern in one file. This is used only when multi-match option is enabled."), MaxSecrets: flag.Uint("max-secrets", 1000, "Maximum number of secrets to find in one container image or file system."), ContainerId: flag.String("container-id", "", "Id of existing container ID"), ContainerNS: flag.String("container-ns", "", "Namespace of existing container to scan, empty for docker runtime"), - Quiet: flag.Bool("quiet", false, "Don't display any output in stdout"), - WorkersPerScan: flag.Int("workers-per-scan", 1, "Number of concrrent workers per scan"), + WorkersPerScan: flag.Int("workers-per-scan", 1, "Number of concurrent workers per scan"), InactiveThreshold: flag.Int("inactive-threshold", 600, "Threshold for Inactive scan in seconds"), + OutFormat: flag.String("output", TableOutput, "Output format: json or table"), + ConsoleUrl: flag.String("console-url", "", "Deepfence Management Console URL"), + ConsolePort: flag.Int("console-port", 443, "Deepfence Management Console Port"), + DeepfenceKey: flag.String("deepfence-key", "", "Deepfence key for auth"), + FailOnCount: flag.Int("fail-on-count", -1, "Exit with status 1 if number of secrets found is >= this value (Default: -1)"), + FailOnHighCount: flag.Int("fail-on-high-count", -1, "Exit with status 1 if number of high secrets found is >= this value (Default: -1)"), + FailOnMediumCount: flag.Int("fail-on-medium-count", -1, "Exit with status 1 if number of medium secrets found is >= this value (Default: -1)"), + FailOnLowCount: flag.Int("fail-on-low-count", -1, "Exit with status 1 if number of low secrets found is >= this value (Default: -1)"), } flag.Var(options.ConfigPath, "config-path", "Searches for config.yaml from given directory. If not set, tries to find it from SecretScanner binary's and current directory. Can be specified multiple times.") flag.Parse() diff --git a/core/util.go b/core/util.go index b0cbabe..3904b69 100644 --- a/core/util.go +++ b/core/util.go @@ -48,30 +48,6 @@ func getSanitizedString(imageName string) string { return sanitizedName } -// GetJsonFilepath Return complete path and filename for json output file -// @parameters -// image - Name of the container image or dir, for which json filename and path will be created -// @returns -// string - Sanitized string which can used as path and filename of json output file -// Error - Errors if path can't be created. Otherwise, returns nil -func GetJsonFilepath(input string) (string, error) { - outputDir := *GetSession().Options.OutputPath - JsonFilename := *GetSession().Options.JsonFilename - if !PathExists(outputDir) { - err := CreateRecursiveDir(outputDir) - if err != nil { - GetSession().Log.Error("GetJsonFilepath: Could not create output dir: %s", err) - return "", err - } - } - if JsonFilename == "" { - JsonFilename = getSanitizedString(input) + "-secrets.json" - } - jsonFilePath := filepath.Join(outputDir, JsonFilename) - GetSession().Log.Info("Complete json file path and name: %s", jsonFilePath) - return jsonFilePath, nil -} - // GetTmpDir Create a temporrary directory to extract the conetents of container image // @parameters // imageName - Name of the container image @@ -124,7 +100,6 @@ func DeleteTmpDir(outputDir string) error { // path - Directory whose contents need to be deleted // wildcard - patterns to match the filenames (e.g. '*') func DeleteFiles(path string, wildCard string) { - var val string files, _ := filepath.Glob(path + wildCard) for _, val = range files { diff --git a/go.mod b/go.mod index 9a11b4c..ccd9791 100644 --- a/go.mod +++ b/go.mod @@ -7,9 +7,13 @@ replace github.com/deepfence/agent-plugins-grpc => ./agent-plugins-grpc require ( github.com/Jeffail/tunny v0.1.4 github.com/deepfence/agent-plugins-grpc v0.0.0-00010101000000-000000000000 + github.com/deepfence/golang_deepfence_sdk/client v0.0.0-20230630084500-8fb0280d6010 + github.com/deepfence/golang_deepfence_sdk/utils v0.0.0-20230630084500-8fb0280d6010 github.com/deepfence/vessel v0.11.1 github.com/fatih/color v1.15.0 github.com/flier/gohs v1.2.2 + github.com/olekukonko/tablewriter v0.0.5 + github.com/sirupsen/logrus v1.9.3 google.golang.org/grpc v1.56.1 gopkg.in/yaml.v3 v3.0.1 ) @@ -39,9 +43,12 @@ require ( github.com/golang/protobuf v1.5.3 // indirect github.com/google/go-cmp v0.5.9 // indirect github.com/google/uuid v1.3.0 // indirect + github.com/hashicorp/go-cleanhttp v0.5.2 // indirect + github.com/hashicorp/go-retryablehttp v0.7.4 // indirect github.com/klauspost/compress v1.16.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.17 // indirect + github.com/mattn/go-runewidth v0.0.9 // indirect github.com/moby/locker v1.0.1 // indirect github.com/moby/sys/mountinfo v0.6.2 // indirect github.com/moby/sys/sequential v0.5.0 // indirect @@ -52,7 +59,6 @@ require ( github.com/opencontainers/runtime-spec v1.1.0-rc.1 // indirect github.com/opencontainers/selinux v1.11.0 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/sirupsen/logrus v1.9.3 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/otel v1.14.0 // indirect go.opentelemetry.io/otel/trace v1.14.0 // indirect diff --git a/go.sum b/go.sum index 22f568b..47b16b0 100644 --- a/go.sum +++ b/go.sum @@ -37,6 +37,10 @@ github.com/cyphar/filepath-securejoin v0.2.3/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxG github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/deepfence/golang_deepfence_sdk/client v0.0.0-20230630084500-8fb0280d6010 h1:GyiH95PstGB/0rkxEI3qUi2XFG+IbSnXW5+fAu5f9lI= +github.com/deepfence/golang_deepfence_sdk/client v0.0.0-20230630084500-8fb0280d6010/go.mod h1:+rchMc4YNjCoHo0YAwKsT+DRBNr1hdDG0WrvAOOCc5k= +github.com/deepfence/golang_deepfence_sdk/utils v0.0.0-20230630084500-8fb0280d6010 h1:LVj2g3fEbS2JBwN6kDgM1+f24Cpnh3EQibKs/GDjtok= +github.com/deepfence/golang_deepfence_sdk/utils v0.0.0-20230630084500-8fb0280d6010/go.mod h1:C3CqMr7oE9RmHZWXIVDWFLuGaNDDaoSBSlILLQJxlew= github.com/deepfence/vessel v0.11.1 h1:RSnPHv/HX9Vrcujxzp6l4cjzF7a/34lVvh+jr8Hq8YA= github.com/deepfence/vessel v0.11.1/go.mod h1:uSMZ7HZePuQzHH2kKdRJ/r8kYPz9ZgkffYhFiccmeHk= github.com/docker/distribution v2.8.1+incompatible h1:Q50tZOPR6T/hjNsyc9g8/syEs6bk8XXApsHjKukMl68= @@ -100,6 +104,12 @@ github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g= +github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= +github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI= +github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= +github.com/hashicorp/go-retryablehttp v0.7.4 h1:ZQgVdpTdAL7WpMIwLzCfbalOcSUdkDZnpUv3/+BxzFA= +github.com/hashicorp/go-retryablehttp v0.7.4/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= @@ -113,6 +123,8 @@ github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovk github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= +github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/moby/locker v1.0.1 h1:fOXqR41zeveg4fFODix+1Ch4mj/gT0NE1XJbp/epuBg= github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc= github.com/moby/sys/mountinfo v0.5.0/go.mod h1:3bMD3Rg+zkqx8MRYPi7Pyb0Ie97QEBmdxbhnCLlSvSU= @@ -125,6 +137,8 @@ github.com/moby/sys/signal v0.7.0/go.mod h1:GQ6ObYZfqacOwTtlXvcmh9A26dVRul/hbOZn github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ= +github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= +github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.0-rc2.0.20221005185240-3a7f492d3f1b h1:YWuSjZCQAPM8UUBLkYUk1e+rZcvWHJmFb6i6rM44Xs8= diff --git a/main.go b/main.go index 491e8a3..ecdacad 100644 --- a/main.go +++ b/main.go @@ -27,6 +27,8 @@ package main import ( "flag" "fmt" + "strconv" + "time" "github.com/deepfence/SecretScanner/core" "github.com/deepfence/SecretScanner/output" @@ -64,8 +66,6 @@ func findSecretsInImage(image string) (*output.JsonImageSecretsOutput, error) { jsonImageSecretsOutput := output.JsonImageSecretsOutput{ImageName: image} jsonImageSecretsOutput.SetTime() jsonImageSecretsOutput.SetImageId(res.ImageId) - jsonImageSecretsOutput.PrintJsonHeader() - jsonImageSecretsOutput.PrintJsonFooter() jsonImageSecretsOutput.SetSecrets(res.Secrets) return &jsonImageSecretsOutput, nil @@ -87,8 +87,6 @@ func findSecretsInDir(dir string) (*output.JsonDirSecretsOutput, error) { jsonDirSecretsOutput := output.JsonDirSecretsOutput{DirName: *session.Options.Local} jsonDirSecretsOutput.SetTime() - jsonDirSecretsOutput.PrintJsonHeader() - jsonDirSecretsOutput.PrintJsonFooter() jsonDirSecretsOutput.SetSecrets(secrets) return &jsonDirSecretsOutput, nil @@ -108,64 +106,105 @@ func findSecretsInContainer(containerId string, containerNS string) (*output.Jso jsonImageSecretsOutput := output.JsonImageSecretsOutput{ContainerId: containerId} jsonImageSecretsOutput.SetTime() jsonImageSecretsOutput.SetImageId(res.ContainerId) - jsonImageSecretsOutput.PrintJsonHeader() - jsonImageSecretsOutput.PrintJsonFooter() jsonImageSecretsOutput.SetSecrets(res.Secrets) return &jsonImageSecretsOutput, nil } type SecretsWriter interface { - WriteSecrets(jsonFilename string) error + WriteJson() error + WriteTable() error + GetSecrets() []output.SecretFound } -func runOnce() { - var output SecretsWriter - var input string +func runOnce(format string) { + var result SecretsWriter + var err error + node_type := "" + node_id := "" // Scan container image for secrets if len(*session.Options.ImageName) > 0 { + node_type = "image" + node_id = *session.Options.ImageName fmt.Printf("Scanning image %s for secrets...\n", *session.Options.ImageName) - jsonOutput, err := findSecretsInImage(*session.Options.ImageName) + result, err = findSecretsInImage(*session.Options.ImageName) if err != nil { core.GetSession().Log.Fatal("main: error while scanning image: %s", err) } - output = jsonOutput } // Scan local directory for secrets if len(*session.Options.Local) > 0 { + node_id = output.GetHostname() fmt.Printf("[*] Scanning local directory: %s\n", color.BlueString(*session.Options.Local)) - jsonOutput, err := findSecretsInDir(*session.Options.Local) + result, err = findSecretsInDir(*session.Options.Local) if err != nil { core.GetSession().Log.Fatal("main: error while scanning dir: %s", err) } - output = jsonOutput } // Scan existing container for secrets if len(*session.Options.ContainerId) > 0 { + node_type = "container_image" + node_id = *session.Options.ContainerId fmt.Printf("Scanning container %s for secrets...\n", *session.Options.ContainerId) - jsonOutput, err := findSecretsInContainer(*session.Options.ContainerId, *session.Options.ContainerNS) + result, err = findSecretsInContainer(*session.Options.ContainerId, *session.Options.ContainerNS) if err != nil { core.GetSession().Log.Fatal("main: error while scanning container: %s", err) } - output = jsonOutput } - if output == nil { + if result == nil { core.GetSession().Log.Error("set either -local or -image-name flag") return } - jsonFilename, err := core.GetJsonFilepath(input) - if err != nil { - core.GetSession().Log.Fatal("main: error while retrieving json output: %s", err) + if len(*core.GetSession().Options.ConsoleUrl) != 0 && len(*core.GetSession().Options.DeepfenceKey) != 0 { + pub, err := output.NewPublisher( + *core.GetSession().Options.ConsoleUrl, + strconv.Itoa(*core.GetSession().Options.ConsolePort), + *core.GetSession().Options.DeepfenceKey, + ) + if err != nil { + core.GetSession().Log.Error(err.Error()) + } + + pub.SendReport(output.GetHostname(), *session.Options.ImageName, *session.Options.ContainerId, node_type) + scanId := pub.StartScan(node_id, node_type) + if len(scanId) == 0 { + scanId = fmt.Sprintf("%s-%d", node_id, time.Now().UnixMilli()) + } + pub.IngestSecretScanResults(scanId, result.GetSecrets()) + core.GetSession().Log.Info("scan id %s", scanId) } - err = output.WriteSecrets(jsonFilename) - if err != nil { - core.GetSession().Log.Fatal("main: error whilewriting secrets: %s", err) + + counts := output.CountBySeverity(result.GetSecrets()) + core.GetSession().Log.Info("result severity counts: %+v", counts) + + fmt.Println("summary:") + fmt.Printf(" total=%d high=%d medium=%d low=%d\n", + counts.Total, counts.High, counts.Medium, counts.Low) + + if format == core.JsonOutput { + err = result.WriteJson() + if err != nil { + core.GetSession().Log.Fatal("main: error while writing secrets: %s", err) + } + } else { + err = result.WriteTable() + if err != nil { + core.GetSession().Log.Fatal("main: error while writing secrets: %s", err) + } } + + output.FailOn( + counts, + *core.GetSession().Options.FailOnHighCount, + *core.GetSession().Options.FailOnMediumCount, + *core.GetSession().Options.FailOnLowCount, + *core.GetSession().Options.FailOnCount, + ) } func main() { @@ -193,6 +232,6 @@ func main() { core.GetSession().Log.Fatal("main: failed to serve through http: %v", err) } } else { - runOnce() + runOnce(*core.GetSession().Options.OutFormat) } } diff --git a/output/output.go b/output/output.go index 80a27ab..9a6e19d 100644 --- a/output/output.go +++ b/output/output.go @@ -6,16 +6,23 @@ import ( "os" "time" - // "strings" "github.com/deepfence/SecretScanner/core" pb "github.com/deepfence/agent-plugins-grpc/srcgo" "github.com/fatih/color" + tw "github.com/olekukonko/tablewriter" ) const ( Indent = " " // Indentation for Json printing ) +// severity +const ( + HIGH = "high" + MEDIUM = "medium" + LOW = "low" +) + type SecretFound struct { LayerID string `json:"Image Layer ID,omitempty"` RuleID int `json:"Matched Rule ID,omitempty"` @@ -32,10 +39,6 @@ type SecretFound struct { MatchedContents string `json:"Matched Contents,omitempty"` } -type SecretstOutput interface { - WriteSecrets(string) error -} - type JsonDirSecretsOutput struct { Timestamp time.Time DirName string `json:"Directory Name"` @@ -66,9 +69,17 @@ func (imageOutput *JsonImageSecretsOutput) SetSecrets(Secrets []SecretFound) { imageOutput.Secrets = Secrets } -func (imageOutput JsonImageSecretsOutput) WriteSecrets(outputFilename string) error { - err := printSecretsToJsonFile(imageOutput, outputFilename) - return err +func (imageOutput *JsonImageSecretsOutput) GetSecrets() []SecretFound { + return imageOutput.Secrets +} + +func (imageOutput JsonImageSecretsOutput) WriteJson() error { + return printSecretsToJson(imageOutput) + +} + +func (imageOutput JsonImageSecretsOutput) WriteTable() error { + return WriteTableOutput(&imageOutput.Secrets) } func (dirOutput *JsonDirSecretsOutput) SetDirName(dirName string) { @@ -82,58 +93,31 @@ func (dirOutput *JsonDirSecretsOutput) SetTime() { func (dirOutput *JsonDirSecretsOutput) SetSecrets(Secrets []SecretFound) { dirOutput.Secrets = Secrets } +func (dirOutput *JsonDirSecretsOutput) GetSecrets() []SecretFound { + return dirOutput.Secrets +} -func (dirOutput JsonDirSecretsOutput) WriteSecrets(outputFilename string) error { - err := printSecretsToJsonFile(dirOutput, outputFilename) - return err +func (dirOutput JsonDirSecretsOutput) WriteJson() error { + return printSecretsToJson(dirOutput) } -func printSecretsToJsonFile(secretsJson interface{}, outputFilename string) error { +func (dirOutput JsonDirSecretsOutput) WriteTable() error { + return WriteTableOutput(&dirOutput.Secrets) +} + +func printSecretsToJson(secretsJson interface{}) error { file, err := json.MarshalIndent(secretsJson, "", Indent) if err != nil { core.GetSession().Log.Error("printSecretsToJsonFile: Couldn't format json output: %s", err) return err } - err = os.WriteFile(outputFilename, file, os.ModePerm) - if err != nil { - core.GetSession().Log.Error("printSecretsToJsonFile: Couldn't write json output to file: %s", err) - return err - } - - // fmt.Println(string(file)) + fmt.Println() + fmt.Println(string(file)) return nil } -func (imageOutput JsonImageSecretsOutput) PrintJsonHeader() { - fmt.Printf("{\n") - fmt.Printf(Indent+"\"Timestamp\": \"%s\",\n", time.Now().Format("2006-01-02 15:04:05.000000000 -07:00")) - fmt.Printf(Indent+"\"Image Name\": \"%s\",\n", imageOutput.ImageName) - fmt.Printf(Indent+"\"Image ID\": \"%s\",\n", imageOutput.ImageId) - fmt.Printf(Indent + "\"Secrets\": [\n") -} - -func (imageOutput JsonImageSecretsOutput) PrintJsonFooter() { - printJsonFooter() -} - -func (dirOutput JsonDirSecretsOutput) PrintJsonHeader() { - fmt.Printf("{\n") - fmt.Printf(Indent+"\"Timestamp\": \"%s\",\n", time.Now().Format("2006-01-02 15:04:05.000000000 -07:00")) - fmt.Printf(Indent+"\"Directory Name\": \"%s\",\n", dirOutput.DirName) - fmt.Printf(Indent + "\"Secrets\": [\n") -} - -func (dirOutput JsonDirSecretsOutput) PrintJsonFooter() { - printJsonFooter() -} - -func printJsonFooter() { - fmt.Printf("\n" + Indent + "]\n") - fmt.Printf("}\n") -} - func PrintColoredSecrets(secrets []SecretFound, isFirstSecret *bool) { for _, secret := range secrets { printColoredSecretJsonObject(secret, isFirstSecret) @@ -220,3 +204,75 @@ func SecretToSecretInfo(out SecretFound) *pb.SecretInfo { }, } } + +func WriteTableOutput(report *[]SecretFound) error { + table := tw.NewWriter(os.Stdout) + table.SetHeader([]string{"Matched Part", "Rule Name", "Severity", "File Name", "Signature"}) + table.SetHeaderLine(true) + table.SetBorder(true) + table.SetAutoWrapText(true) + table.SetAutoFormatHeaders(true) + table.SetColMinWidth(0, 10) + table.SetColMinWidth(1, 10) + table.SetColMinWidth(2, 10) + table.SetColMinWidth(3, 20) + table.SetColMinWidth(4, 20) + + for _, r := range *report { + table.Append([]string{r.PartToMatch, r.RuleName, r.Severity, r.CompleteFilename, r.Regex}) + } + table.Render() + return nil +} + +type SevCount struct { + Total int + High int + Medium int + Low int +} + +func CountBySeverity(report []SecretFound) SevCount { + detail := SevCount{} + + for _, r := range report { + detail.Total += 1 + switch r.Severity { + case HIGH: + detail.High += 1 + case MEDIUM: + detail.Medium += 1 + case LOW: + detail.Low += 1 + } + } + + return detail +} + +func ExitOnSeverity(severity string, count int, failOnCount int) { + core.GetSession().Log.Debug("ExitOnSeverity severity=%s count=%d failOnCount=%d", + severity, count, failOnCount) + if count >= failOnCount { + if len(severity) > 0 { + msg := "Exit secret scan. Number of %s secrets (%d) reached/exceeded the limit (%d).\n" + fmt.Printf(msg, severity, count, failOnCount) + os.Exit(1) + } + msg := "Exit secret scan. Number of secrets (%d) reached/exceeded the limit (%d).\n" + fmt.Printf(msg, count, failOnCount) + os.Exit(1) + } +} + +func FailOn(details SevCount, failOnHighCount int, failOnMediumCount int, failOnLowCount int, failOnCount int) { + if failOnHighCount > 0 { + ExitOnSeverity(HIGH, details.High, failOnHighCount) + } else if failOnMediumCount > 0 { + ExitOnSeverity(MEDIUM, details.Medium, failOnMediumCount) + } else if failOnLowCount > 0 { + ExitOnSeverity(LOW, details.Low, failOnLowCount) + } else if failOnCount > 0 { + ExitOnSeverity("", details.Total, failOnCount) + } +} diff --git a/output/publish-to-console.go b/output/publish-to-console.go index 501744c..c6d4e2c 100644 --- a/output/publish-to-console.go +++ b/output/publish-to-console.go @@ -8,9 +8,16 @@ import ( "fmt" "net" "net/http" - "os" "strings" "time" + + "context" + + "os" + + dsc "github.com/deepfence/golang_deepfence_sdk/client" + oahttp "github.com/deepfence/golang_deepfence_sdk/utils/http" + log "github.com/sirupsen/logrus" ) var ( @@ -84,3 +91,188 @@ func buildClient() (*http.Client, error) { } return client, nil } + +type Publisher struct { + client *oahttp.OpenapiHttpClient + stopScanStatus chan bool +} + +func GetHostname() string { + name, err := os.Hostname() + if err != nil { + return "" + } + return name +} + +func NewPublisher(url string, port string, key string) (*Publisher, error) { + client := oahttp.NewHttpsConsoleClient(url, port) + if err := client.APITokenAuthenticate(key); err != nil { + return nil, err + } + return &Publisher{client: client}, nil +} + +func (p *Publisher) SendReport(hostname, image_name, container_id, node_type string) { + + report := dsc.IngestersReportIngestionData{} + + host := map[string]interface{}{ + "node_id": hostname, + "host_name": hostname, + "node_name": hostname, + "node_type": "host", + "cloud_region": "cli", + "cloud_provider": "cli", + "kubernetes_cluster_id": "", + } + report.HostBatch = []map[string]interface{}{host} + + if node_type != "" { + image := map[string]interface{}{ + "docker_image_name_with_tag": image_name, + "docker_image_id": image_name, + "node_id": image_name, + "node_name": image_name, + "node_type": node_type, + } + s := strings.Split(image_name, ":") + if len(s) == 2 { + image["docker_image_name"] = s[0] + image["docker_image_tag"] = s[1] + } + containerImageEdge := map[string]interface{}{ + "source": hostname, + "destinations": image_name, + } + report.ContainerImageBatch = []map[string]interface{}{image} + report.ContainerImageEdgeBatch = []map[string]interface{}{containerImageEdge} + } + + log.Debugf("report: %+v", report) + + req := p.client.Client().TopologyAPI.IngestSyncAgentReport(context.Background()) + req = req.IngestersReportIngestionData(report) + + resp, err := p.client.Client().TopologyAPI.IngestSyncAgentReportExecute(req) + if err != nil { + log.Error(err) + } + log.Debugf("report response %s", resp.Status) +} + +func (p *Publisher) StartScan(node_id, node_type string) string { + + scanTrigger := dsc.ModelSecretScanTriggerReq{ + Filters: *dsc.NewModelScanFilterWithDefaults(), + NodeIds: []dsc.ModelNodeIdentifier{}, + } + + nodeIds := dsc.ModelNodeIdentifier{NodeId: node_id, NodeType: node_type} + if node_type != "" { + nodeIds.NodeType = "host" + } + + scanTrigger.NodeIds = append(scanTrigger.NodeIds, nodeIds) + + req := p.client.Client().SecretScanAPI.StartSecretScan(context.Background()) + req = req.ModelSecretScanTriggerReq(scanTrigger) + res, resp, err := p.client.Client().SecretScanAPI.StartSecretScanExecute(req) + if err != nil { + log.Error(err) + return "" + } + // defer resp.Body.Close() + // io.Copy(io.Discard, resp.Body) + + log.Debugf("start scan response: %+v", res) + log.Debugf("start scan response status: %s", resp.Status) + + return res.GetScanIds()[0] +} + +func (p *Publisher) PublishScanStatusMessage(scan_id, message, status string) { + data := dsc.IngestersSecretScanStatus{} + data.SetScanId(scan_id) + data.SetScanStatus(status) + data.SetScanMessage(message) + + req := p.client.Client().SecretScanAPI.IngestSecretScanStatus(context.Background()) + req = req.IngestersSecretScanStatus([]dsc.IngestersSecretScanStatus{data}) + + resp, err := p.client.Client().SecretScanAPI.IngestSecretScanStatusExecute(req) + if err != nil { + log.Error(err) + } + + log.Debugf("publish scan status response: %v", resp) +} + +func (p *Publisher) PublishScanError(scan_id, errMsg string) { + p.PublishScanStatusMessage(scan_id, errMsg, "ERROR") +} + +func (p *Publisher) PublishScanStatusPeriodic(scan_id, status string) { + go func() { + p.PublishScanStatusMessage(scan_id, "", status) + ticker := time.NewTicker(30 * time.Second) + for { + select { + case <-ticker.C: + p.PublishScanStatusMessage(scan_id, "", status) + case <-p.stopScanStatus: + return + } + } + }() +} + +func (p *Publisher) StopPublishScanStatus() { + p.stopScanStatus <- true + time.Sleep(5 * time.Second) +} + +func (p *Publisher) IngestSecretScanResults(scan_id string, secrets []SecretFound) error { + data := []dsc.IngestersSecret{} + + for _, secret := range secrets { + rule := dsc.NewIngestersSecretRule() + rule.SetId(int32(secret.RuleID)) + rule.SetName(secret.RuleName) + rule.SetPart(secret.PartToMatch) + rule.SetSignatureToMatch(secret.Regex) + + match := dsc.NewIngestersSecretMatch() + match.SetFullFilename(secret.CompleteFilename) + match.SetMatchedContent(secret.MatchedContents) + match.SetRelativeEndingIndex(int32(secret.MatchToByte)) + match.SetRelativeStartingIndex(int32(secret.MatchFromByte)) + match.SetStartingIndex(int32(secret.PrintBufferStartIndex)) + + severity := dsc.NewIngestersSecretSeverity() + severity.SetLevel(secret.Severity) + severity.SetScore(float32(secret.SeverityScore)) + + s := dsc.NewIngestersSecret() + s.SetImageLayerId(secret.LayerID) + s.SetRule(*rule) + s.SetMatch(*match) + s.SetSeverity(*severity) + s.SetMasked(false) + s.SetScanId(scan_id) + + data = append(data, *s) + } + + req := p.client.Client().SecretScanAPI.IngestSecrets(context.Background()) + req = req.IngestersSecret(data) + + resp, err := p.client.Client().SecretScanAPI.IngestSecretsExecute(req) + if err != nil { + log.Error(err) + } + + log.Debugf("publish scan results response: %v", resp) + + return nil +} diff --git a/scan/process_image.go b/scan/process_image.go index aefb8ae..5dc6fff 100644 --- a/scan/process_image.go +++ b/scan/process_image.go @@ -236,17 +236,11 @@ func ScanSecretsInDir(layer string, baseDir string, fullDir string, session.Log.Error("scanSecretsInDir: %s", err) } else { if len(secrets) > 0 { - if *session.Options.Quiet { - output.PrintColoredSecrets(secrets, isFirstSecret) - } secretsFound = append(secretsFound, secrets...) } } secrets = signature.MatchSimpleSignatures(relPath, file.Filename, file.Extension, layer, &numSecrets) - if *session.Options.Quiet { - output.PrintColoredSecrets(secrets, isFirstSecret) - } secretsFound = append(secretsFound, secrets...) // Don't report secrets if number of secrets exceeds MAX value @@ -362,9 +356,6 @@ func ScanSecretsInDirStream(layer string, baseDir string, fullDir string, session.Log.Error("scanSecretsInDir: %s", err) } else { if len(secrets) > 0 { - if *session.Options.Quiet { - output.PrintColoredSecrets(secrets, isFirstSecret) - } for i := range secrets { res <- secrets[i] } @@ -372,9 +363,6 @@ func ScanSecretsInDirStream(layer string, baseDir string, fullDir string, } secrets = signature.MatchSimpleSignatures(relPath, file.Filename, file.Extension, layer, &numSecrets) - if *session.Options.Quiet { - output.PrintColoredSecrets(secrets, isFirstSecret) - } for i := range secrets { res <- secrets[i] } @@ -821,7 +809,7 @@ func ExtractAndScanFromTar(tarFolder string, imageName string) (*ImageExtraction func CheckScanStatus(scanCtx *ScanContext) error { if scanCtx != nil { - if scanCtx.Aborted.Load() == true { + if scanCtx.Aborted.Load() { close(scanCtx.ScanStatusChan) core.GetSession().Log.Error("Scan aborted due to inactivity, scanid:", scanCtx.ScanID) return fmt.Errorf("Scan aborted due to inactivity") diff --git a/server/http.go b/server/http.go index 623de19..f047e99 100644 --- a/server/http.go +++ b/server/http.go @@ -99,8 +99,6 @@ func runSecretScanStandalone(writer http.ResponseWriter, request *http.Request) jsonImageSecretsOutput := output.JsonImageSecretsOutput{ImageName: req.ImageNameWithTag} jsonImageSecretsOutput.SetTime() jsonImageSecretsOutput.SetImageId(res.ImageId) - jsonImageSecretsOutput.PrintJsonHeader() - jsonImageSecretsOutput.PrintJsonFooter() jsonImageSecretsOutput.SetSecrets(res.Secrets) outByte, err := json.Marshal(jsonImageSecretsOutput) @@ -130,6 +128,7 @@ func processImageWrapper(imageParamsInterface interface{}) interface{} { return nil } +// TODO: remove this code block func processImage(imageName string, scanId string, form url.Values) { tempFolder, err := core.GetTmpDir(imageName) if err != nil {