fix logger and json output

README.md

@@ -56,7 +56,7 @@ docker pull node:8.11
 
 * Scan the container image:
     ```shell
-    docker run -it --rm --name=deepfence-secretscanner -v $(pwd):/home/deepfence/output -v /var/run/docker.sock:/var/run/docker.sock deepfenceio/deepfence_secret_scanner:latest -image-name node:8.11
+    docker run -i --rm --name=deepfence-secretscanner -v /var/run/docker.sock:/var/run/docker.sock deepfenceio/deepfence_secret_scanner:latest -image-name node:8.11 --output json > node.json
     ```
 
 # Credits

core/match.go

@@ -5,6 +5,8 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+
+	log "github.com/sirupsen/logrus"
 )
 
 type MatchFile struct {
@@ -84,7 +86,7 @@ func ContainsBlacklistedString(input []byte) bool {
 	for _, blacklistedString := range session.Config.BlacklistedStrings {
 		blacklistedByteStr := []byte(blacklistedString)
 		if bytes.Contains(input, blacklistedByteStr) {
-			GetSession().Log.Debug("Blacklisted string %s matched", blacklistedString)
+			log.Debugf("Blacklisted string %s matched", blacklistedString)
 			return true
 		}
 	}
@@ -107,7 +109,7 @@ func ContainsBlacklistedString(input []byte) bool {
 //	}
 //	maxFileSize := strconv.FormatUint(uint64(*session.Options.MaximumFileSize), 10)
 //	findCmd += " -type f -size " + maxFileSize + "M"
-//	GetSession().Log.Info("find command: %s", findCmd)
+//	log.Info("find command: %s", findCmd)
 //
 //	return ExecuteCommand(findCmd)
 //}
@@ -118,7 +120,7 @@ func UpdateDirsPermissionsRW(dir string) {
 		if f.IsDir() {
 			err := os.Chmod(path, 0700)
 			if err != nil {
-				GetSession().Log.Error("Failed to change dir %s permission: %s", path, err)
+				log.Errorf("Failed to change dir %s permission: %s", path, err)
 			}
 		}
 		return nil

core/options.go

@@ -15,7 +15,7 @@ const (
 
 type Options struct {
 	Threads           *int
-	DebugLevel        *string
+	Debug             *bool
 	MaximumFileSize   *uint
 	TempDirectory     *string
 	Local             *string
@@ -60,7 +60,7 @@ func (v *repeatableStringValue) Values() []string {
 func ParseOptions() (*Options, error) {
 	options := &Options{
 		Threads:           flag.Int("threads", 0, "Number of concurrent threads (default number of logical CPUs)"),
-		DebugLevel:        flag.String("debug-level", "ERROR", "Debug levels are one of FATAL, ERROR, IMPORTANT, WARN, INFO, DEBUG. Only levels higher than the debug-level are displayed"),
+		Debug:             flag.Bool("debug", false, "enable debug logs"),
 		MaximumFileSize:   flag.Uint("maximum-file-size", 256, "Maximum file size to process in KB"),
 		TempDirectory:     flag.String("temp-directory", os.TempDir(), "Directory to process and store repositories/matches"),
 		Local:             flag.String("local", "", "Specify local directory (absolute path) which to scan. Scans only given directory recursively."),

core/session.go

@@ -2,13 +2,14 @@ package core
 
 import (
 	"context"
-	"fmt"
 	"math/rand"
 	"os"
 	"runtime"
 	"strings"
 	"sync"
 	"time"
+
+	log "github.com/sirupsen/logrus"
 )
 
 type Session struct {
@@ -17,7 +18,6 @@ type Session struct {
 	Options *Options
 	Config  *Config
 	Context context.Context
-	Log     *Logger
 }
 
 var (
@@ -28,16 +28,9 @@ var (
 
 func (s *Session) Start() {
 	rand.Seed(time.Now().Unix())
-
-	s.InitLogger()
 	s.InitThreads()
 }
 
-func (s *Session) InitLogger() {
-	s.Log = &Logger{}
-	s.Log.SetDebugLevel(*s.Options.DebugLevel)
-}
-
 func (s *Session) InitThreads() {
 	if *s.Options.Threads == 0 {
 		numCPUs := runtime.NumCPU()
@@ -49,18 +42,17 @@ func (s *Session) InitThreads() {
 
 func GetSession() *Session {
 	sessionSync.Do(func() {
-		fmt.Println("Initializing....")
 		session = &Session{
 			Context: context.Background(),
 		}
 
 		if session.Options, err = ParseOptions(); err != nil {
-			fmt.Println(err)
+			log.Error(err)
 			os.Exit(1)
 		}
 
 		if session.Config, err = ParseConfig(session.Options); err != nil {
-			fmt.Println(err)
+			log.Error(err)
 			os.Exit(1)
 		}
 

core/util.go

@@ -10,6 +10,8 @@ import (
 	"regexp"
 	"strings"
 	"time"
+
+	log "github.com/sirupsen/logrus"
 )
 
 // CreateRecursiveDir Create directory structure recursively, if they do not exist
@@ -19,14 +21,14 @@ import (
 // Error - Errors if any. Otherwise, returns nil
 func CreateRecursiveDir(completePath string) error {
 	if _, err := os.Stat(completePath); os.IsNotExist(err) {
-		GetSession().Log.Debug("Folder does not exist. Creating folder... %s", completePath)
+		log.Debugf("Folder does not exist. Creating folder... %s", completePath)
 		err = os.MkdirAll(completePath, os.ModePerm)
 		if err != nil {
-			GetSession().Log.Error("createRecursiveDir %q: %s", completePath, err)
+			log.Errorf("createRecursiveDir %q: %s", completePath, err)
 		}
 		return err
 	} else if err != nil {
-		GetSession().Log.Error("createRecursiveDir %q: %s. Deleting temp dir", completePath, err)
+		log.Errorf("createRecursiveDir %q: %s. Deleting temp dir", completePath, err)
 		DeleteTmpDir(completePath)
 		return err
 	}
@@ -69,7 +71,7 @@ func GetTmpDir(imageName string) (string, error) {
 
 	err := CreateRecursiveDir(completeTempPath)
 	if err != nil {
-		GetSession().Log.Error("getTmpDir: Could not create temp dir%s", err)
+		log.Errorf("getTmpDir: Could not create temp dir %s", err)
 		return "", err
 	}
 
@@ -82,13 +84,13 @@ func GetTmpDir(imageName string) (string, error) {
 // @returns
 // Error - Errors if any. Otherwise, returns nil
 func DeleteTmpDir(outputDir string) error {
-	GetSession().Log.Info("Deleting temporary dir %s", outputDir)
+	log.Infof("Deleting temporary dir %s", outputDir)
 	// Output dir will be empty string in case of error, don't delete
 	if outputDir != "" {
 		// deleteFiles(outputDir+"/", "*")
 		err := os.RemoveAll(outputDir)
 		if err != nil {
-			GetSession().Log.Error("deleteTmpDir: Could not delete temp dir: %s", err)
+			log.Errorf("deleteTmpDir: Could not delete temp dir: %s", err)
 			return err
 		}
 	}
@@ -144,7 +146,7 @@ func PathExists(path string) bool {
 
 func LogIfError(text string, err error) {
 	if err != nil {
-		GetSession().Log.Error("%s (%s", text, err.Error())
+		log.Errorf("%s (%s", text, err.Error())
 	}
 }
 

docs/docs/secretscanner/configure/cli.md

@@ -22,7 +22,7 @@ $ ./SecretScanner --help
 
 ### General Configuration
 
- * `--debug-level string`: one of FATAL, ERROR, IMPORTANT, WARN, INFO, DEBUG (default "ERROR"); print messages of this severity or higher.
+ * `--debug bool`: print debug level logs.
  * `--threads int`: Number of concurrent threads to use during scan (default number of logical CPUs).
  * `--temp-directory string`: temporary storage for working data (default "/tmp")
 
@@ -44,10 +44,9 @@ $ ./SecretScanner --help
 
 ### Configure Output
 
-In addition to writing output to **stdout** / **stderr**, SecretScanner can write JSON output to a local file. You may wish to mount a directory on the host into `output-path` in the container so that you can easily obtain the JSON output file.
+SecretScanner can write output as Table and JSON format
 
- * `--json-filename string`: output json file name; required
- * `--output-path string`: location in container where json file will be stored (default `/home/deepfence/output`)
+ * `-output`: Output format: json or table (default "table")
 
 ### Configure GRPC Listener
 

docs/docs/secretscanner/configure/output.md

@@ -4,22 +4,16 @@ title: Configure Output
 
 # Configure Output
 
-SecretScanner can write its JSON output to a container-local file (`--json-file`).
-
-By default, the output is written to `/home/deepfence/output` in the container filesystem.  You can mount a host directory over this location.
+SecretScanner can writes output to `stdout` it can redirected to a file for further analysis.
 
 ```bash
-# Write output to ./my-output/node-secret-scan.json
-
-mkdir ./my-output
+# Write output to ./tmp/node-secret-scan.json
 
 docker run -it --rm --name=deepfence_secret_scanner \
     -v /var/run/docker.sock:/var/run/docker.sock \
+    deepfenceio/deepfence_secret_scanner:latest \
+    --image-name node:latest \
 # highlight-next-line
-    -v $(pwd)/my-output:/home/deepfence/output \
-    deepfenceio/deepfence_secret_scanner:latest --image-name node:latest \
-# highlight-next-line
-    --json-filename node-secret-scan.json
+    --output json > ./tmp/node-secret-scan.json
 ```
 
-You can also override the default output location (`--output-path`) in the container.

docs/docs/secretscanner/quickstart.md

@@ -19,7 +19,7 @@ Pull an image to your local repository, then scan it
 ```bash
 docker pull node:latest
 
-docker run -it --rm --name=deepfence-secretscanner \
+docker run -i --rm --name=deepfence-secretscanner \
 	-v /var/run/docker.sock:/var/run/docker.sock \
 	deepfenceio/deepfence_secret_scanner:latest \
 	-image-name node:latest
@@ -32,11 +32,11 @@ docker rmi node:latest
 You can summarise the results by processing the JSON output, e.g. using `jq`:
 
 ```bash
-docker run -it --rm --name=deepfence-secretscanner \
+docker run -i --rm --name=deepfence-secretscanner \
     -v /var/run/docker.sock:/var/run/docker.sock \
-    -v /tmp:/home/deepfence/output \
     deepfenceio/deepfence_secret_scanner:latest \
-    --image-name node:latest --json-filename=node-secret-scan.json
+    --image-name node:latest \
+    --output json > /tmp/node-secret-scan.json
 
 cat /tmp/node-secret-scan.json | jq '.Secrets[] | { rule: ."Matched Rule Name", file: ."Full File Name" }'
 ```

jobs/scan.go

@@ -12,6 +12,7 @@ import (
 	"github.com/deepfence/golang_deepfence_sdk/utils/tasks"
 
 	pb "github.com/deepfence/agent-plugins-grpc/srcgo"
+	log "github.com/sirupsen/logrus"
 )
 
 var ScanMap sync.Map
@@ -91,12 +92,12 @@ func writeMultiScanData(secrets []*pb.SecretInfo, scan_id string) {
 		}
 		byteJson, err := json.Marshal(secretScanDoc)
 		if err != nil {
-			fmt.Println("Error marshalling json: ", err)
+			log.Errorf("Error marshalling json: ", err)
 			continue
 		}
 		err = writeScanDataToFile(string(byteJson), scanFilename)
 		if err != nil {
-			fmt.Println("Error in sending data to secretScanIndex:" + err.Error())
+			log.Errorf("Error in sending data to secretScanIndex:" + err.Error())
 			continue
 		}
 	}
@@ -112,12 +113,12 @@ func writeSingleScanData(secret *pb.SecretInfo, scan_id string) {
 	}
 	byteJson, err := json.Marshal(secretScanDoc)
 	if err != nil {
-		fmt.Println("Error marshalling json: ", err)
+		log.Errorf("Error marshalling json: ", err)
 		return
 	}
 	err = writeScanDataToFile(string(byteJson), scanFilename)
 	if err != nil {
-		fmt.Println("Error in sending data to secretScanIndex:" + err.Error())
+		log.Errorf("Error in sending data to secretScanIndex:" + err.Error())
 		return
 	}
 }