Merge remote-tracking branch 'origin/master' into general-rule

.github/workflows/test.yml

@@ -1,9 +1,6 @@
 name: Test
 
 on:
-  push:
-    branches:
-      - "*"
   pull_request:
     branches:
       - "*"
@@ -26,4 +23,4 @@ jobs:
         run: make test
 
       - name: Validate Config
-        run: cd cmd/generate/config && go run main.go
+        run: go generate ./... && git diff --exit-code

.pre-commit-hooks.yaml

@@ -9,3 +9,8 @@
   description: Detect hardcoded secrets using Gitleaks
   entry: zricethezav/gitleaks protect --verbose --redact --staged
   language: docker_image
+- id: gitleaks-system
+  name: Detect hardcoded secrets
+  description: Detect hardcoded secrets using Gitleaks
+  entry: gitleaks protect --verbose --redact --staged
+  language: system

CONTRIBUTING.md

@@ -35,7 +35,7 @@ If you want to add a new rule to the [default Gitleaks configuration](https://gi
            RuleID:      "beamer-api-token",
 
            // Regex capture group for the actual secret
-           SecretGroup: 1,
+
 
 
            // Regex used for detecting secrets. See regex section below for more details
@@ -88,15 +88,11 @@ If you want to add a new rule to the [default Gitleaks configuration](https://gi
    validation part. You can use `generateSampleSecret` to create a secret for the
    true positives (`tps` in the example above) used in `validate`.
 
-1. Update `cmd/generate/config/main.go`. Add a line like
-   `configRules = append(configRules, rules.Beamer())` in `main()`. Try and keep
+1. Update `cmd/generate/config/main.go`. Extend `configRules` slice with
+   the `rules.Beamer(),` in `main()`. Try and keep
    this alphabetically pretty please.
 
-1. Change directories into `cmd/generate/config` and run `go run main.go`
-
-   ```
-   cd cmd/generate/config && go run main.go
-   ```
+1. Run `go generate ./...`
 
 1. Check out your new rules in `config/gitleaks.toml` and see if everything looks good.
 

Dockerfile

@@ -1,14 +1,12 @@
-FROM golang:1.19 AS build
+FROM golang:1.21 AS build
 WORKDIR /go/src/github.com/zricethezav/gitleaks
 COPY . .
 RUN VERSION=$(git describe --tags --abbrev=0) && \
 CGO_ENABLED=0 go build -o bin/gitleaks -ldflags "-X="github.com/zricethezav/gitleaks/v8/cmd.Version=${VERSION}
 
-FROM alpine:3.16
-RUN adduser -D gitleaks && \
-    apk add --no-cache bash git openssh-client
+FROM alpine:3.19
+RUN apk add --no-cache bash git openssh-client
 COPY --from=build /go/src/github.com/zricethezav/gitleaks/bin/* /usr/bin/
-USER gitleaks
 
 RUN git config --global --add safe.directory '*'
 

Makefile

@@ -13,11 +13,9 @@ format:
 	go fmt ./...
 
 test: format
-	go vet ./...
 	go test -v ./... --race $(PKG) 
 
 build: format
-	go vet ./...
 	go mod tidy
 	go build $(LDFLAGS)
 

cmd/detect.go

@@ -2,28 +2,19 @@ package cmd
 
 import (
 	"os"
-	"path/filepath"
-	"strings"
 	"time"
 
-	"github.com/rs/zerolog"
 	"github.com/rs/zerolog/log"
 	"github.com/spf13/cobra"
-	"github.com/spf13/viper"
 
-	"github.com/zricethezav/gitleaks/v8/config"
-	"github.com/zricethezav/gitleaks/v8/detect"
 	"github.com/zricethezav/gitleaks/v8/report"
+	"github.com/zricethezav/gitleaks/v8/sources"
 )
 
 func init() {
 	rootCmd.AddCommand(detectCmd)
-	detectCmd.Flags().String("log-opts", "", "git log options")
 	detectCmd.Flags().Bool("no-git", false, "treat git repo as a regular directory and scan those files, --log-opts has no effect on the scan when --no-git is set")
 	detectCmd.Flags().Bool("pipe", false, "scan input from stdin, ex: `cat some_file | gitleaks detect --pipe`")
-	detectCmd.Flags().Bool("follow-symlinks", false, "scan files that are symlinks to other files")
-	detectCmd.Flags().StringSlice("enable-rule", []string{}, "only enable specific rules by id, ex: `gitleaks detect --enable-rule=atlassian-api-token --enable-rule=slack-access-token`")
-	detectCmd.Flags().StringP("gitleaks-ignore-path", "i", ".", "path to .gitleaksignore file or folder containing one")
 }
 
 var detectCmd = &cobra.Command{
@@ -35,117 +26,22 @@ var detectCmd = &cobra.Command{
 func runDetect(cmd *cobra.Command, args []string) {
 	initConfig()
 	var (
-		vc       config.ViperConfig
 		findings []report.Finding
 		err      error
 	)
 
-	// Load config
-	if err = viper.Unmarshal(&vc); err != nil {
-		log.Fatal().Err(err).Msg("Failed to load config")
-	}
-	cfg, err := vc.Translate()
-	if err != nil {
-		log.Fatal().Err(err).Msg("Failed to load config")
-	}
-	cfg.Path, _ = cmd.Flags().GetString("config")
+	// setup config (aka, the thing that defines rules)
+	cfg := Config(cmd)
 
 	// start timer
 	start := time.Now()
 
-	// Setup detector
-	detector := detect.NewDetector(cfg)
-	// set color flag at first
-	if detector.NoColor, err = cmd.Flags().GetBool("no-color"); err != nil {
-		log.Fatal().Err(err).Msg("")
-	}
-	// also init logger again without color
-	if detector.NoColor {
-		log.Logger = log.Output(zerolog.ConsoleWriter{
-			Out:     os.Stderr,
-			NoColor: detector.NoColor,
-		})
-	}
-	detector.Config.Path, err = cmd.Flags().GetString("config")
-	if err != nil {
-		log.Fatal().Err(err).Msg("")
-	}
+	// grab source
 	source, err := cmd.Flags().GetString("source")
 	if err != nil {
 		log.Fatal().Err(err).Msg("")
 	}
-	// if config path is not set, then use the {source}/.gitleaks.toml path.
-	// note that there may not be a `{source}/.gitleaks.toml` file, this is ok.
-	if detector.Config.Path == "" {
-		detector.Config.Path = filepath.Join(source, ".gitleaks.toml")
-	}
-	// set verbose flag
-	if detector.Verbose, err = cmd.Flags().GetBool("verbose"); err != nil {
-		log.Fatal().Err(err).Msg("")
-	}
-	// set redact flag
-	if detector.Redact, err = cmd.Flags().GetUint("redact"); err != nil {
-		log.Fatal().Err(err).Msg("")
-	}
-	if detector.MaxTargetMegaBytes, err = cmd.Flags().GetInt("max-target-megabytes"); err != nil {
-		log.Fatal().Err(err).Msg("")
-	}
-	// set ignore gitleaks:allow flag
-	if detector.IgnoreGitleaksAllow, err = cmd.Flags().GetBool("ignore-gitleaks-allow"); err != nil {
-		log.Fatal().Err(err).Msg("")
-	}
-
-	gitleaksIgnorePath, err := cmd.Flags().GetString("gitleaks-ignore-path")
-	if err != nil {
-		log.Fatal().Err(err).Msg("could not get .gitleaksignore path")
-	}
-
-	if fileExists(gitleaksIgnorePath) {
-		if err = detector.AddGitleaksIgnore(gitleaksIgnorePath); err != nil {
-			log.Fatal().Err(err).Msg("could not call AddGitleaksIgnore")
-		}
-	}
-
-	if fileExists(filepath.Join(gitleaksIgnorePath, ".gitleaksignore")) {
-		if err = detector.AddGitleaksIgnore(filepath.Join(gitleaksIgnorePath, ".gitleaksignore")); err != nil {
-			log.Fatal().Err(err).Msg("could not call AddGitleaksIgnore")
-		}
-	}
-
-	if fileExists(filepath.Join(source, ".gitleaksignore")) {
-		if err = detector.AddGitleaksIgnore(filepath.Join(source, ".gitleaksignore")); err != nil {
-			log.Fatal().Err(err).Msg("could not call AddGitleaksIgnore")
-		}
-	}
-
-	// ignore findings from the baseline (an existing report in json format generated earlier)
-	baselinePath, _ := cmd.Flags().GetString("baseline-path")
-	if baselinePath != "" {
-		err = detector.AddBaseline(baselinePath, source)
-		if err != nil {
-			log.Error().Msgf("Could not load baseline. The path must point of a gitleaks report generated using the default format: %s", err)
-		}
-	}
-
-	// If set, only apply rules that are defined in the flag
-	rules, _ := cmd.Flags().GetStringSlice("enable-rule")
-	if len(rules) > 0 {
-		log.Info().Msg("Overriding enabled rules: " + strings.Join(rules, ", "))
-		ruleOverride := make(map[string]config.Rule)
-		for _, ruleName := range rules {
-			if rule, ok := cfg.Rules[ruleName]; ok {
-				ruleOverride[ruleName] = rule
-			} else {
-				log.Fatal().Msgf("Requested rule %s not found in rules", ruleName)
-			}
-		}
-		detector.Config.Rules = ruleOverride
-	}
-
-	// set follow symlinks flag
-	if detector.FollowSymlinks, err = cmd.Flags().GetBool("follow-symlinks"); err != nil {
-		log.Fatal().Err(err).Msg("")
-	}
+	detector := Detector(cmd, cfg, source)
 
 	// set exit code
 	exitCode, err := cmd.Flags().GetInt("exit-code")
@@ -167,7 +63,11 @@ func runDetect(cmd *cobra.Command, args []string) {
 
 	// start the detector scan
 	if noGit {
-		findings, err = detector.DetectFiles(source)
+		paths, err := sources.DirectoryTargets(source, detector.Sema, detector.FollowSymlinks)
+		if err != nil {
+			log.Fatal().Err(err)
+		}
+		findings, err = detector.DetectFiles(paths)
 		if err != nil {
 			// don't exit on error, just log it
 			log.Error().Err(err).Msg("")
@@ -185,68 +85,16 @@ func runDetect(cmd *cobra.Command, args []string) {
 		if err != nil {
 			log.Fatal().Err(err).Msg("")
 		}
-		findings, err = detector.DetectGit(source, logOpts, detect.DetectType)
+		gitCmd, err := sources.NewGitLogCmd(source, logOpts)
+		if err != nil {
+			log.Fatal().Err(err).Msg("")
+		}
+		findings, err = detector.DetectGit(gitCmd)
 		if err != nil {
 			// don't exit on error, just log it
 			log.Error().Err(err).Msg("")
 		}
 	}
 
-	// log info about the scan
-	if err == nil {
-		log.Info().Msgf("scan completed in %s", FormatDuration(time.Since(start)))
-		if len(findings) != 0 {
-			log.Warn().Msgf("leaks found: %d", len(findings))
-		} else {
-			log.Info().Msg("no leaks found")
-		}
-	} else {
-		log.Warn().Msgf("partial scan completed in %s", FormatDuration(time.Since(start)))
-		if len(findings) != 0 {
-			log.Warn().Msgf("%d leaks found in partial scan", len(findings))
-		} else {
-			log.Warn().Msg("no leaks found in partial scan")
-		}
-	}
-
-	// write report if desired
-	reportPath, _ := cmd.Flags().GetString("report-path")
-	ext, _ := cmd.Flags().GetString("report-format")
-	if reportPath != "" {
-		if err := report.Write(findings, cfg, ext, reportPath); err != nil {
-			log.Fatal().Err(err).Msg("could not write")
-		}
-	}
-
-	if err != nil {
-		os.Exit(1)
-	}
-
-	if len(findings) != 0 {
-		os.Exit(exitCode)
-	}
-}
-
-func fileExists(fileName string) bool {
-	// check for a .gitleaksignore file
-	info, err := os.Stat(fileName)
-	if err != nil && !os.IsNotExist(err) {
-		return false
-	}
-
-	if info != nil && err == nil {
-		if !info.IsDir() {
-			return true
-		}
-	}
-	return false
-}
-
-func FormatDuration(d time.Duration) string {
-	scale := 100 * time.Second
-	// look for the max scale that is smaller than d
-	for scale > d {
-		scale = scale / 10
-	}
-	return d.Round(scale / 100).String()
+	findingSummaryAndExit(findings, cmd, cfg, exitCode, start, err)
 }