diff --git a/.gitignore b/.gitignore index 00c506d8..369de95b 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,16 @@ cli-docs site /pipeleak /pipeleak.exe +/pipeleak-gitlab +/pipeleak-gitlab.exe +/pipeleak-github +/pipeleak-github.exe +/pipeleak-bitbucket +/pipeleak-bitbucket.exe +/pipeleak-devops +/pipeleak-devops.exe +/pipeleak-gitea +/pipeleak-gitea.exe coverage.out coverage.html .vscode diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8a1226d2..33af314f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -11,6 +11,7 @@ The fastest way to start contributing is using GitHub Codespaces: 3. Click "Create codespace on main" (or your branch) The codespace will automatically: + - Set up Go 1.24+ environment - Install golangci-lint for code linting - Install Python and MkDocs for documentation @@ -32,22 +33,31 @@ If you prefer local development: ### Setup Steps 1. Clone the repository: + ```bash git clone https://github.com/CompassSecurity/pipeleak.git cd pipeleak ``` 2. Install golangci-lint: + ```bash go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest ``` 3. Download dependencies: + ```bash go mod download ``` -4. Build the binary: +4. Run it: + + ```bash + go run cmd/pipeleak/main.go + ``` + +5. Optional: Build the binary: ```bash make build ``` diff --git a/Makefile b/Makefile index c553034a..7437c6f9 100644 --- a/Makefile +++ b/Makefile @@ -1,25 +1,60 @@ -.PHONY: help build test test-unit test-e2e lint clean coverage coverage-html serve-docs +.PHONY: help build build-all build-gitlab build-github build-bitbucket build-devops build-gitea test test-unit test-e2e lint clean coverage coverage-html serve-docs # Default target help: @echo "Pipeleak Makefile" @echo "" @echo "Available targets:" - @echo " make build - Build the pipeleak binary" - @echo " make test - Run all tests (unit + e2e)" - @echo " make test-unit - Run unit tests only" - @echo " make test-e2e - Run e2e tests (builds binary first)" - @echo " make coverage - Generate test coverage report" - @echo " make coverage-html - Generate and open HTML coverage report" - @echo " make lint - Run golangci-lint" - @echo " make serve-docs - Generate and serve CLI documentation" - @echo " make clean - Remove built artifacts" - -# Build the pipeleak binary + @echo " make build - Build the main pipeleak binary" + @echo " make build-all - Build all binaries (main + platform-specific)" + @echo " make build-gitlab - Build GitLab-specific binary" + @echo " make build-github - Build GitHub-specific binary" + @echo " make build-bitbucket - Build BitBucket-specific binary" + @echo " make build-devops - Build Azure DevOps-specific binary" + @echo " make build-gitea - Build Gitea-specific binary" + @echo " make test - Run all tests (unit + e2e)" + @echo " make test-unit - Run unit tests only" + @echo " make test-e2e - Run e2e tests (builds binary first)" + @echo " make coverage - Generate test coverage report" + @echo " make coverage-html - Generate and open HTML coverage report" + @echo " make lint - Run golangci-lint" + @echo " make serve-docs - Generate and serve CLI documentation" + @echo " make clean - Remove built artifacts" + +# Build the main pipeleak binary build: @echo "Building pipeleak..." go build -o pipeleak ./cmd/pipeleak +# Build GitLab-specific binary +build-gitlab: + @echo "Building pipeleak-gitlab..." + go build -o pipeleak-gitlab ./cmd/pipeleak-gitlab + +# Build GitHub-specific binary +build-github: + @echo "Building pipeleak-github..." + go build -o pipeleak-github ./cmd/pipeleak-github + +# Build BitBucket-specific binary +build-bitbucket: + @echo "Building pipeleak-bitbucket..." + go build -o pipeleak-bitbucket ./cmd/pipeleak-bitbucket + +# Build Azure DevOps-specific binary +build-devops: + @echo "Building pipeleak-devops..." + go build -o pipeleak-devops ./cmd/pipeleak-devops + +# Build Gitea-specific binary +build-gitea: + @echo "Building pipeleak-gitea..." + go build -o pipeleak-gitea ./cmd/pipeleak-gitea + +# Build all binaries +build-all: build build-gitlab build-github build-bitbucket build-devops build-gitea + @echo "All binaries built successfully" + # Run all tests test: test-unit test-e2e @@ -96,4 +131,9 @@ serve-docs: build clean: @echo "Cleaning up..." rm -f pipeleak pipeleak.exe coverage.out coverage.html + rm -f pipeleak-gitlab pipeleak-gitlab.exe + rm -f pipeleak-github pipeleak-github.exe + rm -f pipeleak-bitbucket pipeleak-bitbucket.exe + rm -f pipeleak-devops pipeleak-devops.exe + rm -f pipeleak-gitea pipeleak-gitea.exe go clean -cache -testcache diff --git a/cmd/pipeleak-bitbucket/main.go b/cmd/pipeleak-bitbucket/main.go new file mode 100644 index 00000000..f44be584 --- /dev/null +++ b/cmd/pipeleak-bitbucket/main.go @@ -0,0 +1,28 @@ +package main + +import ( + "github.com/CompassSecurity/pipeleak/internal/cmd/bitbucket" + "github.com/CompassSecurity/pipeleak/internal/cmd/common" + "github.com/spf13/cobra" +) + +func main() { + common.Run(newRootCmd()) +} + +func newRootCmd() *cobra.Command { + bbCmd := bitbucket.NewBitBucketRootCmd() + bbCmd.Use = "pipeleak-bitbucket" + bbCmd.Short = "Scan BitBucket Pipelines logs and artifacts for secrets" + bbCmd.Long = `Pipeleak-BitBucket scans CI/CD logs and artifacts to detect leaked secrets and pivot from them.` + bbCmd.Version = common.Version + bbCmd.GroupID = "" + + common.SetupPersistentPreRun(bbCmd) + common.AddCommonFlags(bbCmd) + + bbCmd.SetVersionTemplate(`{{.Version}} +`) + + return bbCmd +} diff --git a/cmd/pipeleak-devops/main.go b/cmd/pipeleak-devops/main.go new file mode 100644 index 00000000..91d5c859 --- /dev/null +++ b/cmd/pipeleak-devops/main.go @@ -0,0 +1,28 @@ +package main + +import ( + "github.com/CompassSecurity/pipeleak/internal/cmd/common" + "github.com/CompassSecurity/pipeleak/internal/cmd/devops" + "github.com/spf13/cobra" +) + +func main() { + common.Run(newRootCmd()) +} + +func newRootCmd() *cobra.Command { + adCmd := devops.NewAzureDevOpsRootCmd() + adCmd.Use = "pipeleak-devops" + adCmd.Short = "Scan Azure DevOps Pipelines logs and artifacts for secrets" + adCmd.Long = `Pipeleak-DevOps scans CI/CD logs and artifacts to detect leaked secrets and pivot from them.` + adCmd.Version = common.Version + adCmd.GroupID = "" + + common.SetupPersistentPreRun(adCmd) + common.AddCommonFlags(adCmd) + + adCmd.SetVersionTemplate(`{{.Version}} +`) + + return adCmd +} diff --git a/cmd/pipeleak-gitea/main.go b/cmd/pipeleak-gitea/main.go new file mode 100644 index 00000000..fa4822a0 --- /dev/null +++ b/cmd/pipeleak-gitea/main.go @@ -0,0 +1,28 @@ +package main + +import ( + "github.com/CompassSecurity/pipeleak/internal/cmd/common" + "github.com/CompassSecurity/pipeleak/internal/cmd/gitea" + "github.com/spf13/cobra" +) + +func main() { + common.Run(newRootCmd()) +} + +func newRootCmd() *cobra.Command { + giteaCmd := gitea.NewGiteaRootCmd() + giteaCmd.Use = "pipeleak-gitea" + giteaCmd.Short = "Scan Gitea Actions logs and artifacts for secrets" + giteaCmd.Long = `Pipeleak-Gitea scans CI/CD logs and artifacts to detect leaked secrets and pivot from them.` + giteaCmd.Version = common.Version + giteaCmd.GroupID = "" + + common.SetupPersistentPreRun(giteaCmd) + common.AddCommonFlags(giteaCmd) + + giteaCmd.SetVersionTemplate(`{{.Version}} +`) + + return giteaCmd +} diff --git a/cmd/pipeleak-github/main.go b/cmd/pipeleak-github/main.go new file mode 100644 index 00000000..d1443cd9 --- /dev/null +++ b/cmd/pipeleak-github/main.go @@ -0,0 +1,28 @@ +package main + +import ( + "github.com/CompassSecurity/pipeleak/internal/cmd/common" + "github.com/CompassSecurity/pipeleak/internal/cmd/github" + "github.com/spf13/cobra" +) + +func main() { + common.Run(newRootCmd()) +} + +func newRootCmd() *cobra.Command { + ghCmd := github.NewGitHubRootCmd() + ghCmd.Use = "pipeleak-github" + ghCmd.Short = "Scan GitHub Actions logs and artifacts for secrets" + ghCmd.Long = `Pipeleak-GitHub scans CI/CD logs and artifacts to detect leaked secrets and pivot from them.` + ghCmd.Version = common.Version + ghCmd.GroupID = "" + + common.SetupPersistentPreRun(ghCmd) + common.AddCommonFlags(ghCmd) + + ghCmd.SetVersionTemplate(`{{.Version}} +`) + + return ghCmd +} diff --git a/cmd/pipeleak-gitlab/main.go b/cmd/pipeleak-gitlab/main.go new file mode 100644 index 00000000..edf289c6 --- /dev/null +++ b/cmd/pipeleak-gitlab/main.go @@ -0,0 +1,28 @@ +package main + +import ( + "github.com/CompassSecurity/pipeleak/internal/cmd/common" + "github.com/CompassSecurity/pipeleak/internal/cmd/gitlab" + "github.com/spf13/cobra" +) + +func main() { + common.Run(newRootCmd()) +} + +func newRootCmd() *cobra.Command { + glCmd := gitlab.NewGitLabRootCmd() + glCmd.Use = "pipeleak-gitlab" + glCmd.Short = "Scan GitLab CI/CD logs and artifacts for secrets" + glCmd.Long = `Pipeleak-GitLab scans CI/CD logs and artifacts to detect leaked secrets and pivot from them.` + glCmd.Version = common.Version + glCmd.GroupID = "" + + common.SetupPersistentPreRun(glCmd) + common.AddCommonFlags(glCmd) + + glCmd.SetVersionTemplate(`{{.Version}} +`) + + return glCmd +} diff --git a/goreleaser.yaml b/goreleaser.yaml index 287fcb9e..8473a872 100644 --- a/goreleaser.yaml +++ b/goreleaser.yaml @@ -1,8 +1,7 @@ version: 2 project_name: pipeleak builds: - - - id: "pipeleak" + - id: "pipeleak" main: ./cmd/pipeleak binary: pipeleak goos: @@ -15,4 +14,80 @@ builds: - -s -w - -X github.com/CompassSecurity/pipeleak/internal/cmd.Version={{.Version}} - -X github.com/CompassSecurity/pipeleak/internal/cmd.Commit={{.Commit}} - - -X github.com/CompassSecurity/pipeleak/internal/cmd.Date={{.Date}} \ No newline at end of file + - -X github.com/CompassSecurity/pipeleak/internal/cmd.Date={{.Date}} + # Platform-specific builds - only built on release (tagged builds) + - id: "pipeleak-gitlab" + skip: "{{ .IsSnapshot }}" + main: ./cmd/pipeleak-gitlab + binary: pipeleak-gitlab + goos: + - linux + - windows + goarch: + - amd64 + - arm64 + ldflags: + - -s -w + - -X github.com/CompassSecurity/pipeleak/internal/cmd/common.Version={{.Version}} + - -X github.com/CompassSecurity/pipeleak/internal/cmd/common.Commit={{.Commit}} + - -X github.com/CompassSecurity/pipeleak/internal/cmd/common.Date={{.Date}} + - id: "pipeleak-github" + skip: "{{ .IsSnapshot }}" + main: ./cmd/pipeleak-github + binary: pipeleak-github + goos: + - linux + - windows + goarch: + - amd64 + - arm64 + ldflags: + - -s -w + - -X github.com/CompassSecurity/pipeleak/internal/cmd/common.Version={{.Version}} + - -X github.com/CompassSecurity/pipeleak/internal/cmd/common.Commit={{.Commit}} + - -X github.com/CompassSecurity/pipeleak/internal/cmd/common.Date={{.Date}} + - id: "pipeleak-bitbucket" + skip: "{{ .IsSnapshot }}" + main: ./cmd/pipeleak-bitbucket + binary: pipeleak-bitbucket + goos: + - linux + - windows + goarch: + - amd64 + - arm64 + ldflags: + - -s -w + - -X github.com/CompassSecurity/pipeleak/internal/cmd/common.Version={{.Version}} + - -X github.com/CompassSecurity/pipeleak/internal/cmd/common.Commit={{.Commit}} + - -X github.com/CompassSecurity/pipeleak/internal/cmd/common.Date={{.Date}} + - id: "pipeleak-devops" + skip: "{{ .IsSnapshot }}" + main: ./cmd/pipeleak-devops + binary: pipeleak-devops + goos: + - linux + - windows + goarch: + - amd64 + - arm64 + ldflags: + - -s -w + - -X github.com/CompassSecurity/pipeleak/internal/cmd/common.Version={{.Version}} + - -X github.com/CompassSecurity/pipeleak/internal/cmd/common.Commit={{.Commit}} + - -X github.com/CompassSecurity/pipeleak/internal/cmd/common.Date={{.Date}} + - id: "pipeleak-gitea" + skip: "{{ .IsSnapshot }}" + main: ./cmd/pipeleak-gitea + binary: pipeleak-gitea + goos: + - linux + - windows + goarch: + - amd64 + - arm64 + ldflags: + - -s -w + - -X github.com/CompassSecurity/pipeleak/internal/cmd/common.Version={{.Version}} + - -X github.com/CompassSecurity/pipeleak/internal/cmd/common.Commit={{.Commit}} + - -X github.com/CompassSecurity/pipeleak/internal/cmd/common.Date={{.Date}} diff --git a/internal/cmd/common/common.go b/internal/cmd/common/common.go new file mode 100644 index 00000000..3b79054c --- /dev/null +++ b/internal/cmd/common/common.go @@ -0,0 +1,271 @@ +// Package common provides shared functionality for pipeleak platform-specific binaries. +package common + +import ( + "bytes" + "encoding/json" + "io" + "os" + "runtime" + "time" + + "github.com/CompassSecurity/pipeleak/pkg/format" + "github.com/CompassSecurity/pipeleak/pkg/logging" + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" + "github.com/spf13/cobra" + "golang.org/x/term" +) + +// Version information - set via ldflags during build +var ( + Version = "dev" + Commit = "none" + Date = "unknown" +) + +// Log configuration +var ( + originalTermState *term.State + JsonLogoutput bool + LogFile string + LogColor bool + LogDebug bool + LogLevel string +) + +// TerminalRestorer is a function that can be called to restore terminal state +var TerminalRestorer func() + +// CustomWriter wraps an os.File with proper cross-platform newline handling +type CustomWriter struct { + Writer *os.File +} + +func (cw *CustomWriter) Write(p []byte) (n int, err error) { + originalLen := len(p) + + if bytes.HasSuffix(p, []byte("\n")) { + p = bytes.TrimSuffix(p, []byte("\n")) + } + + // necessary as to: https://github.com/rs/zerolog/blob/master/log.go#L474 + newlineChars := []byte("\n") + if runtime.GOOS == "windows" { + newlineChars = []byte("\n\r") + } + + modified := append(p, newlineChars...) + + written, err := cw.Writer.Write(modified) + if err != nil { + return 0, err + } + + if written != len(modified) { + return 0, io.ErrShortWrite + } + + return originalLen, nil +} + +// TerminalRestoringWriter wraps an io.Writer to restore terminal state on fatal logs +type TerminalRestoringWriter struct { + underlying io.Writer +} + +func (w *TerminalRestoringWriter) Write(p []byte) (n int, err error) { + var logEntry map[string]interface{} + if err := json.Unmarshal(p, &logEntry); err == nil { + if level, ok := logEntry["level"].(string); ok && level == "fatal" { + _, _ = w.underlying.Write(p) + RestoreTerminalState() + os.Exit(1) + } + } + return w.underlying.Write(p) +} + +// FatalHook is a zerolog hook that restores terminal state before fatal exits +type FatalHook struct{} + +func (h FatalHook) Run(e *zerolog.Event, level zerolog.Level, msg string) { + if level == zerolog.FatalLevel { + if TerminalRestorer != nil { + TerminalRestorer() + } + } +} + +// SaveTerminalState saves the current terminal state for later restoration +func SaveTerminalState() { + if term.IsTerminal(int(os.Stdin.Fd())) { + state, err := term.GetState(int(os.Stdin.Fd())) + if err == nil { + originalTermState = state + } + } +} + +// RestoreTerminalState restores the terminal to its saved state +func RestoreTerminalState() { + if originalTermState != nil { + _ = term.Restore(int(os.Stdin.Fd()), originalTermState) + } +} + +// InitLogger initializes the zerolog logger with the configured options +func InitLogger(cmd *cobra.Command) { + defaultOut := &CustomWriter{Writer: os.Stdout} + colorEnabled := LogColor + + if LogFile != "" { + // #nosec G304 - User-provided log file path via --log-file flag, user controls their own filesystem + runLogFile, err := os.OpenFile( + LogFile, + os.O_APPEND|os.O_CREATE|os.O_WRONLY, + format.FileUserReadWrite, + ) + if err != nil { + panic(err) + } + defaultOut = &CustomWriter{Writer: runLogFile} + + rootFlags := cmd.Root().PersistentFlags() + if !rootFlags.Changed("color") { + colorEnabled = false + } + } + + fatalHook := FatalHook{} + + if JsonLogoutput { + // For JSON output, wrap with HitLevelWriter to transform level field + hitWriter := &logging.HitLevelWriter{} + hitWriter.SetOutput(defaultOut) + logging.SetGlobalHitWriter(hitWriter) + log.Logger = zerolog.New(hitWriter).With().Timestamp().Logger().Hook(fatalHook) + } else { + // For console output, use custom FormatLevel to color the hit level + output := zerolog.ConsoleWriter{ + Out: defaultOut, + TimeFormat: time.RFC3339, + NoColor: !colorEnabled, + FormatLevel: formatLevelWithHitColor(colorEnabled), + } + // Wrap with HitLevelWriter to transform JSON before ConsoleWriter processes it + hitWriter := &logging.HitLevelWriter{} + hitWriter.SetOutput(&output) + logging.SetGlobalHitWriter(hitWriter) + log.Logger = zerolog.New(hitWriter).With().Timestamp().Logger().Hook(fatalHook) + } +} + +// formatLevelWithHitColor returns a custom level formatter that adds a distinct color for the "hit" level. +// The hit level uses magenta (color 35) to distinguish it from other log levels. +func formatLevelWithHitColor(colorEnabled bool) zerolog.Formatter { + return func(i interface{}) string { + var level string + if ll, ok := i.(string); ok { + level = ll + } else { + return "" + } + + if !colorEnabled { + return level + } + + // Custom color for hit level - using bright magenta (35) to stand out + if level == "hit" { + return "\x1b[35m" + level + "\x1b[0m" + } + + // Use zerolog's default colors for other levels + switch level { + case "trace": + return "\x1b[90m" + level + "\x1b[0m" + case "debug": + return level + case "info": + return "\x1b[32m" + level + "\x1b[0m" + case "warn": + return "\x1b[33m" + level + "\x1b[0m" + case "error": + return "\x1b[31m" + level + "\x1b[0m" + case "fatal": + return "\x1b[31m" + level + "\x1b[0m" + case "panic": + return "\x1b[31m" + level + "\x1b[0m" + default: + return level + } + } +} + +// SetGlobalLogLevel sets the global log level based on the configured options +func SetGlobalLogLevel(cmd *cobra.Command) { + if LogLevel != "" { + switch LogLevel { + case "trace": + zerolog.SetGlobalLevel(zerolog.TraceLevel) + log.Trace().Msg("Log level set to trace (explicit)") + case "debug": + zerolog.SetGlobalLevel(zerolog.DebugLevel) + log.Debug().Msg("Log level set to debug (explicit)") + case "info": + zerolog.SetGlobalLevel(zerolog.InfoLevel) + log.Info().Msg("Log level set to info (explicit)") + case "warn": + zerolog.SetGlobalLevel(zerolog.WarnLevel) + log.Warn().Msg("Log level set to warn (explicit)") + case "error": + zerolog.SetGlobalLevel(zerolog.ErrorLevel) + log.Error().Msg("Log level set to error (explicit)") + default: + zerolog.SetGlobalLevel(zerolog.InfoLevel) + log.Warn().Str("logLevelSpecified", LogLevel).Msg("Invalid log level, defaulting to info") + } + return + } + + if LogDebug { + zerolog.SetGlobalLevel(zerolog.DebugLevel) + log.Debug().Msg("Log level set to debug (-v)") + return + } + + zerolog.SetGlobalLevel(zerolog.InfoLevel) + log.Info().Msg("Log level set to info (default)") +} + +// AddCommonFlags adds the common logging and output flags to a cobra command +func AddCommonFlags(cmd *cobra.Command) { + cmd.PersistentFlags().BoolVarP(&JsonLogoutput, "json", "", false, "Use JSON as log output format") + cmd.PersistentFlags().StringVarP(&LogFile, "logfile", "l", "", "Log output to a file") + cmd.PersistentFlags().BoolVarP(&LogDebug, "verbose", "v", false, "Enable debug logging (shortcut for --log-level=debug)") + cmd.PersistentFlags().StringVar(&LogLevel, "log-level", "", "Set log level globally (debug, info, warn, error). Example: --log-level=warn") + cmd.PersistentFlags().BoolVar(&LogColor, "color", true, "Enable colored log output (auto-disabled when using --logfile)") +} + +// SetupPersistentPreRun sets up the PersistentPreRun handler for logging initialization +func SetupPersistentPreRun(cmd *cobra.Command) { + cmd.PersistentPreRun = func(c *cobra.Command, args []string) { + InitLogger(c) + SetGlobalLogLevel(c) + go logging.ShortcutListeners(nil) + } +} + +// Run executes the common startup sequence and runs the provided root command +func Run(rootCmd *cobra.Command) { + SaveTerminalState() + defer RestoreTerminalState() + + TerminalRestorer = RestoreTerminalState + + err := rootCmd.Execute() + if err != nil { + os.Exit(1) + } +}