Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
custom webhooks and other webhook modifications (#43)
* tune for optimal webhook behaviour * raises loglevel threshold for webhook notifications * remove ANSI color codes from HTTP payloads for cleaner notifications * modify docs/code to reflect generic capabilities of webhooks * allow for custom webhook payloads * match newlines with rest of project
- Loading branch information
1 parent
71f4b5f
commit eadd127
Showing
7 changed files
with
341 additions
and
325 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,89 +1,90 @@ | ||
| package core | ||
|
|
||
| import ( | ||
| "errors" | ||
| "io/ioutil" | ||
| "os" | ||
| "path" | ||
| "path/filepath" | ||
| "strings" | ||
|
|
||
| "gopkg.in/yaml.v3" | ||
| ) | ||
|
|
||
| type Config struct { | ||
| GitHubAccessTokens []string `yaml:"github_access_tokens"` | ||
| SlackWebhook string `yaml:"slack_webhook,omitempty"` | ||
| BlacklistedExtensions []string `yaml:"blacklisted_extensions"` | ||
| BlacklistedPaths []string `yaml:"blacklisted_paths"` | ||
| BlacklistedEntropyExtensions []string `yaml:"blacklisted_entropy_extensions"` | ||
| Signatures []ConfigSignature `yaml:"signatures"` | ||
| } | ||
|
|
||
| type ConfigSignature struct { | ||
| Name string `yaml:"name"` | ||
| Part string `yaml:"part"` | ||
| Match string `yaml:"match,omitempty"` | ||
| Regex string `yaml:"regex,omitempty"` | ||
| Verifier string `yaml:"verifier,omitempty"` | ||
| } | ||
|
|
||
| func ParseConfig(options *Options) (*Config, error) { | ||
| config := &Config{} | ||
| var ( | ||
| data []byte | ||
| err error | ||
| ) | ||
|
|
||
| if len(*options.ConfigPath) > 0 { | ||
| data, err = ioutil.ReadFile(path.Join(*options.ConfigPath, "config.yaml")) | ||
| if err != nil { | ||
| return config, err | ||
| } | ||
| } else { | ||
| // Trying to first find the configuration next to executable | ||
| // Helps e.g. with Drone where workdir is different than shhgit dir | ||
| ex, err := os.Executable() | ||
| dir := filepath.Dir(ex) | ||
| data, err = ioutil.ReadFile(path.Join(dir, "config.yaml")) | ||
| if err != nil { | ||
| dir, _ = os.Getwd() | ||
| data, err = ioutil.ReadFile(path.Join(dir, "config.yaml")) | ||
| if err != nil { | ||
| return config, err | ||
| } | ||
| } | ||
| } | ||
|
|
||
| err = yaml.Unmarshal(data, config) | ||
| if err != nil { | ||
| return config, err | ||
| } | ||
|
|
||
| if !options.LocalRun && (len(config.GitHubAccessTokens) < 1 || strings.TrimSpace(strings.Join(config.GitHubAccessTokens, "")) == "") { | ||
| return config, errors.New("You need to provide at least one GitHub Access Token. See https://help.github.com/en/articles/creating-a-personal-access-token-for-the-command-line") | ||
| } | ||
|
|
||
| for i := 0; i < len(config.GitHubAccessTokens); i++ { | ||
| config.GitHubAccessTokens[i] = os.ExpandEnv(config.GitHubAccessTokens[i]) | ||
| } | ||
|
|
||
| if len(config.SlackWebhook) > 0 { | ||
| config.SlackWebhook = os.ExpandEnv(config.SlackWebhook) | ||
| } | ||
|
|
||
| return config, nil | ||
| } | ||
|
|
||
| func (c *Config) UnmarshalYAML(unmarshal func(interface{}) error) error { | ||
| *c = Config{} | ||
| type plain Config | ||
|
|
||
| err := unmarshal((*plain)(c)) | ||
|
|
||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| return nil | ||
| } | ||
| package core | ||
|
|
||
| import ( | ||
| "errors" | ||
| "io/ioutil" | ||
| "os" | ||
| "path" | ||
| "path/filepath" | ||
| "strings" | ||
|
|
||
| "gopkg.in/yaml.v3" | ||
| ) | ||
|
|
||
| type Config struct { | ||
| GitHubAccessTokens []string `yaml:"github_access_tokens"` | ||
| Webhook string `yaml:"webhook,omitempty"` | ||
| WebhookPayload string `yaml:"webhook_payload,omitempty"` | ||
| BlacklistedExtensions []string `yaml:"blacklisted_extensions"` | ||
| BlacklistedPaths []string `yaml:"blacklisted_paths"` | ||
| BlacklistedEntropyExtensions []string `yaml:"blacklisted_entropy_extensions"` | ||
| Signatures []ConfigSignature `yaml:"signatures"` | ||
| } | ||
|
|
||
| type ConfigSignature struct { | ||
| Name string `yaml:"name"` | ||
| Part string `yaml:"part"` | ||
| Match string `yaml:"match,omitempty"` | ||
| Regex string `yaml:"regex,omitempty"` | ||
| Verifier string `yaml:"verifier,omitempty"` | ||
| } | ||
|
|
||
| func ParseConfig(options *Options) (*Config, error) { | ||
| config := &Config{} | ||
| var ( | ||
| data []byte | ||
| err error | ||
| ) | ||
|
|
||
| if len(*options.ConfigPath) > 0 { | ||
| data, err = ioutil.ReadFile(path.Join(*options.ConfigPath, "config.yaml")) | ||
| if err != nil { | ||
| return config, err | ||
| } | ||
| } else { | ||
| // Trying to first find the configuration next to executable | ||
| // Helps e.g. with Drone where workdir is different than shhgit dir | ||
| ex, err := os.Executable() | ||
| dir := filepath.Dir(ex) | ||
| data, err = ioutil.ReadFile(path.Join(dir, "config.yaml")) | ||
| if err != nil { | ||
| dir, _ = os.Getwd() | ||
| data, err = ioutil.ReadFile(path.Join(dir, "config.yaml")) | ||
| if err != nil { | ||
| return config, err | ||
| } | ||
| } | ||
| } | ||
|
|
||
| err = yaml.Unmarshal(data, config) | ||
| if err != nil { | ||
| return config, err | ||
| } | ||
|
|
||
| if !options.LocalRun && (len(config.GitHubAccessTokens) < 1 || strings.TrimSpace(strings.Join(config.GitHubAccessTokens, "")) == "") { | ||
| return config, errors.New("You need to provide at least one GitHub Access Token. See https://help.github.com/en/articles/creating-a-personal-access-token-for-the-command-line") | ||
| } | ||
|
|
||
| for i := 0; i < len(config.GitHubAccessTokens); i++ { | ||
| config.GitHubAccessTokens[i] = os.ExpandEnv(config.GitHubAccessTokens[i]) | ||
| } | ||
|
|
||
| if len(config.Webhook) > 0 { | ||
| config.Webhook = os.ExpandEnv(config.Webhook) | ||
| } | ||
|
|
||
| return config, nil | ||
| } | ||
|
|
||
| func (c *Config) UnmarshalYAML(unmarshal func(interface{}) error) error { | ||
| *c = Config{} | ||
| type plain Config | ||
|
|
||
| err := unmarshal((*plain)(c)) | ||
|
|
||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| return nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,54 +1,54 @@ | ||
| package core | ||
|
|
||
| import ( | ||
| "flag" | ||
| "os" | ||
| "path/filepath" | ||
| ) | ||
|
|
||
| type Options struct { | ||
| Threads *int | ||
| Silent *bool | ||
| Debug *bool | ||
| MaximumRepositorySize *uint | ||
| MaximumFileSize *uint | ||
| CloneRepositoryTimeout *uint | ||
| EntropyThreshold *float64 | ||
| MinimumStars *uint | ||
| PathChecks *bool | ||
| ProcessGists *bool | ||
| TempDirectory *string | ||
| CsvPath *string | ||
| SearchQuery *string | ||
| Local *string | ||
| LocalRun bool | ||
| ConfigPath *string | ||
| } | ||
|
|
||
| func ParseOptions() (*Options, error) { | ||
| options := &Options{ | ||
| Threads: flag.Int("threads", 0, "Number of concurrent threads (default number of logical CPUs)"), | ||
| Silent: flag.Bool("silent", false, "Suppress all output except for errors"), | ||
| Debug: flag.Bool("debug", false, "Print debugging information"), | ||
| MaximumRepositorySize: flag.Uint("maximum-repository-size", 5120, "Maximum repository size to process in KB"), | ||
| MaximumFileSize: flag.Uint("maximum-file-size", 512, "Maximum file size to process in KB"), | ||
| CloneRepositoryTimeout: flag.Uint("clone-repository-timeout", 10, "Maximum time it should take to clone a repository in seconds. Increase this if you have a slower connection"), | ||
| EntropyThreshold: flag.Float64("entropy-threshold", 5.0, "Set to 0 to disable entropy checks"), | ||
| MinimumStars: flag.Uint("minimum-stars", 0, "Only process repositories with this many stars. Default 0 will ignore star count"), | ||
| PathChecks: flag.Bool("path-checks", true, "Set to false to disable checking of filepaths, i.e. just match regex patterns of file contents"), | ||
| ProcessGists: flag.Bool("process-gists", true, "Will watch and process Gists. Set to false to disable."), | ||
| TempDirectory: flag.String("temp-directory", filepath.Join(os.TempDir(), Name), "Directory to process and store repositories/matches"), | ||
| CsvPath: flag.String("csv-path", "", "CSV file path to log found secrets to. Leave blank to disable"), | ||
| SearchQuery: flag.String("search-query", "", "Specify a search string to ignore signatures and filter on files containing this string (regex compatible)"), | ||
| Local: flag.String("local", "", "Specify local directory (absolute path) which to scan. Scans only given directory recursively. No need to have Githib tokens with local run."), | ||
| ConfigPath: flag.String("config-path", "", "Searches for config.yaml from given directory. If not set, tries to find if from shhgit binary's and current directory"), | ||
| } | ||
|
|
||
| flag.Parse() | ||
|
|
||
| if len(*options.Local) > 0 { | ||
| options.LocalRun = true | ||
| } | ||
|
|
||
| return options, nil | ||
| } | ||
| package core | ||
|
|
||
| import ( | ||
| "flag" | ||
| "os" | ||
| "path/filepath" | ||
| ) | ||
|
|
||
| type Options struct { | ||
| Threads *int | ||
| Silent *bool | ||
| Debug *bool | ||
| MaximumRepositorySize *uint | ||
| MaximumFileSize *uint | ||
| CloneRepositoryTimeout *uint | ||
| EntropyThreshold *float64 | ||
| MinimumStars *uint | ||
| PathChecks *bool | ||
| ProcessGists *bool | ||
| TempDirectory *string | ||
| CsvPath *string | ||
| SearchQuery *string | ||
| Local *string | ||
| LocalRun bool | ||
| ConfigPath *string | ||
| } | ||
|
|
||
| func ParseOptions() (*Options, error) { | ||
| options := &Options{ | ||
| Threads: flag.Int("threads", 0, "Number of concurrent threads (default number of logical CPUs)"), | ||
| Silent: flag.Bool("silent", false, "Suppress all output except for errors"), | ||
| Debug: flag.Bool("debug", false, "Print debugging information"), | ||
| MaximumRepositorySize: flag.Uint("maximum-repository-size", 5120, "Maximum repository size to process in KB"), | ||
| MaximumFileSize: flag.Uint("maximum-file-size", 512, "Maximum file size to process in KB"), | ||
| CloneRepositoryTimeout: flag.Uint("clone-repository-timeout", 10, "Maximum time it should take to clone a repository in seconds. Increase this if you have a slower connection"), | ||
| EntropyThreshold: flag.Float64("entropy-threshold", 5.0, "Set to 0 to disable entropy checks"), | ||
| MinimumStars: flag.Uint("minimum-stars", 0, "Only process repositories with this many stars. Default 0 will ignore star count"), | ||
| PathChecks: flag.Bool("path-checks", true, "Set to false to disable checking of filepaths, i.e. just match regex patterns of file contents"), | ||
| ProcessGists: flag.Bool("process-gists", true, "Will watch and process Gists. Set to false to disable."), | ||
| TempDirectory: flag.String("temp-directory", filepath.Join(os.TempDir(), Name), "Directory to process and store repositories/matches"), | ||
| CsvPath: flag.String("csv-path", "", "CSV file path to log found secrets to. Leave blank to disable"), | ||
| SearchQuery: flag.String("search-query", "", "Specify a search string to ignore signatures and filter on files containing this string (regex compatible)"), | ||
| Local: flag.String("local", "", "Specify local directory (absolute path) which to scan. Scans only given directory recursively. No need to have Githib tokens with local run."), | ||
| ConfigPath: flag.String("config-path", "", "Searches for config.yaml from given directory. If not set, tries to find if from shhgit binary's and current directory"), | ||
| } | ||
|
|
||
| flag.Parse() | ||
|
|
||
| if len(*options.Local) > 0 { | ||
| options.LocalRun = true | ||
| } | ||
|
|
||
| return options, nil | ||
| } |
Oops, something went wrong.