Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Compile regexp's at build time. Use raw string literals for patterns … #167

Merged
merged 1 commit into from
Oct 31, 2019
Merged

Compile regexp's at build time. Use raw string literals for patterns … #167

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 14 additions & 20 deletions detector/filecontent_credit_card_detector.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,19 @@ type CreditCardDetector struct {
creditCardRegex []*regexp.Regexp
}

var (
creditCardPatterns = []*regexp.Regexp{
regexp.MustCompile(`(?:3[47][0-9]{13})`),
regexp.MustCompile(`(?:3(?:0[0-5]|[68][0-9])[0-9]{11})`),
regexp.MustCompile(`^65[4-9][0-9]{13}|64[4-9][0-9]{13}|6011[0-9]{12}|(622(?:12[6-9]|1[3-9][0-9]|[2-8][0-9][0-9]|9[01][0-9]|92[0-5])[0-9]{10})$`),
regexp.MustCompile(`^(?:2131|1800|35\d{3})\d{11}$`),
regexp.MustCompile(`^(5018|5020|5038|6304|6759|6761|6763)[0-9]{8,15}$`),
regexp.MustCompile(`(?:(?:5[1-5][0-9]{2}|222[1-9]|22[3-9][0-9]|2[3-6][0-9]{2}|27[01][0-9]|2720)[0-9]{12})`),
regexp.MustCompile(`((?:4[0-9]{12})(?:[0-9]{3})?)`),

}
)

func (detector CreditCardDetector) checkCreditCardNumber(content string) string {
if !isLuhnNumber(content) {
return ""
Expand All @@ -18,27 +31,8 @@ func (detector CreditCardDetector) checkCreditCardNumber(content string) string
return ""
}

func initPatternForCreditCard() *CreditCardDetector {

patterns := [...]string{
"(?:3[47][0-9]{13})",
"(?:3(?:0[0-5]|[68][0-9])[0-9]{11})",
"^65[4-9][0-9]{13}|64[4-9][0-9]{13}|6011[0-9]{12}|(622(?:12[6-9]|1[3-9][0-9]|[2-8][0-9][0-9]|9[01][0-9]|92[0-5])[0-9]{10})$",
"^(?:2131|1800|35\\d{3})\\d{11}$",
"^(5018|5020|5038|6304|6759|6761|6763)[0-9]{8,15}$",
"(?:(?:5[1-5][0-9]{2}|222[1-9]|22[3-9][0-9]|2[3-6][0-9]{2}|27[01][0-9]|2720)[0-9]{12})",
"((?:4[0-9]{12})(?:[0-9]{3})?)",
}

var creditCardPatterns = make([]*regexp.Regexp, len(patterns))
for i, pattern := range patterns {
creditCardPatterns[i], _ = regexp.Compile(pattern)
}
return &CreditCardDetector{creditCardPatterns}
}

func NewCreditCardDetector() *CreditCardDetector {
return initPatternForCreditCard()
return &CreditCardDetector{creditCardPatterns}
}

func isLuhnNumber(content string) bool {
Expand Down
105 changes: 53 additions & 52 deletions detector/filename_detector.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,57 @@ import (
log "github.com/Sirupsen/logrus"
)

var (
filenamePatterns = []*regexp.Regexp{
regexp.MustCompile(`^.+_rsa$`),
regexp.MustCompile(`^.+_dsa.*$`),
regexp.MustCompile(`^.+_ed25519$`),
regexp.MustCompile(`^.+_ecdsa$`),
regexp.MustCompile(`^\.\w+_history$`),
regexp.MustCompile(`^.+\.pem$`),
regexp.MustCompile(`^.+\.ppk$`),
regexp.MustCompile(`^.+\.key(pair)?$`),
regexp.MustCompile(`^.+\.pkcs12$`),
regexp.MustCompile(`^.+\.pfx$`),
regexp.MustCompile(`^.+\.p12$`),
regexp.MustCompile(`^.+\.asc$`),
regexp.MustCompile(`^\.?htpasswd$`),
regexp.MustCompile(`^\.?netrc$`),
regexp.MustCompile(`^.*\.tblk$`),
regexp.MustCompile(`^.*\.ovpn$`),
regexp.MustCompile(`^.*\.kdb$`),
regexp.MustCompile(`^.*\.agilekeychain$`),
regexp.MustCompile(`^.*\.keychain$`),
regexp.MustCompile(`^.*\.key(store|ring)$`),
regexp.MustCompile(`^jenkins\.plugins\.publish_over_ssh\.BapSshPublisherPlugin.xml$`),
regexp.MustCompile(`^credentials\.xml$`),
regexp.MustCompile(`^.*\.pubxml(\.user)?$`),
regexp.MustCompile(`^\.?s3cfg$`),
regexp.MustCompile(`^\.gitrobrc$`),
regexp.MustCompile(`^\.?(bash|zsh)rc$`),
regexp.MustCompile(`^\.?(bash_|zsh_)?profile$`),
regexp.MustCompile(`^\.?(bash_|zsh_)?aliases$`),
regexp.MustCompile(`^secret_token.rb$`),
regexp.MustCompile(`^omniauth.rb$`),
regexp.MustCompile(`^carrierwave.rb$`),
regexp.MustCompile(`^schema.rb$`),
regexp.MustCompile(`^database.yml$`),
regexp.MustCompile(`^settings.py$`),
regexp.MustCompile(`^.*(config)(\.inc)?\.php$`),
regexp.MustCompile(`^LocalSettings.php$`),
regexp.MustCompile(`\.?env`),
regexp.MustCompile(`\bdump|dump\b`),
regexp.MustCompile(`\bsql|sql\b`),
regexp.MustCompile(`\bdump|dump\b`),
regexp.MustCompile(`password`),
regexp.MustCompile(`backup`),
regexp.MustCompile(`private.*key`),
regexp.MustCompile(`(oauth).*(token)`),
regexp.MustCompile(`^.*\.log$`),
regexp.MustCompile(`^\.?kwallet$`),
regexp.MustCompile(`^\.?gnucash$`),
}
)
//FileNameDetector represents tests performed against the fileName of the Additions.
//The Paths of the supplied Additions are tested against the configured patterns and if any of them match, it is logged as a failure during the run
type FileNameDetector struct {
Expand All @@ -17,61 +68,11 @@ type FileNameDetector struct {

//DefaultFileNameDetector returns a FileNameDetector that tests Additions against the pre-configured patterns
func DefaultFileNameDetector() Detector {
return NewFileNameDetector("^.+_rsa$",
"^.+_dsa.*$",
"^.+_ed25519$",
"^.+_ecdsa$",
"^\\.\\w+_history$",
"^.+\\.pem$",
"^.+\\.ppk$",
"^.+\\.key(pair)?$",
"^.+\\.pkcs12$",
"^.+\\.pfx$",
"^.+\\.p12$",
"^.+\\.asc$",
"^\\.?htpasswd$",
"^\\.?netrc$",
"^.*\\.tblk$",
"^.*\\.ovpn$",
"^.*\\.kdb$",
"^.*\\.agilekeychain$",
"^.*\\.keychain$",
"^.*\\.key(store|ring)$",
"^jenkins\\.plugins\\.publish_over_ssh\\.BapSshPublisherPlugin.xml$",
"^credentials\\.xml$",
"^.*\\.pubxml(\\.user)?$",
"^\\.?s3cfg$",
"^\\.gitrobrc$",
"^\\.?(bash|zsh)rc$",
"^\\.?(bash_|zsh_)?profile$",
"^\\.?(bash_|zsh_)?aliases$",
"^secret_token.rb$",
"^omniauth.rb$",
"^carrierwave.rb$",
"^schema.rb$",
"^database.yml$",
"^settings.py$",
"^.*(config)(\\.inc)?\\.php$",
"^LocalSettings.php$",
"\\.?env",
"\\bdump|dump\\b",
"\\bsql|sql\\b",
"\\bdump|dump\\b",
"password",
"backup",
"private.*key",
"(oauth).*(token)",
"^.*\\.log$",
"^\\.?kwallet$",
"^\\.?gnucash$")
return NewFileNameDetector(filenamePatterns)
}

//NewFileNameDetector returns a FileNameDetector that tests Additions against the supplied patterns
func NewFileNameDetector(patternStrings ...string) Detector {
var patterns = make([]*regexp.Regexp, len(patternStrings))
for i, p := range patternStrings {
patterns[i], _ = regexp.Compile(p)
}
func NewFileNameDetector(patterns []*regexp.Regexp) Detector {
return FileNameDetector{patterns}
}

Expand Down
4 changes: 3 additions & 1 deletion detector/filename_detector_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ package detector

import (
"testing"
"regexp"

"talisman/gitrepo"

Expand Down Expand Up @@ -156,7 +157,8 @@ func shouldNotFailWithDefaultDetectorAndIgnores(fileName, ignore string, t *test

func shouldFailWithSpecificPattern(fileName, pattern string, t *testing.T) {
results := NewDetectionResults()
NewFileNameDetector(pattern).Test(additionsNamed(fileName), TalismanRCIgnore{}, results)
pt := regexp.MustCompile(pattern)
NewFileNameDetector([]*regexp.Regexp{pt}).Test(additionsNamed(fileName), TalismanRCIgnore{}, results)
assert.True(t, results.HasFailures(), "Expected file %s to fail the check against the %s pattern", fileName, pattern)
}

Expand Down
17 changes: 10 additions & 7 deletions detector/ignores.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,15 +12,21 @@ import (

const (
//LinePattern represents a line in the ignorefile with an optional comment
LinePattern string = "^([^#]+)?\\s*(#(.*))?$"
LinePattern string = `^([^#]+)?\s*(#(.*))?$`

//IgnoreDetectorCommentPattern represents a special comment that ignores only certain detectors
IgnoreDetectorCommentPattern string = "^ignore:([^\\s]+).*$"
IgnoreDetectorCommentPattern string = `^ignore:([^\s]+).*$`

//DefaultRCFileName represents the name of default file in which all the ignore patterns are configured in new version
DefaultRCFileName string = ".talismanrc"
)

var (
commentPattern = regexp.MustCompile(LinePattern)
ignorePattern = regexp.MustCompile(IgnoreDetectorCommentPattern)
emptyStringPattern = regexp.MustCompile(`^\s*$`)
)

//Ignores represents a set of patterns that have been configured to be ignored by the Detectors.
//Detectors are expected to honor these ignores.
type Ignores struct {
Expand Down Expand Up @@ -74,7 +80,6 @@ func NewTalismanRCIgnore(fileContents []byte) (TalismanRCIgnore) {

func NewIgnore(pattern string, comment string) Ignore {
var ignoredDetectors []string
ignorePattern := regexp.MustCompile(IgnoreDetectorCommentPattern)
match := ignorePattern.FindStringSubmatch(comment)
if match != nil {
ignoredDetectors = strings.Split(match[1], ",")
Expand All @@ -92,14 +97,13 @@ func (i FileIgnoreConfig) isEffective(detectorName string) bool {
contains(i.IgnoreDetectors, detectorName)
}


//NewIgnores builds a new Ignores with the patterns specified in the ignoreSpecs
//Empty lines and comments are ignored.
func NewIgnores(lines ...string) Ignores {
var groups []string
var ignores []Ignore
for _, line := range lines {
var commentPattern = regexp.MustCompile(LinePattern)
groups := commentPattern.FindStringSubmatch(line)
groups = commentPattern.FindStringSubmatch(line)
if len(groups) == 4 {
ignores = append(ignores, NewIgnore(strings.TrimSpace(groups[1]), strings.TrimSpace(groups[3])))
}
Expand Down Expand Up @@ -160,7 +164,6 @@ func (i TalismanRCIgnore) effectiveRules(detectorName string) []string {
}

func isEmptyString(str string) bool {
var emptyStringPattern = regexp.MustCompile("^\\s*$")
return emptyStringPattern.MatchString(str)
}
func contains(s []string, e string) bool {
Expand Down
28 changes: 14 additions & 14 deletions detector/pattern_detector.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,20 +14,20 @@ type PatternDetector struct {

var (
detectorPatterns = []*regexp.Regexp{
regexp.MustCompile("(?i)(['\"_]?password['\"]? *[:=][^,;\n]{8,})"),
regexp.MustCompile("(?i)(['\"_]?pw['\"]? *[:=][^,;\n]{8,})"),
regexp.MustCompile("(?i)(['\"_]?pwd['\"]? *[:=][^,;\n]{8,})"),
regexp.MustCompile("(?i)(['\"_]?pass['\"]? *[:=][^,;\n]{8,})"),
regexp.MustCompile("(?i)(['\"_]?pword['\"]? *[:=][^,;\n]{8,})"),
regexp.MustCompile("(?i)(['\"_]?adminPassword['\"]? *[:=\n][^,;]{8,})"),
regexp.MustCompile("(?i)(['\"_]?passphrase['\"]? *[:=\n][^,;]{8,})"),
regexp.MustCompile("(<[^(><.)]?password[^(><.)]*?>[^(><.)]+</[^(><.)]?password[^(><.)]*?>)"),
regexp.MustCompile("(<[^(><.)]?passphrase[^(><.)]*?>[^(><.)]+</[^(><.)]?passphrase[^(><.)]*?>)"),
regexp.MustCompile("(?i)(<ConsumerKey>\\S*<\\/ConsumerKey>)"),
regexp.MustCompile("(?i)(<ConsumerSecret>\\S*<\\/ConsumerSecret>)"),
regexp.MustCompile("(?i)(AWS[ \\w]+key[ \\w]+[:=])"),
regexp.MustCompile("(?i)(AWS[ \\w]+secret[ \\w]+[:=])"),
regexp.MustCompile("(?s)(BEGIN RSA PRIVATE KEY.*END RSA PRIVATE KEY)"),
regexp.MustCompile(`(?i)(['"_]?password['"]? *[:=][^,;\n]{8,})`),
regexp.MustCompile(`(?i)(['"_]?pw['"]? *[:=][^,;\n]{8,})`),
regexp.MustCompile(`(?i)(['"_]?pwd['"]? *[:=][^,;\n]{8,})`),
regexp.MustCompile(`(?i)(['"_]?pass['"]? *[:=][^,;\n]{8,})`),
regexp.MustCompile(`(?i)(['"_]?pword['"]? *[:=][^,;\n]{8,})`),
regexp.MustCompile(`(?i)(['"_]?adminPassword['"]? *[:=\n][^,;]{8,})`),
regexp.MustCompile(`(?i)(['"_]?passphrase['"]? *[:=\n][^,;]{8,})`),
regexp.MustCompile(`(<[^(><.)]?password[^(><.)]*?>[^(><.)]+</[^(><.)]?password[^(><.)]*?>)`),
regexp.MustCompile(`(<[^(><.)]?passphrase[^(><.)]*?>[^(><.)]+</[^(><.)]?passphrase[^(><.)]*?>)`),
regexp.MustCompile(`(?i)(<ConsumerKey>\S*</ConsumerKey>)`),
regexp.MustCompile(`(?i)(<ConsumerSecret>\S*</ConsumerSecret>)`),
regexp.MustCompile(`(?i)(AWS[ \w]+key[ \w]+[:=])`),
regexp.MustCompile(`(?i)(AWS[ \w]+secret[ \w]+[:=])`),
regexp.MustCompile(`(?s)(BEGIN RSA PRIVATE KEY.*END RSA PRIVATE KEY)`),
}
)

Expand Down