diff --git a/go.mod b/go.mod index c421f6aacff7..d375f74b6f0e 100644 --- a/go.mod +++ b/go.mod @@ -18,6 +18,7 @@ require ( github.com/bitfinexcom/bitfinex-api-go v0.0.0-20210608095005-9e0b26f200fb github.com/bradleyfalzon/ghinstallation/v2 v2.1.0 github.com/crewjam/rfc5424 v0.1.0 + github.com/denisenkom/go-mssqldb v0.12.2 github.com/envoyproxy/protoc-gen-validate v0.6.8 github.com/fatih/color v1.13.0 github.com/felixge/fgprof v0.9.3 @@ -26,6 +27,7 @@ require ( github.com/go-git/go-git/v5 v5.4.2 github.com/go-logr/logr v1.2.3 github.com/go-logr/zapr v1.2.3 + github.com/go-sql-driver/mysql v1.6.0 github.com/golang-jwt/jwt v3.2.2+incompatible github.com/google/go-cmp v0.5.9 github.com/google/go-github/v42 v42.0.0 @@ -35,7 +37,9 @@ require ( github.com/joho/godotenv v1.4.0 github.com/jpillora/overseer v1.1.6 github.com/kylelemons/godebug v1.1.0 + github.com/lib/pq v1.10.7 github.com/mattn/go-colorable v0.1.13 + github.com/mattn/go-sqlite3 v1.14.15 github.com/mholt/archiver/v4 v4.0.0-alpha.7 github.com/paulbellamy/ratecounter v0.2.0 github.com/pkg/errors v0.9.1 @@ -83,6 +87,8 @@ require ( github.com/go-git/go-billy/v5 v5.3.1 // indirect github.com/go-ole/go-ole v1.2.6 // indirect github.com/golang-jwt/jwt/v4 v4.4.1 // indirect + github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe // indirect + github.com/golang-sql/sqlexp v0.1.0 // indirect github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/golang/snappy v0.0.4 // indirect @@ -128,6 +134,5 @@ require ( google.golang.org/appengine v1.6.7 // indirect google.golang.org/grpc v1.47.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 5db418b73743..144fe35c53c0 100644 --- a/go.sum +++ b/go.sum @@ -62,6 +62,9 @@ cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9 cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= cloud.google.com/go/storage v1.22.1/go.mod h1:S8N1cAStu7BOeFfE8KAQzmyyLkK8p/vmRq6kuBTW58Y= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/Azure/azure-sdk-for-go/sdk/azcore v0.19.0/go.mod h1:h6H6c8enJmmocHUbLiiGY6sx7f9i+X3m1CHdd5c6Rdw= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.11.0/go.mod h1:HcM1YX14R7CJcghJGOYCgdezslRSVzqwLf/q+4Y2r/0= +github.com/Azure/azure-sdk-for-go/sdk/internal v0.7.0/go.mod h1:yqy467j36fJxcRV2TzfVZ1pCb5vxm4BtZPUdYWe/Xo8= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-autorest/autorest v0.11.24 h1:1fIGgHKqVm54KIPT+q8Zmd1QlVsmHqeUGso5qm2BqqE= @@ -138,8 +141,11 @@ github.com/crewjam/rfc5424 v0.1.0/go.mod h1:RCi9M3xHVOeerf6ULZzqv2xOGRO/zYaVUeRy github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/denisenkom/go-mssqldb v0.12.2 h1:1OcPn5GBIobjWNd+8yjfHNIaFX14B1pWI3F9HZy5KXw= +github.com/denisenkom/go-mssqldb v0.12.2/go.mod h1:lnIw1mZukFRZDJYQ0Pb833QS2IaC3l5HkEfra2LJ+sk= github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= +github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q= github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo= github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= @@ -188,6 +194,8 @@ github.com/go-logr/zapr v1.2.3 h1:a9vnzlIBPQBBkeaR9IuMUfmVOrQlkoC4YfPoFkX3T7A= github.com/go-logr/zapr v1.2.3/go.mod h1:eIauM6P8qSvTw5o2ez6UEAfGjQKrxQTl5EoK+Qa2oG4= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= +github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/gobwas/httphead v0.0.0-20200921212729-da3d93bc3c58/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= github.com/gobwas/ws v1.0.4/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= @@ -198,6 +206,10 @@ github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzw github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= github.com/golang-jwt/jwt/v4 v4.4.1 h1:pC5DB52sCeK48Wlb9oPcdhnjkz1TKt1D/P7WKJ0kUcQ= github.com/golang-jwt/jwt/v4 v4.4.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe h1:lXe2qZdvpiX5WZkZR4hgp4KJVfY3nMkvmwbVkpv1rVY= +github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= +github.com/golang-sql/sqlexp v0.1.0 h1:ZCD6MBpcuOVfGVqsEmY5/4FtYiKz6tSyUv9LPEDei6A= +github.com/golang-sql/sqlexp v0.1.0/go.mod h1:J4ad9Vo8ZCWQ2GMrC4UCQy1JpCbwU9m3EOqtpKwwwHI= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -351,6 +363,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw= +github.com/lib/pq v1.10.7/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lyft/protoc-gen-star v0.6.1/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA= github.com/matryer/is v1.2.0 h1:92UTHpy8CDwaJ08GqLDzhhuixiBUUD1p3AU6PHddz4A= github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA= @@ -362,10 +376,13 @@ github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Ky github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-sqlite3 v1.14.15 h1:vfoHhTN1af61xCRSWzFIWzx2YskyMTwHLrExkBOjvxI= +github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/mholt/archiver/v4 v4.0.0-alpha.7 h1:xzByj8G8tj0Oq7ZYYU4+ixL/CVb5ruWCm0EZQ1PjOkE= github.com/mholt/archiver/v4 v4.0.0-alpha.7/go.mod h1:Fs8qUkO74HHaidabihzYephJH8qmGD/nCP6tE5xC9BM= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32 h1:W6apQkHrMkS0Muv8G/TipAy/FJl/rCYT0+EuS8+Z0z4= @@ -379,6 +396,7 @@ github.com/paulbellamy/ratecounter v0.2.0/go.mod h1:Hfx1hDpSGoqxkVVpBi/IlYD7kChl github.com/pierrec/lz4/v4 v4.1.14 h1:+fL8AQEZtz/ijeNnpduH0bROTu0O3NZAlPjQxGn8LwE= github.com/pierrec/lz4/v4 v4.1.14/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= +github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4/go.mod h1:4OwLy04Bl9Ef3GJJCoec+30X3LQs/0/m4HFRt/2LUSA= github.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7 h1:+/+DxvQaYifJ+grD4klzrS5y+KJXldn/2YTl5JG+vZ8= github.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7/go.mod h1:zO8QMzTeZd5cpnIkz/Gn6iK0jDfGicM1nynOkkPIl28= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -480,6 +498,7 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= @@ -562,6 +581,7 @@ golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLd golang.org/x/net v0.0.0-20210326060303-6b1517762897/go.mod h1:uSPa2vr4CLtc/ILN5odXGNXS6mhrKVzTaCXzk9m6W3k= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= diff --git a/pkg/detectors/jdbc/jdbc.go b/pkg/detectors/jdbc/jdbc.go index 83b0c32a977c..b3ee1b3481d4 100644 --- a/pkg/detectors/jdbc/jdbc.go +++ b/pkg/detectors/jdbc/jdbc.go @@ -2,8 +2,11 @@ package jdbc import ( "context" + "database/sql" + "errors" "regexp" "strings" + "time" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" @@ -15,7 +18,7 @@ type Scanner struct{} var _ detectors.Detector = (*Scanner)(nil) var ( - keyPat = regexp.MustCompile(`(?i)jdbc:[\w]{3,10}:\/\/\w[\s\S]{0,512}?password[=: \"']+(?P[^<{($]*?)[ \s'\"]+`) + keyPat = regexp.MustCompile(`(?i)jdbc:[\w]{3,10}:[^\s"']{0,512}`) ) // Keywords are used for efficiently pre-filtering chunks. @@ -30,38 +33,25 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result matches := keyPat.FindAllStringSubmatch(dataStr, -1) for _, match := range matches { - if match[1] == "" { - continue - } - token := match[0] - password := match[1] - - //TODO if username and password are the same, username will also be redacted... I think this is probably correct. - redact := strings.TrimSpace(strings.Replace(token, password, strings.Repeat("*", len(password)), -1)) + jdbcConn := match[0] s := detectors.Result{ DetectorType: detectorspb.DetectorType_JDBC, - Raw: []byte(token), - Redacted: redact, + Raw: []byte(jdbcConn), + Redacted: tryRedactAnonymousJDBC(jdbcConn), } - //if verify { - // // TODO: can this be verified? Possibly. Could triage verification to other DBMS strings - // s.Verified = false - // client := common.SaneHttpClient() - // req, err := http.NewRequestWithContext(ctx, "GET", "https://jdbcci.com/api/v2/me", nil) - // if err != nil { - // continue - // } - // req.Header.Add("Accept", "application/json;") - // req.Header.Add("Jdbc-Token", token) - // res, err := client.Do(req) - // if err == nil { - // if res.StatusCode >= 200 && res.StatusCode < 300 { - // s.Verified = true - // } - // } - //} + if verify { + s.Verified = false + j, err := newJDBC(jdbcConn) + if err != nil { + continue + } + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + s.Verified = j.ping(ctx) + // TODO: specialized redaction + } if !s.Verified && detectors.IsKnownFalsePositive(string(s.Raw), detectors.DefaultFalsePositives, false) { continue @@ -72,3 +62,76 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result return } + +func tryRedactAnonymousJDBC(conn string) string { + userPass, postfix, found := strings.Cut(conn, "@") + if found { + if index := strings.LastIndex(userPass, ":"); index != -1 { + prefix, pass := userPass[:index], userPass[index+1:] + return prefix + ":" + strings.Repeat("*", len(pass)) + "@" + postfix + } + } + prefix, paramString, found := strings.Cut(conn, "?") + if !found { + return conn + } + var newParams []string + for _, param := range strings.Split(paramString, "&") { + key, val, _ := strings.Cut(param, "=") + if strings.Contains(strings.ToLower(key), "pass") { + newParams = append(newParams, key+"="+strings.Repeat("*", len(val))) + continue + } + newParams = append(newParams, param) + } + return prefix + "?" + strings.Join(newParams, "&") +} + +var supportedSubprotocols = map[string]func(string) (jdbc, error){ + "sqlite": parseSqlite, + "mysql": parseMySQL, + "postgresql": parsePostgres, + "sqlserver": parseSqlServer, +} + +type jdbc interface { + ping(context.Context) bool +} + +func newJDBC(conn string) (jdbc, error) { + // expected format: "jdbc:{subprotocol}:{subname}" + if !strings.HasPrefix(strings.ToLower(conn), "jdbc:") { + return nil, errors.New("expected jdbc prefix") + } + conn = conn[len("jdbc:"):] + subprotocol, subname, found := strings.Cut(conn, ":") + if !found { + return nil, errors.New("expected a colon separated subprotocol and subname") + } + // get the subprotocol parser + parser, ok := supportedSubprotocols[strings.ToLower(subprotocol)] + if !ok { + return nil, errors.New("unsupported subprotocol") + } + return parser(subname) +} + +func ping(ctx context.Context, driverName, conn string) bool { + if err := pingErr(ctx, driverName, conn); err != nil { + return false + } + return true +} + +func pingErr(ctx context.Context, driverName, conn string) error { + db, err := sql.Open(driverName, conn) + if err != nil { + return err + } + defer db.Close() + + if err := db.PingContext(ctx); err != nil { + return err + } + return nil +} diff --git a/pkg/detectors/jdbc/jdbc_integration_test.go b/pkg/detectors/jdbc/jdbc_integration_test.go new file mode 100644 index 000000000000..55c0bdf49349 --- /dev/null +++ b/pkg/detectors/jdbc/jdbc_integration_test.go @@ -0,0 +1,127 @@ +//go:build detectors && integration +// +build detectors,integration + +package jdbc + +import ( + "context" + "fmt" + "os" + "os/exec" + "strings" + "testing" + "time" + + "github.com/kylelemons/godebug/pretty" + "github.com/stretchr/testify/assert" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestMain(m *testing.M) { + code, err := runMain(m) + if err != nil { + panic(err) + } + os.Exit(code) +} + +func runMain(m *testing.M) (int, error) { + if err := startPostgres(); err != nil { + return 0, err + } + defer stopPostgres() + if err := startMySQL(); err != nil { + return 0, err + } + defer stopMySQL() + return m.Run(), nil +} + +func dockerLogLine(hash string, needle string) chan struct{} { + ch := make(chan struct{}, 1) + go func() { + for { + out, err := exec.Command("docker", "logs", hash).CombinedOutput() + if err != nil { + panic(err) + } + if strings.Contains(string(out), needle) { + ch <- struct{}{} + return + } + time.Sleep(1 * time.Second) + } + }() + return ch +} + +func TestJdbcVerified(t *testing.T) { + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + args args + want []detectors.Result + wantErr bool + }{ + { + name: "postgres verified", + args: args{ + ctx: context.Background(), + data: []byte(`jdbc connection string: jdbc:postgresql://localhost:5432/foo?sslmode=disable&password=` + postgresPass), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_JDBC, + Verified: true, + Redacted: "jdbc:postgresql://localhost:5432/foo?sslmode=disable&password=" + strings.Repeat("*", len(postgresPass)), + }, + }, + wantErr: false, + }, + { + name: "mysql verified", + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf(`CONN="jdbc:mysql://%s:%s@tcp(127.0.0.1:3306)/%s"`, mysqlUser, mysqlPass, mysqlDatabase)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_JDBC, + Verified: true, + Redacted: fmt.Sprintf(`jdbc:mysql://%s:%s@tcp(127.0.0.1:3306)/%s`, mysqlUser, strings.Repeat("*", len(mysqlPass)), mysqlDatabase), + }, + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if tt.wantErr { + assert.Error(t, err) + return + } + assert.NoError(t, err) + if os.Getenv("FORCE_PASS_DIFF") == "true" { + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatal("no raw secret present") + } + got[i].Raw = nil + } + if diff := pretty.Compare(got, tt.want); diff != "" { + t.Errorf("Jdbc.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} diff --git a/pkg/detectors/jdbc/jdbc_test.go b/pkg/detectors/jdbc/jdbc_test.go index fc2abff3fe7e..76a33d181572 100644 --- a/pkg/detectors/jdbc/jdbc_test.go +++ b/pkg/detectors/jdbc/jdbc_test.go @@ -21,18 +21,16 @@ func TestJdbc_FromChunk(t *testing.T) { } tests := []struct { name string - s Scanner args args want []detectors.Result wantErr bool }{ { name: "found, unverified", - s: Scanner{}, args: args{ ctx: context.Background(), data: []byte(`jdbc connection string: jdbc:mysql://hello.test.us-east-1.rds.amazonaws.com:3306/testdb?password=testpassword <-`), - verify: true, + verify: false, }, want: []detectors.Result{ { @@ -45,7 +43,6 @@ func TestJdbc_FromChunk(t *testing.T) { }, { name: "found, unverified numeric password", - s: Scanner{}, args: args{ ctx: context.Background(), data: []byte(`jdbc connection string: jdbc:postgresql://host:5342/testdb?password=123456 <-`), @@ -62,15 +59,62 @@ func TestJdbc_FromChunk(t *testing.T) { }, { name: "not found", - s: Scanner{}, args: args{ ctx: context.Background(), data: []byte("You cannot find the secret within"), - verify: true, + verify: false, }, want: nil, wantErr: false, }, + { + name: "sqlite unverified", + args: args{ + ctx: context.Background(), + data: []byte("jdbc:sqlite::memory:"), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_JDBC, + Verified: false, + Redacted: "jdbc:sqlite::memory:", + }, + }, + wantErr: false, + }, + { + name: "found double quoted string, unverified", + args: args{ + ctx: context.Background(), + data: []byte(`CONN="jdbc:postgres://hello.test.us-east-1.rds.amazonaws.com:3306/testdb?password=testpassword"`), + verify: false, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_JDBC, + Verified: false, + Redacted: "jdbc:postgres://hello.test.us-east-1.rds.amazonaws.com:3306/testdb?password=************", + }, + }, + wantErr: false, + }, + { + name: "found single quoted string, unverified", + args: args{ + ctx: context.Background(), + data: []byte(`CONN='jdbc:postgres://hello.test.us-east-1.rds.amazonaws.com:3306/testdb?password=testpassword'`), + verify: false, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_JDBC, + Verified: false, + Redacted: "jdbc:postgres://hello.test.us-east-1.rds.amazonaws.com:3306/testdb?password=************", + }, + }, + wantErr: false, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/pkg/detectors/jdbc/mysql.go b/pkg/detectors/jdbc/mysql.go new file mode 100644 index 000000000000..768d1ed2aedd --- /dev/null +++ b/pkg/detectors/jdbc/mysql.go @@ -0,0 +1,66 @@ +package jdbc + +import ( + "context" + "errors" + "strings" + + _ "github.com/go-sql-driver/mysql" +) + +type mysqlJDBC struct { + conn string + userPass string + host string + database string + params string +} + +func (s *mysqlJDBC) ping(ctx context.Context) bool { + if ping(ctx, "mysql", s.conn) { + return true + } + // try building connection string (should be same as s.conn though) + if ping(ctx, "mysql", s.build()) { + return true + } + // try removing database + s.database = "" + return ping(ctx, "mysql", s.build()) +} + +func (s *mysqlJDBC) build() string { + conn := s.host + "/" + s.database + if s.userPass != "" { + conn = s.userPass + "@" + conn + } + if s.params != "" { + conn = conn + "?" + s.params + } + return conn +} + +func parseMySQL(subname string) (jdbc, error) { + // expected form: [subprotocol:]//[user:password@]HOST[/DB][?key=val[&key=val]] + hostAndDB, params, _ := strings.Cut(subname, "?") + if !strings.HasPrefix(hostAndDB, "//") { + return nil, errors.New("expected host to start with //") + } + userPassAndHostAndDB := strings.TrimPrefix(hostAndDB, "//") + userPass, hostAndDB, found := strings.Cut(userPassAndHostAndDB, "@") + if !found { + hostAndDB = userPass + userPass = "" + } + host, database, found := strings.Cut(hostAndDB, "/") + if !found { + return nil, errors.New("expected host and database to be separated by /") + } + return &mysqlJDBC{ + conn: subname[2:], + userPass: userPass, + host: host, + database: database, + params: params, + }, nil +} diff --git a/pkg/detectors/jdbc/mysql_integration_test.go b/pkg/detectors/jdbc/mysql_integration_test.go new file mode 100644 index 000000000000..b5d503c7a203 --- /dev/null +++ b/pkg/detectors/jdbc/mysql_integration_test.go @@ -0,0 +1,95 @@ +//go:build detectors && integration +// +build detectors,integration + +package jdbc + +import ( + "bytes" + "context" + "errors" + "os/exec" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +const ( + mysqlUser = "coolGuy" + mysqlPass = "23201dabb56ca236f3dc6736c0f9afad" + mysqlDatabase = "stuff" +) + +func TestMySQL(t *testing.T) { + tests := []struct { + input string + wantErr bool + wantPing bool + }{ + { + input: "", + wantErr: true, + }, + { + input: "//" + mysqlUser + ":" + mysqlPass + "@tcp(127.0.0.1:3306)/" + mysqlDatabase, + wantPing: true, + }, + { + input: "//wrongUser:wrongPass@tcp(127.0.0.1:3306)/" + mysqlDatabase, + wantPing: false, + }, + { + input: "//" + mysqlUser + ":wrongPass@tcp(127.0.0.1:3306)/" + mysqlDatabase, + wantPing: false, + }, + { + input: "//" + mysqlUser + ":" + mysqlPass + "@tcp(127.0.0.1:3306)/", + wantPing: true, + }, + { + input: "//" + mysqlUser + ":" + mysqlPass + "@tcp(127.0.0.1:3306)/wrongDB", + wantPing: true, + }, + } + for _, tt := range tests { + t.Run(tt.input, func(t *testing.T) { + j, err := parseMySQL(tt.input) + if tt.wantErr { + assert.Error(t, err) + return + } + assert.NoError(t, err) + assert.Equal(t, tt.wantPing, j.ping(context.Background())) + }) + } +} + +var mysqlDockerHash string + +func startMySQL() error { + cmd := exec.Command( + "docker", "run", "--rm", "-p", "3306:3306", + "-e", "MYSQL_ROOT_PASSWORD=403a96cff2a323f74bfb1c16992895be", + "-e", "MYSQL_USER="+mysqlUser, + "-e", "MYSQL_PASSWORD="+mysqlPass, + "-e", "MYSQL_DATABASE="+mysqlDatabase, + "-e", "MYSQL_ROOT_HOST=%", + "-d", "mysql", + ) + out, err := cmd.Output() + if err != nil { + return err + } + mysqlDockerHash = string(bytes.TrimSpace(out)) + select { + case <-dockerLogLine(mysqlDockerHash, "socket: '/var/run/mysqld/mysqld.sock' port: 3306"): + return nil + case <-time.After(30 * time.Second): + stopMySQL() + return errors.New("timeout waiting for mysql database to be ready") + } +} + +func stopMySQL() { + exec.Command("docker", "kill", mysqlDockerHash).Run() +} diff --git a/pkg/detectors/jdbc/postgres.go b/pkg/detectors/jdbc/postgres.go new file mode 100644 index 000000000000..163e14d09c06 --- /dev/null +++ b/pkg/detectors/jdbc/postgres.go @@ -0,0 +1,81 @@ +package jdbc + +import ( + "context" + "errors" + "fmt" + "strings" + + _ "github.com/lib/pq" +) + +type postgresJDBC struct { + conn string + params map[string]string +} + +func (s *postgresJDBC) ping(ctx context.Context) bool { + // try the provided connection string directly + if ping(ctx, "postgres", s.conn) { + return true + } + // try as a URL + if ping(ctx, "postgres", "postgres://"+s.conn) { + return true + } + // build a connection string + data := map[string]string{ + // default user + "user": "postgres", + } + for key, val := range s.params { + if key == "host" { + if h, p, found := strings.Cut(val, ":"); found { + data["host"] = h + data["port"] = p + continue + } + } + data[key] = val + } + if ping(ctx, "postgres", joinKeyValues(data)) { + return true + } + if s.params["dbname"] != "" { + delete(s.params, "dbname") + return s.ping(ctx) + } + return false +} + +func joinKeyValues(m map[string]string) string { + var data []string + for k, v := range m { + if v == "" { + continue + } + data = append(data, fmt.Sprintf("%s=%s", k, v)) + } + return strings.Join(data, " ") +} + +func parsePostgres(subname string) (jdbc, error) { + // expected form: //HOST/DB?key=value&key=value + hostAndDB, paramString, _ := strings.Cut(subname, "?") + if !strings.HasPrefix(hostAndDB, "//") { + return nil, errors.New("expected host to start with //") + } + hostAndDB = strings.TrimPrefix(hostAndDB, "//") + host, database, _ := strings.Cut(hostAndDB, "/") + + params := map[string]string{ + "host": host, + "dbname": database, + } + for _, param := range strings.Split(paramString, "&") { + key, val, _ := strings.Cut(param, "=") + params[key] = val + } + + return &postgresJDBC{subname[2:], params}, nil +} diff --git a/pkg/detectors/jdbc/postgres_integration_test.go b/pkg/detectors/jdbc/postgres_integration_test.go new file mode 100644 index 000000000000..fb4fe93b68af --- /dev/null +++ b/pkg/detectors/jdbc/postgres_integration_test.go @@ -0,0 +1,95 @@ +//go:build detectors && integration +// +build detectors,integration + +package jdbc + +import ( + "bytes" + "context" + "errors" + "os/exec" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +const ( + postgresUser = "postgres" + postgresPass = "23201dabb56ca236f3dc6736c0f9afad" +) + +func TestPostgres(t *testing.T) { + tests := []struct { + input string + wantErr bool + wantPing bool + }{ + { + input: "//localhost:5432/foo?sslmode=disable&password=" + postgresPass, + wantPing: true, + }, + { + input: "//localhost:5432/foo?sslmode=disable&user=" + postgresUser + "&password=" + postgresPass, + wantPing: true, + }, + { + input: "//localhost/foo?sslmode=disable&port=5432&password=" + postgresPass, + wantPing: true, + }, + { + input: "//localhost:5432/foo?password=" + postgresPass, + wantPing: false, + }, + { + input: "//localhost:5432/foo?sslmode=disable&password=foo", + wantPing: false, + }, + { + input: "//localhost:5432/foo?sslmode=disable&user=foo&password=" + postgresPass, + wantPing: false, + }, + { + input: "invalid", + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.input, func(t *testing.T) { + j, err := parsePostgres(tt.input) + if tt.wantErr { + assert.Error(t, err) + return + } + assert.NoError(t, err) + assert.Equal(t, tt.wantPing, j.ping(context.Background())) + }) + } +} + +var postgresDockerHash string + +func startPostgres() error { + cmd := exec.Command( + "docker", "run", "--rm", "-p", "5432:5432", + "-e", "POSTGRES_PASSWORD="+postgresPass, + "-e", "POSTGRES_USER="+postgresUser, + "-d", "postgres", + ) + out, err := cmd.Output() + if err != nil { + return err + } + postgresDockerHash = string(bytes.TrimSpace(out)) + select { + case <-dockerLogLine(postgresDockerHash, "PostgreSQL init process complete; ready for start up."): + return nil + case <-time.After(30 * time.Second): + stopPostgres() + return errors.New("timeout waiting for postgres database to be ready") + } +} + +func stopPostgres() { + exec.Command("docker", "kill", postgresDockerHash).Run() +} diff --git a/pkg/detectors/jdbc/sqlite.go b/pkg/detectors/jdbc/sqlite.go new file mode 100644 index 000000000000..edf8aa8672a6 --- /dev/null +++ b/pkg/detectors/jdbc/sqlite.go @@ -0,0 +1,37 @@ +package jdbc + +import ( + "context" + "errors" + "strings" + + _ "github.com/mattn/go-sqlite3" +) + +type sqliteJDBC struct { + filename string + params map[string]string + testing bool +} + +func (s *sqliteJDBC) ping(ctx context.Context) bool { + if !s.testing { + // sqlite is not a networked database, so we cannot verify + return false + } + return ping(ctx, "sqlite3", s.filename) +} + +func parseSqlite(subname string) (jdbc, error) { + filename, params, _ := strings.Cut(subname, "?") + if filename == "" { + return nil, errors.New("empty filename") + } + j := &sqliteJDBC{filename: filename, params: map[string]string{}} + for _, keyVal := range strings.Split(params, "&") { + if key, val, found := strings.Cut(keyVal, "="); found { + j.params[key] = val + } + } + return j, nil +} diff --git a/pkg/detectors/jdbc/sqlite_test.go b/pkg/detectors/jdbc/sqlite_test.go new file mode 100644 index 000000000000..4e2578bb323a --- /dev/null +++ b/pkg/detectors/jdbc/sqlite_test.go @@ -0,0 +1,46 @@ +//go:build detectors +// +build detectors + +package jdbc + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" +) + +func parseSqliteTest(subname string) (jdbc, error) { + j, err := parseSqlite(subname) + if err != nil { + return j, err + } + j.(*sqliteJDBC).testing = true + return j, err +} + +func TestParseSqlite(t *testing.T) { + type testStruct struct { + input string + wantErr bool + } + tests := []struct { + input string + wantErr bool + }{ + {input: "", wantErr: true}, + {input: ":memory:"}, + } + + for _, test := range tests { + t.Run(test.input, func(t *testing.T) { + j, err := parseSqliteTest(test.input) + if test.wantErr { + assert.Error(t, err) + } else { + assert.NoError(t, err) + assert.True(t, j.ping(context.Background())) + } + }) + } +} diff --git a/pkg/detectors/jdbc/sqlserver.go b/pkg/detectors/jdbc/sqlserver.go new file mode 100644 index 000000000000..3f545a3b7d07 --- /dev/null +++ b/pkg/detectors/jdbc/sqlserver.go @@ -0,0 +1,25 @@ +package jdbc + +import ( + "context" + "strings" + + _ "github.com/denisenkom/go-mssqldb" +) + +type sqlServerJDBC struct { + conn string +} + +func (s *sqlServerJDBC) ping(ctx context.Context) bool { + if ping(ctx, "mssql", s.conn) { + return true + } + // try URL format + return ping(ctx, "mssql", "sqlserver://"+s.conn) +} + +func parseSqlServer(subname string) (jdbc, error) { + // expected form: //[username:password@]host/instance[?key=val[&key=val]] + return &sqlServerJDBC{strings.TrimPrefix(subname, "//")}, nil +}