Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Added azure COSMOSDB detector #3951

Open
wants to merge 15 commits into
base: main
Choose a base branch
from
Open
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
added table db support
  • Loading branch information
kashifkhan0771 committed Mar 3, 2025
commit 31d8f48917734358f8316a0ab1f63d9a0df55d41
27 changes: 21 additions & 6 deletions pkg/detectors/azure_cosmosdb/azure_cosmosdb.go
Original file line number Diff line number Diff line change
@@ -29,7 +29,7 @@ var (

dbKeyPattern = regexp.MustCompile(`([A-Za-z0-9+/]{86}==)`)
// account name can contain only lowercase letters, numbers and the `-` character, must be between 3 and 44 characters long.
accountUrlPattern = regexp.MustCompile(`([a-z0-9-]{3,44}.documents\.azure\.com)`)
accountUrlPattern = regexp.MustCompile(`([a-z0-9-]{3,44}\.(?:documents|table\.cosmos)\.azure\.com)`)

invalidHosts = simple.NewCache[struct{}]()

@@ -56,7 +56,7 @@ func (s Scanner) Description() string {
}

func (s Scanner) Keywords() []string {
return []string{".documents.azure.com"}
return []string{".documents.azure.com", ".table.cosmos.azure.com"}
}

func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
@@ -83,10 +83,25 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
DetectorType: detectorspb.DetectorType_AzureCosmosDBKeyIdentifiable,
Raw: []byte(key),
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The URL should also be added. #3938 (comment)

RawV2: []byte("key: " + key + " account_url: " + accountUrl), // key: <key> account_url: <account_url>
ExtraData: map[string]string{},
}

if verify {
verified, verificationErr := verifyCosmosDB(s.getClient(), accountUrl, key)
var verified bool
var verificationErr error

client := s.getClient()

// perform verification based on db type
if strings.Contains(accountUrl, ".documents.azure.com") {
verified, verificationErr = verifyCosmosDocumentDB(client, accountUrl, key)
s1.ExtraData["DB Type"] = "Document"

} else if strings.Contains(accountUrl, ".table.cosmos.azure.com") {
verified, verificationErr = verifyCosmosTableDB(client, accountUrl, key)
s1.ExtraData["DB Type"] = "Table"
}

s1.Verified = verified
if verificationErr != nil {
if errors.Is(verificationErr, noHostErr) {
@@ -106,7 +121,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}

// documentation: https://learn.microsoft.com/en-us/rest/api/cosmos-db/list-databases
func verifyCosmosDB(client *http.Client, accountUrl, key string) (bool, error) {
func verifyCosmosDocumentDB(client *http.Client, accountUrl, key string) (bool, error) {
// decode the base64 encoded key
decodedKey, err := base64.StdEncoding.DecodeString(key)
if err != nil {
@@ -119,7 +134,7 @@ func verifyCosmosDB(client *http.Client, accountUrl, key string) (bool, error) {
}

dateRFC1123 := time.Now().UTC().Format("Mon, 02 Jan 2006 15:04:05 GMT")
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit**: time.RFC1123 already has the similar format.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@kashifkhan0771 is this the expiry time ? it is being used in creating signature.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I used the formation for timezone. x-ms-date header require time in GMT
Docs: https://learn.microsoft.com/en-us/rest/api/cosmos-db/common-cosmosdb-rest-request-headers

authHeader := fmt.Sprintf("type=master&ver=1.0&sig=%s", url.QueryEscape(createSignature(decodedKey, dateRFC1123)))
authHeader := fmt.Sprintf("type=master&ver=1.0&sig=%s", url.QueryEscape(createDocumentsSignature(decodedKey, dateRFC1123)))

// required headers
// docs: https://learn.microsoft.com/en-us/rest/api/cosmos-db/common-cosmosdb-rest-request-headers
@@ -152,7 +167,7 @@ func verifyCosmosDB(client *http.Client, accountUrl, key string) (bool, error) {
}
}

func createSignature(decodedKey []byte, dateRFC1123 string) string {
func createDocumentsSignature(decodedKey []byte, dateRFC1123 string) string {
stringToSign := fmt.Sprintf(
"%s\n%s\n%s\n%s\n\n",
strings.ToLower(http.MethodGet),
19 changes: 16 additions & 3 deletions pkg/detectors/azure_cosmosdb/azure_cosmosdb_test.go
Original file line number Diff line number Diff line change
@@ -12,14 +12,22 @@ import (
)

var (
validPattern = `
validDocumentDBPattern = `
Cluster name: Cluster name must be at least 3 characters and at most 40 characters.
Cluster name must only contain lowercase letters, numbers, and hyphens.
The cluster name must not start or end in a hyphen.
// config
cosmosKey: FakeeP35zYGPXaEUfakeU7S8kcOY7NI7id8ddbHfakeAifake8Bbql1mXhMF2t0wQ0FAKEPQrwZZACDb3msoAg==
https://trufflesecurity-fake.documents.azure.com:443
`
validTableDBPattern = `
Cluster name: Cluster name must be at least 3 characters and at most 40 characters.
Cluster name must only contain lowercase letters, numbers, and hyphens.
The cluster name must not start or end in a hyphen.
// config
cosmosKey: FakeeP35zYGPXaEUfakeU7S8kcOY7NI7id8ddbHfakeAifake8Bbql1mXhMF2t0wQ0FAKEPQrwZZACDb3msoAg==
https://trufflesecurity-fake.table.cosmos.azure.com:443
`

invalidPattern = `
FakeeP35zYGPXaEUfakeU7S8kcOY7I7id8ddbHfakeAifake8Bbql1mXhMF2t0wQ0FAKEPQrwZZACDb3msoAg==
@@ -37,10 +45,15 @@ func TestCosmosDB_Pattern(t *testing.T) {
want []string
}{
{
name: "valid pattern",
input: validPattern,
name: "valid document db pattern",
input: validDocumentDBPattern,
want: []string{fmt.Sprintf("key: %s account_url: %s", "FakeeP35zYGPXaEUfakeU7S8kcOY7NI7id8ddbHfakeAifake8Bbql1mXhMF2t0wQ0FAKEPQrwZZACDb3msoAg==", "trufflesecurity-fake.documents.azure.com")},
},
{
name: "valid table db pattern",
input: validTableDBPattern,
want: []string{fmt.Sprintf("key: %s account_url: %s", "FakeeP35zYGPXaEUfakeU7S8kcOY7NI7id8ddbHfakeAifake8Bbql1mXhMF2t0wQ0FAKEPQrwZZACDb3msoAg==", "trufflesecurity-fake.table.cosmos.azure.com")},
},
{
name: "invalid pattern",
input: invalidPattern,
74 changes: 74 additions & 0 deletions pkg/detectors/azure_cosmosdb/table.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
package azure_cosmosdb

import (
"crypto/hmac"
"crypto/sha256"
"encoding/base64"
"fmt"
"io"
"net/http"
"strings"
"time"
)

func verifyCosmosTableDB(client *http.Client, accountUrl, key string) (bool, error) {
// decode the base64 encoded key
decodedKey, err := base64.StdEncoding.DecodeString(key)
if err != nil {
return false, fmt.Errorf("failed to decode key: %v", err)
}

req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("https://%s:443/Tables", accountUrl), nil)
if err != nil {
return false, fmt.Errorf("failed to create request: %v", err)
}

// extract abc123 from abc123.table.cosmos.azure.com
accountName := strings.TrimPrefix(accountUrl, ".table.cosmos.azure.com")

dateRFC1123 := time.Now().UTC().Format("Mon, 02 Jan 2006 15:04:05 GMT")
authHeader := fmt.Sprintf("SharedKeyLite %s:%s", accountName, createTablesSignature(decodedKey, accountName, dateRFC1123))

// required headers
// docs: https://learn.microsoft.com/en-us/rest/api/cosmos-db/common-cosmosdb-rest-request-headers
req.Header.Set("Authorization", authHeader)
req.Header.Set("x-ms-date", dateRFC1123)
req.Header.Set("x-ms-version", "2019-02-02")
req.Header.Set("Accept", "application/json")

resp, err := client.Do(req)
if err != nil {
// lookup foo.table.cosmos.azure.com: no such host
if strings.Contains(err.Error(), "no such host") {
return false, noHostErr
}

return false, err
}
defer func() {
_, _ = io.Copy(io.Discard, resp.Body)
_ = resp.Body.Close()
}()

// Check response status code
switch resp.StatusCode {
case http.StatusOK:
return true, nil
case http.StatusUnauthorized, http.StatusForbidden:
return false, nil
default:
return false, fmt.Errorf("unexpected status code: %d", resp.StatusCode)
}
}

func createTablesSignature(decodedKey []byte, accountName, dateRFC1123 string) string {
// create string to sign (method + date)
stringToSign := fmt.Sprintf("%s\n%s", dateRFC1123, fmt.Sprintf("/%s/Tables", accountName))

// Compute HMAC-SHA256 signature
h := hmac.New(sha256.New, decodedKey)
h.Write([]byte(stringToSign))
signature := base64.StdEncoding.EncodeToString(h.Sum(nil))

return signature
}
Loading
Oops, something went wrong.