diff --git a/.github/workflows/publish-debug-image.yaml b/.github/workflows/publish-debug-image.yaml deleted file mode 100644 index e12899617..000000000 --- a/.github/workflows/publish-debug-image.yaml +++ /dev/null @@ -1,71 +0,0 @@ -name: publish debug image - -on: - pull_request: - types: - - labeled - -env: - DOCKER_REPOSITORY: "tigrisdata/tigris-debug" - QUAY_REPOSITORY: "quay.io/tigrisdata/tigris-debug" - -jobs: - build-and-push-debug-image: - if: ${{ github.event.label.name == 'debug image' }} - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - - steps: - - name: Checkout repository - uses: actions/checkout@v3 - with: - submodules: true - - - name: Fetch tags - run: | - git fetch --prune --unshallow --tags - - - name: Login to Docker Hub - id: login-docker-hub - uses: docker/login-action@v2 - with: - username: ${{ secrets.GH_DOCKER_ACCESS_USER }} - password: ${{ secrets.GH_DOCKER_ACCESS_TOKEN }} - - - name: Login to Quay.io - uses: docker/login-action@v1 - with: - registry: quay.io - username: ${{ secrets.QUAY_REGISTRY_USER }} - password: ${{ secrets.QUAY_REGISTRY_PASSWORD }} - - - name: Docker meta - id: meta - uses: docker/metadata-action@v4 - with: - # list of Docker images to use as base name for tags - images: | - ${{ env.DOCKER_REPOSITORY }} - ${{ env.QUAY_REPOSITORY }} - # generate Docker tags based on the following events/attributes - tags: | - type=ref,event=branch - type=ref,event=pr - - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Build and push Docker images - uses: docker/build-push-action@v3 - with: - context: . - file: docker/Dockerfile - platforms: linux/amd64 - push: ${{ github.event.label.name == 'debug image' }} - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} diff --git a/.github/workflows/push-docker-amd64.yaml b/.github/workflows/push-docker-amd64.yaml new file mode 100644 index 000000000..2ef5dceae --- /dev/null +++ b/.github/workflows/push-docker-amd64.yaml @@ -0,0 +1,25 @@ +name: docker image + +on: + pull_request: + paths: + - docker/Dockerfile* + - scripts/install_*.sh + - .github/workflows/push-docker-amd64.yaml + - .github/workflows/push-docker-image.yaml + push: + branches: + - main + release: + types: [published] + +jobs: + amd64: + uses: ./.github/workflows/push-docker-image.yaml + secrets: inherit + with: + docker_repository: "tigrisdata/tigris" + quay_repository: "quay.io/tigrisdata/tigris" + file: docker/Dockerfile + platforms: amd64 + event_name: ${{ github.event_name }} diff --git a/.github/workflows/push-docker-arm64.yaml b/.github/workflows/push-docker-arm64.yaml new file mode 100644 index 000000000..8c4fc03da --- /dev/null +++ b/.github/workflows/push-docker-arm64.yaml @@ -0,0 +1,25 @@ +name: docker image + +on: + pull_request: + paths: + - docker/Dockerfile* + - scripts/install_*.sh + - .github/workflows/push-docker-arm64.yaml + - .github/workflows/push-docker-image.yaml + push: + branches: + - main + release: + types: [published] + +jobs: + arm64: + uses: ./.github/workflows/push-docker-image.yaml + secrets: inherit + with: + docker_repository: "tigrisdata/tigris" + quay_repository: "quay.io/tigrisdata/tigris" + file: docker/Dockerfile + platforms: arm64 + event_name: ${{ github.event_name }} diff --git a/.github/workflows/push-docker-debug.yaml b/.github/workflows/push-docker-debug.yaml new file mode 100644 index 000000000..5bf2ea297 --- /dev/null +++ b/.github/workflows/push-docker-debug.yaml @@ -0,0 +1,18 @@ +name: docker image + +on: + pull_request: + types: + - labeled + +jobs: + debug: + if: ${{ github.event.label.name == 'debug image' }} + uses: ./.github/workflows/push-docker-image.yaml + secrets: inherit + with: + docker_repository: "tigrisdata/tigris-debug" + quay_repository: "quay.io/tigrisdata/tigris-debug" + file: docker/Dockerfile + platforms: amd64 + event_name: push diff --git a/.github/workflows/push-docker-image.yaml b/.github/workflows/push-docker-image.yaml index 2fb51d8c2..86c2ef081 100644 --- a/.github/workflows/push-docker-image.yaml +++ b/.github/workflows/push-docker-image.yaml @@ -1,28 +1,21 @@ -name: publish docker image +name: docker image on: - pull_request: - paths: - - docker/Dockerfile* - - scripts/install_*.sh - - .github/workflows/push-docker-*.yaml - merge_group: - paths: - - docker/Dockerfile* - - scripts/install_*.sh - - .github/workflows/push-docker-*.yaml - push: - branches: - - main - release: - types: [published] - -env: - DOCKER_REPOSITORY: "tigrisdata/tigris" - QUAY_REPOSITORY: "quay.io/tigrisdata/tigris" + workflow_call: + inputs: + docker_repository: + type: string + quay_repository: + type: string + platforms: + type: string + file: + type: string + event_name: + type: string jobs: - build-and-push-image: + build-and-push: runs-on: ubuntu-latest permissions: contents: read @@ -46,7 +39,8 @@ jobs: password: ${{ secrets.GH_DOCKER_ACCESS_TOKEN }} - name: Login to Quay.io - uses: docker/login-action@v1 + if: ${{ inputs.quay_repository != '' }} + uses: docker/login-action@v2 with: registry: quay.io username: ${{ secrets.QUAY_REGISTRY_USER }} @@ -58,8 +52,8 @@ jobs: with: # list of Docker images to use as base name for tags images: | - ${{ env.DOCKER_REPOSITORY }} - ${{ env.QUAY_REPOSITORY }} + ${{ inputs.docker_repository }} + ${{ inputs.quay_repository }} # generate Docker tags based on the following events/attributes # we generate the latest tag off the beta branch tags: | @@ -67,7 +61,7 @@ jobs: type=ref,event=pr type=semver,pattern={{version}} type=semver,pattern={{major}}.{{minor}} - type=raw,value=latest,enable=${{ github.event_name == 'release' }} + type=raw,value=latest,enable=${{ inputs.event_name == 'release' }} - name: Set up QEMU uses: docker/setup-qemu-action@v2 @@ -79,8 +73,8 @@ jobs: uses: docker/build-push-action@v3 with: context: . - file: docker/Dockerfile - platforms: linux/amd64,linux/arm64 - push: ${{ github.event_name != 'pull_request' }} + file: ${{ inputs.file }} + platforms: ${{ inputs.platforms }} + push: ${{ inputs.event_name != 'pull_request' }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} diff --git a/.github/workflows/push-docker-local-image.yaml b/.github/workflows/push-docker-local-image.yaml deleted file mode 100644 index ffde11cae..000000000 --- a/.github/workflows/push-docker-local-image.yaml +++ /dev/null @@ -1,77 +0,0 @@ -name: publish local docker image - -on: - pull_request: - paths: - - docker/Dockerfile* - - scripts/install_*.sh - - .github/workflows/push-docker-*.yaml - merge_group: - paths: - - docker/Dockerfile* - - scripts/install_*.sh - - .github/workflows/push-docker-*.yaml - push: - branches: - - main - release: - types: [published] - -env: - DOCKER_REPOSITORY: "tigrisdata/tigris-local" - -jobs: - build-and-push-local-image: - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - - steps: - - name: Checkout repository - uses: actions/checkout@v3 - with: - submodules: true - - - name: Fetch tags - run: | - git fetch --prune --unshallow --tags - - - name: Login to Docker Hub - id: login-docker-hub - uses: docker/login-action@v2 - with: - username: ${{ secrets.GH_DOCKER_ACCESS_USER }} - password: ${{ secrets.GH_DOCKER_ACCESS_TOKEN }} - - - name: Docker meta - id: meta - uses: docker/metadata-action@v4 - with: - # list of Docker images to use as base name for tags - images: | - ${{ env.DOCKER_REPOSITORY }} - # generate Docker tags based on the following events/attributes - # we generate the latest tag off the beta branch - tags: | - type=ref,event=branch - type=ref,event=pr - type=semver,pattern={{version}} - type=semver,pattern={{major}}.{{minor}} - type=raw,value=latest,enable=${{ github.event_name == 'release' }} - - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Build and push Docker images - uses: docker/build-push-action@v3 - with: - context: . - file: docker/Dockerfile.local - platforms: linux/amd64,linux/arm64 - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} diff --git a/.github/workflows/push-docker-local.yaml b/.github/workflows/push-docker-local.yaml new file mode 100644 index 000000000..b675b4025 --- /dev/null +++ b/.github/workflows/push-docker-local.yaml @@ -0,0 +1,29 @@ +name: docker image + +on: + pull_request: + paths: + - docker/Dockerfile* + - scripts/install_*.sh + - .github/workflows/push-docker-local.yaml + - .github/workflows/push-docker-image.yaml + merge_group: + paths: + - docker/Dockerfile* + - scripts/install_*.sh + - .github/workflows/push-docker-*.yaml + push: + branches: + - main + release: + types: [published] + +jobs: + local: + uses: ./.github/workflows/push-docker-image.yaml + secrets: inherit + with: + docker_repository: "tigrisdata/tigris-local" + platforms: amd64,arm64 + file: docker/Dockerfile.local + event_name: ${{ github.event_name }} diff --git a/api/proto b/api/proto index fedd435b3..2f9df79ca 160000 --- a/api/proto +++ b/api/proto @@ -1 +1 @@ -Subproject commit fedd435b387b933747a099a1663604c1ea2d70f7 +Subproject commit 2f9df79cab6d3949b60642ee8ca8914c8cd7378d diff --git a/api/server/v1/marshaler.go b/api/server/v1/marshaler.go index 25aec4489..593921955 100644 --- a/api/server/v1/marshaler.go +++ b/api/server/v1/marshaler.go @@ -1004,6 +1004,32 @@ func (x *ListInvoicesRequest) UnmarshalJSON(data []byte) error { return nil } +// UnmarshalJSON on ListAppKeysRequest. Handles query param. +func (x *ListAppKeysRequest) UnmarshalJSON(data []byte) error { + var mp map[string]jsoniter.RawMessage + + if err := jsoniter.Unmarshal(data, &mp); err != nil { + return err + } + + for key, value := range mp { + var v any + + switch key { + case "key_type": + v = &x.KeyType + case "project": + v = &x.Project + default: + continue + } + if err := jsoniter.Unmarshal(value, v); err != nil { + return err + } + } + return nil +} + func (x *GetNamespaceMetadataResponse) MarshalJSON() ([]byte, error) { resp := struct { MetadataKey string `json:"metadataKey,omitempty"` diff --git a/api/server/v1/tx.go b/api/server/v1/tx.go index 7f728ce7a..766fd906e 100644 --- a/api/server/v1/tx.go +++ b/api/server/v1/tx.go @@ -120,6 +120,7 @@ const ( QuotaLimitsMetricsMethodName = ObservabilityMethodPrefix + "QuotaLimits" QuotaUsageMethodName = ObservabilityMethodPrefix + "QuotaUsage" GetInfoMethodName = ObservabilityMethodPrefix + "GetInfo" + WhoAmIMethodName = ObservabilityMethodPrefix + "WhoAmI" // Realtime. PresenceMethodName = realtimeMethodPrefix + "Presence" diff --git a/config/server.test.yaml b/config/server.test.yaml index 77526db35..ccc88dd66 100644 --- a/config/server.test.yaml +++ b/config/server.test.yaml @@ -62,6 +62,12 @@ auth: - issuer: http://tigris_gotrue:8086 algorithm: HS256 audience: https://tigris-testB + api_keys: + auds: + - https://tigris-test + length: 120 + email_suffix: "@apikey.tigrisdata.com" + user_password: hello token_cache_size: 100 primary_audience: https://tigris-test oauth_provider: gotrue diff --git a/server/config/options.go b/server/config/options.go index 4e886b96e..727b516d2 100644 --- a/server/config/options.go +++ b/server/config/options.go @@ -76,6 +76,7 @@ type AuthzConfig struct { type AuthConfig struct { Enabled bool `mapstructure:"enabled" yaml:"enabled" json:"enabled"` Validators []ValidatorConfig `mapstructure:"validators" yaml:"validators" json:"validators"` + ApiKeys ApiKeysConfig `mapstructure:"api_keys" yaml:"api_keys" json:"api_keys"` PrimaryAudience string `mapstructure:"primary_audience" yaml:"primary_audience" json:"primary_audience"` JWKSCacheTimeout time.Duration `mapstructure:"jwks_cache_timeout" yaml:"jwks_cache_timeout" json:"jwks_cache_timeout"` LogOnly bool `mapstructure:"log_only" yaml:"log_only" json:"log_only"` @@ -118,6 +119,13 @@ type ValidatorConfig struct { Audience string `mapstructure:"audience" yaml:"audience" json:"audience"` } +type ApiKeysConfig struct { + Auds []string `mapstructure:"auds" yaml:"auds" json:"auds"` + Length int `mapstructure:"length" yaml:"length" json:"length"` + EmailSuffix string `mapstructure:"email_suffix" yaml:"email_suffix" json:"email_suffix"` + UserPassword string `mapstructure:"user_password" yaml:"user_password" json:"user_password"` +} + type CdcConfig struct { Enabled bool `mapstructure:"enabled" yaml:"enabled" json:"enabled"` StreamInterval time.Duration @@ -162,6 +170,7 @@ type MetricsConfig struct { Auth AuthMetricsConfig `mapstructure:"auth" yaml:"auth" json:"auth"` SecondaryIndex SecondaryIndexMetricsConfig `mapstructure:"secondary_index" yaml:"secondary_index" json:"secondary_index"` Queue QueueMetricsConfig `mapstructure:"queue" yaml:"queue" json:"queue"` + Metronome MetronomeMetricsConfig `mapstructure:"metronome" yaml:"metronome" json:"metronome"` } type LongRequestConfig struct { @@ -231,6 +240,13 @@ type SecondaryIndexMetricsConfig struct { FilteredTags []string `mapstructure:"filtered_tags" yaml:"filtered_tags" json:"filtered_tags"` } +type MetronomeMetricsConfig struct { + Enabled bool `mapstructure:"enabled" yaml:"enabled" json:"enabled"` + Counter CounterConfig `mapstructure:"counter" yaml:"counter" json:"counter"` + Timer TimerConfig `mapstructure:"timer" yaml:"timer" json:"timer"` + FilteredTags []string `mapstructure:"filtered_tags" yaml:"filtered_tags" json:"filtered_tags"` +} + type QueueMetricsConfig struct { Enabled bool `mapstructure:"enabled" yaml:"enabled" json:"enabled"` } @@ -313,6 +329,10 @@ var DefaultConfig = Config{ Audience: "https://tigris-api", }, }, + ApiKeys: ApiKeysConfig{ + Auds: nil, + Length: 120, + }, PrimaryAudience: "https://tigris-api", JWKSCacheTimeout: 5 * time.Minute, TokenValidationCacheSize: 1000, @@ -449,6 +469,17 @@ var DefaultConfig = Config{ }, FilteredTags: nil, }, + Metronome: MetronomeMetricsConfig{ + Enabled: true, + Counter: CounterConfig{ + OkEnabled: true, + ErrorEnabled: true, + }, + Timer: TimerConfig{ + TimerEnabled: true, + HistogramEnabled: false, + }, + }, Session: SessionMetricGroupConfig{ Enabled: true, Counter: CounterConfig{ diff --git a/server/metrics/measurement.go b/server/metrics/measurement.go index 1f44c93b4..242eae919 100644 --- a/server/metrics/measurement.go +++ b/server/metrics/measurement.go @@ -220,6 +220,10 @@ func (m *Measurement) GetFdbErrorTags(err error) map[string]string { return filterTags(standardizeTags(mergeTags(m.tags, getTagsForError(err)), getFdbErrorTagKeys()), config.DefaultConfig.Metrics.Fdb.FilteredTags) } +func (m *Measurement) GetMetronomeTags() map[string]string { + return filterTags(standardizeTags(m.tags, getMetronomeTagKeys()), config.DefaultConfig.Metrics.Metronome.FilteredTags) +} + func (m *Measurement) GetSearchOkTags() map[string]string { return filterTags(standardizeTags(m.tags, getSearchOkTagKeys()), config.DefaultConfig.Metrics.Search.FilteredTags) } @@ -420,10 +424,9 @@ func (m *Measurement) RecordDuration(scope tally.Scope, tags map[string]string) case SecondaryIndexRespTime, SecondaryIndexErrorRespTime: timerEnabled = cfg.SecondaryIndex.Timer.TimerEnabled histogramEnabled = cfg.SecondaryIndex.Timer.HistogramEnabled - case MetronomeCreateAccount, MetronomeAddPlan, MetronomeIngest, MetronomeGetInvoice, MetronomeListInvoices, - MetronomeGetUsage, MetronomeGetCustomer: - timerEnabled = true - histogramEnabled = true + case MetronomeResponseTime, MetronomeErrorResponseTime: + timerEnabled = cfg.Metronome.Timer.TimerEnabled + histogramEnabled = cfg.Metronome.Timer.HistogramEnabled case RequestsRespTimeToFirstDoc: // Response time to first document m.RecordFirstDocumentDuration(scope, tags) diff --git a/server/metrics/metrics.go b/server/metrics/metrics.go index f1cd185ae..a7bd35736 100644 --- a/server/metrics/metrics.go +++ b/server/metrics/metrics.go @@ -16,6 +16,7 @@ package metrics import ( "io" + "sync" "time" "github.com/rs/zerolog/log" @@ -41,6 +42,7 @@ var ( SchemaMetrics tally.Scope MetronomeMetrics tally.Scope GlobalSt *GlobalStatus + once sync.Once ) func getVersion() string { @@ -94,74 +96,76 @@ func SchemaUpdateRepaired(project string, branch string, collection string) { func InitializeMetrics() func() { var closer io.Closer - if cfg := config.DefaultConfig.Metrics; cfg.Enabled { - log.Debug().Msg("Initializing metrics") - Reporter = promreporter.NewReporter(promreporter.Options{ - DefaultSummaryObjectives: getTimerSummaryObjectives(), - }) - root, closer = tally.NewRootScope(tally.ScopeOptions{ - Tags: GetGlobalTags(), - CachedReporter: Reporter, - // Panics with . - Separator: promreporter.DefaultSeparator, - }, 1*time.Second) - - if cfg.Requests.Enabled { - // Request level metrics (HTTP and GRPC) - Requests = root.SubScope("requests") - initializeRequestScopes() - } - if cfg.Fdb.Enabled { - // FDB level metrics - FdbMetrics = root.SubScope("fdb") - initializeFdbScopes() - } - if cfg.Search.Enabled { - // Search level metrics - SearchMetrics = root.SubScope("search") - initializeSearchScopes() - } - if cfg.Session.Enabled { - // Session level metrics - SessionMetrics = root.SubScope("session") - initializeSessionScopes() - } - if cfg.Size.Enabled { - // Size metrics - SizeMetrics = root.SubScope("size") - initializeSizeScopes() - } - if cfg.Network.Enabled { - // Network metrics - NetworkMetrics = root.SubScope("net") - initializeNetworkScopes() - } - if cfg.Auth.Enabled { - // Auth metrics - AuthMetrics = root.SubScope("auth") - initializeAuthScopes() - } + once.Do(func() { + if cfg := config.DefaultConfig.Metrics; cfg.Enabled { + log.Debug().Msg("Initializing metrics") + Reporter = promreporter.NewReporter(promreporter.Options{ + DefaultSummaryObjectives: getTimerSummaryObjectives(), + }) + root, closer = tally.NewRootScope(tally.ScopeOptions{ + Tags: GetGlobalTags(), + CachedReporter: Reporter, + // Panics with . + Separator: promreporter.DefaultSeparator, + }, 1*time.Second) + + if cfg.Requests.Enabled { + // Request level metrics (HTTP and GRPC) + Requests = root.SubScope("requests") + initializeRequestScopes() + } + if cfg.Fdb.Enabled { + // FDB level metrics + FdbMetrics = root.SubScope("fdb") + initializeFdbScopes() + } + if cfg.Search.Enabled { + // Search level metrics + SearchMetrics = root.SubScope("search") + initializeSearchScopes() + } + if cfg.Session.Enabled { + // Session level metrics + SessionMetrics = root.SubScope("session") + initializeSessionScopes() + } + if cfg.Size.Enabled { + // Size metrics + SizeMetrics = root.SubScope("size") + initializeSizeScopes() + } + if cfg.Network.Enabled { + // Network metrics + NetworkMetrics = root.SubScope("net") + initializeNetworkScopes() + } + if cfg.Auth.Enabled { + // Auth metrics + AuthMetrics = root.SubScope("auth") + initializeAuthScopes() + } - if cfg.SecondaryIndex.Enabled { - // Secondary Index metrics - SecondaryIndexMetrics = root.SubScope("secondary_index") - initializeSecondaryIndexScopes() - } + if cfg.SecondaryIndex.Enabled { + // Secondary Index metrics + SecondaryIndexMetrics = root.SubScope("secondary_index") + initializeSecondaryIndexScopes() + } - if cfg.Queue.Enabled { - QueueMetrics = root.SubScope("queue") - initializeQueueScopes() - } + if cfg.Queue.Enabled { + QueueMetrics = root.SubScope("queue") + initializeQueueScopes() + } - // Metrics for Metronome - external billing service - MetronomeMetrics = root.SubScope("metronome") - initializeMetronomeScopes() + // Metrics for Metronome - external billing service + MetronomeMetrics = root.SubScope("metronome") + initializeMetronomeScopes() - initializeQuotaScopes() + initializeQuotaScopes() - SchemaMetrics = root.SubScope("schema") - GlobalSt = NewGlobalStatus() - } + SchemaMetrics = root.SubScope("schema") + GlobalSt = NewGlobalStatus() + } + }) return func() { if closer != nil { diff --git a/server/metrics/metronome.go b/server/metrics/metronome.go index 050824293..669e62cd6 100644 --- a/server/metrics/metronome.go +++ b/server/metrics/metronome.go @@ -17,42 +17,48 @@ package metrics import ( "strconv" + "github.com/tigrisdata/tigris/server/config" "github.com/uber-go/tally" ) var ( - MetronomeCreateAccount tally.Scope - MetronomeAddPlan tally.Scope - MetronomeIngest tally.Scope - MetronomeListInvoices tally.Scope - MetronomeGetInvoice tally.Scope - MetronomeGetUsage tally.Scope - MetronomeGetCustomer tally.Scope + MetronomeRequestOk tally.Scope + MetronomeRequestError tally.Scope + MetronomeResponseTime tally.Scope + MetronomeErrorResponseTime tally.Scope + MetronomeIngest tally.Scope ) func initializeMetronomeScopes() { - MetronomeCreateAccount = MetronomeMetrics.SubScope("create_account") - MetronomeAddPlan = MetronomeMetrics.SubScope("add_plan") + MetronomeRequestOk = MetronomeMetrics.SubScope("count") + MetronomeRequestError = MetronomeMetrics.SubScope("count") + MetronomeResponseTime = MetronomeMetrics.SubScope("response") + MetronomeErrorResponseTime = MetronomeMetrics.SubScope("error_response") MetronomeIngest = MetronomeMetrics.SubScope("ingest") - MetronomeListInvoices = MetronomeMetrics.SubScope("list_invoices") - MetronomeGetInvoice = MetronomeMetrics.SubScope("get_invoice") - MetronomeGetUsage = MetronomeMetrics.SubScope("get_usage") - MetronomeGetCustomer = MetronomeMetrics.SubScope("get_customer") } -func GetResponseCodeTags(code int) map[string]string { +func getMetronomeTagKeys() []string { + return []string{ + "env", + "operation", + "response_code", + } +} + +func GetMetronomeBaseTags(operation string) map[string]string { return map[string]string{ - "response_code": strconv.Itoa(code), + "env": config.GetEnvironment(), + "operation": operation, } } -func GetErrorCodeTags(err error) map[string]string { +func GetMetronomeResponseCodeTags(code int) map[string]string { return map[string]string{ - "error_value": err.Error(), + "response_code": strconv.Itoa(code), } } -func GetIngestEventTags(eventType string) map[string]string { +func GetMetronomeIngestEventTags(eventType string) map[string]string { return map[string]string{ "event_type": eventType, } diff --git a/server/metrics/metronome_test.go b/server/metrics/metronome_test.go index f7f964a6b..09c8bb475 100644 --- a/server/metrics/metronome_test.go +++ b/server/metrics/metronome_test.go @@ -15,14 +15,33 @@ package metrics import ( + "context" "testing" ) func TestMetronomeMetrics(t *testing.T) { InitializeMetrics() - t.Run("Increment metronome counter", func(t *testing.T) { - tags := GetResponseCodeTags(500) - MetronomeListInvoices.Tagged(tags).Counter("request").Inc(1) - MetronomeGetInvoice.Tagged(tags).Counter("error").Inc(1) + t.Run("Test measuring metronome ok requests", func(t *testing.T) { + InitializeMetrics() + ctx := context.TODO() + op := "test_operation" + me := NewMeasurement(MetronomeServiceName, op, MetronomeSpanType, GetMetronomeBaseTags(op)) + me.StartTracing(ctx, false) + me.AddTags(GetMetronomeResponseCodeTags(200)) + me.FinishTracing(ctx) + me.CountOkForScope(MetronomeRequestOk, me.GetMetronomeTags()) + me.RecordDuration(MetronomeResponseTime, me.GetMetronomeTags()) + }) + + t.Run("Test measuring metronome error requests", func(t *testing.T) { + InitializeMetrics() + ctx := context.TODO() + op := "test_operation" + me := NewMeasurement(MetronomeServiceName, op, MetronomeSpanType, GetMetronomeBaseTags(op)) + me.StartTracing(ctx, false) + me.AddTags(GetMetronomeResponseCodeTags(500)) + me.FinishTracing(ctx) + me.CountErrorForScope(MetronomeRequestError, me.GetMetronomeTags()) + me.RecordDuration(MetronomeErrorResponseTime, me.GetMetronomeTags()) }) } diff --git a/server/middleware/auth.go b/server/middleware/auth.go index c1b6938bb..366c7c1e8 100644 --- a/server/middleware/auth.go +++ b/server/middleware/auth.go @@ -32,6 +32,7 @@ import ( "github.com/tigrisdata/tigris/server/defaults" "github.com/tigrisdata/tigris/server/metrics" "github.com/tigrisdata/tigris/server/request" + "github.com/tigrisdata/tigris/server/services/v1/auth" "github.com/tigrisdata/tigris/server/types" "google.golang.org/grpc" ) @@ -135,10 +136,10 @@ func GetJWTValidators(config *config.Config) []*validator.Validator { return jwtValidators } -func measuredAuthFunction(ctx context.Context, jwtValidators []*validator.Validator, config *config.Config, cache gcache.Cache) (context.Context, error) { +func measuredAuthFunction(ctx context.Context, jwtValidators []*validator.Validator, config *config.Config, cache gcache.Cache, a auth.Provider) (context.Context, error) { measurement := metrics.NewMeasurement("auth", "auth", metrics.AuthSpanType, metrics.GetAuthBaseTags(ctx)) measurement.StartTracing(ctx, true) - ctxResult, err := authFunction(ctx, jwtValidators, config, cache) + ctxResult, err := authFunction(ctx, jwtValidators, config, cache, a) if err != nil { measurement.CountErrorForScope(metrics.AuthErrorCount, measurement.GetAuthErrorTags(err)) measurement.FinishWithError(ctxResult, err) @@ -151,7 +152,7 @@ func measuredAuthFunction(ctx context.Context, jwtValidators []*validator.Valida return ctxResult, nil } -func authFunction(ctx context.Context, jwtValidators []*validator.Validator, config *config.Config, cache gcache.Cache) (ctxResult context.Context, err error) { +func authFunction(ctx context.Context, jwtValidators []*validator.Validator, config *config.Config, cache gcache.Cache, a auth.Provider) (ctxResult context.Context, err error) { reqMetadata, err := request.GetRequestMetadataFromContext(ctx) if err != nil { log.Warn().Err(err).Msg("Failed to load request metadata") @@ -178,9 +179,36 @@ func authFunction(ctx context.Context, jwtValidators []*validator.Validator, con if err != nil { return ctx, err } + if strings.Contains(tkn, ".") { + return authenticateUsingAuthToken(ctx, jwtValidators, config, cache, tkn, reqMetadata) + } else if strings.HasPrefix(tkn, auth.ApiKeyPrefix) { + return authenticateUsingApiKey(ctx, jwtValidators, config, cache, tkn, reqMetadata, a) + } + return ctx, errors.Unauthenticated("Failed to authenticate") +} - validatedToken := getCachedToken(ctx, tkn, cache) +func authenticateUsingApiKey(ctx context.Context, _ []*validator.Validator, _ *config.Config, cache gcache.Cache, apiKey string, reqMetadata *request.Metadata, a auth.Provider) (context.Context, error) { + token, err := cache.Get(apiKey) + if err != nil || token == nil { + token, err = a.ValidateApiKey(ctx, apiKey, config.DefaultConfig.Auth.ApiKeys.Auds) + if err != nil { + return ctx, err + } + // put it to cache + err = cache.Set(apiKey, token) + if err != nil { + log.Err(err).Msg("Could not set to the cache") + } + } + castedToken := token.(*types.AccessToken) + reqMetadata.SetNamespace(ctx, castedToken.Namespace) + reqMetadata.SetAccessToken(castedToken) + return ctx, nil +} +func authenticateUsingAuthToken(ctx context.Context, jwtValidators []*validator.Validator, config *config.Config, cache gcache.Cache, tkn string, reqMetadata *request.Metadata) (context.Context, error) { + validatedToken := getCachedToken(ctx, tkn, cache) + var err error // if not found from cache if validatedToken == nil { count := 0 @@ -272,16 +300,17 @@ func getAuthFunction(config *config.Config) func(ctx context.Context) (context.C lruCache := gcache.New(config.Auth.TokenValidationCacheSize). Expiration(time.Duration(config.Auth.TokenValidationCacheTTLSec) * time.Second). Build() + authProvider := auth.NewGotrueProvider() // inline closure to access the state of jwtValidator if config.Tracing.Enabled { return func(ctx context.Context) (context.Context, error) { - return measuredAuthFunction(ctx, jwtValidators, config, lruCache) + return measuredAuthFunction(ctx, jwtValidators, config, lruCache, authProvider) } } return func(ctx context.Context) (context.Context, error) { - return authFunction(ctx, jwtValidators, config, lruCache) + return authFunction(ctx, jwtValidators, config, lruCache, authProvider) } } diff --git a/server/middleware/auth_test.go b/server/middleware/auth_test.go index d6d5b70b0..5d9ec6268 100644 --- a/server/middleware/auth_test.go +++ b/server/middleware/auth_test.go @@ -26,6 +26,7 @@ import ( api "github.com/tigrisdata/tigris/api/server/v1" "github.com/tigrisdata/tigris/errors" "github.com/tigrisdata/tigris/server/config" + "github.com/tigrisdata/tigris/server/services/v1/auth" "github.com/tigrisdata/tigris/util/log" "google.golang.org/grpc/metadata" ) @@ -47,35 +48,36 @@ func TestAuth(t *testing.T) { FoundationDB: config.FoundationDBConfig{}, } cache := gcache.New(10).Expiration(time.Duration(5) * time.Minute).Build() + authProvider := auth.NewGotrueProvider() t.Run("log_only mode: no token", func(t *testing.T) { - ctx, err := authFunction(context.TODO(), []*validator.Validator{{}}, &config.DefaultConfig, cache) + ctx, err := authFunction(context.TODO(), []*validator.Validator{{}}, &config.DefaultConfig, cache, authProvider) require.NotNil(t, ctx) require.Nil(t, err) }) t.Run("enforcing mode: no token", func(t *testing.T) { - _, err := authFunction(context.TODO(), []*validator.Validator{{}}, &enforcedAuthConfig, cache) + _, err := authFunction(context.TODO(), []*validator.Validator{{}}, &enforcedAuthConfig, cache, authProvider) require.NotNil(t, err) require.Equal(t, err, errors.Unauthenticated("request unauthenticated with bearer")) }) t.Run("enforcing mode: Bad authorization string1", func(t *testing.T) { incomingCtx := metadata.NewIncomingContext(context.TODO(), metadata.Pairs("authorization", "bearer")) - _, err := authFunction(incomingCtx, []*validator.Validator{{}}, &enforcedAuthConfig, cache) + _, err := authFunction(incomingCtx, []*validator.Validator{{}}, &enforcedAuthConfig, cache, authProvider) require.NotNil(t, err) require.Equal(t, err, errors.Unauthenticated("bad authorization string")) }) t.Run("enforcing mode: Bad token", func(t *testing.T) { incomingCtx := metadata.NewIncomingContext(context.TODO(), metadata.Pairs("authorization", "bearer somebadtoken")) - _, err := authFunction(incomingCtx, []*validator.Validator{{}}, &enforcedAuthConfig, cache) + _, err := authFunction(incomingCtx, []*validator.Validator{{}}, &enforcedAuthConfig, cache, authProvider) require.NotNil(t, err) - require.Equal(t, err, errors.Unauthenticated("Failed to validate access token, could not be validated")) + require.Equal(t, errors.Unauthenticated("Failed to authenticate"), err) }) t.Run("enforcing mode: Bad token 2", func(t *testing.T) { incomingCtx := metadata.NewIncomingContext(context.TODO(), metadata.Pairs("authorization", "bearer some.bad.token")) - _, err := authFunction(incomingCtx, []*validator.Validator{{}}, &enforcedAuthConfig, cache) + _, err := authFunction(incomingCtx, []*validator.Validator{{}}, &enforcedAuthConfig, cache, authProvider) require.NotNil(t, err) require.Contains(t, err.Error(), "Failed to validate access token") }) diff --git a/server/middleware/authz.go b/server/middleware/authz.go index 40fce4991..65c7f9886 100644 --- a/server/middleware/authz.go +++ b/server/middleware/authz.go @@ -23,7 +23,6 @@ import ( "github.com/tigrisdata/tigris/lib/container" "github.com/tigrisdata/tigris/server/config" "github.com/tigrisdata/tigris/server/request" - "github.com/tigrisdata/tigris/server/types" "google.golang.org/grpc" ) @@ -73,6 +72,7 @@ var ( api.QuotaLimitsMetricsMethodName, api.QuotaUsageMethodName, api.GetInfoMethodName, + api.WhoAmIMethodName, // realtime api.ReadMessagesMethodName, @@ -152,6 +152,7 @@ var ( api.QuotaLimitsMetricsMethodName, api.QuotaUsageMethodName, api.GetInfoMethodName, + api.WhoAmIMethodName, // realtime api.PresenceMethodName, @@ -253,6 +254,7 @@ var ( api.QuotaLimitsMetricsMethodName, api.QuotaUsageMethodName, api.GetInfoMethodName, + api.WhoAmIMethodName, // realtime api.PresenceMethodName, @@ -348,6 +350,7 @@ var ( api.QuotaLimitsMetricsMethodName, api.QuotaUsageMethodName, api.GetInfoMethodName, + api.WhoAmIMethodName, // realtime api.PresenceMethodName, @@ -424,11 +427,11 @@ func authorize(ctx context.Context) (err error) { Msg("Empty role allowed for transition purpose") return nil } + // if !isAuthorizedProject(reqMetadata, accessToken) { + // authorizationErr = errors.PermissionDenied("You are not allowed to perform operation: %s", reqMetadata.GetFullMethod()) + //} var authorizationErr error - if !isAuthorizedProject(reqMetadata, accessToken) { - authorizationErr = errors.PermissionDenied("You are not allowed to perform operation: %s", reqMetadata.GetFullMethod()) - } - if err == nil && !isAuthorizedOperation(reqMetadata.GetFullMethod(), role) { + if !isAuthorizedOperation(reqMetadata.GetFullMethod(), role) { authorizationErr = errors.PermissionDenied("You are not allowed to perform operation: %s", reqMetadata.GetFullMethod()) } @@ -447,13 +450,6 @@ func authorize(ctx context.Context) (err error) { return nil } -func isAuthorizedProject(reqMetadata *request.Metadata, accessToken *types.AccessToken) bool { - if accessToken.Project != "" && reqMetadata.GetProject() != accessToken.Project { - return false - } - return true -} - func isAuthorizedOperation(method string, role string) bool { if methods := getMethodsForRole(role); methods != nil { return methods.Contains(method) diff --git a/server/middleware/authz_test.go b/server/middleware/authz_test.go index 42ac3f59f..497a2d8e2 100644 --- a/server/middleware/authz_test.go +++ b/server/middleware/authz_test.go @@ -97,6 +97,7 @@ func TestAuthzOwnerRole(t *testing.T) { require.True(t, isAuthorizedOperation(api.QuotaLimitsMetricsMethodName, ownerRoleName)) require.True(t, isAuthorizedOperation(api.QuotaUsageMethodName, ownerRoleName)) require.True(t, isAuthorizedOperation(api.GetInfoMethodName, ownerRoleName)) + require.True(t, isAuthorizedOperation(api.WhoAmIMethodName, ownerRoleName)) // realtime require.True(t, isAuthorizedOperation(api.PresenceMethodName, ownerRoleName)) @@ -193,6 +194,7 @@ func TestAuthzEditorRole(t *testing.T) { require.True(t, isAuthorizedOperation(api.QuotaLimitsMetricsMethodName, editorRoleName)) require.True(t, isAuthorizedOperation(api.QuotaUsageMethodName, editorRoleName)) require.True(t, isAuthorizedOperation(api.GetInfoMethodName, editorRoleName)) + require.True(t, isAuthorizedOperation(api.WhoAmIMethodName, editorRoleName)) // realtime require.True(t, isAuthorizedOperation(api.PresenceMethodName, editorRoleName)) @@ -263,6 +265,7 @@ func TestAuthzReadOnlyRole(t *testing.T) { require.True(t, isAuthorizedOperation(api.QuotaLimitsMetricsMethodName, readOnlyRoleName)) require.True(t, isAuthorizedOperation(api.QuotaUsageMethodName, readOnlyRoleName)) require.True(t, isAuthorizedOperation(api.GetInfoMethodName, readOnlyRoleName)) + require.True(t, isAuthorizedOperation(api.WhoAmIMethodName, readOnlyRoleName)) // realtime require.True(t, isAuthorizedOperation(api.ReadMessagesMethodName, readOnlyRoleName)) diff --git a/server/services/v1/auth/auth0.go b/server/services/v1/auth/auth0.go index 4a2bfe70e..ce217639e 100644 --- a/server/services/v1/auth/auth0.go +++ b/server/services/v1/auth/auth0.go @@ -33,6 +33,7 @@ import ( "github.com/tigrisdata/tigris/server/metadata" "github.com/tigrisdata/tigris/server/request" "github.com/tigrisdata/tigris/server/transaction" + "github.com/tigrisdata/tigris/server/types" "golang.org/x/net/context/ctxhttp" ) @@ -328,6 +329,10 @@ func (*auth0) ListGlobalAppKeys(_ context.Context, _ *api.ListGlobalAppKeysReque return nil, errors.Internal("auth0 implementation doesn't support it") } +func (*auth0) ValidateApiKey(_ context.Context, _ string, _ []string) (*types.AccessToken, error) { + return nil, errors.Internal("auth0 implementation doesn't support it") +} + func validateOwnershipAuth0(ctx context.Context, operationName string, appId string, a *auth0) (*management.Client, string, error) { client, err := a.Management.Client.Read(appId) if err != nil { diff --git a/server/services/v1/auth/gotrue.go b/server/services/v1/auth/gotrue.go index 5d367d985..906c7aa4a 100644 --- a/server/services/v1/auth/gotrue.go +++ b/server/services/v1/auth/gotrue.go @@ -27,27 +27,34 @@ import ( "strings" "time" - "github.com/davecgh/go-spew/spew" + "github.com/google/uuid" jsoniter "github.com/json-iterator/go" "github.com/rs/zerolog/log" api "github.com/tigrisdata/tigris/api/server/v1" "github.com/tigrisdata/tigris/errors" "github.com/tigrisdata/tigris/server/config" "github.com/tigrisdata/tigris/server/request" + "github.com/tigrisdata/tigris/server/types" "golang.org/x/net/context/ctxhttp" ) const ( - GotrueAudHeaderKey = "X-JWT-AUD" - ClientIdPrefix = "tid_" - ClientSecretPrefix = "tsec_" - Component = "component" - AppKey = "app_key" - AppKeyUser = "app key" + GotrueAudHeaderKey = "X-JWT-AUD" + ClientIdPrefix = "tid_" + ClientSecretPrefix = "tsec_" + GlobalClientIdPrefix = "tgid_" + GlobalClientSecretPrefix = "tgsec_" + ApiKeyPrefix = "tkey_" + Component = "component" + AppKey = "app_key" + AppKeyUser = "app key" InvitationStatusPending = "PENDING" InvitationStatusAccepted = "ACCEPTED" InvitationStatusExpired = "EXPIRED" + + AppKeyTypeCredentials = "credentials" + AppKeyTypeApiKey = "api_key" ) type gotrue struct { @@ -94,9 +101,22 @@ type UserAppData struct { Name string `json:"name"` Description string `json:"description"` Project string `json:"tigris_project"` + KeyType string `json:"key_type"` +} + +type GetUserResp struct { + InstanceID uuid.UUID `json:"instance_id"` + ID uuid.UUID `json:"id"` + Aud string `json:"aud"` + Role string `json:"role"` + Email string `json:"email"` + EncryptedPassword string `json:"encrypted_password"` + + AppMetaData *UserAppData `json:"app_metadata" db:"app_metadata"` } -func _createAppKey(ctx context.Context, clientId string, clientSecret string, g *gotrue, keyName string, keyDescription string, project string) (string, int64, error) { +// returns currentSub, creationTime, error. +func _createAppKey(ctx context.Context, clientId string, clientSecret string, g *gotrue, keyName string, keyDescription string, project string, keyType string) (string, int64, error) { currentSub, err := GetCurrentSub(ctx) if err != nil { log.Err(err).Msg("Failed to create application: reason - unable to extract current sub") @@ -110,9 +130,10 @@ func _createAppKey(ctx context.Context, clientId string, clientSecret string, g } // make gotrue call + email := _getEmail(clientId, keyType, g) creationTime := time.Now().UnixMilli() payloadBytes, err := jsoniter.Marshal(CreateUserPayload{ - Email: fmt.Sprintf("%s%s", clientId, g.AuthConfig.Gotrue.UsernameSuffix), + Email: email, Password: clientSecret, AppData: UserAppData{ CreatedAt: creationTime, @@ -121,6 +142,7 @@ func _createAppKey(ctx context.Context, clientId string, clientSecret string, g Name: keyName, Description: keyDescription, Project: project, + KeyType: keyType, }, }) if err != nil { @@ -135,6 +157,7 @@ func _createAppKey(ctx context.Context, clientId string, clientSecret string, g Str("namespace", currentNamespace). Str("sub", currentSub). Str("client_id", clientId). + Str("key_type", keyType). Str(Component, AppKey). Msg("appkey created") return currentSub, creationTime, nil @@ -144,14 +167,30 @@ func (g *gotrue) CreateAppKey(ctx context.Context, req *api.CreateAppKeyRequest) if req.GetProject() == "" { return nil, errors.InvalidArgument("Project must be specified") } - clientId := generateClientId(g) - clientSecret := generateClientSecret(g) + if req.GetKeyType() != "" && !(req.GetKeyType() == AppKeyTypeCredentials || req.GetKeyType() == AppKeyTypeApiKey) { + return nil, errors.InvalidArgument("app key supported types are [credentials, api_key]") + } - currentSub, creationTime, err := _createAppKey(ctx, clientId, clientSecret, g, req.GetName(), req.GetDescription(), req.GetProject()) + appKeyType := AppKeyTypeCredentials + if req.GetKeyType() != "" { + appKeyType = req.GetKeyType() + } + var clientId, clientSecret string + if appKeyType == AppKeyTypeCredentials { + clientId = generateClientId(ClientIdPrefix, config.DefaultConfig.Auth.Gotrue.ClientIdLength) + clientSecret = generateClientSecret(g, ClientSecretPrefix) + } else { + clientId = generateClientId(ApiKeyPrefix, config.DefaultConfig.Auth.ApiKeys.Length) + clientSecret = config.DefaultConfig.Auth.ApiKeys.UserPassword + } + currentSub, creationTime, err := _createAppKey(ctx, clientId, clientSecret, g, req.GetName(), req.GetDescription(), req.GetProject(), appKeyType) if err != nil { return nil, err } + if req.GetKeyType() == AppKeyTypeApiKey { + clientSecret = "" // hide the secret + } return &api.CreateAppKeyResponse{ CreatedAppKey: &api.AppKey{ Id: clientId, @@ -166,10 +205,10 @@ func (g *gotrue) CreateAppKey(ctx context.Context, req *api.CreateAppKeyRequest) } func (g *gotrue) CreateGlobalAppKey(ctx context.Context, req *api.CreateGlobalAppKeyRequest) (*api.CreateGlobalAppKeyResponse, error) { - clientId := generateClientId(g) - clientSecret := generateClientSecret(g) + clientId := generateClientId(GlobalClientIdPrefix, config.DefaultConfig.Auth.Gotrue.ClientIdLength) + clientSecret := generateClientSecret(g, GlobalClientSecretPrefix) - currentSub, creationTime, err := _createAppKey(ctx, clientId, clientSecret, g, req.GetName(), req.GetDescription(), "") + currentSub, creationTime, err := _createAppKey(ctx, clientId, clientSecret, g, req.GetName(), req.GetDescription(), "", AppKeyTypeCredentials) if err != nil { return nil, err } @@ -186,8 +225,16 @@ func (g *gotrue) CreateGlobalAppKey(ctx context.Context, req *api.CreateGlobalAp }, nil } -func _updateAppKey(ctx context.Context, g *gotrue, id string, name string, description string) error { - email := fmt.Sprintf("%s%s", id, g.AuthConfig.Gotrue.UsernameSuffix) +func _getEmail(clientId string, appKeyType string, g *gotrue) string { + suffix := g.AuthConfig.Gotrue.UsernameSuffix + if appKeyType == AppKeyTypeApiKey { + suffix = g.AuthConfig.ApiKeys.EmailSuffix + } + return fmt.Sprintf("%s%s", clientId, suffix) +} + +func _updateAppKey(ctx context.Context, g *gotrue, id string, name string, description string, appKeyType string) error { + email := _getEmail(id, appKeyType, g) updateAppKeyUrl := fmt.Sprintf("%s/admin/users/%s", g.AuthConfig.Gotrue.URL, email) currentSub, err := GetCurrentSub(ctx) @@ -250,11 +297,19 @@ func _updateAppKey(ctx context.Context, g *gotrue, id string, name string, descr return nil } +// retrieves the key type from the client id naming scheme. +func getAppKeyType(clientId string) string { + if strings.HasPrefix(clientId, ApiKeyPrefix) { + return AppKeyTypeApiKey + } + return AppKeyTypeCredentials +} + func (g *gotrue) UpdateAppKey(ctx context.Context, req *api.UpdateAppKeyRequest) (*api.UpdateAppKeyResponse, error) { if req.GetProject() == "" { return nil, errors.InvalidArgument("Project must be specified") } - err := _updateAppKey(ctx, g, req.GetId(), req.GetName(), req.GetDescription()) + err := _updateAppKey(ctx, g, req.GetId(), req.GetName(), req.GetDescription(), getAppKeyType(req.GetId())) if err != nil { return nil, err } @@ -273,7 +328,7 @@ func (g *gotrue) UpdateAppKey(ctx context.Context, req *api.UpdateAppKeyRequest) } func (g *gotrue) UpdateGlobalAppKey(ctx context.Context, req *api.UpdateGlobalAppKeyRequest) (*api.UpdateGlobalAppKeyResponse, error) { - err := _updateAppKey(ctx, g, req.GetId(), req.GetName(), req.GetDescription()) + err := _updateAppKey(ctx, g, req.GetId(), req.GetName(), req.GetDescription(), AppKeyTypeCredentials) if err != nil { return nil, err } @@ -291,8 +346,9 @@ func (g *gotrue) UpdateGlobalAppKey(ctx context.Context, req *api.UpdateGlobalAp return result, nil } -func _rotateAppKeySecret(ctx context.Context, g *gotrue, id string) (string, error) { - email := fmt.Sprintf("%s%s", id, g.AuthConfig.Gotrue.UsernameSuffix) +func _rotateAppKeySecret(ctx context.Context, g *gotrue, id string, clientSecretPrefix string) (string, error) { + // no rotation for api key, only for credentials + email := _getEmail(id, AppKeyTypeCredentials, g) updateAppKeyUrl := fmt.Sprintf("%s/admin/users/%s", g.AuthConfig.Gotrue.URL, email) currentSub, err := GetCurrentSub(ctx) @@ -301,7 +357,7 @@ func _rotateAppKeySecret(ctx context.Context, g *gotrue, id string) (string, err return "", errors.Internal("Failed to update app key") } - newSecret := generateClientSecret(g) + newSecret := generateClientSecret(g, clientSecretPrefix) newAppMetadata := &UserAppData{UpdatedBy: currentSub, UpdatedAt: time.Now().UnixMilli()} payload := make(map[string]any) @@ -354,7 +410,7 @@ func (g *gotrue) RotateAppKey(ctx context.Context, req *api.RotateAppKeyRequest) if req.GetProject() == "" { return nil, errors.InvalidArgument("Project must be specified") } - newSecret, err := _rotateAppKeySecret(ctx, g, req.GetId()) + newSecret, err := _rotateAppKeySecret(ctx, g, req.GetId(), ClientSecretPrefix) if err != nil { return nil, err } @@ -369,7 +425,7 @@ func (g *gotrue) RotateAppKey(ctx context.Context, req *api.RotateAppKeyRequest) } func (g *gotrue) RotateGlobalAppKeySecret(ctx context.Context, req *api.RotateGlobalAppKeySecretRequest) (*api.RotateGlobalAppKeySecretResponse, error) { - newSecret, err := _rotateAppKeySecret(ctx, g, req.GetId()) + newSecret, err := _rotateAppKeySecret(ctx, g, req.GetId(), GlobalClientSecretPrefix) if err != nil { return nil, err } @@ -393,7 +449,8 @@ func _deleteAppKey(ctx context.Context, g *gotrue, id string) error { } // make external call - deleteUserUrl := fmt.Sprintf("%s/admin/users/%s%s", g.AuthConfig.Gotrue.URL, id, g.AuthConfig.Gotrue.UsernameSuffix) + email := _getEmail(id, getAppKeyType(id), g) + deleteUserUrl := fmt.Sprintf("%s/admin/users/%s", g.AuthConfig.Gotrue.URL, email) client := &http.Client{} deleteUserReq, err := http.NewRequestWithContext(ctx, http.MethodDelete, deleteUserUrl, nil) if err != nil { @@ -455,9 +512,13 @@ type appKeyInternal struct { CreatedBy string CreatedAt int64 Project string + KeyType string } -func _listAppKeys(ctx context.Context, g *gotrue, project string) ([]*appKeyInternal, error) { +func _listAppKeys(ctx context.Context, g *gotrue, project string, keyType string) ([]*appKeyInternal, error) { + if keyType != "" && !(keyType == AppKeyTypeApiKey || keyType == AppKeyTypeCredentials) { + return nil, errors.InvalidArgument("Invalid key_type. Supported values are [credentials, api_key]") + } currentSub, err := GetCurrentSub(ctx) if err != nil { return nil, errors.Internal("Failed to list applications: reason = %s", err.Error()) @@ -476,6 +537,9 @@ func _listAppKeys(ctx context.Context, g *gotrue, project string) ([]*appKeyInte // make external call getUsersUrl := fmt.Sprintf("%s/admin/users?created_by=%s&tigris_namespace=%s&tigris_project=%s&page=1&per_page=5000", g.AuthConfig.Gotrue.URL, currentSub, currentNamespace, project) + if keyType != "" { + getUsersUrl = fmt.Sprintf("%s&keyType=%s", getUsersUrl, keyType) + } client := &http.Client{} getUsersReq, err := http.NewRequestWithContext(ctx, http.MethodGet, getUsersUrl, nil) if err != nil { @@ -551,7 +615,6 @@ func _listAppKeys(ctx context.Context, g *gotrue, project string) ([]*appKeyInte // parse string time to millis using rfc3339 format createdAtMillis = readDate(createdAtStr) } - appKey := appKeyInternal{ Id: clientId, Name: appMetadata.Name, @@ -561,13 +624,17 @@ func _listAppKeys(ctx context.Context, g *gotrue, project string) ([]*appKeyInte CreatedAt: createdAtMillis, Project: appMetadata.Project, } + appKey.KeyType = AppKeyTypeCredentials + if appMetadata.KeyType != "" { + appKey.KeyType = appMetadata.KeyType + } appKeys[i] = &appKey } return appKeys, nil } func (g *gotrue) ListAppKeys(ctx context.Context, req *api.ListAppKeysRequest) (*api.ListAppKeysResponse, error) { - appKeysInternal, err := _listAppKeys(ctx, g, req.GetProject()) + appKeysInternal, err := _listAppKeys(ctx, g, req.GetProject(), req.GetKeyType()) if err != nil { return nil, errors.Internal("Failed to list app keys") } @@ -577,19 +644,24 @@ func (g *gotrue) ListAppKeys(ctx context.Context, req *api.ListAppKeysRequest) ( Id: internalAppKey.Id, Name: internalAppKey.Name, Description: internalAppKey.Description, - Secret: internalAppKey.Secret, CreatedAt: internalAppKey.CreatedAt, CreatedBy: internalAppKey.CreatedBy, Project: internalAppKey.Project, + KeyType: internalAppKey.KeyType, + } + // expose secret in case of credentials, for backward compatibility defaults to credentials + if req.GetKeyType() == "" || req.GetKeyType() == AppKeyTypeCredentials { + appKeys[i].Secret = internalAppKey.Secret } } + return &api.ListAppKeysResponse{ AppKeys: appKeys, }, nil } func (g *gotrue) ListGlobalAppKeys(ctx context.Context, _ *api.ListGlobalAppKeysRequest) (*api.ListGlobalAppKeysResponse, error) { - appKeysInternal, err := _listAppKeys(ctx, g, "") + appKeysInternal, err := _listAppKeys(ctx, g, "", "") if err != nil { return nil, errors.Internal("Failed to delete app keys") } @@ -642,8 +714,7 @@ func (g *gotrue) GetAccessToken(ctx context.Context, req *api.GetAccessTokenRequ case api.GrantType_REFRESH_TOKEN: return nil, errors.Unimplemented("Use client_credentials to get the access token") case api.GrantType_CLIENT_CREDENTIALS: - spew.Dump(ctx) - accessToken, expiresIn, err := getAccessTokenUsingClientCredentialsGotrue(ctx, fmt.Sprintf("%s%s", req.GetClientId(), g.AuthConfig.Gotrue.UsernameSuffix), req.GetClientSecret(), g) + accessToken, expiresIn, err := getAccessTokenUsingClientCredentialsGotrue(ctx, _getEmail(req.GetClientId(), AppKeyTypeCredentials, g), req.GetClientSecret(), g) if err != nil { return nil, err } @@ -656,6 +727,78 @@ func (g *gotrue) GetAccessToken(ctx context.Context, req *api.GetAccessTokenRequ return nil, errors.InvalidArgument("Failed to GetAccessToken: reason = unsupported grant_type, it has to be one of [refresh_token, client_credentials]") } +func (g *gotrue) ValidateApiKey(ctx context.Context, apiKey string, auds []string) (*types.AccessToken, error) { + // get admin token + adminToken, _, err := getGotrueAdminAccessToken(ctx, g) + if err != nil { + log.Err(err).Msgf("Failed to create gotrue admin access token") + return nil, errors.Internal("Could not form request to validate api key") + } + // admin-get user + client := &http.Client{} + email := _getEmail(apiKey, AppKeyTypeApiKey, g) + getUserReq, err := http.NewRequestWithContext(ctx, http.MethodGet, fmt.Sprintf("%s/admin/users/%s", g.AuthConfig.Gotrue.URL, email), nil) + if err != nil { + log.Err(err).Msgf("Failed to create request to retrieve user") + return nil, errors.Internal("Could not form request to validate api key") + } + getUserReq.Header.Add("Content-Type", "application/json") + getUserReq.Header.Add("Authorization", fmt.Sprintf("bearer %s", adminToken)) + + getUserRes, err := client.Do(getUserReq) + if err != nil { + log.Err(err).Msg("Failed to make get user call") + return nil, errors.Internal("Could not validate api key") + } + defer getUserRes.Body.Close() + + // form the validatedClaims + if getUserRes.StatusCode != http.StatusOK { + log.Error().Int("status", getUserRes.StatusCode).Msg("Received non OK status from gotrue while validating api key") + return nil, errors.Internal("Received non OK status from gotrue while validating api key") + } + + getUserResBody, err := io.ReadAll(getUserRes.Body) + if err != nil { + log.Err(err).Msg("Failed to read get user body while validating api key") + return nil, errors.Internal("Failed to validate the api key") + } + var getUserResp GetUserResp + // parse JSON response + err = json.Unmarshal(getUserResBody, &getUserResp) + if err != nil { + log.Err(err).Msg("Failed to deserialize response into GetUserResp type") + return nil, errors.Internal("Failed to validate the api key") + } + + // validate password + if config.DefaultConfig.Auth.ApiKeys.UserPassword != getUserResp.EncryptedPassword { + return nil, errors.Unauthenticated("Unsupported api-key") + } + + // validate aud + allowedAud := false + for _, supportedAud := range auds { + if supportedAud == getUserResp.Aud { + allowedAud = true + break + } + } + if !allowedAud { + log.Error().Str("api_key_aud", getUserResp.Aud).Strs("supported_auds", auds).Msg("Audience is not supported") + return nil, errors.Unauthenticated("Unsupported audience") + } + + sub := fmt.Sprintf("gt_key|%s", getUserResp.ID) + + return &types.AccessToken{ + Namespace: getUserResp.AppMetaData.TigrisNamespace, + Sub: sub, + Project: getUserResp.AppMetaData.Project, + Role: getUserResp.Role, + }, nil +} + func createUser(ctx context.Context, createUserPayload []byte, aud string, userType string, g *gotrue) error { payloadReader := bytes.NewReader(createUserPayload) @@ -689,7 +832,6 @@ func getAccessTokenUsingClientCredentialsGotrue(ctx context.Context, clientId st payloadValues := url.Values{} payloadValues.Set("username", clientId) payloadValues.Set("password", clientSecret) - client := &http.Client{} getTokenReq, err := http.NewRequestWithContext(ctx, http.MethodPost, getTokenUrl, strings.NewReader(payloadValues.Encode())) @@ -741,22 +883,21 @@ var ( secretChars = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_-+") ) -func generateClientId(g *gotrue) string { - clientIdLength := g.AuthConfig.Gotrue.ClientIdLength - b := make([]rune, clientIdLength) +func generateClientId(prefix string, length int) string { + b := make([]rune, length) for i := range b { b[i] = idChars[generateRandomInt(len(idChars))] } - return fmt.Sprintf("%s%s", ClientIdPrefix, string(b)) + return fmt.Sprintf("%s%s", prefix, string(b)) } -func generateClientSecret(g *gotrue) string { +func generateClientSecret(g *gotrue, prefix string) string { clientSecretLength := g.AuthConfig.Gotrue.ClientSecretLength b := make([]rune, clientSecretLength) for i := range b { b[i] = secretChars[generateRandomInt(len(secretChars))] } - return fmt.Sprintf("%s%s", ClientSecretPrefix, string(b)) + return fmt.Sprintf("%s%s", prefix, string(b)) } func generateRandomInt(max int) int { diff --git a/server/services/v1/auth/gotrue_test.go b/server/services/v1/auth/gotrue_test.go index 18a2934fc..ead86e2b5 100644 --- a/server/services/v1/auth/gotrue_test.go +++ b/server/services/v1/auth/gotrue_test.go @@ -44,8 +44,8 @@ func TestClientCredentialsCharacters(t *testing.T) { // generate 100 random creds and inspect them for i := 0; i < 100; i++ { - id := generateClientId(g) - secret := generateClientSecret(g) + id := generateClientId(ClientIdPrefix, config.DefaultConfig.Auth.Gotrue.ClientIdLength) + secret := generateClientSecret(g, ClientSecretPrefix) require.Truef(t, containsFromTheseChars(id, idCharSet), "Invalid character in id found, id=%s", id) require.Truef(t, containsFromTheseChars(secret, secretCharSet), "Invalid character in secret found, id=%s", secret) } diff --git a/server/services/v1/auth/noop.go b/server/services/v1/auth/noop.go index b310861eb..dfc6d594c 100644 --- a/server/services/v1/auth/noop.go +++ b/server/services/v1/auth/noop.go @@ -19,10 +19,15 @@ import ( api "github.com/tigrisdata/tigris/api/server/v1" "github.com/tigrisdata/tigris/errors" + "github.com/tigrisdata/tigris/server/types" ) type noop struct{} +func (noop) ValidateApiKey(_ context.Context, _ string, _ []string) (*types.AccessToken, error) { + return nil, nil +} + func (noop) GetAccessToken(_ context.Context, _ *api.GetAccessTokenRequest) (*api.GetAccessTokenResponse, error) { return &api.GetAccessTokenResponse{}, nil } diff --git a/server/services/v1/auth/provider.go b/server/services/v1/auth/provider.go index 5cd6b93ab..97846a933 100644 --- a/server/services/v1/auth/provider.go +++ b/server/services/v1/auth/provider.go @@ -24,6 +24,7 @@ import ( "github.com/tigrisdata/tigris/server/config" "github.com/tigrisdata/tigris/server/metadata" "github.com/tigrisdata/tigris/server/transaction" + "github.com/tigrisdata/tigris/server/types" ) const ( @@ -53,6 +54,7 @@ type Provider interface { DeleteAppKey(ctx context.Context, req *api.DeleteAppKeyRequest) (*api.DeleteAppKeyResponse, error) ListAppKeys(ctx context.Context, req *api.ListAppKeysRequest) (*api.ListAppKeysResponse, error) DeleteAppKeys(ctx context.Context, project string) error + ValidateApiKey(ctx context.Context, apiKey string, auds []string) (*types.AccessToken, error) CreateGlobalAppKey(ctx context.Context, req *api.CreateGlobalAppKeyRequest) (*api.CreateGlobalAppKeyResponse, error) UpdateGlobalAppKey(ctx context.Context, req *api.UpdateGlobalAppKeyRequest) (*api.UpdateGlobalAppKeyResponse, error) @@ -61,6 +63,12 @@ type Provider interface { ListGlobalAppKeys(ctx context.Context, req *api.ListGlobalAppKeysRequest) (*api.ListGlobalAppKeysResponse, error) } +func NewGotrueProvider() Provider { + return &gotrue{ + AuthConfig: config.DefaultConfig.Auth, + } +} + func NewProvider(userstore *metadata.UserSubspace, txMgr *transaction.Manager) Provider { var authProvider Provider = &noop{} diff --git a/server/services/v1/billing/metronome.go b/server/services/v1/billing/metronome.go index 7ba97f47b..28cacae4a 100644 --- a/server/services/v1/billing/metronome.go +++ b/server/services/v1/billing/metronome.go @@ -28,7 +28,6 @@ import ( "github.com/tigrisdata/tigris/errors" "github.com/tigrisdata/tigris/server/config" "github.com/tigrisdata/tigris/server/metrics" - "github.com/uber-go/tally" "google.golang.org/protobuf/types/known/timestamppb" ) @@ -61,33 +60,28 @@ func NewMetronomeProvider(conf config.Metronome) (*Metronome, error) { return &Metronome{Config: conf, client: client, billedMetricsByName: bm, billedMetricsById: bu}, nil } -func (*Metronome) measure(ctx context.Context, scope tally.Scope, operation string, f func(ctx context.Context) (*http.Response, error)) { +func (*Metronome) measure(ctx context.Context, operation string, f func(ctx context.Context) (*http.Response, error)) { me := metrics.NewMeasurement( metrics.MetronomeServiceName, operation, metrics.MetronomeSpanType, - map[string]string{}) + metrics.GetMetronomeBaseTags(operation)) me.StartTracing(ctx, true) resp, err := f(ctx) _ = me.FinishTracing(ctx) - me.RecordDuration(scope, map[string]string{}) if resp != nil { defer resp.Body.Close() - // e.g.:- metronome_create_account_request + // e.g.:- metronome_create_account_requests // tags: response_code: 200 - me.IncrementCount(scope, metrics.GetResponseCodeTags(resp.StatusCode), "request", 1) + me.AddTags(metrics.GetMetronomeResponseCodeTags(resp.StatusCode)) } - - availability := int64(100) - var errTags map[string]string if err != nil { - availability = 0 - errTags = metrics.GetErrorCodeTags(err) + me.CountErrorForScope(metrics.MetronomeRequestError, me.GetMetronomeTags()) + } else { + me.CountOkForScope(metrics.MetronomeRequestOk, me.GetMetronomeTags()) } - // e.g.:- metronome_create_account_availability - // tags: error_value: err.Error() - me.IncrementCount(scope, errTags, "availability", availability) + me.RecordDuration(metrics.MetronomeResponseTime, me.GetMetronomeTags()) } func (m *Metronome) CreateAccount(ctx context.Context, namespaceId string, name string) (AccountId, error) { @@ -101,7 +95,7 @@ func (m *Metronome) CreateAccount(ctx context.Context, namespaceId string, name Name: name, } - m.measure(ctx, metrics.MetronomeCreateAccount, "create_account", func(ctx context.Context) (*http.Response, error) { + m.measure(ctx, "create_account", func(ctx context.Context) (*http.Response, error) { resp, err = m.client.CreateCustomerWithResponse(ctx, body) if resp == nil { return nil, err @@ -139,7 +133,7 @@ func (m *Metronome) AddPlan(ctx context.Context, accountId AccountId, planId uui StartingOn: pastMidnight(), } - m.measure(ctx, metrics.MetronomeAddPlan, "add_plan", func(ctx context.Context) (*http.Response, error) { + m.measure(ctx, "add_plan", func(ctx context.Context) (*http.Response, error) { resp, err = m.client.AddPlanToCustomerWithResponse(ctx, accountId, body) if resp == nil { return nil, err @@ -171,7 +165,7 @@ func (m *Metronome) PushUsageEvents(ctx context.Context, events []*UsageEvent) e metrics.MetronomeServiceName, "ingest", metrics.MetronomeSpanType, - metrics.GetIngestEventTags("usage")) + metrics.GetMetronomeIngestEventTags("usage")) me.IncrementCount(metrics.MetronomeIngest, me.GetTags(), "events", int64(len(billingEvents))) return m.pushBillingEvents(ctx, billingEvents) @@ -189,7 +183,7 @@ func (m *Metronome) PushStorageEvents(ctx context.Context, events []*StorageEven metrics.MetronomeServiceName, "ingest", metrics.MetronomeSpanType, - metrics.GetIngestEventTags("storage")) + metrics.GetMetronomeIngestEventTags("storage")) me.IncrementCount(metrics.MetronomeIngest, me.GetTags(), "events", int64(len(billingEvents))) return m.pushBillingEvents(ctx, billingEvents) @@ -222,13 +216,14 @@ func (m *Metronome) pushBillingEvents(ctx context.Context, events []biller.Event page := events[p*pageSize : high] // content encoding - gzip? - m.measure(ctx, metrics.MetronomeIngest, "ingest", func(ctx context.Context) (*http.Response, error) { + m.measure(ctx, "ingest", func(ctx context.Context) (*http.Response, error) { resp, err = m.client.IngestWithResponse(ctx, page) if resp == nil { return nil, err } return resp.HTTPResponse, err }) + if err != nil { return err } @@ -267,13 +262,14 @@ func (m *Metronome) GetInvoices(ctx context.Context, accountId AccountId, r *api params.EndingBefore = &t } - m.measure(ctx, metrics.MetronomeListInvoices, "list_invoices", func(ctx context.Context) (*http.Response, error) { + m.measure(ctx, "list_invoices", func(ctx context.Context) (*http.Response, error) { resp, err = m.client.ListInvoicesWithResponse(ctx, accountId, params) if resp == nil { return nil, err } return resp.HTTPResponse, err }) + if err != nil { return nil, err } @@ -303,7 +299,7 @@ func (m *Metronome) GetInvoiceById(ctx context.Context, accountId AccountId, inv if err != nil { return nil, api.Errorf(api.Code_INVALID_ARGUMENT, "invoiceId is not valid - %s", err.Error()) } - m.measure(ctx, metrics.MetronomeGetInvoice, "get_invoice", func(ctx context.Context) (*http.Response, error) { + m.measure(ctx, "get_invoice", func(ctx context.Context) (*http.Response, error) { resp, err = m.client.GetInvoiceWithResponse(ctx, accountId, invoiceUUID) if resp == nil { return nil, err @@ -389,7 +385,7 @@ func (m *Metronome) GetUsage(ctx context.Context, id AccountId, r *UsageRequest) } } - m.measure(ctx, metrics.MetronomeGetUsage, "get_usage", func(ctx context.Context) (*http.Response, error) { + m.measure(ctx, "get_usage", func(ctx context.Context) (*http.Response, error) { resp, err = m.client.GetUsageBatchWithResponse(ctx, &biller.GetUsageBatchParams{NextPage: r.NextPage}, reqParams) if resp == nil { return nil, err @@ -440,13 +436,14 @@ func (m *Metronome) GetAccountId(ctx context.Context, namespaceId string) (Accou return uuid.Nil, errors.InvalidArgument("namespaceId cannot be empty") } params := &biller.ListCustomersParams{IngestAlias: &namespaceId} - m.measure(ctx, metrics.MetronomeGetCustomer, "get_customer", func(ctx context.Context) (*http.Response, error) { + m.measure(ctx, "get_customer", func(ctx context.Context) (*http.Response, error) { resp, err = m.client.ListCustomersWithResponse(ctx, params) if resp == nil { return nil, err } return resp.HTTPResponse, err }) + if err != nil { return uuid.Nil, err } diff --git a/server/services/v1/billing/metronome_test.go b/server/services/v1/billing/metronome_test.go index aff6a739d..07772c768 100644 --- a/server/services/v1/billing/metronome_test.go +++ b/server/services/v1/billing/metronome_test.go @@ -30,12 +30,10 @@ import ( api "github.com/tigrisdata/tigris/api/server/v1" "github.com/tigrisdata/tigris/server/config" "github.com/tigrisdata/tigris/server/metrics" - "github.com/uber-go/tally" "google.golang.org/protobuf/types/known/timestamppb" ) func TestMetronome_CreateAccount(t *testing.T) { - initializeMetricsForTest() defer gock.Off() cfg := config.DefaultConfig.Billing.Metronome metronome, err := NewMetronomeProvider(cfg) @@ -96,7 +94,6 @@ func TestMetronome_CreateAccount(t *testing.T) { } func TestMetronome_AddDefaultPlan(t *testing.T) { - initializeMetricsForTest() defer gock.Off() cfg := config.DefaultConfig.Billing.Metronome metronome, err := NewMetronomeProvider(cfg) @@ -148,7 +145,7 @@ func TestMetronome_AddDefaultPlan(t *testing.T) { } func TestMetronome_PushStorageEvents(t *testing.T) { - initializeMetricsForTest() + metrics.InitializeMetrics() defer gock.Off() cfg := config.DefaultConfig.Billing.Metronome metronome, err := NewMetronomeProvider(cfg) @@ -254,7 +251,7 @@ func TestMetronome_PushStorageEvents(t *testing.T) { } func TestMetronome_PushUsageEvents(t *testing.T) { - initializeMetricsForTest() + metrics.InitializeMetrics() defer gock.Off() cfg := config.DefaultConfig.Billing.Metronome metronome, err := NewMetronomeProvider(cfg) @@ -360,7 +357,6 @@ func TestMetronome_PushUsageEvents(t *testing.T) { } func TestMetronome_FetchInvoices(t *testing.T) { - initializeMetricsForTest() defer gock.Off() cfg := config.DefaultConfig.Billing.Metronome metronome, err := NewMetronomeProvider(cfg) @@ -440,7 +436,6 @@ func TestMetronome_FetchInvoices(t *testing.T) { } func TestMetronome_GetInvoiceById(t *testing.T) { - initializeMetricsForTest() defer gock.Off() cfg := config.DefaultConfig.Billing.Metronome metronome, err := NewMetronomeProvider(cfg) @@ -538,7 +533,6 @@ func TestMetronome_GetInvoiceById(t *testing.T) { } func TestMetronome_GetUsage(t *testing.T) { - initializeMetricsForTest() defer gock.Off() cfg := config.DefaultConfig.Billing.Metronome cfg.BilledMetrics = map[string]string{ @@ -774,7 +768,6 @@ func TestMetronome_GetUsage(t *testing.T) { } func TestMetronome_GetAccountId(t *testing.T) { - initializeMetricsForTest() defer gock.Off() cfg := config.DefaultConfig.Billing.Metronome metronome, err := NewMetronomeProvider(cfg) @@ -1026,13 +1019,3 @@ func TestMetronome_buildInvoice(t *testing.T) { } }) } - -func initializeMetricsForTest() { - metrics.MetronomeCreateAccount = tally.NewTestScope("create_account", map[string]string{}) - metrics.MetronomeAddPlan = tally.NewTestScope("add_plan", map[string]string{}) - metrics.MetronomeIngest = tally.NewTestScope("ingest", map[string]string{}) - metrics.MetronomeListInvoices = tally.NewTestScope("list_invoices", map[string]string{}) - metrics.MetronomeGetInvoice = tally.NewTestScope("get_invoice", map[string]string{}) - metrics.MetronomeGetUsage = tally.NewTestScope("get_usage", map[string]string{}) - metrics.MetronomeGetCustomer = tally.NewTestScope("get_customer", map[string]string{}) -} diff --git a/server/services/v1/observability.go b/server/services/v1/observability.go index 68db5d552..8480e94db 100644 --- a/server/services/v1/observability.go +++ b/server/services/v1/observability.go @@ -199,6 +199,43 @@ func (*observabilityService) GetInfo(_ context.Context, _ *api.GetInfoRequest) ( }, nil } +func (*observabilityService) WhoAmI(ctx context.Context, _ *api.WhoAmIRequest) (*api.WhoAmIResponse, error) { + currentSub, err := request.GetCurrentSub(ctx) + if err != nil { + log.Err(err).Msg("Failed to read current sub") + return nil, errors.Internal("Failed to read current sub") + } + + currentNamespace, err := request.GetNamespace(ctx) + if err != nil { + log.Err(err).Msg("Failed to read current namespace") + return nil, errors.Internal("Failed to read current namespace") + } + + reqMeta, err := request.GetRequestMetadataFromContext(ctx) + if err != nil { + log.Err(err).Msg("Failed to read current request metadata") + return nil, errors.Internal("Failed to read request metadata") + } + + userType := "machine" + if strings.HasPrefix(currentSub, "auth0") { + userType = "human" + } + + authMethod := "JWT" + if strings.HasPrefix(currentSub, "gt_key|") { + authMethod = "api_key" + } + return &api.WhoAmIResponse{ + Sub: currentSub, + Namespace: currentNamespace, + Role: reqMeta.Role, + AuthMethod: authMethod, + UserType: userType, + }, nil +} + func (o *observabilityService) RegisterHTTP(router chi.Router, inproc *inprocgrpc.Channel) error { mux := runtime.NewServeMux( runtime.WithMarshalerOption(runtime.MIMEWildcard, &api.CustomMarshaler{JSONBuiltin: &runtime.JSONBuiltin{}}), diff --git a/test/v1/server/auth_test.go b/test/v1/server/auth_test.go index 9186b2691..aca87735f 100644 --- a/test/v1/server/auth_test.go +++ b/test/v1/server/auth_test.go @@ -114,7 +114,7 @@ func TestGoTrueAuthProvider(t *testing.T) { createProject2(t, testProject, token).Status(http.StatusOK) // create app key - createdAppKey := createAppKey(e2, token, "test_key", "auth_test") + createdAppKey := createAppKey(e2, token, "test_key", "auth_test", "") require.NotNil(t, createdAppKey) id := createdAppKey.Object().Value("id").String() secret := createdAppKey.Object().Value("secret").String() @@ -246,8 +246,8 @@ func TestGlobalAppKeys(t *testing.T) { require.Equal(t, "test_key", name.Raw()) require.Equal(t, "This key is used for integration test purpose.", description.Raw()) - require.True(t, int(id.Length().Raw()) == 30+len(auth.ClientIdPrefix)) // length + prefix - require.True(t, int(secret.Length().Raw()) == 50+len(auth.ClientSecretPrefix)) // length + prefix + require.Equal(t, 30+len(auth.GlobalClientIdPrefix), int(id.Length().Raw())) // length + prefix + require.Equal(t, 50+len(auth.GlobalClientSecretPrefix), int(secret.Length().Raw())) // length + prefix // update updateGlobalAppKeyPayload := Map{ @@ -276,7 +276,7 @@ func TestGlobalAppKeys(t *testing.T) { Object().Value("app_key") require.Equal(t, id.Raw(), rotatedKey.Object().Value("id").Raw()) require.NotEqual(t, secret.Raw(), rotatedKey.Object().Value("secret").Raw()) - require.True(t, len(rotatedKey.Object().Value("secret").String().Raw()) == 50+len(auth.ClientSecretPrefix)) + require.Equal(t, 50+len(auth.GlobalClientSecretPrefix), len(rotatedKey.Object().Value("secret").String().Raw())) // list globalAppKeys := e.GET(globalAppKeysOperation("get")). @@ -327,9 +327,9 @@ func TestGlobalAndLocalAppKeys(t *testing.T) { _ = createGlobalAppKey(e, token, "g2") // create three local app keys - _ = createAppKey(e, token, "l1", proj) - _ = createAppKey(e, token, "l2", proj) - _ = createAppKey(e, token, "l3", proj) + _ = createAppKey(e, token, "l1", proj, "") + _ = createAppKey(e, token, "l2", proj, "") + _ = createAppKey(e, token, "l3", proj, "") // list globalAppKeys := e.GET(globalAppKeysOperation("get")). @@ -380,12 +380,15 @@ func deleteGlobalAppKey(e *httpexpect.Expect, token string, id string) *httpexpe Expect() } -func createAppKey(e *httpexpect.Expect, token string, name string, project string) *httpexpect.Value { +func createAppKey(e *httpexpect.Expect, token string, name string, project string, keyType string) *httpexpect.Value { createAppKeyPayload := Map{ "name": name, "description": "This key is used for integration test purpose.", "project": project, } + if keyType != "" { + createAppKeyPayload["key_type"] = keyType + } return e.POST(appKeysOperation(project, "create")). WithHeader(Authorization, Bearer+token).WithJSON(createAppKeyPayload). Expect(). @@ -393,6 +396,7 @@ func createAppKey(e *httpexpect.Expect, token string, name string, project strin JSON(). Object().Value("created_app_key") } + func createGlobalAppKey(e *httpexpect.Expect, token string, name string) *httpexpect.Value { createGlobalAppKeyPayload := Map{ "name": name, @@ -416,7 +420,7 @@ func TestMultipleAppsCreation(t *testing.T) { createProject2(t, testProject, token).Status(http.StatusOK) for i := 0; i < 5; i++ { - createdAppKey := createAppKey(e2, token, fmt.Sprintf("test_key_%d", i), testProject) + createdAppKey := createAppKey(e2, token, fmt.Sprintf("test_key_%d", i), testProject, "") require.NotNil(t, createdAppKey) generatedClientId := createdAppKey.Object().Value("id").String().Raw() generatedClientSecret := createdAppKey.Object().Value("secret").String().Raw() @@ -431,9 +435,6 @@ func TestMultipleAppsCreation(t *testing.T) { Status(http.StatusOK). JSON(). Object().Value("app_keys").Array() - for _, key := range appKeys.Raw() { - fmt.Println(key) - } require.Equal(t, 5, int(appKeys.Length().Raw())) for _, value := range appKeys.Iter() { createdAt := int64(value.Object().Value("created_at").Number().Raw()) @@ -454,7 +455,7 @@ func TestListAppKeys(t *testing.T) { for i := 0; i < 5; i++ { projectForThisKey := fmt.Sprintf("%s%d", testProject, i%2) - createdAppKey := createAppKey(e2, token, fmt.Sprintf("test_key_%d", i), projectForThisKey) + createdAppKey := createAppKey(e2, token, fmt.Sprintf("test_key_%d", i), projectForThisKey, "") require.NotNil(t, createdAppKey) } @@ -496,12 +497,127 @@ func TestEmptyListAppKeys(t *testing.T) { require.Equal(t, make(map[string]any), appKeys) } +func TestApiKeyUsage(t *testing.T) { + token := readToken(t, RSATokenFilePath) + createTestNamespace(t, token) + e := expectLow(t, config.GetBaseURL2()) + testProject := "TestApiKey" + + createdApiKey := createAppKey(e, token, "test_api_key", testProject, auth.AppKeyTypeApiKey) + require.NotNil(t, createdApiKey) + key := createdApiKey.Object().Value("id").String().Raw() + require.Equal(t, 125, len(key)) // 120 fixed + 5 prefix + + // use the api key + listProjectsResp := listProjects(t, key) + listProjectsResp.Status(http.StatusOK).JSON() +} + +func TestApiKeyCrud(t *testing.T) { + token := readToken(t, RSATokenFilePath) + createTestNamespace(t, token) + e := expectLow(t, config.GetBaseURL2()) + testProject := "TestApiKeyCrud" + + // create + _ = createAppKey(e, token, "test_api_key_1", testProject, auth.AppKeyTypeApiKey) + createdApiKey := createAppKey(e, token, "test_api_key_2", testProject, auth.AppKeyTypeApiKey) + require.NotNil(t, createdApiKey) + key := createdApiKey.Object().Value("id").String().Raw() + require.Equal(t, 125, len(key)) // 120 fixed + 5 prefix + + // read + appKeys := listAppKeys(t, e, testProject, token) + found := false + for i := 0; i < int(appKeys.Length().Raw()); i++ { + k := appKeys.Element(i) + if k.Object().Value("id").Raw() == key { + require.False(t, found) + found = true + require.Equal(t, "test_api_key_2", k.Object().Value("name").String().Raw()) + require.Equal(t, auth.AppKeyTypeApiKey, k.Object().Value("key_type").String().Raw()) + } + } + require.True(t, found) + + // update + updateAppKeyPayload := Map{ + "id": key, + "name": "[updated] test_api_key", + "description": "[updated]This key is used for integration test purpose.", + } + updateResp := e.POST(appKeysOperation(testProject, "update")). + WithHeader(Authorization, Bearer+token). + WithJSON(updateAppKeyPayload). + Expect(). + Status(http.StatusOK) + require.NotNil(t, updateResp) + + // read back post update + appKeys = listAppKeys(t, e, testProject, token) + found = false + for i := 0; i < int(appKeys.Length().Raw()); i++ { + k := appKeys.Element(i) + if k.Object().Value("id").Raw() == key { + require.False(t, found) + found = true + require.Equal(t, "[updated] test_api_key", k.Object().Value("name").String().Raw()) + require.Equal(t, "[updated]This key is used for integration test purpose.", k.Object().Value("description").String().Raw()) + require.Equal(t, auth.AppKeyTypeApiKey, k.Object().Value("key_type").String().Raw()) + } + } + require.True(t, found) + + // delete + deleteAppKeyPayload := Map{ + "id": key, + } + deletedResponse := e.DELETE(appKeysOperation(testProject, "delete")). + WithHeader(Authorization, Bearer+token).WithJSON(deleteAppKeyPayload). + Expect(). + Status(http.StatusOK). + JSON(). + Object(). + Value("deleted"). + Boolean() + require.True(t, deletedResponse.Raw()) + + // read back + appKeys = listAppKeys(t, e, testProject, token) + found = false + for i := 0; i < int(appKeys.Length().Raw()); i++ { + k := appKeys.Element(i) + if k.Object().Value("id").Raw() == key { + found = true + } + } + require.False(t, found) +} + +func TestWhoAmI(t *testing.T) { + token := readToken(t, RSATokenFilePath) + createTestNamespace(t, token) + e := expectLow(t, config.GetBaseURL2()) + + res := e.GET("/v1/observability/whoami"). + WithHeader(Authorization, Bearer+token). + Expect(). + Status(http.StatusOK) + resJSON := res.JSON().Object() + + require.Equal(t, "JWT", resJSON.Value("auth_method").String().Raw()) + require.Equal(t, "tigris_test", resJSON.Value("namespace").String().Raw()) + require.Equal(t, "gt|5f6812e4-6f4b-44d5-bc06-5ebe33e936ec", resJSON.Value("sub").String().Raw()) + require.Equal(t, "machine", resJSON.Value("user_type").String().Raw()) + +} + func TestCreateAccessToken(t *testing.T) { e2 := expectLow(t, config.GetBaseURL2()) testProject := "auth_test" token := readToken(t, RSATokenFilePath) createTestNamespace(t, token) - createdAppKey := createAppKey(e2, token, "test_key", testProject) + createdAppKey := createAppKey(e2, token, "test_key", testProject, "") require.NotNil(t, createdAppKey) id := createdAppKey.Object().Value("id").String() @@ -613,7 +729,7 @@ func TestAuthFailure(t *testing.T) { Value("error"). Object() require.Equal(t, "UNAUTHENTICATED", authFailureErrorResponse.Value("code").String().Raw()) - require.Equal(t, "Failed to validate access token, could not be validated", authFailureErrorResponse.Value("message").String().Raw()) + require.Equal(t, "Failed to authenticate", authFailureErrorResponse.Value("message").String().Raw()) } func TestUserInvitations(t *testing.T) { diff --git a/test/v1/server/document_test.go b/test/v1/server/document_test.go index 2a0987d50..cdf5dcf89 100644 --- a/test/v1/server/document_test.go +++ b/test/v1/server/document_test.go @@ -26,6 +26,7 @@ import ( "testing" "time" + "github.com/davecgh/go-spew/spew" "github.com/google/uuid" jsoniter "github.com/json-iterator/go" "github.com/stretchr/testify/assert" @@ -5786,6 +5787,7 @@ func TestImport(t *testing.T) { } if c.err == http.StatusOK { + spew.Dump(resp.Body().Raw()) resp.Status(c.err).JSON().Object(). ValueEqual("status", "inserted") diff --git a/test/v1/server/integration_test.go b/test/v1/server/integration_test.go index dd2a2674f..08b23a15a 100644 --- a/test/v1/server/integration_test.go +++ b/test/v1/server/integration_test.go @@ -22,7 +22,9 @@ import ( "os" "testing" + "github.com/stretchr/testify/require" api "github.com/tigrisdata/tigris/api/server/v1" + "github.com/tigrisdata/tigris/server/services/v1/auth" "github.com/tigrisdata/tigris/test/config" "gopkg.in/gavv/httpexpect.v1" ) @@ -277,6 +279,16 @@ func appKeysOperation(project string, operation string) string { return fmt.Sprintf("/v1/projects/%s/apps/keys/%s", project, operation) } +func listAppKeys(t *testing.T, e *httpexpect.Expect, project string, bearer string) *httpexpect.Array { + listApiKeysResp := e.GET(appKeysOperation(project, "get")). + WithQuery("key_type", auth.AppKeyTypeApiKey). + WithHeader(Authorization, Bearer+bearer). + Expect() + listApiKeysRespJSON := listApiKeysResp.Status(http.StatusOK).JSON() + require.NotNil(t, listApiKeysRespJSON) + return listApiKeysRespJSON.Object().Value("app_keys").Array() +} + func globalAppKeysOperation(operation string) string { if operation == "get" { return "/v1/apps/keys"