Skip to content

Commit

Permalink
Merge branch 'master' of github.com:rudderlabs/rudder-server into rel…
Browse files Browse the repository at this point in the history 
…ease/1.22.0-rc.3
  • Loading branch information
@lvrach
lvrach committed Mar 18, 2024
2 parents 9912661 + 3da9179 commit 3ecabc4
Show file tree
Hide file tree
Showing 13 changed files with 434 additions and 462 deletions.
9 changes: 3 additions & 6 deletions archiver/archiver_isolation_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@ import (
backendconfig "github.com/rudderlabs/rudder-server/backend-config"
"github.com/rudderlabs/rudder-server/jobsdb"
"github.com/rudderlabs/rudder-server/runner"
"github.com/rudderlabs/rudder-server/testhelper"
"github.com/rudderlabs/rudder-server/testhelper/destination"
"github.com/rudderlabs/rudder-server/testhelper/health"
"github.com/rudderlabs/rudder-server/utils/misc"
Expand Down Expand Up @@ -125,15 +124,13 @@ func ArchivalScenario(

pool, err := dockertest.NewPool("")
require.NoError(t, err, "Failed to create docker pool")
cleanup := &testhelper.Cleanup{}
defer cleanup.Run()

postgresContainer, err := postgres.Setup(pool, cleanup, postgres.WithShmSize(256*bytesize.MB))
postgresContainer, err := postgres.Setup(pool, t, postgres.WithShmSize(256*bytesize.MB))
require.NoError(t, err, "failed to setup postgres container")

minioResource, err := minio.Setup(pool, cleanup)
minioResource, err := minio.Setup(pool, t)
require.NoError(t, err, "failed to setup minio container")
transformerContainer, err := destination.SetupTransformer(pool, cleanup)
transformerContainer, err := destination.SetupTransformer(pool, t)
require.NoError(t, err, "failed to setup transformer container")

configMap := dummyConfig(numWorkspace, numSourcesPerWorkspace, minioResource)
Expand Down
30 changes: 28 additions & 2 deletions enterprise/reporting/reporting.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import (
"github.com/lib/pq"
"github.com/samber/lo"

"github.com/rudderlabs/rudder-go-kit/bytesize"
"github.com/rudderlabs/rudder-go-kit/config"
"github.com/rudderlabs/rudder-go-kit/logger"
"github.com/rudderlabs/rudder-go-kit/stats"
Expand All @@ -43,6 +44,7 @@ const (
StatReportingHttpReq = "reporting_client_http_request"
StatReportingGetMinReportedAtQueryTime = "reporting_client_get_min_reported_at_query_time"
StatReportingGetReportsQueryTime = "reporting_client_get_reports_query_time"
StatReportingVacuumDuration = "reporting_vacuum_duration"
)

type DefaultReporter struct {
Expand Down Expand Up @@ -354,6 +356,7 @@ func (r *DefaultReporter) mainLoop(ctx context.Context, c types.SyncerConfig) {
getReportsCount := r.stats.NewTaggedStat(StatReportingGetReportsCount, stats.HistogramType, tags)
getAggregatedReportsTimer := r.stats.NewTaggedStat(StatReportingGetAggregatedReportsTime, stats.TimerType, tags)
getAggregatedReportsCount := r.stats.NewTaggedStat(StatReportingGetAggregatedReportsCount, stats.HistogramType, tags)
vacuumDuration := r.stats.NewTaggedStat(StatReportingVacuumDuration, stats.TimerType, tags)

r.getMinReportedAtQueryTime = r.stats.NewTaggedStat(StatReportingGetMinReportedAtQueryTime, stats.TimerType, tags)
r.getReportsQueryTime = r.stats.NewTaggedStat(StatReportingGetReportsQueryTime, stats.TimerType, tags)
Expand Down Expand Up @@ -389,10 +392,10 @@ func (r *DefaultReporter) mainLoop(ctx context.Context, c types.SyncerConfig) {
}
requestChan := make(chan struct{}, r.maxConcurrentRequests.Load())
loopStart := time.Now()
currentMs := time.Now().UTC().Unix() / 60
currentMin := time.Now().UTC().Unix() / 60

getReportsStart := time.Now()
reports, reportedAt, err := r.getReports(currentMs, c.ConnInfo)
reports, reportedAt, err := r.getReports(currentMin, c.ConnInfo)
getReportsTimer.Since(getReportsStart)
getReportsCount.Observe(float64(len(reports)))
if len(reports) == 0 {
Expand All @@ -417,6 +420,8 @@ func (r *DefaultReporter) mainLoop(ctx context.Context, c types.SyncerConfig) {
getAggregatedReportsCount.Observe(float64(len(metrics)))

errGroup, errCtx := errgroup.WithContext(ctx)
// default to -1 to allow unlimited concurrency
errGroup.SetLimit(config.GetInt("Reporting.maxConcurrentRequests", -1))
for _, metric := range metrics {
if r.whActionsOnly && metric.SourceCategory != "warehouse" {
// if whActionsOnly is true, we only send reports for wh actions sources
Expand Down Expand Up @@ -446,6 +451,27 @@ func (r *DefaultReporter) mainLoop(ctx context.Context, c types.SyncerConfig) {
if err != nil {
r.log.Errorf(`[ Reporting ]: Error deleting local reports from %s: %v`, ReportsTable, err)
}

vacuumStart := time.Now()
var sizeEstimate int64
if err := dbHandle.QueryRowContext(
ctx,
fmt.Sprintf(`SELECT pg_table_size(oid) from pg_class where relname='%s';`, ReportsTable),
).Scan(&sizeEstimate); err != nil {
r.log.Errorn(
`[ Reporting ]: Error getting table size estimate`,
logger.NewErrorField(err),
)
}
if sizeEstimate > config.GetInt64("Reporting.vacuumThresholdBytes", 5*bytesize.GB) {
if _, err := dbHandle.ExecContext(ctx, `vacuum full analyze reports;`); err != nil {
r.log.Errorn(
`[ Reporting ]: Error vacuuming reports table`,
logger.NewErrorField(err),
)
}
vacuumDuration.Since(vacuumStart)
}
}

mainLoopTimer.Since(loopStart)
Expand Down
89 changes: 45 additions & 44 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,26 +30,26 @@ replace (
)

require (
cloud.google.com/go/bigquery v1.59.0
cloud.google.com/go/bigquery v1.59.1
cloud.google.com/go/pubsub v1.36.1
cloud.google.com/go/storage v1.37.0
cloud.google.com/go/storage v1.39.1
github.com/Azure/azure-storage-blob-go v0.15.0
github.com/ClickHouse/clickhouse-go v1.5.4
github.com/DATA-DOG/go-sqlmock v1.5.2
github.com/alexeyco/simpletable v1.0.0
github.com/allisson/go-pglock/v2 v2.0.1
github.com/apache/pulsar-client-go v0.12.0
github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de
github.com/aws/aws-sdk-go v1.50.29
github.com/bugsnag/bugsnag-go/v2 v2.2.1
github.com/aws/aws-sdk-go v1.50.38
github.com/bugsnag/bugsnag-go/v2 v2.3.0
github.com/cenkalti/backoff v2.2.1+incompatible
github.com/cenkalti/backoff/v4 v4.2.1
github.com/confluentinc/confluent-kafka-go/v2 v2.3.0
github.com/databricks/databricks-sql-go v1.5.3
github.com/denisenkom/go-mssqldb v0.12.3
github.com/dgraph-io/badger/v4 v4.2.0
github.com/docker/docker v24.0.7+incompatible
github.com/go-chi/chi/v5 v5.0.11
github.com/go-chi/chi/v5 v5.0.12
github.com/go-redis/redis v6.15.9+incompatible
github.com/go-redis/redis/v8 v8.11.5
github.com/gofrs/uuid v4.4.0+incompatible
Expand All @@ -69,22 +69,22 @@ require (
github.com/lib/pq v1.10.9
github.com/linkedin/goavro/v2 v2.12.0
github.com/manifoldco/promptui v0.9.0
github.com/marcboeker/go-duckdb v1.5.6
github.com/minio/minio-go/v7 v7.0.67
github.com/marcboeker/go-duckdb v1.6.1
github.com/minio/minio-go/v7 v7.0.69
github.com/mitchellh/mapstructure v1.5.0
github.com/olekukonko/tablewriter v0.0.5
github.com/onsi/ginkgo/v2 v2.15.0
github.com/onsi/gomega v1.31.1
github.com/ory/dockertest/v3 v3.10.0
github.com/oschwald/maxminddb-golang v1.12.0
github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5
github.com/prometheus/client_model v0.5.0
github.com/prometheus/client_model v0.6.0
github.com/redis/go-redis/v9 v9.4.0
github.com/rs/cors v1.10.1
github.com/rudderlabs/analytics-go v3.3.3+incompatible
github.com/rudderlabs/bing-ads-go-sdk v0.2.1
github.com/rudderlabs/compose-test v0.1.3
github.com/rudderlabs/rudder-go-kit v0.20.2
github.com/rudderlabs/rudder-go-kit v0.23.2
github.com/rudderlabs/rudder-observability-kit v0.0.3
github.com/rudderlabs/sql-tunnels v0.1.6
github.com/samber/lo v1.39.0
Expand All @@ -95,8 +95,8 @@ require (
github.com/spaolacci/murmur3 v1.1.0
github.com/spf13/cast v1.6.0
github.com/spf13/viper v1.18.2
github.com/stretchr/testify v1.8.4
github.com/tidwall/gjson v1.17.0
github.com/stretchr/testify v1.9.0
github.com/tidwall/gjson v1.17.1
github.com/tidwall/sjson v1.2.5
github.com/trinodb/trino-go-client v0.313.0
github.com/urfave/cli/v2 v2.27.1
Expand All @@ -106,23 +106,23 @@ require (
github.com/xitongsys/parquet-go-source v0.0.0-20240122235623-d6294584ab18
go.etcd.io/etcd/api/v3 v3.5.12
go.etcd.io/etcd/client/v3 v3.5.12
go.opentelemetry.io/otel v1.22.0
go.opentelemetry.io/otel v1.24.0
go.uber.org/atomic v1.11.0
go.uber.org/automaxprocs v1.5.3
go.uber.org/goleak v1.3.0
golang.org/x/exp v0.0.0-20240119083558-1b970713d09a
golang.org/x/oauth2 v0.17.0
golang.org/x/oauth2 v0.18.0
golang.org/x/sync v0.6.0
golang.org/x/text v0.14.0
google.golang.org/api v0.162.0
google.golang.org/genproto/googleapis/rpc v0.0.0-20240205150955-31a09d347014
google.golang.org/grpc v1.61.0
google.golang.org/protobuf v1.32.0
google.golang.org/api v0.167.0
google.golang.org/genproto/googleapis/rpc v0.0.0-20240228224816-df926f6c8641
google.golang.org/grpc v1.62.1
google.golang.org/protobuf v1.33.0
)

require (
cloud.google.com/go v0.112.0 // indirect
cloud.google.com/go/compute v1.23.3 // indirect
cloud.google.com/go v0.112.1 // indirect
cloud.google.com/go/compute v1.24.0 // indirect
cloud.google.com/go/compute/metadata v0.2.3 // indirect
cloud.google.com/go/iam v1.1.6 // indirect
github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 // indirect
Expand Down Expand Up @@ -159,6 +159,7 @@ require (
github.com/aws/smithy-go v1.17.0 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/bits-and-blooms/bitset v1.4.0 // indirect
github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect
github.com/bugsnag/panicwrap v1.3.4 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e // indirect
Expand Down Expand Up @@ -197,7 +198,7 @@ require (
github.com/golang-jwt/jwt v3.2.1+incompatible // indirect
github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe // indirect
github.com/golang-sql/sqlexp v0.1.0 // indirect
github.com/golang/glog v1.1.2 // indirect
github.com/golang/glog v1.2.0 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/golang/snappy v0.0.4 // indirect
Expand All @@ -207,7 +208,7 @@ require (
github.com/google/s2a-go v0.1.7 // indirect
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
github.com/googleapis/gax-go/v2 v2.12.0 // indirect
github.com/googleapis/gax-go/v2 v2.12.2 // indirect
github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
Expand Down Expand Up @@ -241,7 +242,7 @@ require (
github.com/mtibben/percent v0.2.1 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.1.0-rc2.0.20221005185240-3a7f492d3f1b // indirect
github.com/opencontainers/runc v1.1.5 // indirect
github.com/opencontainers/runc v1.1.12 // indirect
github.com/openzipkin/zipkin-go v0.4.2 // indirect
github.com/pelletier/go-toml/v2 v2.1.0 // indirect
github.com/pierrec/lz4 v2.0.5+incompatible // indirect
Expand All @@ -250,15 +251,15 @@ require (
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
github.com/prometheus/client_golang v1.18.0 // indirect
github.com/prometheus/common v0.46.0 // indirect
github.com/prometheus/client_golang v1.19.0 // indirect
github.com/prometheus/common v0.50.0 // indirect
github.com/prometheus/procfs v0.12.0 // indirect
github.com/rivo/uniseg v0.1.0 // indirect
github.com/rs/xid v1.5.0 // indirect
github.com/rs/zerolog v1.28.0 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/segmentio/backo-go v1.0.1 // indirect
github.com/shirou/gopsutil/v3 v3.23.12 // indirect
github.com/shirou/gopsutil/v3 v3.24.2 // indirect
github.com/sirupsen/logrus v1.9.3 // indirect
github.com/spf13/afero v1.11.0 // indirect
github.com/spf13/jwalterweatherman v1.1.0 // indirect
Expand All @@ -278,35 +279,35 @@ require (
github.com/xeipuuv/gojsonschema v1.2.0 // indirect
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
github.com/xtgo/uuid v0.0.0-20140804021211-a0b114877d4c // indirect
github.com/yusufpapurcu/wmi v1.2.3 // indirect
github.com/yusufpapurcu/wmi v1.2.4 // indirect
github.com/zeebo/xxh3 v1.0.2 // indirect
go.etcd.io/etcd/client/pkg/v3 v3.5.12 // indirect
go.opencensus.io v0.24.0 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.45.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.22.0 // indirect
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.22.0 // indirect
go.opentelemetry.io/otel/exporters/zipkin v1.22.0 // indirect
go.opentelemetry.io/otel/metric v1.22.0 // indirect
go.opentelemetry.io/otel/sdk v1.22.0 // indirect
go.opentelemetry.io/otel/sdk/metric v1.22.0 // indirect
go.opentelemetry.io/otel/trace v1.22.0 // indirect
go.opentelemetry.io/proto/otlp v1.0.0 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.48.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.48.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.24.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.24.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.24.0 // indirect
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.24.0 // indirect
go.opentelemetry.io/otel/exporters/zipkin v1.24.0 // indirect
go.opentelemetry.io/otel/metric v1.24.0 // indirect
go.opentelemetry.io/otel/sdk v1.24.0 // indirect
go.opentelemetry.io/otel/sdk/metric v1.24.0 // indirect
go.opentelemetry.io/otel/trace v1.24.0 // indirect
go.opentelemetry.io/proto/otlp v1.1.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
go.uber.org/zap v1.26.0 // indirect
golang.org/x/crypto v0.19.0 // indirect
go.uber.org/zap v1.27.0 // indirect
golang.org/x/crypto v0.21.0 // indirect
golang.org/x/mod v0.14.0 // indirect
golang.org/x/net v0.21.0 // indirect
golang.org/x/sys v0.17.0 // indirect
golang.org/x/net v0.22.0 // indirect
golang.org/x/sys v0.18.0 // indirect
golang.org/x/term v0.15.0 // indirect
golang.org/x/time v0.5.0 // indirect
golang.org/x/tools v0.17.0 // indirect
golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
google.golang.org/appengine v1.6.8 // indirect
google.golang.org/genproto v0.0.0-20240125205218-1f4bbc51befe // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20240205150955-31a09d347014 // indirect
google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20240304161311-37d4d3c04a78 // indirect
gopkg.in/alexcesaro/statsd.v2 v2.0.0 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
gopkg.in/jcmturner/aescts.v1 v1.0.1 // indirect
Expand Down

0 comments on commit 3ecabc4

Please sign in to comment.