From 7b6fa35cd688374ed64150e511704107294a8703 Mon Sep 17 00:00:00 2001 From: Rohith BCS Date: Fri, 17 Mar 2023 14:59:23 +0530 Subject: [PATCH 1/4] chore: fix namespace bug (#3110) --- warehouse/testdata/sql/namespace_test.sql | 6 + warehouse/warehouse.go | 26 ++-- warehouse/warehouse_test.go | 164 ++++++++++++++++++++++ 3 files changed, 185 insertions(+), 11 deletions(-) create mode 100644 warehouse/testdata/sql/namespace_test.sql diff --git a/warehouse/testdata/sql/namespace_test.sql b/warehouse/testdata/sql/namespace_test.sql new file mode 100644 index 0000000000..ba45a9fada --- /dev/null +++ b/warehouse/testdata/sql/namespace_test.sql @@ -0,0 +1,6 @@ +BEGIN; +INSERT INTO wh_schemas(id,wh_upload_id, source_id, namespace, destination_id, destination_type, + schema, error, created_at, updated_at) +VALUES (1,1, 'test-sourceID', 'test-namespace', 'test-destinationID', 'POSTGRES','{}', NULL, + '2022-12-06 15:23:37.100685', '2022-12-06 15:23:37.100685'); +COMMIT; diff --git a/warehouse/warehouse.go b/warehouse/warehouse.go index 0a7feebdf2..631842b8cc 100644 --- a/warehouse/warehouse.go +++ b/warehouse/warehouse.go @@ -303,7 +303,7 @@ func (wh *HandleT) backendConfigSubscriber(ctx context.Context) { destination = wh.attachSSHTunnellingInfo(ctx, destination) } - namespace := wh.getNamespace(destination.Config, source, destination, wh.destType) + namespace := wh.getNamespace(source, destination) warehouse := model.Warehouse{ WorkspaceID: workspaceID, Source: source, @@ -400,9 +400,9 @@ func deepCopy(src, dest interface{}) error { // 1. user set name from destinationConfig // 2. from existing record in wh_schemas with same source + dest combo // 3. convert source name -func (wh *HandleT) getNamespace(configI interface{}, source backendconfig.SourceT, destination backendconfig.DestinationT, destType string) string { - configMap := configI.(map[string]interface{}) - if destType == warehouseutils.CLICKHOUSE { +func (wh *HandleT) getNamespace(source backendconfig.SourceT, destination backendconfig.DestinationT) string { + configMap := destination.Config + if wh.destType == warehouseutils.CLICKHOUSE { if _, ok := configMap["database"].(string); ok { return configMap["database"].(string) } @@ -411,18 +411,22 @@ func (wh *HandleT) getNamespace(configI interface{}, source backendconfig.Source if configMap["namespace"] != nil { namespace, _ := configMap["namespace"].(string) if len(strings.TrimSpace(namespace)) > 0 { - return warehouseutils.ToProviderCase(destType, warehouseutils.ToSafeNamespace(destType, namespace)) + return warehouseutils.ToProviderCase(wh.destType, warehouseutils.ToSafeNamespace(wh.destType, namespace)) } } // TODO: Move config to global level based on use case - namespacePrefix := config.GetString(fmt.Sprintf("Warehouse.%s.customDatasetPrefix", warehouseutils.WHDestNameMap[destType]), "") + namespacePrefix := config.GetString(fmt.Sprintf("Warehouse.%s.customDatasetPrefix", warehouseutils.WHDestNameMap[wh.destType]), "") if namespacePrefix != "" { - return warehouseutils.ToProviderCase(destType, warehouseutils.ToSafeNamespace(destType, fmt.Sprintf(`%s_%s`, namespacePrefix, source.Name))) + return warehouseutils.ToProviderCase(wh.destType, warehouseutils.ToSafeNamespace(wh.destType, fmt.Sprintf(`%s_%s`, namespacePrefix, source.Name))) } - if _, exists := warehouseutils.GetNamespace(source, destination, wh.dbHandle); !exists { - return warehouseutils.ToProviderCase(destType, warehouseutils.ToSafeNamespace(destType, source.Name)) + var ( + namespace string + exists bool + ) + if namespace, exists = warehouseutils.GetNamespace(source, destination, wh.dbHandle); !exists { + return warehouseutils.ToProviderCase(wh.destType, warehouseutils.ToSafeNamespace(wh.destType, source.Name)) } - return "" + return namespace } func (wh *HandleT) setDestInProgress(warehouse model.Warehouse, jobID int64) { @@ -968,7 +972,7 @@ func minimalConfigSubscriber() { dbHandle: dbHandle, destType: destination.DestinationDefinition.Name, } - namespace := wh.getNamespace(destination.Config, source, destination, wh.destType) + namespace := wh.getNamespace(source, destination) connectionsMapLock.Lock() if connectionsMap[destination.ID] == nil { connectionsMap[destination.ID] = map[string]model.Warehouse{} diff --git a/warehouse/warehouse_test.go b/warehouse/warehouse_test.go index cd683b4494..44440931f9 100644 --- a/warehouse/warehouse_test.go +++ b/warehouse/warehouse_test.go @@ -7,6 +7,7 @@ import ( "testing" "time" + backendconfig "github.com/rudderlabs/rudder-server/config/backend-config" postgreslegacy "github.com/rudderlabs/rudder-server/warehouse/integrations/postgres-legacy" "github.com/rudderlabs/rudder-server/warehouse/integrations/postgres" @@ -187,3 +188,166 @@ func TestUploadJob_ProcessingStats(t *testing.T) { }) } } + +func Test_GetNamespace(t *testing.T) { + testcases := []struct { + config map[string]interface{} + source backendconfig.SourceT + destination backendconfig.DestinationT + destType string + result string + setConfig bool + }{ + { + source: backendconfig.SourceT{}, + destination: backendconfig.DestinationT{ + Config: map[string]interface{}{ + "database": "test_db", + }, + }, + destType: warehouseutils.CLICKHOUSE, + result: "test_db", + setConfig: false, + }, + { + source: backendconfig.SourceT{}, + destination: backendconfig.DestinationT{ + Config: map[string]interface{}{}, + }, + destType: warehouseutils.CLICKHOUSE, + result: "rudder", + setConfig: false, + }, + { + source: backendconfig.SourceT{}, + destination: backendconfig.DestinationT{ + Config: map[string]interface{}{ + "namespace": "test_namespace", + }, + }, + destType: "test-destinationType-1", + result: "test_namespace", + setConfig: false, + }, + { + source: backendconfig.SourceT{}, + destination: backendconfig.DestinationT{ + Config: map[string]interface{}{ + "namespace": " test_namespace ", + }, + }, + destType: "test-destinationType-1", + result: "test_namespace", + setConfig: false, + }, + { + source: backendconfig.SourceT{}, + destination: backendconfig.DestinationT{ + Config: map[string]interface{}{ + "namespace": "##", + }, + }, + destType: "test-destinationType-1", + result: "stringempty", + setConfig: false, + }, + { + source: backendconfig.SourceT{}, + destination: backendconfig.DestinationT{ + Config: map[string]interface{}{ + "namespace": "##evrnvrv$vtr&^", + }, + }, + destType: "test-destinationType-1", + result: "evrnvrv_vtr", + setConfig: false, + }, + { + source: backendconfig.SourceT{}, + destination: backendconfig.DestinationT{ + Config: map[string]interface{}{}, + }, + destType: "test-destinationType-1", + result: "config_result", + setConfig: true, + }, + { + source: backendconfig.SourceT{ + Name: "test-source", + ID: "test-sourceID", + }, + destination: backendconfig.DestinationT{ + Config: map[string]interface{}{}, + ID: "test-destinationID", + }, + destType: "test-destinationType-1", + result: "test-namespace", + setConfig: false, + }, + { + source: backendconfig.SourceT{ + Name: "test-source", + ID: "random-sourceID", + }, + destination: backendconfig.DestinationT{ + Config: map[string]interface{}{}, + ID: "random-destinationID", + }, + destType: "test-destinationType-1", + result: "test_source", + setConfig: false, + }, + { + source: backendconfig.SourceT{ + Name: "test-source", + ID: "test-sourceID", + }, + destination: backendconfig.DestinationT{ + Config: map[string]interface{}{}, + ID: "test-destinationID", + }, + destType: "test-destinationType-1", + result: "config_result_test_source", + setConfig: true, + }, + } + + for _, tc := range testcases { + tc := tc + t.Run("should return namespace", func(t *testing.T) { + t.Parallel() + + pool, err := dockertest.NewPool("") + require.NoError(t, err) + + pgResource, err := destination.SetupPostgres(pool, t) + require.NoError(t, err) + + err = (&migrator.Migrator{ + Handle: pgResource.DB, + MigrationsTable: "wh_schema_migrations", + }).Migrate("warehouse") + + require.NoError(t, err) + store := memstats.New() + wh := HandleT{ + destType: tc.destType, + stats: store, + dbHandle: pgResource.DB, + } + if tc.setConfig { + config.Set(fmt.Sprintf("Warehouse.%s.customDatasetPrefix", warehouseutils.WHDestNameMap[tc.destType]), "config_result") + } + + sqlStatement, err := os.ReadFile("testdata/sql/namespace_test.sql") + require.NoError(t, err) + + _, err = pgResource.DB.Exec(string(sqlStatement)) + require.NoError(t, err) + + namespace := wh.getNamespace(tc.source, tc.destination) + require.Equal(t, tc.result, namespace) + config.Reset() + }) + } +} From 2abecaa6020d033e2ac8dd0a57c6badeb40e57bc Mon Sep 17 00:00:00 2001 From: Akash Chetty Date: Fri, 17 Mar 2023 22:17:45 +0530 Subject: [PATCH 2/4] fix: minio error while creating bucket if already exists (#3109) --- services/filemanager/miniomanager.go | 11 +++++++---- warehouse/validations/validate_test.go | 2 +- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/services/filemanager/miniomanager.go b/services/filemanager/miniomanager.go index 2d8df3d250..c2cc5734e4 100644 --- a/services/filemanager/miniomanager.go +++ b/services/filemanager/miniomanager.go @@ -35,10 +35,13 @@ func (manager *MinioManager) Upload(ctx context.Context, file *os.File, prefixes ctx, cancel := context.WithTimeout(ctx, manager.getTimeout()) defer cancel() - if err = minioClient.MakeBucket(ctx, manager.Config.Bucket, minio.MakeBucketOptions{Region: "us-east-1"}); err != nil { - exists, errBucketExists := minioClient.BucketExists(ctx, manager.Config.Bucket) - if !(errBucketExists == nil && exists) { - return UploadOutput{}, err + exists, err := minioClient.BucketExists(ctx, manager.Config.Bucket) + if err != nil { + return UploadOutput{}, fmt.Errorf("checking bucket: %w", err) + } + if !exists { + if err = minioClient.MakeBucket(ctx, manager.Config.Bucket, minio.MakeBucketOptions{Region: "us-east-1"}); err != nil { + return UploadOutput{}, fmt.Errorf("creating bucket: %w", err) } } diff --git a/warehouse/validations/validate_test.go b/warehouse/validations/validate_test.go index 84f39035bd..5cdbb6cfe9 100644 --- a/warehouse/validations/validate_test.go +++ b/warehouse/validations/validate_test.go @@ -476,7 +476,7 @@ func TestValidator(t *testing.T) { "accessKeyID": "temp-access-key", "secretAccessKey": "test-secret-key", }, - wantError: errors.New("upload file: uploading file: The Access Key Id you provided does not exist in our records."), + wantError: errors.New("upload file: uploading file: checking bucket: The Access Key Id you provided does not exist in our records."), }, { name: "no privilege", From dd626b310fe0e90c1a01eb811ddf4cacc8d2900f Mon Sep 17 00:00:00 2001 From: Akash Chetty Date: Fri, 17 Mar 2023 22:38:33 +0530 Subject: [PATCH 3/4] fix(warehouse): snowflakes user identifies table getting skipped (#3113) --- warehouse/integrations/docker-compose.test.yml | 1 + warehouse/upload.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/warehouse/integrations/docker-compose.test.yml b/warehouse/integrations/docker-compose.test.yml index d08493032b..e55e48bb40 100644 --- a/warehouse/integrations/docker-compose.test.yml +++ b/warehouse/integrations/docker-compose.test.yml @@ -242,6 +242,7 @@ services: - BIGQUERY_INTEGRATION_TEST_CREDENTIALS - REDSHIFT_INTEGRATION_TEST_CREDENTIALS - SNOWFLAKE_INTEGRATION_TEST_CREDENTIALS + - SNOWFLAKE_RBAC_INTEGRATION_TEST_CREDENTIALS - DATABRICKS_INTEGRATION_TEST_CREDENTIALS - BIGQUERY_INTEGRATION_TEST_SCHEMA - REDSHIFT_INTEGRATION_TEST_SCHEMA diff --git a/warehouse/upload.go b/warehouse/upload.go index e0bb9f3bed..395287ec7e 100644 --- a/warehouse/upload.go +++ b/warehouse/upload.go @@ -1192,7 +1192,7 @@ func (job *UploadJob) loadUserTables(loadFilesTableMap map[tableNameT]bool) ([]e } // Skip loading user tables if identifies table schema is not present - if identifiesSchema := job.GetTableSchemaInUpload(warehouseutils.IdentifiesTable); len(identifiesSchema) == 0 { + if identifiesSchema := job.GetTableSchemaInUpload(job.identifiesTableName()); len(identifiesSchema) == 0 { return []error{}, nil } From 722ca9ed1eed87730d957c049937d6a86e26d96b Mon Sep 17 00:00:00 2001 From: "Rudder Bot (server team)" <103839104+rudder-server-bot@users.noreply.github.com> Date: Mon, 20 Mar 2023 12:23:48 +0200 Subject: [PATCH 4/4] chore: release 1.7.0 (#3106) --- CHANGELOG.md | 69 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 79b2b57b54..a6b68eb4c7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,74 @@ # Changelog +## [1.7.0](https://github.com/rudderlabs/rudder-server/compare/v1.6.0...v1.7.0) (2023-03-17) + + +### Features + +* default retention period set to 7 days for rudder backups ([#3038](https://github.com/rudderlabs/rudder-server/issues/3038)) ([0d9af35](https://github.com/rudderlabs/rudder-server/commit/0d9af35aa8fcc690b4c8601ae8f59bb86fcae9ec)) +* **gateway:** support new event type extract ([#2999](https://github.com/rudderlabs/rudder-server/issues/2999)) ([63dc940](https://github.com/rudderlabs/rudder-server/commit/63dc9402b3e263eeb97c62e8902a0d68718b9556)) +* kafka over ssh ([#3007](https://github.com/rudderlabs/rudder-server/issues/3007)) ([99262c3](https://github.com/rudderlabs/rudder-server/commit/99262c3736d5ab3b03e5b0ad7c582d73e168f7a7)) +* **warehouse:** added duplicates stats for snowflake ([#3097](https://github.com/rudderlabs/rudder-server/issues/3097)) ([0eeaeb5](https://github.com/rudderlabs/rudder-server/commit/0eeaeb5287d0481427df7bfbb13bdbc1538ed87a)) +* **warehouse:** snowflake roles support. ([#3031](https://github.com/rudderlabs/rudder-server/issues/3031)) ([98a413c](https://github.com/rudderlabs/rudder-server/commit/98a413c5df1e3011b2c774d3ea4d9054eab44a72)) +* **warehouse:** temp table support for postgres ([#2964](https://github.com/rudderlabs/rudder-server/issues/2964)) ([9a80f45](https://github.com/rudderlabs/rudder-server/commit/9a80f459f5289df3d99d25fc7b09ed1804bd6522)) + + +### Bug Fixes + +* enabled data race ([#3041](https://github.com/rudderlabs/rudder-server/issues/3041)) ([7136be4](https://github.com/rudderlabs/rudder-server/commit/7136be40e7cf02886c0f99053ff32c0c6996e14e)) +* inconsistent and leaky retry delay logic in router ([#3002](https://github.com/rudderlabs/rudder-server/issues/3002)) ([20c8644](https://github.com/rudderlabs/rudder-server/commit/20c8644298e2d4cc3338b8aa5d2f7d9cd717ba62)) +* kafka create topic ([#3102](https://github.com/rudderlabs/rudder-server/issues/3102)) ([10ccbf3](https://github.com/rudderlabs/rudder-server/commit/10ccbf332ed661a501a47afe6f7687d78fc05cff)) +* limiter deadlock while trying to notify a dynamic priority item ([#3056](https://github.com/rudderlabs/rudder-server/issues/3056)) ([ade2e4d](https://github.com/rudderlabs/rudder-server/commit/ade2e4d456ec42262b037ee3839f1823561d2bee)) +* minio error while creating bucket if already exists ([#3109](https://github.com/rudderlabs/rudder-server/issues/3109)) ([2abecaa](https://github.com/rudderlabs/rudder-server/commit/2abecaa6020d033e2ac8dd0a57c6badeb40e57bc)) +* payload limiter gauge reports invalid value type ([#3048](https://github.com/rudderlabs/rudder-server/issues/3048)) ([828f70d](https://github.com/rudderlabs/rudder-server/commit/828f70d271832ff56a463d9618c9a02dd1f408c7)) +* pending events all ([#3075](https://github.com/rudderlabs/rudder-server/issues/3075)) ([1d9f63b](https://github.com/rudderlabs/rudder-server/commit/1d9f63b3df72a038a6b1f4f5d402a04ab00befa5)) +* **router:** retry backoff causing out-of-order job processing ([#3098](https://github.com/rudderlabs/rudder-server/issues/3098)) ([eda4525](https://github.com/rudderlabs/rudder-server/commit/eda4525edaa6f3af6ae05bd41a11ddb6d94ab202)) +* user/anonymous id read at gateway ([#3051](https://github.com/rudderlabs/rudder-server/issues/3051)) ([828f70d](https://github.com/rudderlabs/rudder-server/commit/828f70d271832ff56a463d9618c9a02dd1f408c7)) +* **warehouse:** added support for filtering on the uploads and calculating aborted events for task_run_id ([#2975](https://github.com/rudderlabs/rudder-server/issues/2975)) ([8ab58b8](https://github.com/rudderlabs/rudder-server/commit/8ab58b80914a602d786e9fb37747770f97f325f0)) +* **warehouse:** grouping uploads for processing pickup ([#3039](https://github.com/rudderlabs/rudder-server/issues/3039)) ([4832630](https://github.com/rudderlabs/rudder-server/commit/4832630e58db4aeaaaf5c65c634247ad5da8e12a)) +* **warehouse:** infinites retries ([#3050](https://github.com/rudderlabs/rudder-server/issues/3050)) ([d3320fa](https://github.com/rudderlabs/rudder-server/commit/d3320fa6f407d65368cbbf32278fcff5927aa7f9)) +* **warehouse:** snowflake default timestamp to timestamp with time zone ([#3100](https://github.com/rudderlabs/rudder-server/issues/3100)) ([cef4a18](https://github.com/rudderlabs/rudder-server/commit/cef4a18578332a16455cde4127731c6ea0bf7ba7)) +* **warehouse:** snowflakes user identifies table getting skipped ([#3113](https://github.com/rudderlabs/rudder-server/issues/3113)) ([dd626b3](https://github.com/rudderlabs/rudder-server/commit/dd626b310fe0e90c1a01eb811ddf4cacc8d2900f)) + + +### Miscellaneous + +* abort job after 5 attempts at deletion-worker ([#3012](https://github.com/rudderlabs/rudder-server/issues/3012)) ([ed83eaa](https://github.com/rudderlabs/rudder-server/commit/ed83eaac2f56a7825a7ddd0e830bab0f21b13713)) +* add live events stats ([#2930](https://github.com/rudderlabs/rudder-server/issues/2930)) ([7ca1b0b](https://github.com/rudderlabs/rudder-server/commit/7ca1b0bb03bbb8499d87bcc84b7c1919f9ac69f7)) +* add transformations challenge announcement ([#3072](https://github.com/rudderlabs/rudder-server/issues/3072)) ([f128b93](https://github.com/rudderlabs/rudder-server/commit/f128b931047dfb1bef0ec50d32994ab3a3028f3f)) +* badgerDB improvements for debugger ([#3101](https://github.com/rudderlabs/rudder-server/issues/3101)) ([0c525d0](https://github.com/rudderlabs/rudder-server/commit/0c525d0ba9ef85decfeba3b5be5aba6a2d2dd446)) +* debugger's badgerdb cache optimisations ([#3042](https://github.com/rudderlabs/rudder-server/issues/3042)) ([e6e9933](https://github.com/rudderlabs/rudder-server/commit/e6e99337b218f844bb893371246b54a3b7f710b8)) +* **deps:** bump github.com/minio/minio-go/v7 from 7.0.48 to 7.0.49 ([#3018](https://github.com/rudderlabs/rudder-server/issues/3018)) ([f4ea7b3](https://github.com/rudderlabs/rudder-server/commit/f4ea7b31c0d9b8da1b8f0f71e14d10c390ae3ae2)) +* **deps:** bump github.com/onsi/ginkgo/v2 from 2.1.6 to 2.9.0 ([#3068](https://github.com/rudderlabs/rudder-server/issues/3068)) ([6bdbb7a](https://github.com/rudderlabs/rudder-server/commit/6bdbb7aa3d79d06bce27a230d32ee4af2bc05b89)) +* **deps:** bump github.com/prometheus/common from 0.37.0 to 0.41.0 ([#3062](https://github.com/rudderlabs/rudder-server/issues/3062)) ([bb04a8b](https://github.com/rudderlabs/rudder-server/commit/bb04a8b21af714ed7ed03d4769db6fa92b781b67)) +* **deps:** bump github.com/urfave/cli/v2 from 2.20.3 to 2.25.0 ([#3067](https://github.com/rudderlabs/rudder-server/issues/3067)) ([6b429b7](https://github.com/rudderlabs/rudder-server/commit/6b429b7baa9795668960b07b2a73e56cad8fb7f9)) +* **deps:** bump go.uber.org/goleak from 1.2.0 to 1.2.1 ([#3017](https://github.com/rudderlabs/rudder-server/issues/3017)) ([2eb92ca](https://github.com/rudderlabs/rudder-server/commit/2eb92ca6d89738b33830e4d31dbcb82baf31744a)) +* don't include prereleases in docker latest tag ([#3006](https://github.com/rudderlabs/rudder-server/issues/3006)) ([20c8644](https://github.com/rudderlabs/rudder-server/commit/20c8644298e2d4cc3338b8aa5d2f7d9cd717ba62)) +* enable pipeline level sync frequency ([#3094](https://github.com/rudderlabs/rudder-server/issues/3094)) ([ea3bbd5](https://github.com/rudderlabs/rudder-server/commit/ea3bbd502d7ed5f6f9be0e08e1a9af389589831a)) +* fix namespace bug ([#3110](https://github.com/rudderlabs/rudder-server/issues/3110)) ([7b6fa35](https://github.com/rudderlabs/rudder-server/commit/7b6fa35cd688374ed64150e511704107294a8703)) +* improve regulation-worker status capture ([#2837](https://github.com/rudderlabs/rudder-server/issues/2837)) ([6f1d07d](https://github.com/rudderlabs/rudder-server/commit/6f1d07df234a2a2c32cf0ecaac465a08ddfca2bc)) +* increase parallel loads ([#3073](https://github.com/rudderlabs/rudder-server/issues/3073)) ([7dcc756](https://github.com/rudderlabs/rudder-server/commit/7dcc756f7fc39a7b66a69f0df42feb3957ba821c)) +* **jobsdb:** omit workspaceId tag when it doesn't correspond to an actual workspace ([#3057](https://github.com/rudderlabs/rudder-server/issues/3057)) ([f936260](https://github.com/rudderlabs/rudder-server/commit/f936260589a1058025ca67ea5dda6f7425e25157)) +* migrate stats to otel ([#2989](https://github.com/rudderlabs/rudder-server/issues/2989)) ([a4243de](https://github.com/rudderlabs/rudder-server/commit/a4243de1d52dff56e2e1619b00d4ba9a6291f560)) +* perform rss/wss-aware cgroups memory usage calculation ([#3052](https://github.com/rudderlabs/rudder-server/issues/3052)) ([1b6af85](https://github.com/rudderlabs/rudder-server/commit/1b6af85e6410184d1f84fd57c6a76f7e30a5c97f)) +* prefer using lo.BufferWithTimeout ([#2998](https://github.com/rudderlabs/rudder-server/issues/2998)) ([20c8644](https://github.com/rudderlabs/rudder-server/commit/20c8644298e2d4cc3338b8aa5d2f7d9cd717ba62)) +* reduce parquet file size datalake ([#3035](https://github.com/rudderlabs/rudder-server/issues/3035)) ([4cb5907](https://github.com/rudderlabs/rudder-server/commit/4cb59075ab6430a3b4f898bb349e343d40d85944)) +* remove workerID tag ([#3055](https://github.com/rudderlabs/rudder-server/issues/3055)) ([b732919](https://github.com/rudderlabs/rudder-server/commit/b7329199fc9e1db4097c6d21376deec8effb2310)) +* upgrade all dependencies ([#2996](https://github.com/rudderlabs/rudder-server/issues/2996)) ([20c8644](https://github.com/rudderlabs/rudder-server/commit/20c8644298e2d4cc3338b8aa5d2f7d9cd717ba62)) +* use gcra rate limiter gateway ([#3086](https://github.com/rudderlabs/rudder-server/issues/3086)) ([61d9275](https://github.com/rudderlabs/rudder-server/commit/61d927500b8262e06d92d08c24bb2310ca4b3cef)) +* use official bitnami images for arm64 ([#3047](https://github.com/rudderlabs/rudder-server/issues/3047)) ([3aeb4f6](https://github.com/rudderlabs/rudder-server/commit/3aeb4f60139b99938476ecbdb1b786caf1f7c41c)) +* use token on protoc setup to avoid rate-limit ([#3083](https://github.com/rudderlabs/rudder-server/issues/3083)) ([0f89b26](https://github.com/rudderlabs/rudder-server/commit/0f89b265f524b04f9f0445e3502b02ac25420ccc)) +* use upload_id for staging files ([#3066](https://github.com/rudderlabs/rudder-server/issues/3066)) ([3ec2433](https://github.com/rudderlabs/rudder-server/commit/3ec2433b671428a7f5504af8e9cb85b2f4373c51)) +* **warehouse:** added support for observability for loading tables and made dedup optional for Redshift ([#3037](https://github.com/rudderlabs/rudder-server/issues/3037)) ([63fd288](https://github.com/rudderlabs/rudder-server/commit/63fd28852af9278ebaa7e9c55ee59a1449c139f1)) +* **warehouse:** added warehouse handling for s3 with glue and other improvements ([#2940](https://github.com/rudderlabs/rudder-server/issues/2940)) ([3495797](https://github.com/rudderlabs/rudder-server/commit/34957976512f658f5165d296671ee4adfc24d321)) +* **warehouse:** additional error mappings ([#2994](https://github.com/rudderlabs/rudder-server/issues/2994)) ([20c8644](https://github.com/rudderlabs/rudder-server/commit/20c8644298e2d4cc3338b8aa5d2f7d9cd717ba62)) +* **warehouse:** allow empty properties schema for event models ([#3095](https://github.com/rudderlabs/rudder-server/issues/3095)) ([b9deb4a](https://github.com/rudderlabs/rudder-server/commit/b9deb4abe620ca80bf64ce5b2b0b278d4e36d654)) +* **warehouse:** default warehouse priority set to 100 ([#3026](https://github.com/rudderlabs/rudder-server/issues/3026)) ([20c8644](https://github.com/rudderlabs/rudder-server/commit/20c8644298e2d4cc3338b8aa5d2f7d9cd717ba62)) +* **warehouse:** encoding package with readers, loaders, writers ([#3077](https://github.com/rudderlabs/rudder-server/issues/3077)) ([06c0a71](https://github.com/rudderlabs/rudder-server/commit/06c0a7179ff1f65a72ab6e518aaa3487c21ce9a5)) +* **warehouse:** naming conventions ([#3076](https://github.com/rudderlabs/rudder-server/issues/3076)) ([f2e99c7](https://github.com/rudderlabs/rudder-server/commit/f2e99c7f4b56e1d595722133fc479df505e96029)) +* **warehouse:** use first_event_at while pickup for warehouse processing jobs ([#3036](https://github.com/rudderlabs/rudder-server/issues/3036)) ([7aeed3b](https://github.com/rudderlabs/rudder-server/commit/7aeed3bfcf9f3679eabfb9f6689f49ba04c62ad9)) +* **warehouse:** warehouse integration tests improvements ([#3059](https://github.com/rudderlabs/rudder-server/issues/3059)) ([e57362e](https://github.com/rudderlabs/rudder-server/commit/e57362e514a2731a7f9419467fc626efdc3d316f)) + ## [1.6.3](https://github.com/rudderlabs/rudder-server/compare/v1.6.2...v1.6.3) (2023-03-01)