From 68ceed6216fb5e0545c02c82d74324a2f2223917 Mon Sep 17 00:00:00 2001 From: Limarev Denis Date: Wed, 7 Jun 2023 18:10:15 +0500 Subject: [PATCH 1/2] add batch router config param for date customization --- router/batchrouter/handle.go | 4 +++- router/batchrouter/handle_lifecycle.go | 8 +++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/router/batchrouter/handle.go b/router/batchrouter/handle.go index bf449e7df2..7ae5b2a175 100644 --- a/router/batchrouter/handle.go +++ b/router/batchrouter/handle.go @@ -25,6 +25,7 @@ import ( "github.com/rudderlabs/rudder-go-kit/config" "github.com/rudderlabs/rudder-go-kit/logger" "github.com/rudderlabs/rudder-go-kit/stats" + backendconfig "github.com/rudderlabs/rudder-server/backend-config" "github.com/rudderlabs/rudder-server/jobsdb" "github.com/rudderlabs/rudder-server/router/batchrouter/asyncdestinationmanager" @@ -89,6 +90,7 @@ type Handle struct { warehouseServiceMaxRetryTime time.Duration transformerURL string datePrefixOverride string + customDatePrefix string // state @@ -398,7 +400,7 @@ func (brt *Handle) upload(provider string, batchJobs *BatchedJobs, isWarehouse b default: datePrefixLayout = time.Now().Format("2006-01-02") } - keyPrefixes := []string{folderName, batchJobs.Connection.Source.ID, datePrefixLayout} + keyPrefixes := []string{folderName, batchJobs.Connection.Source.ID, brt.customDatePrefix + datePrefixLayout} _, fileName := filepath.Split(gzipFilePath) var ( diff --git a/router/batchrouter/handle_lifecycle.go b/router/batchrouter/handle_lifecycle.go index a201e62562..46086cdc2c 100644 --- a/router/batchrouter/handle_lifecycle.go +++ b/router/batchrouter/handle_lifecycle.go @@ -16,6 +16,10 @@ import ( "github.com/rudderlabs/rudder-go-kit/config" "github.com/rudderlabs/rudder-go-kit/logger" "github.com/rudderlabs/rudder-go-kit/stats" + "github.com/tidwall/gjson" + "golang.org/x/exp/slices" + "golang.org/x/sync/errgroup" + backendconfig "github.com/rudderlabs/rudder-server/backend-config" "github.com/rudderlabs/rudder-server/jobsdb" "github.com/rudderlabs/rudder-server/router/batchrouter/asyncdestinationmanager" @@ -33,9 +37,6 @@ import ( "github.com/rudderlabs/rudder-server/utils/types" "github.com/rudderlabs/rudder-server/warehouse/client" warehouseutils "github.com/rudderlabs/rudder-server/warehouse/utils" - "github.com/tidwall/gjson" - "golang.org/x/exp/slices" - "golang.org/x/sync/errgroup" ) // Setup initializes the batch router @@ -108,6 +109,7 @@ func (brt *Handle) Setup( config.RegisterDurationConfigVariable(3, &brt.warehouseServiceMaxRetryTime, true, time.Hour, []string{"BatchRouter.warehouseServiceMaxRetryTime", "BatchRouter.warehouseServiceMaxRetryTimeinHr"}...) brt.transformerURL = config.GetString("DEST_TRANSFORM_URL", "http://localhost:9090") config.RegisterStringConfigVariable("", &brt.datePrefixOverride, true, "BatchRouter.datePrefixOverride") + config.RegisterStringConfigVariable("", &brt.customDatePrefix, true, "BatchRouter.customDatePrefix") ctx, cancel := context.WithCancel(context.Background()) brt.backgroundGroup, brt.backgroundCtx = errgroup.WithContext(ctx) From fefeca717d81e2794469fdb15b7565e0bffe0cdd Mon Sep 17 00:00:00 2001 From: Limarev Denis Date: Thu, 8 Jun 2023 16:23:19 +0500 Subject: [PATCH 2/2] fix imports --- router/batchrouter/handle.go | 3 +++ router/batchrouter/handle_lifecycle.go | 3 +-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/router/batchrouter/handle.go b/router/batchrouter/handle.go index 7bead7798d..bd01a5b11b 100644 --- a/router/batchrouter/handle.go +++ b/router/batchrouter/handle.go @@ -22,6 +22,9 @@ import ( "github.com/tidwall/sjson" "golang.org/x/exp/slices" + "github.com/rudderlabs/rudder-go-kit/config" + "github.com/rudderlabs/rudder-go-kit/logger" + "github.com/rudderlabs/rudder-go-kit/stats" kitsync "github.com/rudderlabs/rudder-go-kit/sync" backendconfig "github.com/rudderlabs/rudder-server/backend-config" "github.com/rudderlabs/rudder-server/jobsdb" diff --git a/router/batchrouter/handle_lifecycle.go b/router/batchrouter/handle_lifecycle.go index a68ad0ce85..cc00e6502f 100644 --- a/router/batchrouter/handle_lifecycle.go +++ b/router/batchrouter/handle_lifecycle.go @@ -13,12 +13,11 @@ import ( "time" "github.com/google/uuid" + "github.com/rudderlabs/rudder-go-kit/bytesize" "github.com/rudderlabs/rudder-go-kit/config" "github.com/rudderlabs/rudder-go-kit/logger" "github.com/rudderlabs/rudder-go-kit/stats" - kitsync "github.com/rudderlabs/rudder-go-kit/sync" - backendconfig "github.com/rudderlabs/rudder-server/backend-config" "github.com/rudderlabs/rudder-server/jobsdb" "github.com/rudderlabs/rudder-server/router/batchrouter/asyncdestinationmanager"