fix: error message for deltalake during test connection (#3883)

rudderlabs · Sep 19, 2023 · 1fa2f45 · 1fa2f45
1 parent e188b94
commit 1fa2f45
Show file tree

Hide file tree

Showing 5 changed files with 20 additions and 16 deletions.
diff --git a/warehouse/integrations/clickhouse/clickhouse_test.go b/warehouse/integrations/clickhouse/clickhouse_test.go
@@ -630,6 +630,7 @@ func TestClickhouse_TestConnection(t *testing.T) {
 	workspaceID := "test_workspace_id"
 	namespace := "test_namespace"
 	provider := "MINIO"
+	timeout := 5 * time.Second
 
 	dsn := fmt.Sprintf("tcp://%s:%d?compress=false&database=%s&password=%s&secure=false&skip_verify=true&username=%s",
 		"localhost", clickhousePort, databaseName, password, user,
@@ -653,16 +654,16 @@ func TestClickhouse_TestConnection(t *testing.T) {
 		},
 		{
 			name:    "Success",
-			timeout: warehouseutils.TestConnectionTimeout,
+			timeout: timeout,
 		},
 		{
 			name:     "TLS config",
-			timeout:  warehouseutils.TestConnectionTimeout,
+			timeout:  timeout,
 			tlConfig: "test-tls-config",
 		},
 		{
 			name:      "No such host",
-			timeout:   warehouseutils.TestConnectionTimeout,
+			timeout:   timeout,
 			wantError: errors.New(`dial tcp: lookup clickhouse`),
 			host:      "clickhouse",
 		},

diff --git a/warehouse/integrations/deltalake/deltalake.go b/warehouse/integrations/deltalake/deltalake.go
@@ -1203,7 +1203,7 @@ func (*Deltalake) IsEmpty(context.Context, model.Warehouse) (bool, error) {
 func (d *Deltalake) TestConnection(ctx context.Context, _ model.Warehouse) error {
 	err := d.DB.PingContext(ctx)
 	if errors.Is(err, context.DeadlineExceeded) {
-		return fmt.Errorf("connection timeout: %w", err)
+		return errors.New("connection timeout: verify the availability of the SQL warehouse/cluster on Databricks (this process may take up to 15 minutes). Once the SQL warehouse/cluster is ready, please attempt your connection again")
 	}
 	if err != nil {
 		return fmt.Errorf("pinging: %w", err)

diff --git a/warehouse/utils/utils.go b/warehouse/utils/utils.go
@@ -184,10 +184,9 @@ var DiscardsSchema = map[string]string{
 }
 
 const (
-	LoadFileTypeCsv       = "csv"
-	LoadFileTypeJson      = "json"
-	LoadFileTypeParquet   = "parquet"
-	TestConnectionTimeout = 15 * time.Second
+	LoadFileTypeCsv     = "csv"
+	LoadFileTypeJson    = "json"
+	LoadFileTypeParquet = "parquet"
 )
 
 func Init() {

diff --git a/warehouse/validations/validate.go b/warehouse/validations/validate.go
@@ -249,7 +249,7 @@ func (os *objectStorage) Validate(ctx context.Context) error {
 func (c *connections) Validate(ctx context.Context) error {
 	defer c.manager.Cleanup(ctx)
 
-	ctx, cancel := context.WithTimeout(ctx, warehouseutils.TestConnectionTimeout)
+	ctx, cancel := context.WithTimeout(ctx, queryTimeout)
 	defer cancel()
 
 	return c.manager.TestConnection(ctx, createDummyWarehouse(c.destination))
@@ -467,7 +467,7 @@ func createFileManager(dest *backendconfig.DestinationT) (filemanager.FileManage
 		return nil, fmt.Errorf("creating file manager: %w", err)
 	}
 
-	fileManager.SetTimeout(objectStorageValidationTimeout)
+	fileManager.SetTimeout(objectStorageTimeout)
 
 	return fileManager, nil
 }
@@ -485,7 +485,7 @@ func createManager(ctx context.Context, dest *backendconfig.DestinationT) (manag
 		return nil, fmt.Errorf("getting manager: %w", err)
 	}
 
-	operations.SetConnectionTimeout(warehouseutils.TestConnectionTimeout)
+	operations.SetConnectionTimeout(queryTimeout)
 
 	if err = operations.Setup(ctx, warehouse, &dummyUploader{
 		dest: dest,

diff --git a/warehouse/validations/validations.go b/warehouse/validations/validations.go
@@ -22,10 +22,11 @@ const (
 )
 
 var (
-	connectionTestingFolder        string
-	pkgLogger                      logger.Logger
-	fileManagerFactory             filemanager.Factory
-	objectStorageValidationTimeout time.Duration
+	connectionTestingFolder string
+	pkgLogger               logger.Logger
+	fileManagerFactory      filemanager.Factory
+	objectStorageTimeout    time.Duration
+	queryTimeout            time.Duration
 )
 
 var (
@@ -50,7 +51,10 @@ func Init() {
 	connectionTestingFolder = config.GetString("RUDDER_CONNECTION_TESTING_BUCKET_FOLDER_NAME", misc.RudderTestPayload)
 	pkgLogger = logger.NewLogger().Child("warehouse").Child("validations")
 	fileManagerFactory = filemanager.New
-	objectStorageValidationTimeout = 15 * time.Second
+	objectStorageTimeout = config.GetDuration("Warehouse.Validations.ObjectStorageTimeout", 15, time.Second)
+
+	// Since we have a cp-router default timeout of 30 seconds, keeping the query timeout to 25 seconds
+	queryTimeout = config.GetDuration("Warehouse.Validations.QueryTimeout", 25, time.Second)
 }
 
 // Validate the destination by running all the validation steps