Integration test

go.mod

@@ -31,11 +31,13 @@ require (
 	github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect
 	github.com/fsnotify/fsnotify v1.4.7
 	github.com/garyburd/redigo v1.6.0 // indirect
+	github.com/go-ini/ini v1.63.2 // indirect
 	github.com/go-redis/redis v6.15.7+incompatible
 	github.com/gofrs/uuid v3.2.0+incompatible // indirect
 	github.com/golang-migrate/migrate/v4 v4.11.0
 	github.com/golang/mock v1.5.0
 	github.com/golang/protobuf v1.5.0
+	github.com/gomodule/redigo v1.8.5
 	github.com/gorilla/mux v1.7.4
 	github.com/gotestyourself/gotestyourself v2.2.0+incompatible // indirect
 	github.com/hashicorp/go-retryablehttp v0.6.6
@@ -47,6 +49,7 @@ require (
 	github.com/kr/text v0.2.0 // indirect
 	github.com/lib/pq v1.3.0
 	github.com/mattn/go-ieproxy v0.0.0-20200203040449-2dbc853185d9 // indirect
+	github.com/minio/minio-go v6.0.14+incompatible
 	github.com/minio/minio-go/v6 v6.0.49
 	github.com/mkmik/multierror v0.3.0
 	github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect
@@ -83,6 +86,7 @@ require (
 	go.uber.org/zap v1.14.0
 	golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d // indirect
 	golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5
+	golang.org/x/sync v0.0.0-20201207232520-09787c993a3a
 	golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e // indirect
 	golang.org/x/time v0.0.0-20191024005414-555d28b269f0
 	google.golang.org/api v0.39.0

go.sum

@@ -215,6 +215,8 @@ github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeME
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
+github.com/go-ini/ini v1.63.2 h1:kwN3umicd2HF3Tgvap4um1ZG52/WyKT9GGdPx0CJk6Y=
+github.com/go-ini/ini v1.63.2/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
@@ -276,6 +278,8 @@ github.com/golang/snappy v0.0.0-20170215233205-553a64147049/go.mod h1:/XxbfmMg8l
 github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
 github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/gomodule/redigo v1.8.5 h1:nRAxCa+SVsyjSBrtZmG/cqb6VbTmuRzpg/PoTFlpumc=
+github.com/gomodule/redigo v1.8.5/go.mod h1:P9dn9mFrCBvWhGE1wpxx6fgq7BAeLBk+UUUzlpkBYO0=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
@@ -443,6 +447,8 @@ github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd
 github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
 github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/minio/minio-go v6.0.14+incompatible h1:fnV+GD28LeqdN6vT2XdGKW8Qe/IfjJDswNVuni6km9o=
+github.com/minio/minio-go v6.0.14+incompatible/go.mod h1:7guKYtitv8dktvNUGrhzmNlA5wrAABTQXCoesZdFQO8=
 github.com/minio/minio-go/v6 v6.0.49 h1:bU4kIa/qChTLC1jrWZ8F+8gOiw1MClubddAJVR4gW3w=
 github.com/minio/minio-go/v6 v6.0.49/go.mod h1:qD0lajrGW49lKZLtXKtCB4X/qkMf0a5tBvN2PaZg7Gg=
 github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU=

testdata/workspaceConfigTemplate.json

@@ -101,14 +101,14 @@
                         "password": "password",
                         "port": "{{.postgresPort}}",
                         "sslMode": "disable",
-                        "namespace": "example",
-                        "bucketProvider": "S3",
-                        "bucketName": "priyamrudderservertest",
-                        "accessKeyID": "",
-                        "secretAccessKey": "",
+                        "namespace": "",
+                        "bucketProvider": "MINIO",
+                        "bucketName": "{{.minioBucketName}}",
+                        "accessKeyID": "MYACCESSKEY",
+                        "secretAccessKey": "MYSECRETKEY",
                         "useSSL": false,
-                        "endPoint": "",
-                        "syncFrequency": "1",
+                        "endPoint": "{{.minioEndpoint}}",
+                        "syncFrequency": "30",
                         "useRudderStorage": false
                     },
                     "secretConfig": {},
@@ -470,6 +470,152 @@
                     },
                     "isConnectionEnabled": true,
                     "isProcessorEnabled": true
+                },
+                {
+                    "config": {
+                        "address": "{{.address}}",
+                        "database": "",
+                        "password": "",
+                        "prefix": "",
+                        "secure": false,
+                        "skipVerify": false,
+                        "caCertificate": "",
+                        "clusterMode": false
+                    },
+                    "secretConfig": {},
+                    "id": "xxxyyyzzYjAZfPBa5yqqTvqBVqu",
+                    "name": "Des Redis Integration Test 1",
+                    "enabled": true,
+                    "workspaceId": "{{.workspaceId}}",
+                    "deleted": false,
+                    "createdAt": "2021-08-27T06:45:46.149Z",
+                    "updatedAt": "2021-08-27T06:45:46.149Z",
+                    "transformations": [],
+                    "destinationDefinition": {
+                        "config": {
+                            "destConfig": {
+                                "defaultConfig": [
+                                    "address",
+                                    "database",
+                                    "password",
+                                    "prefix",
+                                    "secure",
+                                    "skipVerify",
+                                    "caCertificate",
+                                    "clusterMode"
+                                ]
+                            },
+                            "secretKeys": [
+                                "password",
+                                "caCertificate"
+                            ],
+                            "excludeKeys": [],
+                            "includeKeys": [],
+                            "transformAt": "processor",
+                            "transformAtV1": "processor",
+                            "supportedSourceTypes": [
+                                "android",
+                                "ios",
+                                "web",
+                                "unity",
+                                "amp",
+                                "cloud",
+                                "warehouse",
+                                "reactnative",
+                                "flutter"
+                            ],
+                            "supportedMessageTypes": [
+                                "identify"
+                            ],
+                            "saveDestinationResponse": true
+                        },
+                        "configSchema": null,
+                        "responseRules": {},
+                        "id": "xxxyyyzzl28zT3nxr8SWZp6xaWT",
+                        "name": "REDIS",
+                        "displayName": "Redis",
+                        "category": null,
+                        "createdAt": "2021-02-08T15:26:17.995Z",
+                        "updatedAt": "2021-08-27T08:09:46.976Z"
+                    },
+                    "isConnectionEnabled": true,
+                    "isProcessorEnabled": true
+                },
+                {
+                    "config": {
+                        "hostName": "localhost",
+                        "port": "{{.kafkaPort}}",
+                        "topic": "dev_integration_test",
+                        "sslEnabled": false,
+                        "caCertificate": "key",
+                        "useSASL": false,
+                        "saslType": "plain",
+                        "username": "",
+                        "password": ""
+                    },
+                    "secretConfig": {},
+                    "id": "xxxyyyzzhyrw8v0CrTMrDZ4ovej",
+                    "name": "Des Kafka Integration Test 1",
+                    "enabled": true,
+                    "workspaceId": "{{.workspaceId}}",
+                    "deleted": false,
+                    "createdAt": "2021-08-27T06:44:37.836Z",
+                    "updatedAt": "2021-08-27T09:09:11.873Z",
+                    "transformations": [],
+                    "destinationDefinition": {
+                        "config": {
+                            "destConfig": {
+                                "defaultConfig": [
+                                    "hostName",
+                                    "port",
+                                    "topic",
+                                    "sslEnabled",
+                                    "caCertificate",
+                                    "useSASL",
+                                    "saslType",
+                                    "username",
+                                    "password"
+                                ]
+                            },
+                            "secretKeys": [
+                                "password"
+                            ],
+                            "excludeKeys": [],
+                            "includeKeys": [],
+                            "transformAt": "processor",
+                            "transformAtV1": "processor",
+                            "supportedSourceTypes": [
+                                "android",
+                                "ios",
+                                "web",
+                                "unity",
+                                "amp",
+                                "cloud",
+                                "warehouse",
+                                "reactnative",
+                                "flutter"
+                            ],
+                            "supportedMessageTypes": [
+                                "alias",
+                                "group",
+                                "identify",
+                                "page",
+                                "screen",
+                                "track"
+                            ],
+                            "saveDestinationResponse": true
+                        },
+                        "configSchema": null,
+                        "responseRules": null,
+                        "id": "xxxyyyzzsyAyCQ2QIxrKU9X5AoR",
+                        "name": "KAFKA",
+                        "displayName": "Apache Kafka",
+                        "category": null,
+                        "createdAt": "2020-05-12T17:13:54.165Z",
+                        "updatedAt": "2021-08-27T08:09:44.485Z"
+                    },
+                    "isConnectionEnabled": true,
+                    "isProcessorEnabled": true
                 }
             ],
             "sourceDefinition":