Development environment for multi cluster setup (#2848)

Makefile

@@ -374,38 +374,52 @@ install-schema-es:
 	curl -X PUT "http://127.0.0.1:9200/temporal_visibility_v1_dev" --write-out "\n"
 
 install-schema-cdc: temporal-cassandra-tool
-	@printf $(COLOR)  "Set up temporal_active key space..."
+	@printf $(COLOR)  "Install Cassandra schema (active)..."
 	./temporal-cassandra-tool drop -k temporal_active -f
 	./temporal-cassandra-tool create -k temporal_active --rf 1
 	./temporal-cassandra-tool -k temporal_active setup-schema -v 0.0
 	./temporal-cassandra-tool -k temporal_active update-schema -d ./schema/cassandra/temporal/versioned
-	./temporal-cassandra-tool drop -k temporal_visibility_active -f
-	./temporal-cassandra-tool create -k temporal_visibility_active --rf 1
-	./temporal-cassandra-tool -k temporal_visibility_active setup-schema -v 0.0
-	./temporal-cassandra-tool -k temporal_visibility_active update-schema -d ./schema/cassandra/visibility/versioned
 
-	@printf $(COLOR) "Set up temporal_standby key space..."
+	@printf $(COLOR)  "Install Cassandra schema (standby)..."
 	./temporal-cassandra-tool drop -k temporal_standby -f
 	./temporal-cassandra-tool create -k temporal_standby --rf 1
 	./temporal-cassandra-tool -k temporal_standby setup-schema -v 0.0
 	./temporal-cassandra-tool -k temporal_standby update-schema -d ./schema/cassandra/temporal/versioned
-	./temporal-cassandra-tool drop -k temporal_visibility_standby -f
-	./temporal-cassandra-tool create -k temporal_visibility_standby --rf 1
-	./temporal-cassandra-tool -k temporal_visibility_standby setup-schema -v 0.0
-	./temporal-cassandra-tool -k temporal_visibility_standby update-schema -d ./schema/cassandra/visibility/versioned
+
+	@printf $(COLOR)  "Install Cassandra schema (other)..."
+	./temporal-cassandra-tool drop -k temporal_other -f
+	./temporal-cassandra-tool create -k temporal_other --rf 1
+	./temporal-cassandra-tool -k temporal_other setup-schema -v 0.0
+	./temporal-cassandra-tool -k temporal_other update-schema -d ./schema/cassandra/temporal/versioned
+
+	@printf $(COLOR) "Install Elasticsearch schemas..."
+	curl --fail -X PUT "http://127.0.0.1:9200/_cluster/settings" -H "Content-Type: application/json" --data-binary @./schema/elasticsearch/visibility/cluster_settings_v7.json --write-out "\n"
+	curl --fail -X PUT "http://127.0.0.1:9200/_template/temporal_visibility_v1_template" -H "Content-Type: application/json" --data-binary @./schema/elasticsearch/visibility/index_template_v7.json --write-out "\n"
+# No --fail here because create index is not idempotent operation.
+	curl -X PUT "http://127.0.0.1:9200/temporal_visibility_v1_dev_active" --write-out "\n"
+	curl -X PUT "http://127.0.0.1:9200/temporal_visibility_v1_dev_standby" --write-out "\n"
+	curl -X PUT "http://127.0.0.1:9200/temporal_visibility_v1_dev_other" --write-out "\n"
 
 ##### Run server #####
+DOCKER_COMPOSE_FILES     := -f ./develop/docker-compose/docker-compose.yml -f ./develop/docker-compose/docker-compose.$(GOOS).yml
+DOCKER_COMPOSE_CDC_FILES := -f ./develop/docker-compose/docker-compose.cdc.yml -f ./develop/docker-compose/docker-compose.cdc.$(GOOS).yml
 start-dependencies:
-	docker-compose -f ./develop/docker-compose/docker-compose.yml -f ./develop/docker-compose/docker-compose.$(GOOS).yml up
+	docker-compose $(DOCKER_COMPOSE_FILES) up
 
 stop-dependencies:
-	docker-compose -f ./develop/docker-compose/docker-compose.yml -f ./develop/docker-compose/docker-compose.$(GOOS).yml down
+	docker-compose $(DOCKER_COMPOSE_FILES) down
+
+start-dependencies-cdc:
+	docker-compose $(DOCKER_COMPOSE_FILES) $(DOCKER_COMPOSE_CDC_FILES) up
+
+stop-dependencies-cdc:
+	docker-compose $(DOCKER_COMPOSE_FILES) $(DOCKER_COMPOSE_CDC_FILES) down
 
 start: temporal-server
-	./temporal-server start
+	./temporal-server --zone cass start
 
 start-es: temporal-server
-	./temporal-server --zone es start
+	./temporal-server --zone cass-es start
 
 start-mysql: temporal-server
 	./temporal-server --zone mysql start

common/config/config_test.go

@@ -32,7 +32,7 @@ import (
 
 func TestToString(t *testing.T) {
 	var cfg Config
-	err := Load("", "../../config", "", &cfg)
+	err := Load("", "../../config", "cass", &cfg)
 	assert.NoError(t, err)
 	assert.NotEmpty(t, cfg.String())
 }

config/base.yaml

@@ -1,3 +1,3 @@
 log:
   stdout: true
-  level: info
+  level: info

config/development_active.yaml

@@ -1,16 +1,24 @@
 persistence:
   defaultStore: cass-default
-  visibilityStore: cass-visibility
+  visibilityStore:
+  advancedVisibilityStore: es-visibility
   numHistoryShards: 4
   datastores:
     cass-default:
       cassandra:
         hosts: "127.0.0.1"
         keyspace: "temporal_active"
-    cass-visibility:
-      cassandra:
-        hosts: "127.0.0.1"
-        keyspace: "temporal_visibility_active"
+    es-visibility:
+      elasticsearch:
+        version: "v7"
+        logLevel: "error"
+        url:
+          scheme: "http"
+          host: "127.0.0.1:9200"
+        indices:
+          visibility: temporal_visibility_v1_dev_active
+          # secondary_visibility: temporal_visibility_v2_dev
+        closeIdleConnectionsInterval: 15s
 global:
   membership:
     maxJoinDuration: 30s
@@ -19,6 +27,9 @@ global:
     port: 7936
   metrics:
     prometheus:
+#      # specify framework to use new approach for initializing metrics and/or use opentelemetry
+#      framework: "opentelemetry"
+      framework: "tally"
       timerType: "histogram"
       listenAddress: "127.0.0.1:8000"
 
@@ -83,6 +94,8 @@ archival:
       filestore:
         fileMode: "0666"
         dirMode: "0766"
+      gstorage:
+        credentialsPath: "/tmp/gcloud/keyfile.json"
   visibility:
     state: "enabled"
     enableRead: true
@@ -102,3 +115,7 @@ namespaceDefaults:
 
 publicClient:
   hostPort: "localhost:7233"
+
+dynamicConfigClient:
+  filepath: "config/dynamicconfig/development_es.yaml"
+  pollInterval: "10s"

config/development_archival.yaml → config/development_cass-archival.yaml

@@ -19,6 +19,9 @@ global:
     port: 7936
   metrics:
     prometheus:
+#      # specify framework to use new approach for initializing metrics and/or use opentelemetry
+#      framework: "opentelemetry"
+      framework: "tally"
       timerType: "histogram"
       listenAddress: "127.0.0.1:8000"
 
@@ -94,6 +97,5 @@ publicClient:
   hostPort: "localhost:7233"
 
 dynamicConfigClient:
-  filepath: "./config/dynamicconfig/development.yaml"
-  pollInterval: "10s"
-
+  filepath: "./config/dynamicconfig/development_cass.yaml"
+  pollInterval: "10s"

config/development_es.yaml → config/development_cass-es.yaml

@@ -22,10 +22,14 @@ persistence:
 global:
   membership:
     maxJoinDuration: 30s
+    broadcastAddress: "127.0.0.1"
   pprof:
-      port: 7936
+    port: 7936
   metrics:
     prometheus:
+#      # specify framework to use new approach for initializing metrics and/or use opentelemetry
+#      framework: "opentelemetry"
+      framework: "tally"
       timerType: "histogram"
       listenAddress: "127.0.0.1:8000"
 
@@ -78,6 +82,8 @@ archival:
       filestore:
         fileMode: "0666"
         dirMode: "0766"
+      gstorage:
+        credentialsPath: "/tmp/gcloud/keyfile.json"
   visibility:
     state: "enabled"
     enableRead: true
@@ -89,16 +95,15 @@ archival:
 namespaceDefaults:
   archival:
     history:
-      state: "enabled"
+      state: "disabled"
       URI: "file:///tmp/temporal_archival/development"
     visibility:
-      state: "enabled"
+      state: "disabled"
       URI: "file:///tmp/temporal_vis_archival/development"
 
 publicClient:
   hostPort: "localhost:7233"
 
 dynamicConfigClient:
   filepath: "config/dynamicconfig/development_es.yaml"
-  pollInterval: "10s"
-
+  pollInterval: "10s"

config/development.yaml → config/development_cass.yaml

@@ -121,6 +121,5 @@ publicClient:
   hostPort: "localhost:7233"
 
 dynamicConfigClient:
-  filepath: "./config/dynamicconfig/development.yaml"
-  pollInterval: "10s"
-
+  filepath: "./config/dynamicconfig/development_cass.yaml"
+  pollInterval: "10s"

config/development_mysql-es.yaml

@@ -28,10 +28,14 @@ persistence:
 global:
   membership:
     maxJoinDuration: 30s
+    broadcastAddress: "127.0.0.1"
   pprof:
-      port: 7936
+    port: 7936
   metrics:
     prometheus:
+#      # specify framework to use new approach for initializing metrics and/or use opentelemetry
+#      framework: "opentelemetry"
+      framework: "tally"
       timerType: "histogram"
       listenAddress: "127.0.0.1:8000"
 
@@ -84,6 +88,8 @@ archival:
       filestore:
         fileMode: "0666"
         dirMode: "0766"
+      gstorage:
+        credentialsPath: "/tmp/gcloud/keyfile.json"
   visibility:
     state: "enabled"
     enableRead: true
@@ -95,16 +101,15 @@ archival:
 namespaceDefaults:
   archival:
     history:
-      state: "enabled"
+      state: "disabled"
       URI: "file:///tmp/temporal_archival/development"
     visibility:
-      state: "enabled"
+      state: "disabled"
       URI: "file:///tmp/temporal_vis_archival/development"
 
 publicClient:
   hostPort: "localhost:7233"
 
 dynamicConfigClient:
   filepath: "config/dynamicconfig/development_sql_es.yaml"
-  pollInterval: "10s"
-
+  pollInterval: "10s"

config/development_mysql.yaml

@@ -29,10 +29,14 @@ persistence:
 global:
   membership:
     maxJoinDuration: 30s
+    broadcastAddress: "127.0.0.1"
   pprof:
     port: 7936
   metrics:
     prometheus:
+#      # specify framework to use new approach for initializing metrics and/or use opentelemetry
+#      framework: "opentelemetry"
+      framework: "tally"
       timerType: "histogram"
       listenAddress: "127.0.0.1:8000"
 
@@ -85,6 +89,8 @@ archival:
       filestore:
         fileMode: "0666"
         dirMode: "0766"
+      gstorage:
+        credentialsPath: "/tmp/gcloud/keyfile.json"
   visibility:
     state: "enabled"
     enableRead: true
@@ -103,9 +109,8 @@ namespaceDefaults:
       URI: "file:///tmp/temporal_vis_archival/development"
 
 publicClient:
-  hostPort: "127.0.0.1:7233"
+  hostPort: "localhost:7233"
 
 dynamicConfigClient:
   filepath: "config/dynamicconfig/development_sql.yaml"
-  pollInterval: "10s"
-
+  pollInterval: "10s"

config/development_other.yaml

@@ -1,16 +1,24 @@
 persistence:
   defaultStore: cass-default
-  visibilityStore: cass-visibility
+  visibilityStore:
+  advancedVisibilityStore: es-visibility
   numHistoryShards: 4
   datastores:
     cass-default:
       cassandra:
         hosts: "127.0.0.1"
         keyspace: "temporal_other"
-    cass-visibility:
-      cassandra:
-        hosts: "127.0.0.1"
-        keyspace: "temporal_visibility_other"
+    es-visibility:
+      elasticsearch:
+        version: "v7"
+        logLevel: "error"
+        url:
+          scheme: "http"
+          host: "127.0.0.1:9200"
+        indices:
+          visibility: temporal_visibility_v1_dev_other
+          # secondary_visibility: temporal_visibility_v2_dev
+        closeIdleConnectionsInterval: 15s
 
 global:
   membership:
@@ -19,6 +27,9 @@ global:
     port: 9936
   metrics:
     prometheus:
+#      # specify framework to use new approach for initializing metrics and/or use opentelemetry
+#      framework: "opentelemetry"
+      framework: "tally"
       timerType: "histogram"
       listenAddress: "127.0.0.1:8000"
 
@@ -53,6 +64,11 @@ clusterMetadata:
   masterClusterName: "active"
   currentClusterName: "other"
   clusterInformation:
+    active:
+      enabled: true
+      initialFailoverVersion: 1
+      rpcName: "frontend"
+      rpcAddress: "localhost:7233"
     other:
       enabled: true
       initialFailoverVersion: 3
@@ -83,6 +99,8 @@ archival:
       filestore:
         fileMode: "0666"
         dirMode: "0766"
+      gstorage:
+        credentialsPath: "/tmp/gcloud/keyfile.json"
   visibility:
     state: "enabled"
     enableRead: true
@@ -102,3 +120,7 @@ namespaceDefaults:
 
 publicClient:
   hostPort: "localhost:9233"
+
+dynamicConfigClient:
+  filepath: "config/dynamicconfig/development_es.yaml"
+  pollInterval: "10s"