Enable dynamodb global-refdb

Remove zookeeper refdb and replace it with dynamodb also taking care of
creating the relevant DynamoDB tables on request.

The DynamoDB refdb is now used for both multi-site and high-availability
plugin.

Bug: Issue 15127
Change-Id: I5012c15ae2c06ecc55cc2693fc516ce34842a58e

.gitignore

@@ -5,6 +5,5 @@ gerrit/etc/*key*
 gerrit.config
 secure.config
 replication.config
-zookeeper-refdb.config
 .idea
 **/*.pem

Makefile.common

@@ -23,6 +23,7 @@ endif
 	aws s3 cp ../common-templates/cf-primary-asg.yml s3://$(TEMPLATE_BUCKET_NAME)/
 	aws s3 cp ../common-templates/cf-efs-stack.yml s3://$(TEMPLATE_BUCKET_NAME)/
 	aws s3 cp ../common-templates/cf-ecs-service-cpu-autoscaling.yml s3://$(TEMPLATE_BUCKET_NAME)/
+	aws s3 cp ../common-templates/cf-dynamodb-stack.yml s3://$(TEMPLATE_BUCKET_NAME)/
 
 set-optional-params-metrics-cloudwatch:
 ifdef METRICS_CLOUDWATCH_ENABLED
@@ -62,10 +63,6 @@ set-optional-params-multisite:
 ifdef MULTISITE_ENABLED
 		$(eval MULTISITE_OPTIONAL_PARAMS := ParameterKey=MultiSiteEnabled,ParameterValue=$(MULTISITE_ENABLED))
 		$(eval MULTISITE_OPTIONAL_PARAMS := $(MULTISITE_OPTIONAL_PARAMS) ParameterKey=MultiSiteKafkaBrokers,ParameterValue=\"$(MULTISITE_KAFKA_BROKERS)\")
-		$(eval MULTISITE_OPTIONAL_PARAMS := $(MULTISITE_OPTIONAL_PARAMS) ParameterKey=MultiSiteZookeeperConnectString,ParameterValue=$(MULTISITE_ZOOKEEPER_CONNECT_STRING))
-endif
-ifdef MULTISITE_ZOOKEEPER_ROOT_NODE
-		$(eval MULTISITE_OPTIONAL_PARAMS := $(MULTISITE_OPTIONAL_PARAMS) ParameterKey=MultiSiteZookeeperRootNode,ParameterValue=$(MULTISITE_ZOOKEEPER_ROOT_NODE))
 endif
 
 set-ldap-account-pattern:

common-templates/cf-dynamodb-stack.yml

@@ -0,0 +1,60 @@
+AWSTemplateFormatVersion: '2010-09-09'
+Description: DynamoDB tables for refs-db validation
+Parameters:
+  DynamoDBLocksTableName:
+    Description: The name of the DynamoDB locks table
+    Type: String
+  DynamoDBRefsTableName:
+    Description: The name of the DynamoDB refs table
+    Type: String
+
+Mappings:
+  DynamoDB:
+    CapacityUnits:
+      Read: 10
+      Write: 10
+
+Resources:
+  RefsDbTable:
+    Type: AWS::DynamoDB::Table
+    Properties:
+      TableName: !Ref DynamoDBRefsTableName
+      AttributeDefinitions:
+        - AttributeName: refPath
+          AttributeType: S
+      KeySchema:
+        - AttributeName: refPath
+          KeyType: HASH
+      ProvisionedThroughput:
+        ReadCapacityUnits: !FindInMap ['DynamoDB', 'CapacityUnits', 'Read']
+        WriteCapacityUnits: !FindInMap ['DynamoDB', 'CapacityUnits', 'Write']
+
+  LocksTable:
+    Type: AWS::DynamoDB::Table
+    Properties:
+      TableName: !Ref DynamoDBLocksTableName
+      AttributeDefinitions:
+        - AttributeName: lockKey
+          AttributeType: S
+        - AttributeName: lockValue
+          AttributeType: S
+      KeySchema:
+        - AttributeName: lockKey
+          KeyType: HASH
+        - AttributeName: lockValue
+          KeyType: RANGE
+      ProvisionedThroughput:
+        ReadCapacityUnits: !FindInMap ['DynamoDB', 'CapacityUnits', 'Read']
+        WriteCapacityUnits: !FindInMap ['DynamoDB', 'CapacityUnits', 'Write']
+
+Outputs:
+  RefsDbTable:
+    Description: The refs-db table
+    Value: !Ref RefsDbTable
+    Export:
+      Name: !Join [ ':', [ !Ref 'AWS::StackName', 'RefsDbTable' ] ]
+  LocksTable:
+    Description: The locks-db table
+    Value: !Ref LocksTable
+    Export:
+      Name: !Join [ ':', [ !Ref 'AWS::StackName', 'LocksTable' ] ]

common-templates/cf-gerrit-task-execution-role.yml

@@ -31,6 +31,17 @@ Resources:
                   - 'logs:PutLogEvents'
                   # Allow the ECS tasks to push metrics to CloudWatch
                   - 'cloudwatch:PutMetricData'
+                  # Allow the ECS tasks to perform CRUD actions to dynamodb table:
+                  # Used for global refs-db operations.
+                  - 'dynamodb:DeleteItem'
+                  - 'dynamodb:CreateTable'
+                  - 'dynamodb:DescribeTable'
+                  - 'dynamodb:GetItem'
+                  - 'dynamodb:ListTables'
+                  - 'dynamodb:PutItem'
+                  - 'dynamodb:Query'
+                  - 'dynamodb:Scan'
+                  - 'dynamodb:UpdateItem'
                 Resource: '*'
           - PolicyName: AmazonECSTaskSecretManagerRolePolicy
             PolicyDocument:

dual-primary/Makefile

@@ -15,7 +15,10 @@ GIT_GC_SOURCE_PATH=/mnt/efs/gerrit-shared/git
 
 SINGLE_SITE_PLUGINS=javamelody high-availability healthcheck metrics-reporter-cloudwatch
 
-MULTI_SITE_PLUGINS=$(SINGLE_SITE_PLUGINS) multi-site kafka-events websession-broker zookeeper-refdb~zk-3.5
+HA_SITE_PLUGINS=$(SINGLE_SITE_PLUGINS) aws-dynamodb-refdb
+HA_SITE_PLUGINS_LIBS_LINKS=high-availability
+
+MULTI_SITE_PLUGINS=$(SINGLE_SITE_PLUGINS) multi-site kafka-events websession-broker
 MULTI_SITE_PLUGINS_LIBS_LINKS=multi-site replication
 MULTI_SITE_MAVEN_LIBS=events-broker~$(EVENTSBROKER_LIB_VER)
 
@@ -54,7 +57,8 @@ cluster: cluster-keys set-optional-gerrit-primary-volume \
 			set-optional-params-for-replica-filesystem \
 			set-optional-params-for-replica-auto-scaling-capacity \
 			set-optional-params-for-replica-capacity-provider \
-			set-optional-network-params
+			set-optional-network-params \
+			set-optional-refs-db-params
 ifdef CLUSTER_INSTANCE_TYPE
 		$(eval CLUSTER_OPTIONAL_PARAMS := $(CLUSTER_OPTIONAL_PARAMS) ParameterKey=InstanceType,ParameterValue=$(CLUSTER_INSTANCE_TYPE))
 endif
@@ -79,6 +83,9 @@ endif
 ifdef LOAD_BALANCER_SCHEME
 		$(eval CLUSTER_OPTIONAL_PARAMS := $(CLUSTER_OPTIONAL_PARAMS) ParameterKey=PrimariesGerritLoadBalancerScheme,ParameterValue=$(LOAD_BALANCER_SCHEME))
 endif
+ifdef CREATE_REFS_DB_TABLES
+		$(eval CLUSTER_OPTIONAL_PARAMS := $(CLUSTER_OPTIONAL_PARAMS) ParameterKey=CreateRefsDBTables,ParameterValue=$(CREATE_REFS_DB_TABLES))
+endif
 
 	$(AWS_FC_COMMAND) create-stack \
 		--stack-name $(CLUSTER_STACK_NAME) \
@@ -94,12 +101,14 @@ endif
 		$(GERRIT_OPTIONAL_PRIMARY_VOLUME) \
 		$(GERRIT_OPTIONAL_PARAMS_REPLICA_FILESYSTEM) \
 		$(GERRIT_OPTIONAL_PARAMS_REPLICA_AUTO_SCALING_CAPACITY) \
-		$(GERRIT_OPTIONAL_PARAMS_REPLICA_CAPACITY_PROVIDER)
+		$(GERRIT_OPTIONAL_PARAMS_REPLICA_CAPACITY_PROVIDER) \
+		$(REFS_DB_OPTIONAL_PARAMS)
 
 service-primary-1: set-optional-params-metrics-cloudwatch set-optional-params-smtp \
 					set-optional-params-multisite set-ldap-account-pattern \
 					set-optional-gerrit-ulimits set-optional-jgit-conf \
-					set-high-availability-optional-params
+					set-high-availability-optional-params \
+					set-optional-refs-db-params
 ifdef GERRIT_PRIMARY1_INSTANCE_ID
 		$(eval PRIMARY1_SERVICE_OPTIONAL_PARAMS := $(PRIMARY1_SERVICE_OPTIONAL_PARAMS) ParameterKey=InstanceId,ParameterValue=$(GERRIT_PRIMARY1_INSTANCE_ID))
 endif
@@ -150,12 +159,14 @@ endif
 		$(LDAP_ACCOUNT_PATTERN_PARAM) \
 		$(METRICS_CW_OPTIONAL_PARAMS) \
 		$(SMTP_OPTIONAL_PARAMS) \
-		$(GERRIT_ULIMITS)
+		$(GERRIT_ULIMITS) \
+		$(REFS_DB_OPTIONAL_PARAMS)
 
 service-primary-2: set-optional-params-metrics-cloudwatch set-optional-params-smtp \
 					set-optional-params-multisite set-ldap-account-pattern \
 					set-optional-gerrit-ulimits set-optional-jgit-conf \
-					set-high-availability-optional-params
+					set-high-availability-optional-params \
+					set-optional-refs-db-params
 ifdef GERRIT_PRIMARY2_INSTANCE_ID
 		$(eval PRIMARY2_SERVICE_OPTIONAL_PARAMS := $(PRIMARY2_SERVICE_OPTIONAL_PARAMS) ParameterKey=InstanceId,ParameterValue=$(GERRIT_PRIMARY2_INSTANCE_ID))
 endif
@@ -207,7 +218,8 @@ endif
 		$(LDAP_ACCOUNT_PATTERN_PARAM) \
 		$(METRICS_CW_OPTIONAL_PARAMS) \
 		$(SMTP_OPTIONAL_PARAMS) \
-		$(GERRIT_ULIMITS)
+		$(GERRIT_ULIMITS) \
+		$(REFS_DB_OPTIONAL_PARAMS)
 
 service-replication:
 	$(AWS_FC_COMMAND) create-stack \
@@ -498,7 +510,7 @@ delete-all: delete-dns-routing wait-for-dns-routing-deletion \
 						$(optional_git_gc_targets_deletion) \
 						delete-cluster wait-for-cluster-deletion
 
-delete-all-including-retained-stack: confirm-persistent-stack-deletion delete-all delete-git-primary-persistent-stack delete-git-replica-persistent-stack delete-network-persistent-stack
+delete-all-including-retained-stack: confirm-persistent-stack-deletion delete-all delete-git-primary-persistent-stack delete-git-replica-persistent-stack delete-network-persistent-stack delete-dynamodb-persistent-stack
 
 delete-git-primary-persistent-stack:
 
@@ -521,7 +533,7 @@ gerrit-publish:
 ifeq ($(MULTISITE_ENABLED),true)
 	$(MAKE) -C ../gerrit gerrit-publish RECIPE=dual-primary PLUGINS="$(MULTI_SITE_PLUGINS)" PLUGINS_LIBS_LINKS="$(MULTI_SITE_PLUGINS_LIBS_LINKS)" MAVEN_LIBS="$(MULTI_SITE_MAVEN_LIBS)"
 else
-	$(MAKE) -C ../gerrit gerrit-publish RECIPE=dual-primary PLUGINS="$(SINGLE_SITE_PLUGINS)"
+	$(MAKE) -C ../gerrit gerrit-publish RECIPE=dual-primary PLUGINS="$(HA_SITE_PLUGINS)" PLUGINS_LIBS_LINKS="$(HA_SITE_PLUGINS_LIBS_LINKS)"
 endif
 
 haproxy-publish:
@@ -541,3 +553,28 @@ set-high-availability-optional-params:
 ifdef AUTOREINDEX_POLL_INTERVAL
 		$(eval HA_OPTIONAL_PARAMS := ParameterKey=AutoReindexPollInterval,ParameterValue=$(AUTOREINDEX_POLL_INTERVAL))
 endif
+
+set-optional-refs-db-params:
+	$(eval REFS_DB_OPTIONAL_PARAMS=)
+ifdef DYNAMODB_LOCKS_TABLE_NAME
+		$(eval REFS_DB_OPTIONAL_PARAMS := $(REFS_DB_OPTIONAL_PARAMS) ParameterKey=DynamoDBLocksTableName,ParameterValue=$(DYNAMODB_LOCKS_TABLE_NAME))
+endif
+ifdef DYNAMODB_REFS_TABLE_NAME
+		$(eval REFS_DB_OPTIONAL_PARAMS := $(REFS_DB_OPTIONAL_PARAMS) ParameterKey=DynamoDBRefsTableName,ParameterValue=$(DYNAMODB_REFS_TABLE_NAME))
+endif
+
+delete-dynamodb-persistent-stack:
+	$(eval DYNAMODB_STACK_NAME=$(shell $(AWS_FC_COMMAND) list-stacks --stack-status-filter CREATE_COMPLETE --query "StackSummaries[*].StackName" | jq -r '.[]| select(startswith("$(CLUSTER_STACK_NAME)-DynamoDBPersistentStack"))'))
+
+	$(if $(DYNAMODB_STACK_NAME), \
+		$(AWS_FC_COMMAND) delete-stack \
+			--stack-name $(DYNAMODB_STACK_NAME) \
+			--region $(AWS_REGION) && \
+		echo "*** Wait for DynamoDB stack '$(DYNAMODB_STACK_NAME)' deletion" && \
+		$(AWS_FC_COMMAND) wait stack-delete-complete \
+			--stack-name $(DYNAMODB_STACK_NAME) \
+			--region $(AWS_REGION) && \
+		echo "*** DynamoDB stack '$(DYNAMODB_STACK_NAME)' deleted" \
+		, \
+		echo "No DynamoDB stack found. Nothing to do." \
+	)

dual-primary/README.md

@@ -98,6 +98,10 @@ SUBNET2_AZ=us-east-1b
 SUBNET2_CIDR=10.0.32.0/24
 ```
 
+Note that if the refs-db dynamodb tables were created as part of the initial
+stack (`CREATE_REFS_DB_TABLES` was set to `true`), you will need to explicitly
+set it to `false` to avoid attempting to create the same tables again.
+
 3. Deploy the *green* stack:
 
 ```bash
@@ -211,6 +215,19 @@ EBS on multiple EC2 instances.
 Default: `10m`
 high-availability docs [here](https://gerrit.googlesource.com/plugins/high-availability/+/refs/heads/master/src/main/resources/Documentation/config.md)
 
+* `DYNAMODB_LOCKS_TABLE_NAME`. Optional. The name of the dynamoDB table used to
+  store distribute locking.
+  Default: `locksTable`
+  See DynamoDB lock client [here](https://github.com/awslabs/amazon-dynamodb-lock-client)
+
+* `DYNAMODB_REFS_TABLE_NAME`. Optional. The name of the dynamoDB table used to
+  store git refs and their associated sha1.
+  Default: `refsDb`
+
+* `CREATE_REFS_DB_TABLES`. Optional. Whether to create the DynamoDB refs and
+  lock tables.
+  Default: `false`
+
 ##### Shared filesystem for replicas
 
 Similarly to primary nodes, replicas share a data via an EFS filesystem which is
@@ -356,7 +373,7 @@ of gerrit specific events that are produced and consumed by the members of the m
 for more information on this.
 
 ##### Requirements
-* Kafka brokers and Zookeeper are required by this recipe and are expected to exist
+* Kafka brokers and DynamoDB are required by this recipe and are expected to exist
 and accessible with server-side TLS security enabled by the primary instances
 resulting from the deployment of this recipe.
 * Replication service must be enabled to allow syncing of Git data.
@@ -365,15 +382,9 @@ These are the parameters that can be specified to enable/disable multi-site:
 
 * `MULTISITE_ENABLED`: Optional. Whether this Gerrit is part of a multi-site
 cluster deployment. "false" by default.
-* `MULTISITE_ZOOKEEPER_CONNECT_STRING`: Required when "MULTISITE_ENABLED=true".
-Connection string to Zookeeper.
 * `MULTISITE_KAFKA_BROKERS`: Required when "MULTISITE_ENABLED=true".
 Comma separated list of Kafka broker hosts (host:port)
 to use for publishing events to the message broker.
-* `MULTISITE_ZOOKEEPER_ROOT_NODE` Optional. Root node to use in Zookeeper to
-store/retrieve information.
-Constraint: a slash-separated ('/') string not starting with a slash ('/')
-"gerrit/multi-site" by default.
 * `MULTISITE_GLOBAL_PROJECTS`: Optional. Comma separated list of patterns (see [projects.pattern](https://gerrit.googlesource.com/plugins/multi-site/+/refs/heads/stable-3.2/src/main/resources/Documentation/config.md))
 to specify which projects are available across all sites. This parametes applies to both multi-site
 and replication service remote destinations.
@@ -433,10 +444,12 @@ Note that this will *not* delete:
 * EFS stack
 * VPC and subnets (if created as part of this deployment, rather than externally
 provided)
+* Refs-DB DynamoDB stack (if created as part of this deployment, rather than
+  externally provided))
 
 Note that you can completely delete the stack, including explicitly retained
-resources such as the EFS Git filesystem, VPC and subnets, by issuing the more
-aggressive command:
+resources such as the EFS Git filesystem, VPC and subnets and DynamoDB stack by
+issuing the more aggressive command:
 
 ```
 make [AWS_REGION=a-valid-aws-region] [AWS_PREFIX=some-cluster-prefix] delete-all-including-retained-stack

dual-primary/cf-cluster.yml

@@ -163,8 +163,22 @@ Parameters:
   PrimariesGerritCertificateArn:
     Description: SSL Certificates ARN for the load balancer serving requests to the primary gerrit instances
     Type: String
+  DynamoDBLocksTableName:
+    Description: The name of the DynamoDB locks table
+    Type: String
+    Default: 'locksTable'
+  DynamoDBRefsTableName:
+    Description: The name of the DynamoDB refs table
+    Type: String
+    Default: 'refsDb'
+  CreateRefsDBTables:
+    Description: Whether to create DynamoDB tables for refs-db
+    Type: String
+    Default: false
+    AllowedValues: [true, false]
 
 Conditions:
+  ShouldCreateDynamoDBTables: !Equals [!Ref CreateRefsDBTables, "true"]
   isProvisionedThroughput: !Equals [!Ref PrimaryFileSystemThroughputMode, "provisioned"]
   CreatePrimaryEFS: !Equals [!Ref PrimaryFileSystemID, ""]
   CreateReplicaEFS: !Equals [!Ref ReplicaFileSystemID, ""]
@@ -459,6 +473,9 @@ Resources:
               - 'ecr:GetDownloadUrlForLayer'
               - 'ec2:AttachVolume'
               - 'ec2:DescribeVolumes'
+              - "dynamodb:CreateTable"
+              - "dynamodb:DeleteTable"
+              - "dynamodb:ListTables"
             Resource: '*'
 
   PrimaryGitFileSystemPermanentStack:
@@ -513,6 +530,17 @@ Resources:
         Subnet2CIDR: !Ref 'Subnet2CIDR'
         Subnet2AZProp: !Ref 'Subnet2AZProp'
 
+  DynamoDBPersistentStack:
+    Type: AWS::CloudFormation::Stack
+    DeletionPolicy: Retain
+    Condition: ShouldCreateDynamoDBTables
+    Properties:
+      TemplateURL: !Join [ '', ['https://', !Ref TemplateBucketName, '.s3.amazonaws.com/cf-dynamodb-stack.yml'] ]
+      TimeoutInMinutes: '25'
+      Parameters:
+        DynamoDBRefsTableName: !Ref DynamoDBRefsTableName
+        DynamoDBLocksTableName: !Ref DynamoDBLocksTableName
+
 Outputs:
   ClusterName:
     Description: The name of the ECS cluster

dual-primary/cf-service-primary.yml

@@ -172,16 +172,14 @@ Parameters:
     Description: Comma separated list of Kafka broker hosts (host:port) to use for publishing events to the message broker
     Type: CommaDelimitedList
     Default: ''
-  MultiSiteZookeeperConnectString:
-    Description: Connection string to Zookeeper
+  DynamoDBLocksTableName:
+    Description: The name of the DynamoDB locks table
     Type: String
-    Default: ''
-  MultiSiteZookeeperRootNode:
-    Description: Root node to use in Zookeeper to store/retrieve information
+    Default: 'locksTable'
+  DynamoDBRefsTableName:
+    Description: The name of the DynamoDB refs table
     Type: String
-    ConstraintDescription: Choose a slash-separated ('/') string not starting with a slash ('/')
-    AllowedPattern: '^[^\/].*'
-    Default: 'gerrit/multi-site'
+    Default: 'refsDb'
   AutoReindexPollInterval:
     Description: Interval between reindexing of all changes, accounts and groups.
     Type: String
@@ -334,10 +332,12 @@ Resources:
                       Value: !Ref MultiSiteEnabled
                     - Name: MULTISITE_KAFKA_BROKERS
                       Value: !Join [',', !Ref MultiSiteKafkaBrokers]
-                    - Name: MULTISITE_ZOOKEEPER_CONNECT_STRING
-                      Value: !Ref MultiSiteZookeeperConnectString
-                    - Name: MULTISITE_ZOOKEEPER_ROOT_NODE
-                      Value: !Ref MultiSiteZookeeperRootNode
+                    - Name: REFS_DB_ENABLED
+                      Value: true
+                    - Name: DYNAMODB_LOCKS_TABLE_NAME
+                      Value: !Ref DynamoDBLocksTableName
+                    - Name: DYNAMODB_REFS_TABLE_NAME
+                      Value: !Ref DynamoDBRefsTableName
                   Ulimits:
                     - Name: nofile
                       HardLimit: !Ref FileDescriptorsHardLimit