CBG-2711 add 3 node cluster option to jenkins

base/main_test_bucket_pool_config.go

@@ -54,6 +54,9 @@ const (
 
 	// wait this long when requesting a test bucket from the pool before giving up and failing the test.
 	waitForReadyBucketTimeout = time.Minute
+
+	// Creates buckets with a specific number of number of replicas
+	tbpEnvBucketNumReplicas = "SG_TEST_BUCKET_NUM_REPLICAS"
 )
 
 var tbpDefaultBucketSpec = BucketSpec{
@@ -136,6 +139,19 @@ func tbpNumBuckets() int {
 	return numBuckets
 }
 
+// tbpNumReplicasreturns the number of replicas to use in each bucket.
+func tbpNumReplicas() uint32 {
+	numReplicas := os.Getenv(tbpEnvBucketNumReplicas)
+	if numReplicas == "" {
+		return 0
+	}
+	replicas, err := strconv.Atoi(numReplicas)
+	if err != nil {
+		FatalfCtx(context.TODO(), "Couldn't parse %s: %v", tbpEnvBucketPoolSize, err)
+	}
+	return uint32(replicas)
+}
+
 // tbpNumCollectionsPerBucket returns the configured number of collections prepared in a bucket.
 func tbpNumCollectionsPerBucket() int {
 	numCollectionsPerBucket := tbpDefaultCollectionPoolSize

base/main_test_cluster.go

@@ -137,7 +137,7 @@ func (c *tbpClusterV2) insertBucket(name string, quotaMB int) error {
 			RAMQuotaMB:   uint64(quotaMB),
 			BucketType:   gocb.CouchbaseBucketType,
 			FlushEnabled: true,
-			NumReplicas:  0,
+			NumReplicas:  tbpNumReplicas(),
 		},
 	}
 

integration-test/docker-compose.yml

@@ -0,0 +1,35 @@
+# Copyright 2023-Present Couchbase, Inc.
+#
+# Use of this software is governed by the Business Source License included
+# in the file licenses/BSL-Couchbase.txt.  As of the Change Date specified
+# in that file, in accordance with the Business Source License, use of this
+# software will be governed by the Apache License, Version 2.0, included in
+# the file licenses/APL2.txt.
+
+services:
+  couchbase:
+    container_name: couchbase
+    image: "couchbase/server:${COUCHBASE_DOCKER_IMAGE_NAME:-enterprise-7.1.4}"
+    ports:
+      - 8091:8091
+      - 8092:8092
+      - 8093:8093
+      - 8094:8094
+      - 8095:8095
+      - 8096:8096
+      - 11207:11207
+      - 11210:11210
+      - 11211:11211
+      - 18091:18091
+      - 18092:18092
+      - 18093:18093
+      - 18094:18094
+    volumes:
+      - "${DOCKER_CBS_ROOT_DIR:-.}/cbs:/root"
+      - "${WORKSPACE_ROOT:-.}:/workspace"
+  couchbase-replica1:
+    container_name: couchbase-replica1
+    image: "couchbase/${COUCHBASE_DOCKER_IMAGE_NAME:-server:enterprise-7.1.4}"
+  couchbase-replica2:
+    container_name: couchbase-replica2
+    image: "couchbase/server:${COUCHBASE_DOCKER_IMAGE_NAME:-enterprise-7.1.4}"

integration-test/start_server.sh

@@ -0,0 +1,86 @@
+#!/bin/bash
+# Copyright 2023-Present Couchbase, Inc.
+#
+# Use of this software is governed by the Business Source License included
+# in the file licenses/BSL-Couchbase.txt.  As of the Change Date specified
+# in that file, in accordance with the Business Source License, use of this
+# software will be governed by the Apache License, Version 2.0, included in
+# the file licenses/APL2.txt.
+
+set -eux -o pipefail
+
+function usage() {
+    echo "Usage: $0 [-m] [-h] containername"
+}
+
+if [ $# -gt 2 ]; then
+    echo "Expected maximally two arguments"
+    exit 1
+fi
+
+while [[ $# -gt 0 ]]; do
+    key="$1"
+    case $key in
+        -m | --multi-node)
+            MULTI_NODE=true
+            shift
+            ;;
+        -h | --help)
+            echo "Usage: $0 [-m] [-h] containername"
+            exit 1
+            ;;
+        *)
+            COUCHBASE_DOCKER_IMAGE_NAME="$1"
+            shift
+            ;;
+    esac
+done
+
+WORKSPACE_ROOT="$(pwd)"
+DOCKER_CBS_ROOT_DIR="$(pwd)"
+if [ "${CBS_ROOT_DIR:-}" != "" ]; then
+    DOCKER_CBS_ROOT_DIR="${CBS_ROOT_DIR}"
+fi
+
+if [[ -n "${JENKINS_URL:-}" ]]; then
+    DOCKER_COMPOSE="docker-compose" # use docker-compose v1 for Jenkins AWS Linux 2
+else
+    DOCKER_COMPOSE="docker compose"
+fi
+cd -- "${BASH_SOURCE%/*}/"
+${DOCKER_COMPOSE} down || true
+export SG_TEST_COUCHBASE_SERVER_DOCKER_NAME=couchbase
+# Start CBS
+docker stop ${SG_TEST_COUCHBASE_SERVER_DOCKER_NAME} || true
+docker rm ${SG_TEST_COUCHBASE_SERVER_DOCKER_NAME} || true
+# --volume: Makes and mounts a CBS folder for storing a CBCollect if needed
+
+if [[ -z "${MULTI_NODE:-}" ]]; then
+    docker run -d --name ${SG_TEST_COUCHBASE_SERVER_DOCKER_NAME} --volume "${DOCKER_CBS_ROOT_DIR}/cbs:/root" --volume "${WORKSPACE_ROOT}:/workspace" -p 8091-8096:8091-8096 -p 11207:11207 -p 11210:11210 -p 11211:11211 -p 18091-18094:18091-18094 "couchbase/server:${COUCHBASE_DOCKER_IMAGE_NAME}"
+else
+    ${DOCKER_COMPOSE} up -d --force-recreate --renew-anon-volumes --remove-orphans
+fi
+
+# Test to see if Couchbase Server is up
+# Each retry min wait 5s, max 10s. Retry 20 times with exponential backoff (delay 0), fail at 120s
+curl --retry-all-errors --connect-timeout 5 --max-time 10 --retry 20 --retry-delay 0 --retry-max-time 120 'http://127.0.0.1:8091'
+
+# Set up CBS
+
+docker exec couchbase couchbase-cli cluster-init --cluster-username Administrator --cluster-password password --cluster-ramsize 3072 --cluster-index-ramsize 3072 --cluster-fts-ramsize 256 --services data,index,query
+docker exec couchbase couchbase-cli setting-index --cluster couchbase://localhost --username Administrator --password password --index-threads 4 --index-log-level verbose --index-max-rollback-points 10 --index-storage-setting default --index-memory-snapshot-interval 150 --index-stable-snapshot-interval 40000
+
+curl -u Administrator:password -v -X POST http://127.0.0.1:8091/node/controller/rename -d 'hostname=127.0.0.1'
+
+if [[ -n "${MULTI_NODE:-}" ]]; then
+    REPLICA1_NAME=couchbase-replica1
+    REPLICA2_NAME=couchbase-replica2
+    CLI_ARGS=(-c couchbase://couchbase -u Administrator -p password)
+    docker exec ${REPLICA1_NAME} couchbase-cli node-init "${CLI_ARGS[@]}"
+    docker exec ${REPLICA2_NAME} couchbase-cli node-init "${CLI_ARGS[@]}"
+    REPLICA1_IP=$(docker inspect --format '{{json .NetworkSettings.Networks}}' ${REPLICA1_NAME} | jq -r 'first(.[]) | .IPAddress')
+    REPLICA2_IP=$(docker inspect --format '{{json .NetworkSettings.Networks}}' ${REPLICA2_NAME} | jq -r 'first(.[]) | .IPAddress')
+    docker exec ${SG_TEST_COUCHBASE_SERVER_DOCKER_NAME} couchbase-cli server-add "${CLI_ARGS[@]}" --server-add "$REPLICA2_IP" --server-add-username Administrator --server-add-password password --services data,index,query
+    docker exec ${SG_TEST_COUCHBASE_SERVER_DOCKER_NAME} couchbase-cli server-add "${CLI_ARGS[@]}" --server-add "$REPLICA1_IP" --server-add-username Administrator --server-add-password password --services data,index,query
+    docker exec ${SG_TEST_COUCHBASE_SERVER_DOCKER_NAME} couchbase-cli rebalance "${CLI_ARGS[@]}"
+fi

jenkins-integration-build.sh

@@ -62,6 +62,13 @@ else
     go install -v github.com/AlekSi/gocov-xml@latest
 fi
 
+if [[ -n "${JENKINS_URL:-}" ]]; then
+    # last 1.x version, when updating aws linux 2 docker, docker-compose becomes docker compose
+    sudo yum install -y jq
+    sudo curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
+    sudo chmod +x /usr/local/bin/docker-compose
+fi
+
 if [ "${SG_TEST_X509:-}" == "true" -a "${COUCHBASE_SERVER_PROTOCOL}" != "couchbases" ]; then
     echo "Setting SG_TEST_X509 requires using couchbases:// protocol, aborting integration tests"
     exit 1
@@ -112,34 +119,15 @@ if [ "${RUN_WALRUS}" == "true" ]; then
     go test -coverprofile=coverage_walrus_ce.out -coverpkg=github.com/couchbase/sync_gateway/... $GO_TEST_FLAGS github.com/couchbase/sync_gateway/${TARGET_PACKAGE} > verbose_unit_ce.out.raw 2>&1 | true
 fi
 
-WORKSPACE_ROOT="$(pwd)"
-DOCKER_CBS_ROOT_DIR="$(pwd)"
-if [ "${CBS_ROOT_DIR:-}" != "" ]; then
-    DOCKER_CBS_ROOT_DIR="${CBS_ROOT_DIR}"
+# Run CBS
+if [[ -z ${MULTI_NODE:-} ]]; then
+    # Run CBS
+    ./integration-test/start_server.sh "${COUCHBASE_SERVER_VERSION}"
+else
+    ./integration-test/start_server.sh -m "${COUCHBASE_SERVER_VERSION}"
+    export SG_TEST_BUCKET_NUM_REPLICAS=1
 fi
 
-export SG_TEST_COUCHBASE_SERVER_DOCKER_NAME=couchbase
-# Start CBS
-docker stop ${SG_TEST_COUCHBASE_SERVER_DOCKER_NAME} || true
-docker rm ${SG_TEST_COUCHBASE_SERVER_DOCKER_NAME} || true
-# --volume: Makes and mounts a CBS folder for storing a CBCollect if needed
-docker run -d --name ${SG_TEST_COUCHBASE_SERVER_DOCKER_NAME} --volume ${DOCKER_CBS_ROOT_DIR}/cbs:/root --volume ${WORKSPACE_ROOT}:/workspace --net=host couchbase/server:${COUCHBASE_SERVER_VERSION}
-
-# Test to see if Couchbase Server is up
-# Each retry min wait 5s, max 10s. Retry 20 times with exponential backoff (delay 0), fail at 120s
-curl --retry-all-errors --connect-timeout 5 --max-time 10 --retry 20 --retry-delay 0 --retry-max-time 120 'http://127.0.0.1:8091'
-
-# Set up CBS
-curl -u Administrator:password -v -X POST http://127.0.0.1:8091/nodes/self/controller/settings -d 'path=%2Fopt%2Fcouchbase%2Fvar%2Flib%2Fcouchbase%2Fdata&' -d 'index_path=%2Fopt%2Fcouchbase%2Fvar%2Flib%2Fcouchbase%2Fdata&' -d 'cbas_path=%2Fopt%2Fcouchbase%2Fvar%2Flib%2Fcouchbase%2Fdata&' -d 'eventing_path=%2Fopt%2Fcouchbase%2Fvar%2Flib%2Fcouchbase%2Fdata&'
-curl -u Administrator:password -v -X POST http://127.0.0.1:8091/node/controller/rename -d 'hostname=127.0.0.1'
-curl -u Administrator:password -v -X POST http://127.0.0.1:8091/node/controller/setupServices -d 'services=kv%2Cn1ql%2Cindex'
-curl -u Administrator:password -v -X POST http://127.0.0.1:8091/pools/default -d 'memoryQuota=3072' -d 'indexMemoryQuota=3072' -d 'ftsMemoryQuota=256'
-curl -u Administrator:password -v -X POST http://127.0.0.1:8091/settings/web -d 'password=password&username=Administrator&port=SAME'
-curl -u Administrator:password -v -X POST http://localhost:8091/settings/indexes -d indexerThreads=4 -d logLevel=verbose -d maxRollbackPoints=10 \
-    -d storageMode=plasma -d memorySnapshotInterval=150 -d stableSnapshotInterval=40000
-
-sleep 10
-
 # Set up test environment variables for CBS runs
 export SG_TEST_USE_XATTRS=${XATTRS}
 export SG_TEST_USE_GSI=${GSI}