Support Travis-ci Continuous Integration testing

.travis.yml

@@ -0,0 +1,72 @@
+language: python
+python:
+  - "2.7"
+cache:
+  pip: true
+  directories: 
+    - $TRAVIS_BUILD_DIR/docker
+    - $TRAVIS_BUILD_DIR/tmp/pex
+    - $TRAVIS_BUILD_DIR/tmp/pip
+services:
+  - docker
+install: 'pip install virtualenv'
+jobs:
+  include:
+#    - stage: flake8
+#      script:
+#        - pip install flake8
+#        - make flake8
+    - stage: build
+      script:
+        - make docker
+        - docker run --rm -it mongodb_consistent_backup:latest --version
+        - mkdir -p $TRAVIS_BUILD_DIR/docker
+        - docker save mongodb_consistent_backup:latest >$TRAVIS_BUILD_DIR/docker/mongodb_consistent_backup.tar
+    - stage: test-cluster-3.4
+      script:
+        - docker load -i $TRAVIS_BUILD_DIR/docker/mongodb_consistent_backup.tar
+        - $TRAVIS_BUILD_DIR/scripts/travis-ci/run-cluster.sh 3.4
+    - stage: test-replset-3.4
+      script:
+        - docker load -i $TRAVIS_BUILD_DIR/docker/mongodb_consistent_backup.tar
+        - $TRAVIS_BUILD_DIR/scripts/travis-ci/run-replset.sh 3.4
+    - stage: test-cluster-3.2
+      script:
+        - docker load -i $TRAVIS_BUILD_DIR/docker/mongodb_consistent_backup.tar
+        - $TRAVIS_BUILD_DIR/scripts/travis-ci/run-cluster.sh 3.2
+#    - stage: test-cluster-3.2-sccc
+#      script:
+#        - docker load -i $TRAVIS_BUILD_DIR/docker/mongodb_consistent_backup.tar
+#        - $TRAVIS_BUILD_DIR/scripts/travis-ci/run-cluster.sh 3.2
+    - stage: test-replset-3.2
+      script:
+        - docker load -i $TRAVIS_BUILD_DIR/docker/mongodb_consistent_backup.tar
+        - $TRAVIS_BUILD_DIR/scripts/travis-ci/run-replset.sh 3.2
+#    - stage: test-cluster-3.0
+#      script:
+#        - docker load -i $TRAVIS_BUILD_DIR/docker/mongodb_consistent_backup.tar
+#        - $TRAVIS_BUILD_DIR/scripts/travis-ci/run-cluster.sh 3.0 SCCC
+    - stage: test-replset-3.0
+      script:
+        - docker load -i $TRAVIS_BUILD_DIR/docker/mongodb_consistent_backup.tar
+        - $TRAVIS_BUILD_DIR/scripts/travis-ci/run-replset.sh 3.0
+    - stage: test-archive-none
+      script:
+        - docker load -i $TRAVIS_BUILD_DIR/docker/mongodb_consistent_backup.tar
+        - $TRAVIS_BUILD_DIR/scripts/travis-ci/run-replset.sh 3.4 --archive.method=none
+    - stage: test-archive-zbackup
+      script:
+        - docker load -i $TRAVIS_BUILD_DIR/docker/mongodb_consistent_backup.tar
+        - $TRAVIS_BUILD_DIR/scripts/travis-ci/run-replset.sh 3.4 --archive.method=zbackup
+#    - stage: test-upload-gs
+#      script:
+#        - docker load -i $TRAVIS_BUILD_DIR/docker/mongodb_consistent_backup.tar
+#        - $TRAVIS_BUILD_DIR/scripts/travis-ci/run-replset.sh 3.4 --upload.method=gs
+#    - stage: test-upload-rsync
+#      script:
+#        - docker load -i $TRAVIS_BUILD_DIR/docker/mongodb_consistent_backup.tar
+#        - $TRAVIS_BUILD_DIR/scripts/travis-ci/run-replset.sh 3.4 --upload.method=rsync
+#    - stage: test-upload-s3
+#      script:
+#        - docker load -i $TRAVIS_BUILD_DIR/docker/mongodb_consistent_backup.tar
+#        - $TRAVIS_BUILD_DIR/scripts/travis-ci/run-replset.sh 3.4 --upload.method=s3

Dockerfile

@@ -2,8 +2,8 @@ FROM centos:centos7
 ARG RELEASE
 MAINTAINER Tim Vaillancourt <tim.vaillancourt@percona.com>
 RUN yum install -y https://www.percona.com/redir/downloads/percona-release/redhat/latest/percona-release-0.1-4.noarch.rpm epel-release && \
-	yum install -y Percona-Server-MongoDB-34-tools zbackup && yum clean all && \
-	curl -Lo /usr/bin/mongodb-consistent-backup https://github.com/Percona-Lab/mongodb_consistent_backup/releases/download/$RELEASE/mongodb-consistent-backup.el7.centos.x86_64 && \
+	yum install -y Percona-Server-MongoDB-34-tools zbackup && yum clean all
+RUN curl -Lo /usr/bin/mongodb-consistent-backup https://github.com/Percona-Lab/mongodb_consistent_backup/releases/download/$RELEASE/mongodb-consistent-backup.el7.centos.x86_64 && \
 	chmod +x /usr/bin/mongodb-consistent-backup
 ENTRYPOINT ["mongodb-consistent-backup"]
 CMD ["--help"]

Makefile

@@ -22,6 +22,10 @@ install: bin/mongodb-consistent-backup
 	install -m 0644 LICENSE $(SHAREDIR)/$(NAME)/LICENSE
 	install -m 0644 README.rst $(SHAREDIR)/$(NAME)/README.rst
 
+flake8:
+	# Ignore long-lines and space-aligned = and : for now
+	flake8 --ignore E221,E241,E501 $(PWD)/$(NAME)
+
 uninstall:
 	rm -f $(BINDIR)/mongodb-consistent-backup
 	rm -rf $(SHAREDIR)/$(NAME)
@@ -36,6 +40,7 @@ rpm: bin/mongodb-consistent-backup
 
 docker: bin/mongodb-consistent-backup
 	docker build --no-cache --tag $(DOCKER_TAG) --build-arg "RELEASE=$(VERSION)" .
+	docker tag $(DOCKER_TAG) $(NAME):latest
 
 clean:
 	rm -rf bin build $(NAME).egg-info tmp 2>/dev/null

README.rst

@@ -1,6 +1,12 @@
 MongoDB Consistent Backup Tool - mongodb-consistent-backup
 ----------------------------------------------------------
 
+.. image:: https://github-release-version.herokuapp.com/github/Percona-Lab/mongodb_consistent_backup/release.svg?style=flat
+    :target: https://github.com/Percona-Lab/mongodb_consistent_backup/releases/latest
+
+.. image:: https://travis-ci.org/Percona-Lab/mongodb_consistent_backup.svg?branch=master
+    :target: https://travis-ci.org/Percona-Lab/mongodb_consistent_backup
+
 About
 ~~~~~
 

requirements.txt

@@ -7,3 +7,4 @@ boto==2.47.0
 filechunkio==1.8
 python-dateutil==2.2
 yconf==0.3.4
+google_compute_engine

scripts/travis-ci/docker-compose.yml

@@ -0,0 +1,44 @@
+version: '2'
+services:
+  mongo-rs0-1:
+    image: "percona/percona-server-mongodb:${MONGO_VERSION}"
+    command: mongod --port=27017 --replSet=rs0 --dbpath=/data/db ${DATA_MONGOD_FLAGS}
+  mongo-rs0-2:
+    image: "percona/percona-server-mongodb:${MONGO_VERSION}"
+    command: mongod --port=27017 --replSet=rs0 --dbpath=/data/db ${DATA_MONGOD_FLAGS}
+  mongo-rs0-3:
+    image: "percona/percona-server-mongodb:${MONGO_VERSION}"
+    command: mongod --port=27017 --replSet=rs0 --dbpath=/data/db ${DATA_MONGOD_FLAGS}
+  mongo-cs-1:
+    image: "percona/percona-server-mongodb:${MONGO_VERSION}"
+    command: mongod --port=27017 --configsvr --dbpath=/data/db ${CONFIGSVR_FLAGS}
+  mongo-cs-2:
+    image: "percona/percona-server-mongodb:${MONGO_VERSION}"
+    command: mongod --port=27017 --configsvr --dbpath=/data/db ${CONFIGSVR_FLAGS}
+  mongo-cs-3:
+    image: "percona/percona-server-mongodb:${MONGO_VERSION}"
+    command: mongod --port=27017 --configsvr --dbpath=/data/db ${CONFIGSVR_FLAGS}
+  mongo-mongos:
+    image: "percona/percona-server-mongodb:${MONGO_VERSION}"
+    command: mongos --port=27017 --configdb=${MONGOS_CONFIGDB}
+    depends_on:
+    - mongo-rs0-1
+    - mongo-rs0-2
+    - mongo-rs0-3
+    - mongo-cs-1
+    - mongo-cs-2
+    - mongo-cs-3
+  backup-cluster:
+    image: mongodb_consistent_backup:latest
+    entrypoint: mongodb-consistent-backup
+    command: --backup.location=/tmp --backup.name=test-cluster --host=${MONGOS_CONFIGDB} --replication.max_lag_secs=15 ${MCB_EXTRA}
+    depends_on:
+    - mongo-mongos
+  backup-replset:
+    image: mongodb_consistent_backup:latest
+    entrypoint: mongodb-consistent-backup
+    command: --backup.location=/tmp --backup.name=test-replset --host=rs0/mongo-rs0-1:27017,mongo-rs0-2:27017,mongo-rs0-3:27017 --replication.max_lag_secs=15 ${MCB_EXTRA}
+    depends_on:
+    - mongo-rs0-1
+    - mongo-rs0-2
+    - mongo-rs0-3

scripts/travis-ci/func.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+function doMongo() {
+	set +x
+	set +e
+	local service=$1
+	local host=$2
+	local cmd=$3
+	local tries=0
+	while [ $tries -le 5 ]; do
+  	  bash -x -c "docker-compose run --rm ${service} /usr/bin/mongo ${host} --quiet --eval '${cmd}'"
+	  [ $? = 0 ] && break
+	  echo "# Retrying mongo command to ${host}"
+	  tries=$(($tries + 1))
+	  sleep 3
+	done
+	set -e
+	set -x
+}

scripts/travis-ci/run-cluster.sh

@@ -0,0 +1,77 @@
+#!/bin/bash
+
+set -e
+set -x
+
+function print_usage() {
+	echo "Usage $0: [MONGO_VERSION] [CONFIGSVR_TYPE (CSRS or SCCC)] [mongodb-consistent-backup EXTRA FLAGS...]"
+}
+
+MONGO_VERSION=${1:-3.2}
+CONFIGSVR_TYPE=${2:-CSRS}
+MCB_EXTRA="${@:3}"
+
+pushd $(dirname $0)
+	source $PWD/func.sh
+
+	export MONGO_VERSION=${MONGO_VERSION}
+	export DATA_MONGOD_FLAGS="--shardsvr"
+	export MCB_EXTRA=${MCB_EXTRA}
+
+	CONFIGSVR_REPLSET=csReplSet
+	if [ "${CONFIGSVR_TYPE}" == "CSRS" ]; then
+		export CONFIGSVR_FLAGS="--replSet=${CONFIGSVR_REPLSET}"
+		export MONGOS_CONFIGDB="${CONFIGSVR_REPLSET}/mongo-cs-1:27017,mongo-cs-2:27017,mongo-cs-3:27017"
+		echo "# Using CSRS-based config servers: '${MONGOS_CONFIGDB}'"
+	elif [ "${CONFIGSVR_TYPE}" == "SCCC" ]; then
+		export CONFIGSVR_FLAGS=
+		export MONGOS_CONFIGDB="mongo-cs-1:27017,mongo-cs-2:27017"
+		echo "# Using SCCC-based config servers: '${MONGOS_CONFIGDB}'"
+	else
+		echo "Unsupported CONFIGSVR_TYPE field: '${CONFIGSVR_TYPE}'! Supported: CSRS (default) or SCCC"
+		print_usage
+		exit 1
+	fi
+
+	# start mongo-mongos service (which starts the whole cluster)
+	echo "# Starting instances with docker-compose"
+	docker-compose up -d mongo-mongos
+
+	echo "# Waiting 10 seconds"
+	sleep 10
+
+	if [ "${CONFIGSVR_TYPE}" == "CSRS" ]; then
+		echo "# Initiating csReplSet (config server replica set)"
+		doMongo mongo-mongos mongo-cs-1:27017 'rs.initiate({
+		  _id: "csReplSet",
+		  configsvr: true,
+		  members: [
+		    { _id: 0, host: "mongo-cs-1:27017" },
+		    { _id: 1, host: "mongo-cs-2:27017" },
+		    { _id: 2, host: "mongo-cs-3:27017" }
+		  ]
+		})'
+	fi
+
+	echo "# Initiating rs0"
+	doMongo mongo-mongos mongo-rs0-1:27017 'rs.initiate({
+	  _id: "rs0",
+	  members: [
+	    { _id: 0, host: "mongo-rs0-1:27017" },
+	    { _id: 1, host: "mongo-rs0-2:27017" },
+	    { _id: 2, host: "mongo-rs0-3:27017", priority: 0 }
+	  ]
+	})'
+
+	echo "# Waiting 10 seconds"
+	sleep 10
+
+	echo "# Adding shard rs0"
+  	doMongo mongo-mongos mongo-mongos:27017 'sh.addShard("rs0/mongo-rs0-1:27017,mongo-rs0-2:27017,mongo-rs0-3:27017")'
+
+        echo "# Starting mongodb_consistent_backup, cluster mode (in docker)"
+        docker-compose up --abort-on-container-exit backup-cluster
+
+        echo "# Stopping instances with docker-compose"
+        docker-compose down
+popd

scripts/travis-ci/run-replset.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+
+set -e
+set -x
+
+MONGO_VERSION=${1:-3.2}
+MCB_EXTRA="${@:2}"
+
+pushd $(dirname $0)
+	source $PWD/func.sh
+
+	export MONGO_VERSION=${MONGO_VERSION}
+	export DATA_MONGOD_FLAGS=
+	export MONGOS_CONFIGDB=
+	export CONFIGSVR_FLAGS=
+	export MCB_EXTRA=${MCB_EXTRA}
+
+	echo "# Starting instances with docker-compose"
+	docker-compose up -d mongo-rs0-1
+	docker-compose up -d mongo-rs0-2
+	docker-compose up -d mongo-rs0-3
+
+	echo "# Waiting 10 seconds"
+	sleep 10
+
+	echo "# Initiating rs0"
+	doMongo mongo-rs0-1 mongo-rs0-1:27017 'rs.initiate({
+	  _id: "rs0",
+	  members: [
+	    { _id: 0, host: "mongo-rs0-1:27017" },
+	    { _id: 1, host: "mongo-rs0-2:27017" },
+	    { _id: 2, host: "mongo-rs0-3:27017", priority: 0 }
+	  ]
+	})'
+
+	echo "# Waiting 10 seconds"
+	sleep 10
+
+        echo "# Starting mongodb_consistent_backup, replset-only mode (in docker)"
+        docker-compose up --abort-on-container-exit backup-replset
+
+        echo "# Stopping instances with docker-compose"
+        docker-compose down
+popd