unittest using externally managed database (#9400)

* MM-12083: unittest using externally managed database * cherry-pick Makefile changes from @cpanato * Jenkins changes (#9915) * add docker compose * udpate * when using minio dont need to set the region * update * add wait for it script * using old minio * add new jenkins file * update makefile * add dockerfile * rename the docker-compose proj to avoid colision (#9917) * rename the docker-compose proj to avoid colision * enable debug * enable debug to double checkt the branchs and fix docker-compose name (#9919) * add ee hash to check (#9920) * fix name (#9921) * update jenkins file to push from branch and prs * if a new push comes in stop the running build * split mysql and postgres variables * add script to run jenkins-like env in local dev env * update docker-compose project name to use uuid to make it more randon * fix DCNAME definition * update elasticsearch docker image * revert test * tidy up stages, and wait for mysql differently * update docker image and add check for postgres * checking if is ready * update docker compose to have a wait for deps * add readme and rename dockerfile * fix -unittest setup * using mm docker image * restore parallel unit tests at the package level Spin up a dedicated database for each package under test to avoid races in accessing the same tables. Simplify the interface for configuring the test database to just a DSN instead of multiple exports for each field. * try to work around root mysql access in CI * update local-test-env.sh too * MYSQL_ROOT_HOST: % * fix missing quotes * setting some memory limits for mysql * revert memory docker compose does not support * fix env name for postgres * expose errors in app/export_test.go * fix test label, better error checking on teardown * increase query timeout for tests * fix export_test * update local dev script * add configurable mysql root passwd
mattermost · Dec 10, 2018 · 8374b62 · 8374b62
1 parent 204dde9
commit 8374b62
Show file tree

Hide file tree

Showing 16 changed files with 846 additions and 300 deletions.
diff --git a/Makefile b/Makefile
@@ -2,6 +2,7 @@
 
 ROOT := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
 
+IS_CI ?= false
 # Build Flags
 BUILD_NUMBER ?= $(BUILD_NUMBER:)
 BUILD_DATE = $(shell date -u)
@@ -100,47 +101,55 @@ all: run ## Alias for 'run'.
 include build/*.mk
 
 start-docker: ## Starts the docker containers for local development.
+ifeq ($(IS_CI),false)
 	@echo Starting docker containers
 
-	@if [ $(shell docker ps -a | grep -ci mattermost-mysql) -eq 0 ]; then \
+	@if [ $(shell docker ps -a --no-trunc --quiet --filter name=^/mattermost-mysql$$ | wc -l) -eq 0 ]; then \
 		echo starting mattermost-mysql; \
-		docker run --name mattermost-mysql -p 3306:3306 -e MYSQL_ROOT_PASSWORD=mostest \
-		-e MYSQL_USER=mmuser -e MYSQL_PASSWORD=mostest -e MYSQL_DATABASE=mattermost_test -d mysql:5.7 > /dev/null; \
-	elif [ $(shell docker ps | grep -ci mattermost-mysql) -eq 0 ]; then \
+		docker run --name mattermost-mysql -p 3306:3306 \
+			-e MYSQL_ROOT_PASSWORD=mostest \
+			-e MYSQL_USER=mmuser \
+			-e MYSQL_PASSWORD=mostest \
+			-e MYSQL_DATABASE=mattermost_test \
+			-d mysql:5.7 > /dev/null; \
+	elif [ $(shell docker ps --no-trunc --quiet --filter name=^/mattermost-mysql$$ | wc -l) -eq 0 ]; then \
 		echo restarting mattermost-mysql; \
 		docker start mattermost-mysql > /dev/null; \
 	fi
 
-	@if [ $(shell docker ps -a | grep -ci mattermost-postgres) -eq 0 ]; then \
+	@if [ $(shell docker ps -a --no-trunc --quiet --filter name=^/mattermost-postgres$$ | wc -l) -eq 0 ]; then \
 		echo starting mattermost-postgres; \
-		docker run --name mattermost-postgres -p 5432:5432 -e POSTGRES_USER=mmuser -e POSTGRES_PASSWORD=mostest -e POSTGRES_DB=mattermost_test \
-		-d postgres:9.4 > /dev/null; \
-	elif [ $(shell docker ps | grep -ci mattermost-postgres) -eq 0 ]; then \
+		docker run --name mattermost-postgres -p 5432:5432 \
+			-e POSTGRES_USER=mmuser \
+			-e POSTGRES_PASSWORD=mostest \
+			-e POSTGRES_DB=mattermost_test \
+			-d postgres:9.4 > /dev/null; \
+	elif [ $(shell docker ps --no-trunc --quiet --filter name=^/mattermost-postgres$$ | wc -l) -eq 0 ]; then \
 		echo restarting mattermost-postgres; \
 		docker start mattermost-postgres > /dev/null; \
 	fi
 
-	@if [ $(shell docker ps -a | grep -ci mattermost-inbucket) -eq 0 ]; then \
+	@if [ $(shell docker ps -a --no-trunc --quiet --filter name=^/mattermost-inbucket$$ | wc -l) -eq 0 ]; then \
 		echo starting mattermost-inbucket; \
 		docker run --name mattermost-inbucket -p 9000:10080 -p 2500:10025 -d jhillyerd/inbucket:release-1.2.0 > /dev/null; \
-	elif [ $(shell docker ps | grep -ci mattermost-inbucket) -eq 0 ]; then \
+	elif [ $(shell docker ps --no-trunc --quiet --filter name=^/mattermost-inbucket$$ | wc -l) -eq 0 ]; then \
 		echo restarting mattermost-inbucket; \
 		docker start mattermost-inbucket > /dev/null; \
 	fi
 
-	@if [ $(shell docker ps -a | grep -ci mattermost-minio) -eq 0 ]; then \
+	@if [ $(shell docker ps -a --no-trunc --quiet --filter name=^/mattermost-minio$$ | wc -l) -eq 0 ]; then \
 		echo starting mattermost-minio; \
 		docker run --name mattermost-minio -p 9001:9000 -e "MINIO_ACCESS_KEY=minioaccesskey" \
 		-e "MINIO_SECRET_KEY=miniosecretkey" -d minio/minio:RELEASE.2018-05-25T19-49-13Z server /data > /dev/null; \
 		docker exec -it mattermost-minio /bin/sh -c "mkdir -p /data/mattermost-test" > /dev/null; \
-	elif [ $(shell docker ps | grep -ci mattermost-minio) -eq 0 ]; then \
+	elif [ $(shell docker ps --no-trunc --quiet --filter name=^/mattermost-minio$$ | wc -l) -eq 0 ]; then \
 		echo restarting mattermost-minio; \
 		docker start mattermost-minio > /dev/null; \
 	fi
 
 ifeq ($(BUILD_ENTERPRISE_READY),true)
 	@echo Ldap test user test.one
-	@if [ $(shell docker ps -a | grep -ci mattermost-openldap) -eq 0 ]; then \
+	@if [ $(shell docker ps -a --no-trunc --quiet --filter name=^/mattermost-openldap$$ | wc -l) -eq 0 ]; then \
 		echo starting mattermost-openldap; \
 		docker run --name mattermost-openldap -p 389:389 -p 636:636 \
 			-e LDAP_TLS_VERIFY_CLIENT="never" \
@@ -155,48 +164,61 @@ ifeq ($(BUILD_ENTERPRISE_READY),true)
 		docker exec -ti mattermost-openldap bash -c 'echo -e "dn: uid=test.two,ou=testusers,dc=mm,dc=test,dc=com\nobjectclass: iNetOrgPerson\nsn: User\ncn: Test2\nmail: success+testtwo@simulator.amazonses.com" | ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest';\
 		docker exec -ti mattermost-openldap bash -c 'ldappasswd -s Password1 -D "cn=admin,dc=mm,dc=test,dc=com" -x "uid=test.two,ou=testusers,dc=mm,dc=test,dc=com" -w mostest';\
 		docker exec -ti mattermost-openldap bash -c 'echo -e "dn: cn=tgroup,ou=testusers,dc=mm,dc=test,dc=com\nobjectclass: groupOfUniqueNames\nuniqueMember: uid=test.one,ou=testusers,dc=mm,dc=test,dc=com" | ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest';\
-	elif [ $(shell docker ps | grep -ci mattermost-openldap) -eq 0 ]; then \
+	elif [ $(shell docker ps --no-trunc --quiet --filter name=^/mattermost-openldap$$ | wc -l) -eq 0 ]; then \
 		echo restarting mattermost-openldap; \
 		docker start mattermost-openldap > /dev/null; \
 		sleep 10; \
 	fi
 
-	@if [ $(shell docker ps -a | grep -ci mattermost-elasticsearch) -eq 0 ]; then \
+	@if [ $(shell docker ps -a --no-trunc --quiet --filter name=^/mattermost-elasticsearch$$ | wc -l) -eq 0 ]; then \
 		echo starting mattermost-elasticsearch; \
 		docker run --name mattermost-elasticsearch -p 9200:9200 -e "http.host=0.0.0.0" -e "transport.host=127.0.0.1" -e "ES_JAVA_OPTS=-Xms250m -Xmx250m" -d grundleborg/elasticsearch:latest > /dev/null; \
-	elif [ $(shell docker ps | grep -ci mattermost-elasticsearch) -eq 0 ]; then \
+	elif [ $(shell docker ps --no-trunc --quiet --filter name=^/mattermost-elasticsearch$$ | wc -l) -eq 0 ]; then \
 		echo restarting mattermost-elasticsearch; \
 		docker start mattermost-elasticsearch> /dev/null; \
 	fi
 
-	@if [ $(shell docker ps -a | grep -ci mattermost-redis) -eq 0 ]; then \
+	@if [ $(shell docker ps -a --no-trunc --quiet --filter name=^/mattermost-redis$$ | wc -l) -eq 0 ]; then \
 		echo starting mattermost-redis; \
 		docker run --name mattermost-redis -p 6379:6379 -d redis > /dev/null; \
-	elif [ $(shell docker ps | grep -ci mattermost-redis) -eq 0 ]; then \
+	elif [ $(shell docker ps --no-trunc --quiet --filter name=^/mattermost-redis$$ | wc -l) -eq 0 ]; then \
 		echo restarting mattermost-redis; \
 		docker start mattermost-redis > /dev/null; \
 	fi
 endif
+else
+	@echo CI Build: skipping docker start
+endif
 
 stop-docker: ## Stops the docker containers for local development.
 	@echo Stopping docker containers
 
-	@if [ $(shell docker ps -a | grep -ci mattermost-mysql) -eq 1 ]; then \
+	@if [ $(shell docker ps -a --no-trunc --quiet --filter name=^/mattermost-mysql$$ | wc -l) -eq 1 ]; then \
 		echo stopping mattermost-mysql; \
 		docker stop mattermost-mysql > /dev/null; \
 	fi
 
-	@if [ $(shell docker ps -a | grep -ci mattermost-postgres) -eq 1 ]; then \
+	@if [ $(shell docker ps -a --no-trunc --quiet --filter name=^/mattermost-mysql-unittest$$ | wc -l) -eq 1 ]; then \
+		echo stopping mattermost-mysql-unittest; \
+		docker stop mattermost-mysql-unittest > /dev/null; \
+	fi
+
+	@if [ $(shell docker ps -a --no-trunc --quiet --filter name=^/mattermost-postgres$$ | wc -l) -eq 1 ]; then \
 		echo stopping mattermost-postgres; \
 		docker stop mattermost-postgres > /dev/null; \
 	fi
 
-	@if [ $(shell docker ps -a | grep -ci mattermost-openldap) -eq 1 ]; then \
+	@if [ $(shell docker ps -a --no-trunc --quiet --filter name=^/mattermost-postgres-unittest$$ | wc -l) -eq 1 ]; then \
+		echo stopping mattermost-postgres-unittest; \
+		docker stop mattermost-postgres-unittest > /dev/null; \
+	fi
+
+	@if [ $(shell docker ps -a --no-trunc --quiet --filter name=^/mattermost-openldap$$ | wc -l) -eq 1 ]; then \
 		echo stopping mattermost-openldap; \
 		docker stop mattermost-openldap > /dev/null; \
 	fi
 
-	@if [ $(shell docker ps -a | grep -ci mattermost-inbucket) -eq 1 ]; then \
+	@if [ $(shell docker ps -a --no-trunc --quiet --filter name=^/mattermost-inbucket$$ | wc -l) -eq 1 ]; then \
 		echo stopping mattermost-inbucket; \
 		docker stop mattermost-inbucket > /dev/null; \
 	fi
@@ -206,26 +228,43 @@ stop-docker: ## Stops the docker containers for local development.
 		docker stop mattermost-minio > /dev/null; \
 	fi
 
-	@if [ $(shell docker ps -a | grep -ci mattermost-elasticsearch) -eq 1 ]; then \
+	@if [ $(shell docker ps -a --no-trunc --quiet --filter name=^/mattermost-elasticsearch$$ | wc -l) -eq 1 ]; then \
 		echo stopping mattermost-elasticsearch; \
 		docker stop mattermost-elasticsearch > /dev/null; \
 	fi
 
+	@if [ $(shell docker ps -a --no-trunc --quiet --filter name=^/mattermost-redis$$ | wc -l) -eq 1 ]; then \
+		echo stopping mattermost-redis; \
+		docker stop mattermost-redis > /dev/null; \
+	fi
+
 clean-docker: ## Deletes the docker containers for local development.
 	@echo Removing docker containers
 
-	@if [ $(shell docker ps -a | grep -ci mattermost-mysql) -eq 1 ]; then \
+	@if [ $(shell docker ps -a --no-trunc --quiet --filter name=^/mattermost-mysql$$ | wc -l) -eq 1 ]; then \
 		echo removing mattermost-mysql; \
 		docker stop mattermost-mysql > /dev/null; \
 		docker rm -v mattermost-mysql > /dev/null; \
 	fi
 
-	@if [ $(shell docker ps -a | grep -ci mattermost-postgres) -eq 1 ]; then \
+	@if [ $(shell docker ps -a --no-trunc --quiet --filter name=^/mattermost-mysql-unittest$$ | wc -l) -eq 1 ]; then \
+		echo removing mattermost-mysql-unittest; \
+		docker stop mattermost-mysql-unittest > /dev/null; \
+		docker rm -v mattermost-mysql-unittest > /dev/null; \
+	fi
+
+	@if [ $(shell docker ps -a --no-trunc --quiet --filter name=^/mattermost-postgres$$ | wc -l) -eq 1 ]; then \
 		echo removing mattermost-postgres; \
 		docker stop mattermost-postgres > /dev/null; \
 		docker rm -v mattermost-postgres > /dev/null; \
 	fi
 
+	@if [ $(shell docker ps -a --no-trunc --quiet --filter name=^/mattermost-postgres-unittest$$ | wc -l) -eq 1 ]; then \
+		echo removing mattermost-postgres-unittest; \
+		docker stop mattermost-postgres-unittest > /dev/null; \
+		docker rm -v mattermost-postgres-unittest > /dev/null; \
+	fi
+
 	@if [ $(shell docker ps -a | grep -ci mattermost-openldap) -eq 1 ]; then \
 		echo removing mattermost-openldap; \
 		docker stop mattermost-openldap > /dev/null; \

diff --git a/app/export_test.go b/app/export_test.go
@@ -88,7 +88,8 @@ func TestExportUserChannels(t *testing.T) {
 	th.App.Srv.Store.Channel().SaveMember(&channelMember)
 	th.App.Srv.Store.Preference().Save(&preferences)
 	th.App.UpdateChannelMemberNotifyProps(notifyProps, channel.Id, user.Id)
-	exportData, _ := th.App.buildUserChannelMemberships(user.Id, team.Id)
+	exportData, err := th.App.buildUserChannelMemberships(user.Id, team.Id)
+	require.Nil(t, err)
 	assert.Equal(t, len(*exportData), 3)
 	for _, data := range *exportData {
 		if *data.Name == channelName {
@@ -158,15 +159,15 @@ func TestExportCustomEmoji(t *testing.T) {
 
 	filePath := "../demo.json"
 
-	fileWriter, _ := os.Create(filePath)
+	fileWriter, err := os.Create(filePath)
+	require.Nil(t, err)
 	defer os.Remove(filePath)
 
 	pathToEmojiDir := "../data/emoji/"
 	dirNameToExportEmoji := "exported_emoji_test"
-
-	err := th.App.ExportCustomEmoji(fileWriter, filePath, pathToEmojiDir, dirNameToExportEmoji)
 	defer os.RemoveAll("../" + dirNameToExportEmoji)
-	if err != nil {
+
+	if err := th.App.ExportCustomEmoji(fileWriter, filePath, pathToEmojiDir, dirNameToExportEmoji); err != nil {
 		t.Fatal(err)
 	}
 }
diff --git a/build/Dockerfile.buildenv b/build/Dockerfile.buildenv
@@ -0,0 +1,3 @@
+FROM golang:1.11
+
+RUN apt-get update && apt-get install -y make git apt-transport-https ca-certificates curl software-properties-common build-essential zip wget xmlsec1
diff --git a/build/Jenkinsfile → build/Jenkinsfile.k8s b/build/Jenkinsfile → build/Jenkinsfile.k8s