CI testing via GitLab (#248)

* gitlab support * disable dialog when no tty * split builds into stages * add vmware wrapper * fix for dovecot port rename * avg: add -y when no tty * switch pkg from quarterly -> latest * prevent variable interpolation * IP whitelist the staging & monitor servers" * reduce busy fs timeout * vpopmail: use unique nrpe function name
msimerson · Aug 17, 2017 · d1071c8 · d1071c8
1 parent bebe601
commit d1071c8
Show file tree

Hide file tree

Showing 19 changed files with 481 additions and 39 deletions.
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -1,2 +1,92 @@
-jailip:
-  script: "test/get_jail_ip.sh"
+image: auchida/freebsd:latest
+
+# Instructions for creating your own test runner:
+# https://github.com/msimerson/Mail-Toaster-6/wiki/Develop-CI-Testing/
+
+# these get run before/after EVERY script / stage
+# before_script:
+# after_script
+
+stages:
+  - host_setup
+  - mailservices
+  - mailstore
+  - mta
+  - webmail
+  - extras
+  - test
+
+host_setup:
+  stage: host_setup
+  script:
+    - sh test/get_jail_ip.sh
+    - echo "export TOASTER_HOSTNAME=`hostname`" >> mail-toaster.conf
+    - echo "export TOASTER_MAIL_DOMAIN=`hostname`" >> mail-toaster.conf
+    - pkg update && pkg install -y ca_root_nss
+    - sh provision-host.sh
+    - sh provision-base.sh
+  tags:
+    - freebsd
+    - ssh
+
+mailservices:
+  stage: mailservices
+  script:
+    - sh provision-dns.sh
+    - sh provision-mysql.sh
+    - sh provision-redis.sh
+  tags:
+    - freebsd
+    - ssh
+
+mailstore:
+  stage: mailstore
+  script:
+    - sh provision-vpopmail.sh
+    - sh provision-dovecot.sh
+  tags:
+    - freebsd
+    - ssh
+
+mta:
+  stage: mta
+  script:
+    - sh provision-clamav.sh
+    - sh provision-rspamd.sh
+    - sh provision-geoip.sh
+    - sh provision-haraka.sh
+  tags:
+    - freebsd
+    - ssh
+
+webmail:
+  stage: webmail
+  script:
+    - sh provision-haproxy.sh
+    - sh provision-webmail.sh
+    - sh provision-roundcube.sh
+    - sh provision-rainloop.sh
+    - sh provision-squirrelmail.sh
+  tags:
+    - freebsd
+    - ssh
+
+extras:
+  stage: extras
+  script:
+    - sh provision-avg.sh
+    - sh provision-spamassassin.sh
+    - sh provision-sqwebmail.sh
+  tags:
+    - freebsd
+    - ssh
+  allow_failure: true
+
+test:
+  stage: test
+  script:
+    - sh provision-monitor.sh
+  tags:
+    - freebsd
+    - ssh
+  allow_failure: true
diff --git a/.travis.yml b/.travis.yml
@@ -1,12 +1,5 @@
 language: bash
 
-# required until the apt addon shellcheck is available
-#sudo: required
-#dist: trusty
-
-# services:
-#    - docker
-
 cache:
   directories:
     - /home/travis/.stack
@@ -22,22 +15,22 @@ before_install:
 #   - echo "deb http://archive.ubuntu.com/ubuntu/ wily universe" | sudo tee -a /etc/apt/sources.list
 #   - sudo apt-get update -qq
 #   - sudo apt-get install -y shellcheck
-#   - docker build -t tnpi/mt6:latest .
-#   - docker run -d --name mt6 tnpi/mt6:latest /
 
 install:
     - test -f mail-toaster.sh
 
-before_script:
+#before_script:
 
 script:
    - bash -n *.sh
    - shellcheck -e SC2009,SC2039,SC2086,SC2153,SC2016,SC1004,SC2119 *.sh
+   - bash -n include/*.sh
+   - shellcheck -e SC2009,SC2039,SC2086,SC2153,SC2016,SC1004,SC2119 include/*.sh
    - bash -n qmail/run.sh
    - shellcheck -e SC2039 qmail/run.sh
    - bash test/*.sh
 
 # after_success:
 # after_failure:
-after_script:
+# after_script:
 
diff --git a/include/php.sh b/include/php.sh
@@ -51,7 +51,7 @@ configure_php_ini()
 
 	tell_status "getting the timezone"
 	TZ=$(md5 -q /etc/localtime)
-	TIMEZONE=$(find /usr/share/zoneinfo -type f | xargs md5 -r | grep "$TZ" | awk '{print $2}' |cut -c21-)
+	TIMEZONE=$(find /usr/share/zoneinfo -type f -print0 | xargs md5 -r | grep "$TZ" | awk '{print $2}' |cut -c21-)
 
 	if [ -z "$TIMEZONE" ]; then
 		TIMEZONE="America\/Los_Angeles"

diff --git a/include/shell.sh b/include/shell.sh
@@ -34,7 +34,6 @@ export HISTCONTROL=ignoredups:erasedups
 export HISTIGNORE="&:[bf]g:exit"
 shopt -s histappend
 shopt -s cdspell
-bind Space:magic-space
 alias h="history 200"
 alias ll="ls -alFG"
 PS1="$(whoami)@$(hostname -s):\\w # "

diff --git a/mail-toaster.sh b/mail-toaster.sh
@@ -5,9 +5,11 @@ create_default_config()
 	local _HOSTNAME;
 	local _EMAIL_DOMAIN;
 
-	echo "editing prefs"
-	_HOSTNAME=$(dialog --stdout --nocancel --backtitle "mail-toaster.sh" --title TOASTER_HOSTNAME --inputbox "the hostname of this [virtual] machine" 8 70 "mail.example.com")
-	_EMAIL_DOMAIN=$(dialog --stdout --nocancel --backtitle "mail-toaster.sh" --title TOASTER_MAIL_DOMAIN --inputbox "the primary email domain" 8 70 "example.com")
+	if [ -t 0 ]; then
+		echo "editing prefs"
+		_HOSTNAME=$(dialog --stdout --nocancel --backtitle "mail-toaster.sh" --title TOASTER_HOSTNAME --inputbox "the hostname of this [virtual] machine" 8 70 "mail.example.com")
+		_EMAIL_DOMAIN=$(dialog --stdout --nocancel --backtitle "mail-toaster.sh" --title TOASTER_MAIL_DOMAIN --inputbox "the primary email domain" 8 70 "example.com")
+	fi
 
 	# for Travis CI (Linux) where dialog doesn't exist
 	if [ -z "$_HOSTNAME"     ]; then _HOSTNAME=$(hostname); fi
@@ -59,7 +61,7 @@ export BOURNE_SHELL=${BOURNE_SHELL:="bash"}
 export JAIL_NET_PREFIX=${JAIL_NET_PREFIX:="172.16.15"}
 export JAIL_NET_MASK=${JAIL_NET_MASK:="/12"}
 export JAIL_NET_INTERFACE=${JAIL_NET_INTERFACE:="lo1"}
-export JAIL_ORDERED_LIST="syslog base dns mysql clamav spamassassin dspam vpopmail haraka webmail monitor haproxy rspamd avg dovecot redis geoip nginx lighttpd apache postgres minecraft joomla php7 memcached sphinxsearch elasticsearch nictool sqwebmail dhcp letsencrypt tinydns roundcube squirrelmail rainloop rsnapshot mediawiki smf wordpress whmcs squirrelcart horde grafana unifi mongodb"
+export JAIL_ORDERED_LIST="syslog base dns mysql clamav spamassassin dspam vpopmail haraka webmail monitor haproxy rspamd avg dovecot redis geoip nginx lighttpd apache postgres minecraft joomla php7 memcached sphinxsearch elasticsearch nictool sqwebmail dhcp letsencrypt tinydns roundcube squirrelmail rainloop rsnapshot mediawiki smf wordpress whmcs squirrelcart horde grafana unifi mongodb gitlab gitlab_runner"
 
 export ZFS_VOL=${ZFS_VOL:="zroot"}
 export ZFS_JAIL_MNT=${ZFS_JAIL_MNT:="/jails"}
@@ -434,13 +436,13 @@ rename_staged_to_ready()
 	local _zfs_rename="zfs rename $ZFS_JAIL_VOL/stage $_new_vol"
 	echo "$_zfs_rename"
 	until $_zfs_rename; do
-		if [ "$_tries" -gt 25 ]; then
+		if [ "$_tries" -gt 15 ]; then
 			echo "trying to force rename"
 			_zfs_rename="zfs rename -f $ZFS_JAIL_VOL/stage $_new_vol"
 		fi
 		echo "waiting for ZFS filesystem to quiet ($_tries)"
 		_tries=$((_tries + 1))
-		sleep 5
+		sleep 4
 	done
 }
 
@@ -463,7 +465,7 @@ rename_active_to_last()
 		fi
 		echo "waiting for ZFS filesystem to quiet ($_tries)"
 		_tries=$((_tries + 1))
-		sleep 5
+		sleep 4
 	done
 }
 
@@ -935,3 +937,16 @@ jail_rename()
 
     echo "Don't forget to update your PF and/or Haproxy rules"
 }
+
+configure_pkg_latest()
+{
+	local REPODIR="$1/usr/local/etc/pkg/repos"
+	if [ -f "$REPODIR/FreeBSD.conf" ]; then return; fi
+
+	mkdir -p "$REPODIR"
+	tee "$REPODIR/FreeBSD.conf" <<'EO_PKG'
+FreeBSD: {
+  url: "pkg+http://pkg.FreeBSD.org/${ABI}/latest"
+}
+EO_PKG
+}
diff --git a/provision-avg.sh b/provision-avg.sh
@@ -48,7 +48,11 @@ install_avg()
 	tell_status "installing avg"
 	tar -C "$STAGE_MNT/tmp" -xzf avg2013ffb-r3115-a6155.i386.tar.gz || exit
 	mkdir -p "$STAGE_MNT/usr/local/etc/rc.d" || exit
-	stage_exec /tmp/avg2013ffb-r3115-a6155.i386/install.sh
+	if [ -t 0 ]; then
+		stage_exec /tmp/avg2013ffb-r3115-a6155.i386/install.sh
+	else
+		stage_exec /tmp/avg2013ffb-r3115-a6155.i386/install.sh -y
+	fi
 }
 
 configure_avg()

diff --git a/provision-base.sh b/provision-base.sh
@@ -25,6 +25,11 @@ create_base_filesystem()
 
 freebsd_update()
 {
+	if [ ! -t 0 ]; then
+		echo "No tty, can't update FreeBSD with freebsd-update"
+		return
+	fi
+
 	tell_status "apply FreeBSD security updates to base jail"
 	sed -i .bak -e 's/^Components.*/Components world/' "$BASE_MNT/etc/freebsd-update.conf"
 	freebsd-update -b "$BASE_MNT" -f "$BASE_MNT/etc/freebsd-update.conf" fetch install
@@ -184,6 +189,7 @@ configure_base()
 		sendmail_enable=NONE \
 		update_motd=NO
 
+	configure_pkg_latest "$BASE_MNT"
 	configure_ssl_dirs
 	configure_tls_dhparams
 	disable_cron_jobs

diff --git a/provision-dns.sh b/provision-dns.sh
@@ -19,7 +19,6 @@ get_mt6_data()
 	   local-data: \"stage        A $(get_jail_ip stage)\"
 	   local-data: \"$(get_reverse_ip stage) PTR stage\""
 
-	local _octet=${JAIL_NET_START:=1}
 	for _j in $JAIL_ORDERED_LIST
 	do
 		echo "
@@ -64,6 +63,7 @@ tweak_unbound_conf()
 {
 	tell_status "configuring unbound.conf"
 	# control.conf for the munin stats plugin
+	# shellcheck disable=1004
 	sed -i .bak \
 		-e 's/# interface: 192.0.2.153$/interface: 0.0.0.0/' \
 		-e 's/# interface: 192.0.2.154$/interface: ::0/' \
@@ -121,7 +121,7 @@ configure_unbound()
 	cp "$UNBOUND_DIR/unbound.conf.sample" "$UNBOUND_DIR/unbound.conf" || exit
 	if [ -f 'unbound.conf.local' ]; then
 		tell_status "moving unbound.conf.local to data volume"
-		mv unbound.conf.local $ZFS_DATA_MNT/dns/ || exit
+		mv unbound.conf.local "$ZFS_DATA_MNT/dns/" || exit
 	fi
 
 	if [ -f "$ZFS_DATA_MNT/dns/unbound.conf.local" ]; then
@@ -171,7 +171,7 @@ promote_staged_jail dns
 
 if ! grep "^nameserver $(get_jail_ip dns)" /etc/resolv.conf;
 then
-	# shellcheck disable=2039,2094
 	echo "switching host resolver to $(get_jail_ip dns)"
+	# shellcheck disable=2039,2094
 	echo -e "nameserver $(get_jail_ip dns)\n$(cat /etc/resolv.conf)" > /etc/resolv.conf
 fi
diff --git a/provision-dovecot.sh b/provision-dovecot.sh
@@ -12,8 +12,8 @@ mt6-include vpopmail
 
 install_dovecot()
 {
-	tell_status "installing dovecot v2 package"
-	stage_pkg_install dovecot2 || stage_pkg_install dovecot || exit
+	tell_status "installing dovecot package"
+	stage_pkg_install dovecot || exit
 
 	tell_status "configure dovecot port options"
 	stage_make_conf dovecot2_SET 'mail_dovecot2_SET=VPOPMAIL LIBWRAP EXAMPLES'
@@ -211,9 +211,10 @@ configure_tls_certs()
 
 	tell_status "installing dovecot TLS certificates"
 	cp /etc/ssl/certs/server.crt "$_ssldir/certs/${TOASTER_MAIL_DOMAIN}.pem" || exit
-	cat /etc/ssl/dhparam.pem >> "$_ssldir/certs/${TOASTER_MAIL_DOMAIN}.pem"
+	# sunset after Dovecot 2.3 released
+	cat /etc/ssl/dhparam.pem >> "$_ssldir/certs/${TOASTER_MAIL_DOMAIN}.pem" || exit
+	# /sunset
 	cp /etc/ssl/private/server.key "$_ssldir/private/${TOASTER_MAIL_DOMAIN}.pem" || exit
-
 }
 
 configure_dovecot()
@@ -243,6 +244,42 @@ start_dovecot()
 	stage_exec service dovecot start || exit
 }
 
+test_imap()
+{
+	stage_pkg_install empty
+
+	REMOTE_IP=$(get_jail_ip dovecot)
+	POST_USER="postmaster@$(hostname)"
+	POST_PASS=$(jexec vpopmail /usr/local/vpopmail/bin/vuserinfo -C "${POST_USER}")
+	rm -f in out
+
+	#empty -v -f -i in -o out openssl s_client -quiet -crlf -connect $REMOTE_IP:993
+	empty -v -f -i in -o out telnet "$REMOTE_IP" 143
+	empty -v -w -i out -o in "ready"             ". LOGIN $POST_USER $POST_PASS\n"
+	empty -v -w -i out -o in "Logged in"         ". LIST \"\" \"*\"\n"
+	empty -v -w -i out -o in "List completed"    ". SELECT INBOX\n"
+	empty -v -w -i out -o in "Select completed"  ". FETCH 1 BODY\n"
+	empty -v -w -i out -o in "OK Fetch completed" ". logout\n"
+	echo "Logout completed"
+}
+
+test_pop3()
+{
+	stage_pkg_install empty
+
+	REMOTE_IP=$(get_jail_ip dovecot)
+	POST_USER="postmaster@$(hostname)"
+	POST_PASS=$(jexec vpopmail /usr/local/vpopmail/bin/vuserinfo -C "${POST_USER}")
+	rm -f in out
+
+	#empty -v -f -i in -o out openssl s_client -quiet -crlf -connect $REMOTE_IP:995
+	empty -v -f -i in -o out telnet "$REMOTE_IP" 110
+	empty -v -w -i out -o in "\+OK." "user $POST_USER\n"
+	empty -v -w -i out -o in "\+OK" "pass $POST_PASS\n"
+	empty -v -w -i out -o in "OK Logged in" "list\n"
+	empty -v -w -i out -o in "." "quit\n"
+}
+
 test_dovecot()
 {
 	tell_status "testing dovecot"