diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f904ebca..b9c903cb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,2 +1,92 @@ -jailip: - script: "test/get_jail_ip.sh" +image: auchida/freebsd:latest + +# Instructions for creating your own test runner: +# https://github.com/msimerson/Mail-Toaster-6/wiki/Develop-CI-Testing/ + +# these get run before/after EVERY script / stage +# before_script: +# after_script + +stages: + - host_setup + - mailservices + - mailstore + - mta + - webmail + - extras + - test + +host_setup: + stage: host_setup + script: + - sh test/get_jail_ip.sh + - echo "export TOASTER_HOSTNAME=`hostname`" >> mail-toaster.conf + - echo "export TOASTER_MAIL_DOMAIN=`hostname`" >> mail-toaster.conf + - pkg update && pkg install -y ca_root_nss + - sh provision-host.sh + - sh provision-base.sh + tags: + - freebsd + - ssh + +mailservices: + stage: mailservices + script: + - sh provision-dns.sh + - sh provision-mysql.sh + - sh provision-redis.sh + tags: + - freebsd + - ssh + +mailstore: + stage: mailstore + script: + - sh provision-vpopmail.sh + - sh provision-dovecot.sh + tags: + - freebsd + - ssh + +mta: + stage: mta + script: + - sh provision-clamav.sh + - sh provision-rspamd.sh + - sh provision-geoip.sh + - sh provision-haraka.sh + tags: + - freebsd + - ssh + +webmail: + stage: webmail + script: + - sh provision-haproxy.sh + - sh provision-webmail.sh + - sh provision-roundcube.sh + - sh provision-rainloop.sh + - sh provision-squirrelmail.sh + tags: + - freebsd + - ssh + +extras: + stage: extras + script: + - sh provision-avg.sh + - sh provision-spamassassin.sh + - sh provision-sqwebmail.sh + tags: + - freebsd + - ssh + allow_failure: true + +test: + stage: test + script: + - sh provision-monitor.sh + tags: + - freebsd + - ssh + allow_failure: true diff --git a/.travis.yml b/.travis.yml index 759dcbb7..ba6c43a8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,12 +1,5 @@ language: bash -# required until the apt addon shellcheck is available -#sudo: required -#dist: trusty - -# services: -# - docker - cache: directories: - /home/travis/.stack @@ -22,22 +15,22 @@ before_install: # - echo "deb http://archive.ubuntu.com/ubuntu/ wily universe" | sudo tee -a /etc/apt/sources.list # - sudo apt-get update -qq # - sudo apt-get install -y shellcheck -# - docker build -t tnpi/mt6:latest . -# - docker run -d --name mt6 tnpi/mt6:latest / install: - test -f mail-toaster.sh -before_script: +#before_script: script: - bash -n *.sh - shellcheck -e SC2009,SC2039,SC2086,SC2153,SC2016,SC1004,SC2119 *.sh + - bash -n include/*.sh + - shellcheck -e SC2009,SC2039,SC2086,SC2153,SC2016,SC1004,SC2119 include/*.sh - bash -n qmail/run.sh - shellcheck -e SC2039 qmail/run.sh - bash test/*.sh # after_success: # after_failure: -after_script: +# after_script: diff --git a/include/php.sh b/include/php.sh index 350a7759..cb113273 100644 --- a/include/php.sh +++ b/include/php.sh @@ -51,7 +51,7 @@ configure_php_ini() tell_status "getting the timezone" TZ=$(md5 -q /etc/localtime) - TIMEZONE=$(find /usr/share/zoneinfo -type f | xargs md5 -r | grep "$TZ" | awk '{print $2}' |cut -c21-) + TIMEZONE=$(find /usr/share/zoneinfo -type f -print0 | xargs md5 -r | grep "$TZ" | awk '{print $2}' |cut -c21-) if [ -z "$TIMEZONE" ]; then TIMEZONE="America\/Los_Angeles" diff --git a/include/shell.sh b/include/shell.sh index 4ffc90e1..d7e470ce 100644 --- a/include/shell.sh +++ b/include/shell.sh @@ -34,7 +34,6 @@ export HISTCONTROL=ignoredups:erasedups export HISTIGNORE="&:[bf]g:exit" shopt -s histappend shopt -s cdspell -bind Space:magic-space alias h="history 200" alias ll="ls -alFG" PS1="$(whoami)@$(hostname -s):\\w # " diff --git a/mail-toaster.sh b/mail-toaster.sh index 830ef2ed..32525b49 100755 --- a/mail-toaster.sh +++ b/mail-toaster.sh @@ -5,9 +5,11 @@ create_default_config() local _HOSTNAME; local _EMAIL_DOMAIN; - echo "editing prefs" - _HOSTNAME=$(dialog --stdout --nocancel --backtitle "mail-toaster.sh" --title TOASTER_HOSTNAME --inputbox "the hostname of this [virtual] machine" 8 70 "mail.example.com") - _EMAIL_DOMAIN=$(dialog --stdout --nocancel --backtitle "mail-toaster.sh" --title TOASTER_MAIL_DOMAIN --inputbox "the primary email domain" 8 70 "example.com") + if [ -t 0 ]; then + echo "editing prefs" + _HOSTNAME=$(dialog --stdout --nocancel --backtitle "mail-toaster.sh" --title TOASTER_HOSTNAME --inputbox "the hostname of this [virtual] machine" 8 70 "mail.example.com") + _EMAIL_DOMAIN=$(dialog --stdout --nocancel --backtitle "mail-toaster.sh" --title TOASTER_MAIL_DOMAIN --inputbox "the primary email domain" 8 70 "example.com") + fi # for Travis CI (Linux) where dialog doesn't exist if [ -z "$_HOSTNAME" ]; then _HOSTNAME=$(hostname); fi @@ -59,7 +61,7 @@ export BOURNE_SHELL=${BOURNE_SHELL:="bash"} export JAIL_NET_PREFIX=${JAIL_NET_PREFIX:="172.16.15"} export JAIL_NET_MASK=${JAIL_NET_MASK:="/12"} export JAIL_NET_INTERFACE=${JAIL_NET_INTERFACE:="lo1"} -export JAIL_ORDERED_LIST="syslog base dns mysql clamav spamassassin dspam vpopmail haraka webmail monitor haproxy rspamd avg dovecot redis geoip nginx lighttpd apache postgres minecraft joomla php7 memcached sphinxsearch elasticsearch nictool sqwebmail dhcp letsencrypt tinydns roundcube squirrelmail rainloop rsnapshot mediawiki smf wordpress whmcs squirrelcart horde grafana unifi mongodb" +export JAIL_ORDERED_LIST="syslog base dns mysql clamav spamassassin dspam vpopmail haraka webmail monitor haproxy rspamd avg dovecot redis geoip nginx lighttpd apache postgres minecraft joomla php7 memcached sphinxsearch elasticsearch nictool sqwebmail dhcp letsencrypt tinydns roundcube squirrelmail rainloop rsnapshot mediawiki smf wordpress whmcs squirrelcart horde grafana unifi mongodb gitlab gitlab_runner" export ZFS_VOL=${ZFS_VOL:="zroot"} export ZFS_JAIL_MNT=${ZFS_JAIL_MNT:="/jails"} @@ -434,13 +436,13 @@ rename_staged_to_ready() local _zfs_rename="zfs rename $ZFS_JAIL_VOL/stage $_new_vol" echo "$_zfs_rename" until $_zfs_rename; do - if [ "$_tries" -gt 25 ]; then + if [ "$_tries" -gt 15 ]; then echo "trying to force rename" _zfs_rename="zfs rename -f $ZFS_JAIL_VOL/stage $_new_vol" fi echo "waiting for ZFS filesystem to quiet ($_tries)" _tries=$((_tries + 1)) - sleep 5 + sleep 4 done } @@ -463,7 +465,7 @@ rename_active_to_last() fi echo "waiting for ZFS filesystem to quiet ($_tries)" _tries=$((_tries + 1)) - sleep 5 + sleep 4 done } @@ -935,3 +937,16 @@ jail_rename() echo "Don't forget to update your PF and/or Haproxy rules" } + +configure_pkg_latest() +{ + local REPODIR="$1/usr/local/etc/pkg/repos" + if [ -f "$REPODIR/FreeBSD.conf" ]; then return; fi + + mkdir -p "$REPODIR" + tee "$REPODIR/FreeBSD.conf" <<'EO_PKG' +FreeBSD: { + url: "pkg+http://pkg.FreeBSD.org/${ABI}/latest" +} +EO_PKG +} diff --git a/provision-avg.sh b/provision-avg.sh index 9b802347..db4f6b97 100755 --- a/provision-avg.sh +++ b/provision-avg.sh @@ -48,7 +48,11 @@ install_avg() tell_status "installing avg" tar -C "$STAGE_MNT/tmp" -xzf avg2013ffb-r3115-a6155.i386.tar.gz || exit mkdir -p "$STAGE_MNT/usr/local/etc/rc.d" || exit - stage_exec /tmp/avg2013ffb-r3115-a6155.i386/install.sh + if [ -t 0 ]; then + stage_exec /tmp/avg2013ffb-r3115-a6155.i386/install.sh + else + stage_exec /tmp/avg2013ffb-r3115-a6155.i386/install.sh -y + fi } configure_avg() diff --git a/provision-base.sh b/provision-base.sh index f9531946..2975f6d3 100644 --- a/provision-base.sh +++ b/provision-base.sh @@ -25,6 +25,11 @@ create_base_filesystem() freebsd_update() { + if [ ! -t 0 ]; then + echo "No tty, can't update FreeBSD with freebsd-update" + return + fi + tell_status "apply FreeBSD security updates to base jail" sed -i .bak -e 's/^Components.*/Components world/' "$BASE_MNT/etc/freebsd-update.conf" freebsd-update -b "$BASE_MNT" -f "$BASE_MNT/etc/freebsd-update.conf" fetch install @@ -184,6 +189,7 @@ configure_base() sendmail_enable=NONE \ update_motd=NO + configure_pkg_latest "$BASE_MNT" configure_ssl_dirs configure_tls_dhparams disable_cron_jobs diff --git a/provision-dns.sh b/provision-dns.sh index c6f71d5b..a85ac19f 100755 --- a/provision-dns.sh +++ b/provision-dns.sh @@ -19,7 +19,6 @@ get_mt6_data() local-data: \"stage A $(get_jail_ip stage)\" local-data: \"$(get_reverse_ip stage) PTR stage\"" - local _octet=${JAIL_NET_START:=1} for _j in $JAIL_ORDERED_LIST do echo " @@ -64,6 +63,7 @@ tweak_unbound_conf() { tell_status "configuring unbound.conf" # control.conf for the munin stats plugin + # shellcheck disable=1004 sed -i .bak \ -e 's/# interface: 192.0.2.153$/interface: 0.0.0.0/' \ -e 's/# interface: 192.0.2.154$/interface: ::0/' \ @@ -121,7 +121,7 @@ configure_unbound() cp "$UNBOUND_DIR/unbound.conf.sample" "$UNBOUND_DIR/unbound.conf" || exit if [ -f 'unbound.conf.local' ]; then tell_status "moving unbound.conf.local to data volume" - mv unbound.conf.local $ZFS_DATA_MNT/dns/ || exit + mv unbound.conf.local "$ZFS_DATA_MNT/dns/" || exit fi if [ -f "$ZFS_DATA_MNT/dns/unbound.conf.local" ]; then @@ -171,7 +171,7 @@ promote_staged_jail dns if ! grep "^nameserver $(get_jail_ip dns)" /etc/resolv.conf; then - # shellcheck disable=2039,2094 echo "switching host resolver to $(get_jail_ip dns)" + # shellcheck disable=2039,2094 echo -e "nameserver $(get_jail_ip dns)\n$(cat /etc/resolv.conf)" > /etc/resolv.conf fi diff --git a/provision-dovecot.sh b/provision-dovecot.sh index eaeb170a..d08e651b 100755 --- a/provision-dovecot.sh +++ b/provision-dovecot.sh @@ -12,8 +12,8 @@ mt6-include vpopmail install_dovecot() { - tell_status "installing dovecot v2 package" - stage_pkg_install dovecot2 || stage_pkg_install dovecot || exit + tell_status "installing dovecot package" + stage_pkg_install dovecot || exit tell_status "configure dovecot port options" stage_make_conf dovecot2_SET 'mail_dovecot2_SET=VPOPMAIL LIBWRAP EXAMPLES' @@ -211,9 +211,10 @@ configure_tls_certs() tell_status "installing dovecot TLS certificates" cp /etc/ssl/certs/server.crt "$_ssldir/certs/${TOASTER_MAIL_DOMAIN}.pem" || exit - cat /etc/ssl/dhparam.pem >> "$_ssldir/certs/${TOASTER_MAIL_DOMAIN}.pem" + # sunset after Dovecot 2.3 released + cat /etc/ssl/dhparam.pem >> "$_ssldir/certs/${TOASTER_MAIL_DOMAIN}.pem" || exit + # /sunset cp /etc/ssl/private/server.key "$_ssldir/private/${TOASTER_MAIL_DOMAIN}.pem" || exit - } configure_dovecot() @@ -243,6 +244,42 @@ start_dovecot() stage_exec service dovecot start || exit } +test_imap() +{ + stage_pkg_install empty + + REMOTE_IP=$(get_jail_ip dovecot) + POST_USER="postmaster@$(hostname)" + POST_PASS=$(jexec vpopmail /usr/local/vpopmail/bin/vuserinfo -C "${POST_USER}") + rm -f in out + + #empty -v -f -i in -o out openssl s_client -quiet -crlf -connect $REMOTE_IP:993 + empty -v -f -i in -o out telnet "$REMOTE_IP" 143 + empty -v -w -i out -o in "ready" ". LOGIN $POST_USER $POST_PASS\n" + empty -v -w -i out -o in "Logged in" ". LIST \"\" \"*\"\n" + empty -v -w -i out -o in "List completed" ". SELECT INBOX\n" + empty -v -w -i out -o in "Select completed" ". FETCH 1 BODY\n" + empty -v -w -i out -o in "OK Fetch completed" ". logout\n" + echo "Logout completed" +} + +test_pop3() +{ + stage_pkg_install empty + + REMOTE_IP=$(get_jail_ip dovecot) + POST_USER="postmaster@$(hostname)" + POST_PASS=$(jexec vpopmail /usr/local/vpopmail/bin/vuserinfo -C "${POST_USER}") + rm -f in out + + #empty -v -f -i in -o out openssl s_client -quiet -crlf -connect $REMOTE_IP:995 + empty -v -f -i in -o out telnet "$REMOTE_IP" 110 + empty -v -w -i out -o in "\+OK." "user $POST_USER\n" + empty -v -w -i out -o in "\+OK" "pass $POST_PASS\n" + empty -v -w -i out -o in "OK Logged in" "list\n" + empty -v -w -i out -o in "." "quit\n" +} + test_dovecot() { tell_status "testing dovecot" diff --git a/provision-gitlab-runner.sh b/provision-gitlab-runner.sh new file mode 100755 index 00000000..43075365 --- /dev/null +++ b/provision-gitlab-runner.sh @@ -0,0 +1,98 @@ +#!/bin/sh + +# shellcheck disable=1091 +. mail-toaster.sh || exit + +# https://wiki.freebsd.org/Docker +# https://docs.gitlab.com/runner/install/freebsd.html + +install_gitlab_runner_pkg() +{ + tell_status "installing GitLab Runner package" + stage_pkg_install gitlab-runner +} + +install_gitlab_runner_port() +{ + tell_status "installing GitLab Runner port" + stage_pkg_install dialog4ports go + stage_port_install devel/gitlab-runner || exit +} + +install_gitlab_runner_latest() +{ + tell_status "installing GitLab Runner Latest" + stage_exec fetch -m -o /usr/local/bin/gitlab-runner https://gitlab-ci-multi-runner-downloads.s3.amazonaws.com/latest/binaries/gitlab-ci-multi-runner-freebsd-amd64 + stage_exec chmod +x /usr/local/bin/gitlab-runner +} + +install_docker_freebsd() +{ + # argggg, never mind. After getting FreeBSD up and running inside + # a Docker, I realize that Docker work work for my purposes because + # there's no ZFS pools mounted inside the docker + pkg install docker-freebsd + zfs create -o mountpoint=/usr/docker zroot/docker + pw groupadd docker + pw usermod gitlab-runner -G docker + #proc /proc procfs rw,noauto 0 0 + #mount /proc + #docker run -it auchida/freebsd /bin/sh + sysrc docker_enable=YES + service docker start +} + +install_gitlab_runner() +{ + tell_status "setting up gitlab-runner user" + stage_exec pw group add -n gitlab-runner -m + stage_exec pw user add -n gitlab-runner -g gitlab-runner -s /usr/local/bin/bash + stage_exec mkdir /home/gitlab-runner + stage_exec chown gitlab-runner:gitlab-runner /home/gitlab-runner + touch "$STAGE_MNT/var/log/gitlab_runner.log" + stage_exec chown gitlab-runner:gitlab-runner /var/log/gitlab_runner.log + + install_gitlab_runner_pkg + # Version: 1.11.1 + # Git revision: 08a9e6f + # Git branch: 9-0-stable + + install_gitlab_runner_port + # Version: 9.3.0 + # Git revision: 3df822b + # Git branch: 9-3-stable + + # install_gitlab_runner_latest + # Version: 9.4.2 + # Git revision: 6d06f2e + # Git branch: 9-4-stable +} + +configure_gitlab_runner() +{ + tell_status "configuring GitLab Runner!" + stage_sysrc gitlab_runner_enable=YES + stage_sysrc gitlab_runner_dir=/home/gitlab-runner +} + +start_gitlab_runner() +{ + tell_status "starting up GitLab Runner!" + stage_exec service gitlab_runner start +} + +test_gitlab_runner() +{ + tell_status "testing nothing!" + + echo "it worked" +} + +base_snapshot_exists || exit +create_staged_fs gitlab_runner +start_staged_jail gitlab_runner +install_gitlab_runner +configure_gitlab_runner +start_gitlab_runner +test_gitlab_runner +promote_staged_jail gitlab_runner diff --git a/provision-gitlab.sh b/provision-gitlab.sh new file mode 100755 index 00000000..87da5d41 --- /dev/null +++ b/provision-gitlab.sh @@ -0,0 +1,111 @@ +#!/bin/sh + +# shellcheck disable=1091 +. mail-toaster.sh || exit + +# https://github.com/gitlabhq/gitlab-recipes/blob/master/install/freebsd/freebsd-10.md +# https://github.com/t-zuehlsdorff/gitlabhq/blob/master/doc/install/installation-freebsd.md +# http://gitlab.toco-domains.de/FreeBSD/GitLab-docu/blob/master/install/9.1-freebsd.md + +install_redis() +{ + tell_status "configuring Redis" + + tee -a "$STAGE_MNT/usr/local/etc/redis.conf" < "" ) } extforward.forwarder = ( - "172.16.15.12" => "trust", + "$(get_jail_ip haproxy)" => "trust", ) EO_LIGHTTPD diff --git a/provision-vpopmail.sh b/provision-vpopmail.sh index 2fc0d532..f412fc91 100755 --- a/provision-vpopmail.sh +++ b/provision-vpopmail.sh @@ -133,7 +133,7 @@ install_vpopmail_mysql_grants() | jexec mysql /usr/local/bin/mysql || exit } -install_nrpe() +install_vpop_nrpe() { if [ -z "$TOASTER_NRPE" ]; then echo "TOASTER_NRPE unset, skipping nrpe plugin" @@ -186,7 +186,7 @@ install_vpopmail() fi install_qmailadmin - install_nrpe + install_vpop_nrpe } configure_qmail() diff --git a/provision-webmail.sh b/provision-webmail.sh index 565552d7..291cd656 100755 --- a/provision-webmail.sh +++ b/provision-webmail.sh @@ -76,7 +76,7 @@ $HTTP["url"] =~ "^/cgi-bin" { cgi.assign = ( "" => "" ) } extforward.forwarder = ( - "172.16.15.12" => "trust", + "$(get_jail_ip haproxy)" => "trust", ) EO_LIGHTTPD_MT6 diff --git a/test/vmware.sh b/test/vmware.sh new file mode 100755 index 00000000..7e74eb54 --- /dev/null +++ b/test/vmware.sh @@ -0,0 +1,69 @@ +#!/bin/sh + +# commands: list, start, stop, reset, suspend, pause, unpause +# listSnapshot, snapshot, deleteSnapshot, revertToSnapshot +# runProgramInGuest +# upgradevm, installTools, checkToolsState, deleteVM, clone +# vmrun -T fusion start + +FREEBSD="/Users/Shared/Virtual Machines/FreeBSD 11.vmwarevm" +VERSION="11.1p1" +VMRUN="/Applications/VMware Fusion.app/Contents/Library/vmrun" +GUESTUSER="root" +GUESTPASS="passWord" + +_err() +{ + echo "ERROR: $1" + exit 1 +} + +start() +{ + "$VMRUN" -T fusion start "$FREEBSD" || _err "start failed" + echo "started" +} + +stop() +{ + "$VMRUN" -T fusion stop "$FREEBSD" || _err "start failed" + echo "stopped" +} + +listSnapshots() +{ + "$VMRUN" listSnapshots "$FREEBSD" +} + +runProgramInGuest() +{ + PROG="$1" + if [ -z "$PROG" ]; then + PROG="/home/matt/hi.sh" + fi + + "$VMRUN" -gu "$GUESTUSER" -gp "$GUESTPASS" runProgramInGuest "$FREEBSD" "$PROG" +} + +revertToSnapshot() +{ + "$VMRUN" revertToSnapshot "$FREEBSD" "$1" || _err "revert failed" + echo "reverted to $1" +} + +listProcessesInGuest() +{ + "$VMRUN" -gu "$GUESTUSER" -gp "$GUESTPASS" listProcessesInGuest "$FREEBSD" +} + +cleanstart() { + stop + revertToSnapshot "$VERSION" + start +} + +if [ "$1" = "cleanstart" ]; then + cleanstart +else + echo "$0 cleanstart" +fi