Simplify Travis-CI FreeBSD build

.travis.yml

@@ -74,8 +74,8 @@ matrix:
     install:       .travis/$JOB install
     script:        .travis/$JOB script
   - stage: "Stage 1"
-    if: type IN (push, api, cron)
     env: JOB=cmake-freebsd
+    if: type IN (push, api, cron)
     sudo: required
     install:       .travis/$JOB-stage1 install
     script:        .travis/$JOB-stage1 script
@@ -85,8 +85,8 @@ matrix:
     services: [docker]
     script:        .travis/$JOB
   - stage: "Stage 2"
-    if: type IN (push, api, cron)
     env: JOB=cmake-freebsd
+    if: type IN (push, api, cron)
     sudo: required
     install:       .travis/$JOB-stage2 install
     script:        .travis/$JOB-stage2 script

.travis/cmake-freebsd-env.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+
+NPROC=`nproc`
+
+SCREEN_SESSION=freebsd
+SSH_PORT=10022
+
+RUN() {
+  ssh -t -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no root@localhost -p $SSH_PORT "$@"
+}

other/travis/freebsd-install → .travis/cmake-freebsd-install.sh

@@ -30,7 +30,7 @@ mkdir -p /opt/freebsd/cache
 cd /opt/freebsd/cache
 
 # Make sure to update DL_SHA512 when bumping the version
-FREEBSD_VERSION="11.1"
+FREEBSD_VERSION="11.2"
 IMAGE_NAME=FreeBSD-${FREEBSD_VERSION}-RELEASE-amd64.raw
 
 # Sends keys to the VM as they are
@@ -56,7 +56,7 @@ start_vm()
   # Start emulator. 2000mb RAM should be enough, right? The build machine has over 7gb.
   screen -L -S $SCREEN_SESSION -d -m \
     qemu-system-x86_64 -curses -m 2000 -smp $NPROC \
-    -net user,hostfwd=tcp::${SSH_PORT}-:22 -net nic $IMAGE_NAME
+    -net user,hostfwd=tcp::${SSH_PORT}-:22 -net nic "$IMAGE_NAME"
 
   # Wait for the boot screen options
   wait_for "Autoboot in"
@@ -73,7 +73,8 @@ start_vm()
 stop_vm()
 {
   # Turn it off
-  RUN poweroff
+  send_keys 'poweroff
+'
 
   # Wait for qemu process to terminate
   while ps aux | grep qemu | grep -vq grep

.travis/cmake-freebsd-stage1

@@ -2,14 +2,174 @@
 
 ACTION="$1"
 
-set -eu
+set -eux
 
-. other/travis/env.sh
-. other/travis/env-freebsd.sh
+. .travis/cmake-freebsd-env.sh
 
 travis_install() {
-  . other/travis/freebsd-install
-  . other/travis/freebsd-install-stage1
+  . .travis/cmake-freebsd-install.sh
+
+  git tag -l --sort=version:refname > GIT_TAGS
+
+  OLD_PWD="$PWD"
+
+  mkdir -p /opt/freebsd/cache
+  cd /opt/freebsd/cache
+
+  # === Get the VM image, set it up and cache ===
+
+  # Create image if it's not cached or if this build script has changed
+  sha256sum "$OLD_PWD/.travis/cmake-freebsd-env.sh"      > /tmp/sha
+  sha256sum "$OLD_PWD/.travis/cmake-freebsd-install.sh" >> /tmp/sha
+  sha256sum "$OLD_PWD/.travis/cmake-freebsd-stage1"     >> /tmp/sha
+  if [ ! -f "./$IMAGE_NAME.tgz" ] || [ ! -f ./cmake-freebsd-stage1-all.sha256 ] || [ "`cat cmake-freebsd-stage1-all.sha256`" != "`cat /tmp/sha`" ]; then
+
+    rm -rf ./*
+
+    # https://download.freebsd.org/ftp/releases/VM-IMAGES/11.2-RELEASE/amd64/Latest/
+    DL_SHA512="0c3c232c7023c5036daeb5fbf68c2ddecf9703c74e317afcf19da91e83d0afcc526785571e2868894ce15cdb56b74fafa1ce9fd216469db91e021ac2ef8911e5"
+    # Selecting random mirror from https://www.freebsd.org/doc/handbook/mirrors-ftp.html
+    # Note that not all mirrors listed on that page are working, so we have removed them
+    # I'm so sorry, there are no arrays in sh and we are not using bash...
+    DL_MIRROR_1=1
+    DL_MIRROR_2=4
+    # There are 2 mirrors
+    DL_MIRROR_RANDOM=`expr $(date +%s) % 2 + 1`
+    DL_URL="ftp://ftp$(eval echo \$DL_MIRROR_$DL_MIRROR_RANDOM).us.freebsd.org/pub/FreeBSD/releases/VM-IMAGES/${FREEBSD_VERSION}-RELEASE/amd64/Latest/${IMAGE_NAME}.xz"
+
+    wget "$DL_URL"
+
+    if ! ( echo "$DL_SHA512  $IMAGE_NAME.xz" | sha512sum -c --status - ) ; then
+        echo "Error: sha512 of $IMAGE_NAME.xz doesn't match the known one"
+        exit 1
+    fi
+
+    unxz "$IMAGE_NAME.xz"
+
+    # With this we don't have to guess how long a command will run for and sleeping
+    # for that amount of time, risking either under sleeping or over sleeping, instead
+    # we will sleep exactly until the command is finished by printing out a unique
+    # string after the command is executed and then checking if it was printed.
+    execute_shell_and_wait()
+    {
+      # $RANDOM is a bash built-in, so we try to avoid name collision here by using ugly RANDOM_STR name
+      RANDOM_STR=$(< /dev/urandom tr -dc _A-Za-z0-9 | head -c16)
+      send_keys "$1;echo $RANDOM_STR
+
+"
+      # \[1B is a control escape sequence for a new line in the terminal.
+      # We want to wait for <new-line>$RANDOM_STR instead of just $RANDOM_STR because
+      # $RANDOM_STR we have inputted with send_keys above would appear in the screenlog.0
+      # file and we don't want to match our input, we want to match the echo's output.
+      # The .\? optionally matches any character. Sometimes it happens that there is some
+      # random character inserved between the new line control escape sequence and $RANDOM_STR.
+      wait_for "\[1B.\?$RANDOM_STR"
+    }
+
+    start_vm
+
+    # Login as root user
+    send_keys 'root
+
+'
+
+    # Wait for the prompt
+    wait_for "root@.*:~"
+
+    # Configure network, ssh and start changing password
+    execute_shell_and_wait 'echo "ifconfig_em0=DHCP"          >> /etc/rc.conf'
+    execute_shell_and_wait 'echo "Port 22"                    >> /etc/ssh/sshd_config'
+    execute_shell_and_wait 'echo "PermitRootLogin yes"        >> /etc/ssh/sshd_config'
+    execute_shell_and_wait 'echo "PasswordAuthentication yes" >> /etc/ssh/sshd_config'
+    execute_shell_and_wait 'echo "PermitEmptyPasswords yes"   >> /etc/ssh/sshd_config'
+    execute_shell_and_wait 'echo "sshd_enable=YES"            >> /etc/rc.conf'
+    send_keys 'sh /etc/rc.d/netif restart && sh /etc/rc.d/sshd start && passwd
+'
+
+    # Wait for the password prompt
+    wait_for "Changing local password for root"
+
+    # Reset password to empty for the passwordless ssh to work
+    send_keys '
+'
+    wait_for "New Password"
+    send_keys '
+'
+
+    # Update system
+    RUN env PAGER=cat env ASSUME_ALWAYS_YES=YES freebsd-update --not-running-from-cron fetch
+    # It fails if there is nothing to install, so we make it always succeed with true
+    RUN env PAGER=cat env ASSUME_ALWAYS_YES=YES freebsd-update --not-running-from-cron install || true
+
+    # Update packages
+    RUN env PAGER=cat env ASSUME_ALWAYS_YES=YES pkg upgrade
+
+    # Install and set bash as the default shell for the root user
+    RUN env PAGER=cat env ASSUME_ALWAYS_YES=YES pkg install bash
+    RUN chsh -s /usr/local/bin/bash root
+
+    # Install required toxcore dependencies
+    RUN PAGER=cat ASSUME_ALWAYS_YES=YES pkg install git \
+                                                    opus \
+                                                    libvpx \
+                                                    libsodium \
+                                                    gmake \
+                                                    cmake \
+                                                    pkgconf \
+                                                    opencv \
+                                                    portaudio \
+                                                    libsndfile \
+                                                    texinfo \
+                                                    autotools
+
+    # === Cache the VM image ===
+
+    stop_vm
+
+    # Create cache
+    tar -Sczvf "$IMAGE_NAME.tgz" "$IMAGE_NAME"
+    rm "$IMAGE_NAME"
+    rm screenlog.0
+
+    cp "$OLD_PWD/GIT_TAGS" .
+    sha256sum "$OLD_PWD/.travis/cmake-freebsd-env.sh"      > cmake-freebsd-stage1-all.sha256
+    sha256sum "$OLD_PWD/.travis/cmake-freebsd-install.sh" >> cmake-freebsd-stage1-all.sha256
+    sha256sum "$OLD_PWD/.travis/cmake-freebsd-stage1"     >> cmake-freebsd-stage1-all.sha256
+
+    ls -lh
+  fi
+
+  # === Update the image on new version (tag) of toxcore ===
+
+  if ! diff -u ./GIT_TAGS "$OLD_PWD/GIT_TAGS" ; then
+    # Extract the cached image
+    tar -Sxzvf "$IMAGE_NAME.tgz"
+
+    start_vm
+
+    # Update system
+    RUN PAGER=cat ASSUME_ALWAYS_YES=YES freebsd-update --not-running-from-cron fetch
+    RUN PAGER=cat ASSUME_ALWAYS_YES=YES freebsd-update --not-running-from-cron install || true
+
+    # Update packages
+    RUN PAGER=cat ASSUME_ALWAYS_YES=YES pkg upgrade
+
+    # === Cache the updated VM image ===
+
+    stop_vm
+
+    # Create/Update cache
+    rm "$IMAGE_NAME.tgz"
+    tar -Sczvf "$IMAGE_NAME.tgz" "$IMAGE_NAME"
+    rm "$IMAGE_NAME"
+    rm screenlog.0
+
+    cp "$OLD_PWD/GIT_TAGS" .
+
+    ls -lh
+  fi
+
+  cd "$OLD_PWD"
 }
 
 travis_script() {

.travis/cmake-freebsd-stage2

@@ -2,18 +2,71 @@
 
 ACTION="$1"
 
-set -eu
+set -eux
 
-. other/travis/env.sh
-. other/travis/env-freebsd.sh
+. .travis/cmake-freebsd-env.sh
 
 travis_install() {
-  . other/travis/freebsd-install
-  . other/travis/freebsd-install-stage2
+  . .travis/cmake-freebsd-install.sh
+
+  OLD_PWD="$PWD"
+
+  mkdir -p /opt/freebsd/cache
+  cd /opt/freebsd/cache
+
+  # Extract the cached image
+  tar -Sxzvf "$IMAGE_NAME.tgz"
+
+  # Get the image we will be using out of the cached directory
+  mv "$IMAGE_NAME" ..
+  ls -lh
+
+  cd ..
+
+  ls -lh
+
+  # === Get VM ready to build the code ===
+
+  start_vm
+
+  # Display FreeBSD kernel info and last login
+  RUN uname -a
+  RUN last
+
+  cd "$OLD_PWD"
+
+  # Copy over toxcore code from Travis to qemu
+  scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -P $SSH_PORT -r ./* root@localhost:~
+
+  RUN ls -lh
 }
 
 travis_script() {
-  . other/travis/toxcore-script
+  CACHEDIR="none"
+  . ".travis/flags-$CC.sh"
+
+  add_ld_flag -Wl,-z,defs
+
+  # Make compilation error on a warning
+  add_flag -Werror
+
+  RUN 'cmake -B_build -H. \
+    -DCMAKE_C_FLAGS="$C_FLAGS" \
+    -DCMAKE_CXX_FLAGS="$CXX_FLAGS" \
+    -DCMAKE_EXE_LINKER_FLAGS="$LD_FLAGS" \
+    -DCMAKE_SHARED_LINKER_FLAGS="$LD_FLAGS" \
+    -DCMAKE_INSTALL_PREFIX:PATH="_install" \
+    -DTRACE=ON \
+    -DMUST_BUILD_TOXAV=ON \
+    -DSTRICT_ABI=ON \
+    -DTEST_TIMEOUT_SECONDS=120 \
+    -DUSE_IPV6=OFF'
+
+  # We created the VM with the same number of cores as the host, so the host-ran `nproc` here is fine
+  RUN 'gmake "-j$NPROC" -k install -C_build'
+  RUN 'gmake "-j$NPROC" test ARGS="-j50" -C_build || \
+    gmake "-j$NPROC" -C_build test ARGS="-j50 --rerun-failed" CTEST_OUTPUT_ON_FAILURE=1 || \
+    true'
 }
 
 if [ "-z" "$ACTION" ]; then