diff --git a/.travis.yml b/.travis.yml index d279c4bc7e4..ea1ffea7b19 100644 --- a/.travis.yml +++ b/.travis.yml @@ -74,8 +74,8 @@ matrix: install: .travis/$JOB install script: .travis/$JOB script - stage: "Stage 1" - if: type IN (push, api, cron) env: JOB=cmake-freebsd + if: type IN (push, api, cron) sudo: required install: .travis/$JOB-stage1 install script: .travis/$JOB-stage1 script @@ -85,8 +85,8 @@ matrix: services: [docker] script: .travis/$JOB - stage: "Stage 2" - if: type IN (push, api, cron) env: JOB=cmake-freebsd + if: type IN (push, api, cron) sudo: required install: .travis/$JOB-stage2 install script: .travis/$JOB-stage2 script diff --git a/.travis/cmake-freebsd-env.sh b/.travis/cmake-freebsd-env.sh new file mode 100644 index 00000000000..8ae7e738a3b --- /dev/null +++ b/.travis/cmake-freebsd-env.sh @@ -0,0 +1,10 @@ +#!/bin/sh + +NPROC=`nproc` + +SCREEN_SESSION=freebsd +SSH_PORT=10022 + +RUN() { + ssh -t -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no root@localhost -p $SSH_PORT "$@" +} diff --git a/other/travis/freebsd-install b/.travis/cmake-freebsd-install.sh similarity index 96% rename from other/travis/freebsd-install rename to .travis/cmake-freebsd-install.sh index 55ae1c4320f..59a78965367 100755 --- a/other/travis/freebsd-install +++ b/.travis/cmake-freebsd-install.sh @@ -30,7 +30,7 @@ mkdir -p /opt/freebsd/cache cd /opt/freebsd/cache # Make sure to update DL_SHA512 when bumping the version -FREEBSD_VERSION="11.1" +FREEBSD_VERSION="11.2" IMAGE_NAME=FreeBSD-${FREEBSD_VERSION}-RELEASE-amd64.raw # Sends keys to the VM as they are @@ -56,7 +56,7 @@ start_vm() # Start emulator. 2000mb RAM should be enough, right? The build machine has over 7gb. screen -L -S $SCREEN_SESSION -d -m \ qemu-system-x86_64 -curses -m 2000 -smp $NPROC \ - -net user,hostfwd=tcp::${SSH_PORT}-:22 -net nic $IMAGE_NAME + -net user,hostfwd=tcp::${SSH_PORT}-:22 -net nic "$IMAGE_NAME" # Wait for the boot screen options wait_for "Autoboot in" @@ -73,7 +73,8 @@ start_vm() stop_vm() { # Turn it off - RUN poweroff + send_keys 'poweroff +' # Wait for qemu process to terminate while ps aux | grep qemu | grep -vq grep diff --git a/.travis/cmake-freebsd-stage1 b/.travis/cmake-freebsd-stage1 index 3954e2af861..d1aeab627c2 100755 --- a/.travis/cmake-freebsd-stage1 +++ b/.travis/cmake-freebsd-stage1 @@ -2,14 +2,174 @@ ACTION="$1" -set -eu +set -eux -. other/travis/env.sh -. other/travis/env-freebsd.sh +. .travis/cmake-freebsd-env.sh travis_install() { - . other/travis/freebsd-install - . other/travis/freebsd-install-stage1 + . .travis/cmake-freebsd-install.sh + + git tag -l --sort=version:refname > GIT_TAGS + + OLD_PWD="$PWD" + + mkdir -p /opt/freebsd/cache + cd /opt/freebsd/cache + + # === Get the VM image, set it up and cache === + + # Create image if it's not cached or if this build script has changed + sha256sum "$OLD_PWD/.travis/cmake-freebsd-env.sh" > /tmp/sha + sha256sum "$OLD_PWD/.travis/cmake-freebsd-install.sh" >> /tmp/sha + sha256sum "$OLD_PWD/.travis/cmake-freebsd-stage1" >> /tmp/sha + if [ ! -f "./$IMAGE_NAME.tgz" ] || [ ! -f ./cmake-freebsd-stage1-all.sha256 ] || [ "`cat cmake-freebsd-stage1-all.sha256`" != "`cat /tmp/sha`" ]; then + + rm -rf ./* + + # https://download.freebsd.org/ftp/releases/VM-IMAGES/11.2-RELEASE/amd64/Latest/ + DL_SHA512="0c3c232c7023c5036daeb5fbf68c2ddecf9703c74e317afcf19da91e83d0afcc526785571e2868894ce15cdb56b74fafa1ce9fd216469db91e021ac2ef8911e5" + # Selecting random mirror from https://www.freebsd.org/doc/handbook/mirrors-ftp.html + # Note that not all mirrors listed on that page are working, so we have removed them + # I'm so sorry, there are no arrays in sh and we are not using bash... + DL_MIRROR_1=1 + DL_MIRROR_2=4 + # There are 2 mirrors + DL_MIRROR_RANDOM=`expr $(date +%s) % 2 + 1` + DL_URL="ftp://ftp$(eval echo \$DL_MIRROR_$DL_MIRROR_RANDOM).us.freebsd.org/pub/FreeBSD/releases/VM-IMAGES/${FREEBSD_VERSION}-RELEASE/amd64/Latest/${IMAGE_NAME}.xz" + + wget "$DL_URL" + + if ! ( echo "$DL_SHA512 $IMAGE_NAME.xz" | sha512sum -c --status - ) ; then + echo "Error: sha512 of $IMAGE_NAME.xz doesn't match the known one" + exit 1 + fi + + unxz "$IMAGE_NAME.xz" + + # With this we don't have to guess how long a command will run for and sleeping + # for that amount of time, risking either under sleeping or over sleeping, instead + # we will sleep exactly until the command is finished by printing out a unique + # string after the command is executed and then checking if it was printed. + execute_shell_and_wait() + { + # $RANDOM is a bash built-in, so we try to avoid name collision here by using ugly RANDOM_STR name + RANDOM_STR=$(< /dev/urandom tr -dc _A-Za-z0-9 | head -c16) + send_keys "$1;echo $RANDOM_STR + +" + # \[1B is a control escape sequence for a new line in the terminal. + # We want to wait for $RANDOM_STR instead of just $RANDOM_STR because + # $RANDOM_STR we have inputted with send_keys above would appear in the screenlog.0 + # file and we don't want to match our input, we want to match the echo's output. + # The .\? optionally matches any character. Sometimes it happens that there is some + # random character inserved between the new line control escape sequence and $RANDOM_STR. + wait_for "\[1B.\?$RANDOM_STR" + } + + start_vm + + # Login as root user + send_keys 'root + +' + + # Wait for the prompt + wait_for "root@.*:~" + + # Configure network, ssh and start changing password + execute_shell_and_wait 'echo "ifconfig_em0=DHCP" >> /etc/rc.conf' + execute_shell_and_wait 'echo "Port 22" >> /etc/ssh/sshd_config' + execute_shell_and_wait 'echo "PermitRootLogin yes" >> /etc/ssh/sshd_config' + execute_shell_and_wait 'echo "PasswordAuthentication yes" >> /etc/ssh/sshd_config' + execute_shell_and_wait 'echo "PermitEmptyPasswords yes" >> /etc/ssh/sshd_config' + execute_shell_and_wait 'echo "sshd_enable=YES" >> /etc/rc.conf' + send_keys 'sh /etc/rc.d/netif restart && sh /etc/rc.d/sshd start && passwd +' + + # Wait for the password prompt + wait_for "Changing local password for root" + + # Reset password to empty for the passwordless ssh to work + send_keys ' +' + wait_for "New Password" + send_keys ' +' + + # Update system + RUN env PAGER=cat env ASSUME_ALWAYS_YES=YES freebsd-update --not-running-from-cron fetch + # It fails if there is nothing to install, so we make it always succeed with true + RUN env PAGER=cat env ASSUME_ALWAYS_YES=YES freebsd-update --not-running-from-cron install || true + + # Update packages + RUN env PAGER=cat env ASSUME_ALWAYS_YES=YES pkg upgrade + + # Install and set bash as the default shell for the root user + RUN env PAGER=cat env ASSUME_ALWAYS_YES=YES pkg install bash + RUN chsh -s /usr/local/bin/bash root + + # Install required toxcore dependencies + RUN PAGER=cat ASSUME_ALWAYS_YES=YES pkg install git \ + opus \ + libvpx \ + libsodium \ + gmake \ + cmake \ + pkgconf \ + opencv \ + portaudio \ + libsndfile \ + texinfo \ + autotools + + # === Cache the VM image === + + stop_vm + + # Create cache + tar -Sczvf "$IMAGE_NAME.tgz" "$IMAGE_NAME" + rm "$IMAGE_NAME" + rm screenlog.0 + + cp "$OLD_PWD/GIT_TAGS" . + sha256sum "$OLD_PWD/.travis/cmake-freebsd-env.sh" > cmake-freebsd-stage1-all.sha256 + sha256sum "$OLD_PWD/.travis/cmake-freebsd-install.sh" >> cmake-freebsd-stage1-all.sha256 + sha256sum "$OLD_PWD/.travis/cmake-freebsd-stage1" >> cmake-freebsd-stage1-all.sha256 + + ls -lh + fi + + # === Update the image on new version (tag) of toxcore === + + if ! diff -u ./GIT_TAGS "$OLD_PWD/GIT_TAGS" ; then + # Extract the cached image + tar -Sxzvf "$IMAGE_NAME.tgz" + + start_vm + + # Update system + RUN PAGER=cat ASSUME_ALWAYS_YES=YES freebsd-update --not-running-from-cron fetch + RUN PAGER=cat ASSUME_ALWAYS_YES=YES freebsd-update --not-running-from-cron install || true + + # Update packages + RUN PAGER=cat ASSUME_ALWAYS_YES=YES pkg upgrade + + # === Cache the updated VM image === + + stop_vm + + # Create/Update cache + rm "$IMAGE_NAME.tgz" + tar -Sczvf "$IMAGE_NAME.tgz" "$IMAGE_NAME" + rm "$IMAGE_NAME" + rm screenlog.0 + + cp "$OLD_PWD/GIT_TAGS" . + + ls -lh + fi + + cd "$OLD_PWD" } travis_script() { diff --git a/.travis/cmake-freebsd-stage2 b/.travis/cmake-freebsd-stage2 index 57b8a0f8f65..92e681f001f 100755 --- a/.travis/cmake-freebsd-stage2 +++ b/.travis/cmake-freebsd-stage2 @@ -2,18 +2,71 @@ ACTION="$1" -set -eu +set -eux -. other/travis/env.sh -. other/travis/env-freebsd.sh +. .travis/cmake-freebsd-env.sh travis_install() { - . other/travis/freebsd-install - . other/travis/freebsd-install-stage2 + . .travis/cmake-freebsd-install.sh + + OLD_PWD="$PWD" + + mkdir -p /opt/freebsd/cache + cd /opt/freebsd/cache + + # Extract the cached image + tar -Sxzvf "$IMAGE_NAME.tgz" + + # Get the image we will be using out of the cached directory + mv "$IMAGE_NAME" .. + ls -lh + + cd .. + + ls -lh + + # === Get VM ready to build the code === + + start_vm + + # Display FreeBSD kernel info and last login + RUN uname -a + RUN last + + cd "$OLD_PWD" + + # Copy over toxcore code from Travis to qemu + scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -P $SSH_PORT -r ./* root@localhost:~ + + RUN ls -lh } travis_script() { - . other/travis/toxcore-script + CACHEDIR="none" + . ".travis/flags-$CC.sh" + + add_ld_flag -Wl,-z,defs + + # Make compilation error on a warning + add_flag -Werror + + RUN 'cmake -B_build -H. \ + -DCMAKE_C_FLAGS="$C_FLAGS" \ + -DCMAKE_CXX_FLAGS="$CXX_FLAGS" \ + -DCMAKE_EXE_LINKER_FLAGS="$LD_FLAGS" \ + -DCMAKE_SHARED_LINKER_FLAGS="$LD_FLAGS" \ + -DCMAKE_INSTALL_PREFIX:PATH="_install" \ + -DTRACE=ON \ + -DMUST_BUILD_TOXAV=ON \ + -DSTRICT_ABI=ON \ + -DTEST_TIMEOUT_SECONDS=120 \ + -DUSE_IPV6=OFF' + + # We created the VM with the same number of cores as the host, so the host-ran `nproc` here is fine + RUN 'gmake "-j$NPROC" -k install -C_build' + RUN 'gmake "-j$NPROC" test ARGS="-j50" -C_build || \ + gmake "-j$NPROC" -C_build test ARGS="-j50 --rerun-failed" CTEST_OUTPUT_ON_FAILURE=1 || \ + true' } if [ "-z" "$ACTION" ]; then diff --git a/other/travis/env-freebsd.sh b/other/travis/env-freebsd.sh deleted file mode 100644 index 882e9da56c5..00000000000 --- a/other/travis/env-freebsd.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/sh - -CMAKE=cmake -NPROC=`nproc` -CURDIR=/root -RUN_TESTS=true -MAKE=gmake -# A lot of tests fail and run for the full 2 minutes allowed, resulting in -# Travis build timing out, so we restrict it to just 1 test run until enough -# tests are fixed so that they succeed and don't run the full 2 minutes. -MAX_TEST_RETRIES=1 - -SCREEN_SESSION=freebsd -SSH_PORT=10022 - -RUN() { - ssh -t -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no root@localhost -p $SSH_PORT "$@" -} - -TESTS() { - COUNT="$1"; shift - RUN "$@" || { - if [ $COUNT -gt 1 ]; then - TESTS `expr $COUNT - 1` "$@" - else - # FIXME: We allow the tests to fail for now, but this should be changed to - # "false" once we fix tests under FreeBSD - true - fi - } -} diff --git a/other/travis/env.sh b/other/travis/env.sh deleted file mode 100644 index 2d3e646c556..00000000000 --- a/other/travis/env.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/sh - -# Globally used environment variables. -export CACHE_DIR=$HOME/cache -export OPAMROOT=$CACHE_DIR/.opam -export LD_LIBRARY_PATH=$CACHE_DIR/lib -export PKG_CONFIG_PATH=$CACHE_DIR/lib/pkgconfig -export ASTYLE=$CACHE_DIR/astyle/build/gcc/bin/astyle -export CFLAGS="-O3 -DTRAVIS_ENV=1" -export CXXFLAGS="-O3 -DTRAVIS_ENV=1" -export MAKE=make - -BUILD_DIR=_build -MAX_TEST_RETRIES=10 - -# Workaround for broken Travis image. -export TERM=xterm diff --git a/other/travis/freebsd-install-stage1 b/other/travis/freebsd-install-stage1 deleted file mode 100755 index 46b403743e2..00000000000 --- a/other/travis/freebsd-install-stage1 +++ /dev/null @@ -1,164 +0,0 @@ -#!/bin/sh - -git tag -l --sort=version:refname > GIT_TAGS - -OLD_PWD="$PWD" - -mkdir -p /opt/freebsd/cache -cd /opt/freebsd/cache - -# === Get the VM image, set it up and cache === - -# Create image if it's not cached or if this build script has changed -if [ ! -f ./$IMAGE_NAME.tgz ] || [ ! -f ./freebsd-install-stage1.sha256 ] || [ "`cat freebsd-install-stage1.sha256`" != "`sha256sum $OLD_PWD/other/travis/freebsd-install-stage1`" ]; then - - rm -rf ./* - - # https://download.freebsd.org/ftp/releases/VM-IMAGES/11.1-RELEASE/amd64/Latest/ - DL_SHA512="c569776334131fdc85cd25a2a0d5aecafdc3e4b2e6e010dffaa2488d934293ce4f091f23481079dd91ad20dfd2dfc3d3487707096c59448f1d8914c5d7d6b582" - # Selecting random mirror from https://www.freebsd.org/doc/handbook/mirrors-ftp.html - # Note that not all mirrors listed on that page are working, so we have removed them - # I'm so sorry, there are no arrays in sh and we are not using bash... - DL_MIRROR_1=1 - DL_MIRROR_2=4 - DL_MIRROR_3=5 - DL_MIRROR_4=6 - DL_MIRROR_5=8 - DL_MIRROR_6=10 - DL_MIRROR_7=14 - DL_MIRROR_8=15 - # There are 8 mirrors - DL_MIRROR_RANDOM=`expr $(date +%s) % 8 + 1` - DL_URL=ftp://ftp$(eval echo \$DL_MIRROR_$DL_MIRROR_RANDOM).us.freebsd.org/pub/FreeBSD/releases/VM-IMAGES/${FREEBSD_VERSION}-RELEASE/amd64/Latest/${IMAGE_NAME}.xz - - wget $DL_URL - - if ! ( echo "$DL_SHA512 $IMAGE_NAME.xz" | sha512sum -c --status - ) ; then - echo "Error: sha512 of $IMAGE_NAME.xz doesn't match the known one" - exit 1 - fi - - unxz $IMAGE_NAME.xz - - # With this we don't have to guess how long a command will run for and sleeping - # for that amount of time, risking either under sleeping or over sleeping, instead - # we will sleep exactly until the command is finished by printing out a unique - # string after the command is executed and then checking if it was printed. - execute_shell_and_wait() - { - # $RANDOM is a bash built-in, so we try to avoid name collision here by using ugly RANDOM_STR name - RANDOM_STR=$(< /dev/urandom tr -dc _A-Za-z0-9 | head -c16) - send_keys "$1;echo $RANDOM_STR - -" - # \[1B is a control escape sequence for a new line in the terminal. - # We want to wait for $RANDOM_STR instead of just $RANDOM_STR because - # $RANDOM_STR we have inputted with send_keys above would appear in the screenlog.0 - # file and we don't want to match our input, we want to match the echo's output. - # The .\? optionally matches any character. Sometimes it happens that there is some - # random character inserved between the new line control escape sequence and $RANDOM_STR. - wait_for "\[1B.\?$RANDOM_STR" - } - - start_vm - - # Login as root user - send_keys 'root - -' - - # Wait for the prompt - wait_for "root@:~" - - # Configure network, ssh and start changing password - execute_shell_and_wait 'echo "ifconfig_em0=DHCP" >> /etc/rc.conf' - execute_shell_and_wait 'echo "Port 22" >> /etc/ssh/sshd_config' - execute_shell_and_wait 'echo "PermitRootLogin yes" >> /etc/ssh/sshd_config' - execute_shell_and_wait 'echo "PasswordAuthentication yes" >> /etc/ssh/sshd_config' - execute_shell_and_wait 'echo "PermitEmptyPasswords yes" >> /etc/ssh/sshd_config' - execute_shell_and_wait 'echo "sshd_enable=YES" >> /etc/rc.conf' - send_keys 'sh /etc/rc.d/netif restart && sh /etc/rc.d/sshd start && passwd -' - - # Wait for the password prompt - wait_for "Changing local password for root" - - # Reset password to empty for the passwordless ssh to work - send_keys ' -' - wait_for "New Password" - send_keys ' -' - - # Update system - RUN env PAGER=cat env ASSUME_ALWAYS_YES=YES freebsd-update --not-running-from-cron fetch - # It fails if there is nothing to install, so we make it always succeed with true - RUN env PAGER=cat env ASSUME_ALWAYS_YES=YES freebsd-update --not-running-from-cron install || true - - # Update packages - RUN env PAGER=cat env ASSUME_ALWAYS_YES=YES pkg upgrade - - # Install and set bash as the default shell for the root user - RUN env PAGER=cat env ASSUME_ALWAYS_YES=YES pkg install bash - RUN chsh -s /usr/local/bin/bash root - - # Install required toxcore dependencies - RUN PAGER=cat ASSUME_ALWAYS_YES=YES pkg install git \ - opus \ - libvpx \ - libsodium \ - gmake \ - cmake \ - pkgconf \ - opencv \ - portaudio \ - libsndfile \ - texinfo \ - autotools - - # === Cache the VM image === - - stop_vm - - # Create cache - tar -Sczvf $IMAGE_NAME.tgz $IMAGE_NAME - rm $IMAGE_NAME - rm screenlog.0 - - cp "$OLD_PWD/GIT_TAGS" . - sha256sum "$OLD_PWD/other/travis/freebsd-install-stage1" > freebsd-install-stage1.sha256 - - ls -lh -fi - -# === Update the image on new version (tag) of toxcore === - -if ! diff -u ./GIT_TAGS "$OLD_PWD/GIT_TAGS" ; then - # Extract the cached image - tar -Sxzvf $IMAGE_NAME.tgz - - start_vm - - # Update system - RUN PAGER=cat ASSUME_ALWAYS_YES=YES freebsd-update --not-running-from-cron fetch - RUN PAGER=cat ASSUME_ALWAYS_YES=YES freebsd-update --not-running-from-cron install || true - - # Update packages - RUN PAGER=cat ASSUME_ALWAYS_YES=YES pkg upgrade - - # === Cache the updated VM image === - - stop_vm - - # Create/Update cache - rm $IMAGE_NAME.tgz - tar -Sczvf $IMAGE_NAME.tgz $IMAGE_NAME - rm $IMAGE_NAME - rm screenlog.0 - - cp "$OLD_PWD/GIT_TAGS" . - - ls -lh -fi - -cd "$OLD_PWD" diff --git a/other/travis/freebsd-install-stage2 b/other/travis/freebsd-install-stage2 deleted file mode 100755 index 61f8d013761..00000000000 --- a/other/travis/freebsd-install-stage2 +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/sh - -OLD_PWD="$PWD" - -mkdir -p /opt/freebsd/cache -cd /opt/freebsd/cache - -# Extract the cached image -tar -Sxzvf $IMAGE_NAME.tgz - -# Get the image we will be using out of the cached directory -mv $IMAGE_NAME .. -ls -lh - -cd .. - -ls -lh - -# === Get VM ready to build the code === - -start_vm - -# Display FreeBSD kernel info and last login -RUN uname -a -RUN last - -cd "$OLD_PWD" - -# Copy over toxcore code from Travis to qemu -scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -P $SSH_PORT -r ./* root@localhost:~ - -RUN ls -lh diff --git a/other/travis/toxcore-script b/other/travis/toxcore-script deleted file mode 100755 index ae2af416afb..00000000000 --- a/other/travis/toxcore-script +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/sh - -# Enable test coverage recording. -export CFLAGS="$CFLAGS -fprofile-arcs -ftest-coverage" -export CXXFLAGS="$CXXFLAGS -fprofile-arcs -ftest-coverage" - -set_opt() { - opts="$opts -D$1="`expr ON \& \( $i % 2 \) \| OFF` - set +e # result can be 0 - i=`expr $i / 2` - set -e -} - -if perl -e ' - use Socket qw(PF_INET6 SOCK_STREAM pack_sockaddr_in6 IN6ADDR_LOOPBACK); - - socket(S, PF_INET6, SOCK_STREAM, 0) || exit 1; - bind(S, pack_sockaddr_in6(0, IN6ADDR_LOOPBACK)) || exit 1; -'; then - USE_IPV6=yes -else - USE_IPV6=no -fi - - -# Build toxcore and run tests. -RUN $CMAKE \ - -B$BUILD_DIR \ - -H. \ - -DCMAKE_INSTALL_PREFIX:PATH=$CURDIR/_install \ - -DDEBUG=ON \ - -DMUST_BUILD_TOXAV=ON \ - -DSTRICT_ABI=ON \ - -DTEST_TIMEOUT_SECONDS=120 \ - -DTRACE=ON \ - -DUSE_IPV6=$USE_IPV6 - -export CTEST_OUTPUT_ON_FAILURE=1 - -RUN $MAKE -C$BUILD_DIR -j$NPROC -k install -if $RUN_TESTS; then - TESTS $MAX_TEST_RETRIES $MAKE -C$BUILD_DIR -j$NPROC test ARGS="-j50 --rerun-failed" CTEST_OUTPUT_ON_FAILURE="$CTEST_OUTPUT_ON_FAILURE" -fi - -if [ -f "$ASTYLE" ]; then - other/astyle/format-source . $ASTYLE -fi