diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d1063dcd5a515..aa85e7c41335e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -40,7 +40,7 @@ default: # submodules (a commit in this repo does not affect their builds anyway) and # many components that are otherwise slow to build. variables: - CMAKE_FLAGS: "-DWITH_SSL=system -DPLUGIN_COLUMNSTORE=NO -DPLUGIN_ROCKSDB=NO -DPLUGIN_S3=NO -DPLUGIN_MROONGA=NO -DPLUGIN_CONNECT=NO -DPLUGIN_MROONGA=NO -DPLUGIN_TOKUDB=NO -DPLUGIN_PERFSCHEMA=NO -DWITH_WSREP=OFF" + CMAKE_FLAGS: "-DWITH_SSL=system -DPLUGIN_COLUMNSTORE=NO -DPLUGIN_ROCKSDB=NO -DPLUGIN_S3=NO -DPLUGIN_MROONGA=NO -DPLUGIN_CONNECT=NO -DPLUGIN_TOKUDB=NO -DWITH_WSREP=OFF" # Major version dictates which branches share the same ccache. E.g. 10.6-abc # and 10.6-xyz will have the same cache. MARIADB_MAJOR_VERSION: "11.1" @@ -580,6 +580,43 @@ flawfinder: - flawfinder-all-vulnerabilities.html - flawfinder-min-level5.json +fedora upgrade-from: + stage: test + image: fedora:latest + variables: + GIT_STRATEGY: pull + dependencies: + - fedora + needs: + - fedora + parallel: + matrix: + # Get latest versions of all major versions + - UPGRADE_PATH: + - "10.6>11.0" + - "10.3" + - "10.4" + - "10.4.8" # Test unexpected table rebuilds as in https://jira.mariadb.org/browse/MDEV-28727 + - "10.5" + - "10.6" + - "10.11" + - "11.0" + script: + - | + if [[ $UPGRADE_PATH == *">"* ]]; then + SOURCE=$(echo "$UPGRADE_PATH" | cut -d'>' -f1) + TARGET=$(echo "$UPGRADE_PATH" | cut -d'>' -f2) + else + SOURCE=$(echo "$UPGRADE_PATH") + fi + + # Test upgrade from earlier minor version + - ./tests/upgrade_from/test_upgrade.sh $SOURCE $TARGET + artifacts: + paths: + - "*.sql" + - "test_upgrade*.log" + # Once all RPM builds and tests have passed, also run the DEB builds and tests # @NOTE: This is likely to work well only on salsa.debian.org as the Gitlab.com # runners are too small for everything this stage does. diff --git a/tests/upgrade_from/test_upgrade.sh b/tests/upgrade_from/test_upgrade.sh new file mode 100755 index 0000000000000..e356231afc8e7 --- /dev/null +++ b/tests/upgrade_from/test_upgrade.sh @@ -0,0 +1,276 @@ +#!/bin/bash + +# This script installs an existing version of MariaDB from the archives, +# adds the appropriate users, starts the server, and stores the database +# system files before and after mariadb-upgrade is executed. +# mariadb-upgrade should do nothing since the minor version is the same. + +# Adapted from: https://mariadb.com/kb/en/installing-mariadb-binary-tarballs/#installing-mariadb-as-root-in-usrlocalmysql + +# Exit on error +set -e -o pipefail + +GREEN='\033[0;32m' +NC='\033[0m' # No Color + +# Initialize variables +source_version="" +rpm_dir="" +target_version="" + +function usage { + echo Usage: + echo " $0 source_version [target-version] [--rpm-dir rpm/]" + echo " (target_version and --rpm-dir are mutually exclusive)" + +} + +rpm_dir="" +# Parse long arguments +while [[ $# -gt 0 ]]; do + case "$1" in + --rpm-dir) + rpm_dir="$2" + shift 2 + ;; + *) + break + ;; + esac +done + +# Parse positional arguments +if [[ $# -lt 1 ]]; then + echo "Missing required positional argument: source_version" + usage + exit 1 +fi + +source_version="$1" +shift + +if [[ $# -gt 0 ]]; then + target_version="$1" + shift +fi + +if [[ -n "$rpm_dir" && -n "$target_version" ]]; then + usage + exit 1 +fi + +# Check for extra arguments +if [[ $# -gt 0 ]]; then + echo "Unknown arguments: $*" + usage + exit 1 +fi + +function log { + local text="$*" + echo -e "${GREEN}${text}${NC}" +} + +if [[ -z "$target_version" ]]; then + echo "No target version specified" + if [[ -z "$rpm_dir" ]]; then + echo "Using default RPM directory" + rpm_dir="rpm/" + fi + if [ ! -d $rpm_dir ]; then + echo "Error: directory $rpm_dir not found" + usage + exit 1 + fi +fi + +if [[ -n "$target_version" ]]; then + log "Source version: $source_version" + log "Target version: $target_version" +else + log "Source version: $source_version" + log "RPM Directory: $rpm_dir" +fi + +function wait_for_socket { + SOCK="$1" + LIMIT="${2:-10}" + log "Waiting $LIMIT seconds for Unix socket to be created at $SOCK ..." + for ii in $(seq $LIMIT); do + [ -S $SOCK ] && break || sleep 1 + done + if [ $ii == $LIMIT ]; then + log "Unix socket was not created within $LIMIT seconds" + false + fi +} + +# Install dependencies +log Installing dependencies +dnf install -y wget libxcrypt-compat libaio ncurses-compat-libs numactl-libs + +install_mariadb_from_archive() { + version=$1 + # Find latest RPM repository from https://archive.mariadb.org/ + # The RPMs for each MariaDB version are build for the latest distribution version at the time of release. + # Certain required libraries are missing in the latest version of the distribuiton this test is being run on (currently Fedora). + # The missing libraries depned on the latest distribution version, so that is found by scanning the MariaDB archive directory. + + log Finding MariaDB RPM repository for version $version + if ! $(wget --spider -r --no-parent --level 1 --quiet https://archive.mariadb.org/mariadb-$version/yum/fedora/ -P /tmp/); then + log Could not find RPMs + exit 1 + fi + latest_distro=$(ls -1v /tmp/archive.mariadb.org/mariadb-$version/yum/fedora/ | tail -n 1) + rpm_repository=https://archive.mariadb.org/mariadb-$version/yum/fedora/$latest_distro/$(uname -m)/ + log RPM repository: $rpm_repository + # log Fedora distribution: $latest_distro # Currently only supports tests on Fedora + + # Use custom repository + log Using custom MariaDB repository + cat </etc/yum.repos.d/MariaDB.repo +[mariadb] +name=MariaDB +baseurl=$rpm_repository +gpgkey=https://ftp.osuosl.org/pub/mariadb/yum/RPM-GPG-KEY-MariaDB +gpgcheck=1 +EOF + + cat /etc/yum.repos.d/MariaDB.repo + + # Get version of MariaDB server. This is necessary to check which additional dependencies must be installed. + # Full version has been defined + major_version=$(echo $version | cut -d '.' -f1-2) + minor_version=$(echo $version | cut -d '.' -f3) + # Only major version has been defined, get minor version from RPM + if [ -z $minor_version ]; then + rpm_version=$(dnf list MariaDB-server | grep -oP 'MariaDB-server.x86_64\s+\K[0-9]+\.[0-9]+\.[0-9]+(?=-)') + minor_version=$(echo $rpm_version | cut -d '.' -f3) + fi + # Install missing dependencies that are version/distro specific + log "Installing missing libraries" + [[ $latest_distro -le 32 ]] && dnf install -y https://download-ib01.fedoraproject.org/pub/epel/8/Everything/x86_64/Packages/b/boost169-program-options-1.69.0-5.el8.x86_64.rpm + [[ $latest_distro == 33 ]] && dnf install -y http://springdale.princeton.edu/data/springdale/7/x86_64/os/Computational/boost173-program-options-1.73.0-7.sdl7.x86_64.rpm + [[ $latest_distro == 34 ]] && dnf install -y https://repo.almalinux.org/almalinux/9/AppStream/x86_64/os/Packages/boost-program-options-1.75.0-8.el9.x86_64.rpm \ + https://vault.centos.org/centos/8/AppStream/x86_64/os/Packages/liburing-1.0.7-3.el8.x86_64.rpm + [[ $latest_distro == 35 ]] && dnf install -y https://download-ib01.fedoraproject.org/pub/fedora/linux/updates/36/Everything/x86_64/Packages/b/boost-program-options-1.76.0-12.fc36.x86_64.rpm + if [[ $major_version == "10.4" ]] && [[ $minor_version -ge 24 ]]; then + log RPMs not available for version 10.4.24+ from this repository. You may try testing by installing from the .tar.gz binaries. + exit 1 + fi + + log Begin installation of MariaDB version $version + dnf install -y MariaDB-server +} + +install_mariadb_from_archive $source_version + +# Set variables and binaries +MYSQLD=$([ -f /usr/sbin/mariadbd ] && echo "/usr/sbin/mariadbd" || echo "/usr/sbin/mysqld") +MYSQL_BIN=/usr/bin +TMP_DATA_DIR=/tmp/var/lib/mysql +rm -rf $TMP_DATA_DIR test_upgrade.log # clean up from old tests +SOCK=/var/lib/mysql/mysql.sock + +start_mariadb_server() { + $MYSQLD --version # check version + + log Creating system tables with mysql_install_db + $MYSQL_BIN/mysql_install_db --user=mysql --datadir=$TMP_DATA_DIR + chown -R mysql $TMP_DATA_DIR + + # Start server + log Starting mariadb daemon + sudo -u mysql $MYSQLD --datadir=$TMP_DATA_DIR & + wait_for_socket $SOCK 10 + $MYSQL_BIN/mysql --skip-column-names -e "SELECT @@version, @@version_comment" # Show version +} + +shutdown_and_wait() { + while $MYSQL_BIN/mysql -e 'SELECT 1' 2>&1 >/dev/null; do + $MYSQL_BIN/mysql -e 'SHUTDOWN' + sleep 1 + [ ! -S $SOCK ] && break + done +} + +log Start server with source version +start_mariadb_server + +log Dump database contents in installed state +$MYSQL_BIN/mysqldump --all-databases --all-tablespaces --triggers --routines --events --skip-extended-insert >old-installed-database.sql +log Check if tables need upgrade +$MYSQL_BIN/mysqlcheck --all-databases --check-upgrade | tee -a test_upgrade.log + +# Generate user tables filled with random data +$MYSQL_BIN/mysql -e "CREATE DATABASE random_data;" || true +$MYSQL_BIN/mysql -e "USE random_data;" +# Varchars +$MYSQL_BIN/mysql -D random_data -e "CREATE TABLE random_strings (string VARCHAR(255) NOT NULL);" || true +for i in {1..10}; do + $MYSQL_BIN/mysql -D random_data -e "INSERT INTO random_strings ( string ) VALUES ( MD5(RAND()) );" +done +$MYSQL_BIN/mysql -D random_data -e "SELECT count(*) FROM random_strings;" +# Integers +$MYSQL_BIN/mysql -D random_data -e "CREATE TABLE random_numbers (number INT NOT NULL);" || true +for i in {1..10}; do + $MYSQL_BIN/mysql -D random_data -e "INSERT INTO random_numbers ( number ) VALUES ( RAND() );" +done +$MYSQL_BIN/mysql -D random_data -e "SELECT count(*) FROM random_numbers;" + +# Run mysql_upgrade which should have no effect +log Do upgrade +$MYSQL_BIN/mysql_upgrade -u root -v | tee -a test_upgrade.log +log Dump database contents in upgraded state +$MYSQL_BIN/mysqldump --all-databases --all-tablespaces --triggers --routines --events --skip-extended-insert >old-upgraded-database.sql +$MYSQL_BIN/mysql --skip-column-names -e "SELECT @@version, @@version_comment" # Show version +log Shutdown +shutdown_and_wait + +log Uninstall source version +dnf remove -y MariaDB-server + +log ------------------------------------------------ +if [[ -n "$target_version" ]]; then + install_mariadb_from_archive $target_version +elif [[ -n "$rpm_dir" ]]; then + log Begin installation of MariaDB from local RPMs + log "rpm_dir: $rpm_dir" + dnf install -y $rpm_dir/*.rpm # from previous job +fi + +start_mariadb_server + +log Dump database contents in installed state +$MYSQL_BIN/mysqldump --all-databases --all-tablespaces --triggers --routines --events --skip-extended-insert >new-installed-database.sql || true +# The step above fails on: mariadb-dump: Couldn't execute 'show events': Cannot proceed, because event scheduler is disabled (1577) + +log Run check for upgrade +$MYSQL_BIN/mysqlcheck --all-databases --check-upgrade --check --extended | tee -a test_upgrade.log + +log Run upgrade +$MYSQL_BIN/mysql_upgrade -u root | tee -a test_upgrade.log # minor version is the same, so no upgrade done + +log Force upgrade +$MYSQL_BIN/mysql_upgrade -u root --force | tee -a test_upgrade.log + +log Dump database contents in upgraded state +$MYSQL_BIN/mysqldump --all-databases --all-tablespaces --triggers --routines --events --skip-extended-insert >new-upgraded-database.sql +$MYSQL_BIN/mysql --skip-column-names -e "SELECT @@version, @@version_comment" # Show version + +log Shutdown +shutdown_and_wait +echo + +# Test whether a table rebuild was triggered +if grep -E "Needs upgrade|Table rebuild required" test_upgrade.log; then + log "=================================" + log "= A table rebuild was triggered =" + log "=================================" + exit 1 +else + log "==================================" + log "= No table rebuild was triggered =" + log "==================================" + exit 0 +fi