bb.org: dbdeployer ci test

buildbot.mariadb.org/dockerfiles/eco-dbdeployer-ubuntu-2004.dockerfile

@@ -0,0 +1,35 @@
+FROM ubuntu:20.04
+
+LABEL maintainer="MariaDB Buildbot maintainers"
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update \
+    && apt-get upgrade -y \
+    && apt-get install -y make sudo curl libsnappy1v5 libaio1 vim-tiny perl-modules libnuma1 binutils \
+        xz-utils wget less net-tools lsof libreadline5 python3-pip \
+    && rm -rf /var/lib/apt/lists/* 
+
+# Create buildbot user
+RUN useradd -ms /bin/bash buildbot && \
+    mkdir -p /buildbot && \
+    chown -R buildbot /buildbot /usr/local && \
+    curl -o /buildbot/buildbot.tac https://raw.githubusercontent.com/MariaDB/mariadb.org-tools/master/buildbot.mariadb.org/dockerfiles/buildbot.tac
+
+VOLUME /dbdeployer
+
+WORKDIR /buildbot
+
+# Upgrade pip and install packages
+RUN pip3 install -U pip virtualenv
+RUN pip3 install buildbot-worker && \
+    pip3 --no-cache-dir install 'twisted[tls]'
+
+RUN curl https://github.com/Yelp/dumb-init/releases/download/v1.2.2/dumb-init_1.2.2_$(dpkg --print-architecture).deb -Lo /tmp/init.deb && dpkg -i /tmp/init.deb
+
+USER buildbot
+ENV USER=buildbot
+ENV HOME=/buildbot
+
+CMD ["/usr/bin/dumb-init", "twistd", "--pidfile=", "-ny", "buildbot.tac"]
+

buildbot.mariadb.org/dockerfiles/ecofiles/test-dbdeployer.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+set -xeuvo pipefail
+
+if [ $# -lt 2 ]
+then
+	echo "insufficent arguments - two arguments minimum expected"
+	exit 1
+fi
+
+
+# Asset ID for download:
+# curl -L -s -H 'Accept: application/vnd.github.v3+json' https://api.github.com/repos/datacharmer/dbdeployer/releases/latest | grep -B 4 linux.tar.gz\"
+case "${1}" in
+	dbdeployerfetch)
+		f=dbdeployer-"${2}".linux
+		[ ! -f "/dbdeployer/$f" ] || \
+			curl -L -s -H 'Accept: application/octet-stream' https://api.github.com/repos/datacharmer/dbdeployer/releases/assets/"${2}" \
+			| tar -zxf - -C /dbdeployer
+		ln -s /dbdeployer/dbdeployer-${2}.linux dbeployer
+		;;
+	init)
+		mkdir /tmp/opt
+		./dbdeployer init --skip-all-downloads --skip-shell-completion --sandbox-home=/tmp/sandboxes --sandbox-binary=/tmp/opt
+		file=/tmp/$(basename "${2}")
+                [ -f "$file" ] ||  curl --output "${file}" "$2"
+                ./dbdeployer unpack --prefix=ma "${file}"
+                rm "${file}"
+		;;
+	*)
+		./dbdeployer $@
+		;;
+esac
+

buildbot.mariadb.org/master.cfg

@@ -91,7 +91,7 @@ builders_install=["ubuntu-1804-deb-autobake-install", "centos-7-rpm-autobake-ins
 
 builders_upgrade=["ubuntu-1804-deb-autobake-major-upgrade", "ubuntu-1804-deb-autobake-minor-upgrade", "centos-7-rpm-autobake-major-upgrade", "centos-7-rpm-autobake-minor-upgrade"]
 
-builders_eco=["ubuntu-2004-eco-php", "debian-10-eco-pymysql", "debian-10-eco-mysqljs"]
+builders_eco=["ubuntu-2004-eco-php", "debian-10-eco-pymysql", "debian-10-eco-mysqljs", "ubuntu-2004-eco-dbdeployer"]
 
 
 # Configure the Schedulers, which decide how to react to incoming changes.
@@ -185,6 +185,7 @@ c['workers'].append(worker.DockerLatentWorker("hz-bbw1-docker-tarball-debian-10"
                     volumes=['/srv/buildbot/master_packages:/packages'],
                     max_builds=1,
                     properties={ 'jobs':4, 'save_packages':True }))
+
 c['workers'].append(worker.DockerLatentWorker("hz-bbw1-docker-eco-php-ubuntu-2004", None,
                     docker_host=config["private"]["docker_workers"]["hz-bbw1-docker"],
                     tls=docker.tls.TLSConfig(verify=True, ca_cert='/srv/buildbot/tlscerts/ca-hz-bbw1.pem', client_cert=('/srv/buildbot/tlscerts/cert-hz-bbw1.pem', '/srv/buildbot/tlscerts/key-hz-bbw1.pem')),
@@ -196,6 +197,19 @@ c['workers'].append(worker.DockerLatentWorker("hz-bbw1-docker-eco-php-ubuntu-200
                     max_builds=1,
                     volumes=['/srv/buildbot/eco/code:/code', '/srv/buildbot/eco/build:/build'],
                     properties={ 'jobs':7, 'save_packages':False }))
+
+c['workers'].append(worker.DockerLatentWorker("hz-bbw1-docker-eco-dbdeployer-ubuntu-2004", None,
+                    docker_host=config["private"]["docker_workers"]["hz-bbw1-docker"],
+                    tls=docker.tls.TLSConfig(verify=True, ca_cert='/srv/buildbot/tlscerts/ca-hz-bbw1.pem', client_cert=('/srv/buildbot/tlscerts/cert-hz-bbw1.pem', '/srv/buildbot/tlscerts/key-hz-bbw1.pem')),
+                    dockerfile=open("dockerfiles/eco-dbdeployer-ubuntu-2004.dockerfile").read(),
+                    followStartupLogs=False,
+                    masterFQDN='buildbot.mariadb.org',
+                    hostconfig={ 'shm_size':'6G' },
+                    build_wait_timeout=0,
+                    max_builds=1,
+                    volumes=['/srv/buildbot/eco/dbdeployer:/dbdeployer'],
+                    properties={ 'jobs':7, 'save_packages':False }))
+
 c['workers'].append(worker.DockerLatentWorker("hz-bbw1-docker-eco-pymysql-python-3-9-slim-buster", None,
                     docker_host=config["private"]["docker_workers"]["hz-bbw1-docker"],
                     tls=docker.tls.TLSConfig(verify=True, ca_cert='/srv/buildbot/tlscerts/ca-hz-bbw1.pem', client_cert=('/srv/buildbot/tlscerts/cert-hz-bbw1.pem', '/srv/buildbot/tlscerts/key-hz-bbw1.pem')),
@@ -2755,6 +2769,32 @@ f_eco_php.addStep(steps.ShellCommand(
     name="test PHP-8.0",
     command=["sh", "-xc", "/buildbot/test-php.sh PHP-8.0"]))
 
+## f_eco_dbdeployer
+f_eco_dbdeployer = util.BuildFactory()
+f_eco_dbdeployer.addStep(steps.ShellCommand(
+    name="fetch_test_script",
+    command=["sh", "-xc", "curl https://raw.githubusercontent.com/MariaDB/mariadb.org-tools/master/buildbot.mariadb.org/dockerfiles/ecofiles/test-dbdeployer.sh -o /buildbot/test-dbdeployer.sh && chmod a+x /buildbot/test-dbdeployer.sh"]))
+# Asset ID:
+# curl -L -s -H 'Accept: application/vnd.github.v3+json' https://api.github.com/repos/datacharmer/dbdeployer/releases/latest | grep -B 4 linux.tar.gz\"
+f_eco_dbdeployer.addStep(steps.ShellCommand(
+    name="download if needed dbdeployer 1.57.0",
+    command=["sh", "-xc", "/buildbot/test-dbdeployer.sh dbdeployerfetch 29350892"]))
+f_eco_dbdeployer.addStep(steps.ShellCommand(
+    name="fetching mariadb tarball",
+    command=["sh", "-xc", util.Interpolate("/buildbot/test-dbdeployer.sh init \"https://ci.mariadb.org/%(prop:tarbuildnum)s/%(prop:parentbuildername)s/%(prop:mariadb_binary)s\"")]))
+f_eco_dbdeployer.addStep(steps.ShellCommand(
+    name="deploy single ma",
+    command=["sh", "-xc", util.Interpolate("/buildbot/test-dbdeployer.sh deploy single ma%(prop:mariadb_version)s")]))
+f_eco_dbdeployer.addStep(steps.ShellCommand(
+    name="deploy replication ma",
+    command=["sh", "-xc", util.Interpolate("/buildbot/test-dbdeployer.sh deploy replication ma%(prop:mariadb_version)s")]))
+f_eco_dbdeployer.addStep(steps.ShellCommand(
+    name="global test",
+    command=["sh", "-xc", "/buildbot/test-dbdeployer.sh global test"]))
+f_eco_dbdeployer.addStep(steps.ShellCommand(
+    name="global replication",
+    command=["sh", "-xc", "/buildbot/test-dbdeployer.sh global test-replication"]))
+
 ## f_eco_pymysql
 f_eco_pymysql = util.BuildFactory()
 f_eco_pymysql.addStep(steps.ShellCommand(
@@ -3041,6 +3081,15 @@ c['builders'].append(
       canStartBuild=canStartBuild,
       factory=f_eco_php))
 
+c['builders'].append(
+    util.BuilderConfig(name="ubuntu-2004-eco-dbdeployer",
+      workernames=["hz-bbw1-docker-eco-dbdeployer-ubuntu-2004"],
+      tags=["Ubuntu", "ecosystem", "dbdeployer"],
+      collapseRequests=True,
+      nextBuild=nextBuild,
+      canStartBuild=canStartBuild,
+      factory=f_eco_dbdeployer))
+
 c['builders'].append(
     util.BuilderConfig(name="debian-10-eco-pymysql",
       workernames=["hz-bbw1-docker-eco-pymysql-python-3-9-slim-buster"],