Skip to content

Commit

Permalink
Merge pull request #57988 from lopsided98/buildbot-update
Browse files Browse the repository at this point in the history 
buildbot: 1.8.1 -> 2.1.0
  • Loading branch information
@veprbl
veprbl committed Mar 24, 2019
2 parents 04727cc + 69f2787 commit 6f95ac3
Show file tree
Hide file tree
Showing 10 changed files with 162 additions and 132 deletions.
23 changes: 23 additions & 0 deletions nixos/doc/manual/release-notes/rl-1909.xml
View file
Expand Up @@ -42,6 +42,29 @@
</itemizedlist>
</section>

<section xmlns="http://docbook.org/ns/docbook"
xmlns:xlink="http://www.w3.org/1999/xlink"
xmlns:xi="http://www.w3.org/2001/XInclude"
version="5.0"
xml:id="sec-release-19.09-incompatibilities">
<title>Backward Incompatibilities</title>

<para>
When upgrading from a previous release, please be aware of the following
incompatible changes:
</para>

<itemizedlist>
<listitem>
<para>
Buildbot no longer supports Python 2, as support was dropped upstream in
version 2.0.0. Configurations may need to be modified to make them
compatible with Python 3.
</para>
</listitem>
</itemizedlist>
</section>

<section xmlns="http://docbook.org/ns/docbook"
xmlns:xlink="http://www.w3.org/1999/xlink"
xmlns:xi="http://www.w3.org/2001/XInclude"
Expand Down
6 changes: 3 additions & 3 deletions nixos/modules/services/continuous-integration/buildbot/master.nix
View file
Expand Up @@ -199,10 +199,10 @@ in {

package = mkOption {
type = types.package;
default = pkgs.pythonPackages.buildbot-full;
defaultText = "pkgs.pythonPackages.buildbot-full";
default = pkgs.python3Packages.buildbot-full;
defaultText = "pkgs.python3Packages.buildbot-full";
description = "Package to use for buildbot.";
example = literalExample "pkgs.python3Packages.buildbot-full";
example = literalExample "pkgs.python3Packages.buildbot";
};

packages = mkOption {
Expand Down
6 changes: 3 additions & 3 deletions nixos/modules/services/continuous-integration/buildbot/worker.nix
View file
Expand Up @@ -118,10 +118,10 @@ in {

package = mkOption {
type = types.package;
default = pkgs.pythonPackages.buildbot-worker;
defaultText = "pkgs.pythonPackages.buildbot-worker";
default = pkgs.python3Packages.buildbot-worker;
defaultText = "pkgs.python3Packages.buildbot-worker";
description = "Package to use for buildbot worker.";
example = literalExample "pkgs.python3Packages.buildbot-worker";
example = literalExample "pkgs.python2Packages.buildbot-worker";
};

packages = mkOption {
Expand Down
205 changes: 99 additions & 106 deletions nixos/tests/buildbot.nix
View file
Expand Up @@ -5,116 +5,109 @@

with import ../lib/testing.nix { inherit system pkgs; };

let
# Test ensures buildbot master comes up correctly and workers can connect
mkBuildbotTest = python: makeTest {
name = "buildbot";

nodes = {
bbmaster = { pkgs, ... }: {
services.buildbot-master = {
enable = true;
package = python.pkgs.buildbot-full;

# NOTE: use fake repo due to no internet in hydra ci
factorySteps = [
"steps.Git(repourl='git://gitrepo/fakerepo.git', mode='incremental')"
"steps.ShellCommand(command=['bash', 'fakerepo.sh'])"
];
changeSource = [
"changes.GitPoller('git://gitrepo/fakerepo.git', workdir='gitpoller-workdir', branch='master', pollinterval=300)"
];
};
networking.firewall.allowedTCPPorts = [ 8010 8011 9989 ];
environment.systemPackages = with pkgs; [ git python.pkgs.buildbot-full ];
};

bbworker = { pkgs, ... }: {
services.buildbot-worker = {
enable = true;
masterUrl = "bbmaster:9989";
};
environment.systemPackages = with pkgs; [ git python.pkgs.buildbot-worker ];
# Test ensures buildbot master comes up correctly and workers can connect
makeTest {
name = "buildbot";

nodes = {
bbmaster = { pkgs, ... }: {
services.buildbot-master = {
enable = true;

# NOTE: use fake repo due to no internet in hydra ci
factorySteps = [
"steps.Git(repourl='git://gitrepo/fakerepo.git', mode='incremental')"
"steps.ShellCommand(command=['bash', 'fakerepo.sh'])"
];
changeSource = [
"changes.GitPoller('git://gitrepo/fakerepo.git', workdir='gitpoller-workdir', branch='master', pollinterval=300)"
];
};
networking.firewall.allowedTCPPorts = [ 8010 8011 9989 ];
environment.systemPackages = with pkgs; [ git python3Packages.buildbot-full ];
};

gitrepo = { pkgs, ... }: {
services.openssh.enable = true;
networking.firewall.allowedTCPPorts = [ 22 9418 ];
environment.systemPackages = with pkgs; [ git ];
bbworker = { pkgs, ... }: {
services.buildbot-worker = {
enable = true;
masterUrl = "bbmaster:9989";
};
environment.systemPackages = with pkgs; [ git python3Packages.buildbot-worker ];
};

testScript = ''
#Start up and populate fake repo
$gitrepo->waitForUnit("multi-user.target");
print($gitrepo->execute(" \
git config --global user.name 'Nobody Fakeuser' && \
git config --global user.email 'nobody\@fakerepo.com' && \
rm -rvf /srv/repos/fakerepo.git /tmp/fakerepo && \
mkdir -pv /srv/repos/fakerepo ~/.ssh && \
ssh-keyscan -H gitrepo > ~/.ssh/known_hosts && \
cat ~/.ssh/known_hosts && \
cd /srv/repos/fakerepo && \
git init && \
echo -e '#!/bin/sh\necho fakerepo' > fakerepo.sh && \
cat fakerepo.sh && \
touch .git/git-daemon-export-ok && \
git add fakerepo.sh .git/git-daemon-export-ok && \
git commit -m fakerepo && \
git daemon --verbose --export-all --base-path=/srv/repos --reuseaddr & \
"));
# Test gitrepo
$bbmaster->waitForUnit("network-online.target");
#$bbmaster->execute("nc -z gitrepo 9418");
print($bbmaster->execute(" \
rm -rfv /tmp/fakerepo && \
git clone git://gitrepo/fakerepo /tmp/fakerepo && \
pwd && \
ls -la && \
ls -la /tmp/fakerepo \
"));
# Test start master and connect worker
$bbmaster->waitForUnit("buildbot-master.service");
$bbmaster->waitUntilSucceeds("curl -s --head http://bbmaster:8010") =~ /200 OK/;
$bbworker->waitForUnit("network-online.target");
$bbworker->execute("nc -z bbmaster 8010");
$bbworker->execute("nc -z bbmaster 9989");
$bbworker->waitForUnit("buildbot-worker.service");
print($bbworker->execute("ls -la /home/bbworker/worker"));
# Test stop buildbot master and worker
print($bbmaster->execute(" \
systemctl -l --no-pager status buildbot-master && \
systemctl stop buildbot-master \
"));
$bbworker->fail("nc -z bbmaster 8010");
$bbworker->fail("nc -z bbmaster 9989");
print($bbworker->execute(" \
systemctl -l --no-pager status buildbot-worker && \
systemctl stop buildbot-worker && \
ls -la /home/bbworker/worker \
"));
# Test buildbot daemon mode
$bbmaster->execute("buildbot create-master /tmp");
$bbmaster->execute("mv -fv /tmp/master.cfg.sample /tmp/master.cfg");
$bbmaster->execute("sed -i 's/8010/8011/' /tmp/master.cfg");
$bbmaster->execute("buildbot start /tmp");
$bbworker->execute("nc -z bbmaster 8011");
$bbworker->waitUntilSucceeds("curl -s --head http://bbmaster:8011") =~ /200 OK/;
$bbmaster->execute("buildbot stop /tmp");
$bbworker->fail("nc -z bbmaster 8011");
'';

meta.maintainers = with pkgs.stdenv.lib.maintainers; [ nand0p ];

gitrepo = { pkgs, ... }: {
services.openssh.enable = true;
networking.firewall.allowedTCPPorts = [ 22 9418 ];
environment.systemPackages = with pkgs; [ git ];
};
};
in {
python2 = mkBuildbotTest pkgs.python2;
python3 = mkBuildbotTest pkgs.python3;

testScript = ''
#Start up and populate fake repo
$gitrepo->waitForUnit("multi-user.target");
print($gitrepo->execute(" \
git config --global user.name 'Nobody Fakeuser' && \
git config --global user.email 'nobody\@fakerepo.com' && \
rm -rvf /srv/repos/fakerepo.git /tmp/fakerepo && \
mkdir -pv /srv/repos/fakerepo ~/.ssh && \
ssh-keyscan -H gitrepo > ~/.ssh/known_hosts && \
cat ~/.ssh/known_hosts && \
cd /srv/repos/fakerepo && \
git init && \
echo -e '#!/bin/sh\necho fakerepo' > fakerepo.sh && \
cat fakerepo.sh && \
touch .git/git-daemon-export-ok && \
git add fakerepo.sh .git/git-daemon-export-ok && \
git commit -m fakerepo && \
git daemon --verbose --export-all --base-path=/srv/repos --reuseaddr & \
"));
# Test gitrepo
$bbmaster->waitForUnit("network-online.target");
#$bbmaster->execute("nc -z gitrepo 9418");
print($bbmaster->execute(" \
rm -rfv /tmp/fakerepo && \
git clone git://gitrepo/fakerepo /tmp/fakerepo && \
pwd && \
ls -la && \
ls -la /tmp/fakerepo \
"));
# Test start master and connect worker
$bbmaster->waitForUnit("buildbot-master.service");
$bbmaster->waitUntilSucceeds("curl -s --head http://bbmaster:8010") =~ /200 OK/;
$bbworker->waitForUnit("network-online.target");
$bbworker->execute("nc -z bbmaster 8010");
$bbworker->execute("nc -z bbmaster 9989");
$bbworker->waitForUnit("buildbot-worker.service");
print($bbworker->execute("ls -la /home/bbworker/worker"));
# Test stop buildbot master and worker
print($bbmaster->execute(" \
systemctl -l --no-pager status buildbot-master && \
systemctl stop buildbot-master \
"));
$bbworker->fail("nc -z bbmaster 8010");
$bbworker->fail("nc -z bbmaster 9989");
print($bbworker->execute(" \
systemctl -l --no-pager status buildbot-worker && \
systemctl stop buildbot-worker && \
ls -la /home/bbworker/worker \
"));
# Test buildbot daemon mode
$bbmaster->execute("buildbot create-master /tmp");
$bbmaster->execute("mv -fv /tmp/master.cfg.sample /tmp/master.cfg");
$bbmaster->execute("sed -i 's/8010/8011/' /tmp/master.cfg");
$bbmaster->execute("buildbot start /tmp");
$bbworker->execute("nc -z bbmaster 8011");
$bbworker->waitUntilSucceeds("curl -s --head http://bbmaster:8011") =~ /200 OK/;
$bbmaster->execute("buildbot stop /tmp");
$bbworker->fail("nc -z bbmaster 8011");
'';

meta.maintainers = with pkgs.stdenv.lib.maintainers; [ nand0p ];
}
25 changes: 19 additions & 6 deletions pkgs/development/python-modules/buildbot/default.nix
View file
@@ -1,8 +1,9 @@
{ stdenv, lib, buildPythonPackage, fetchPypi, makeWrapper, isPy3k,
{ stdenv, lib, buildPythonPackage, /*fetchPypi,*/ fetchFromGitHub, makeWrapper, isPy3k,
python, twisted, jinja2, zope_interface, future, sqlalchemy,
sqlalchemy_migrate, dateutil, txaio, autobahn, pyjwt, pyyaml, treq,
txrequests, txgithub, pyjade, boto3, moto, mock, python-lz4, setuptoolsTrial,
isort, pylint, flake8, buildbot-worker, buildbot-pkg, glibcLocales }:
isort, pylint, flake8, buildbot-worker, buildbot-pkg, parameterized,
glibcLocales }:

let
withPlugins = plugins: buildPythonPackage {
Expand All @@ -24,12 +25,21 @@ let

package = buildPythonPackage rec {
pname = "buildbot";
version = "1.8.1";
version = "2.1.0";

src = fetchPypi {
/*src = fetchPypi {
inherit pname version;
sha256 = "1zadmyrlk7p9h1akmbzwa7p90s7jwsxvdx4xn9i54dnda450m3a7";
};
sha256 = "1745hj9s0c0fcdjv6w05bma76xqg1fv42v0dslmi4d8yz9phf37w";
};*/
# Temporarily use GitHub source because PyPi archive is missing some files
# needed for the tests to pass. This has been fixed upstream.
# See: https://github.com/buildbot/buildbot/commit/30f5927cf9a80f98ed909241a149469dec3ce68d
src = fetchFromGitHub {
owner = "buildbot";
repo = "buildbot";
rev = "v${version}";
sha256 = "022ybhdvp0hp2z0cwgx7n41jyh56bpxj3fwm4z7ppzj1qhm7lb65";
} + "/master";

propagatedBuildInputs = [
# core
Expand Down Expand Up @@ -63,6 +73,7 @@ let
flake8
buildbot-worker
buildbot-pkg
parameterized
glibcLocales
];

Expand All @@ -84,6 +95,8 @@ let
export PATH="$out/bin:$PATH"
'';

disabled = !isPy3k;

passthru = {
inherit withPlugins;
};
Expand Down
4 changes: 2 additions & 2 deletions pkgs/development/python-modules/buildbot/pkg.nix
View file
Expand Up @@ -2,11 +2,11 @@

buildPythonPackage rec {
pname = "buildbot-pkg";
version = "1.8.1";
version = "2.1.0";

src = fetchPypi {
inherit pname version;
sha256 = "16gjdzkris6475bvsgvb0v6rkn4xb6f55s468q37n0l1r6n8snc3";
sha256 = "03lv97q4pp2izjfbwfv4zmf2fyiz7jyp537bi3gc6rhfbrfgib1i";
};

postPatch = ''
Expand Down
11 changes: 6 additions & 5 deletions pkgs/development/python-modules/buildbot/plugins.nix
View file
Expand Up @@ -10,7 +10,8 @@

src = fetchPypi {
inherit pname version format;
sha256 = "03cgjhwpgbm0qgis1cdy9g4vc11hsrya9grcx4j35784rny7lbfl";
python = "py3";
sha256 = "011sagw8zp1z12vzkxi44w3w2lbxncz5yahkrbxj8hp6iwfzfm5v";
};

meta = with lib; {
Expand All @@ -27,7 +28,7 @@

src = fetchPypi {
inherit pname version;
sha256 = "0pfp2n4ys99jglshdrp2f6jm73c4ym3dfwl6qjvbc7y7nsi74824";
sha256 = "11gz4ry1law3l64ii383cj5fnbw9409czp2ybzkqafr4xi1qbk9h";
};

propagatedBuildInputs = [ buildbot-pkg ];
Expand All @@ -47,7 +48,7 @@

src = fetchPypi {
inherit pname version;
sha256 = "0gnxq9niw64q36dm917lhhcl8zp0wjwaamjp07zidnrb5c3pjbsz";
sha256 = "0w4iwpj1rg20fbli0ppqz70l1mc9ilg0crq8g3xrf29f9z8d1w27";
};

propagatedBuildInputs = [ buildbot-pkg ];
Expand All @@ -67,7 +68,7 @@

src = fetchPypi {
inherit pname version;
sha256 = "1b06aa8m1pzqq2d8imrq5mazc7llrlbgm7jzi8h6jjd2gahdjgz5";
sha256 = "0xyvxamw45qhnfml3x5hfg9nai1jhdwbmq4pm8csf3ad0cw6vqya";
};

propagatedBuildInputs = [ buildbot-pkg ];
Expand All @@ -87,7 +88,7 @@

src = fetchPypi {
inherit pname version;
sha256 = "1v8411bw0cs206vwfnqx1na7dzg77h9aff4wlm11hkbdsy9ayv2d";
sha256 = "1szcrx8vslskifzxaq7lrfg2arilaq1w1aqr0nc8pjclj7idp92c";
};

propagatedBuildInputs = [ buildbot-pkg ];
Expand Down

0 comments on commit 6f95ac3

Please sign in to comment.