Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
Switch to a common BuildFactory creation function 
There are minor differences between build steps but they are mostly the
same.  The configuration file gets inconsistent very easily due to so
many duplicated BuildFactories.

Extract common BuildFactory setup code into create_build_factory().
Things to consider:

 * Git repository URL
 * Git branch
 * make(1) program name (e.g. gmake on OpenBSD and Solaris)
 * ./configure arguments
 * Out-of-tree builds using a separate directory

Locks are used to avoid hammering the git servers.  Automatically find
locks based on the git repository URL.  This saves us from specifying
the right lock variable for each BuildFactory.

This patch also runs "make check" on each BuildFactory.  This has been
requested by submaintainers who found their pull requests broke "make
check" - they'd like to find out earlier so run it on each BuildFactory,
not just the qemu.git/master default one.

Signed-off-by: Stefan Hajnoczi <stefanha@gmail.com>
  • Loading branch information
@stefanha
stefanha committed Jan 30, 2013
1 parent 1d093c4 commit 58a2d8a
Showing 1 changed file with 54 additions and 117 deletions.
171 changes: 54 additions & 117 deletions qemu-master.cfg
View file
Expand Up @@ -641,12 +641,22 @@ slow_lock = SlaveLock("cpu")
# Only perform one git fetch per repo. For two reasons:
# - do not stress the repos
# - do not stress VMs which host several buildslaves
git_fetch_qemuorg_lock = MasterLock("fetch_git_qemuorg", 1)
git_fetch_repoorcz_lock = MasterLock("fetch_git_repoorcz", 1)
git_fetch_kernelorg_lock = MasterLock("fetch_git_kernelorg", 1)
git_fetch_github_lock = MasterLock("fetch_git_github", 1)
git_fetch_linaro_lock = MasterLock("fetch_git_linaro", 1)

repo_locks = {
"git.qemu-project.org": MasterLock("fetch_git_qemuorg", 1),
"repo.or.cz": MasterLock("fetch_git_repoorcz", 1),
"git.kernel.org": MasterLock("fetch_git_kernelorg", 1),
"github.com": MasterLock("fetch_git_github", 1),
"git.linaro.org": MasterLock("fetch_git_linaro", 1),
}

import urlparse
def repo_locks_from_url(repourl):
"""Return a list of exclusive access locks given a git repo URL"""
host = urlparse.urlparse(repourl).netloc
if host in repo_locks:
return [repo_locks[host].access('exclusive')]
else:
return []


#####################################################################
Expand All @@ -671,155 +681,82 @@ from buildbot.steps.shell import Configure, Compile, ShellCommand, Test
# test=None
#)

f_default = factory.BuildFactory()
f_default.addStep(Git(repourl="git://git.qemu-project.org/qemu.git", timeout=2400, mode='copy', retry=(5 * 60, 3), locks=[git_fetch_qemuorg_lock.access('exclusive')]))
f_default.addStep(Configure(command="./configure", env={'LANG': 'C'}))
f_default.addStep(Compile(command=["make", "CFLAGS=-O2"], env={'LANG': 'C'}, timeout=2400))
f_default.addStep(Test(command=["make", "check"], env={'LANG': 'C'}, maxTime=2400))

f_disable_kvm = factory.BuildFactory()
f_disable_kvm.addStep(Git(repourl="git://git.qemu-project.org/qemu.git", timeout=2400, mode='copy', retry=(5 * 60, 3), locks=[git_fetch_qemuorg_lock.access('exclusive')]))
f_disable_kvm.addStep(Configure(command="./configure --disable-kvm", env={'LANG': 'C'}))
f_disable_kvm.addStep(Compile(command=["make", "CFLAGS=-O2"], env={'LANG': 'C'}, timeout=2400))

###
# f_out_of_tree_default: mkdir outoftree; cd outoftree; ../configure
f_out_of_tree_default = factory.BuildFactory()
f_out_of_tree_default.addStep(Git(repourl="git://git.qemu-project.org/qemu.git", timeout=2400, mode='copy', retry=(5 * 60, 3), locks=[git_fetch_qemuorg_lock.access('exclusive')]))
f_out_of_tree_default.addStep(ShellCommand(command="mkdir -p outoftree", env={'LANG': 'C'}))
f_out_of_tree_default.addStep(Configure(command="cd outoftree; ../configure", env={'LANG': 'C'}))
f_out_of_tree_default.addStep(Compile(command="cd outoftree; make", env={'LANG': 'C'}, timeout=2400))

# f_out_of_tree_disable_kvm: mkdir outoftree; cd outoftree; ../configure --disable-kvm
f_out_of_tree_disable_kvm = factory.BuildFactory()
f_out_of_tree_disable_kvm.addStep(Git(repourl="git://git.qemu-project.org/qemu.git", timeout=2400, mode='copy', retry=(5 * 60, 3), locks=[git_fetch_qemuorg_lock.access('exclusive')]))
f_out_of_tree_disable_kvm.addStep(ShellCommand(command="mkdir -p outoftree", env={'LANG': 'C'}))
f_out_of_tree_disable_kvm.addStep(Configure(command="cd outoftree; ../configure --disable-kvm", env={'LANG': 'C'}))
f_out_of_tree_disable_kvm.addStep(Compile(command="cd outoftree; make", env={'LANG': 'C'}, timeout=2400))
def create_build_factory(repourl, branch="HEAD", make="make", configure_args=[], outoftree=False):
"""Return a BuildFactory pre-configured with common build steps"""
f = factory.BuildFactory()
f.addStep(Git(repourl=repourl, timeout=2400, mode='copy', retry=(5 * 60, 3), locks=repo_locks_from_url(repourl)))
workdir = f.workdir
if outoftree:
f.addStep(ShellCommand(command="mkdir -p outoftree", env={'LANG': 'C'}))
workdir = os.path.join(workdir, 'outoftree')
f.addStep(Configure(command=["./configure"] + list(configure_args), env={'LANG': 'C'}, workdir=workdir))
f.addStep(Compile(command=[make, "CFLAGS=-O2"], env={'LANG': 'C'}, timeout=2400, workdir=workdir))
f.addStep(Test(command=[make, "check"], env={'LANG': 'C'}, maxTime=2400, workdir=workdir))
return f

f_default = create_build_factory("git://git.qemu-project.org/qemu.git")
f_disable_kvm = create_build_factory("git://git.qemu-project.org/qemu.git", configure_args=["--disable-kvm"])
f_out_of_tree_default = create_build_factory("git://git.qemu-project.org/qemu.git", outoftree=True)
f_out_of_tree_disable_kvm = create_build_factory("git://git.qemu-project.org/qemu.git", configure_args=["--disable-kvm"], outoftree=True)

# anessh / branch: for-upstream
f_virtfs = factory.BuildFactory()
f_virtfs.addStep(Git(repourl="git://github.com/kvaneesh/qemu.git", branch="for-upstream", timeout=2400, mode='copy', retry=(5 * 60, 3), locks=[git_fetch_github_lock.access('exclusive')]))
f_virtfs.addStep(Configure(command="./configure", env={'LANG': 'C'}))
f_virtfs.addStep(Compile(command=["make", "CFLAGS=-O2"], env={'LANG': 'C'}, timeout=2400))
f_virtfs = create_build_factory("git://github.com/kvaneesh/qemu.git", branch="for-upstream")

# block - kwolf / branch: block
f_block = factory.BuildFactory()
f_block.addStep(Git(repourl="git://repo.or.cz/qemu/kevin.git", branch="block", timeout=2400, mode='copy', retry=(5 * 60, 3), locks=[git_fetch_repoorcz_lock.access('exclusive')]))
f_block.addStep(Configure(command="./configure", env={'LANG': 'C'}))
f_block.addStep(Compile(command=["make", "CFLAGS=-O2"], env={'LANG': 'C'}, timeout=2400))

f_block_openbsd = factory.BuildFactory()
f_block_openbsd.addStep(Git(repourl="git://repo.or.cz/qemu/kevin.git", branch="block", timeout=2400, mode='copy', retry=(5 * 60, 3), locks=[git_fetch_repoorcz_lock.access('exclusive')]))
f_block_openbsd.addStep(ShellCommand(command=["./configure", "--extra-cflags=-Wno-redundant-decls"], env={'LANG': 'C'}))
f_block_openbsd.addStep(Compile(command=["gmake", "CFLAGS=-O2"], env={'LANG': 'C'}, timeout=2400))

f_block_mingw32 = factory.BuildFactory()
f_block_mingw32.addStep(Git(repourl="git://repo.or.cz/qemu/kevin.git", branch="block", timeout=2400, mode='copy', retry=(5 * 60, 3), locks=[git_fetch_repoorcz_lock.access('exclusive')]))
f_block_mingw32.addStep(Configure(command=["./configure", "--cross-prefix=i686-pc-mingw32-"], env={'LANG': 'C'}))
f_block_mingw32.addStep(Compile(command=["make", "CFLAGS=-O2"], env={'LANG': 'C'}, timeout=2400))
f_block = create_build_factory("git://repo.or.cz/qemu/kevin.git", branch="block")
f_block_openbsd = create_build_factory("git://repo.or.cz/qemu/kevin.git", branch="block", make="gmake", configure_args=["--extra-cflags=-Wno-redundant-decls"])
f_block_mingw32 = create_build_factory("git://repo.or.cz/qemu/kevin.git", branch="block", configure_args=["--cross-prefix=i686-pc-mingw32-"])

# pci - mst / branch: pci
f_pci = factory.BuildFactory()
f_pci.addStep(Git(repourl="git://github.com/mstsirkin/qemu.git", branch="pci", timeout=2400, mode='copy', retry=(5 * 60, 3), locks=[git_fetch_github_lock.access('exclusive')]))
f_pci.addStep(Configure(command="./configure", env={'LANG': 'C'}))
f_pci.addStep(Compile(command=["make", "CFLAGS=-O2"], env={'LANG': 'C'}, timeout=2400))
f_pci = create_build_factory("git://github.com/mstsirkin/qemu.git", branch="pci")

# qmp - luiz / branch: qmp
f_qmp = factory.BuildFactory()
f_qmp.addStep(Git(repourl="git://repo.or.cz/qemu/qmp-unstable.git", branch="queue/qmp", timeout=2400, mode='copy', retry=(5 * 60, 3), locks=[git_fetch_repoorcz_lock.access('exclusive')]))
f_qmp.addStep(Configure(command="./configure", env={'LANG': 'C'}))
f_qmp.addStep(Compile(command=["make", "CFLAGS=-O2"], env={'LANG': 'C'}, timeout=2400))
f_qmp = create_build_factory("git://repo.or.cz/qemu/qmp-unstable.git", branch="queue/qmp")

## monitor - luiz / branch: monitor
#f_monitor = factory.BuildFactory()
#f_monitor.addStep(Git(repourl="git://repo.or.cz/qemu/qmp-unstable.git", branch="queue/monitor", timeout=2400, mode='copy', retry=(5 * 60, 3), locks=[git_fetch_repoorcz_lock.access('exclusive')]))
#f_monitor.addStep(Configure(command="./configure", env={'LANG': 'C'}))
#f_monitor.addStep(Compile(command=["make", "CFLAGS=-O2"], env={'LANG': 'C'}, timeout=2400))
#f_monitor = create_build_factory("git://repo.or.cz/qemu/qmp-unstable.git", branch="queue/monitor")

# trivial-patches - stefanha / branch: trivial-patches
f_trivialp = factory.BuildFactory()
f_trivialp.addStep(Git(repourl="git://repo.or.cz/qemu/stefanha.git", branch="trivial-patches", timeout=2400, mode='copy', retry=(5 * 60, 3), locks=[git_fetch_repoorcz_lock.access('exclusive')]))
f_trivialp.addStep(Configure(command="./configure", env={'LANG': 'C'}))
f_trivialp.addStep(Compile(command=["make", "CFLAGS=-O2"], env={'LANG': 'C'}, timeout=2400))
f_trivialp = create_build_factory("git://repo.or.cz/qemu/stefanha.git", branch="trivial-patches")

# agraf / s390-next
f_s390 = factory.BuildFactory()
f_s390.addStep(Git(repourl="git://repo.or.cz/qemu/agraf.git", branch="s390-next", timeout=2400, mode='copy', retry=(5 * 60, 3), locks=[git_fetch_repoorcz_lock.access('exclusive')]))
f_s390.addStep(Configure(command="./configure", env={'LANG': 'C'}))
f_s390.addStep(Compile(command=["make", "CFLAGS=-O2"], env={'LANG': 'C'}, timeout=2400))
f_s390 = create_build_factory("git://repo.or.cz/qemu/agraf.git", branch="s390-next")

# agraf / ppc-next
f_ppc = factory.BuildFactory()
f_ppc.addStep(Git(repourl="git://repo.or.cz/qemu/agraf.git", branch="ppc-next", timeout=2400, mode='copy', retry=(5 * 60, 3), locks=[git_fetch_repoorcz_lock.access('exclusive')]))
f_ppc.addStep(Configure(command="./configure", env={'LANG': 'C'}))
f_ppc.addStep(Compile(command=["make", "CFLAGS=-O2"], env={'LANG': 'C'}, timeout=2400))
f_ppc = create_build_factory("git://repo.or.cz/qemu/agraf.git", branch="ppc-next")

# agraf / xen-next
f_xen_next = factory.BuildFactory()
f_xen_next.addStep(Git(repourl="git://repo.or.cz/qemu/agraf.git", branch="xen-next", timeout=2400, mode='copy', retry=(5 * 60, 3), locks=[git_fetch_repoorcz_lock.access('exclusive')]))
f_xen_next.addStep(Configure(command="./configure", env={'LANG': 'C'}))
f_xen_next.addStep(Compile(command=["make", "CFLAGS=-O2"], env={'LANG': 'C'}, timeout=2400))
f_xen_next = create_build_factory("git://repo.or.cz/qemu/agraf.git", branch="xen-next")

# mingw32 build (based on Gerd Hoffmann's buildbot config)
f_mingw32 = factory.BuildFactory()
f_mingw32.addStep(Git(repourl="git://git.qemu-project.org/qemu.git", timeout=2400, mode='copy', retry=(5 * 60, 3), locks=[git_fetch_qemuorg_lock.access('exclusive')]))
f_mingw32.addStep(Configure(command=["./configure", "--cross-prefix=i686-pc-mingw32-"], env={'LANG': 'C'}))
f_mingw32.addStep(Compile(command=["make", "CFLAGS=-O2"], env={'LANG': 'C'}, timeout=2400))
f_mingw32 = create_build_factory("git://git.qemu-project.org/qemu.git", configure_args=["--cross-prefix=i686-pc-mingw32-"])

# openbsd build (based on Gerd Hoffmann's buildbot config)
f_openbsd = factory.BuildFactory()
f_openbsd.addStep(Git(repourl="git://git.qemu-project.org/qemu.git", timeout=2400, mode='copy', retry=(5 * 60, 3), locks=[git_fetch_qemuorg_lock.access('exclusive')]))
f_openbsd.addStep(ShellCommand(command=["./configure", "--extra-cflags=-Wno-redundant-decls"], env={'LANG': 'C'}))
f_openbsd.addStep(Compile(command=["gmake", "CFLAGS=-O2"], env={'LANG': 'C'}, timeout=2400))
f_openbsd = create_build_factory("git://git.qemu-project.org/qemu.git", make="gmake", configure_args=["--extra-cflags=-Wno-redundant-decls"])

# xen40 branches
f_xen40 = factory.BuildFactory()
f_xen40.addStep(Git(repourl="git://git.qemu-project.org/qemu.git", timeout=2400, mode='copy', retry=(5 * 60, 3), locks=[git_fetch_qemuorg_lock.access('exclusive')]))
f_xen40.addStep(Configure(command="./configure --enable-xen --extra-cflags=-I/home/build/install/xen-4.0.x/include/ --extra-ldflags=-L/home/build/install/xen-4.0.x/lib64/", env={'LANG': 'C'}))
f_xen40.addStep(Compile(command=["make", "CFLAGS=-O2"], env={'LANG': 'C'}, timeout=2400))
f_xen40 = create_build_factory("git://git.qemu-project.org/qemu.git", configure_args=["--enable-xen", "--extra-cflags=-I/home/build/install/xen-4.0.x/include/", "--extra-ldflags=-L/home/build/install/xen-4.0.x/lib64/"])

# xen41 branches
f_xen41 = factory.BuildFactory()
f_xen41.addStep(Git(repourl="git://git.qemu-project.org/qemu.git", timeout=2400, mode='copy', retry=(5 * 60, 3), locks=[git_fetch_qemuorg_lock.access('exclusive')]))
f_xen41.addStep(Configure(command="./configure --enable-xen --extra-cflags=-I/home/build/install/xen-4.1.x/include/ --extra-ldflags=-L/home/build/install/xen-4.1.x/lib64/", env={'LANG': 'C'}))
f_xen41.addStep(Compile(command=["make", "CFLAGS=-O2"], env={'LANG': 'C'}, timeout=2400))
f_xen41 = create_build_factory("git://git.qemu-project.org/qemu.git", configure_args=["--enable-xen", "--extra-cflags=-I/home/build/install/xen-4.1.x/include/", "--extra-ldflags=-L/home/build/install/xen-4.1.x/lib64/"])

# xen42 branches
f_xen42 = factory.BuildFactory()
f_xen42.addStep(Git(repourl="git://git.qemu-project.org/qemu.git", timeout=2400, mode='copy', retry=(5 * 60, 3), locks=[git_fetch_qemuorg_lock.access('exclusive')]))
f_xen42.addStep(Configure(command="./configure --enable-xen --extra-cflags=-I/home/build/install/xen-4.2.x/include/ --extra-ldflags=-L/home/build/install/xen-4.2.x/lib/", env={'LANG': 'C'}))
f_xen42.addStep(Compile(command=["make", "CFLAGS=-O2"], env={'LANG': 'C'}, timeout=2400))
f_xen42 = create_build_factory("git://git.qemu-project.org/qemu.git", configure_args=["--enable-xen", "--extra-cflags=-I/home/build/install/xen-4.2.x/include/", "--extra-ldflags=-L/home/build/install/xen-4.2.x/lib/"])

# xen_unstable branches
f_xen_unstable = factory.BuildFactory()
f_xen_unstable.addStep(Git(repourl="git://git.qemu-project.org/qemu.git", timeout=2400, mode='copy', retry=(5 * 60, 3), locks=[git_fetch_qemuorg_lock.access('exclusive')]))
f_xen_unstable.addStep(Configure(command="./configure --enable-xen --extra-cflags=-I/home/build/install/xen-unstable/include/ --extra-ldflags=-L/home/build/install/xen-unstable/lib/", env={'LANG': 'C'}))
f_xen_unstable.addStep(Compile(command=["make", "CFLAGS=-O2"], env={'LANG': 'C'}, timeout=2400))
f_xen_unstable = create_build_factory("git://git.qemu-project.org/qemu.git", configure_args=["--enable-xen", "--extra-cflags=-I/home/build/install/xen-unstable/include/", "--extra-ldflags=-L/home/build/install/xen-unstable/lib/"])

# macosx build (based on Gerd Hoffmann's buildbot config)
f_macosx = factory.BuildFactory()
f_macosx.addStep(Git(repourl="git://git.qemu-project.org/qemu.git", timeout=2400, mode='copy', retry=(5 * 60, 3), locks=[git_fetch_qemuorg_lock.access('exclusive')]))
f_macosx.addStep(Configure(command=["./configure", "--prefix=/sw"], env={'LANG': 'C'}))
f_macosx.addStep(Compile(command=["make", "CFLAGS=-O2"], env={'LANG': 'C'}, timeout=2400))
f_macosx = create_build_factory("git://git.qemu-project.org/qemu.git", configure_args=["--prefix=/sw"])

# linaro qemu-arm / target-arm.next
f_arm_target = factory.BuildFactory()
f_arm_target.addStep(Git(repourl="git://git.linaro.org/people/pmaydell/qemu-arm.git", branch="target-arm.next", timeout=2400, mode='copy', retry=(5 * 60, 3), locks=[git_fetch_linaro_lock.access('exclusive')]))
f_arm_target.addStep(Configure(command="./configure", env={'LANG': 'C'}))
f_arm_target.addStep(Compile(command=["make", "CFLAGS=-O2"], env={'LANG': 'C'}, timeout=2400))
f_arm_target = create_build_factory("git://git.linaro.org/people/pmaydell/qemu-arm.git", branch="target-arm.next")

# linaro qemu-arm / arm-devs.next
f_arm_devs = factory.BuildFactory()
f_arm_devs.addStep(Git(repourl="git://git.linaro.org/people/pmaydell/qemu-arm.git", branch="arm-devs.next", timeout=2400, mode='copy', retry=(5 * 60, 3), locks=[git_fetch_linaro_lock.access('exclusive')]))
f_arm_devs.addStep(Configure(command="./configure", env={'LANG': 'C'}))
f_arm_devs.addStep(Compile(command=["make", "CFLAGS=-O2"], env={'LANG': 'C'}, timeout=2400))
f_arm_devs = create_build_factory("git://git.linaro.org/people/pmaydell/qemu-arm.git", branch="arm-devs.next")

# afaerber qemu-cpu / qom-cpu
f_afaerber = factory.BuildFactory()
f_afaerber.addStep(Git(repourl="git://github.com/afaerber/qemu-cpu.git", branch="qom-cpu", timeout=2400, mode='copy', retry=(5 * 60, 3), locks=[git_fetch_github_lock.access('exclusive')]))
f_afaerber.addStep(Configure(command="./configure", env={'LANG': 'C'}))
f_afaerber.addStep(Compile(command=["make", "CFLAGS=-O2"], env={'LANG': 'C'}, timeout=2400))
f_afaerber = create_build_factory("git://github.com/afaerber/qemu-cpu.git", branch="qom-cpu")


#####################################################################
Expand Down

0 comments on commit 58a2d8a

Please sign in to comment.