master.cfg

#!/usr/bin/env python
# -*- python -*-
# vim: syntax=python

import time

# basis for the revision string
base_rev_major = "0."
base_rev = base_rev_major
master_repo = 'https://github.com/springlobby/springlobby.git'

WIN_STRIP = "/home/buildbot/buildbot/springlobby/mxe/build/mxe/usr/bin/i686-w64-mingw32.static.posix-strip"
WIN_EXE_DIR = "build-msw-default/src/"

from buildbot.plugins import util
from admin_users import users as admins

class SpringAuth(util.CustomAuth):
    def check_credentials(self, user, password):
        if user in admins.keys():
            if password == admins[user]:
                return True
        else:
            return False


authz = util.Authz(
    allowRules=[
        util.AnyControlEndpointMatcher(role="admins"),
    ],
    roleMatchers=[
        util.RolesFromEmails(admins=["turboss","abma"]),
    ]
)

c = BuildmasterConfig = {}

c['workers'] = []
c['builders'] = []
c['schedulers'] = []
c['status'] = []
c['buildbotNetUsageData'] = None

c['db_url'] = "sqlite:///state.sqlite"

c['www'] = {
	'port': 8010,
	'avatar_methods': [],
	'auth': SpringAuth(),
	'authz':authz,
	'plugins': {
		'waterfall_view': {},
		'console_view': {},
		'grid_view': {}
	}
}

# WORKERS #

from buildbot.worker import Worker

# import the pw vars from another file. this way we can keep config in git
import pw

c['workers'].append(Worker("testbot", pw.test, max_builds=1))

# CONFIGS #

# maybe see: /opt/mingw32/usr/i686-w64-mingw32/share/cmake/mxe-conf.cmake
common_win_opts = [
    '-DCMAKE_BUILD_TYPE=RelWithDebInfo',
    '-DAUX_VERSION=msw',
    '-DCMAKE_INSTALL_PREFIX:PATH=.',
]
no_opti = ['-DCMAKE_CXX_FLAGS:STRING=-O0']
common_nix_opts = []
build_configs = dict()  # name -> options_string
build_configs['msw-default'] = common_win_opts + ['-DCMAKE_CXX_FLAGS:STRING=-DNDEBUG']
build_configs['msw-debug'] = common_win_opts + ['-DCMAKE_BUILD_TYPE=DEBUG', '-DCMAKE_CXX_FLAGS:STRING=-g3 -ggdb -O0']
build_configs['msw-no-optionals'] = common_win_opts + no_opti

build_configs['linux-default'] = no_opti + common_nix_opts
build_configs['linux-no-optionals'] = ['-DCMAKE_BUILD_TYPE=PROFILE'] + no_opti + common_nix_opts

common_mac_opts = "-DCMAKE_TOOLCHAIN_FILE:STRING=../cmake/Toolchain-osx.cmake"
# build_configs['osx-default'] = ' ' + common_mac_opts
# builds with extra env file sourced
build_env_configs = dict()  # name -> options_string
# disabled cause compiler segfaults
# build_env_configs['gcc-svn-linux-default'] = "-DCMAKE_CXX_COMPILER=/opt/gcc-svn/bin/g++ -DCMAKE_C_COMPILER=/opt/gcc-svn/bin/gcc -DCMAKE_LINKER=/opt/gcc-svn/bin/ld" + no_opti + common_nix_opts
build_env_configs['wx2.9-linux-default'] = no_opti + common_nix_opts
build_env_configs['clang-linux-default'] = [
                                               '-DCMAKE_CXX_COMPILER=/opt/llvm/bin/clang++',
                                               '-DCMAKE_C_COMPILER=/opt/llvm/bin/clang',
                                               '-DCMAKE_LINKER=/opt/llvm/bin/llvm-ld'] + no_opti + common_nix_opts
env_builds = []
# env_builds.append( ( "/opt/gcc-svn.PATH", "gcc-svn-linux-default" ) )
env_builds.append(("/opt/wx2.9.PATH", "wx2.9-linux-default"))
env_builds.append(("/opt/llvm.PATH", "clang-linux-default"))


# CHANGESOURCES #

def changeIsImportant(change):
    for name in change.files:
        if (name.endswith('.cpp') or name.endswith('.c') or
                name.endswith('.hpp') or name.endswith('.h') or
                name.endswith('.cmake') or name.endswith('.hh') or
                name.endswith('.inl') or name.endswith('CMakeLists.txt')):
            return True
    return False


from buildbot.changes.pb import PBChangeSource

c['change_source'] = []
c['change_source'].append(PBChangeSource())

# SCHEDULERS #

from buildbot.schedulers.filter import ChangeFilter
from buildbot.schedulers.basic import AnyBranchScheduler
from buildbot.schedulers.forcesched import ForceScheduler
from buildbot.schedulers import timed

c['schedulers'].append(AnyBranchScheduler(
    name="develop_Scheduler",
    # change_filter=ChangeFilter(category_re = '.*develop.*'),
    treeStableTimer=60,
    fileIsImportant=changeIsImportant,
    builderNames=["release"]))


c['schedulers'].append(ForceScheduler(
    name='force_sched',
    builderNames=["docs", "release", "mxe"]))


# BUILDERS #

from buildbot.process.factory import BuildFactory
from buildbot.steps.source.git import Git
from buildbot.steps.shell import Compile, ShellCommand, WithProperties


class SyncSubmodules(ShellCommand):
    name = "syncsubmodules"
    description = 'syncing .gitmodules with .git/config'
    descriptionDone = 'synced submodule paths'
    command = ['git', 'submodule', 'sync', '--recursive']


class InitSubmodules(ShellCommand):
    name = "submodules"
    description = 'updating submodules'
    descriptionDone = 'updated submodules'
    command = ['git', 'submodule', 'update', '--init', '--recursive']


class BuildDocs(ShellCommand):
    name = "execute doxygen"
    description = ["generating doxygen documentation"]
    descriptionDone = ["docs generated"]
    command = ["/bin/sh", "./tools/update-docs.sh"]


class WindowsBinary(ShellCommand):
    name = "windows binary"
    description = ["making windows binary"]
    descriptionDone = ["windows binary"]
    command = ["zip", "-jr", WithProperties("build-msw-default/springlobby-" + base_rev + "%(buildnumber)s-win32.zip"),
               "build-msw-default/wininst"]


class WindowsLocales(ShellCommand):
    name = "windows locale"
    description = ["adding locales to win zip"]
    descriptionDone = ["windows locale added"]
    command = ["/bin/sh", "./tools/buildbot-add-locale-win-zip.sh",
               WithProperties("springlobby-" + base_rev + "%(buildnumber)s-win32.zip")]

class UpdateEbuild(ShellCommand):
    name = "update ebuild"
    description = "updating ebuild"
    descriptionDone = "updated ebuild"
    command = ["/bin/bash", "./tools/update-ebuilds.sh", WithProperties(base_rev + "%(buildnumber)s")]


class UpdateTranslations(ShellCommand):
    name = "update translations"
    description = "updating translations"
    descriptionDone = "updated translations"
    command = ["/bin/bash", "./tools/update-translations-buildbot.sh"]


class UploadBuild(ShellCommand):
    name = "upload"

    def __init__(self, configname='linux-default', **kwargs):
        self.configname = configname
        ShellCommand.__init__(self, **kwargs)

        # mandatory for later (automatic) re-creation of step object
        self.addFactoryArguments(configname=configname)
        self.description = "uploading exe"
        self.descriptionDone = "uploaded exe"
        self.command = ["./tools/upload-build.py", "build-%s" % configname, WithProperties("%s", "branch")]


# common build config steps ---------------------------------------------@
class buildConfig(Compile):
    name = 'buildConfig'

    def __init__(self, configname='linux-default', jobs=1, release=False, **kwargs):
        self.configname = configname
        self.release = release
        Compile.__init__(self, **kwargs)
        self.haltOnFailure = True

        # mandatory for later (automatic) re-creation of step object
        self.addFactoryArguments(configname=configname)
        self.addFactoryArguments(jobs=jobs)
        self.addFactoryArguments(release=release)

        # self.name                      = self.configname + " build"
        self.description = ["building " + self.configname + " config"]
        self.descriptionDone = ["built " + self.configname + " config"]
        if self.release:
            self.command = ['make', 'BUILDBOT_RELEASE=1', '-k', '-j%d' % jobs, '-C', 'build-%s' % (self.configname)]
        else:
            self.command = ['make', '-k', '-j%d' % jobs, '-C', 'build-%s' % (self.configname)]


class buildEnvConfig(Compile):
    name = 'buildConfig'

    def __init__(self, env_file, configname, jobs=1, release=False, **kwargs):
        self.configname = configname
        self.release = release
        self.env_file = env_file
        Compile.__init__(self, **kwargs)
        self.haltOnFailure = True

        # mandatory for later (automatic) re-creation of step object
        self.addFactoryArguments(configname=configname)
        self.addFactoryArguments(jobs=jobs)
        self.addFactoryArguments(release=release)
        self.addFactoryArguments(env_file=env_file)
        if 'gcc-svn' in configname:
            self.addFactoryArguments(flunkOnFailure=False)
            self.addFactoryArguments(haltOnFailure=False)

        # self.name                      = self.configname + " build"
        self.description = ["building " + self.configname + " config"]
        self.descriptionDone = ["built " + self.configname + " config"]
        if self.release:
            self.command = ' '.join(
                ['source', self.env_file, '&&', 'make', 'BUILDBOT_RELEASE=1', '-k', '-j%d' % jobs, '-C',
                 'build-%s' % (self.configname)])
        else:
            self.command = ' '.join(
                ['source', self.env_file, '&&', 'make', '-k', '-j%d' % jobs, '-C', 'build-%s' % (self.configname)])

# TODO fail and stop the build if this fails
class CreateBuildDir(ShellCommand):
    name = "build dir create"

    def __init__(self, configname='linux-default', **kwargs):
        self.configname = configname
        ShellCommand.__init__(self, **kwargs)

        # mandatory for later (automatic) re-creation of step object
        self.addFactoryArguments(configname=configname)
        self.haltOnFailure = True

        self.description = ["creating dir %s" % (self.configname)]
        self.descriptionDone = ["%s dir created" % (self.configname)]
        self.command = ["/bin/bash", "./buildbot/create-build-dirs.sh", self.configname,
                        WithProperties(base_rev + "%(buildnumber)s"), ] + build_configs[self.configname]


class CreateEnvBuildDir(ShellCommand):
    name = "build dir create"

    def __init__(self, env_file, configname='linux-default', **kwargs):
        self.configname = configname
        self.env_file = env_file
        ShellCommand.__init__(self, **kwargs)

        # mandatory for later (automatic) re-creation of step object
        self.addFactoryArguments(configname=configname)
        self.addFactoryArguments(env_file=env_file)
        self.haltOnFailure = True

        self.description = ["creating dir %s" % (self.configname)]
        self.descriptionDone = ["%s dir created" % (self.configname)]
        self.command = ["source", self.env_file, "&&", "/bin/bash", "./buildbot/create-build-dirs.sh", self.configname,
                        str(time.time()), build_env_configs[self.configname]]


class CreateWinInstaller(ShellCommand):
    name = "create win installer"

    def __init__(self, configname='msw-default', **kwargs):
        self.configname = configname
        ShellCommand.__init__(self, **kwargs)

        # mandatory for later (automatic) re-creation of step object
        self.addFactoryArguments(configname=configname)
        self.haltOnFailure = True

        self.description = ["creating dir %s" % (self.configname)]
        self.descriptionDone = ["%s dir created" % (self.configname)]
        self.command = ["installer/make_installer.sh",
                        WithProperties("build-msw-default/springlobby-%(commit-description)s-win32.zip")]

# common build config steps ---------------------------------------------@
JOBS = '-j2'


class ReleaseFactory(BuildFactory):
    def __init__(self):
        BuildFactory.__init__(self)
        self.addStep(Git(repourl=master_repo, getDescription={"tags": True}))
        self.addStep(SyncSubmodules())
        self.addStep(InitSubmodules())
        self.addStep(CreateBuildDir('msw-default'))
        self.addStep(buildConfig(configname='msw-default', jobs=2, release=True))
        self.addStep(Compile(command=["make", 'BUILDBOT_RELEASE=1', JOBS, "-C", "build-msw-default", "pack", "dist"]))
        self.addStep(CreateWinInstaller())
        self.addStep(UploadBuild(configname='msw-default'))


class FullBuildFactory(BuildFactory):
    def __init__(self, dude):
        BuildFactory.__init__(self)
        self.addStep(Git(repourl=master_repo))
        self.addStep(SyncSubmodules())
        self.addStep(InitSubmodules())
        for name in set(build_configs.keys()) - set(['osx-default']):
            self.addStep(CreateBuildDir(name))
            self.addStep(buildConfig(configname=name, jobs=2))
        for env in env_builds:
            self.addStep(CreateEnvBuildDir(env[0], env[1]))
            self.addStep(buildEnvConfig(env[0], env[1], jobs=2))
        self.addStep(Compile(command=["make", "-j2", "-C", "build-linux-default", "distcheck"]))


class TestBuildFactory(BuildFactory):
    def __init__(self, dude):
        BuildFactory.__init__(self)
        self.addStep(Git(repourl=master_repo))
        self.addStep(SyncSubmodules())
        self.addStep(InitSubmodules())
        for env in env_builds:
            self.addStep(CreateEnvBuildDir(env[0], env[1]))
            self.addStep(buildEnvConfig(env[0], env[1], jobs=2))


class WinTempBuildFactory(BuildFactory):
    def __init__(self, who):
        BuildFactory.__init__(self)
        self.addStep(Git(repourl=master_repo))
        self.addStep(SyncSubmodules())
        self.addStep(InitSubmodules())
        self.addStep(CreateBuildDir('msw-default'))
        self.addStep(buildConfig(configname='msw-default', jobs=2))
        self.addStep(UploadBuild(configname='msw-default'))


class WinTempDebugBuildFactory(BuildFactory):
    def __init__(self, who):
        BuildFactory.__init__(self)
        self.addStep(Git(repourl=master_repo))
        self.addStep(SyncSubmodules())
        self.addStep(InitSubmodules())
        self.addStep(CreateBuildDir('msw-debug'))
        self.addStep(buildConfig(configname='msw-debug', jobs=2))
        self.addStep(UploadBuild(configname='msw-debug'))


class OSXBuildFactory(BuildFactory):
    def __init__(self, who):
        BuildFactory.__init__(self)
        self.addStep(Git(repourl=master_repo))
        self.addStep(SyncSubmodules())
        self.addStep(InitSubmodules())
        self.addStep(CreateBuildDir('osx-default'))
        self.addStep(buildConfig(configname='osx-default', jobs=1))
        self.addStep(UploadBuild(configname='osx-default'))


class CreateMXEBuildDir(ShellCommand):
    name = "createmxebuilddir"
    description = 'creating build dir'
    descriptionDone = 'created build dir'
    command = ['tools/mxe_create_builddir.sh']


class CompileMXEBuildDir(ShellCommand):
    name = "createmxebuilddir"
    description = 'compiling mxe'
    descriptionDone = 'compiled mxe'
    command = ['tools/mxe_compile.sh']


class MXEFactory(BuildFactory):
    def __init__(self):
        BuildFactory.__init__(self)
        self.addStep(Git(repourl=master_repo, mode="full"))
        self.addStep(CreateMXEBuildDir())
        self.addStep(CompileMXEBuildDir())


win32buildenv = {
	'CMAKE': '/home/buildbot/workers/springlobby/mxe/build/mxe/usr/bin/i686-w64-mingw32.static.posix-cmake',
	'CMAKE_PARAMS': '-DCMAKE_STRIP:PATH=/home/buildbot/workers/springlobby/mxe/build/mxe/usr/bin/i686-w64-mingw32.static.posix-strip',
	"LC_ALL": "C",
}

f2 = BuildFactory()
f2.addStep(Git(repourl=master_repo))
f2.addStep(BuildDocs())


c['builders'].append(
	{'name': "release",
	'workernames': ["testbot"],
	'builddir': "release",
	'factory': ReleaseFactory(),
	'env': win32buildenv
})

c['builders'].append({
	'name': "docs",
	'workernames': ["testbot"],
	'builddir': "docs",
	'factory': f2,
})

c['builders'].append({
	'name': 'mxe',
	'workernames': ["testbot"],
	'builddir': 'mxe',
	'factory': MXEFactory()
})

# CHANGE SOURCE #

from buildbot.changes.gitpoller import GitPoller

c['change_source'].append(GitPoller(master_repo, branches=['master']))

c['protocols'] = {'pb': {'port': 9999}}

####### DEBUGGING OPTIONS

# c['debugPassword'] = "debugpassword"
# c['manhole'] = buildbot.manhole.PasswordManhole("tcp:9999:interface=127.0.0.1", "admin", "password")

####### PROJECT IDENTITY

c['projectName'] = "SpringLobby"
c['projectURL'] = "https://springlobby.springrts.org/"
c['buildbotURL'] = "https://springlobby.springrts.com/buildbot/"