master.cfg

# -*- python -*-
# ex: set syntax=python:

# This is the buildmaster config file. It must be installed as
# 'master.cfg' in your buildmaster's base directory.

# This is the dictionary that the buildmaster pays attention to. We also use
# a shorter alias to save typing.
c = BuildmasterConfig = {}

#=========================================================================
#  Shared Globals
#=========================================================================
##### Web Locators
# Buildbot http server
BB_PORT=str(8010)
# URL where buildbot's internal web server is visible
BUILDBOT_URL="http://lsst-buildx.ncsa.illinois.edu:"+BB_PORT+"/"

# Web access to DM git repositories
LSST_DMS="git@git.lsstcorp.org:LSST/DMS/"
# FUTURE: Web access for DM documentation
DOXY_URL="http://lsst-web.ncsa.illinois.edu/~buildbot/doxygen"
DOXY_DEST="lsst-build3.ncsa.illinois.edu:/lsst/home/buildbot/public_html/doxygen"

# Slave systems - all are available only 2 are currently used
#BUILDBOT_SLAVE1="lsst-buildx1.ncsa.illinois.edu"
BUILDBOT_SLAVE2="lsst-buildx2.ncsa.illinois.edu"
#BUILDBOT_SLAVE3="lsst-buildx3.ncsa.illinois.edu"
BUILDBOT_SLAVE8="lsst-dev.ncsa.illinois.edu"

# Slave Aliases (can have multiple aliases hosted on same slave)
REBUILD_SLAVE         = BUILDBOT_SLAVE2
STACK_SLAVE           = BUILDBOT_SLAVE8

# Master's Authentication file
MASTER_WORK_DIR = "/usr/local/home/buildbot/master"
HTPASSWD = MASTER_WORK_DIR+"/.htpasswd"

# Slaves' Primary Directories 
BUILDBOT_HOME = "/lsst/home/buildbot/RHEL6/"
SLAVE_HOME_DIR = BUILDBOT_HOME+"/buildslaves"
SLAVE_BUILDS_DIR = BUILDBOT_HOME+"/builds"
SLAVE_SCRIPTS_DIR = BUILDBOT_HOME+ "/scripts"
LDEV_FAILED_LOGS = "/lsst/home/lsstsw/build/FailedLogs"
BUILDBOT_SCRIPTS = "/home/lsstsw/buildbot-scripts"

# Make sure whatever email provided below is listserv-allowed.
BUILDBOT_EMAIL="square@lists.lsst.org"
# Due to BB requirement that a statically addressed email MUST be sent out if
# if the mail_notifier is invoked, a non-archiving dead-letter box was created.
DEAD_LETTER_FROM="buildbot-robot@lsstcorp.org"
DEAD_LETTER_TO="lsst-buildbot@lsstcorp.org"

####### BUILDSLAVES
# The 'slaves' list defines the set of recognized buildslaves. Each element is
# a BuildSlave object, specifying a unique slave name and password.  The same
# slave name and password must be configured on the slave.
from buildbot.buildslave import BuildSlave

####
# These slave names match the subdirectory where the build logs are maintained.
#        and where the buildslave's configuration is found. Keep this template.
#
# Passwords are recorded in BbConfig.py so master.cfg can be git-archived.
import BbConfig
SlavePassword = BbConfig.getSecret()
EMAIL_ON_MISSING=["sqre-admin@lists.lsst.org", "bglick@illinois.edu"]

c['slaves'] = [
    BuildSlave("BUILDBOT_SLAVE1", SlavePassword, notify_on_missing=EMAIL_ON_MISSING, missing_timeout=300, max_builds=1),
    BuildSlave("BUILDBOT_SLAVE2", SlavePassword, notify_on_missing=EMAIL_ON_MISSING, missing_timeout=300, max_builds=1),
    BuildSlave("BUILDBOT_SLAVE3", SlavePassword, notify_on_missing=EMAIL_ON_MISSING, missing_timeout=300, max_builds=1),
    BuildSlave("BUILDBOT_SLAVE8", SlavePassword, notify_on_missing=EMAIL_ON_MISSING, missing_timeout=300, max_builds=1)
]

# 'slavePortnum' defines the TCP port to listen on for connections from slaves.
# This must match the value configured in the buildslaves (w/ --master option)
c['slavePortnum'] = 9989


####### SCHEDULERS

# Configure the Schedulers, which decide how to react to incoming changes.  

from buildbot.schedulers import timed
from buildbot.schedulers.forcesched import ForceScheduler
from buildbot.schedulers.forcesched import BaseParameter
from buildbot.schedulers.forcesched import NestedParameter
from buildbot.schedulers.forcesched import StringParameter
from buildbot.schedulers.forcesched import UserNameParameter
from buildbot.schedulers.forcesched import CodebaseParameter

c['schedulers'] = []
#c['schedulers'].append(Nightly(name="testConnect",
#     hour=15, minute=10, builderNames=["testConnect"], branch='master'))
c['schedulers'].append(ForceScheduler(name="demoForce",builderNames=["testConnect"]))
c['schedulers'].append(ForceScheduler(
    name="force",
    builderNames=[ "DM_stack" ], 
    codebases=[CodebaseParameter(
        codebase="", branch=None, revision=None, repository=None, project=None )],
    properties= [ NestedParameter( name="",
        fields=[ 
            UserNameParameter( name="email", label="Your full email:",
                required=True ),
            StringParameter( name="branches", label="Branches (blank separated):", size=30 )])] 
))

c['schedulers'].append(
    timed.Nightly(name='every1hour',
                  branch='master', # default branch
                  builderNames=[ "DM_stack" ],
                  properties = { "email": "everyman <dm-devel@lists.lsst.org>", "branches": "" },
                  hour=[1,19],
                  minute=42,
#                  onlyIfChanged=True))
              ))

####### LOCKS

from buildbot import locks

# Ensure that a single build executes at a time on the DM Production stack
build_lock = locks.SlaveLock("slave_builds", maxCount = 1, 
     maxCountForSlave = {'LSST_SLAVE8': 1})

####### BUILDERS

# The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
# what steps, and which slaves can execute them.  Note that any particular build will
# only take place on one slave.

from buildbot.process.factory import BuildFactory
from buildbot.steps.source.git import Git
from buildbot.steps.shell import ShellCommand
from buildbot.process.properties import Property
from buildbot.config import BuilderConfig
from buildbot.status.results import SUCCESS, WARNINGS, FAILURE

c['builders'] = []


#######
# DM stack    *** This is the actual Production Stack on lsst-dev ***
#
# This slave needs to conform to the directory layout of the pre-existing
# 'lsstsw' account which itself mimics the lsstsw installation's layout.
#  It is not buildbot friendly.
#
#  The slavebuilddir base is one level higher than the invoking account
#  home directory so that the package's residuals are positioned as required.
#
# This build assumes the full DM stack structure exists. If that is not true,
# the build needs to fail and backup recovery methods need to recover the full
# stack from the last valid state. This stack is the lsst-dev shared DM stack.

factory = BuildFactory()
factory.addStep(ShellCommand( 
         command=[BUILDBOT_SCRIPTS+"/lsstswBuild.sh",
             "--branch", Property("branches"),
             "--build_number", Property("buildnumber")],
         workdir="lsstsw", description="Build:", timeout=16200,
         descriptionSuffix=[ Property("branches",default=" "), "master" ],
         ))

c['builders'].append(BuilderConfig(
    name="DM_stack",
    description=["DM stack build"],
    slavename="BUILDBOT_SLAVE8",
    locks=[build_lock.access('exclusive')],
    builddir='builds/DM_stack',
    slavebuilddir='/lsst/home',
    factory=factory))

#######
# testConnect    this is a demo which only  tests a VM buildslave connection
factory = BuildFactory()
factory.addStep(ShellCommand( command=["pwd"], description="PrintWD"))
factory.addStep(ShellCommand( command=["ls", "-al"], description="SimpleLs"))
factory.addStep(ShellCommand( 
         command=[SLAVE_SCRIPTS_DIR+"/bbLsstswBuild.sh",
             "--branch",  Property("branches"),
             "--builder_name", Property("buildername"),
             "--build_number", Property("buildnumber"),
             "--step_number", Property("stepnumber"),
             "--email", Property("email")],
         workdir="work", description="bbLsstswBuild", timeout=16200,
         ))

c['builders'].append(BuilderConfig(
    name="testConnect",
    description=["check com link"],
    slavename="BUILDBOT_SLAVE2",
    builddir='builds/testConnect',
    slavebuilddir=SLAVE_BUILDS_DIR+'/testConnect',
    factory=factory))


####### STATUS TARGETS

# 'status' is a list of Status Targets. The results of each build will be
# pushed to these targets. buildbot/status/*.py has a variety to choose from,
# including web pages, email senders, and IRC bots.

c['status'] = []

#-- Send Build Status via email
from buildbot.status.builder import Results
import cgi, datetime
import smtplib
from buildbot.status import html
from buildbot.status import mail
from string import maketrans
import os
import re
import email.utils as email_utils

# Build the Body of the email message used by the formatter
def build_mailBody( mode, name, build, result, master_status, buildProperties, buildEmail ):
    buildBranches = buildProperties['branches']
    buildNumber = buildProperties['buildnumber']
    realName, emailAddr = email_utils.parseaddr( buildEmail )
    buildSlaveHostName = build.getSlavename()
    eupsPath = "%s/lsstsw/stack" %( buildProperties['workdir'] )
    stdio = MASTER_WORK_DIR+"/builds/"+name+"/"+str(buildNumber)+"-log-shell-stdio"
    # Find eups tag number within the stdio file
    f = open(stdio, 'r')
    a = re.compile("(?<=BUILD ID: )b[0-9]+")
    eupsTag = re.findall(a, f.read())
    print 'FEFEFE ',eupsTag
    f.close()

    text = list()
    text.append(u'<h3>Build status: %s</h3>' % result.upper())
    text.append(u'<table cellspacing="10">')
    text.append(u"<tr><td>Build </td><td><b>%s . . . . @ %s</b></td>" %(name,buildNumber))
    text.append(u"<tr><td>Build Branch(es) </td><td><b>%s master</b></td>" %(buildBranches))
    text.append(u"<tr><td>Eups Tag </td><td><b>%s</b></td></tr>" % eupsTag[0])
    text.append(u"<tr><td>Eups Path </td><td><b>%s</b></td></tr>" % eupsPath)
    if master_status.getURLForThing(build):
        text.append(u'<tr><td>Stack Build Summary & stdio</td><td><a href="%s">%s</a></td></tr>'
                    % (master_status.getURLForThing(build),
                       master_status.getURLForThing(build)))
        if result == 'failure':
            text.append(u'<tr><td>Failed Package Info at</td><td><b>%s:%s/%s/</b></td></tr>'
                    % (BUILDBOT_SLAVE8, LDEV_FAILED_LOGS, buildNumber))
            text.append(u'<tr><td>.......Log</td><td><b>*/_build.log</b></td></tr>')
            text.append(u'<tr><td>.......Manifest</td><td><b>*/_build.tags</b></td></tr>')
            text.append(u'<tr><td>.......Bot Script</td><td><b>*/_build.sh</b> -- for info, not use</td></tr>')
            text.append(u'<tr><td>.......Unittest</td><td><b>*/*.failed</b></td></tr>')

        text.append(u'<tr><td>Build Reason</td><td>%s</td></tr>' % build.getReason())
        if  realName != None:
            text.append(u'<tr><td>Built on behalf of </td><td>%s  %s </td></tr>' % ( realName, emailAddr ))
        text.append(u'</table>')

        logs = build.getLogs()
        text.append(u"<h3>Build Log</h3>")
        limit_lines = 40
        for log in logs:
            if log.getName() != 'stdio':
                continue
            status, dummy = log.getStep().getResults()
            # status=0: SUCCESS status=1:WARNINGS  status=2:FAILURE
            name = "%s.%s" % (log.getStep().getName(), log.getName())
            status, dummy = log.getStep().getResults()
            content = log.getText().splitlines() # Note: can be VERY LARGE
            url = u'%s/steps/%s/logs/%s' % (master_status.getURLForThing(build),
                                           log.getStep().getName(),
                                           log.getName())
            unilist = list()
            # Tags indicating error output in log: '^*** ' or  ' ERROR '
            p = re.compile(r"\*\*\* *| ERROR *")
            if p :
                errorLines = []
                for line in content:
                    if p.search(line):
                         errorLines.append(line)
                if len(errorLines) == 0:
                    # No SCONS errors so send last 50 lines of log
                    text.append(u'<h4>Showing tail of build stdio: <a href="%s">logs</a>, Status: %s</h4>' % (url, status))
                    sendTo = ( emailAddr if emailAddr != '' else BUILDBOT_EMAIL )
                    for line in content[len(content)-limit_lines:]:
                        unilist.append(cgi.escape(unicode(line,'utf-8')))
                else:
                    # SCONS error so send the aggregated error messages
                    text.append(u'<h4>Failure: showing error messages in: <a href="%s">stdio</a>, Status: %s</h4>' % (url, status))
                    sendTo = ( emailAddr if emailAddr != '' else BUILDBOT_EMAIL )
                    for line in errorLines:
                        unilist.append(cgi.escape(unicode(line,'utf-8')))
            text.append(u'<pre>'.join([uniline for uniline in unilist]))
            text.append(u'</pre>')
            text.append(u'<h3>A directory where a build failure occurred is overwritten next build; its failed-build logs are moved to: "%s:%s/%s/"</h4>' % (BUILDBOT_SLAVE8, LDEV_FAILED_LOGS, buildNumber))
    return text,sendTo

# Notify mail_list
def html_MasterFormatter(mode, name, build, results, master_status ):
    """
    Provide a customized message to Buildbot's MailNotifier. 
    If:  
      * Build status was 'failure' or 'warning', send to [dm-devel]; or
      * Build status was 'success' and build was 'forced', send only to 
        user-supplied email.

    The end of the log is included. Message content is formatted as html.
    """
    result = Results[results]
    buildProperties = build.getProperties()
    scheduler = buildProperties["scheduler"]
    buildEmail = buildProperties["email"]
    realName, emailAddr = email_utils.parseaddr( buildEmail )

    # Build complete email message and load into 'text'
    text,receivers = build_mailBody(mode, name, build, result, master_status, buildProperties, buildEmail)

    # if Build NOT 'forced' && 'success' status, exit without mailing.
    # if Build git-change forced && 'success' status, exit without mailing.
    if result == "failure"  or  ( scheduler == "force" and realName != "everyman" ): 
         print "BB will send dynamic Mail Notification"
         reason = build.getReason()
         print buildProperties
         print "BuildName:%s Result:%s Reason:%s realname:%s  emailAddr:%s" %(name, result, reason, realName, emailAddr)
         # Following sends out mail to dynamic email list 
         sender = BUILDBOT_EMAIL
         message = """\
From: %s
To: %s
MIME-Version: 1.0
Content-type: text/html
Subject: %s ... Status summary for your review.

<br>%s
""" % ( sender, receivers, name, u"\n".join(text) )
         try:
             smtpObj = smtplib.SMTP('localhost')
             smtpObj.sendmail(sender, receivers.split(','), message)
             print "Successfully sent email to:", receivers
         except SMTPException:
             print "Error: unable to send email"
    else:
         print "BB will NOT send dynamic Mail Notification"

# Always send the official mailNotifier mail to static email list
    return {
        'body': u"\n".join(text),
        'type': 'html'
        }

# Notify mail_list on Master or Branch build failures
c['status'].append(mail.MailNotifier(fromaddr=DEAD_LETTER_FROM,
    mode=["failing", "passing", "warnings"],
    subject="%(builder)s ... build status",
    sendToInterestedUsers=False,
    extraRecipients=[ DEAD_LETTER_TO ],
    messageFormatter=html_MasterFormatter))


from buildbot.status.html import WebStatus
from buildbot.status.web.authz import Authz
from buildbot.status.web.auth import HTPasswdAuth

# this takes a BuildStatus object
# so we compare the owner of the build with the logged-in web user
# and let (1) people stop their own builds (2) BBadmins stop anyone's build

def canStopBuild( username, build_status ):
     buildInfo = build_status.getProperties()
     if "owner" in buildInfo and buildInfo["owner"].startswith(username):
         return True
     if username in ('raa', 'frossie', 'jhoblitt'):
        return True
     else:
        return False     

# this takes a BuildRequest object so we have to chase down the original owner
# again we let people cancel their own build requests; and BBadmins cancel any.

def canCancelPendingBuild( username, build_request ):
     buildInfo = build_request.original_request.properties.getProperties()
     if "owner" in buildInfo and buildInfo["owner"].startswith(username):
         return True
     if username in ('raa', 'frossie', 'jhoblitt'):
        return True
     else:
        return False     

authz_cfg=Authz(auth=(HTPasswdAuth(HTPASSWD)),
    # change any of these to True to enable; see the manual for more options
    gracefulShutdown = False,
    forceBuild = True,
    forceAllBuilds = False,
    pingBuilder = False,
    stopBuild = canStopBuild,
    stopAllBuilds = False,
    cancelPendingBuild = canCancelPendingBuild,
)
c['status'].append(WebStatus(http_port='tcp:'+BB_PORT, authz=authz_cfg))

####### PROJECT IDENTITY

# the 'title' string will appear at the top of this buildbot
# installation's WebStatus home page (linked to the
# 'titleURL') and is embedded in the title of the waterfall HTML page.

c['title'] = "LSST Data Management"
c['titleURL'] = "https://dev.lsstcorp.org/buildx"

# the 'buildbotURL' string should point to the location where the buildbot's
# internal web server (usually the WebStatus page) is visible. This
# typically uses the port number set in the Waterfall 'status' entry, but
# with an externally-visible host name which the buildbot cannot figure out
# without some help.

c['buildbotURL'] = BUILDBOT_URL

####### DB URL

c['db'] = {
    # This specifies what database buildbot uses to store its state.  You can leave
    # this at its default for all but the largest installations.
    'db_url' : "sqlite:///state.sqlite",
}

### Notifications

#
# git clone https://github.com/pricingassistant/buildbot-status-hipchat.git
# ln -s buildbot-status-hipchat/hipchat.py

import hipchat
c['status'].append(hipchat.HipChatStatusPush("d5574564a9db79569265a4c5dfdfd2", "856354"))