Permalink
Browse files

add buildlogger (separate test output)

  • Loading branch information...
1 parent 7b39ed7 commit 41db66ef99fbc0e499a366db8d341e787d17a7b1 Dan Crosta committed Feb 27, 2012
Showing with 376 additions and 12 deletions.
  1. +7 −0 SConscript.smoke
  2. +336 −0 buildscripts/buildlogger.py
  3. +33 −12 buildscripts/smoke.py
View
@@ -13,6 +13,13 @@ def add_exe( v ):
smokeEnv = testEnv.Clone()
smokeEnv['ENV']['PATH']=os.environ['PATH']
+
+# copy in any envrionment variables beginning with MONGO_; these
+# are used by buildscripts/buildlogger.py
+for name, value in os.environ.items():
+ if name.startswith('MONGO_'):
+ smokeEnv['ENV'][name] = value
+
smokeEnv.Alias( "dummySmokeSideEffect", [], [] )
smokeFlags = []
View
@@ -0,0 +1,336 @@
+"""
+buildlogger.py
+
+Wrap a command (specified on the command line invocation of buildlogger.py)
+and send output in batches to the buildlogs web application via HTTP POST.
+
+The script configures itself from environment variables:
+
+ required env vars:
+ MONGO_BUILDER_NAME (e.g. "Nightly Linux 64-bit")
+ MONGO_BUILD_NUMBER (an integer)
+ MONGO_TEST_FILENAME (not required when invoked with -g)
+
+ optional env vars:
+ MONGO_PHASE (e.g. "core", "slow nightly", etc)
+ MONGO_* (any other environment vars are passed to the web app)
+
+This script has two modes: a "test" mode, intended to wrap the invocation of
+an individual test file, and a "global" mode, intended to wrap the mongod
+instances that run throughout the duration of a mongo test phase (the logs
+from "global" invocations are displayed interspersed with the logs of each
+test, in order to let the buildlogs web app display the full output sensibly.)
+"""
+
+import functools
+import os
+import os.path
+import subprocess
+import sys
+import time
+import traceback
+import urllib2
+
+try:
+ import json
+except:
+ try:
+ import simplejson as json
+ except:
+ json = None
+
+# try to load the shared secret from settings.py
+# which will be one, two, or three directories up
+# from this file's location
+here = os.path.abspath(os.path.dirname(__file__))
+possible_paths = [
+ os.path.abspath(os.path.join(here, '..')),
+ os.path.abspath(os.path.join(here, '..', '..')),
+ os.path.abspath(os.path.join(here, '..', '..', '..')),
+]
+sys.path.extend(possible_paths)
+try:
+ from settings import buildlogger_secret
+except ImportError:
+ buildlogger_secret = None
+
+for path in possible_paths:
+ sys.path.remove(path)
+
+
+URL_ROOT = 'http://buildlogs.mongodb.org/'
+TIMEOUT_SECONDS = 10
+
+digest_handler = urllib2.HTTPDigestAuthHandler()
+digest_handler.add_password(
+ realm='buildlogs',
+ uri=URL_ROOT,
+ user='buildbot',
+ passwd=buildlogger_secret)
+url_opener = urllib2.build_opener(digest_handler)
+
+def url(endpoint):
+ if not endpoint.endswith('/'):
+ endpoint = '%s/' % endpoint
+
+ return '%s/%s' % (URL_ROOT.rstrip('/'), endpoint)
+
+def post(endpoint, data, headers=None):
+ data = json.dumps(data)
+
+ headers = headers or {}
+ headers.update({'Content-Type': 'application/json'})
+
+ req = urllib2.Request(url=url(endpoint), data=data, headers=headers)
+ response = url_opener.open(req, timeout=TIMEOUT_SECONDS)
+ response_headers = dict(response.info())
+ if response_headers.get('content-type') == 'application/json':
+ return json.load(response)
+ return response.read()
+
+def traceback_to_stderr(func):
+ """
+ decorator which logs any exceptions encountered to stderr
+ and returns none.
+ """
+ @functools.wraps(func)
+ def wrapper(*args, **kwargs):
+ try:
+ return func(*args, **kwargs)
+ except urllib2.HTTPError, err:
+ sys.stderr.write('error: HTTP code %d\n----\n' % err.code)
+ for k, v in dict(err.info()).items():
+ sys.stderr.write("%s: %s\n" % (k, v))
+ sys.stderr.write('\n')
+ sys.stderr.write(err.read())
+ sys.stderr.write('\n----\n')
+ sys.stderr.flush()
+ except:
+ sys.stderr.write('Traceback from buildlogger:\n')
+ traceback.print_exc(file=sys.stderr)
+ sys.stderr.flush()
+ return None
+ return wrapper
+
+
+@traceback_to_stderr
+def get_or_create_build(builder, buildnum, extra={}):
+ data = {'builder': builder, 'buildnum': buildnum}
+ data.update(extra)
+ response = post('build', data)
+ return response['id']
+
+@traceback_to_stderr
+def create_test(build_id, test_filename, test_command, test_phase):
+ response = post('build/%s/test' % build_id, {
+ 'test_filename': test_filename,
+ 'command': test_command,
+ 'phase': test_phase,
+ })
+ return response['id']
+
+@traceback_to_stderr
+def append_test_logs(build_id, test_id, log_lines):
+ post('build/%s/test/%s' % (build_id, test_id), data=log_lines)
+ return True
+
+@traceback_to_stderr
+def append_global_logs(build_id, log_lines):
+ """
+ "global" logs are for the mongod(s) started by smoke.py
+ that last the duration of a test phase -- since there
+ may be output in here that is important but spans individual
+ tests, the buildlogs webapp handles these logs specially.
+ """
+ post('build/%s' % build_id, data=log_lines)
+ return True
+
+@traceback_to_stderr
+def finish_test(build_id, test_id, failed=False):
+ post('build/%s/test/%s' % (build_id, test_id), data=[], headers={
+ 'X-Sendlogs-Test-Done': 'true',
+ 'X-Sendlogs-Test-Failed': failed and 'true' or 'false',
+ })
+ return True
+
+def run_and_echo(command):
+ """
+ this just calls the command, and returns its return code,
+ allowing stdout and stderr to work as normal. it is used
+ as a fallback when environment variables or python
+ dependencies cannot be configured, or when the logging
+ webapp is unavailable, etc
+ """
+ return subprocess.call(command)
+
+def wrap_test(command):
+ """
+ call the given command, intercept its stdout and stderr,
+ and send results in batches of 100 lines or 10s to the
+ buildlogger webapp
+ """
+
+ # get builder name and build number from environment
+ builder = os.environ.get('MONGO_BUILDER_NAME')
+ buildnum = os.environ.get('MONGO_BUILD_NUMBER')
+
+ if builder is None or buildnum is None:
+ return run_and_echo(command)
+
+ try:
+ buildnum = int(buildnum)
+ except ValueError:
+ sys.stderr.write('buildlogger: build number ("%s") was not an int\n' % buildnum)
+ sys.stderr.flush()
+ return run_and_echo(command)
+
+ # test takes some extra info
+ phase = os.environ.get('MONGO_PHASE', 'unknown')
+ test_filename = os.environ.get('MONGO_TEST_FILENAME', 'unknown')
+
+ build_info = dict((k, v) for k, v in os.environ.items() if k.startswith('MONGO_'))
+ build_info.pop('MONGO_BUILDER_NAME', None)
+ build_info.pop('MONGO_BUILD_NUMBER', None)
+ build_info.pop('MONGO_PHASE', None)
+ build_info.pop('MONGO_TEST_FILENAME', None)
+
+ build_id = get_or_create_build(builder, buildnum, extra=build_info)
+ if not build_id:
+ return run_and_echo(command)
+
+ test_id = create_test(build_id, test_filename, ' '.join(command), phase)
+ if not test_id:
+ return run_and_echo(command)
+
+ buf = []
+ def callback(line):
+ if line is None and buf:
+ # callback is called with None when the
+ # command is finished
+ append_test_logs(build_id, test_id, buf)
+
+ buf.append((time.time(), line))
+ if len(buf) > 100 or (buf and time.time() - buf[0][0] > 10):
+ append_test_logs(build_id, test_id, buf)
+
+ # this is like "buf = []", but doesn't change
+ # the "buf" reference -- necessary to make
+ # the closure work
+ buf[:] = []
+
+ # the peculiar formatting here matches what is printed by
+ # smoke.py when starting tests
+ output_url = '%s/build/%s/test/%s/' % (URL_ROOT.rstrip('/'), build_id, test_id)
+ sys.stdout.write(' (output suppressed; see %s)\n' % output_url)
+ sys.stdout.flush()
+
+ returncode = loop_and_callback(command, callback)
+
+ failed = bool(returncode != 0)
+ finish_test(build_id, test_id, failed)
+
+ return returncode
+
+def wrap_global(command):
+ """
+ call the given command, intercept its stdout and stderr,
+ and send results in batches of 100 lines or 10s to the
+ buildlogger webapp. see :func:`append_global_logs` for the
+ difference between "global" and "test" log output.
+ """
+
+ # get builder name and build number from environment
+ builder = os.environ.get('MONGO_BUILDER_NAME')
+ buildnum = os.environ.get('MONGO_BUILD_NUMBER')
+
+ if builder is None or buildnum is None:
+ return run_and_echo(command)
+
+ try:
+ buildnum = int(buildnum)
+ except ValueError:
+ sys.stderr.write('int(os.environ["MONGO_BUILD_NUMBER"]):\n')
+ sys.stderr.write(traceback.format_exc())
+ sys.stderr.flush()
+ return run_and_echo(command)
+
+ build_info = dict((k, v) for k, v in os.environ.items() if k.startswith('MONGO_'))
+ build_info.pop('MONGO_BUILDER_NAME', None)
+ build_info.pop('MONGO_BUILD_NUMBER', None)
+
+ build_id = get_or_create_build(builder, buildnum, extra=build_info)
+ if not build_id:
+ return run_and_echo(command)
+
+ buf = []
+ def callback(line):
+ if line is None and buf:
+ # callback is called with None when the
+ # command is finished
+ append_global_logs(build_id, buf)
+
+ buf.append((time.time(), line))
+ if len(buf) > 100 or (buf and time.time() - buf[0][0] > 10):
+ append_global_logs(build_id, buf)
+
+ # this is like "buf = []", but doesn't change
+ # the "buf" reference -- necessary to make
+ # the closure work
+ buf[:] = []
+
+ return loop_and_callback(command, callback)
+
+def loop_and_callback(command, callback):
+ """
+ run the given command (a sequence of arguments, ordinarily
+ from sys.argv), and call the given callback with each line
+ of stdout or stderr encountered. after the command is finished,
+ callback is called once more with None instead of a string.
+ """
+ proc = subprocess.Popen(
+ command,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT,
+ )
+
+ while proc.poll() is None:
+ line = proc.stdout.readline().strip('\r\n')
+ callback(line)
+
+ callback(None)
+ return proc.returncode
+
+
+if __name__ == '__main__':
+ # argv[0] is 'buildlogger.py'
+ del sys.argv[0]
+
+ if sys.argv[0] in ('-g', '--global'):
+ # then this is wrapping a "global" command, and should
+ # submit global logs to the build, not test logs to a
+ # test within the build
+ del sys.argv[0]
+ wrapper = wrap_global
+
+ else:
+ wrapper = wrap_test
+
+ # if we are missing credentials or the json module, then
+ # we can't use buildlogger; so just echo output, but also
+ # log why we can't work.
+ if json is None:
+ sys.stderr.write('buildlogger: could not import a json module\n')
+ sys.stderr.flush()
+ wrapper = run_and_echo
+
+ elif buildlogger_secret is None:
+ sys.stderr.write('buildlogger: could not find settings.py with buildlogger_secret\n')
+ sys.stderr.flush()
+ wrapper = run_and_echo
+
+ # otherwise wrap a test command as normal; the
+ # wrapper functions return the return code of
+ # the wrapped command, so that should be our
+ # exit code as well.
+ sys.exit(wrapper(sys.argv))
+
Oops, something went wrong.

0 comments on commit 41db66e

Please sign in to comment.