Permalink
Browse files

first commit

  • Loading branch information...
0 parents commit 432300ddbb3cc3bc6a1fab238a584437a1f63878 @jonallengriffin jonallengriffin committed Mar 15, 2012
@@ -0,0 +1,3 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
@@ -0,0 +1,120 @@
+import os
+import sys
+
+# File mode creation mask of the daemon.
+UMASK = 0
+
+# Default working directory for the daemon.
+WORKDIR = "/"
+
+# Default maximum for the number of available file descriptors.
+MAXFD = 1024
+
+def createDaemon(pidfile, logfile='/dev/null'):
+ """Detach a process from the controlling terminal and run it in the
+ background as a daemon. http://code.activestate.com/recipes/278731/
+ """
+
+ try:
+ # Fork a child process so the parent can exit. This returns control to
+ # the command-line or shell. It also guarantees that the child will not
+ # be a process group leader, since the child receives a new process ID
+ # and inherits the parent's process group ID. This step is required
+ # to insure that the next call to os.setsid is successful.
+ pid = os.fork()
+ except OSError, e:
+ raise Exception, "%s [%d]" % (e.strerror, e.errno)
+
+ if (pid == 0): # The first child.
+ # To become the session leader of this new session and the process group
+ # leader of the new process group, we call os.setsid(). The process is
+ # also guaranteed not to have a controlling terminal.
+ os.setsid()
+
+ # Is ignoring SIGHUP necessary?
+ #
+ # It's often suggested that the SIGHUP signal should be ignored before
+ # the second fork to avoid premature termination of the process. The
+ # reason is that when the first child terminates, all processes, e.g.
+ # the second child, in the orphaned group will be sent a SIGHUP.
+ #
+ # "However, as part of the session management system, there are exactly
+ # two cases where SIGHUP is sent on the death of a process:
+ #
+ # 1) When the process that dies is the session leader of a session that
+ # is attached to a terminal device, SIGHUP is sent to all processes
+ # in the foreground process group of that terminal device.
+ # 2) When the death of a process causes a process group to become
+ # orphaned, and one or more processes in the orphaned group are
+ # stopped, then SIGHUP and SIGCONT are sent to all members of the
+ # orphaned group." [2]
+ #
+ # The first case can be ignored since the child is guaranteed not to have
+ # a controlling terminal. The second case isn't so easy to dismiss.
+ # The process group is orphaned when the first child terminates and
+ # POSIX.1 requires that every STOPPED process in an orphaned process
+ # group be sent a SIGHUP signal followed by a SIGCONT signal. Since the
+ # second child is not STOPPED though, we can safely forego ignoring the
+ # SIGHUP signal. In any case, there are no ill-effects if it is ignored.
+ #
+ # import signal # Set handlers for asynchronous events.
+ # signal.signal(signal.SIGHUP, signal.SIG_IGN)
+
+ try:
+ # Fork a second child and exit immediately to prevent zombies. This
+ # causes the second child process to be orphaned, making the init
+ # process responsible for its cleanup. And, since the first child is
+ # a session leader without a controlling terminal, it's possible for
+ # it to acquire one by opening a terminal in the future (System V-
+ # based systems). This second fork guarantees that the child is no
+ # longer a session leader, preventing the daemon from ever acquiring
+ # a controlling terminal.
+ pid = os.fork() # Fork a second child.
+
+ if pidfile:
+ # update the pid in the pid logfile
+ fp = open(pidfile, 'w')
+ fp.write("%d\n" % pid)
+ fp.close()
+
+ except OSError, e:
+ raise Exception, "%s [%d]" % (e.strerror, e.errno)
+
+ if (pid == 0): # The second child.
+ # Since the current working directory may be a mounted filesystem, we
+ # avoid the issue of not being able to unmount the filesystem at
+ # shutdown time by changing it to the root directory.
+ os.chdir(WORKDIR)
+ # We probably don't want the file mode creation mask inherited from
+ # the parent, so we give the child complete control over permissions.
+ os.umask(UMASK)
+ else:
+ # exit() or _exit()? See below.
+ os._exit(0) # Exit parent (the first child) of the second child.
+ else:
+ # exit() or _exit()?
+ # _exit is like exit(), but it doesn't call any functions registered
+ # with atexit (and on_exit) or any registered signal handlers. It also
+ # closes any open file descriptors. Using exit() may cause all stdio
+ # streams to be flushed twice and any temporary files may be unexpectedly
+ # removed. It's therefore recommended that child branches of a fork()
+ # and the parent branch(es) of a daemon use _exit().
+ os._exit(0) # Exit parent of the first child.
+
+ # Redirect the standard I/O file descriptors to the specified file. Since
+ # the daemon has no controlling terminal, most daemons redirect stdin,
+ # stdout, and stderr to /dev/null. This is done to prevent side-effects
+ # from reads and writes to the standard I/O file descriptors.
+
+ si = open('/dev/null', 'r')
+ so = open(logfile, 'a+', 0)
+ se = open(logfile, 'a+', 0)
+ os.dup2(si.fileno(), sys.stdin.fileno())
+ os.dup2(so.fileno(), sys.stdout.fileno())
+ os.dup2(se.fileno(), sys.stderr.fileno())
+
+ # Set custom file descriptors so that they get proper buffering.
+ sys.stdout, sys.stderr = so, se
+
+ return(0)
+
@@ -0,0 +1,162 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import calendar
+import httplib
+import json
+import logging
+import logging.handlers
+import os
+import platform
+from Queue import Empty
+import subprocess
+import sys
+import time
+from urlparse import urlparse
+
+from translatorexceptions import *
+from translatorqueues import *
+
+DEBUG = False
+
+
+class LogHandler(object):
+
+ def __init__(self, queue, parent_pid, logdir):
+ self.queue = queue
+ self.parent_pid = parent_pid
+ self.logdir = logdir
+
+ def get_logger(self, name, filename):
+ filepath = os.path.join(self.logdir, filename)
+ if os.access(filepath, os.F_OK):
+ os.remove(filepath)
+ logger = logging.getLogger(name)
+ logger.setLevel(logging.DEBUG)
+ logger.addHandler(logging.handlers.RotatingFileHandler(filepath, maxBytes=300000, backupCount=2))
+ return logger
+
+ def get_url_info(self, url):
+ """Return a (code, content_length) tuple from making an
+ HTTP HEAD request for the given url.
+ """
+
+ try:
+ content_length = -1
+ p = urlparse(url)
+
+ conn = httplib.HTTPConnection(p[1])
+ conn.request('HEAD', p[2])
+ res = conn.getresponse()
+ code = res.status
+
+ if code == 200:
+ for header in res.getheaders():
+ if header[0] == 'content-length':
+ content_length = int(header[1])
+
+ return (code, content_length)
+
+ except AttributeError:
+ # this can happen when we didn't get a valid url from pulse
+ return (-1, -1)
+
+ except Exception, inst:
+ # XXX: something bad happened; we should log this
+ # return (-1, -1)
+ raise
+
+ def process_builddata(self, data):
+ if not data.get('logurl'):
+ # should log this
+ return
+
+ # If it's been less than 15s since we checked for this particular
+ # log, put this item back in the queue without checking again.
+ now = calendar.timegm(time.gmtime())
+ last_check = data.get('last_check', 0)
+ if last_check and now - last_check < 15:
+ self.queue.put(data)
+ return
+
+ code, content_length = self.get_url_info(str(data['logurl']))
+ if DEBUG:
+ if data.get('last_check'):
+ print '...reprocessing logfile', code, data.get('logurl')
+ print '...', data.get('key')
+ print '...', now - data.get('insertion_time', 0), 'seconds since insertion_time'
+ else:
+ print 'processing logfile', code, data.get('logurl')
+ if code == 200:
+ self.publish_unittest_message(data)
+ else:
+ if now - data.get('insertion_time', 0) > 600:
+ # Currently, this is raised for unittests from beta and aurora
+ # builds at least, as their log files get stored in a place
+ # entirely different than the builds. This should change soon
+ # per bug 713846, so I've not adapted the code to handle this.
+ raise LogTimeoutError(data.get('key', 'unknown'), data.get('logurl'))
+ else:
+ # re-insert this into the queue
+ data['last_check'] = now
+ if DEBUG:
+ print 'requeueing after check'
+ self.queue.put(data)
+
+ def publish_unittest_message(self, data):
+ # The original routing key has the format build.foo.bar.finished;
+ # we only use 'foo' in the new routing key.
+ original_key = data['key'].split('.')[1]
+ tree = data['tree']
+ platform = data['platform']
+ buildtype = data['buildtype']
+ os = data['os']
+ test = data['test']
+ product = data['product'] if data['product'] else 'unknown'
+ key_parts = ['talos' if data['talos'] else 'unittest',
+ tree,
+ platform,
+ os,
+ buildtype,
+ test,
+ product,
+ original_key]
+
+ publish_message(TranlsatorPublisher, data, '.'.join(key_parts))
+
+ def start(self):
+ self.errorLogger = self.get_logger('LogHandlerErrorLog', 'log_handler_error.log')
+ while True:
+ try:
+ data = None
+
+ # Check if the parent process is still alive, and if so,
+ # look for another log to process.
+ if 'windows' in platform.system().lower():
+ proc = subprocess.Popen(['tasklist', '-FI', 'PID eq %d' % self.parent_pid],
+ stderr=subprocess.STDOUT,
+ stdout=subprocess.PIPE)
+ if not proc.wait():
+ result = proc.stdout.read()
+ if not str(self.parent_pid) in result:
+ raise OSError
+ else:
+ raise Exception("Unable to call tasklist")
+ else:
+ os.kill(self.parent_pid, 0)
+ data = self.queue.get_nowait()
+ self.process_builddata(data)
+ except Empty:
+ time.sleep(5)
+ continue
+ except OSError:
+ # if the parent process isn't alive, shutdown gracefully
+ # XXX: Need to drain the queue to a file before shutting
+ # down, so we can pick up where we left off when we resume.
+ sys.exit(0)
+ except Exception, inst:
+ obj_to_log = data
+ if data.get('payload') and data['payload'].get('build') and data['payload']['build'].get('properties'):
+ obj_to_log = data['payload']['build']['properties']
+ self.errorLogger.exception(json.dumps(obj_to_log, indent=2))
@@ -0,0 +1,69 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import re
+
+
+buildtypes = [ 'opt', 'debug', 'pgo' ]
+
+def guess_platform(builder):
+ for platform in sorted(platforms.keys(), reverse=True):
+ if platform in builder:
+ return platform
+
+ for key in platforms:
+ for os in platforms[key]:
+ if os in builder:
+ return os
+
+def convert_os(data):
+ if re.search(r'OS\s*X\s*10.5', data['buildername'], re.I):
+ return 'leopard'
+ if re.search(r'OS\s*X\s*10.6', data['buildername'], re.I):
+ return 'snowleopard'
+ if re.search(r'OS\s*X\s*10.7', data['buildername'], re.I):
+ return 'lion'
+ if re.search(r'WINNT\s*5.2', data['buildername'], re.I):
+ return 'xp'
+ return 'unknown'
+
+os_conversions = {
+ 'leopard-o': lambda x: 'leopard',
+ 'tegra_android-o': lambda x: 'tegra_android',
+ 'macosx': convert_os,
+ 'macosx64': convert_os,
+ 'win32': convert_os,
+}
+
+platforms = {
+ 'linux64-rpm': ['fedora64'],
+ 'linux64': ['fedora64'],
+ 'linuxqt': ['fedora'],
+ 'linux-rpm': ['fedora'],
+ 'linux': ['fedora', 'linux'],
+ 'win32': ['xp', 'win7'],
+ 'win64': ['w764'],
+ 'macosx64': ['macosx64', 'snowleopard', 'leopard', 'lion'],
+ 'macosx': ['macosx', 'leopard'],
+ 'android-xul': ['tegra_android-xul'],
+ 'android': ['tegra_android'],
+}
+
+tags = [
+ '',
+ 'build',
+ 'dep',
+ 'dtrace',
+ 'l10n',
+ 'nightly',
+ 'nomethodjit',
+ 'notracejit',
+ 'release',
+ 'shark',
+ 'spidermonkey',
+ 'valgrind',
+ 'warnaserr',
+ 'warnaserrdebug',
+ 'xulrunner',
+ ]
Oops, something went wrong.

0 comments on commit 432300d

Please sign in to comment.