Skip to content
Browse files

- added to remote host management for tegras a reformat SDcard helper

- adding to AWS support code the start of pending job metrics so that kittenherder can know when to stop/start EC2 instances
  • Loading branch information...
1 parent 915b39e commit 33abf5e145cbbc9cc23921c7dab2bdea9f1d1e9f @bear bear committed Jul 6, 2012
Showing with 76 additions and 14 deletions.
  1. +3 −1 README
  2. +6 −8 kitten.py
  3. +14 −5 kittenmonitor.py
  4. +37 −0 releng/__init__.py
  5. +16 −0 releng/remote.py
View
4 README
@@ -89,4 +89,6 @@ Dependencies
dnspython
http://www.dnspython.org/
pip install dnspython
-
+
+ sqlalchemy
+ mysql-python
View
14 kitten.py
@@ -15,17 +15,11 @@
bear Mike Taylor <bear@mozilla.com>
"""
-import sys, os
-import re
-import time
-import json
-import random
import logging
-import datetime
-from multiprocessing import get_logger, log_to_stderr
+from multiprocessing import get_logger
-from releng import initOptions, initLogs, runCommand, initKeystore, relative
+from releng import initOptions, initLogs, initKeystore, relative
import releng.remote
@@ -126,9 +120,13 @@ def check(kitten):
if options.stop:
print host.graceful_shutdown()
+ if host.isTegra and options.sdcard:
+ host.formatSDCard()
+
_options = { 'reboot': ('-r', '--reboot', False, 'reboot host if required'),
'info': ('-i', '--info', False, 'show passive info only, do not ssh to host'),
'stop': ('', '--stop', False, 'stop buildbot for host'),
+ 'sdcard': ('', '--sdcard', False, 'reformat tegra sdcard'),
}
View
19 kittenmonitor.py
@@ -41,7 +41,7 @@
from boto.ec2 import connect_to_region
-from releng import initOptions, initLogs, dbRedis
+from releng import initOptions, initLogs, dbRedis, dbMysql
log = logging.getLogger()
@@ -282,6 +282,12 @@ def sendAlertEmail(alerts, options):
server.sendmail(addr, [addr], msg.as_string())
server.quit()
+def getPendingCounts(options):
+ db = dbMysql(options)
+
+ print db.pendingJobs()
+
+
def awsUpdate(options):
secrets = json.load(open(options.secrets))
conn = connect_to_region(options.region,
@@ -292,13 +298,14 @@ def awsUpdate(options):
if conn is not None:
reservations = conn.get_all_instances()
+ farm = 'ec2'
+ farmKey = 'farm:%s' % farm
current = {}
for reservation in reservations:
for instance in reservation.instances:
if 'moz-state' in instance.tags:
dNow = datetime.now()
ts = dNow.strftime('%Y-%m-%dT%H:%M:%SZ')
- farm = 'ec2'
currStatus = { 'state': instance.state,
'id': instance.id,
@@ -316,9 +323,7 @@ def awsUpdate(options):
currStatus[tag.lower()] = instance.tags[tag]
hostKey = '%s:%s:%s' % (farm, currStatus['name'], currStatus['id'])
- farmKey = 'farm:%s' % farm
-
- print hostKey, farmKey, currStatus['moz-state']
+ log.debug('%s %s %s' % hostKey, farmKey, currStatus['moz-state'])
db.sadd(farmKey, hostKey)
@@ -357,6 +362,8 @@ def awsUpdate(options):
'logpath': ('-l', '--logpath', None, 'Path where log file is to be written'),
'redis': ('-r', '--redis', 'localhost:6379', 'Redis connection string'),
'redisdb': ('', '--redisdb', '10', 'Redis database'),
+ 'mysql': ('', '--mysql', None, "MySql host:port"),
+ 'mysqldb': ('', '--mysqldb', None, 'MySql database'),
'email': ('-e', '--email', False, 'send result email'),
'region': ('', '--region' , 'us-west-1', 'EC2 Region'),
}
@@ -369,6 +376,8 @@ def awsUpdate(options):
db = dbRedis(options)
+ getPendingCounts(options)
+
awsUpdate(options)
options.redisdb = 8 # metrics data
View
37 releng/__init__.py
@@ -16,18 +16,21 @@
"""
import os, sys
+import time
import types
import json
import gzip
import urllib2
import logging
+import signal
import StringIO
import subprocess
from optparse import OptionParser
from logging.handlers import RotatingFileHandler
from multiprocessing import get_logger
+import sqlalchemy as sa
import redis
import version
@@ -62,6 +65,40 @@ def relative(delta):
else:
return '%d hours ago' % (delta.seconds / 3600)
+_pending_jobs_select = """
+SELECT buildername, count(*) FROM buildrequests WHERE
+ complete=0 AND
+ claimed_at=0 AND
+ submitted_at > :yesterday
+ GROUP BY buildername"""
+
+class dbMysql(object):
+ def __init__(self, options):
+ secrets = json.load(open(options.secrets))
+ self.engine = None
+ self.config = { 'port': 5432,
+ 'database': options.mysqldb,
+ }
+
+ if ':' in options.mysql:
+ self.config['host'], self.config['port'] = options.mysql.split(':')
+ else:
+ self.config['host'] = options.mysql
+
+ if 'mysql' in secrets:
+ self.config['user'] = secrets['mysql']['user']
+ self.config['password'] = secrets['mysql']['password']
+
+ self.connect = 'mysql://%(user)s:%(password)s@%(host)s:5432/%(database)s' % self.config
+ self.engine = sa.create_engine(self.connect)
+
+ def pendingJobs(self):
+ if self.engine is not None:
+ result = self.engine.execute(sa.text(_pending_jobs_select), yesterday=time.time()-86400)
+ return result.fetchall()
+ else:
+ return None
+
class dbRedis(object):
def __init__(self, options):
if ':' in options.redis:
View
16 releng/remote.py
@@ -24,6 +24,7 @@
import socket
import logging
import datetime
+import telnetlib
import ssh
import requests
import dns.resolver
@@ -97,6 +98,7 @@ def __init__(self, hostname, remoteEnv, verbose=False):
if hostname.startswith('tegra'):
self.isTegra = True
self.farm = 'tegra'
+ self.bbdir = '/builds/%s' % hostname
else:
if 'ec2' in hostname:
self.farm = 'ec2'
@@ -501,6 +503,20 @@ class TegraHost(UnixishHost):
def reboot(self):
self.rebootPDU()
+ def formatSDCard(self):
+ log.info('formatting SDCard')
+ tn = telnetlib.Telnet(self.fqdn, 20701)
+ log.debug('telnet: %s' % tn.read_until('$>'))
+ tn.write('exec newfs_msdos -F 32 /dev/block/vold/179:9\n')
+ out = tn.read_until('$>')
+
+ log.debug('telnet: %s' % out)
+ if 'return code [0]' in out:
+ log.info('SDCard formatted, rebooting Tegra')
+ tn.write('exec rebt\n')
+ else:
+ log.error('SDCard format failed')
+
class AWSHost(UnixishHost):
prompt = "]$ "
bbdir = "/builds/slave"

0 comments on commit 33abf5e

Please sign in to comment.
Something went wrong with that request. Please try again.