Permalink
Browse files

set the configuration of the pybit-client to use external configurati…

…on and enable this as debconf for Debian packaging.
  • Loading branch information...
2 parents 640ecaf + 7ef9df3 commit 5858d327026bc721063366e5b41976609072ba1b @codehelp codehelp committed Nov 19, 2012
@@ -11,5 +11,7 @@
"password": "guest",
"vhost": "/",
"dput": "-U",
- "dput_dest": ""
+ "dput_dest": "",
+ "dput_dest": "tcl",
+ "poll_time": 60
}
@@ -3,5 +3,6 @@
"repobase": "/srv/tcl/distros/debian",
"path" : "/srv/tcl/distros/debian/incoming",
"dryrun" : true,
- "rule" : "lightwriters"
+ "rule" : "lightwriters",
+ "sleeptime": 3
}
View
@@ -116,6 +116,10 @@ if __name__ == '__main__':
groupConfigFile.add_option("--clientid", dest="clientid",
help="id to use for build-client control queue, defaults to 1 but is unique per amqp server.",
metavar=META + "CLIENTID")
+
+ groupConfigFile.add_option("--poll_time", dest="poll_time",
+ help="interval at which to poll the queue, defaults to 60 s.",
+ metavar=META + "POLL_TIME")
parser.add_option_group(groupConfigFile)
View
@@ -32,6 +32,8 @@ import os
import sys
import logging
import signal
+import subprocess
+import time
from pybit.daemonlogger import LoggingDaemonContext, FileLikeLogger
from pyinotify import ProcessEvent
@@ -55,24 +57,32 @@ def getDaemonLogger (filePath, format = None) :
return logging.getLogger()
class EventHandler(pyinotify.ProcessEvent):
+
def process_IN_CREATE(self, event):
if os.path.isfile(event.pathname) and event.pathname.endswith(".changes"):
-
- cmd = "reprepro -b %s processincoming %s" % (self.settings['repobase'], self.settings['rule'])
- log_cmd = "run"
- if ('dryrun' in self.settings and
- self.settings['dryrun'] == False):
- log_cmd = "dryrun"
-
- logging.debug("%s: %s" % (log_cmd, cmd))
- if ('dryrun' in self.settings and
+ logging.debug("Sleeping for %ss",self.sleeptime)
+ time.sleep(self.sleeptime)
+ cmd_args = ["reprepro", "-b", self.settings['repobase'], "processincoming", self.settings['rule']]
+ logging.debug(" ".join(cmd_args))
+ if ('dryrun' not in self.settings or
self.settings['dryrun'] == False):
- os.system(cmd)
+ process = subprocess.Popen(cmd_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ (stdout, stderr) = process.communicate()
+
+ if process.returncode:
+ logging.debug(stderr)
+ logging.debug("reprepo command failed with code: %s" % process.returncode)
+ else:
+ logging.debug(stdout)
def __init__(self, settings):
self.settings = settings
+ self.sleeptime = 3
+ if 'sleeptime' in self.settings:
+ self.sleeptime = self.settings['sleeptime']
ProcessEvent.__init__(self)
-
+
+
def run(settings):
try:
signal.signal(signal.SIGINT, signal_handler)
@@ -81,7 +91,8 @@ def run(settings):
raise Exception('Error configuring signal handler: ' + str(e))
wm = pyinotify.WatchManager()
mask = pyinotify.IN_CREATE
-
+ if 'dryrun' in settings and settings['dryrun'] == True:
+ logging.debug("Starting in dryrun mode")
handler = EventHandler(settings)
notifier = pyinotify.Notifier(wm, handler)
wdd = wm.add_watch(settings['path'], mask, rec=True)
@@ -101,7 +112,9 @@ if __name__ == '__main__':
groupConfigFile.add_option("--dry-run", dest="dryrun", action="store_true",
help="Controls if we simulate or do we actually run.", metavar=META + "DRYRUN")
-
+ groupConfigFile.add_option("--sleeptime", dest="sleeptime",
+ help="The number of seconds we wait after a changes file is created before we run the reprepro command.",
+ metavar=META + "SLEEPTIME")
parser.add_option("--conf_file", dest="conf_file", default="watcher/watcher.conf",
help="Config file to read settings from, defaults to watcher.conf which will be read from configs/watcher and /etc/pybit/watcher in turn.",
metavar=META + "CONF_FILE")
View
@@ -1,21 +1,3 @@
-import re
-import os
-import errno
-import json
-import time
-import logging
-import jsonpickle
-from amqplib import client_0_8 as amqp
-import pybit
-from pybit.models import TaskComplete, PackageInstance, ClientMessage, BuildRequest, CommandRequest, AMQPConnection,\
- CancelRequest
-from debianclient import DebianBuildClient
-from subversion import SubversionClient
-import multiprocessing
-import socket
-import requests
-from requests.auth import HTTPBasicAuth
-
# Copyright 2012:
#
# Nick Davidson <nickd@toby-churchill.com>,
@@ -38,6 +20,25 @@
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
# MA 02110-1301, USA.
+
+import re
+import os
+import errno
+import json
+import time
+import logging
+import jsonpickle
+from amqplib import client_0_8 as amqp
+import pybit
+from pybit.models import TaskComplete, PackageInstance, ClientMessage, BuildRequest, CommandRequest, AMQPConnection,\
+ CancelRequest
+from debianclient import DebianBuildClient
+from subversion import SubversionClient
+import multiprocessing
+import socket
+import requests
+from requests.auth import HTTPBasicAuth
+
class PyBITClient(object):
def _clean_current(self):
@@ -46,6 +47,7 @@ def _clean_current(self):
self.current_msg = None
self.current_request = None
self.overall_success = None
+ self.subprocess_message
def set_status(self, status, request=None, client = None):
if request is None:
@@ -73,9 +75,19 @@ def get_status(self, request = None):
status_list = jsonpickle.decode(r.content)
if (len(status_list) > 0):
return status_list[-1].status
+
+ def republish_job(self, buildreq):
+ routing_key = pybit.get_build_route_name(buildreq.packageinstance.distribution.name,
+ buildreq.packageinstance.arch.name,
+ buildreq.packageinstance.suite.name,
+ buildreq.packageinstance.format.name)
+ self.message_chan.basic_publish(amqp.Message(buildreq),
+ exchange=pybit.exchange_name,
+ routing_key=routing_key,
+ mandatory=True)
def wait(self):
- time.sleep(5)
+ time.sleep(self.poll_time)
if self.state == "IDLE" :
msg = None
if self.message_chan is not None:
@@ -120,13 +132,18 @@ def move_state(self, new_state):
overall_success = self.overall_success
current_msg = self.current_msg
current_req = self.current_request
+ subprocess_message = self.subprocess_message
self._clean_current()
if current_msg is not None :
self.message_chan.basic_ack(current_msg.delivery_tag)
if overall_success == True:
self.set_status(ClientMessage.done, current_req)
elif overall_success == False:
- self.set_status(ClientMessage.failed, current_req)
+ if subprocess_message == 'build-dep-wait':
+ self.set_status(ClientMessage.blocked, current_req)
+ self.republish_job(current_req)
+ else:
+ self.set_status(ClientMessage.failed, current_req)
logging.debug ("Moved from %s to %s" % (self.old_state, self.state))
else:
@@ -177,15 +194,16 @@ def build_handler(self, msg, decoded):
self.move_state("UPLOAD")
else:
self.overall_success = False
+ self.subprocess_message = decoded.message
self.move_state("CLEAN")
def upload_handler(self, msg, decoded):
if isinstance(decoded, TaskComplete) :
self.overall_success = decoded.success
+ self.subprocess_message = decoded.message
self.process.join()
- self.overall_success = False
self.move_state("CLEAN")
def clean_handler(self, msg, decoded):
@@ -216,7 +234,9 @@ def __init__(self, arch, distribution, pkg_format, suite, conn_info, settings) :
self.command_chan = None
self.message_chan = None
self.settings = settings
-
+ self.poll_time = 60
+ if 'poll_time' in self.settings:
+ self.poll_time = self.settings['poll_time']
self.routing_key = pybit.get_build_route_name(self.distribution,
self.arch, self.suite, self.pkg_format)
View
@@ -123,7 +123,7 @@ def serve_static_package_instances():
app.mount('/status',lookups.get_status_app(settings, db))
app.mount('/arch',lookups.get_arch_app(settings, db))
app.mount('/format', lookups.get_format_app(settings, db))
- app.mount('/buildd', buildd.get_buildd_app(settings, db))
+ app.mount('/buildd', buildd.get_buildd_app(settings, db, controller))
app.mount('/package', package.get_packages_app(settings, db, controller))
app.mount('/packageinstance', packageinstance.get_packageinstance_app(settings, db))
return app
View
@@ -26,9 +26,10 @@
import jsonpickle
import common
from common import requires_auth
+from controller import Controller
-def get_buildd_app(settings, db):
- app = Bottle(config= {'settings' : settings, 'db' : db})
+def get_buildd_app(settings, db, controller):
+ app = Bottle(config= {'settings' : settings, 'db' : db, 'controller': controller})
@app.route('/', method='GET')
def get_buildd():
try:
@@ -89,12 +90,15 @@ def delete_buildd_id(buildd_id):
raise Exception('Exception encountered: ' + str(e))
return None
- @app.route('/<buildd_id:int>/status', method='GET')
- def get_buildd_status(buildd_id):
+ @app.route('/<buildd_name>/status', method='GET')
+ def get_buildd_status(buildd_name):
+ #TODO: FIXME
try:
- response.content_type = "application/json"
- #TODO - CODEME
- return template("Returning status of buildd: {{buildd_id}}",buildd_id=buildd_id)
+ res = app.config['controller'].buildd_command_queue_exists(buildd_name)
+ if res == True :
+ return "Active"
+ else :
+ return "Not Active"
except Exception as e:
raise Exception('Exception encountered: ' + str(e))
return None
Oops, something went wrong.

0 comments on commit 5858d32

Please sign in to comment.