Permalink
Browse files

bring dbconfig branch into line with master, with changes for dbconfi…

…g upgrade script.
  • Loading branch information...
2 parents 17363b5 + c29b3b7 commit 69990c81965a9c5635278248e682dd8a3b7087e2 @codehelp codehelp committed Feb 12, 2013
View
@@ -9,4 +9,7 @@
Order deny,allow
Allow from all
</Directory>
+ LogLevel info
+ ErrorLog ${APACHE_LOG_DIR}/pybit-error.log
+ CustomLog ${APACHE_LOG_DIR}/pybit-access.log combined
</VirtualHost>
View
@@ -102,9 +102,9 @@ def main():
test_suite = Suite (0, suite)
test_transport = Transport (0, method_type, uri, vcs_id)
test_package = Package(0, version, package)
- test_packageinstance = PackageInstance(1, test_package, test_arch, test_suite, pkg_format, distribution, True)
+ test_packageinstance = PackageInstance(1, test_package, test_arch, None, test_suite, distribution, pkg_format, True)
test_job = Job(2, test_packageinstance,None)
- test_req = BuildRequest(test_job,test_transport,None,commands)
+ test_req = BuildRequest(test_job,test_transport,None)
test_req.stamp_request()
# clean up in case the last test failed.
if (method_type == "svn") :
@@ -14,6 +14,7 @@
"password" : "pass"
},
"controller": {
+ "debug": false,
"rabbit_url": "localhost:5672",
"rabbit_userid": "guest",
"rabbit_password": "guest",
View
@@ -136,6 +136,7 @@ COMMENT ON TABLE Blacklist
CREATE TABLE PackageInstance (
id SERIAL PRIMARY KEY NOT NULL,
Package_id bigint NOT NULL,
+ BuildEnv_id bigint,
Arch_id bigint NOT NULL,
Suite_id bigint NOT NULL,
Dist_id bigint NOT NULL,
@@ -311,3 +312,7 @@ ALTER TABLE SuiteArches ADD CONSTRAINT FK_SuiteArches_Arch
ALTER TABLE BuildRequest ADD CONSTRAINT FK_BuildRequest_Job
FOREIGN KEY (job) REFERENCES Job (id)
;
+
+ALTER TABLE PackageInstance ADD CONSTRAINT FK_PackageInstance_BuildEnv
+ FOREIGN KEY (BuildEnv_id) REFERENCES BuildEnv (id)
+;
View
@@ -0,0 +1,12 @@
+create or replace function run_update() returns void as
+$$
+begin
+if (SELECT count(*) from schema_version WHERE id=4)
+then
+ ALTER TABLE PackageInstance ADD COLUMN buildenv_id bigint;
+ UPDATE schema_version SET id=5;
+end if;
+end;
+$$ LANGUAGE plpgsql;
+
+SELECT run_update();
View
@@ -1,8 +1,8 @@
-pybit (0.4.3-1) experimental; urgency=low
+pybit (0.5.0-1) experimental; urgency=low
* New upstream release.
- -- Neil Williams <codehelp@debian.org> Wed, 06 Feb 2013 13:23:06 +0000
+ -- Neil Williams <codehelp@debian.org> Tue, 12 Feb 2013 14:14:37 +0000
pybit (0.4.2-1) experimental; urgency=low
@@ -29,7 +29,7 @@ client feeds status messages back to the controller.
</refsect1>
<refsect1 id="usage">
<title>Usage</title>
- <para>pybit-client runs as a single-instance daemon by default. The standard
+ <para>pybit-client runs as a single-instance daemon by default. The standard
daemon control interface is supported:
</para>
<programlisting>
@@ -8,3 +8,4 @@ pybit-client ./usr/bin/
buildd-test.py ./usr/share/pybitclient/
debian/pybit-client.1 ./usr/share/man/man1/
debian/pybitclientlog ./etc/logrotate.d/
+debian/script ./usr/share/bug/pybit-client/
@@ -1,9 +1,11 @@
db/schema.sql ./usr/share/dbconfig-common/data/pybit-web/install/pgsql/
debian/pgsql ./usr/share/dbconfig-common/scripts/pybit-web/install/
-db/updates/v1.sql ./usr/share/dbconfig-common/data/pybit-web/upgrade/pgsql/0.3.1
-db/updates/v2.sql ./usr/share/dbconfig-common/data/pybit-web/upgrade/pgsql/0.4.3
-db/updates/v3.sql ./usr/share/dbconfig-common/data/pybit-web/upgrade/pgsql/0.4.3
-db/updates/v4.sql ./usr/share/dbconfig-common/data/pybit-web/upgrade/pgsql/0.4.3
+debian/upgrades/0.5.0 ./usr/share/dbconfig-common/data/pybit-web/upgrade/pgsql/
+db/updates/v1.sql ./usr/share/pybit-web/upgrade/pgsql/0.3.1
+db/updates/v2.sql ./usr/share/pybit-web/upgrade/pgsql/0.4.3
+db/updates/v3.sql ./usr/share/pybit-web/upgrade/pgsql/0.4.3
+db/updates/v4.sql ./usr/share/pybit-web/upgrade/pgsql/0.4.3
+db/updates/v5.sql ./usr/share/pybit-web/upgrade/pgsql/0.4.3
pybitweb/static/ ./usr/share/pybit-web/
configs/web/web.conf ./usr/share/pybit-web/
debian/pybit-web-json.pl ./usr/share/pybit-web/
View
@@ -0,0 +1,42 @@
+#!/bin/bash
+set -e
+
+if [ -z "$YESNO" ]; then
+ YESNO=$"yYnN"
+fi
+
+cat <<EOF
+I can automatically include various information about your pybit-client
+configuration and debug output in your bug report. This information may help
+to diagnose your problem.
+
+It may also help if you provide /var/log/apache2/pybit-error.log from
+the machine running the pybit-web package.
+
+EOF
+
+if [ -f "/var/log/pybitclient.log" ]; then
+ yesno "May I include /var/log/pybitclient.log ? [Y/n] " yep
+ if [ "$REPLY" = "yep" ]; then
+ echo >&3
+ echo "-- /var/log/pybitclient.log --" >&3
+ echo >&3
+ cat $config >&3
+ else
+ echo >&3
+ echo "-- ( /var/log/pybitclient.log present, but not submitted) --" >&3
+ echo >&3
+ fi
+fi
+
+yesno "May I include /etc/pybit/client/client.conf ? [Y/n] " yep
+if [ "$REPLY" = "yep" ]; then
+ echo >&3
+ echo "-- /etc/pybit/client/client.conf --" >&3
+ echo >&3
+ cat /etc/dpkg-cross/cross-compile >&3
+else
+ echo >&3
+ echo "-- (/etc/pybit/client/client.conf not submitted) --" >&3
+ echo >&3
+fi
View
@@ -0,0 +1,26 @@
+#!/bin/sh
+
+set -e
+
+# This is still a dbconfig-common script because we need to ask the
+# admin before upgrading. Each sql file is idempotent and the files need
+# to be executed in sequence. So always run each upgrade SQL.
+
+. /usr/share/dbconfig-common/dpkg/postinst.pgsql
+. /usr/share/dbconfig-common/dpkg/common
+dbc_share="/usr/share/dbconfig-common"
+dbc_package=$1
+dbc_basepackage=`echo $dbc_package | cut -d_ -f1`
+dbc_dbtype=pgsql
+dbc_sqldir=/usr/share/$dbc_basepackage/upgrade/$dbc_dbtype
+dbc_dbname=`grep -v '#' /etc/dbconfig-common/pybit-web.conf |grep dbc_dbname|cut -d= -f2|sed -e "s:'::g"`
+
+if [ -d "$dbc_sqldir" ]; then
+ for FILE in ` find $dbc_sqldir/* -xtype f -print0 | xargs --no-run-if-empty -0 ls -1 | sort -n | uniq`; do
+ echo "updating tables in database for $dbc_dbname using $FILE "
+ echo su -s /bin/sh $dbc_dbname -c "psql -d $dbc_dbname -f $FILE 2> /dev/null"
+ dbc_logline "$FILE done"
+ done
+else
+ dbc_logline "$dbc_sqldir not found\n"
+fi
View
@@ -62,6 +62,11 @@ def __init__(self,arch_id,name):
self.id = arch_id
self.name = name
+class BuildEnv(Model):
+ def __init__(self,build_env_id,name):
+ self.id = build_env_id
+ self.name = name
+
class Dist(Model):
def __init__(self,dist_id,name):
self.id = dist_id
@@ -101,10 +106,11 @@ def __init__(self,transport_id,method,uri,vcs_id):
self.vcs_id = vcs_id
class PackageInstance(Model):
- def __init__(self, packageinstance_id, package, arch, suite, distribution, pkg_format, master) :
+ def __init__(self, packageinstance_id, package, arch, build_env, suite, distribution, pkg_format, master) :
self.id = packageinstance_id
self.package = package
self.arch = arch
+ self.build_env = build_env
self.suite = suite
self.distribution = distribution
self.format = pkg_format
@@ -125,6 +131,12 @@ def __init__(self,suitearch_id,suite,arch,master_weight=0):
self.arch = arch
self.master_weight = master_weight
+class BuildEnvSuiteArch(Model):
+ def __init__(self,buildenv_suitearch_id,buildenv,suitearch):
+ self.id = buildenv_suitearch_id
+ self.buildenv = buildenv
+ self.suitearch = suitearch
+
class BuildRequest(Model):
def __init__(self,job,transport,web_host):
self.job = job
@@ -141,6 +153,11 @@ def get_buildstamp (self) :
def get_suite(self):
return self.job.packageinstance.suite.name
+ def get_buildenv(self):
+ if self.job.packageinstance.build_env is None:
+ return None
+ return self.job.packageinstance.build_env.name
+
def get_package(self):
return self.job.packageinstance.package.name
View
@@ -31,13 +31,30 @@ def fetch_source(self, buildreq, conn_data):
retval = "wrong_method"
if not retval :
self.workdir = os.path.join (self.settings["buildroot"],
- buildreq.get_suite(), buildreq.transport.method)
+ buildreq.get_suite(), buildreq.transport.method, buildreq.get_package())
+ if not os.path.isdir (self.workdir):
+ pybitclient.mkdir_p (self.workdir)
+ apt_path = os.path.join (self.workdir, "lists", "partial")
+ pybitclient.mkdir_p (apt_path)
+ apt_path = os.path.join (self.workdir, "archives", "partial")
+ pybitclient.mkdir_p (apt_path)
+ apt_path = os.path.join (self.workdir, "etc", "apt", "preferences.d")
+ pybitclient.mkdir_p (apt_path)
+ apt_path = os.path.join (self.workdir, "sources.list")
+ src_list = os.open (apt_path, os.O_CREAT | os.O_WRONLY)
+ url = "deb-src %s %s main" % (buildreq.transport.uri, buildreq.get_suite())
+ os.write (src_list, url)
+ cfg_str = "-o Apt::Get::AllowUnauthenticated=true -o Dir=%s -o Dir::State=%s -o Dir::Etc::SourceList=%s/sources.list -o Dir::Cache=%s" % \
+ (self.workdir, self.workdir, self.workdir, self.workdir)
+ command = "(cd %s && apt-get %s update 2>/dev/null || true)" % (self.workdir, cfg_str)
+ if not retval :
+ if pybitclient.run_cmd (command, self.settings["dry_run"], None) :
+ retval = "update_apt"
if (buildreq.get_version() is not None):
- command = "(cd %s && apt-get -d source %s=%s && dpkg-source -x %s_%s.dsc)" % (self.workdir,
- buildreq.get_package(), buildreq.get_version(), buildreq.get_package(), buildreq.get_version() )
+ command = "(cd %s/.. && apt-get %s -d source %s=%s )" % (self.workdir, cfg_str,
+ buildreq.get_package(), buildreq.get_version() )
else :
- command = "(cd %s && apt-get -d source %s && dpkg-source -x %s_%s.dsc)" % (self.workdir,
- buildreq.get_package(), buildreq.get_package(), buildreq.get_version() )
+ command = "(cd %s && apt-get %s -d source %s )" % (self.workdir, cfg_str, buildreq.get_package() )
if not retval :
if pybitclient.run_cmd (command, self.settings["dry_run"], None) :
retval = "fetch_source"
@@ -57,9 +74,15 @@ def clean_source (self, buildreq, conn_data) :
if buildreq.transport.method != "apt":
retval = "wrong_method"
if not retval :
+ src_dir = os.path.join (self.settings["buildroot"], buildreq.get_suite(), buildreq.transport.method)
+ src_changes = "%s/%s_%s.dsc" % (src_dir, buildreq.get_package(), buildreq.get_version() )
+ command = "dcmd rm %s" % (src_changes)
+ if pybitclient.run_cmd (command, self.settings["dry_run"], None):
+ retval = "source-clean-fail"
+ if not retval :
self.cleandir = os.path.join (self.settings["buildroot"], buildreq.get_suite(), buildreq.transport.method,
buildreq.get_package())
- command = "rm -rf %s" % (self.cleandir)
+ command = "rm -rf %s/" % (self.cleandir)
if pybitclient.run_cmd (command, self.settings["dry_run"], None) :
retval = "failed_clean"
if not retval :
@@ -3,7 +3,7 @@
#
# debian.py
#
-# Copyright 2012 Neil Williams <codehelp@debian.org>
+# Copyright 2012, 2013 Neil Williams <codehelp@debian.org>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -73,7 +73,7 @@ def build_command_handler (self, buildreq, conn_data) :
# chroot and therefore copied to ${HOME} so that schroot copies it again,
# into the chroot itself.
orig_sh = "/usr/share/pybitclient/sbuild-orig.sh"
- command = "(cp %s %s/sbuild-orig.sh ; schroot -n -u root -c %s -- %s/sbuild-orig.sh %s %s ; rm %s/sbuild-orig.sh)" % (orig_sh,
+ command = "(cp %s %s/sbuild-orig.sh ; schroot --directory / -n -u root -c %s -- %s/sbuild-orig.sh %s %s ; rm %s/sbuild-orig.sh)" % (orig_sh,
self.settings["buildroot"], buildreq.get_suite(), self.settings["buildroot"], package_dir, parts[2], self.settings["buildroot"])
if pybitclient.run_cmd (command, self.settings["dry_run"], logfile):
retval = "custom-command-error"
@@ -117,14 +117,14 @@ def orig_source_handler (self, buildreq, conn_data) :
else :
command = "(cd ../ ; apt-get -d source %s/%s)" % (buildreq.get_package(), buildreq.get_suite())
if pybitclient.run_cmd (command, self.settings["dry_run"], logfile):
- logging.debug("apt-get source failed, proceeding anyway incase its a tcl update of a debian package.")
+ logging.debug("I: apt-get source failed, proceeding anyway incase its an update of a debian package.")
return retval
def build_master (self, buildreq, conn_data):
retval = None
logfile = self.get_buildlog (self.settings["buildroot"], buildreq)
if (not isinstance(buildreq, BuildRequest)):
- logging.debug ("E: not able to identify package name.")
+ logging.warn ("E: not able to identify package name.")
retval = "misconfigured"
return self._overall_success(retval, conn_data)
srcdir = os.path.join (self.settings["buildroot"],
@@ -136,10 +136,13 @@ def build_master (self, buildreq, conn_data):
# once for the dep check and once before the build. The choice depends
# on whether two network trips are more efficient than rewriting the
# lvm snapshot before even trying to do any build.
+ chroot_name = buildreq.get_suite()
+ if (buildreq.get_buildenv() is not None):
+ chroot_name = "%s-%s" % (buildreq.get_buildenv(), buildreq.get_suite())
if self.settings["use_lvm"] :
- update_name = "%s-source" % buildreq.get_suite()
+ update_name = "%s-source" % chroot_name
else :
- update_name = buildreq.get_suite()
+ update_name = chroot_name
retval = self.update_environment (update_name, buildreq, conn_data)
# need an extra uscan stage to deal with non-native packages
# this requires the upstream release to be accessible to the client.
@@ -152,7 +155,7 @@ def build_master (self, buildreq, conn_data):
if self.settings["use_lvm"] and (os.path.isdir(package_dir) or self.settings["dry_run"]) :
control = os.path.join (package_dir, 'debian', 'control')
dep_check = "/usr/lib/pbuilder/pbuilder-satisfydepends-classic --control"
- command = "schroot -u root -c %s -- %s %s" % (buildreq.get_suite(), dep_check, os.path.realpath(control))
+ command = "schroot --directory / -u root -c %s -- %s %s" % (chroot_name, dep_check, os.path.realpath(control))
if pybitclient.run_cmd (command, self.settings["dry_run"], logfile):
retval = "build-dep-wait"
if not retval :
@@ -164,7 +167,7 @@ def build_master (self, buildreq, conn_data):
if pybitclient.run_cmd (command, self.settings["dry_run"], logfile):
retval = "build_dsc"
if not retval :
- command = "sbuild -A -n -s -d %s %s/%s_%s.dsc" % (buildreq.get_suite(),
+ command = "sbuild -A -n -s -d %s %s/%s_%s.dsc" % (chroot_name,
srcdir, buildreq.get_package(), buildreq.get_version())
ret = pybitclient.run_cmd (command, self.settings["dry_run"], logfile)
if (ret == 3 or ret == 1):
@@ -175,13 +178,12 @@ def build_master (self, buildreq, conn_data):
changes = "%s/%s_%s_%s.changes" % (self.settings["buildroot"], buildreq.get_package(),
buildreq.get_version(), buildreq.get_arch())
if not self.settings["dry_run"] and not os.path.isfile (changes) :
- logging.debug("build_master: Failed to find %s file." % (changes))
+ logging.warn("E: build_master: Failed to find %s file." % (changes))
retval = "build_changes"
if not retval and checkValue ('debsignkey', self.settings) :
command = "debsign -k%s %s" % (self.settings['debsignkey'], changes)
if pybitclient.run_cmd (command, self.settings["dry_run"], logfile):
retval = "build_sign"
-
return self._overall_success(retval, conn_data)
@@ -191,7 +193,7 @@ def upload (self, buildreq, conn_data):
changes = "%s/%s_%s_%s.changes" % (self.settings["buildroot"], buildreq.get_package(),
buildreq.get_version(), buildreq.get_arch())
if not os.path.isfile (changes) and not self.settings["dry_run"]:
- logging.debug("upload: Failed to find %s file." % (changes))
+ logging.warn("E: upload: Failed to find %s file." % (changes))
retval = "upload_changes"
if not retval :
command = "dput -c %s %s %s %s" % (self.dput_cfg,
@@ -223,9 +225,11 @@ def build_slave (self, buildreq, conn_data):
command = "(cd %s ; dpkg-buildpackage -nc -S -d -uc -us)" % (package_dir)
if pybitclient.run_cmd (command, self.settings["dry_run"], logfile):
retval = "build_dsc"
+ chroot_name = buildreq.get_suite()
+ if (buildreq.get_buildenv() is not None):
+ chroot_name = "%s-%s" % (buildreq.get_buildenv(), buildreq.get_suite())
if not retval :
- command = "sbuild -n --apt-update -d %s %s/%s_%s.dsc" % (
- buildreq.get_suite(), srcdir,
+ command = "sbuild -n --apt-update -d %s %s/%s_%s.dsc" % (chroot_name, srcdir,
buildreq.get_package(), buildreq.get_version())
ret = pybitclient.run_cmd (command, self.settings["dry_run"], logfile)
if (ret == 3 or ret == 768):
@@ -237,7 +241,7 @@ def build_slave (self, buildreq, conn_data):
buildreq.get_package(), buildreq.get_version(),
buildreq.get_arch())
if not self.settings["dry_run"] and not os.path.isfile (changes) :
- logging.debug ("build_slave: Failed to find %s file." % (changes))
+ logging.warn ("E: build_slave: Failed to find %s file." % (changes))
retval = "build_changes"
if not retval and checkValue ('debsignkey', self.settings) :
command = "debsign -k%s %s" % (self.settings['debsignkey'], changes)
View
@@ -37,7 +37,7 @@ def fetch_source(self, buildreq, conn_data):
elif (buildreq.transport.uri is not None):
command = "git clone %s %s" % (buildreq.transport.uri, self.workdir)
else:
- logging.debug ("Could not fetch source, no method URI found")
+ logging.warn ("E: Could not fetch source, no method URI found")
retval = "unrecognised uri"
if not retval :
if pybitclient.run_cmd (command, self.settings["dry_run"], None) :
Oops, something went wrong.

0 comments on commit 69990c8

Please sign in to comment.