Skip to content
Browse files

Checkpoint changes to packaging scripts.

  • Loading branch information...
1 parent 5d2ac55 commit acf722bb9e67e63d42a00feaf4b3f93f80963797 @kreuter kreuter committed
Showing with 149 additions and 46 deletions.
  1. +9 −4 buildscripts/makealldists.py
  2. +113 −13 buildscripts/makedist.py
  3. +27 −29 buildscripts/mergerepositories.py
View
13 buildscripts/makealldists.py
@@ -188,7 +188,9 @@ def __main__():
("ubuntu", "9.4"),
("ubuntu", "8.10"),
("debian", "5.0"),
- ("centos", "5.4"),)
+ ("centos", "5.4"),
+ ("fedora", "11"),
+ ("fedora", "12"))
arches = ("x86", "x86_64")
# mongos = branches.split(',')
# Run a makedist for each distro/version/architecture tuple above.
@@ -199,6 +201,9 @@ def __main__():
procs = []
count = 0
for ((distro, distro_version), arch, spec) in gen([dists, arches, [branches]]):
+ # FIXME: now x86 fedoras on RackSpace circa 04/10.
+ if distro == "fedora" and arch == "x86":
+ continue
count+=1
opts = makedistopts
if distro in ["debian", "ubuntu"]:
@@ -258,9 +263,9 @@ def __main__():
if r != 0:
raise Exception("mergerepositories.py exited %d" % r)
print repodir
- pushrepo(repodir)
- shutil.rmtree(outputroot)
- shutil.rmtree(repodir)
+# pushrepo(repodir)
+# shutil.rmtree(outputroot)
+# shutil.rmtree(repodir)
return 0
View
126 buildscripts/makedist.py
@@ -50,6 +50,7 @@
from libcloud.providers import get_driver
from libcloud.drivers.ec2 import EC2NodeDriver, NodeImage
from libcloud.base import Node, NodeImage, NodeSize, NodeState
+from libcloud.ssh import ParamikoSSHClient
EC2 = get_driver(Provider.EC2)
EC2Driver=EC2NodeDriver(settings.id, settings.key)
@@ -226,6 +227,9 @@ def __enter__(self):
def __exit__(self, type, value, traceback):
self.stop()
+ def setup(self):
+ pass
+
def getHostname(self):
return self.node.public_ip[0] # FIXME private_ip?
# return self.ec2_internal_hostname if self.use_internal_name else self.ec2_external_hostname
@@ -242,7 +246,7 @@ def __init__(self, **kwargs):
(("ubuntu", "9.4", "*"), "root"),
(("ubuntu", "8.10", "*"), "root"),
(("ubuntu", "8.4", "*"), "ubuntu"),
- (("fedora", "8", "*"), "root"),
+ (("fedora", "*", "*"), "root"),
(("centos", "*", "*"), "root"))),
]
@@ -416,10 +420,10 @@ class ScriptFileConfigurator (BaseConfigurator):
yum -y install {pkg_prereq_str}
"""
rpm_build_commands="""
-for d in BUILD BUILDROOT RPMS SOURCES SPECS SRPMS; do mkdir -p /usr/src/redhat/$d; done
-cp -v "{pkg_name}{pkg_name_suffix}-{pkg_version}/rpm/mongo.spec" /usr/src/redhat/SPECS/{pkg_name}{pkg_name_suffix}.spec
-tar -cpzf /usr/src/redhat/SOURCES/"{pkg_name}{pkg_name_suffix}-{pkg_version}".tar.gz "{pkg_name}{pkg_name_suffix}-{pkg_version}"
-rpmbuild -ba /usr/src/redhat/SPECS/{pkg_name}{pkg_name_suffix}.spec
+for d in BUILD BUILDROOT RPMS SOURCES SPECS SRPMS; do mkdir -p {rpmbuild_dir}/$d; done
+cp -v "{pkg_name}{pkg_name_suffix}-{pkg_version}/rpm/mongo.spec" {rpmbuild_dir}/SPECS/{pkg_name}{pkg_name_suffix}.spec
+tar -cpzf {rpmbuild_dir}/SOURCES/"{pkg_name}{pkg_name_suffix}-{pkg_version}".tar.gz "{pkg_name}{pkg_name_suffix}-{pkg_version}"
+rpmbuild -ba --target={distro_arch} {rpmbuild_dir}/SPECS/{pkg_name}{pkg_name_suffix}.spec
# FIXME: should install the rpms, check if mongod is running.
"""
# FIXME: this is clean, but adds 40 minutes or so to the build process.
@@ -495,8 +499,8 @@ def __init__(self, **kwargs):
self.configuration += [("pkg_product_dir",
((("ubuntu", "*", "*"), self.deb_productdir),
(("debian", "*", "*"), self.deb_productdir),
- (("fedora", "*", "*"), self.rpm_productdir),
- (("centos", "*", "*"), self.rpm_productdir))),
+ (("fedora", "*", "*"), "~/rpmbuild/RPMS"),
+ (("centos", "*", "*"), "/usr/src/redhat/RPMS"))),
("pkg_prereqs",
((("ubuntu", "9.4", "*"),
self.versioned_deb_boost_prereqs + self.unversioned_deb_xulrunner_prereqs + self.common_deb_prereqs),
@@ -510,7 +514,7 @@ def __init__(self, **kwargs):
self.unversioned_deb_boost_prereqs + self.old_versioned_deb_xulrunner_prereqs + self.common_deb_prereqs),
(("debian", "5.0", "*"),
self.versioned_deb_boost_prereqs + self.unversioned_deb_xulrunner_prereqs + self.common_deb_prereqs),
- (("fedora", "8", "*"),
+ (("fedora", "*", "*"),
self.fedora_prereqs),
(("centos", "5.4", "*"),
self.centos_preqres))),
@@ -573,6 +577,11 @@ def __init__(self, **kwargs):
("pkg_name_conflicts",
((("*", "*", "*"), ["", "-stable", "-unstable", "-snapshot", "-oldstable"]),
)),
+ ("rpmbuild_dir",
+ ((("fedora", "*", "*"), "~/rpmbuild"),
+ (("centos", "*", "*"), "/usr/src/redhat"),
+ (("*", "*","*"), ''),
+ )),
]
@@ -668,8 +677,8 @@ def genscript(self):
# suppose this works elsewhere
pkg_name_conflicts = ", ".join([self.pkg_name+conflict for conflict in pkg_name_conflicts]),
mongo_arch=self.mongo_arch,
- mongo_pub_version=mongo_pub_version
- )
+ mongo_pub_version=mongo_pub_version,
+ rpmbuild_dir=self.configurator.default('rpmbuild_dir'))
script+='rm -rf mongo'
return script
@@ -693,7 +702,95 @@ def __exit__(self, type, value, traceback):
class Configurator(SshConnectionConfigurator, EC2InstanceConfigurator, ScriptFileConfigurator, BaseHostConfigurator):
def __init__(self, **kwargs):
super(Configurator, self).__init__(**kwargs)
+
+class rackspaceNode(object):
+ def __init__(self, configurator, **kwargs):
+ driver = get_driver(Provider.RACKSPACE)
+ self.conn = driver(settings.rackspace_account, settings.rackspace_api_key)
+ name=self.imgname+'-'+str(os.getpid())
+ images=filter(lambda x: (x.name.find(self.imgname) > -1), self.conn.list_images())
+ sizes=self.conn.list_sizes()
+ sizes.sort(cmp=lambda x,y: int(x.ram)<int(y.ram))
+ node = None
+ if len(images) != 1:
+ raise Exception("too many images with \"%s\" in the name" % self.imgname)
+ image = images[0]
+ self.node = self.conn.create_node(image=image, name=name, size=sizes[0])
+ print self.node
+ self.password = self.node.extra['password']
+
+ def initwait(self):
+ while 1:
+ n=None
+ # EC2 sometimes takes a while to report a node.
+ for i in range(6):
+ nodes = [n for n in self.conn.list_nodes() if (n.id==self.node.id)]
+ if len(nodes)>0:
+ n=nodes[0]
+ break
+ else:
+ time.sleep(10)
+ if not n:
+ raise Exception("couldn't find node with id %s" % self.node.id)
+ if n.state == NodeState.PENDING:
+ time.sleep(10)
+ else:
+ self.node = n
+ break
+ print "ok"
+ # Now wait for the node's sshd to be accepting connections.
+ print "waiting for ssh"
+ sshwait = True
+ if sshwait == False:
+ return
+ while sshwait:
+ s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ try:
+ try:
+ s.connect((self.node.public_ip[0], 22))
+ sshwait = False
+ print "connected on port 22 (ssh)"
+ time.sleep(15) # arbitrary timeout, in case the
+ # remote sshd is slow.
+ except socket.error, err:
+ pass
+ finally:
+ s.close()
+ time.sleep(3) # arbitrary timeout
+ print "ok"
+
+ def getHostname(self):
+ return self.node.public_ip[0]
+
+ def setup(self):
+ self.putSshKey()
+
+ def putSshKey(self):
+ keyfile=settings.makedist['ssh_keyfile']
+ ssh = ParamikoSSHClient(hostname = self.node.public_ip[0], password = self.password)
+ ssh.connect()
+ print "putting ssh public key"
+ ssh.put(".ssh/authorized_keys", contents=open(keyfile+'.pub').read(), chmod=0600)
+ print "ok"
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, arg0, arg1, arg2):
+ if arg1:
+ print ">>>>>>>>"
+ print arg1
+ print arg2
+ print "<<<<<<<<"
+ print "shutting down node %s" % self.node
+ self.node.destroy()
+
+class fedoraNode(rackspaceNode):
+ def __init__(self, configurator, **kwargs):
+ self.imgname='Fedora '+kwargs['distro_version']
+ super(fedoraNode, self).__init__(configurator, **kwargs)
+
def parse_mongo_version_spec (spec):
foo = spec.split(":")
mongo_version = foo[0] # this can be a commit id, a
@@ -759,9 +856,12 @@ def main():
print """# Going to run the following on a fresh AMI:"""
print f.read()
time.sleep(10)
- with EC2Instance(configurator, **kwargs) as ec2:
- ec2.initwait()
- kwargs["ssh_host"] = ec2.getHostname()
+ # FIXME: it's not the best to have two different pathways for
+ # the different hosting services, but...
+ with EC2Instance(configurator, **kwargs) if kwargs['distro_name'] != 'fedora' else fedoraNode(configurator, **kwargs) as host:
+ host.initwait()
+ host.setup()
+ kwargs["ssh_host"] = host.getHostname()
with SshConnection(configurator, **kwargs) as ssh:
ssh.runRemotely(["uname -a; ls /"])
ssh.runRemotely(["mkdir", "pkg"])
View
56 buildscripts/mergerepositories.py
@@ -39,18 +39,13 @@ def tryEC2():
node.destroy()
-# I don't think libcloud's Nodes implement __enter__ and __exit__, and
-# I like the with statement for ensuring that we don't leak nodes when
-# we don't have to.
-class ec2node(object):
+class node(object):
def initWait(self):
- print "waiting for node to spin up"
- # Wait for EC2 to tell us the node is running.
while 1:
n=None
# EC2 sometimes takes a while to report a node.
for i in range(6):
- nodes = [n for n in EC2Driver.list_nodes() if (n.id==self.node.id)]
+ nodes = [n for n in self.list_nodes() if (n.id==self.node.id)]
if len(nodes)>0:
n=nodes[0]
break
@@ -77,7 +72,7 @@ def initWait(self):
sshwait = False
print "connected on port 22 (ssh)"
time.sleep(15) # arbitrary timeout, in case the
- # remote sshd is slow.
+ # remote sshd is slow.
except socket.error, err:
pass
finally:
@@ -85,7 +80,6 @@ def initWait(self):
time.sleep(3) # arbitrary timeout
print "ok"
-
def __enter__(self):
return self
@@ -93,6 +87,13 @@ def __exit__(self, arg0, arg1, arg2):
print "shutting down node %s" % self.node
self.node.destroy()
+# I don't think libcloud's Nodes implement __enter__ and __exit__, and
+# I like the with statement for ensuring that we don't leak nodes when
+# we don't have to.
+class ec2node(node):
+ def list_nodes(self):
+ return EC2Driver.list_nodes()
+
class ubuntuNode(ec2node):
def __init__(self):
image=NodeImage('ami-bf07ead6', 'ubuntu 10.4', EC2)
@@ -107,28 +108,25 @@ def __init__(self):
self.node = EC2Driver.create_node(image=image, name="ubuntu-test", size=size, securitygroup=['default', 'dist-slave', 'buildbot-slave'], keyname='kp1')
+class rackspaceNode(node):
+ def list_nodes(self):
+ self.conn.list_nodes()
-
-def tryRackSpace():
- driver=get_driver(Provider.RACKSPACE)
- conn = driver(settings.rackspace_account, settings.rackspace_api_key)
- string='Fedora 11'
- images=filter(lambda x: (x.name.find(string) > -1), conn.list_images())
- sizes=conn.list_sizes()
- sizes.sort(cmp=lambda x,y: int(x.ram)<int(y.ram))
- node = None
- if len(images) != 1:
- raise "too many images with \"%s\" in the name" % string
- try:
+class fedora11Node(rackspaceNode):
+ def __init__(self):
+ driver = get_driver(Provider.RACKSPACE)
+ self.conn = driver(settings.rackspace_account, settings.rackspace_api_key)
+ string='Fedora 11'
+ images=filter(lambda x: (x.name.find(string) > -1), self.conn.list_images())
+ sizes=self.conn.list_sizes()
+ sizes.sort(cmp=lambda x,y: int(x.ram)<int(y.ram))
+ node = None
+ if len(images) != 1:
+ raise "too many images with \"%s\" in the name" % string
image = images[0]
- node = conn.create_node(image=image, name=string, size=sizes[0])
- print node
- print node.extras['password']
- while node.state == NodeState.PENDING:
- time.sleep(10)
- finally:
- if node:
- node.destroy()
+ self.node = self.conn.create_node(image=image, name=string, size=sizes[0])
+ print self.node
+ self.password = self.node.extra['password']
class Err(Exception):
pass

0 comments on commit acf722b

Please sign in to comment.
Something went wrong with that request. Please try again.