etc/global.spec etc/stage.spec @@ -55,7 +55,7 @@ if __name__ == "__main__": # parse out the command line arguments try: - opts,args = getopt.getopt(sys.argv[1:], "ldhvV", ["lax","debug","help", "verbose", "version" ]) + opts,args = getopt.getopt(sys.argv[1:], "ldhvV", ["lax","debug","help", "verbose", "version","replace"]) except getopt.GetoptError: usage() sys.exit(2) @@ -65,7 +65,7 @@ if __name__ == "__main__": verbose=False myopts=[] strict=True - readconfig=True + replace=False # check preconditions if len(args) == 0: @@ -82,9 +82,9 @@ if __name__ == "__main__": if o in ("-V", "--version"): print "Catalyst version "+__version__ sys.exit(1) - if o == "--noconfig": - print "Warning: --noconfig option specified - skipping /etc/catalyst.conf..." - readconfig=False + if o == "--replace": + print "Warning: --replace option specified - any existing output stage will be replaced..." + replace=True if o in ("-d", "--debug"): debug=True print "DEBUG: opts",opts @@ -108,15 +108,21 @@ if __name__ == "__main__": elif host in [ "i386", "i486", "i586", "i686" ]: settings["hostarch"] = "x86" - if readconfig: - if os.path.exists("/etc/catalyst.conf"): - print "Reading in configuration from /etc/catalyst.conf..." - settings.collect("/etc/catalyst.conf") + if os.path.exists("/etc/catalyst/global.spec"): + print "Reading in configuration from /etc/catalyst/global.spec..." + settings.collect("/etc/catalyst/global.spec") + else: + print "Warning: /etc/catalyst/global.spec not found." + + if replace: + if settings.has_key("options"): + settings["options"] = settings["options"] + " replace" else: - print "Warning: /etc/catalyst.conf not found." + settings["options"] = "replace" # the new version of catalyst does not have any configuration file # parse command-line supplied configuration files and spec files in the order they were specified + for arg in args: settings.collect(arg) catalyst @@ -354,7 +354,7 @@ def ismount(path): a.close() for line in mylines: mysplit=line.split() - if os.path.normpath(path) == os.path.normpath(mysplit2): + if os.path.normpath(path) == os.path.normpath(mysplit[2]): print "popen shows",path,"still mounted. Line: ",line return 1 return 0 modules/catalyst_support.py @@ -20,10 +20,6 @@ mv $[foo] $[oni] class FlexDataError(Exception): def __init__(self, message): if message: - (type,value)=sys.exc_info()[:2] - if value!=None: - print - print traceback.print_exc(file=sys.stdout) print print "!!! catalyst: "+message print @@ -46,7 +42,7 @@ class collection: self.evaluated={} - def expandAll(self): + def expand_all(self): # try to expand all variables to find any undefined elements, to record all blanks or throw an exception for key in self.keys(): myvar = self[key] @@ -78,11 +74,7 @@ class collection: string = self.raw[myvar] if type(string) != types.StringType: - raise FlexDataError("expandString received non-string: "+repr(string)+", myvar = "+repr(myvar)) - - if myvar == None: - # this is to make our exception error messages happy if they trigger - myvar = "(None)" + raise FlexDataError("expandString received non-string: "+repr(string)+", myvar = "+repr(myvar)+" (did you forget '>>' in front of multi-line reference?)") mysplit = string.strip().split(" ") if len(mysplit) == 2 and mysplit[0] == "<<": @@ -109,12 +101,15 @@ class collection: if varname in stack: raise KeyError, "Circular reference of '"+varname+"' by '"+stack[-1]+"' ( Call stack: "+repr(stack)+' )' if self.evaluated.has_key(varname): - if type(self.evaluated[varname]) == types.ListType: - raise FlexDataError,"Trying to expand multi-line value \""+varname+"\" in single-line value \""+myvar+"\"" + # if myvar == None, we are being called from self.expand_all() and we don't care where we are being expanded from + if myvar != None and type(self.evaluated[varname]) == types.ListType: + raise FlexDataError,"Trying to expand multi-line value "+repr(varname)+" in single-line value "+repr(myvar) + print "DEBUG: looking at self.evaluated["+varname+"]" ex += self.evaluated[varname] elif self.raw.has_key(varname): - if type(self.raw[varname]) == types.ListType: - raise FlexDataError,"Trying to expand multi-line value \""+varname+"\" in single-line value \""+myvar+"\"" + # if myvar == None, we are being called from self.expand_all() and we don't care where we are being expanded from + if myvar != None and type(self.raw[varname]) == types.ListType: + raise FlexDataError,"Trying to expand multi-line value "+repr(varname)+" in single-line value "+repr(myvar) newstack = stack[:] newstack.append(varname) ex += self.expandString(self.raw[varname],varname,newstack) @@ -167,7 +162,7 @@ class collection: raise FlexDataError,"Circular reference of '"+mysplit[1]+"' by '"+stack[-1]+"' ( Call stack: "+repr(stack)+' )' newstack = stack[:] newstack.append(mysplit[1]) - newlines += self.expandMulti(mysplit[1],newstack) + newlines += self.expandMulti(self.expandString(string=mysplit[1]),newstack) else: newlines.append(self.expandString(string=line)) self.evaluated[myvar] = newlines @@ -221,7 +216,7 @@ class collection: else: continue - def parseline(self,openfile=None): + def parseline(self,openfile=None,dups=False): # parseline() will parse a line and return None on EOF, return [] on a blank line with no data, or will # return a list of string elements if there is data on the line, split along whitespace: [ "foo:", "bar", "oni" ] @@ -260,7 +255,8 @@ class collection: if len(mysplit) == 2 and mysplit[0][-1] == ":" and mysplit[1] == "[": if self.debug: print "DEBUG: MULTI-LINE BLOCK" - myvar = mysplit[0] + # for myvar, remove trailing colon: + myvar = mysplit[0][:-1] mylines = [] while 1: curline = openfile.readline() @@ -271,6 +267,8 @@ class collection: if self.debug: print "DEBUG: end multi-line block" # record value and quit + if not dups and self.raw.has_key(myvar): + raise FlexDataError,"Error - \""+myvar+"\" already defined." self.raw[myvar] = mylines break else: @@ -278,7 +276,10 @@ class collection: mylines.append(curline[:-1]) elif mysplit[0][-1] == ":": #basic element - rejoin all data elements with spaces and add to self.raw - self.raw[mysplit[0][:-1]] = " ".join(mysplit[1:]) + mykey = mysplit[0][:-1] + if not dups and self.raw.has_key(mykey): + raise FlexDataError,"Error - \""+mykey+"\" already defined." + self.raw[mykey] = " ".join(mysplit[1:]) return mysplit def collect(self,filename): modules/flexdata.py @@ -64,8 +64,6 @@ class target: self.require(["target","version"]) - self.settings["workdir"]="/var/tmp/catalyst/$[target]-$[version]" - if not os.path.exists(self.settings["workdir"]): os.makedirs(self.settings["workdir"]) @@ -75,7 +73,8 @@ class target: self.cmd(self.bin("rm") + " -rf "+self.settings["workdir"]) def cmd(self,mycmd,myexc="",badval=None): - print "Executing \""+mycmd.split(" ")[0]+"\"..." + print "Executing \""+mycmd+"\"..." + #print "Executing \""+mycmd.split(" ")[0]+"\"..." try: sys.stdout.flush() retval=spawn_bash(mycmd,self.env) @@ -91,6 +90,21 @@ class target: class chroot(target): + def kill_chroot_pids(self): + cdir=self.settings["chrootdir"] + for pid in os.listdir("/proc"): + if not os.path.isdir("/proc/"+pid): + continue + try: + mylink = os.readlink("/proc/"+pid+"/exe") + except OSError: + # not a pid directory + continue + if mylink[0:len(cdir)] == cdir: + #we got something in our chroot + print "Killing process "+pid+" ("+mylink+")" + self.cmd("/bin/kill -9 "+pid) + def exec_in_chroot(self,key): if not self.settings.has_key(key): @@ -111,10 +125,17 @@ class chroot(target): outfd.close() - retval = spawn(["/bin/bash", outfile ], env=self.env ) + if self.settings["arch"] == "x86" and os.uname[4] == "x86_64": + cmds = [self.bin("linux32"),self.bin("chroot"),self.settings["chrootdir"],self.bin("bash")] + else: + cmds = [self.bin("chroot"),self.settings["chrootdir"],self.bin("bash")] + + cmds.append("/tmp/"+key+".sh") - return retval + retval = spawn(cmds, env=self.env ) + if retval != 0: + raise CatalystError, "Command failure: "+" ".join(cmds) def __init__(self,settings): target.__init__(self,settings) @@ -219,6 +240,11 @@ class chroot(target): class snapshot(target): def __init__(self,settings): target.__init__(self,settings) + + if os.path.exists("/etc/catalyst/snapshot.spec"): + print "Reading in configuration from /etc/catalyst/snapshot.spec..." + self.settings.collect("/etc/catalyst/snapshot.spec") + self.require(["portname","portdir","version","target"]) self.require(["storedir/snapshot"]) @@ -228,9 +254,16 @@ class snapshot(target): print "Removing existing temporary work directory..." self.cmd( self.bin("rm") + " -rf " + self.settings["workdir"] ) - # do not overwrite snapshot if it already exists - if os.path.exists(self.settings["storedir/snapshot"]): - raise CatalystError,"Snapshot "+self.settings["storedir/snapshot"]+" already exists. Aborting." + if "replace" in self.settings["options"].split(): + if os.path.exists(self.settings["storedir/snapshot"]): + print "Removing existing snapshot..." + self.cmd( self.bin("rm") + " -f " + self.settings["storedir/snapshot"]) + + elif os.path.exists(self.settings["storedir/snapshot"]): + print "Snapshot already exists at "+self.settings["storedir/snapshot"]+". Skipping..." + return + else: + print self.settings["storedir/snapshot"],"does not exist - creating it..." # create required directories for dir in [ os.path.dirname(self.settings["storedir/snapshot"]), self.settings["workdir"]+"/portage" ]: @@ -241,8 +274,10 @@ class snapshot(target): rsync_opts = "-a --delete --exclude /packages/ --exclude /distfiles/ --exclude /local/ --exclude CVS/ --exclude /.git/" rsync_cmd = self.bin("rsync") + " " + rsync_opts + " " + os.path.normpath(self.settings["portdir"])+"/ " + os.path.normpath(self.settings["workdir"]+"/portage")+"/" self.cmd(rsync_cmd,"Snapshot failure") + tmpfile=os.path.dirname(self.settings["storedir/snapshot"])+"/."+os.path.basename(self.settings["storedir/snapshot"]) - self.cmd( self.bin("tar") + " -cjf " + self.settings["storedir/snapshot"]+" -C "+self.settings["workdir"]+" portage","Snapshot creation failure") + self.cmd( self.bin("tar") + " -cjf " + tmpfile +" -C "+self.settings["workdir"]+" portage","Snapshot creation failure") + self.cmd( self.bin("mv") + " " + tmpfile + " " + self.settings["storedir/snapshot"], "Couldn't move snapshot to final position" ) # workdir cleanup is handled by catalyst calling our cleanup() method @@ -250,19 +285,23 @@ class stage(chroot): def __init__(self,settings): chroot.__init__(self,settings) - + + if os.path.exists("/etc/catalyst/stage.spec"): + print "Reading in configuration from /etc/catalyst/stage.spec..." + self.settings.collect("/etc/catalyst/stage.spec") + # In the constructor, we want to define settings but not reference them if we can help it, certain things # like paths may not be able to be fully expanded yet until we define our goodies like "source", etc. - self.require(["arch","subarch","profile","storedir/srcstage","storedir/deststage","storedir/snapshot"]) + self.require(["arch","subarch","profile","storedir/srcstage","storedir/deststage","storedir/snapshot","CHOST"]) # look for user-specified USE, if none specified then fallback to HOSTUSE if specified if not self.settings.has_key("USE"): if self.settings.has_key("HOSTUSE"): self.settings["USE"]="$[HOSTUSE]" - # If DISTDIR, USE or CFLAGS not specified, alert the user that they might be missing them - self.recommend(["DISTDIR","USE","CFLAGS","MAKEOPTS"]) + # If distdir, USE or CFLAGS not specified, alert the user that they might be missing them + self.recommend(["distdir","USE","CFLAGS","MAKEOPTS"]) # We also use an initial "~" as a trigger to build an unstable version of the portage tree. This # means we need to use ~arch rather than arch in ACCEPT_KEYWORDS. So if someone specified "~pentium4" @@ -288,9 +327,19 @@ class stage(chroot): self.mountmap["/usr/portage/distfiles"]=self.settings["distdir"] self.settings["chrootdir"]="$[workdir]/chroot" - self.settings["controller_file"]="$[sharedir]/targets/$[target]/$[target]-controller.sh" def run(self): + + if "replace" in self.settings["options"].split(): + if os.path.exists(settings["storedir/deststage"]): + print "Removing existing stage..." + self.cmd( self.bin("rm") + " -f " + self.settings["storedir/deststage"]) + # do not overwrite snapshot if it already exists + elif os.path.exists(self.settings["storedir/deststage"]): + print "Stage "+repr(self.settings["storedir/deststage"])+" already exists - skipping..." + return + #raise CatalystError,"Snapshot "+self.settings["storedir/snapshot"]+" already exists. Aborting." + # look for required files for loc in [ "storedir/srcstage", "storedir/snapshot" ]: if not os.path.exists(self.settings[loc]): @@ -314,16 +363,20 @@ class stage(chroot): self.bind() + self.exec_in_chroot("chroot/prerun") self.exec_in_chroot("chroot/run") self.exec_in_chroot("chroot/postrun") self.unbind() + # remove our tweaks... + + self.chroot_cleanup() + + # now let the spec-defined clean script do all the heavy lifting... + self.exec_in_chroot("chroot/clean") - # remove our tweaks ... - self.chroot_cleanup() - except: self.kill_chroot_pids() @@ -348,7 +401,7 @@ class stage(chroot): self.cmd(unpack_cmd,"Error unpacking source stage.") def unpack_snapshot(self): - destdir=normpath(self.settings["chrootdir"]+"/usr/portage") + destdir=os.path.normpath(self.settings["chrootdir"]+"/usr/portage") cleanup_errmsg="Error removing existing snapshot directory." cleanup_msg="Cleaning up existing portage tree (This can take a long time) ..." unpack_cmd="tar xjpf "+self.settings["storedir/snapshot"]+" -C "+self.settings["chrootdir"]+"/usr" @@ -360,9 +413,6 @@ class stage(chroot): print "Unpacking \""+self.settings["storedir/snapshot"]+" ..." self.cmd(unpack_cmd,unpack_errmsg) - def config_profile_link(self): - self.controller("profile") - def proxy_config(self): # SETUP PROXY SETTINGS - ENVSCRIPT DOESN'T ALWAYS WORK. THIS DOES. print "Writing out proxy settings..." @@ -401,9 +451,10 @@ class stage(chroot): # Back up original files. for file in [ "/etc/resolv.conf", "/etc/hosts" ]: respath=self.settings["chrootdir"]+file - if os.path.exists(respath): - self.cmd("/bin/cp "+respath+" "+respath+".bck","Couldn't back up "+file) - self.cmd("/bin/cp "+file+" "+respath,"Couldn't copy "+file+" into place.") + if os.path.exists(file): + if os.path.exists(respath): + self.cmd("/bin/cp "+respath+" "+respath+".bck","Couldn't back up "+file) + self.cmd("/bin/cp "+file+" "+respath,"Couldn't copy "+file+" into place.") def portage_config(self): self.cmd("/bin/rm -f "+self.settings["chrootdir"]+"/etc/make.conf","Could not remove "+self.settings["chrootdir"]+"/etc/make.conf") @@ -421,8 +472,7 @@ class stage(chroot): self.locale_config() self.network_config() self.portage_config() - self.config_profile_link() - + def chroot_cleanup(self): # Philosophy: Catalyst should only do the bare minimum cleanup in Python. The bulk of the cleanup work should be defined in @@ -443,25 +493,6 @@ class stage(chroot): # Run our "clean" bash script, which should do all of the heavy lifting... - self.controller("clean") - - def controller(self,stage): - if not os.path.exists(self.settings["controller_file"]): - raise CatalystError,"Controller file "+self.settings["controller_file"]+" not found. Aborting." - try: - cmd("/bin/bash "+self.settings["controller_file"]+" "+stage,stage+" script failed.",env=self.env) - except: - self.unbind() - raise CatalystError, "Build failed, could not execute "+stage - - def kill_chroot_pids(self): - print "Checking for processes running in chroot and killing them." - cmdpath=self.settings["sharedir"]+"/misc/kill-chroot-pids.sh") - if not os.path.exists(cmdpath): - raise CatalystError,"Couldn't find "+cmdpath+" - Exiting." - self.cmd("/bin/bash "+cmdpath, "kill-chroot-pids script failed.") - - def capture(self): """capture target in a tarball""" # IF TARGET EXISTS, REMOVE IT - WE WILL CREATE A NEW ONE modules/targets.py @@ -2,4 +2,3 @@ arch: amd64 CFLAGS: -O2 -pipe CHOST: x86_64-pc-linux-gnu HOSTUSE: mmx sse sse2 -profile: default/linux/$[arch]/2008.0 specs/arch/amd64.spec @@ -1,5 +1,4 @@ arch: x86 CFLAGS: -O2 -march=athlon-xp -pipe CHOST: i686-pc-linux-gnu -profile: default/linux/$[arch]/2008.0 HOSTUSE: mmx sse 3dnow specs/arch/athlon-xp.spec @@ -1,5 +1,4 @@ arch: x86 CFLAGS: -O2 -march=prescott -pipe CHOST: i686-pc-linux-gnu -profile: default/linux/$[arch]/2008.0 HOSTUSE: mmx sse sse2 specs/arch/core32.spec @@ -2,4 +2,3 @@ arch: amd64 CFLAGS: -march=nocona -O2 -pipe CHOST: x86_64-pc-linux-gnu HOSTUSE: mmx sse sse2 -profile: default/linux/$[arch]/2008.0 specs/arch/core64.spec @@ -1,4 +1,3 @@ arch: x86 CFLAGS: -O2 -march=i686 -pipe CHOST: i686-pc-linux-gnu -profile: default/linux/$[arch]/2008.0 specs/arch/i686.spec @@ -1,5 +1,4 @@ arch: x86 CFLAGS: -O2 -march=pentium4 -pipe CHOST: i686-pc-linux-gnu -profile: default/linux/$[arch]/2008.0 HOSTUSE: mmx sse sse2 specs/arch/pentium4.spec @@ -1,4 +1,3 @@ arch: x86 CFLAGS: -O2 -mtune=i686 -pipe CHOST: i386-pc-linux-gnu -profile: default/linux/$[arch]/2008.0 specs/arch/x86.spec @@ -1,9 +1,2 @@ # what I want -portname: funtoo -portdir: /root/git/funtoo-portage -version: 2008.08.29 target: snapshot - -# config -storedir: /home/mirror/linux -storedir/snapshot: $[storedir]/snapshots/$[portname]-$[version].tar.bz2 specs/targets/snapshot.spec @@ -1,9 +1,5 @@ target: stage1 -subarch: ~amd64 -# stuff we specify - our info: -version: 2008.08.29 - # REQUIRED: # RECOMMENDED, otherwise use HOSTUSE. USE: $[HOSTUSE] @@ -43,9 +39,9 @@ en_US.UTF-8 UTF-8 # do any cleanup that you need with things bind mounted here: chroot/postrun: [ - $[chroot/setup] + >> chroot/setup [ -e /var/tmp/ccache ] && emerge -C dev-util/ccache -[ +] # do the stuff here that you need to do, with bind mounts unmounted (for safety:) @@ -91,7 +87,7 @@ print ] chroot/run: [ -$[chroot/setup] +>> chroot/setup if [ -d /var/tmp/ccache ] then export CCACHE_DIR=/var/tmp/ccache @@ -99,24 +95,28 @@ then fi cat > /tmp/build.py << EOF -$[chroot/pythonjunk] +>> chroot/pythonjunk EOF export buildpkgs="$(python /tmp/build.py)" export STAGE1_USE="$(portageq envvar STAGE1_USE)" export USE="-* bindist build ${STAGE1_USE}" export FEATURES="nodoc noman noinfo" - +[ -e /usr/bin/ccache ] && export FEATURES="ccache $FEATURES" ## Sanity check profile -if [ -z "${clst_buildpkgs}" ] +if [ -z "${buildpkgs}" ] then echo "Your profile seems to be broken." echo "Could not build a list of build packages." echo "Double check your /etc/make.profile link and the 'packages' files." exit 1 +else + echo "WE ARE BUILDING: ${buildpkgs}" fi +export ROOT="$[ROOT]" +install -d $[ROOT] emerge baselayout -emerge --noreplace --oneshot ${clst_buildpkgs} +emerge --noreplace --oneshot ${buildpkgs} ] specs/targets/stage1.spec @@ -1,18 +1,6 @@ target: stage2 -subarch: ~amd64 -# stuff we specify - our info: -version: 2008.08.29 - -# REQUIRED: -# RECOMMENDED, otherwise use HOSTUSE. USE: $[HOSTUSE] -# arch stuff -arch: amd64 -CFLAGS: -O2 -pipe -CHOST: x86_64-pc-linux-gnu -HOSTUSE: mmx sse sse2 -profile: default/linux/$[arch]/2008.0 chroot/setup: [ /usr/sbin/env-update specs/targets/stage2.spec @@ -1,8 +1,4 @@ target: stage3 -subarch: ~amd64 - -# stuff we specify - our info: -version: 2008.08.29 # RECOMMENDED, otherwise use HOSTUSE. USE: $[HOSTUSE] @@ -43,7 +39,7 @@ en_US.UTF-8 UTF-8 # do any cleanup that you need with things bind mounted here: chroot/postrun: [ - $[chroot/setup] +<< chroot/setup [ -e /var/tmp/ccache ] && emerge -C dev-util/ccache [ specs/targets/stage3.spec etc/catalyst.conf misc/kill-chroot-pids.sh 0cc3c221ae363e4f137f7065abe08d180aba8ce0 Daniel Robbins drobbins@funtoo.org http://github.com/funtoo/metro/commit/6a1e20aea2f8168b2cd3d0fd961f3e7836899502 6a1e20aea2f8168b2cd3d0fd961f3e7836899502 2008-09-02T18:39:49-07:00 2008-09-02T18:39:49-07:00 it's somewhat working! 6f8a20e37c800628db38746db66cd33c9d6c7ee0 Daniel Robbins drobbins@funtoo.org