Skip to content

Commit

Permalink
minor code cleanup
Browse files Browse the repository at this point in the history
  • Loading branch information
Mark Fiers committed Sep 28, 2012
1 parent 4a2848e commit 765b199
Showing 1 changed file with 74 additions and 56 deletions.
130 changes: 74 additions & 56 deletions lib/python/moa/plugin/system/moautil.py
View file
Original file line number Diff line number Diff line change
@@ -1,54 +1,58 @@
# Copyright 2009-2011 Mark Fiers
# The New Zealand Institute for Plant & Food Research
#
#
# This file is part of Moa - http://github.com/mfiers/Moa
#
#
# Licensed under the GPL license (see 'COPYING')
#
#
"""
**moautil** - Some extra utilities - copy/move jobs
---------------------------------------------------
----------------------------------------------------
"""

import glob
import os
import re
import sys
import glob
import shutil
import sys
import tarfile
import optparse

import moa.args
import moa.logger
l = moa.logger.getLogger(__name__)
#l.setLevel(moa.logger.DEBUG)

import moa.ui
import moa.args
from moa.sysConf import sysConf


l = moa.logger.getLogger(__name__)


@moa.args.private
@moa.args.command
def archive_incl(job, args):
"""
Toggle a directory to be included in an moa archive.
"""
moa.ui.message("%s {{green}}will{{reset}} be included in an archive" % job.wd)

moa.ui.message("%s {{green}}will{{reset}} be included in an archive" % (
job.wd))

moaConfDir = os.path.join(job.wd, '.moa')

if not os.path.exists(moaConfDir):
return #default is to include
return # default is to include
moaNoArchiveFile = os.path.join(moaConfDir, 'noarchive')
if os.path.exists(moaNoArchiveFile):
os.unlink(moaNoArchiveFile)


@moa.args.private
@moa.args.command
def archive_excl(job, args):
"""
Toggle a directory to be included in an moa archive.
"""
moa.ui.message("%s will {{bold}}NOT{{reset}} be included in an archive" % job.wd)
moa.ui.message("%s will {{bold}}NOT{{reset}} be included in an archive" %
job.wd)

moaConfDir = os.path.join(job.wd, '.moa')
if not os.path.exists(moaConfDir):
os.makedirs(moaConfDir)
Expand All @@ -60,14 +64,15 @@ def archive_excl(job, args):

@moa.args.argument('name', nargs='?', help='archive name')
@moa.args.addFlag('-t', '--template', help='Store this archive as a template')
@moa.args.addFlag('-s', '--sync', help='Alternative approach to deal with sync type jobs - only include _ref directories')
@moa.args.addFlag('-s', '--sync', help='Alternative approach to deal with ' +
'sync type jobs - only include _ref directories')
@moa.args.forceable
@moa.args.localRecursive
@moa.args.command
def archive(job, args):
"""
Archive a job, or tree with jobs for later reuse.
This command stores only those files that are necessary for
execution of this job, that is: templates & configuration. In &
output files, and any other file are ignored. An exception to this
Expand All @@ -78,20 +83,20 @@ def archive(job, args):
parameter - in which case all (moa job containing) subdirectories
are included in the archive.
"""


if args.name:
archiveName = args.name
else:
archiveName = job.conf.get('jobid', None)
if archiveName == None:
if archiveName is None:
moa.ui.exitError('When not in a moa job, you must define an ' +
'archive name')

if args.template:
if ('/' in archiveName):
moa.ui.exitError("You can not specify a path when using --template")

moa.ui.exitError(
"You can not specify a path when using --template")

archivePath = os.path.abspath(os.path.expanduser(
sysConf.plugins.moautil.get('dir', '~/.config/moa/archive')))
if not os.path.exists(archivePath):
Expand All @@ -101,7 +106,7 @@ def archive(job, args):
archivePath, archiveName = os.path.split(archiveName)
else:
archivePath = '.'

if archiveName[-2:] == 'gz':
moa.ui.exitError("Do not specify an extension for the archive")

Expand All @@ -116,21 +121,22 @@ def archive(job, args):

if not args.recursive and not sysConf.job.isMoa():
moa.ui.exitError('Nothing to archive')

l.info("archiving %s" % archiveFile)

TF = tarfile.open(
name = archiveFile,
mode = 'w:gz')
name=archiveFile,
mode='w:gz')

def _addFiles(tf, path, job):
moa.ui.message("Archiving %s" % path)
for pattern in job.data.moaFiles:
for fl in glob.glob(os.path.join(path, pattern)):
if fl[-1] == '~': continue
if fl[-1] == '~':
continue
l.debug("adding to tarfile: %s" % fl)
tf.add(fl)

if args.recursive:
for path, dirs, files in os.walk('.'):
if os.path.exists(os.path.join(path, '.moa', 'noarchive')):
Expand All @@ -145,10 +151,11 @@ def _addFiles(tf, path, job):
else:
_addFiles(TF, '.', job)

@moa.args.argument('todir', metavar='to', nargs='?', help='copy to this path')
@moa.args.argument('fromdir', metavar='from', nargs=1, help='copy from this path')
@moa.args.addFlag('-o', '--overwrite', help='if the target dir exists - overwrite (instead' +
'of copying into that dir')

@moa.args.argument('todir', metavar='to', nargs='?', help='copy to')
@moa.args.argument('fromdir', metavar='from', nargs=1, help='copy from')
@moa.args.addFlag('-o', '--overwrite', help='if the target dir exists' +
' - overwrite (instead of copying into that dir')
@moa.args.localRecursive
@moa.args.command
def cp(job, args):
Expand All @@ -162,39 +169,43 @@ def cp(job, args):
conjunction with the -r (recursive) flag the complete tree is
copied.
"""

#remember the files copied
sysConf.moautil.filesCopied = []

if args.todir:
dirTo = args.todir
else: dirTo = '.'
else:
dirTo = '.'

dirFrom = args.fromdir[0]

if dirFrom[-1] == '/': dirFrom = dirFrom[:-1]

if dirFrom[-1] == '/':
dirFrom = dirFrom[:-1]
fromBase = os.path.basename(dirFrom)

if dirTo[-1] == '/': dirTo = dirTo[:-1]
if dirTo[-1] == '/':
dirTo = dirTo[:-1]

toBase = os.path.basename(dirTo)

print toBase

#print fromBase, toBase
# trick - the second argument is a number
# renumber the target directory
if re.match("^[0-9]+$", toBase) and re.match("^[0-9]+\..+$", fromBase):
print toBase, fromBase
toBase = re.sub("^[0-9]*\.", toBase + '.', fromBase)
dirTo = os.path.join(os.path.dirname(dirTo), toBase)

elif args.overwrite and os.path.exists(dirTo):
#if the 'to' directory exists - create a new sub directory
dirTo = os.path.join(dirTo, fromBase)
#if the 'to' directory exists - create a new sub directory
dirTo = os.path.join(dirTo, fromBase)

l.info("Copying from %s to %s" % (dirFrom, dirTo))

if not args.recursive:
if not args.recursive:
if not os.path.isdir(dirFrom):
moa.ui.exitError(
"Need %s to be a directory" % dirFrom)
Expand All @@ -212,11 +223,12 @@ def cp(job, args):
thisToPath = path.replace(dirFrom, dirTo)
_copyMoaDir(fromJob, thisToPath)


def _copyMoaDir(job, toDir):
#l.info("Copying from %s to %s" % (job.wd, toDir))
for pattern in job.data.moaFiles:
for fromFile in glob.glob(os.path.join(job.wd, pattern)):

for fromFile in glob.glob(os.path.join(job.wd, pattern)):
toFile = fromFile.replace(job.wd, toDir)
if os.path.exists(fromFile) and not os.path.isfile(fromFile):
l.critical("Uncertain about copying %s" % fromFile)
Expand All @@ -228,14 +240,16 @@ def _copyMoaDir(job, toDir):
shutil.copyfile(fromFile, toFile)
sysConf.moautil.filesCopied.append(toFile)


def hook_git_finish_cp():
files = sysConf.moautil.get('filesCopied', [])
repo = sysConf.git.getRepo(sysConf.job)
repo.index.add(map(os.path.abspath, files))
repo.index.commit("moa cp %s" % " ".join(sysConf.newargs))

@moa.args.argument('todir', metavar='to', nargs='?', help='copy to this path')
@moa.args.argument('fromdir', metavar='from', nargs=1, help='copy from this path')

@moa.args.argument('todir', metavar='to', nargs='?', help='copy to')
@moa.args.argument('fromdir', metavar='from', nargs=1, help='copy from')
@moa.args.localRecursive
@moa.args.command
def mv(job, args):
Expand All @@ -250,7 +264,7 @@ def mv(job, args):
#l.info("copying from" + fr)
if fr[-1] == '/':
fr = fr[:-1]

if args.todir:
to = args.todir
else:
Expand All @@ -263,31 +277,34 @@ def mv(job, args):
l.critical("Cannot resolve %s (%s)" % (fr, newfr))
sys.exit(1)
fr = newfr[0]

if re.match('^\d+$', to):
if re.search('^\d+', fr):
to = re.sub('^\d+', to, fr)
else:
to = '%s.%s' % (to, fr)

moa.ui.message("Moving %s to %s" % (fr, to))
if sysConf.git.active and sysConf.git.repo:
l.debug('git is active - deferring move to later')
l.debug('git is active - deferring move to later')
sysConf.moautil.mv.fr = fr
sysConf.moautil.mv.to = to
else:
shutil.move(fr, to)



def hook_git_finish_mv():

#make sure the 'from' directory is under git control
sysConf.git.commitDir(sysConf.moautil.mv.fr, 'Preparing for git mv')
#seems that we need to call git directly gitpython does not work
os.system('git mv %s %s' % (sysConf.moautil.mv.fr, sysConf.moautil.mv.to))
os.system('git commit %s %s -m "moa mv %s %s"' % (sysConf.moautil.mv.fr, sysConf.moautil.mv.to,
sysConf.moautil.mv.fr, sysConf.moautil.mv.to))
os.system('git commit %s %s -m "moa mv %s %s"' % (
sysConf.moautil.mv.fr, sysConf.moautil.mv.to,
sysConf.moautil.mv.fr, sysConf.moautil.mv.to))
#repo.index.add(map(os.path.abspath, files))
#repo.index.commit("moa cp %s" % " ".join(sysConf.newargs))


#Unittest scripts
RENTEST = '''
Expand Down Expand Up @@ -323,7 +340,8 @@ def hook_git_finish_mv():
moa simple -t "test2" -- echo "subtest"
cd ../../
moa cp 10.test 40.test 2>/dev/null
[[ ! -d "40.test/05.subtest" ]] || (echo "subdirectory should not be there"; false)
[[ ! -d "40.test/05.subtest" ]] || \
(echo "subdirectory should not be there"; false)
moa cp -r 10.test 50.test 2>/dev/null
[[ -d "50.test/05.subtest" ]] || (echo "subdirectory is not there?"; false)
'''

0 comments on commit 765b199

Please sign in to comment.