Browse files

write tryprune.py

  • Loading branch information...
1 parent 224b933 commit ee0607586c243def746d3ae8eb0555f0696e57de @epico epico committed Jul 25, 2011
Showing with 37 additions and 12 deletions.
  1. +2 −2 docs/fileformat
  2. +3 −0 lib/myconfig.py
  3. +32 −10 tryprune.py
View
4 docs/fileformat
@@ -46,9 +46,9 @@ Status File Format
The lines are sorted by <score>.
Prune Status Files
1. 'merged.db', 'kmm_merged.text' , 'pruned.db', 'kmm_pruned.text', 'interpolation.text' are generated when running prune tools in 'finals/try<name>' sub-directory.
- 2. 'prune.status' file are generated also, like:
+ 2. 'cwd.status' file are generated also, like:
{'PruneEpoch': 4, 'PruneMergeNumber': 1000,
'PruneK':2, 'PruneCDF': 0.6}
Evaluate Status Files
- 1. 'evaluate.status' file are generated, like:
+ 1. 'cwd.status' file are generated, like:
{'EvaluateEpoch': 5, 'EvaluateAverageLambda': 0.66, 'EvaluateCorrectionRate': 0.77}
View
3 lib/myconfig.py
@@ -96,3 +96,6 @@ def getTextPostfix(self):
def getFinalModelFileName(self):
return 'interpolation.text'
+
+ def getFinalStatusFileName(self):
+ return 'cwd.status'
View
42 tryprune.py 100644 → 100755
@@ -6,6 +6,7 @@
from subprocess import Popen, PIPE
from argparse import ArgumentParser
from myconfig import MyConfig
+import utils
config = MyConfig()
@@ -90,7 +91,7 @@ def mergeOneModel(mergedmodel, onemodel, score):
sys.exit('Corrupted model found when merging:' + onemodel)
#end processing
-def mergeSomeModels(tryname, mergedmodel, sortedindexname, mergenum):
+def mergeSomeModels(mergedmodel, sortedindexname, mergenum):
last_score = 1.
#begin processing
indexfile = open(sortedindexname, 'r')
@@ -113,11 +114,11 @@ def mergeSomeModels(tryname, mergedmodel, sortedindexname, mergenum):
#validate merged model
validateModel(mergedmodel)
-def pruneModel(modelfile, k, CDF):
+def pruneModel(prunedmodel, k, CDF):
#begin processing
cmdline = ['./prune_k_mixture_model', \
'-k', k, '--CDF', CDF,
- modelfile]
+ prunedmodel]
subprocess = Popen(cmdline, shell=False, close_fds=True)
#check os.waitpid doc
@@ -126,6 +127,9 @@ def pruneModel(modelfile, k, CDF):
sys.exit('Corrupted model found when pruning:' + modelfile)
#end processing
+ #validate pruned model
+ validateModel(prunedmodel)
+
if __name__ == '__main__':
parser = ArgumentParser(description='Try prune models.')
parser.add_argument('--modeldir', action='store', \
@@ -150,33 +154,51 @@ def pruneModel(modelfile, k, CDF):
args = parser.parse_args()
print(args)
tryname = 'try' + args.tryname
+
+ trydir = os.path.join(config.getFinalDir(), tryname)
+
+ #check try<name> directory
+ if os.access(trydir, os.F_OK):
+ sys.exit('try' + tryname + ' exists.')
+
+ os.makedirs(trydir)
+ cwdstatuspath = os.path.join(trydir, config.getFinalStatusFileName())
+ cwdstatus = {}
+ cwdstatus['PruneMergeNumber'] = args.mergenumber
+ cwdstatus['PruneK'] = args.k
+ cwdstatus['PruneCDF'] = args.CDF
+ utils.store_status(cwdstatuspath, cwdstatus)
+
#merge model candidates
print('merging')
- mergedmodel = os.path.join(config.getFinalDir(), tryname, 'merged.db')
+ mergedmodel = os.path.join(trydir, 'merged.db')
sortedindexname = os.path.join(args.modeldir, \
config.getSortedEstimateIndex())
- mergeSomeModels(tryname, mergedmodel, sortedindexname, args.mergenumber)
+ mergeSomeModels(mergedmodel, sortedindexname, args.mergenumber)
#export textual format
print('exporting')
- exportfile = os.path.join(config.getFinalDir(), tryname, 'kmm_merged.text')
+ exportfile = os.path.join(trydir, 'kmm_merged.text')
exportModel(mergedmodel, exportfile)
#prune merged model
print('pruning')
- prunedmodel = os.path.join(config.getFinalDir(), tryname, 'pruned.db')
+ prunedmodel = os.path.join(trydir, 'pruned.db')
#backup merged model
shutil.copyfile(mergedmodel, prunedmodel)
pruneModel(prunedmodel, args.k, args.CDF)
#export textual format
print('exporting')
- exportfile = os.path.join(config.getFinalDir(), tryname, 'kmm_pruned.text')
+ exportfile = os.path.join(trydir, 'kmm_pruned.text')
exportModel(prunedmodel, exportfile)
#convert to interpolation
print('converting')
kmm_model = exportfile
- inter_model = os.path.join(config.getFinalDir(), tryname, \
- config.getFinalModelFileName())
+ inter_model = os.path.join(trydir, config.getFinalModelFileName())
convertModel(kmm_model, inter_model)
+
+ #sign status epoch
+ utils.sign_epoch(cwdstatus, 'Prune')
+ utils.store_status(cwdstatuspath, cwdstatus)

0 comments on commit ee06075

Please sign in to comment.