Permalink
Browse files

improve speed of tryprune.py tool

  • Loading branch information...
1 parent 4ae9a17 commit 3897e71c43e5da9829e342d92d038964acc68316 @epico epico committed Aug 5, 2011
Showing with 22 additions and 3 deletions.
  1. +3 −0 lib/myconfig.py
  2. +19 −3 tryprune.py
View
@@ -48,6 +48,9 @@ def getEstimateIndex(self):
def getSortedEstimateIndex(self):
return 'estimate.sorted.index'
+ def getInMemoryFileSystem(self):
+ return '/dev/shm'
+
def getEvalsText(self):
evals_text = m_evals_dir + '/data/evals.text'
return evals_text
View
@@ -148,11 +148,16 @@ def pruneModel(prunedmodel, k, CDF):
help='CDF parameter of k mixture model prune', \
default=0.99, type=float)
+ parser.add_argument('--fast', action='store_const', \
+ help='Use in-memory filesystem to speed up prune',\
+ const=True, default=False)
+
parser.add_argument('tryname', action='store', \
help='the storage directory')
args = parser.parse_args()
print(args)
+
tryname = 'try' + args.tryname
trydir = os.path.join(config.getFinalModelDir(), tryname)
@@ -187,10 +192,21 @@ def pruneModel(prunedmodel, k, CDF):
#prune merged model
print('pruning')
+
prunedmodel = os.path.join(trydir, 'pruned.db')
- #backup merged model
- shutil.copyfile(mergedmodel, prunedmodel)
- pruneModel(prunedmodel, args.k, args.CDF)
+ if args.fast:
+ shmmodel = os.path.join(config.getInMemoryFileSystem(), 'pruned.db')
+ if os.access(shmmodel, os.F_OK):
+ os.unlink(shmmodel)
+ #copy to memory
+ shutil.copyfile(mergedmodel, shmmodel)
+ pruneModel(shmmodel, args.k, args.CDF)
+ #copy to filesystem
+ shutil.copyfile(shmmodel, prunedmodel)
+ else:
+ #backup merged model
+ shutil.copyfile(mergedmodel, prunedmodel)
+ pruneModel(prunedmodel, args.k, args.CDF)
#validate pruned model
print('validating')

0 comments on commit 3897e71

Please sign in to comment.