improve speed of tryprune.py tool

lib/myconfig.py

@@ -48,6 +48,9 @@ def getEstimateIndex(self):
     def getSortedEstimateIndex(self):
         return 'estimate.sorted.index'
 
+    def getInMemoryFileSystem(self):
+        return '/dev/shm'
+
     def getEvalsText(self):
         evals_text = m_evals_dir + '/data/evals.text'
         return evals_text

tryprune.py

@@ -148,11 +148,16 @@ def pruneModel(prunedmodel, k, CDF):
                             help='CDF parameter of k mixture model prune', \
                             default=0.99, type=float)
 
+    parser.add_argument('--fast', action='store_const', \
+                            help='Use in-memory filesystem to speed up prune',\
+                            const=True, default=False)
+
     parser.add_argument('tryname', action='store', \
                             help='the storage directory')
 
     args = parser.parse_args()
     print(args)
+
     tryname = 'try' + args.tryname
 
     trydir = os.path.join(config.getFinalModelDir(), tryname)
@@ -187,10 +192,21 @@ def pruneModel(prunedmodel, k, CDF):
 
     #prune merged model
     print('pruning')
+
     prunedmodel = os.path.join(trydir, 'pruned.db')
-    #backup merged model
-    shutil.copyfile(mergedmodel, prunedmodel)
-    pruneModel(prunedmodel, args.k, args.CDF)
+    if args.fast:
+        shmmodel = os.path.join(config.getInMemoryFileSystem(), 'pruned.db')
+        if os.access(shmmodel, os.F_OK):
+            os.unlink(shmmodel)
+        #copy to memory
+        shutil.copyfile(mergedmodel, shmmodel)
+        pruneModel(shmmodel, args.k, args.CDF)
+        #copy to filesystem
+        shutil.copyfile(shmmodel, prunedmodel)
+    else:
+        #backup merged model
+        shutil.copyfile(mergedmodel, prunedmodel)
+        pruneModel(prunedmodel, args.k, args.CDF)
 
     #validate pruned model
     print('validating')