Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

fixes estimate.py

  • Loading branch information...
commit c676243f85f22ace3ca18dcb07e79db2b133f21c 1 parent 5b2b302
@epico epico authored
Showing with 20 additions and 5 deletions.
  1. +17 −3 estimate.py
  2. +1 −0  generate.py
  3. +2 −2 lib/myconfig.py
View
20 estimate.py
@@ -22,7 +22,7 @@ def handleError(error):
sys.exit(error)
-def handleOneModel(modelfile):
+def handleOneModel(modelfile, reportfile):
modelfilestatuspath = modelfile + config.getStatusPostfix()
modelfilestatus = utils.load_status(modelfilestatuspath)
if not utils.check_epoch(modelfilestatus, 'Generate'):
@@ -30,6 +30,8 @@ def handleOneModel(modelfile):
if utils.check_epoch(modelfilestatus, 'Estimate'):
return
+ reporthandle = open(reportfile, 'wb')
+
result_line_prefix = "average lambda:"
avg_lambda = 0.
@@ -44,14 +46,21 @@ def handleOneModel(modelfile):
close_fds=True)
for line in subprocess.stdout.readlines():
+ reporthandle.writelines([line])
#remove trailing '\n'
+ line = line.decode('utf-8')
line = line.rstrip(os.linesep)
if line.startswith(result_line_prefix):
avg_lambda = float(line[len(result_line_prefix):])
- os.waitpid(subprocess.pid, 0)
+ reporthandle.close()
+
+ (pid, status) = os.waitpid(subprocess.pid, 0)
+ if status != 0:
+ sys.exit('estimate k mixture model returns error.')
#end processing
+ print('average lambda:', avg_lambda)
modelfilestatus['EstimateScore'] = avg_lambda
utils.sign_epoch(modelfilestatus, 'Estimate')
utils.store_status(modelfilestatuspath, modelfilestatus)
@@ -63,13 +72,16 @@ def walkThroughModels(path):
filepath = os.path.join(root, onefile)
if onefile.endswith(config.getModelPostfix()):
subpath = os.path.relpath(filepath, path)
+ reportfile = filepath + config.getReportPostfix()
print("Processing " + subpath)
- handleOneModel(filepath)
+ handleOneModel(filepath, reportfile)
print("Processed " + subpath)
elif onefile.endswith(config.getStatusPostfix()):
pass
elif onefile.endswith(config.getIndexPostfix()):
pass
+ elif onefile.endswith(config.getReportPostfix()):
+ pass
else:
print('Unexpected file:' + filepath)
@@ -102,6 +114,8 @@ def gatherModels(path, indexname):
pass
elif onefile.endswith(config.getIndexPostfix()):
pass
+ elif onefile.endswith(config.getReportPostfix()):
+ pass
else:
print('Unexpected file:' + filepath)
indexfile.close()
View
1  generate.py
@@ -148,6 +148,7 @@ def storeModelStatus(modelfile, textnum, nexttextnum):
#end processing
#save current progress in status file
+ modelnum += 1
indexstatus['GenerateTextEnd'] = nexttextnum
indexstatus['GenerateModelEnd'] = modelnum
View
4 lib/myconfig.py
@@ -39,7 +39,7 @@ def getEvalsDir(self):
return self.m_evals_dir
def getEstimatesModel(self):
- estimates_model = m_tools_dir + '/data/estimates.db'
+ estimates_model = self.m_tools_dir + '/data/estimates.db'
return estimates_model
def getEstimateIndex(self):
@@ -62,7 +62,7 @@ def getMinimumFileSize(self):
#the trained corpus size of model candidates
def getCandidateModelSize(self):
- candidate_model_size = 11.9 * 1024 * 1024
+ candidate_model_size = 11.9 * 1024 * 1024 * 3
return candidate_model_size
def getModelPostfix(self):
Please sign in to comment.
Something went wrong with that request. Please try again.