Permalink
Browse files

fixes pep8 warnings

  • Loading branch information...
1 parent 04428f2 commit 2575950b486c9716174a14a8b5ad8c04fa086a14 @epico epico committed Jul 26, 2011
Showing with 45 additions and 14 deletions.
  1. +6 −1 estimate.py
  2. +11 −5 evaluate.py
  3. +6 −3 generate.py
  4. +1 −1 lib/myconfig.py
  5. +10 −1 lib/utils.py
  6. +5 −3 segment.py
  7. +6 −0 tryprune.py
View
@@ -17,9 +17,11 @@
os.chdir(libpinyin_sub_dir)
#chdir done
+
def handleError(error):
sys.exit(error)
+
def handleOneModel(modelfile):
modelfilestatuspath = modelfile + config.getStatusPostfix()
modelfilestatus = utils.load_status(modelfilestatuspath)
@@ -54,6 +56,7 @@ def handleOneModel(modelfile):
utils.sign_epoch(modelfilestatus, 'Estimate')
utils.store_status(modelfilestatuspath, modelfilestatus)
+
def walkThroughModels(path):
for root, dirs, files in os.walk(path, topdown=True, onerror=handleError):
for onefile in files:
@@ -70,6 +73,7 @@ def walkThroughModels(path):
else:
print('Unexpected file:' + filepath)
+
def gatherModels(path, indexname):
indexfilestatuspath = indexname + config.getStatusPostfix()
indexfilestatus = utils.load_status(indexfilestatuspath)
@@ -106,11 +110,12 @@ def gatherModels(path, indexname):
utils.sign_epoch(indexfilestatus, 'Estimate')
utils.store_status(indexfilestatuspath, indexfilestatus)
+
def sortModels(indexname, sortedindexname):
sortedindexfilestatuspath = sortedindexname + config.getStatusPostfix()
sortedindexfilestatus = utils.load_status(sortedindexfilestatuspath)
if utils.check_epoch(sortedindexfilestatus, 'Estimate'):
- return
+ return
#begin processing
records = []
View
@@ -17,9 +17,10 @@
libpinyindir = config.getEvalsDir()
os.chdir(libpinyindir)
-datafiles = [ 'gb_char.table', 'gbk_char.table', \
- config.getFinalModelFileName(), 'evals.text', \
- 'deleted_bigram.db']
+datafiles = ['gb_char.table', 'gbk_char.table', \
+ config.getFinalModelFileName(), 'evals.text', \
+ 'deleted_bigram.db']
+
def checkData():
cwd = os.getcwd()
@@ -29,6 +30,7 @@ def checkData():
sys.exit('missing one data file:' + onefile)
os.chdir(cwd)
+
def cleanUpData():
#begin processing
cmdline = ['/usr/bin/make', '-f', 'Makefile.data', 'clean']
@@ -38,6 +40,7 @@ def cleanUpData():
sys.exit('make clean for data files failed.')
#end processing
+
def buildData():
#begin processing
cmdline = ['/usr/bin/make', '-f', 'Makefile.data', 'build']
@@ -47,6 +50,7 @@ def buildData():
sys.exit('make build for data files failed.')
#end processing
+
def estimateModel():
#change to utils/training subdir
cwd = os.getcwd()
@@ -73,6 +77,7 @@ def estimateModel():
os.chdir(cwd)
return avg_lambda
+
def modifyCodeforLambda(lambdaparam):
#begin processing
cmdline = ['/usr/bin/make', '-f', 'Makefile.data', 'rebuild', \
@@ -83,6 +88,7 @@ def modifyCodeforLambda(lambdaparam):
sys.exit('make rebuild for data files failed.')
#end processing
+
def evaluateModel():
#change to utils/training subdir
cwd = os.getcwd()
@@ -159,7 +165,7 @@ def evaluateModel():
cwdstatus['EvaluateCorrectionRate'] = rate
utils.store_status(cwdstatuspath, cwdstatus)
-
- utils.sign_epoch(cwdstatus, 'Evaluate');
+
+ utils.sign_epoch(cwdstatus, 'Evaluate')
utils.store_status(cwdstatuspath, cwdstatus)
print('done')
View
@@ -16,6 +16,7 @@
os.chdir(libpinyin_sub_dir)
#chdir done
+
def handleError(error):
sys.exit(error)
@@ -47,12 +48,13 @@ def generateOneText(infile, modelfile, reportfile):
f.writelines(lines)
f.close()
- os.waitpid(subprocess.pid, 0);
+ os.waitpid(subprocess.pid, 0)
#end processing
utils.sign_epoch(infilestatus, 'Generate')
utils.store_status(infilestatuspath, infilestatus)
+
#Note: should check the corpus file size, and skip the too small text file.
def handleOneIndex(indexpath, subdir, indexname):
print(indexpath, subdir, indexname)
@@ -111,8 +113,8 @@ def handleOneIndex(indexpath, subdir, indexname):
textnum = nexttextnum
modelnum += 1
modeldir = os.path.join(config.getModelDir(), subdir, indexname)
- modelfile = os.path.join(modeldir, \
- config.getCandidateModelName(modelnum))
+ modelfile = os.path.join( \
+ modeldir, config.getCandidateModelName(modelnum))
reportfile = modelfile + config.getReportPostfix()
if os.access(modelfile, os.F_OK):
os.unlink(modelfile)
@@ -129,6 +131,7 @@ def handleOneIndex(indexpath, subdir, indexname):
utils.sign_epoch(indexstatus, 'Generate')
utils.store_status(indexstatuspath, indexstatus)
+
def walkThroughIndex(path):
for root, dirs, files in os.walk(path, topdown=True, onerror=handleError):
for onefile in files:
View
@@ -77,7 +77,7 @@ def getMaximumOccursAllowed(self):
def getMaximumIncreaseRatesAllowed(self):
return 3.
-
+
def getSegmentPostfix(self):
return '.segmented'
View
@@ -5,35 +5,41 @@
config = MyConfig()
+
#Exceptions
class EpochError(Exception):
def __init__(self, value):
self.value = value
+
def __str__(self):
return repr(self.value)
#Utils
+
#File Load/Store/Length
def read_file(infile):
with open(infile, 'r') as f:
data = ''.join(f.readlines())
f.close()
return data
+
def write_file(outfile, data):
with open(outfile, 'w') as f:
f.writelines([data])
f.close()
return
+
def get_file_length(infile):
f = open(infile, 'r')
f.seek(0, whence=io.SEEK_END)
length = f.tell()
f.close()
return length
+
#JSON Load/Store
def load_status(infile):
data = '{}'
@@ -42,10 +48,12 @@ def load_status(infile):
return json.loads(data)
+
def store_status(outfile, obj):
write_file(outfile, json.dumps(obj))
return
+
def check_epoch(obj, passname):
epochname = passname + 'Epoch'
if not epochname in obj:
@@ -54,14 +62,15 @@ def check_epoch(obj, passname):
config_epoch = int(config.getEpochs()[epochname])
if object_epoch > config_epoch:
raise EpochError('Un-excepted larger epoch en-countered.\n' + \
- 'Please increace the epoch in myconfig.py\n' )
+ 'Please increace the epoch in myconfig.py\n')
if object_epoch < config_epoch:
return False
if object_epoch == config_epoch:
return True
return None
+
def sign_epoch(obj, passname):
epochname = passname + 'Epoch'
obj[epochname] = config.getEpochs()[epochname]
View
@@ -16,9 +16,11 @@
os.chdir(libpinyin_sub_dir)
#chdir done
+
def handleError(error):
sys.exit(error)
+
def segmentOneText(infile, outfile, reportfile):
infilestatuspath = infile + config.getStatusPostfix()
infilestatus = utils.load_status(infilestatuspath)
@@ -48,6 +50,7 @@ def segmentOneText(infile, outfile, reportfile):
utils.sign_epoch(infilestatus, 'Segment')
utils.store_status(infilestatuspath, infilestatus)
+
def handleOneIndex(indexpath):
indexstatuspath = indexpath + config.getStatusPostfix()
indexstatus = utils.load_status(indexstatuspath)
@@ -66,13 +69,14 @@ def handleOneIndex(indexpath):
config.getSegmentReportPostfix()
print("Processing " + title + '#' + textpath)
segmentOneText(infile, outfile, reportfile)
- print("Processed "+ title + '#' + textpath)
+ print("Processed " + title + '#' + textpath)
indexfile.close()
#end processing
utils.sign_epoch(indexstatus, 'Segment')
utils.store_status(indexstatuspath, indexstatus)
+
def walkThroughIndex(path):
for root, dirs, files in os.walk(path, topdown=True, onerror=handleError):
for onefile in files:
@@ -95,5 +99,3 @@ def walkThroughIndex(path):
print(args)
walkThroughIndex(args.indexdir)
print('done')
-
-
View
@@ -17,6 +17,7 @@
os.chdir(libpinyin_sub_dir)
#chdir done
+
def validateModel(modelfile):
#begin processing
cmdline = ['./validate_k_mixture_model', \
@@ -29,6 +30,7 @@ def validateModel(modelfile):
sys.exit('Corrupted model found when validating:' + modelfile)
#end processing
+
def exportModel(modelfile, textmodel):
#begin processing
cmdline = ['./export_k_mixture_model', \
@@ -47,6 +49,7 @@ def exportModel(modelfile, textmodel):
sys.exit('Corrupted model found when exporting:' + modelfile)
#end processing
+
def convertModel(kmm_model, inter_model):
#begin processing
cmdline = ['./k_mixture_model_to_interpolation']
@@ -66,6 +69,7 @@ def convertModel(kmm_model, inter_model):
sys.exit('Corrupted model found when converting:' + kmm_model)
#end processing
+
def mergeOneModel(mergedmodel, onemodel, score):
#validate first
validateModel(onemodel)
@@ -90,6 +94,7 @@ def mergeOneModel(mergedmodel, onemodel, score):
sys.exit('Corrupted model found when merging:' + onemodel)
#end processing
+
def mergeSomeModels(mergedmodel, sortedindexname, mergenum):
last_score = 1.
#begin processing
@@ -113,6 +118,7 @@ def mergeSomeModels(mergedmodel, sortedindexname, mergenum):
#validate merged model
validateModel(mergedmodel)
+
def pruneModel(prunedmodel, k, CDF):
#begin processing
cmdline = ['./prune_k_mixture_model', \

0 comments on commit 2575950

Please sign in to comment.