Permalink
Browse files

ask what to do when data path or model file already exists

  • Loading branch information...
ottokart committed Apr 10, 2017
1 parent 3f7c794 commit 3b37a3e04de19f69560dece7fce589dc745050f8
Showing with 46 additions and 6 deletions.
  1. +19 −4 data.py
  2. +14 −0 main.py
  3. +13 −2 main2.py
View
23 data.py
@@ -8,6 +8,7 @@
import cPickle
import codecs
import fnmatch
import shutil
DATA_PATH = "../data"
@@ -256,11 +257,25 @@ def create_dev_test_train_split_and_vocabulary(root_path, create_vocabulary, tra
else:
sys.exit("The path to stage1 source data directory with txt files is missing")
if not os.path.exists(DATA_PATH):
os.makedirs(DATA_PATH)
else:
sys.exit("Data already exists")
replace = False
if os.path.exists(DATA_PATH):
while True:
resp = raw_input("Data path '%s' already exists. Do you want to:\n[r]eplace the files in existing data path?\n[e]xit?\n>" % DATA_PATH)
resp = resp.lower().strip()
if resp not in ('r', 'e'):
continue
if resp == 'e':
sys.exit()
elif resp == 'r':
replace = True
break
if replace and os.path.exists(DATA_PATH):
shutil.rmtree(DATA_PATH)
os.makedirs(DATA_PATH)
create_dev_test_train_split_and_vocabulary(path, True, TRAIN_FILE, DEV_FILE, TEST_FILE, PRETRAINED_EMBEDDINGS_PATH)
# Stage 2
View
14 main.py
@@ -97,7 +97,21 @@ def get_minibatch(file_name, batch_size, shuffle, with_pauses=False):
y = T.imatrix('y')
lr = T.scalar('lr')
continue_with_previous = False
if os.path.isfile(model_file_name):
while True:
resp = raw_input("Found an existing model with the name %s. Do you want to:\n[c]ontinue training the existing model?\n[r]eplace the existing model and train a new one?\n[e]xit?\n>" % model_file_name)
resp = resp.lower().strip()
if resp not in ('c', 'r', 'e'):
continue
if resp == 'e':
sys.exit()
elif resp == 'c':
continue_with_previous = True
break
if continue_with_previous:
print "Loading previous model state"
net, state = models.load(model_file_name, MINIBATCH_SIZE, x)
View
@@ -60,13 +60,24 @@
p = T.matrix('p')
lr = T.scalar('lr')
continue_with_previous = False
if os.path.isfile(model_file_name):
print "Loading previous model state"
while True:
resp = raw_input("Found an existing model with the name %s. Do you want to:\n[c]ontinue training the existing model?\n[r]eplace the existing model and train a new one?\n[e]xit?\n>" % model_file_name)
resp = resp.lower().strip()
if resp not in ('c', 'r', 'e'):
continue
if resp == 'e':
sys.exit()
elif resp == 'c':
continue_with_previous = True
break
if continue_with_previous:
net, state = models.load(model_file_name, MINIBATCH_SIZE, x, p)
gsums, learning_rate, validation_ppl_history, starting_epoch, rng = state
best_ppl = min(validation_ppl_history)
else:
rng = np.random
rng.seed(1)

0 comments on commit 3b37a3e

Please sign in to comment.