use the not compressed version of imdb. This take 1s to load instead …

…of 45s
lisa-lab · Jan 9, 2015 · 1d12bee · 1d12bee
1 parent 6b7d587
commit 1d12bee
Show file tree

Hide file tree

Showing 3 changed files with 9 additions and 5 deletions.
diff --git a/code/imdb.py b/code/imdb.py
@@ -77,7 +77,7 @@ def get_dataset_file(dataset, default_dataset, origin):
     return dataset
 
 
-def load_data(path="imdb.pkl.gz", n_words=100000, valid_portion=0.1):
+def load_data(path="imdb.pkl", n_words=100000, valid_portion=0.1):
     ''' Loads the dataset
 
     :type dataset: string
@@ -91,9 +91,13 @@ def load_data(path="imdb.pkl.gz", n_words=100000, valid_portion=0.1):
     # Load the dataset
     path = get_dataset_file(
         path, "imdb.pkl.gz",
-        "http://www.iro.umontreal.ca/~lisa/deep/data/imdb.pkl.gz")
+        "http://www.iro.umontreal.ca/~lisa/deep/data/imdb.pkl")
+
+    if path.endswith(".gz"):
+        f = gzip.open(path, 'rb')
+    else:
+        f = open(path, 'rb')
 
-    f = gzip.open(path, 'rb')
     train_set = cPickle.load(f)
     test_set = cPickle.load(f)
     f.close()

diff --git a/code/lstm.py b/code/lstm.py
@@ -471,7 +471,7 @@ def train(dim_proj=100,
                 else:
                     params = unzip(tparams)
                 numpy.savez(saveto, history_errs=history_errs, **params)
-                pkl.dump(model_options, open('%s.pkl' % saveto, 'wb'))
+                pkl.dump(model_options, open('%s.pkl' % saveto, 'wb'), -1)
                 print 'Done'
 
             if numpy.mod(uidx, validFreq) == 0:

diff --git a/data/download.sh b/data/download.sh
@@ -15,7 +15,7 @@ fi
 
 $DL_CMD http://www.iro.umontreal.ca/~lisa/deep/data/mnist/mnist.pkl.gz
 $DL_CMD http://www.iro.umontreal.ca/~lisa/deep/data/mnist/mnist_py3k.pkl.gz
-$DL_CMD http://www.iro.umontreal.ca/~lisa/deep/data/imdb.pkl.gz
+$DL_CMD http://www.iro.umontreal.ca/~lisa/deep/data/imdb.pkl.gz && gunzip imdb.pkl.gz
 $DL_CMD http://www.iro.umontreal.ca/~lisa/deep/data/Nottingham.zip && unzip -u Nottingham.zip
 $DL_CMD http://www.iro.umontreal.ca/~lisa/deep/midi.zip && unzip -u midi.zip -d ../code && echo "extracted Modified Python MIDI package (GPL)"
 $DL_CMD http://www-etud.iro.umontreal.ca/~mesnilgr/atis/atis.fold0.pkl.gz