release load SVHN #422
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -39,6 +39,7 @@ | |
| import sys | ||
| import tarfile | ||
| import zipfile | ||
| import time | ||
|
|
||
| import numpy as np | ||
| import tensorflow as tf | ||
|
|
@@ -320,6 +321,106 @@ def unpickle(file): | |
| return X_train, y_train, X_test, y_test | ||
|
|
||
|
|
||
| def load_cropped_svhn(path='data', include_extra=True): | ||
| """Load Cropped SVHN. | ||
|
|
||
| The Cropped Street View House Numbers (SVHN) Dataset contains 32x32x3 RGB images. | ||
| Digit '1' has label 1, '9' has label 9 and '0' has label 0 (the original dataset uses 10 to represent '0'), see `ufldl website <http://ufldl.stanford.edu/housenumbers/>`__. | ||
|
|
||
| Parameters | ||
| ---------- | ||
| path : str | ||
| The path that the data is downloaded to. | ||
| include_extra : boolean | ||
| If True (default), add extra images to the training set. | ||
|
|
||
| Returns | ||
| ------- | ||
| X_train, y_train, X_test, y_test: tuple | ||
| Return splitted training/test set respectively. | ||
|
|
||
| Examples | ||
| --------- | ||
| >>> X_train, y_train, X_test, y_test = tl.files.load_cropped_svhn(include_extra=False) | ||
| >>> tl.vis.save_images(X_train[0:100], [10, 10], 'svhn.png') | ||
|
|
||
| """ | ||
|
|
||
| import scipy.io | ||
|
|
||
| start_time = time.time() | ||
|
|
||
| path = os.path.join(path, 'cropped_svhn') | ||
| logging.info("Load or Download Cropped SVHN > {} | include extra images: {}".format(path, include_extra)) | ||
| url = "http://ufldl.stanford.edu/housenumbers/" | ||
|
|
||
| np_file = os.path.join(path, "train_32x32.npz") | ||
| if file_exists(np_file) is False: | ||
| filename = "train_32x32.mat" | ||
| filepath = maybe_download_and_extract(filename, path, url) | ||
| mat = scipy.io.loadmat(filepath) | ||
| X_train = mat['X'] / 255.0 # to [0, 1] | ||
| X_train = np.transpose(X_train, (3, 0, 1, 2)) | ||
| y_train = np.squeeze(mat['y'], axis=1) | ||
| y_train[y_train == 10] = 0 # replace 10 to 0 | ||
| np.savez(np_file, X=X_train, y=y_train) | ||
| del_file(filepath) | ||
| else: | ||
| v = np.load(np_file) | ||
| X_train = v['X'] | ||
| y_train = v['y'] | ||
| logging.info(" n_train: {}".format(len(y_train))) | ||
|
|
||
| np_file = os.path.join(path, "test_32x32.npz") | ||
| if file_exists(np_file) is False: | ||
| filename = "test_32x32.mat" | ||
| filepath = maybe_download_and_extract(filename, path, url) | ||
| mat = scipy.io.loadmat(filepath) | ||
| X_test = mat['X'] / 255.0 | ||
| X_test = np.transpose(X_test, (3, 0, 1, 2)) | ||
| y_test = np.squeeze(mat['y'], axis=1) | ||
| y_test[y_test == 10] = 0 | ||
| np.savez(np_file, X=X_test, y=y_test) | ||
| del_file(filepath) | ||
| else: | ||
| v = np.load(np_file) | ||
| X_test = v['X'] | ||
| y_test = v['y'] | ||
| logging.info(" n_test: {}".format(len(y_test))) | ||
|
|
||
| if include_extra: | ||
| logging.info(" getting extra 531131 images, please wait ...") | ||
| np_file = os.path.join(path, "extra_32x32.npz") | ||
| if file_exists(np_file) is False: | ||
| logging.info(" the first time to load extra images will take long time to convert the file format ...") | ||
| filename = "extra_32x32.mat" | ||
| filepath = maybe_download_and_extract(filename, path, url) | ||
| mat = scipy.io.loadmat(filepath) | ||
| X_extra = mat['X'] / 255.0 | ||
| X_extra = np.transpose(X_extra, (3, 0, 1, 2)) | ||
| y_extra = np.squeeze(mat['y'], axis=1) | ||
| y_extra[y_extra == 10] = 0 | ||
| np.savez(np_file, X=X_extra, y=y_extra) | ||
| del_file(filepath) | ||
| else: | ||
| v = np.load(np_file) | ||
| X_extra = v['X'] | ||
| y_extra = v['y'] | ||
| # print(X_train.shape, X_extra.shape) | ||
| logging.info(" adding n_extra {} to n_train {}".format(len(y_extra), len(y_train))) | ||
| t = time.time() | ||
| X_train = np.concatenate((X_train, X_extra), 0) | ||
| y_train = np.concatenate((y_train, y_extra), 0) | ||
| # X_train = np.append(X_train, X_extra, axis=0) | ||
| # y_train = np.append(y_train, y_extra, axis=0) | ||
| logging.info(" added n_extra {} to n_train {} took {}s".format(len(y_extra), len(y_train), time.time() - t)) | ||
| else: | ||
| logging.info(" no extra images are included") | ||
| logging.info(" image size:%s n_train:%d n_test:%d" % (str(X_train.shape[1:4]), len(y_train), len(y_test))) | ||
| logging.info(" took: {}s".format(int(time.time() - start_time))) | ||
| return X_train, y_train, X_test, y_test | ||
|
|
||
|
|
||
| def load_ptb_dataset(path='data'): | ||
| """Load Penn TreeBank (PTB) dataset. | ||
|
|
||
|
|
@@ -656,19 +757,19 @@ def load_flickr25k_dataset(tag='sky', path="data", n_threads=50, printable=False | |
| url = 'http://press.liacs.nl/mirflickr/mirflickr25k/' | ||
|
|
||
| # download dataset | ||
| if folder_exists(os.path.join(path, "mirflickr")) is False: | ||
| logging.info("[*] Flickr25k is nonexistent in {}".format(path)) | ||
| maybe_download_and_extract(filename, path, url, extract=True) | ||
| del_file(os.path.join(path, filename)) | ||
|
|
||
| # return images by the given tag. | ||
| # 1. image path list | ||
| folder_imgs = os.path.join(path, "mirflickr") | ||
| path_imgs = load_file_list(path=folder_imgs, regx='\\.jpg', printable=False) | ||
| path_imgs.sort(key=natural_keys) | ||
|
|
||
| # 2. tag path list | ||
| folder_tags = os.path.join(path, "mirflickr", "meta", "tags") | ||
| path_tags = load_file_list(path=folder_tags, regx='\\.txt', printable=False) | ||
| path_tags.sort(key=natural_keys) | ||
|
|
||
|
|
@@ -679,7 +780,7 @@ def load_flickr25k_dataset(tag='sky', path="data", n_threads=50, printable=False | |
| logging.info("[Flickr25k] reading images with tag: {}".format(tag)) | ||
| images_list = [] | ||
| for idx, _v in enumerate(path_tags): | ||
| tags = read_file(os.path.join(folder_tags, path_tags[idx])).split('\n') | ||
| # logging.info(idx+1, tags) | ||
| if tag is None or tag in tags: | ||
| images_list.append(path_imgs[idx]) | ||
|
|
@@ -722,6 +823,8 @@ def load_flickr1M_dataset(tag='sky', size=10, path="data", n_threads=50, printab | |
| >>> images = tl.files.load_flickr1M_dataset(tag='zebra') | ||
|
|
||
| """ | ||
| import shutil | ||
|
There was a problem hiding this comment. we should almost always put |
||
|
|
||
| path = os.path.join(path, 'flickr1M') | ||
| logging.info("[Flickr1M] using {}% of images = {}".format(size * 10, size * 100000)) | ||
| images_zip = [ | ||
|
|
@@ -734,20 +837,21 @@ def load_flickr1M_dataset(tag='sky', size=10, path="data", n_threads=50, printab | |
| for image_zip in images_zip[0:size]: | ||
| image_folder = image_zip.split(".")[0] | ||
| # logging.info(path+"/"+image_folder) | ||
| if folder_exists(os.path.join(path, image_folder)) is False: | ||
| # logging.info(image_zip) | ||
| logging.info("[Flickr1M] {} is missing in {}".format(image_folder, path)) | ||
| maybe_download_and_extract(image_zip, path, url, extract=True) | ||
| del_file(os.path.join(path, image_zip)) | ||
| # os.system("mv {} {}".format(os.path.join(path, 'images'), os.path.join(path, image_folder))) | ||
| shutil.move(os.path.join(path, 'images'), os.path.join(path, image_folder)) | ||
| else: | ||
| logging.info("[Flickr1M] {} exists in {}".format(image_folder, path)) | ||
|
|
||
| # download tag | ||
| if folder_exists(os.path.join(path, "tags")) is False: | ||
| logging.info("[Flickr1M] tag files is nonexistent in {}".format(path)) | ||
| maybe_download_and_extract(tag_zip, path, url, extract=True) | ||
| del_file(os.path.join(path, tag_zip)) | ||
| else: | ||
| logging.info("[Flickr1M] tags exists in {}".format(path)) | ||
|
|
||
|
|
@@ -761,17 +865,19 @@ def load_flickr1M_dataset(tag='sky', size=10, path="data", n_threads=50, printab | |
| for folder in images_folder_list[0:size * 10]: | ||
| tmp = load_file_list(path=folder, regx='\\.jpg', printable=False) | ||
| tmp.sort(key=lambda s: int(s.split('.')[-2])) # ddd.jpg | ||
| images_list.extend([os.path.join(folder, x) for x in tmp]) | ||
|
|
||
| # 2. tag path list | ||
| tag_list = [] | ||
| tag_folder_list = load_folder_list(os.path.join(path, "tags")) | ||
|
|
||
| # tag_folder_list.sort(key=lambda s: int(s.split("/")[-1])) # folder/images/ddd | ||
| tag_folder_list.sort(key=lambda s: int(os.path.basename(s))) | ||
|
|
||
| for folder in tag_folder_list[0:size * 10]: | ||
| tmp = load_file_list(path=folder, regx='\\.txt', printable=False) | ||
| tmp.sort(key=lambda s: int(s.split('.')[-2])) # ddd.txt | ||
| tmp = [os.path.join(folder, s) for s in tmp] | ||
| tag_list += tmp | ||
|
|
||
| # 3. select images | ||
|
|
||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
another
import timeon #L1130 could be removed now.