From 338dd13542387387028b0f3adbfc296756734d5a Mon Sep 17 00:00:00 2001 From: wanghaoshuang Date: Mon, 10 Jul 2017 11:56:57 +0800 Subject: [PATCH 1/6] Add voc2012 dataset for image segment --- python/paddle/v2/dataset/__init__.py | 5 +- python/paddle/v2/dataset/tests/vocseg_test.py | 42 +++++++++++ python/paddle/v2/dataset/voc_seg.py | 74 +++++++++++++++++++ 3 files changed, 120 insertions(+), 1 deletion(-) create mode 100644 python/paddle/v2/dataset/tests/vocseg_test.py create mode 100644 python/paddle/v2/dataset/voc_seg.py diff --git a/python/paddle/v2/dataset/__init__.py b/python/paddle/v2/dataset/__init__.py index 80ff6295c34e8..cdd85cce37188 100644 --- a/python/paddle/v2/dataset/__init__.py +++ b/python/paddle/v2/dataset/__init__.py @@ -24,8 +24,11 @@ import uci_housing import sentiment import wmt14 +import mq2007 +import flowers +import voc_seg __all__ = [ 'mnist', 'imikolov', 'imdb', 'cifar', 'movielens', 'conll05', 'sentiment' - 'uci_housing', 'wmt14' + 'uci_housing', 'wmt14', 'mq2007', 'flowers', 'voc_seg' ] diff --git a/python/paddle/v2/dataset/tests/vocseg_test.py b/python/paddle/v2/dataset/tests/vocseg_test.py new file mode 100644 index 0000000000000..1a773fa18b2a7 --- /dev/null +++ b/python/paddle/v2/dataset/tests/vocseg_test.py @@ -0,0 +1,42 @@ +# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import paddle.v2.dataset.voc_seg +import unittest + + +class TestVOC(unittest.TestCase): + def check_reader(self, reader): + sum = 0 + label = 0 + for l in reader(): + self.assertEqual(l[0].size, l[1].size) + sum += 1 + return sum + + def test_train(self): + count = self.check_reader(paddle.v2.dataset.voc_seg.train()) + self.assertEqual(count, 2913) + + def test_test(self): + count = self.check_reader(paddle.v2.dataset.voc_seg.test()) + self.assertEqual(count, 1464) + + def test_val(self): + count = self.check_reader(paddle.v2.dataset.voc_seg.val()) + self.assertEqual(count, 1449) + + +if __name__ == '__main__': + unittest.main() diff --git a/python/paddle/v2/dataset/voc_seg.py b/python/paddle/v2/dataset/voc_seg.py new file mode 100644 index 0000000000000..9b79f726d235b --- /dev/null +++ b/python/paddle/v2/dataset/voc_seg.py @@ -0,0 +1,74 @@ +# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Image dataset for segmentation. +The 2012 dataset contains images from 2008-2011 for which additional segmentations have been prepared. As in previous years the assignment to training/test sets has been maintained. The total number of images with segmentation has been increased from 7,062 to 9,993. +""" + +import tarfile +import numpy as np +from common import download +from paddle.v2.image import * + +__all__ = ['train', 'test', 'val'] + +VOC_URL = 'http://host.robots.ox.ac.uk/pascal/VOC/voc2012/VOCtrainval_11-May-2012.tar' +VOC_MD5 = '6cd6e144f989b92b3379bac3b3de84fd' +SET_FILE = 'VOCdevkit/VOC2012/ImageSets/Segmentation/{}.txt' +DATA_FILE = 'VOCdevkit/VOC2012/JPEGImages/{}.jpg' +LABEL_FILE = 'VOCdevkit/VOC2012/SegmentationClass/{}.png' + + +def reader_creator(filename, sub_name): + + tarobject = tarfile.open(filename) + name2mem = {} + for ele in tarobject.getmembers(): + name2mem[ele.name] = ele + + def reader(): + set_file = SET_FILE.format(sub_name) + sets = tarobject.extractfile(name2mem[set_file]) + for line in sets: + line = line.strip() + data_file = DATA_FILE.format(line) + label_file = LABEL_FILE.format(line) + data = tarobject.extractfile(name2mem[data_file]).read() + label = tarobject.extractfile(name2mem[label_file]).read() + data = load_image_bytes(data) + label = load_image_bytes(label) + yield data, label + + return reader + + +def train(): + """ + Create a train dataset reader containing 2913 images. + """ + return reader_creator(download(VOC_URL, 'voc_seg', VOC_MD5), 'trainval') + + +def test(): + """ + Create a test dataset reader containing 1464 images. + """ + return reader_creator(download(VOC_URL, 'voc_seg', VOC_MD5), 'train') + + +def val(): + """ + Create a val dataset reader containing 1449 images. + """ + return reader_creator(download(VOC_URL, 'voc_seg', VOC_MD5), 'val') From c4f301ded74b1f4c0dd1526a76ece9e8e26d2048 Mon Sep 17 00:00:00 2001 From: wanghaoshuang Date: Wed, 12 Jul 2017 10:34:59 +0800 Subject: [PATCH 2/6] Modify comments and fix code format. --- python/paddle/v2/dataset/voc_seg.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/python/paddle/v2/dataset/voc_seg.py b/python/paddle/v2/dataset/voc_seg.py index 9b79f726d235b..595ab41cd80f4 100644 --- a/python/paddle/v2/dataset/voc_seg.py +++ b/python/paddle/v2/dataset/voc_seg.py @@ -13,7 +13,10 @@ # limitations under the License. """ Image dataset for segmentation. -The 2012 dataset contains images from 2008-2011 for which additional segmentations have been prepared. As in previous years the assignment to training/test sets has been maintained. The total number of images with segmentation has been increased from 7,062 to 9,993. +The 2012 dataset contains images from 2008-2011 for which additional +segmentations have been prepared. As in previous years the assignment +to training/test sets has been maintained. The total number of images +with segmentation has been increased from 7,062 to 9,993. """ import tarfile @@ -23,7 +26,9 @@ __all__ = ['train', 'test', 'val'] -VOC_URL = 'http://host.robots.ox.ac.uk/pascal/VOC/voc2012/VOCtrainval_11-May-2012.tar' +VOC_URL = 'http://host.robots.ox.ac.uk/pascal/VOC/voc2012/\ + VOCtrainval_11-May-2012.tar' + VOC_MD5 = '6cd6e144f989b92b3379bac3b3de84fd' SET_FILE = 'VOCdevkit/VOC2012/ImageSets/Segmentation/{}.txt' DATA_FILE = 'VOCdevkit/VOC2012/JPEGImages/{}.jpg' @@ -55,20 +60,20 @@ def reader(): def train(): """ - Create a train dataset reader containing 2913 images. + Create a train dataset reader containing 2913 images in HWC order. """ return reader_creator(download(VOC_URL, 'voc_seg', VOC_MD5), 'trainval') def test(): """ - Create a test dataset reader containing 1464 images. + Create a test dataset reader containing 1464 images in HWC order. """ return reader_creator(download(VOC_URL, 'voc_seg', VOC_MD5), 'train') def val(): """ - Create a val dataset reader containing 1449 images. + Create a val dataset reader containing 1449 images in HWC order. """ return reader_creator(download(VOC_URL, 'voc_seg', VOC_MD5), 'val') From 1ba879beadc33e84ff3d5c62ee5ac188027d7638 Mon Sep 17 00:00:00 2001 From: wanghaoshuang Date: Wed, 12 Jul 2017 14:38:03 +0800 Subject: [PATCH 3/6] Use PIL to read image in palette mode --- python/paddle/v2/dataset/tests/vocseg_test.py | 2 +- python/paddle/v2/dataset/voc_seg.py | 10 +++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/python/paddle/v2/dataset/tests/vocseg_test.py b/python/paddle/v2/dataset/tests/vocseg_test.py index 1a773fa18b2a7..8217ff45b13e2 100644 --- a/python/paddle/v2/dataset/tests/vocseg_test.py +++ b/python/paddle/v2/dataset/tests/vocseg_test.py @@ -21,7 +21,7 @@ def check_reader(self, reader): sum = 0 label = 0 for l in reader(): - self.assertEqual(l[0].size, l[1].size) + self.assertEqual(l[0].size, 3 * l[1].size) sum += 1 return sum diff --git a/python/paddle/v2/dataset/voc_seg.py b/python/paddle/v2/dataset/voc_seg.py index 595ab41cd80f4..0df4423ff0489 100644 --- a/python/paddle/v2/dataset/voc_seg.py +++ b/python/paddle/v2/dataset/voc_seg.py @@ -20,14 +20,16 @@ """ import tarfile +import io import numpy as np from common import download from paddle.v2.image import * +from PIL import Image __all__ = ['train', 'test', 'val'] VOC_URL = 'http://host.robots.ox.ac.uk/pascal/VOC/voc2012/\ - VOCtrainval_11-May-2012.tar' +VOCtrainval_11-May-2012.tar' VOC_MD5 = '6cd6e144f989b92b3379bac3b3de84fd' SET_FILE = 'VOCdevkit/VOC2012/ImageSets/Segmentation/{}.txt' @@ -51,8 +53,10 @@ def reader(): label_file = LABEL_FILE.format(line) data = tarobject.extractfile(name2mem[data_file]).read() label = tarobject.extractfile(name2mem[label_file]).read() - data = load_image_bytes(data) - label = load_image_bytes(label) + data = Image.open(io.BytesIO(data)) + label = Image.open(io.BytesIO(label)) + data = np.array(data) + label = np.array(label) yield data, label return reader From 4a5c3714eaec33628259dd3c481f3d36597e0c58 Mon Sep 17 00:00:00 2001 From: wanghaoshuang Date: Thu, 13 Jul 2017 15:10:25 +0800 Subject: [PATCH 4/6] fix python dependency for voc2012 dataset --- python/paddle/v2/dataset/voc_seg.py | 10 ++++++---- python/setup.py.in | 3 ++- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/python/paddle/v2/dataset/voc_seg.py b/python/paddle/v2/dataset/voc_seg.py index 0df4423ff0489..617e212d67fbe 100644 --- a/python/paddle/v2/dataset/voc_seg.py +++ b/python/paddle/v2/dataset/voc_seg.py @@ -22,7 +22,7 @@ import tarfile import io import numpy as np -from common import download +from paddle.v2.dataset.common import download from paddle.v2.image import * from PIL import Image @@ -36,6 +36,8 @@ DATA_FILE = 'VOCdevkit/VOC2012/JPEGImages/{}.jpg' LABEL_FILE = 'VOCdevkit/VOC2012/SegmentationClass/{}.png' +CACHE_DIR = 'voc2012' + def reader_creator(filename, sub_name): @@ -66,18 +68,18 @@ def train(): """ Create a train dataset reader containing 2913 images in HWC order. """ - return reader_creator(download(VOC_URL, 'voc_seg', VOC_MD5), 'trainval') + return reader_creator(download(VOC_URL, CACHE_DIR, VOC_MD5), 'trainval') def test(): """ Create a test dataset reader containing 1464 images in HWC order. """ - return reader_creator(download(VOC_URL, 'voc_seg', VOC_MD5), 'train') + return reader_creator(download(VOC_URL, CACHE_DIR, VOC_MD5), 'train') def val(): """ Create a val dataset reader containing 1449 images in HWC order. """ - return reader_creator(download(VOC_URL, 'voc_seg', VOC_MD5), 'val') + return reader_creator(download(VOC_URL, CACHE_DIR, VOC_MD5), 'val') diff --git a/python/setup.py.in b/python/setup.py.in index 271ee6e552698..310ac403a98fb 100644 --- a/python/setup.py.in +++ b/python/setup.py.in @@ -19,7 +19,8 @@ setup_requires=["requests", "recordio", "matplotlib", "rarfile", - "scipy>=0.19.0"] + "scipy>=0.19.0", + "Pillow"] if '${CMAKE_SYSTEM_PROCESSOR}' not in ['arm', 'armv7-a', 'aarch64']: setup_requires+=["opencv-python"] From 302c4f11d164311d6352d39e162d4b79bac6459e Mon Sep 17 00:00:00 2001 From: wanghaoshuang Date: Tue, 18 Jul 2017 10:48:23 +0800 Subject: [PATCH 5/6] rename voc_seg to voc2012 --- python/paddle/v2/dataset/__init__.py | 2 +- .../paddle/v2/dataset/tests/{vocseg_test.py => voc2012_test.py} | 2 +- python/paddle/v2/dataset/{voc_seg.py => voc2012.py} | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename python/paddle/v2/dataset/tests/{vocseg_test.py => voc2012_test.py} (97%) rename python/paddle/v2/dataset/{voc_seg.py => voc2012.py} (100%) diff --git a/python/paddle/v2/dataset/__init__.py b/python/paddle/v2/dataset/__init__.py index cdd85cce37188..f99116b0129da 100644 --- a/python/paddle/v2/dataset/__init__.py +++ b/python/paddle/v2/dataset/__init__.py @@ -30,5 +30,5 @@ __all__ = [ 'mnist', 'imikolov', 'imdb', 'cifar', 'movielens', 'conll05', 'sentiment' - 'uci_housing', 'wmt14', 'mq2007', 'flowers', 'voc_seg' + 'uci_housing', 'wmt14', 'mq2007', 'flowers', 'voc2012' ] diff --git a/python/paddle/v2/dataset/tests/vocseg_test.py b/python/paddle/v2/dataset/tests/voc2012_test.py similarity index 97% rename from python/paddle/v2/dataset/tests/vocseg_test.py rename to python/paddle/v2/dataset/tests/voc2012_test.py index 8217ff45b13e2..31e72ebf5eac0 100644 --- a/python/paddle/v2/dataset/tests/vocseg_test.py +++ b/python/paddle/v2/dataset/tests/voc2012_test.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import paddle.v2.dataset.voc_seg +import paddle.v2.dataset.voc2012 import unittest diff --git a/python/paddle/v2/dataset/voc_seg.py b/python/paddle/v2/dataset/voc2012.py similarity index 100% rename from python/paddle/v2/dataset/voc_seg.py rename to python/paddle/v2/dataset/voc2012.py From ceb9a73aaad48f063ae4dcccf4aafb0ce0a3f709 Mon Sep 17 00:00:00 2001 From: wanghaoshuang Date: Tue, 18 Jul 2017 11:02:53 +0800 Subject: [PATCH 6/6] fix import err --- python/paddle/v2/dataset/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/paddle/v2/dataset/__init__.py b/python/paddle/v2/dataset/__init__.py index f99116b0129da..90830515c1e8e 100644 --- a/python/paddle/v2/dataset/__init__.py +++ b/python/paddle/v2/dataset/__init__.py @@ -26,7 +26,7 @@ import wmt14 import mq2007 import flowers -import voc_seg +import voc2012 __all__ = [ 'mnist', 'imikolov', 'imdb', 'cifar', 'movielens', 'conll05', 'sentiment'