From 67fd7ceb6fd4bde65d5a05f71465b2d6649058b0 Mon Sep 17 00:00:00 2001 From: Qiusheng Wu Date: Wed, 11 Apr 2018 13:49:10 -0400 Subject: [PATCH 1/4] fix image and tile format inconsistency When a local GeoTIFF or a remote Cloud Optimized GeoTIFF is provided, the image tiles are generated in jpg format. Therefore, the `label-maker package` should look for image tiles in jpg format rather than tif format. --- label_maker/package.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/label_maker/package.py b/label_maker/package.py index ecfa085..ba3e40f 100644 --- a/label_maker/package.py +++ b/label_maker/package.py @@ -58,6 +58,8 @@ def package_directory(dest_folder, classes, imagery, ml_type, seed=False, train_ # open the images and load those plus the labels into the final arrays o = urlparse(imagery) _, image_format = op.splitext(o.path) + if image_format == '.tif': # if a local GeoTIFF or a remote Cloud Optimized GeoTIFF is provided, use jpg as tile format + image_format = '.jpg' for tile in tiles: image_file = op.join(dest_folder, 'tiles', '{}{}'.format(tile, image_format)) try: From 0b17f9ab5b1e38a87b75a834af30a36dbb760f2f Mon Sep 17 00:00:00 2001 From: Qiusheng Wu Date: Thu, 12 Apr 2018 00:16:10 -0400 Subject: [PATCH 2/4] add unit testing for local geotiff package --- .gitignore | 1 + .../config.intergration.geotiff_package.json | 12 +++++++ .../geotiff/geotiff-download-url.txt | 1 + test/integration/test_geotiff_package.py | 35 +++++++++++++++++++ 4 files changed, 49 insertions(+) create mode 100644 test/fixtures/integration/config.intergration.geotiff_package.json create mode 100644 test/fixtures/integration/geotiff/geotiff-download-url.txt create mode 100644 test/integration/test_geotiff_package.py diff --git a/.gitignore b/.gitignore index 63815e9..8a4ec3d 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,4 @@ data config.json stdout* /integration* +.idea/ diff --git a/test/fixtures/integration/config.intergration.geotiff_package.json b/test/fixtures/integration/config.intergration.geotiff_package.json new file mode 100644 index 0000000..fbb4178 --- /dev/null +++ b/test/fixtures/integration/config.intergration.geotiff_package.json @@ -0,0 +1,12 @@ +{ + "country": "sao_tome_and_principe", + "bounding_box": [6.7184,0.3367,6.7378,0.3590], + "zoom": 19, + "classes": [ + { "name": "Roads", "filter": ["has", "highway"] }, + { "name": "Buildings", "filter": ["has", "building"] } + ], + "imagery": "geotiff/3041615b-2bdb-40c5-b834-36f580baca29.tif", + "background_ratio": 1, + "ml_type": "classification" +} diff --git a/test/fixtures/integration/geotiff/geotiff-download-url.txt b/test/fixtures/integration/geotiff/geotiff-download-url.txt new file mode 100644 index 0000000..2a4570a --- /dev/null +++ b/test/fixtures/integration/geotiff/geotiff-download-url.txt @@ -0,0 +1 @@ +http://oin-hotosm.s3.amazonaws.com/593ede5ee407d70011386139/0/3041615b-2bdb-40c5-b834-36f580baca29.tif diff --git a/test/integration/test_geotiff_package.py b/test/integration/test_geotiff_package.py new file mode 100644 index 0000000..d5bcc7b --- /dev/null +++ b/test/integration/test_geotiff_package.py @@ -0,0 +1,35 @@ +"""Test that the following CLI command returns the expected outputs +label-maker package -d integration-od/sao_tome -c test/fixtures/integration/config.intergration.geotiff_package.json""" +import unittest +from os import makedirs +from shutil import copytree, rmtree +import subprocess + +import numpy as np + +class TestObjectDetectionPackage(unittest.TestCase): + """Tests for local GeoTIFF package creation""" + @classmethod + def setUpClass(cls): + makedirs('integration-od') + copytree('test/fixtures/integration/geotiff', 'integration-od/geotiff') + copytree('test/fixtures/integration/sao_tome', 'integration-od/sao_tome') + + @classmethod + def tearDownClass(cls): + rmtree('integration-od') + + def test_cli(self): + """Verify data.npz produced by CLI""" + cmd = 'label-maker package -d integration-od/sao_tome -c test/fixtures/integration/config.intergration.geotiff_package.json' + cmd = cmd.split(' ') + subprocess.run(cmd, universal_newlines=True) + + data = np.load('integration-od/sao_tome/data.npz') + + self.assertEqual(data['x_train'].shape, (332, 256, 256, 3)) + self.assertEqual(data['x_test'].shape, (83, 256, 256, 3)) + + # validate our label data with exact matches in shape + self.assertEqual(data['y_train'].shape, (332, 3)) + self.assertEqual(data['y_test'].shape, (83, 3)) From 621461ea0d12128318789fa6c9654a57030f6f48 Mon Sep 17 00:00:00 2001 From: Qiusheng Wu Date: Mon, 16 Apr 2018 22:34:21 -0400 Subject: [PATCH 3/4] fix test example for packaging local geotiff --- .../config.intergration.geotiff_package.json | 6 ++-- .../geotiff/geotiff-download-url.txt | 1 - test/fixtures/integration/labels-tif.npz | Bin 0 -> 902 bytes test/integration/test_geotiff_package.py | 28 ++++++++++-------- 4 files changed, 19 insertions(+), 16 deletions(-) delete mode 100644 test/fixtures/integration/geotiff/geotiff-download-url.txt create mode 100644 test/fixtures/integration/labels-tif.npz diff --git a/test/fixtures/integration/config.intergration.geotiff_package.json b/test/fixtures/integration/config.intergration.geotiff_package.json index fbb4178..aa89309 100644 --- a/test/fixtures/integration/config.intergration.geotiff_package.json +++ b/test/fixtures/integration/config.intergration.geotiff_package.json @@ -1,12 +1,12 @@ { "country": "sao_tome_and_principe", - "bounding_box": [6.7184,0.3367,6.7378,0.3590], - "zoom": 19, + "bounding_box": [6.72746119738703,0.3382909054246151,6.72776258384623,0.33878086940393715], + "zoom": 20, "classes": [ { "name": "Roads", "filter": ["has", "highway"] }, { "name": "Buildings", "filter": ["has", "building"] } ], - "imagery": "geotiff/3041615b-2bdb-40c5-b834-36f580baca29.tif", + "imagery": "../drone.tif", "background_ratio": 1, "ml_type": "classification" } diff --git a/test/fixtures/integration/geotiff/geotiff-download-url.txt b/test/fixtures/integration/geotiff/geotiff-download-url.txt deleted file mode 100644 index 2a4570a..0000000 --- a/test/fixtures/integration/geotiff/geotiff-download-url.txt +++ /dev/null @@ -1 +0,0 @@ -http://oin-hotosm.s3.amazonaws.com/593ede5ee407d70011386139/0/3041615b-2bdb-40c5-b834-36f580baca29.tif diff --git a/test/fixtures/integration/labels-tif.npz b/test/fixtures/integration/labels-tif.npz new file mode 100644 index 0000000000000000000000000000000000000000..95e41cfdb7cce94b80c5993e0c786e7431abd1cb GIT binary patch literal 902 zcmWIWW@Zs#00FkO6MT9O1s=%&vOyRmYHDI^VPULmYGiC|V5DngpqE!rIoU7NHz1Oc z!HuC>Jteg`xk%kgLER?PLS08eJuSbeq$n{jKEEg>6(sJKm{Xhz6fe$5EJy|NHH>vM zbrfn9kN_9Z8Zcl)qXUR`nc;xCOz6;W4va7}frJ^oTm}s~1|q@?>N5JrCL@ywGwzfI z^TGc{5DQzn3-Bh^gjr}Nz+wU21oW6gxWNRN7{P8pi$`=5(DMtzgmY*nz`{Ggo0Scu OiUkNcfNC Date: Mon, 16 Apr 2018 22:48:12 -0400 Subject: [PATCH 4/4] fix local geotiff path --- .../integration/config.intergration.geotiff_package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/fixtures/integration/config.intergration.geotiff_package.json b/test/fixtures/integration/config.intergration.geotiff_package.json index aa89309..dd3cd1c 100644 --- a/test/fixtures/integration/config.intergration.geotiff_package.json +++ b/test/fixtures/integration/config.intergration.geotiff_package.json @@ -6,7 +6,7 @@ { "name": "Roads", "filter": ["has", "highway"] }, { "name": "Buildings", "filter": ["has", "building"] } ], - "imagery": "../drone.tif", + "imagery": "integration-tif/drone.tif", "background_ratio": 1, "ml_type": "classification" }