From 3623c0efd6c368c5dc24f5d2da1eb732bd6e0030 Mon Sep 17 00:00:00 2001 From: geoyee Date: Wed, 22 Dec 2021 15:21:10 +0800 Subject: [PATCH 1/2] Init temporary test --- .gitignore | 1 + .../utils/paddlepaddle_split_dataset_list.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 42e7cdc..0342037 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ __pycache__/ .qt_for_python/ .vscode/settings.json +data/ diff --git a/deep-learning-datasets-maker/utils/paddlepaddle_split_dataset_list.py b/deep-learning-datasets-maker/utils/paddlepaddle_split_dataset_list.py index 5471c07..3d6b2e0 100644 --- a/deep-learning-datasets-maker/utils/paddlepaddle_split_dataset_list.py +++ b/deep-learning-datasets-maker/utils/paddlepaddle_split_dataset_list.py @@ -105,7 +105,7 @@ def generate_list(args): except: line = left + '\n' - f.write(line) + f.write(line.replace("\\", "/")) # print(line) # test start = end From 93f6102d9b77192d7d5ee222be9d2aa99d40088f Mon Sep 17 00:00:00 2001 From: geoyee Date: Wed, 22 Dec 2021 16:26:49 +0800 Subject: [PATCH 2/2] Init create COCO --- deep-learning-datasets-maker/split_rs_data.py | 34 +-- .../utils/COCO/__init__.py | 4 +- .../utils/COCO/pycococreatortools/__init__.py | 2 +- .../utils/COCO/shape_to_coco.py | 21 +- .../utils/COCO/slice_dataset.py | 150 ++++++------- .../utils/COCO/tif_process.py | 202 ++++++------------ .../utils/COCO/visualize_coco.py | 46 ++-- 7 files changed, 188 insertions(+), 271 deletions(-) diff --git a/deep-learning-datasets-maker/split_rs_data.py b/deep-learning-datasets-maker/split_rs_data.py index e9c1d80..6468ce4 100644 --- a/deep-learning-datasets-maker/split_rs_data.py +++ b/deep-learning-datasets-maker/split_rs_data.py @@ -21,7 +21,6 @@ * * ***************************************************************************/ """ -from numpy import double from qgis.PyQt.QtCore import QSettings, QTranslator, QCoreApplication from qgis.PyQt.QtGui import QIcon from qgis.PyQt.QtWidgets import QAction, QFileDialog, QTabWidget @@ -44,7 +43,7 @@ import os import os.path as osp from .utils import * -from .utils.COCO import * +from .utils.COCO import clip_from_file, slice, from_mask_to_coco # import argparse @@ -379,36 +378,21 @@ def mkdir_p(path): generate_list(args) if self.dlg.checkBoxCOCO.isChecked(): - # COCO Dataset Paths dataset_COCO = osp.join(dataset_path, "COCO") mkdir_p(dataset_COCO) Ras_COCO_path = osp.join(dataset_COCO, "image/") annotations_COCO_path = osp.join(dataset_COCO, "annotations/") - train_COCO_path = osp.join(dataset_COCO, "train/") - eval_COCO_path = osp.join(dataset_COCO, "eval/") - test_COCO_path = osp.join(dataset_COCO, "test/") - mkdir_p(Ras_COCO_path) mkdir_p(annotations_COCO_path) - mkdir_p(train_COCO_path) - mkdir_p(eval_COCO_path) - mkdir_p(test_COCO_path) - - # img_path = "/".join(ras_path.split("/")[:-1]) - # shp_path = "/".join(vec_path.split("/")[:-1]) - img_path = ras_path - shp_path = vec_path - - # root_path_test = "" - clip_from_file(SplittingSize, dataset_COCO, - img_path, shp_path) - slice(dataset_COCO, train=Training_Set, - eval=Val_Set, test=Testing_Set) - from_mask_to_coco(dataset_COCO, "train", - "image", "annotations") - # from_mask_to_coco(ROOT_DIR, 'eval', "image", "annotations") - # from_mask_to_coco(ROOT_DIR, 'test', "image", "annotations") + + # TODO: the cut image is not repeated, + # and the cut image is used to directly generate coco format + clip_from_file(SplittingSize, dataset_COCO, ras_path, vec_path) + slice(dataset_COCO, train=Training_Set, eval=Val_Set, test=Testing_Set) + from_mask_to_coco(dataset_COCO, 'train', "image", "annotations") + from_mask_to_coco(dataset_COCO, 'eval', "image", "annotations") + from_mask_to_coco(dataset_COCO, 'test', "image", "annotations") iface.messageBar().pushMessage( "You will find the dataset in " + dataset_path, diff --git a/deep-learning-datasets-maker/utils/COCO/__init__.py b/deep-learning-datasets-maker/utils/COCO/__init__.py index 6150221..61a1258 100644 --- a/deep-learning-datasets-maker/utils/COCO/__init__.py +++ b/deep-learning-datasets-maker/utils/COCO/__init__.py @@ -1,3 +1 @@ -from .shape_to_coco import clip_from_file, slice, from_mask_to_coco -from .tif_process import * -from .slice_dataset import * \ No newline at end of file +from .shape_to_coco import clip_from_file, slice, from_mask_to_coco \ No newline at end of file diff --git a/deep-learning-datasets-maker/utils/COCO/pycococreatortools/__init__.py b/deep-learning-datasets-maker/utils/COCO/pycococreatortools/__init__.py index a0ac584..52a0e14 100644 --- a/deep-learning-datasets-maker/utils/COCO/pycococreatortools/__init__.py +++ b/deep-learning-datasets-maker/utils/COCO/pycococreatortools/__init__.py @@ -1 +1 @@ -from .pycococreatortools import resize_binary_mask, close_contour, binary_mask_to_rle, binary_mask_to_polygon, create_image_info, create_annotation_info \ No newline at end of file +from .pycococreatortools import * \ No newline at end of file diff --git a/deep-learning-datasets-maker/utils/COCO/shape_to_coco.py b/deep-learning-datasets-maker/utils/COCO/shape_to_coco.py index ab6ba1d..0ed88e6 100644 --- a/deep-learning-datasets-maker/utils/COCO/shape_to_coco.py +++ b/deep-learning-datasets-maker/utils/COCO/shape_to_coco.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 + import datetime import json import os @@ -22,11 +23,11 @@ clip_size = 512 INFO = { - "description": "Greenhouse Dataset", + "description": "Image Dataset", "url": "", "version": "0.1.0", - "year": 2019, - "contributor": "DuncanChen", + "year": 2021, + "contributor": "", "date_created": datetime.datetime.utcnow().isoformat(' ') } @@ -41,7 +42,7 @@ CATEGORIES = [ { 'id': 1, - 'name': 'greenhouse', + 'name': 'image', 'supercategory': 'building', }, ] @@ -91,7 +92,7 @@ def from_mask_to_coco(root, MARK, IMAGE, ANNOTATION): # go through each image for image_filename in image_files: image = Image.open(image_filename) - image_info = pycococreatortools.create_image_info( + image_info = create_image_info( image_id, os.path.basename(image_filename), image.size) coco_output["images"].append(image_info) @@ -109,7 +110,7 @@ def from_mask_to_coco(root, MARK, IMAGE, ANNOTATION): binary_mask = np.asarray(Image.open(annotation_filename) .convert('1')).astype(np.uint8) - annotation_info = pycococreatortools.create_annotation_info( + annotation_info = create_annotation_info( segmentation_id, image_id, category_info, binary_mask, image.size, tolerance=2) @@ -120,17 +121,17 @@ def from_mask_to_coco(root, MARK, IMAGE, ANNOTATION): image_id = image_id + 1 - with open('{}/instances_greenhouse_{}2019.json'.format(ROOT_DIR, MARK), 'w') as output_json_file: + with open('{}/instances_image_{}2019.json'.format(ROOT_DIR, MARK), 'w') as output_json_file: json.dump(coco_output, output_json_file) else: print(ROOT_DIR + ' does not exit!') -# def main(): +def main(): clip_from_file(clip_size, ROOT, img_path, shp_path) slice(ROOT_DIR, train=0.6, eval=0.2, test=0.2) from_mask_to_coco(ROOT_DIR, 'train', "image", "annotations") from_mask_to_coco(ROOT_DIR, 'eval', "image", "annotations") from_mask_to_coco(ROOT_DIR, 'test', "image", "annotations") -# if __name__ == "__main__": -# main() +if __name__ == "__main__": + main() diff --git a/deep-learning-datasets-maker/utils/COCO/slice_dataset.py b/deep-learning-datasets-maker/utils/COCO/slice_dataset.py index cbd1849..bb06d16 100644 --- a/deep-learning-datasets-maker/utils/COCO/slice_dataset.py +++ b/deep-learning-datasets-maker/utils/COCO/slice_dataset.py @@ -1,75 +1,75 @@ -import os -import numpy as np -import shutil -import re -import fnmatch - -ann_path = 'annotations' -img_path = 'image' - -def filter_for_annotations(root, files, image_filename): - # file_types = ['*.png'] - file_types = ['*.tif'] - file_types = r'|'.join([fnmatch.translate(x) for x in file_types]) - basename_no_extension = os.path.splitext(os.path.basename(image_filename))[0] - # file_name_prefix = basename_no_extension + '.*' - files = [os.path.join(root, f) for f in files] - files = [f for f in files if re.match(file_types, f)] - # files = [f for f in files if re.match(file_name_prefix, os.path.splitext(os.path.basename(f))[0])] - files = [f for f in files if basename_no_extension == os.path.splitext(os.path.basename(f))[0].split('_', 1)[0]] - - return files - -def copy_data(input_path, id, num, mark = 'train'): - if num != 0: - list = os.listdir(input_path + '/' + img_path) - ann_list = os.listdir(input_path + '/' + ann_path) - if not os.path.isdir(input_path + '/' + mark + '/' + img_path): - os.makedirs(input_path + '/' + mark + '/' + img_path) - if not os.path.isdir(input_path + '/' + mark + '/' + ann_path): - os.makedirs(input_path + '/' + mark + '/' + ann_path) - - for i in range(num): - shutil.copy(input_path + '/' + img_path + '/' + list[id[i]], input_path + '/' + mark + '/' + img_path - + '/' + list[id[i]]) - print('From src: ' + img_path + '/' + list[id[i]] + ' =>dst:' + '/' + mark + '/' + img_path - + '/' + list[id[i]]) - annotation_files = filter_for_annotations(input_path, ann_list, list[id[i]]) - for j in range(len(annotation_files)): - shutil.copy(input_path + '/' + ann_path + '/' + os.path.basename(annotation_files[j]), - input_path + '/' + mark + '/' + ann_path + '/' + os.path.basename(annotation_files[j])) - - f = open(input_path + '/' + mark + '/' + mark + '.txt', 'w') - f.write(str(id)) - f.close() - -def slice(input_path, train=0.8, eval=0.2, test=0.0): - """ - slice the dataset into training, eval and test sub_dataset. - :param input_path: path to the original dataset. - :param train: the ratio of the training subset. - :param eval: the ratio of the eval subset. - :param test: the ratio of the test subset. - """ - list = os.listdir(input_path + '/' + img_path) - ann_list = os.listdir(input_path + '/' + ann_path) - num_list = len(list) - n_train = int(num_list * train) - if test == 0: - n_eval = num_list - n_train - n_test = 0 - else: - n_eval = int(num_list * eval) - n_test = num_list - n_train - n_eval - - img_id = np.arange(num_list) - np.random.shuffle(img_id) - train_id, eval_id, test_id = img_id[:n_train], img_id[n_train: n_train+n_eval], img_id[n_train+n_eval:] - copy_data(input_path, train_id, n_train, 'train') - copy_data(input_path, eval_id, n_eval, 'eval') - copy_data(input_path, test_id, n_test, 'test') - -if __name__ == '__main__': - input_path = r'./example_data/original_data/dataset' - # slice(input_path, train=0.6, eval=0.2, test=0.2) - slice(input_path) +import os +import numpy as np +import shutil +import re +import fnmatch + +ann_path = 'annotations' +img_path = 'image' + +def filter_for_annotations(root, files, image_filename): + # file_types = ['*.png'] + file_types = ['*.tif'] + file_types = r'|'.join([fnmatch.translate(x) for x in file_types]) + basename_no_extension = os.path.splitext(os.path.basename(image_filename))[0] + # file_name_prefix = basename_no_extension + '.*' + files = [os.path.join(root, f) for f in files] + files = [f for f in files if re.match(file_types, f)] + # files = [f for f in files if re.match(file_name_prefix, os.path.splitext(os.path.basename(f))[0])] + files = [f for f in files if basename_no_extension == os.path.splitext(os.path.basename(f))[0].split('_', 1)[0]] + + return files + +def copy_data(input_path, id, num, mark = 'train'): + if num != 0: + list = os.listdir(input_path + '/' + img_path) + ann_list = os.listdir(input_path + '/' + ann_path) + if not os.path.isdir(input_path + '/' + mark + '/' + img_path): + os.makedirs(input_path + '/' + mark + '/' + img_path) + if not os.path.isdir(input_path + '/' + mark + '/' + ann_path): + os.makedirs(input_path + '/' + mark + '/' + ann_path) + + for i in range(num): + shutil.copy(input_path + '/' + img_path + '/' + list[id[i]], input_path + '/' + mark + '/' + img_path + + '/' + list[id[i]]) + print('From src: ' + img_path + '/' + list[id[i]] + ' =>dst:' + '/' + mark + '/' + img_path + + '/' + list[id[i]]) + annotation_files = filter_for_annotations(input_path, ann_list, list[id[i]]) + for j in range(len(annotation_files)): + shutil.copy(input_path + '/' + ann_path + '/' + os.path.basename(annotation_files[j]), + input_path + '/' + mark + '/' + ann_path + '/' + os.path.basename(annotation_files[j])) + + f = open(input_path + '/' + mark + '/' + mark + '.txt', 'w') + f.write(str(id)) + f.close() + +def slice(input_path, train=0.8, eval=0.2, test=0.0): + """ + slice the dataset into training, eval and test sub_dataset. + :param input_path: path to the original dataset. + :param train: the ratio of the training subset. + :param eval: the ratio of the eval subset. + :param test: the ratio of the test subset. + """ + list = os.listdir(input_path + '/' + img_path) + ann_list = os.listdir(input_path + '/' + ann_path) + num_list = len(list) + n_train = int(num_list * train) + if test == 0: + n_eval = num_list - n_train + n_test = 0 + else: + n_eval = int(num_list * eval) + n_test = num_list - n_train - n_eval + + img_id = np.arange(num_list) + np.random.shuffle(img_id) + train_id, eval_id, test_id = img_id[:n_train], img_id[n_train: n_train+n_eval], img_id[n_train+n_eval:] + copy_data(input_path, train_id, n_train, 'train') + copy_data(input_path, eval_id, n_eval, 'eval') + copy_data(input_path, test_id, n_test, 'test') + +if __name__ == '__main__': + input_path = r'./example_data/original_data/dataset' + # slice(input_path, train=0.6, eval=0.2, test=0.2) + slice(input_path) diff --git a/deep-learning-datasets-maker/utils/COCO/tif_process.py b/deep-learning-datasets-maker/utils/COCO/tif_process.py index 3e96599..d75b24c 100644 --- a/deep-learning-datasets-maker/utils/COCO/tif_process.py +++ b/deep-learning-datasets-maker/utils/COCO/tif_process.py @@ -7,13 +7,17 @@ from PIL import Image, ImageDraw import os -from osgeo import gdal, gdalnumeric, ogr import numpy as np - -# import ogr import glob +from osgeo import gdalnumeric + +try: + from osgeo import gdal, ogr +except ImportError: + import gdal + import ogr -# gdal.UseExceptions() +gdal.UseExceptions() class GeoTiff(object): @@ -31,12 +35,10 @@ def __init__(self, tif_path): # if you want to mask tif with shape file >>> tif.mask_tif_with_shapefile('shapefile.shp', 'save_path.tif') """ - self.dataset = gdal.Open(tif_path, 1) + self.dataset = gdal.Open(tif_path) self.bands_count = self.dataset.RasterCount # get each band - self.bands = [ - self.dataset.GetRasterBand(i + 1) for i in range(self.bands_count) - ] + self.bands = [self.dataset.GetRasterBand(i + 1) for i in range(self.bands_count)] self.col = self.dataset.RasterXSize self.row = self.dataset.RasterYSize self.geotransform = self.dataset.GetGeoTransform() @@ -69,11 +71,10 @@ def __getitem__(self, *args): num_row = self.row if end_row is None else (end_row - start_row) num_col = self.col if end_col is None else (end_col - start_col) # dataset read image array - res = self.dataset.ReadAsArray( - start_col, start_row, num_col, num_row) + res = self.dataset.ReadAsArray(start_col, start_row, num_col, num_row) return res else: - raise NotImplementedError("the param should be [a: b, c: d] !") + raise NotImplementedError('the param should be [a: b, c: d] !') def clip_tif_with_grid(self, clip_size, begin_id, out_dir): """ @@ -88,12 +89,12 @@ def clip_tif_with_grid(self, clip_size, begin_id, out_dir): if not os.path.exists(out_dir): # check the dir os.makedirs(out_dir) - print("create dir", out_dir) + print('create dir', out_dir) row_num = int(self.row / clip_size) col_num = int(self.col / clip_size) - gtiffDriver = gdal.GetDriverByName("GTiff") + gtiffDriver = gdal.GetDriverByName('GTiff') if gtiffDriver is None: raise ValueError("Can't find GeoTiff Driver") @@ -102,31 +103,17 @@ def clip_tif_with_grid(self, clip_size, begin_id, out_dir): for j in range(col_num): # if begin_id+i*col_num+j in self.mark: # continue - clipped_image = np.array( - self[ - i * clip_size: (i + 1) * clip_size, - j * clip_size: (j + 1) * clip_size, - ] - ) + clipped_image = np.array(self[i * clip_size: (i + 1) * clip_size, j * clip_size: (j + 1) * clip_size]) clipped_image = clipped_image.astype(np.int8) try: - save_path = os.path.join( - out_dir, "%d.tif" % (begin_id + i * col_num + j) - ) - save_image_with_georef( - clipped_image, - gtiffDriver, - self.dataset, - j * clip_size, - i * clip_size, - save_path, - ) - print("clip successfully！(%d/%d)" % - (count, row_num * col_num)) + save_path = os.path.join(out_dir, '%d.tif' % (begin_id+i*col_num+j)) + save_image_with_georef(clipped_image, gtiffDriver, + self.dataset, j*clip_size, i*clip_size, save_path) + print('clip successfully! (%d/%d)' % (count, row_num * col_num)) count += 1 except Exception: - raise IOError("clip failed!%d" % count) + raise IOError('clip failed!%d' % count) return row_num * col_num @@ -143,12 +130,12 @@ def clip_mask_with_grid(self, clip_size, begin_id, out_dir): if not os.path.exists(out_dir): # check the dir os.makedirs(out_dir) - print("create dir", out_dir) + print('create dir', out_dir) row_num = int(self.row / clip_size) col_num = int(self.col / clip_size) - gtiffDriver = gdal.GetDriverByName("GTiff") + gtiffDriver = gdal.GetDriverByName('GTiff') if gtiffDriver is None: raise ValueError("Can't find GeoTiff Driver") @@ -157,13 +144,7 @@ def clip_mask_with_grid(self, clip_size, begin_id, out_dir): count = 1 for i in range(row_num): for j in range(col_num): - clipped_image = np.array( - self.mask[ - 0, - i * clip_size: (i + 1) * clip_size, - j * clip_size: (j + 1) * clip_size, - ] - ) + clipped_image = np.array(self.mask[0, i * clip_size: (i + 1) * clip_size, j * clip_size: (j + 1) * clip_size]) ins_list = np.unique(clipped_image) # if len(ins_list) <= 1: # self.mark.append(begin_id+i*col_num+j) @@ -174,26 +155,13 @@ def clip_mask_with_grid(self, clip_size, begin_id, out_dir): if ins_list[id] > 0: bg_img[np.where(clipped_image == ins_list[id])] = 255 try: - save_path = os.path.join( - out_dir, - "%d_%s_%d.tif" - % (begin_id + i * col_num + j, "greenhouse", id), - ) - save_image_with_georef( - bg_img, - gtiffDriver, - self.dataset, - j * clip_size, - i * clip_size, - save_path, - ) - print( - "clip mask successfully！(%d/%d)" - % (count, row_num * col_num) - ) + save_path = os.path.join(out_dir, '%d_%s_%d.tif' % (begin_id+i*col_num+j, 'image', id)) + save_image_with_georef(bg_img, gtiffDriver, + self.dataset, j*clip_size, i*clip_size, save_path) + print('clip mask successfully! (%d/%d)' % (count, row_num * col_num)) count += 1 except Exception: - raise IOError("clip failed!%d" % count) + raise IOError('clip failed!%d' % count) def world2Pixel(self, x, y): """ @@ -220,11 +188,11 @@ def mask_tif_with_shapefile(self, shapefile_path, label=255): Returns: """ - driver = ogr.GetDriverByName("ESRI Shapefile") + driver = ogr.GetDriverByName('ESRI Shapefile') dataSource = driver.Open(shapefile_path, 0) if dataSource is None: - raise IOError("could not open!") - gtiffDriver = gdal.GetDriverByName("GTiff") + raise IOError('could not open!') + gtiffDriver = gdal.GetDriverByName('GTiff') if gtiffDriver is None: raise ValueError("Can't find GeoTiff Driver") @@ -245,13 +213,13 @@ def mask_tif_with_shapefile(self, shapefile_path, label=255): geom = feature.GetGeometryRef() feature_type = geom.GetGeometryName() - if feature_type == "POLYGON" or "MULTIPOLYGON": + if feature_type == 'POLYGON' or 'MULTIPOLYGON': # multi polygon operation # 1. use label to mask the max polygon # 2. use -label to mask the other polygon for j in range(geom.GetGeometryCount()): sub_polygon = geom.GetGeometryRef(j) - if feature_type == "MULTIPOLYGON": + if feature_type == 'MULTIPOLYGON': sub_polygon = sub_polygon.GetGeometryRef(0) for p_i in range(sub_polygon.GetPointCount()): px = sub_polygon.GetX(p_i) @@ -259,16 +227,15 @@ def mask_tif_with_shapefile(self, shapefile_path, label=255): points.append((px, py)) for p in points: - origin_pixel_x, origin_pixel_y = self.world2Pixel( - p[0], p[1]) + origin_pixel_x, origin_pixel_y = self.world2Pixel(p[0], p[1]) # the pixel in new image new_pixel_x, new_pixel_y = origin_pixel_x, origin_pixel_y pixels.append((new_pixel_x, new_pixel_y)) - rasterize.polygon(pixels, i + 1) + rasterize.polygon(pixels, i+1) pixels = [] points = [] - if feature_type != "MULTIPOLYGON": + if feature_type != 'MULTIPOLYGON': label = -abs(label) # restore the label value @@ -280,34 +247,28 @@ def mask_tif_with_shapefile(self, shapefile_path, label=255): points.append((px, py)) for p in points: - origin_pixel_x, origin_pixel_y = self.world2Pixel( - p[0], p[1]) + origin_pixel_x, origin_pixel_y = self.world2Pixel(p[0], p[1]) # the pixel in new image new_pixel_x, new_pixel_y = origin_pixel_x, origin_pixel_y pixels.append((new_pixel_x, new_pixel_y)) feature.Destroy() # delete feature - if feature_type == "LINESTRING": - rasterize.line(pixels, i + 1) - if feature_type == "POINT": + if feature_type == 'LINESTRING': + rasterize.line(pixels, i+1) + if feature_type == 'POINT': # pixel x, y - rasterize.point(pixels, i + 1) + rasterize.point(pixels, i+1) mask = np.array(rasterPoly) self.mask = mask[np.newaxis, :] # extend an axis to three def clip_tif_and_shapefile(self, clip_size, begin_id, shapefile_path, out_dir): self.mask_tif_with_shapefile(shapefile_path) - self.clip_mask_with_grid( - clip_size=clip_size, begin_id=begin_id, out_dir=out_dir + "/annotations" - ) - pic_id = self.clip_tif_with_grid( - clip_size=clip_size, begin_id=begin_id, out_dir=out_dir + "/image" - ) + self.clip_mask_with_grid(clip_size=clip_size, begin_id=begin_id, out_dir=out_dir + '/annotations') + pic_id = self.clip_tif_with_grid(clip_size=clip_size, begin_id=begin_id, out_dir=out_dir + '/image') return pic_id - def channel_first_to_last(image): """ @@ -320,7 +281,6 @@ def channel_first_to_last(image): new_image = np.transpose(image, axes=[1, 2, 0]) return new_image - def channel_last_to_first(image): """ @@ -333,10 +293,7 @@ def channel_last_to_first(image): new_image = np.transpose(image, axes=[2, 0, 1]) return new_image - -def save_image_with_georef( - image, driver, original_ds, offset_x=0, offset_y=0, save_path=None -): +def save_image_with_georef(image, driver, original_ds, offset_x=0, offset_y=0, save_path=None): """ Args: @@ -353,18 +310,17 @@ def save_image_with_georef( # get Geo Reference ds = gdalnumeric.OpenArray(image) gdalnumeric.CopyDatasetInfo(original_ds, ds, xoff=offset_x, yoff=offset_y) - ds = driver.CreateCopy(save_path, ds, gdal.GF_Write) + driver.CreateCopy(save_path, ds) # write by band clip = image.astype(np.int8) # write the dataset - if len(image.shape) == 3: + if len(image.shape)==3: for i in range(image.shape[0]): ds.GetRasterBand(i + 1).WriteArray(clip[i]) else: ds.GetRasterBand(1).WriteArray(clip) del ds - def define_ref_predict(tif_dir, mask_dir, save_dir): """ define reference for raster referred to a geometric raster. @@ -376,28 +332,26 @@ def define_ref_predict(tif_dir, mask_dir, save_dir): Returns: """ - tif_list = glob.glob(os.path.join(tif_dir, "*.tif")) + tif_list = glob.glob(os.path.join(tif_dir, '*.tif')) - mask_list = glob.glob(os.path.join(mask_dir, "*.png")) - mask_list += glob.glob(os.path.join(mask_dir, "*.jpg")) - mask_list += glob.glob(os.path.join(mask_dir, "*.tif")) + mask_list = glob.glob(os.path.join(mask_dir, '*.png')) + mask_list += (glob.glob(os.path.join(mask_dir, '*.jpg'))) + mask_list += (glob.glob(os.path.join(mask_dir, '*.tif'))) tif_list.sort() mask_list.sort() os.makedirs(save_dir, exist_ok=True) - gtiffDriver = gdal.GetDriverByName("GTiff") + gtiffDriver = gdal.GetDriverByName('GTiff') if gtiffDriver is None: raise ValueError("Can't find GeoTiff Driver") for i in range(len(tif_list)): - save_name = tif_list[i].split("\\")[-1] + save_name = tif_list[i].split('\\')[-1] save_path = os.path.join(save_dir, save_name) tif = GeoTiff(tif_list[i]) mask = np.array(Image.open(mask_list[i])) mask = channel_last_to_first(mask) - save_image_with_georef( - mask, gtiffDriver, tif.dataset, save_path=save_path) - + save_image_with_georef(mask, gtiffDriver, tif.dataset, save_path=save_path) class GeoShaplefile(object): def __init__(self, file_path=""): @@ -407,13 +361,12 @@ def __init__(self, file_path=""): self.feature_type = "" self.feature_num = 0 self.open_shapefile() - def open_shapefile(self): - driver = ogr.GetDriverByName("ESRI Shapefile") + driver = ogr.GetDriverByName('ESRI Shapefile') dataSource = driver.Open(self.file_path, 0) if dataSource is None: - raise IOError("could not open!") - gtiffDriver = gdal.GetDriverByName("GTiff") + raise IOError('could not open!') + gtiffDriver = gdal.GetDriverByName('GTiff') if gtiffDriver is None: raise ValueError("Can't find GeoTiff Driver") @@ -426,41 +379,22 @@ def open_shapefile(self): # feature type self.feature_type = geom.GetGeometryName() - # def clip_from_file(clip_size, root, img_path, shp_path): -# img_list = os.listdir(root + "/" + img_path) +# img_list = os.listdir(root + '/' + img_path) # n_img = len(img_list) # pic_id = 0 # for i in range(n_img): -# tif = GeoTiff(root + "/" + img_path + "/" + img_list[i]) -# img_id = img_list[i].split(".", 1)[0] -# pic_num = tif.clip_tif_and_shapefile( -# clip_size, -# pic_id, -# root + "/" + shp_path + "/" + img_id + "/" + img_id + ".shp", -# root + "/dataset", -# ) +# tif = GeoTiff(root + '/' + img_path + '/' + img_list[i]) +# img_id = img_list[i].split('.', 1)[0] +# pic_num = tif.clip_tif_and_shapefile(clip_size, pic_id, root + '/' + shp_path + '/' + img_id + '/' + img_id + '.shp', root + '/dataset') # pic_id += pic_num - def clip_from_file(clip_size, root, img_path, shp_path): - img_list = os.listdir(img_path) - n_img = len(img_list) - pic_id = 0 - for i in range(n_img): - tif = GeoTiff(img_path) - img_id = "0" - pic_num = tif.clip_tif_and_shapefile( - clip_size, - pic_id, - shp_path, - "/dataset", - ) - pic_id += pic_num - - -# if __name__ == '__main__': -# root = r'./example_data/original_data' -# img_path = 'img' -# shp_path = 'shp' -# clip_from_file(512, root, img_path, shp_path) + tif = GeoTiff(img_path) + tif.clip_tif_and_shapefile(clip_size, 0, shp_path, root) + +if __name__ == '__main__': + root = r'./example_data/original_data' + img_path = 'img' + shp_path = 'shp' + clip_from_file(512, root, img_path, shp_path) diff --git a/deep-learning-datasets-maker/utils/COCO/visualize_coco.py b/deep-learning-datasets-maker/utils/COCO/visualize_coco.py index 1f85b29..fcd676a 100644 --- a/deep-learning-datasets-maker/utils/COCO/visualize_coco.py +++ b/deep-learning-datasets-maker/utils/COCO/visualize_coco.py @@ -1,24 +1,24 @@ -from pycocotools.coco import COCO -import numpy as np -import skimage.io as io -import matplotlib.pyplot as plt -import pylab -import os - -ROOT_DIR = r'./example_data/original_data/dataset/eval' -image_directory = os.path.join(ROOT_DIR, "image") -annotation_file = os.path.join(ROOT_DIR, "instances_greenhouse_eval2019.json") - -example_coco = COCO(annotation_file) - -category_ids = example_coco.getCatIds(catNms=['square']) -image_ids = example_coco.getImgIds(catIds=category_ids) -image_data = example_coco.loadImgs(image_ids[0])[0] - -image = io.imread(image_directory + '/' + image_data['file_name']) -plt.imshow(image); plt.axis('off') -pylab.rcParams['figure.figsize'] = (8.0, 10.0) -annotation_ids = example_coco.getAnnIds(imgIds=image_data['id'], catIds=category_ids, iscrowd=None) -annotations = example_coco.loadAnns(annotation_ids) -example_coco.showAnns(annotations) +from pycocotools.coco import COCO +import numpy as np +import skimage.io as io +import matplotlib.pyplot as plt +import pylab +import os + +ROOT_DIR = r'./example_data/original_data/dataset/eval' +image_directory = os.path.join(ROOT_DIR, "image") +annotation_file = os.path.join(ROOT_DIR, "instances_image_eval2019.json") + +example_coco = COCO(annotation_file) + +category_ids = example_coco.getCatIds(catNms=['square']) +image_ids = example_coco.getImgIds(catIds=category_ids) +image_data = example_coco.loadImgs(image_ids[0])[0] + +image = io.imread(image_directory + '/' + image_data['file_name']) +plt.imshow(image); plt.axis('off') +pylab.rcParams['figure.figsize'] = (8.0, 10.0) +annotation_ids = example_coco.getAnnIds(imgIds=image_data['id'], catIds=category_ids, iscrowd=None) +annotations = example_coco.loadAnns(annotation_ids) +example_coco.showAnns(annotations) plt.show() \ No newline at end of file