Skip to content
Permalink
Browse files

Add no_data args parameters. Default workers related to rasters file num

  • Loading branch information...
ocourtin committed Aug 10, 2019
1 parent 571eb1b commit 8defac049e1c94d3ee8d48b65b3f5bacb6654b1f
Showing with 33 additions and 13 deletions.
  1. +10 −3 docs/tools.md
  2. +23 −10 robosat_pink/tools/tile.py
@@ -204,8 +204,10 @@ Web UI:
```
## rsp tile
```
usage: rsp tile [-h] --zoom ZOOM [--ts TS] [--label] [--config CONFIG]
[--workers WORKERS] [--web_ui_base_url WEB_UI_BASE_URL]
usage: rsp tile [-h] [--nodata_threshold [0-100]] [--nodata_border]
[--nodata_value [0-255]] --zoom ZOOM [--ts TS] [--label]
[--config CONFIG] [--workers WORKERS]
[--web_ui_base_url WEB_UI_BASE_URL]
[--web_ui_template WEB_UI_TEMPLATE] [--no_web_ui]
rasters [rasters ...] out
@@ -215,6 +217,11 @@ optional arguments:
Inputs:
rasters path to raster files to tile [required]
No Data:
--nodata_threshold [0-100] Skip tile if nodata pixel ratio > threshold. [default: 100]
--nodata_border if set, keep no data border tiles
--nodata_value [0-255] No data pixel value [default: 0]
Output:
--zoom ZOOM zoom level of tiles [required]
--ts TS tile size in pixels [default: 512]
@@ -225,7 +232,7 @@ Labels:
--config CONFIG path to config file [required in label mode]
Performances:
--workers WORKERS number of workers [default: CPU]
--workers WORKERS number of workers [default: raster files]
Web UI:
--web_ui_base_url WEB_UI_BASE_URL alternate Web UI base URL
@@ -31,6 +31,13 @@ def add_parser(subparser, formatter_class):
inp = parser.add_argument_group("Inputs")
inp.add_argument("rasters", type=str, nargs="+", help="path to raster files to tile [required]")

nd = parser.add_argument_group("No Data")
help = "Skip tile if nodata pixel ratio > threshold. [default: 100]"
nd.add_argument("--nodata_threshold", type=int, default=100, choices=range(0, 101), metavar="[0-100]", help=help)
nd.add_argument("--nodata_border", action="store_true", help="if set, keep no data border tiles")
help = "No data pixel value [default: 0]"
nd.add_argument("--nodata_value", type=int, default=0, choices=range(0, 256), metavar="[0-255]", help=help)

out = parser.add_argument_group("Output")
out.add_argument("--zoom", type=int, required=True, help="zoom level of tiles [required]")
out.add_argument("--ts", type=int, default=512, help="tile size in pixels [default: 512]")
@@ -41,7 +48,7 @@ def add_parser(subparser, formatter_class):
lab.add_argument("--config", type=str, help="path to config file [required in label mode]")

perf = parser.add_argument_group("Performances")
perf.add_argument("--workers", type=int, help="number of workers [default: CPU]")
perf.add_argument("--workers", type=int, help="number of workers [default: raster files]")

ui = parser.add_argument_group("Web UI")
ui.add_argument("--web_ui_base_url", type=str, help="alternate Web UI base URL")
@@ -51,24 +58,27 @@ def add_parser(subparser, formatter_class):
parser.set_defaults(func=main)


def is_nodata(image, no_data=0, threshold=5):
def skip_nodata(image, keep_border, no_data, threshold):

if (
if not keep_border and (
np.all(image[0, :, :] == no_data)
or np.all(image[-1, :, :] == no_data)
or np.all(image[:, 0, :] == no_data)
or np.all(image[:, -1, :] == no_data)
):
return True # pixel border is no_data, on all bands

if threshold == 100:
return False

C, W, H = image.shape
return np.sum(image[:, :, :] == no_data) > ((threshold * C * 100) / (W * H))


def main(args):

if not args.workers:
args.workers = os.cpu_count()
args.workers = min(os.cpu_count(), len(args.rasters))

if args.label:
config = load_config(args.config)
@@ -139,9 +149,11 @@ def worker(path):
image = np.moveaxis(data, 0, 2) # C,H,W -> H,W,C

tile_key = (str(tile.x), str(tile.y), str(tile.z))
if not args.label and len(tiles_map[tile_key]) == 1 and is_nodata(image):
progress.update()
continue

if not args.label and len(tiles_map[tile_key]) == 1:
if skip_nodata(image, args.nodata_border, args.nodata_value, args.nodata_threshold):
progress.update()
continue

if len(tiles_map[tile_key]) > 1:
out = os.path.join(splits_path, str(tiles_map[tile_key].index(path)))
@@ -171,7 +183,7 @@ def worker(path):
# Aggregate remaining tiles splits
with futures.ThreadPoolExecutor(args.workers) as executor:

def worker(tile_key):
def worker(tile_key, nodata):

if len(tiles_map[tile_key]) == 1:
return
@@ -192,7 +204,7 @@ def worker(tile_key):
assert image.shape == split.shape
image[:, :, :] += split[:, :, :]

if not args.label and is_nodata(image):
if not args.label and skip_nodata(image, nodata["border"], nodata["value"], nodata["threshold"]):
progress.update()
return

@@ -210,7 +222,8 @@ def worker(tile_key):
progress.update()
return tile

for tiled in executor.map(worker, tiles_map.keys()):
nodata = {"border": args.nodata_border, "value": args.nodata_value, "threshold": args.nodata_threshold}
for tiled in executor.map(worker, tiles_map.keys(), nodata):
if tiled is not None:
tiles.append(tiled)

0 comments on commit 8defac0

Please sign in to comment.
You can’t perform that action at this time.