diff --git a/auxiliary/bulk_tiles/README.md b/auxiliary/bulk_tiles/README.md index 0a8e606a..71f4480b 100644 --- a/auxiliary/bulk_tiles/README.md +++ b/auxiliary/bulk_tiles/README.md @@ -11,6 +11,7 @@ optional arguments: -h, --help show this help message and exit -c COLORMAP_DIR, --colormap_dir COLORMAP_DIR Directory containing colormaps + -d, --discrete Use discrete color values -i INPUT_DIR, --input_dir INPUT_DIR Directory containing raw input TIFF files -j JSON_CONFIG, --json_config JSON_CONFIG diff --git a/auxiliary/bulk_tiles/bulk_tiles.py b/auxiliary/bulk_tiles/bulk_tiles.py index 67fac65e..db2ec038 100755 --- a/auxiliary/bulk_tiles/bulk_tiles.py +++ b/auxiliary/bulk_tiles/bulk_tiles.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """ This script bulk processes a directory of TIFs into TMS tiles @@ -15,7 +15,7 @@ from lxml import etree -def process_tiffs(input_dir, process_dir, colormap_dir, legends_dir, prefix=''): +def process_tiffs(input_dir, process_dir, colormap_dir, legends_dir, discrete, prefix=''): output_files = [] if os.path.isdir(input_dir): input_files = sorted(glob.glob(input_dir + '/' + prefix + '*.tif', recursive=True)) @@ -25,7 +25,12 @@ def process_tiffs(input_dir, process_dir, colormap_dir, legends_dir, prefix=''): colormap_files = sorted(glob.glob(colormap_dir+'/*.txt', recursive=True)) colormap_dict = {} for colormap_file in colormap_files: - colormap_key = os.path.basename(colormap_file).split('_')[1].split('.')[0] + if 'units_' in colormap_file: + continue + elif 'colormap_' in colormap_file: + colormap_key = os.path.basename(colormap_file).split('_')[1].split('.')[0] + else: + colormap_key = os.path.basename(colormap_file).split('.')[0] colormap_dict[colormap_key] = colormap_file else: print('Warning: ' + colormap_dir + ' directory does not exist') @@ -39,7 +44,10 @@ def process_tiffs(input_dir, process_dir, colormap_dir, legends_dir, prefix=''): # Figure out the colormap map to use for the file colormap = None for key in colormap_dict: - if key in input_file: + if os.path.basename(input_file).split('.')[0] == key: + colormap = colormap_dict[key] + break + elif key in input_file: colormap = colormap_dict[key] if colormap is None: print('Warning: Skipping...no colormap found for ' + input_file) @@ -50,7 +58,9 @@ def process_tiffs(input_dir, process_dir, colormap_dir, legends_dir, prefix=''): # Colorize the TIFF file with its respective colormap rastertolegend = str(Path(os.path.dirname(os.path.realpath(__file__)) + '/../rastertolegend/rastertolegend.py') .absolute()) - color_tiff = ['python', rastertolegend, input_file, colormap, '-discrete'] + color_tiff = ['python3', rastertolegend, input_file, colormap] + if discrete: + color_tiff.append('-discrete') print("Running:", " ".join(color_tiff)) process = subprocess.Popen(color_tiff, stdout=subprocess.PIPE, stderr=subprocess.PIPE) process.wait() @@ -66,7 +76,7 @@ def process_tiffs(input_dir, process_dir, colormap_dir, legends_dir, prefix=''): print('Moving ' + color_output + ' to ' + legends_dir + '/' + legend_file) shutil.move(color_output, legends_dir + '/' + legend_file) else: - output_file = process_dir + '/' + str(Path(color_output).name) + output_file = process_dir + '/' + str(Path(input_file).name) print('Moving ' + color_output + ' to ' + output_file) shutil.move(color_output, output_file) output_files.append(output_file) @@ -88,7 +98,7 @@ def create_tiles(input_files, output_dir): print('Output directory: ' + output_subdir) rasterstotiles = str(Path(os.path.dirname(os.path.realpath(__file__)) + '/../rasterstotiles/rasterstotiles.py') .absolute()) - tile_tiff = ['python', rasterstotiles, '-o', output_subdir, tiff] + tile_tiff = ['python3', rasterstotiles, '-o', output_subdir, tiff] print("Running:", " ".join(tile_tiff)) process = subprocess.Popen(tile_tiff, stdout=subprocess.PIPE, stderr=subprocess.PIPE) process.wait() @@ -137,7 +147,7 @@ def create_configs(output_dirs, json_config, prefix): sublayer['togglesWithHeader'] = False sublayer['minZoom'] = int(minZoom) sublayer['maxNativeZoom'] = int(maxNativeZoom) - sublayer['maxZoom'] = 10 + sublayer['maxZoom'] = 16 sublayer['boundingBox'] = [minx, miny, maxx, maxy] sublayers.append(sublayer) @@ -155,6 +165,12 @@ def create_configs(output_dirs, json_config, prefix): dest='colormap_dir', help='Directory containing colormaps', action='store') +parser.add_argument( + '-d', + '--discrete', + dest='discrete', + help='Use discrete color values', + action='store_true') parser.add_argument( '-i', '--input_dir', @@ -199,7 +215,7 @@ def create_configs(output_dirs, json_config, prefix): args = parser.parse_args() output_dirs = glob.glob(args.output_dir + '/*', recursive=True) -output_tiffs = process_tiffs(args.input_dir, args.process_dir, args.colormap_dir, args.legends_dir, args.prefix) +output_tiffs = process_tiffs(args.input_dir, args.process_dir, args.colormap_dir, args.legends_dir, args.discrete, args.prefix) output_dirs = create_tiles(output_tiffs, args.output_dir) # Generate JSON layer configurations if specified diff --git a/auxiliary/quantize_colormap/README.md b/auxiliary/quantize_colormap/README.md new file mode 100644 index 00000000..705dcac6 --- /dev/null +++ b/auxiliary/quantize_colormap/README.md @@ -0,0 +1,20 @@ +# quantize_colormap.py + +```shell +usage: quantize_colormap.py [-h] [-c COLORMAP] [-i INPUT] [-n NODATA] [-o OUTPUT_DIR] [-0] [-m] + +Generate quantized MPL colormaps from tiff files. + +options: + -h, --help show this help message and exit + -c COLORMAP, --colormap COLORMAP + matplotlib colormap to use + -i INPUT, --input INPUT + TIFF file or directory containing raw input TIFF files + -n NODATA, --nodata NODATA + nodata value + -o OUTPUT_DIR, --output_dir OUTPUT_DIR + Directory containing output colormaps + -0, --ignore_zeroes Ignore zero and under values + -m, --ignore_min Ignore min value +``` diff --git a/auxiliary/quantize_colormap/quantize_colormap.py b/auxiliary/quantize_colormap/quantize_colormap.py new file mode 100755 index 00000000..98b52661 --- /dev/null +++ b/auxiliary/quantize_colormap/quantize_colormap.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python3 + +import os +import sys +import argparse +import glob +import matplotlib as mpl +import matplotlib.pyplot as plt +from matplotlib import cm +import numpy as np +from osgeo import gdal + +# https://matplotlib.org/stable/gallery/color/colormap_reference.html +cmaps = [('Perceptually Uniform Sequential', [ + 'viridis', 'plasma', 'inferno', 'magma', 'cividis']), + ('Sequential', [ + 'Greys', 'Purples', 'Blues', 'Greens', 'Oranges', 'Reds', + 'YlOrBr', 'YlOrRd', 'OrRd', 'PuRd', 'RdPu', 'BuPu', + 'GnBu', 'PuBu', 'YlGnBu', 'PuBuGn', 'BuGn', 'YlGn']), + ('Sequential (2)', [ + 'binary', 'gist_yarg', 'gist_gray', 'gray', 'bone', 'pink', + 'spring', 'summer', 'autumn', 'winter', 'cool', 'Wistia', + 'hot', 'afmhot', 'gist_heat', 'copper']), + ('Diverging', [ + 'PiYG', 'PRGn', 'BrBG', 'PuOr', 'RdGy', 'RdBu', + 'RdYlBu', 'RdYlGn', 'Spectral', 'coolwarm', 'bwr', 'seismic']), + ('Cyclic', ['twilight', 'twilight_shifted', 'hsv']), + ('Qualitative', [ + 'Pastel1', 'Pastel2', 'Paired', 'Accent', + 'Dark2', 'Set1', 'Set2', 'Set3', + 'tab10', 'tab20', 'tab20b', 'tab20c']), + ('Miscellaneous', [ + 'flag', 'prism', 'ocean', 'gist_earth', 'terrain', 'gist_stern', + 'gnuplot', 'gnuplot2', 'CMRmap', 'cubehelix', 'brg', + 'gist_rainbow', 'rainbow', 'jet', 'turbo', 'nipy_spectral', + 'gist_ncar'])] + + +class MplColorHelper: + + def __init__(self, cmap_name, start_val, stop_val): + self.cmap_name = cmap_name + self.cmap = plt.get_cmap(cmap_name) + self.norm = mpl.colors.Normalize(vmin=start_val, vmax=stop_val) + self.scalarMap = cm.ScalarMappable(norm=self.norm, cmap=self.cmap) + + def get_rgb(self, val): + return self.scalarMap.to_rgba(val) + + +def quantize_colormap_from_tiff(tiff, output_dir, colormap, nodata, ignore_zeroes, ignore_min): + print('Processing', tiff) + output_file = output_dir + '/' + os.path.basename(tiff).split('.')[0] + '.txt' + + # open the dataset and retrieve raster data as an array + dataset = gdal.Open(tiff) + array = dataset.ReadAsArray() + + # get min max values + min = np.min(array) + max = np.max(array) + + # remove 0 and negative values + if ignore_zeroes: + array = array[array > 0] + # remove min values + if ignore_min: + array = array[array != min] + + # use the numpy percentile function to calculate quantile thresholds + print('Calculating quantiles...') + print(max, min) + percentiles = [] + for i in np.arange(100, 0, -10).tolist(): + percentiles.append(np.percentile(array, i)) + percentiles.append(np.min(array)) + print(percentiles) + + print('Generating', output_file) + out = open(output_file, 'w+') + mch = MplColorHelper(colormap, 0, len(percentiles)) + for i in range(0, len(percentiles)): + val = percentiles[i] + rgba = (mch.scalarMap.to_rgba(len(percentiles)-1-i)) + r = round(rgba[0] * 255) + g = round(rgba[1] * 255) + b = round(rgba[2] * 255) + a = round(rgba[3] * 255) + out_str = ' '.join([str(round(val)), str(r), str(g), str(b), str(a)]) + out.write(out_str + '\n') + + if not str(nodata) == 'nv': + if float(nodata) <= min: + out.write(f'{str(nodata)} 0 0 0 0\n') + else: + out.write(f'nv 0 0 0 0\n') + out.seek(0) + print(out.read()) + out.close() + + +parser = argparse.ArgumentParser(description='Generate quantized MPL colormaps from tiff files.') +parser.add_argument( + '-c', + '--colormap', + default='viridis', + dest='colormap', + help='matplotlib colormap to use', + action='store') +parser.add_argument( + '-i', + '--input', + default='Raw/', + dest='input', + help='TIFF file or directory containing raw input TIFF files', + action='store') +parser.add_argument( + '-n', + '--nodata', + dest='nodata', + default='nv', + help='nodata value', + action='store') +parser.add_argument( + '-o', + '--output_dir', + default='Colormaps/', + dest='output_dir', + help='Directory containing output colormaps', + action='store') +parser.add_argument( + '-0', + '--ignore_zeroes', + default=False, + dest='ignore_zeroes', + help='Ignore zero and under values', + action='store_true') +parser.add_argument( + '-m', + '--ignore_min', + default=False, + dest='ignore_min', + help='Ignore min value', + action='store_true') + +args = parser.parse_args() + +if not os.path.isdir(args.output_dir): + print(args.output_dir, 'is not a directory.') + sys.exit() + +if os.path.isfile(args.input): + quantize_colormap_from_tiff(args.input, args.output_dir, args.colormap, args.nodata, args.ignore_zeroes, args.ignore_min) +else: + print('Searching', args.input) + input_files = glob.glob(args.input + '*.tif*', recursive=True) + for input_file in input_files: + quantize_colormap_from_tiff(input_file, args.output_dir, args.colormap, args.nodata, args.ignore_zeroes, args.ignore_min) + +sys.exit()