-
Notifications
You must be signed in to change notification settings - Fork 40
/
utilities.py
400 lines (361 loc) · 15.2 KB
/
utilities.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
import io
import math
import xml.etree.ElementTree
from collections import defaultdict
import numpy
import PIL
import PIL.Image
import PIL.ImageColor
import PIL.ImageDraw
from ..constants import (TILE_FORMAT_IMAGE, TILE_FORMAT_NUMPY, TILE_FORMAT_PIL,
TileOutputMimeTypes, TileOutputPILFormat)
# Turn off decompression warning check
PIL.Image.MAX_IMAGE_PIXELS = None
def _encodeImage(image, encoding='JPEG', jpegQuality=95, jpegSubsampling=0,
format=(TILE_FORMAT_IMAGE, ), tiffCompression='raw',
**kwargs):
"""
Convert a PIL or numpy image into raw output bytes and a mime type.
:param image: a PIL image.
:param encoding: a valid PIL encoding (typically 'PNG' or 'JPEG'). Must
also be in the TileOutputMimeTypes map.
:param jpegQuality: the quality to use when encoding a JPEG.
:param jpegSubsampling: the subsampling level to use when encoding a JPEG.
:param format: the desired format or a tuple of allowed formats. Formats
are members of (TILE_FORMAT_PIL, TILE_FORMAT_NUMPY, TILE_FORMAT_IMAGE).
:param tiffCompression: the compression format to use when encoding a TIFF.
:returns:
:imageData: the image data in the specified format and encoding.
:imageFormatOrMimeType: the image mime type if the format is
TILE_FORMAT_IMAGE, or the format of the image data if it is
anything else.
"""
if not isinstance(format, (tuple, set, list)):
format = (format, )
imageData = image
imageFormatOrMimeType = TILE_FORMAT_PIL
if TILE_FORMAT_NUMPY in format:
imageData, _ = _imageToNumpy(image)
imageFormatOrMimeType = TILE_FORMAT_NUMPY
elif TILE_FORMAT_PIL in format:
imageData = _imageToPIL(image)
imageFormatOrMimeType = TILE_FORMAT_PIL
elif TILE_FORMAT_IMAGE in format:
if encoding not in TileOutputMimeTypes:
raise ValueError('Invalid encoding "%s"' % encoding)
imageFormatOrMimeType = TileOutputMimeTypes[encoding]
image = _imageToPIL(image)
if image.width == 0 or image.height == 0:
imageData = b''
else:
encoding = TileOutputPILFormat.get(encoding, encoding)
output = io.BytesIO()
params = {}
if encoding == 'JPEG' and image.mode not in ('L', 'RGB'):
image = image.convert('RGB' if image.mode != 'LA' else 'L')
if encoding == 'JPEG':
params['quality'] = jpegQuality
params['subsampling'] = jpegSubsampling
elif encoding in {'TIFF', 'TILED'}:
params['compression'] = {
'none': 'raw',
'lzw': 'tiff_lzw',
'deflate': 'tiff_adobe_deflate',
}.get(tiffCompression, tiffCompression)
elif encoding == 'PNG':
params['compress_level'] = 2
image.save(output, encoding, **params)
imageData = output.getvalue()
return imageData, imageFormatOrMimeType
def _imageToPIL(image, setMode=None):
"""
Convert an image in PIL, numpy, or image file format to a PIL image.
:param image: input image.
:param setMode: if specified, the output image is converted to this mode.
:returns: a PIL image.
"""
if isinstance(image, numpy.ndarray):
mode = 'L'
if len(image.shape) == 3:
# Fallback for hyperspectral data to just use the first three bands
if image.shape[2] > 4:
image = image[:, :, :3]
mode = ['L', 'LA', 'RGB', 'RGBA'][image.shape[2] - 1]
if len(image.shape) == 3 and image.shape[2] == 1:
image = numpy.resize(image, image.shape[:2])
if image.dtype == numpy.uint16:
image = numpy.floor_divide(image, 256).astype(numpy.uint8)
# TODO: The scaling of float data needs to be identical across all
# tiles of an image. This means that we need a reference to the parent
# tile source or some other way of regulating it.
# elif image.dtype.kind == 'f':
# if numpy.max(image) > 1:
# maxl2 = math.ceil(math.log(numpy.max(image) + 1) / math.log(2))
# image = image / ((2 ** maxl2) - 1)
# image = (image * 255).astype(numpy.uint8)
elif image.dtype != numpy.uint8:
image = image.astype(numpy.uint8)
image = PIL.Image.fromarray(image, mode)
elif not isinstance(image, PIL.Image.Image):
image = PIL.Image.open(io.BytesIO(image))
if setMode is not None and image.mode != setMode:
image = image.convert(setMode)
return image
def _imageToNumpy(image):
"""
Convert an image in PIL, numpy, or image file format to a numpy array. The
output numpy array always has three dimensions.
:param image: input image.
:returns: a numpy array and a target PIL image mode.
"""
if not isinstance(image, numpy.ndarray):
if not isinstance(image, PIL.Image.Image):
image = PIL.Image.open(io.BytesIO(image))
if image.mode not in ('L', 'LA', 'RGB', 'RGBA'):
image = image.convert('RGBA')
mode = image.mode
image = numpy.asarray(image)
else:
if len(image.shape) == 3:
mode = ['L', 'LA', 'RGB', 'RGBA'][(image.shape[2] - 1) if image.shape[2] <= 4 else 3]
else:
mode = 'L'
if len(image.shape) == 2:
image = numpy.resize(image, (image.shape[0], image.shape[1], 1))
return image, mode
def _letterboxImage(image, width, height, fill):
"""
Given a PIL image, width, height, and fill color, letterbox or pillarbox
the image to make it the specified dimensions. The image is never
cropped. The original image will be returned if no action is needed.
:param image: the source image.
:param width: the desired width in pixels.
:param height: the desired height in pixels.
:param fill: a fill color.
"""
if ((image.width >= width and image.height >= height) or
not fill or str(fill).lower() == 'none'):
return image
corner = False
if fill.lower().startswith('corner:'):
corner, fill = True, fill.split(':', 1)[1]
color = PIL.ImageColor.getcolor(fill, image.mode)
width = max(width, image.width)
height = max(height, image.height)
result = PIL.Image.new(image.mode, (width, height), color)
result.paste(image, (
int((width - image.width) / 2) if not corner else 0,
int((height - image.height) / 2) if not corner else 0))
return result
def _vipsCast(image, mustBe8Bit=False, originalScale=None):
"""
Cast a vips image to a format we want.
:param image: a vips image
:param mustBe9Bit: if True, then always cast to unsigned 8-bit.
:param originalScale:
:returns: a vips image
"""
import pyvips
formats = {
pyvips.BandFormat.CHAR: (pyvips.BandFormat.UCHAR, 2**7, 1),
pyvips.BandFormat.COMPLEX: (pyvips.BandFormat.USHORT, 0, 65535),
pyvips.BandFormat.DOUBLE: (pyvips.BandFormat.USHORT, 0, 65535),
pyvips.BandFormat.DPCOMPLEX: (pyvips.BandFormat.USHORT, 0, 65535),
pyvips.BandFormat.FLOAT: (pyvips.BandFormat.USHORT, 0, 65535),
pyvips.BandFormat.INT: (pyvips.BandFormat.USHORT, 2**31, 2**-16),
pyvips.BandFormat.USHORT: (pyvips.BandFormat.UCHAR, 0, 2**-8),
pyvips.BandFormat.SHORT: (pyvips.BandFormat.USHORT, 2**15, 1),
pyvips.BandFormat.UINT: (pyvips.BandFormat.USHORT, 0, 2**-16),
}
if image.format not in formats or (image.format == pyvips.BandFormat.USHORT and not mustBe8Bit):
return image
target, offset, multiplier = formats[image.format]
if image.format == pyvips.BandFormat.DOUBLE:
maxVal = image.max()
# These thresholds are higher than 256 and 65536 because bicubic and
# other interpolations can cause value spikes
if maxVal >= 2 and maxVal < 2**9:
multiplier = 256
elif maxVal >= 256 and maxVal < 2**17:
multiplier = 1
if mustBe8Bit and target != pyvips.BandFormat.UCHAR:
target = pyvips.BandFormat.UCHAR
multiplier /= 256
# logger.debug('Casting image from %r to %r', image.format, target)
image = ((image.cast(pyvips.BandFormat.DOUBLE) + offset) * multiplier).cast(target)
return image
def _gdalParameters(defaultCompression=None, eightbit=None, **kwargs):
"""
Return an array of gdal translation parameters.
:param defaultCompression: if not specified, use this value.
:param eightbit: True or False to indicate that the bit depth per sample is
known. None for unknown.
Optional parameters that can be specified in kwargs:
:param tileSize: the horizontal and vertical tile size.
:param compression: one of 'jpeg', 'deflate' (zip), 'lzw', 'packbits',
'zstd', or 'none'.
:param quality: a jpeg quality passed to gdal. 0 is small, 100 is high
quality. 90 or above is recommended.
:param level: compression level for zstd, 1-22 (default is 10).
:param predictor: one of 'none', 'horizontal', or 'float' used for lzw and
deflate.
:returns: a dictionary of parameters.
"""
options = {
'tileSize': 256,
'compression': 'lzw',
'quality': 90,
}
if eightbit is not None:
options['predictor'] = 'yes' if eightbit else 'none'
predictor = {
'none': 'NO',
'horizontal': 'STANDARD',
'float': 'FLOATING_POINT',
'yes': 'YES',
}
options.update({k: v for k, v in kwargs.items() if v not in (None, '')})
cmdopt = ['-of', 'COG', '-co', 'BIGTIFF=IF_SAFER']
cmdopt += ['-co', 'BLOCKSIZE=%d' % options['tileSize']]
cmdopt += ['-co', 'COMPRESS=%s' % options['compression'].upper()]
cmdopt += ['-co', 'QUALITY=%s' % options['quality']]
if 'predictor' in options:
cmdopt += ['-co', 'PREDICTOR=%s' % predictor[options['predictor']]]
if 'level' in options:
cmdopt += ['-co', 'LEVEL=%s' % options['level']]
return cmdopt
def _vipsParameters(forTiled=True, defaultCompression=None, **kwargs):
"""
Return a dictionary of vips conversion parameters.
:param forTiled: True if this is for a tiled image. False for an
associated image.
:param defaultCompression: if not specified, use this value.
Optional parameters that can be specified in kwargs:
:param tileSize: the horizontal and vertical tile size.
:param compression: one of 'jpeg', 'deflate' (zip), 'lzw', 'packbits',
'zstd', or 'none'.
:param quality: a jpeg quality passed to vips. 0 is small, 100 is high
quality. 90 or above is recommended.
:param level: compression level for zstd, 1-22 (default is 10).
:param predictor: one of 'none', 'horizontal', or 'float' used for lzw and
deflate.
:returns: a dictionary of parameters.
"""
if not forTiled:
convertParams = {
'compression': defaultCompression or 'jpeg',
'Q': 90,
'predictor': 'horizontal',
'tile': False,
}
if 'mime' in kwargs and kwargs.get('mime') != 'image/jpeg':
convertParams['compression'] = 'lzw'
return convertParams
convertParams = {
'tile': True,
'tile_width': 256,
'tile_height': 256,
'pyramid': True,
'bigtiff': True,
'compression': defaultCompression or 'jpeg',
'Q': 90,
'predictor': 'horizontal',
}
for vkey, kwkeys in {
'tile_width': {'tileSize'},
'tile_height': {'tileSize'},
'compression': {'compression', 'tiffCompression'},
'Q': {'quality', 'jpegQuality'},
'level': {'level'},
'predictor': {'predictor'},
}.items():
for kwkey in kwkeys:
if kwkey in kwargs and kwargs[kwkey] not in {None, ''}:
convertParams[vkey] = kwargs[kwkey]
if convertParams['compression'] == 'jp2k':
convertParams['compression'] = 'none'
if convertParams['compression'] == 'webp' and kwargs.get('quality') == 0:
convertParams['lossless'] = True
convertParams.pop('Q', None)
if convertParams['predictor'] == 'yes':
convertParams['predictor'] = 'horizontal'
if convertParams['compression'] == 'jpeg':
convertParams['rgbjpeg'] = True
return convertParams
def etreeToDict(t):
"""
Convert an xml etree to a nested dictionary without schema names in the
keys. If you have an xml string, this can be converted to a dictionary via
xml.etree.etreeToDict(ElementTree.fromstring(xml_string)).
:param t: an etree.
:returns: a python dictionary with the results.
"""
# Remove schema
tag = t.tag.split('}', 1)[1] if t.tag.startswith('{') else t.tag
d = {tag: {}}
children = list(t)
if children:
entries = defaultdict(list)
for entry in map(etreeToDict, children):
for k, v in entry.items():
entries[k].append(v)
d = {tag: {k: v[0] if len(v) == 1 else v
for k, v in entries.items()}}
if t.attrib:
d[tag].update({(k.split('}', 1)[1] if k.startswith('{') else k): v
for k, v in t.attrib.items()})
text = (t.text or '').strip()
if text and len(d[tag]):
d[tag]['text'] = text
elif text:
d[tag] = text
return d
def dictToEtree(d, root=None):
"""
Convert a dictionary in the style produced by etreeToDict back to an etree.
Make an xml string via xml.etree.ElementTree.tostring(dictToEtree(
dictionary), encoding='utf8', method='xml'). Note that this function and
etreeToDict are not perfect conversions; numerical values are quoted in
xml. Plain key-value pairs are ambiguous whether they should be attributes
or text values. Text fields are collected together.
:param d: a dictionary.
:prarm root: the root node to attach this dictionary to.
:returns: an etree.
"""
if root is None:
if len(d) == 1:
k, v = next(iter(d.items()))
root = xml.etree.ElementTree.Element(k)
dictToEtree(v, root)
return root
root = xml.etree.ElementTree.Element('root')
for k, v in d.items():
if isinstance(v, list):
for l in v:
elem = xml.etree.ElementTree.SubElement(root, k)
dictToEtree(l, elem)
elif isinstance(v, dict):
elem = xml.etree.ElementTree.SubElement(root, k)
dictToEtree(v, elem)
else:
if k == 'text':
root.text = v
else:
root.set(k, v)
return root
def nearPowerOfTwo(val1, val2, tolerance=0.02):
"""
Check if two values are different by nearly a power of two.
:param val1: the first value to check.
:param val2: the second value to check.
:param tolerance: the maximum difference in the log2 ratio's mantissa.
:return: True if the valeus are nearly a power of two different from each
other; false otherwise.
"""
# If one or more of the values is zero or they have different signs, then
# return False
if val1 * val2 <= 0:
return False
log2ratio = math.log(float(val1) / float(val2)) / math.log(2)
# Compare the mantissa of the ratio's log2 value.
return abs(log2ratio - round(log2ratio)) < tolerance