From ffc643c5e324b8d35a90ad2bb71feb763dbfaa16 Mon Sep 17 00:00:00 2001 From: Ngoguey42 Date: Wed, 12 Aug 2020 12:33:55 +0200 Subject: [PATCH 1/2] Updates to Footprint and UT --- buzzard/_footprint.py | 473 ++++++++++++++++++++++++++- buzzard/test/test_footprint_convs.py | 308 +++++++++++++++++ 2 files changed, 768 insertions(+), 13 deletions(-) create mode 100644 buzzard/test/test_footprint_convs.py diff --git a/buzzard/_footprint.py b/buzzard/_footprint.py index f68c6be..0108d20 100644 --- a/buzzard/_footprint.py +++ b/buzzard/_footprint.py @@ -310,24 +310,114 @@ def clip(self, startx, starty, endx, endy): rsize=rsize, ) - def _morpho(self, scount): - aff = self._aff * affine.Affine.translation(-scount, -scount) + def _morpho(self, left, right, top, bottom): + if left == right == top == bottom == 0: + return self + aff = self._aff * affine.Affine.translation(-left, -top) return Footprint( gt=aff.to_gdal(), - rsize=(self.rsize + 2 * scount), + rsize=(self.rsize + [left + right, top + bottom]), ) - def erode(self, count): - """Construct a new Footprint from self, eroding all edges by :code:`count` pixels""" - assert count >= 0 - assert count == int(count) - return self._morpho(-count) + def erode(self, *args): + """erode(self, inward_count, /) + erode(self, inward_count_x, inward_count_y, /) + erode(self, inward_count_left, inward_count_right, inward_count_top, inward_count_bottom, /) - def dilate(self, count): - """Construct a new Footprint from self, dilating all edges by :code:`count` pixels""" - assert count >= 0 - assert count == int(count) - return self._morpho(count) + Erode self's edges by the given pixel count to construct a new Footprint. + + A negative erosion is a dilation. + + Parameters + ---------- + *args: int or (int, int) or (int, int, int, int) + When `int`, erode all 4 directions by that much pixels + When `(int, int)`, erode x and y by a different number of pixel + When `(int, int, int, int)`, erode all 4 directions with a different number of pixel + + Returns + ------- + Footprint + + """ + if len(args) == 1: + left = right = top = bottom = args[0] + elif len(args) == 2: + left, top = right, bottom = args + elif len(args) == 4: + left, right, top, bottom = args + else: # pragma: no cover + raise ValueError('Expecting one, two or four positional parameters') + + v = int(left) + if v != left: # pragma: no cover + raise ValueError('left should be an integer') + left = v + v = int(right) + if v != right: # pragma: no cover + raise ValueError('right should be an integer') + right = v + v = int(top) + if v != top: # pragma: no cover + raise ValueError('top should be an integer') + top = v + v = int(bottom) + if v != bottom: # pragma: no cover + raise ValueError('bottom should be an integer') + bottom = v + + left, right, top, bottom = map(lambda x: -int(x), [left, right, top, bottom]) + return self._morpho(left, right, top, bottom) + + def dilate(self, *args): + """dilate(self, outward_count, /) + dilate(self, outward_count_x, outward_count_y, /) + dilate(self, outward_count_left, outward_count_right, outward_count_top, outward_count_bottom, /) + + Dilate self's edges by the given pixel count to construct a new Footprint. + + A negative dilation is an erosion. + + Parameters + ---------- + *args: int or (int, int) or (int, int, int, int) + When `int`, dilate all 4 directions by that much pixels + When `(int, int)`, dilate x and y by a different number of pixel + When `(int, int, int, int)`, dilate all 4 directions with a different number of pixel + + Returns + ------- + Footprint + + """ + if len(args) == 1: + left = right = top = bottom = args[0] + elif len(args) == 2: + left, top = right, bottom = args + elif len(args) == 4: + left, right, top, bottom = args + else: # pragma: no cover + raise ValueError('Expecting one, two or four positional parameters') + + v = int(left) + if v != left: # pragma: no cover + raise ValueError('left should be an integer') + left = v + v = int(right) + if v != right: # pragma: no cover + raise ValueError('right should be an integer') + right = v + v = int(top) + if v != top: # pragma: no cover + raise ValueError('top should be an integer') + top = v + v = int(bottom) + if v != bottom: # pragma: no cover + raise ValueError('bottom should be an integer') + bottom = v + + left, right, top, bottom = map(lambda x: int(x), [left, right, top, bottom]) + return self._morpho(left, right, top, bottom) def intersection(self, *objects, **kwargs): """intersection(self, *objects, scale='self', rotation='auto', alignment='auto', \ @@ -2207,6 +2297,342 @@ def __hash__(self): tuple(self._rsize.tolist()), )) + # Convolutions ****************************************************************************** ** + def forward_conv2d(self, kernel_size, stride=1, padding=0, dilation=1): + """Shift, scale and dilate the Footprint as if it went throught a 2d convolution kernel. + + The arithmetic followed is the one from `pytorch`, but the arithmetics in other + deep-learning libraries are mostly the same. + + This function is a `many to one` mapping, two footprints with different `rsizes` can produce + the same Footprint when `stride > 1`. + + Parameters + ---------- + kernel_size: int or (int, int) + See `torch.nn.Conv2d` documentation. + stride: int or (int, int) + See `torch.nn.Conv2d` documentation. + padding: int or (int, int) + See `torch.nn.Conv2d` documentation. + dilation: int or (int, int) + See `torch.nn.Conv2d` documentation. + + Returns + ------- + Footprint + + Example + ------- + >>> fp0 = buzz.Footprint(tl=(0, 0), size=(1024, 1024), rsize=(1024, 1024)) + ... fp1 = fp0.forward_conv2d(kernel_size=2, stride=2) + ... print(fp1) + Footprint(tl=(0.5, -0.5), size=(1024, 1024), rsize=(512, 512)) + + """ + + # *********************************************************************** ** + fp0 = self + kernel_size, stride, padding, dilation = _parse_conv2d_params( + kernel_size, stride, padding, dilation + ) + kernel_size = 1 + (kernel_size - 1) * dilation + + # *********************************************************************** ** + # rf_rad: Receptive field radius (2,) + # pxlrvec: Pixel Left-Right Vector (2,) + # pxtbvev: Pixel Top-Bottprint Vector (2,) + rf_rad = (kernel_size - 1) / 2 + tl1 = ( + fp0.tl + + # A padding shift toward top-left + - fp0.pxlrvec * padding[0] + - fp0.pxtbvec * padding[1] + + # A convolution kernel shift toward bottom-left + + fp0.pxlrvec * rf_rad[0] + + fp0.pxtbvec * rf_rad[1] + ) + + # *********************************************************************** ** + rsize0_padded = fp0.rsize + padding * 2 + if np.any(rsize0_padded < kernel_size): # pragma: no cover + fmt = ("Can't conv2d with kernel:{:.0f}x{:.0f} dil:{:.0f}x{:.0f} pad:{:.0f}x{:.0f} on " + "input-shape:{:.0f}x{:.0f} because on at least one dimension: " + "padded-input-shape:{:.0f}x{:.0f} < kernel-span:{:.0f}x{:.0f}." + ) + raise ValueError(fmt.format( + *np.flipud((kernel_size - 1) / dilation + 1), + *np.flipud(dilation), + *np.flipud(padding), + *fp0.shape, + *np.flipud(rsize0_padded), + *np.flipud(kernel_size), + )) + rsize1 = 1 + np.floor((rsize0_padded - (kernel_size - 1) - 1) / stride) + + # *********************************************************************** ** + aff = fp0._aff + aff = aff * affine.Affine.scale(*stride) + _, dx, rx, _, ry, dy = aff.to_gdal() + if np.all(stride == 1): + assert np.all(aff.to_gdal() == fp0.gt), (aff.to_gdal(), fp0.gt) + + # *********************************************************************** ** + return Footprint( + gt=(tl1[0], dx, rx, tl1[1], ry, dy), + rsize=rsize1, + ) + + def backward_conv2d(self, kernel_size, stride=1, padding=0, dilation=1): + """Shift, scale and dilate the Footprint as if it went backward throught a 2d convolution + kernel. + + The arithmetic followed is the one from `pytorch`, but the arithmetics in other + deep-learning libraries are mostly the same. + + This function is a `one to one` mapping, two different input footprints will produce two + different output Footprints. It means that the `backward_conv2d` of a `forward_conv2d` may + not reproduce the initial Footprint, some pixels on the bottom and right edges may be + missing. + + Parameters + ---------- + kernel_size: int or (int, int) + See `torch.nn.Conv2d` documentation. + stride: int or (int, int) + See `torch.nn.Conv2d` documentation. + padding: int or (int, int) + See `torch.nn.Conv2d` documentation. + dilation: int or (int, int) + See `torch.nn.Conv2d` documentation. + + Returns + ------- + Footprint + + Example + ------- + >>> fp1 = buzz.Footprint(tl=(0.5, -0.5), size=(1024, 1024), rsize=(512, 512)) + ... fp0 = fp1.backward_conv2d(kernel_size=2, stride=2) + ... print(fp0) + Footprint(tl=(0, 0), size=(1024, 1024), rsize=(1024, 1024)) + + """ + + # *********************************************************************** ** + fp1 = self + kernel_size, stride, padding, dilation = _parse_conv2d_params( + kernel_size, stride, padding, dilation + ) + kernel_size = 1 + (kernel_size - 1) * dilation + + # *********************************************************************** ** + # Inverse of the operation of `forward_conv2d` + rf_rad = (kernel_size - 1) / 2 + tl0 = ( + fp1.tl + + + fp1.pxlrvec / stride[0] * padding[0] + + fp1.pxtbvec / stride[1] * padding[1] + + - fp1.pxlrvec / stride[0] * rf_rad[0] + - fp1.pxtbvec / stride[1] * rf_rad[1] + ) + + # *********************************************************************** ** + # Inverse of the operation of `forward_conv2d` + rsize0 = (fp1.rsize - 1) * stride - (padding * 2 - (kernel_size - 1) - 1) + + # *********************************************************************** ** + aff = fp1._aff + aff = aff * affine.Affine.scale(*(1 / stride)) + _, dx, rx, _, ry, dy = aff.to_gdal() + if np.all(stride == 1): + assert np.all(aff.to_gdal() == fp1.gt), (aff.to_gdal(), fp1.gt) + + # *********************************************************************** ** + return Footprint( + gt=(tl0[0], dx, rx, tl0[1], ry, dy), + rsize=rsize0, + ) + + def forward_convtranspose2d(self, kernel_size, stride=1, padding=0, dilation=1, + output_padding=0): + """Shift, scale and dilate the Footprint as if it went throught a 2d transposed convolution + kernel. + + The arithmetic followed is the one from `pytorch`, but the arithmetics in other + deep-learning libraries are mostly the same. + + A 2d transposed convolution has 4 internal steps: + 1. Apply stride (interleave the input pixels with zeroes) + 2. Add padding + 3. Apply a 2d convolution stride:1, pad:0 + 4. Add output-padding + + This function is a `one to one` mapping, two different input footprints will produce two + different output Footprints. + + Parameters + ---------- + kernel_size: int or (int, int) + See `torch.nn.ConvTranspose2d` documentation. + stride: int or (int, int) + See `torch.nn.ConvTranspose2d` documentation. + padding: int or (int, int) + See `torch.nn.ConvTranspose2d` documentation. + dilation: int or (int, int) + See `torch.nn.ConvTranspose2d` documentation. + output_padding: int or (int, int) + See `torch.nn.ConvTranspose2d` documentation. + + Returns + ------- + Footprint + + Example + ------- + >>> fp0 = buzz.Footprint(tl=(0, 0), size=(1024, 1024), rsize=(512, 512)) + ... fp1 = fp0.forward_convtranspose2d(kernel_size=3, stride=2, padding=1) + ... print(fp1) + Footprint(tl=(0, 0), size=(1023, 1023), rsize=(1023, 1023)) + + """ + + # *********************************************************************** ** + fp0 = self + kernel_size, stride, padding, dilation, output_padding = _parse_conv2d_params( + kernel_size, stride, padding, dilation, output_padding, + allow_neg_padding=False, + ) + padding_input = dilation * (kernel_size - 1) - padding + kernel_size = 1 + (kernel_size - 1) * dilation + + # *********************************************************************** ** + # See `forward_conv2d` for details + rf_rad = (kernel_size - 1) / 2 + tl1 = ( + fp0.tl + - fp0.pxlrvec / stride[0] * padding_input[0] + - fp0.pxtbvec / stride[1] * padding_input[1] + + fp0.pxlrvec / stride[0] * rf_rad[0] + + fp0.pxtbvec / stride[1] * rf_rad[1] + ) + + # *********************************************************************** ** + rsize_inner = fp0.rsize + (fp0.rsize - 1) * (stride - 1) + padding_input * 2 + if np.any(rsize_inner < kernel_size): # pragma: no cover + fmt = ("Can't convtranspose2d with " + "kernel:{:.0f}x{:.0f} dil:{:.0f}x{:.0f} pad:{:.0f}x{:.0f} on " + "input-shape:{:.0f}x{:.0f} because on at least one dimension: " + "inner-shape:{:.0f}x{:.0f} < kernel-span:{:.0f}x{:.0f}." + ) + raise ValueError(fmt.format( + *np.flipud((kernel_size - 1) / dilation + 1), + *np.flipud(dilation), + *np.flipud(padding), + *fp0.shape, + *np.flipud(rsize_inner), + *np.flipud(kernel_size), + )) + rsize1 = 1 + np.floor((rsize_inner - (kernel_size - 1) - 1)) + output_padding + + # *********************************************************************** ** + aff = fp0._aff + aff = aff * affine.Affine.scale(*(1 / stride)) + _, dx, rx, _, ry, dy = aff.to_gdal() + if np.all(stride == 1): + assert np.all(aff.to_gdal() == fp0.gt), (aff.to_gdal(), fp0.gt) + + # *********************************************************************** ** + return Footprint( + gt=(tl1[0], dx, rx, tl1[1], ry, dy), + rsize=rsize1, + ) + + def backward_convtranspose2d(self, kernel_size, stride=1, padding=0, dilation=1, + output_padding=0): + """Shift, scale and dilate the Footprint as if it went backward throught a 2d transposed + convolution kernel. + + The arithmetic followed is the one from `pytorch`, but the arithmetics in other + deep-learning libraries are mostly the same. + + A 2d transposed convolution has 4 internal steps: + 1. Apply stride (interleave the input pixels with zeroes) + 2. Add padding + 3. Apply a 2d convolution stride:1, pad:0 + 4. Add output-padding + + This function is a `one to one` mapping, two different input Footprints will produce two + different output Footprints. + + Parameters + ---------- + kernel_size: int or (int, int) + See `torch.nn.ConvTranspose2d` documentation. + stride: int or (int, int) + See `torch.nn.ConvTranspose2d` documentation. + padding: int or (int, int) + See `torch.nn.ConvTranspose2d` documentation. + dilation: int or (int, int) + See `torch.nn.ConvTranspose2d` documentation. + output_padding: int or (int, int) + See `torch.nn.ConvTranspose2d` documentation. + + Returns + ------- + Footprint + + Example + ------- + >>> fp0 = buzz.Footprint(tl=(0, 0), size=(1023, 1023), rsize=(1023, 1023)) + ... fp1 = fp0.backward_convtranspose2d(kernel_size=3, stride=2, padding=1) + ... print(fp1) + Footprint(tl=(0, 0), size=(1024, 1024), rsize=(512, 512)) + + """ + + # *********************************************************************** ** + fp1 = self + kernel_size, stride, padding, dilation, output_padding = _parse_conv2d_params( + kernel_size, stride, padding, dilation, output_padding, + allow_neg_padding=False, + ) + padding_input = dilation * (kernel_size - 1) - padding + kernel_size = 1 + (kernel_size - 1) * dilation + + # *********************************************************************** ** + # Inverse of the operation of `forward_convtranspose2d` + rf_rad = (kernel_size - 1) / 2 + tl0 = ( + fp1.tl + + fp1.pxlrvec * padding_input[0] + + fp1.pxtbvec * padding_input[1] + - fp1.pxlrvec * rf_rad[0] + - fp1.pxtbvec * rf_rad[1] + ) + + # *********************************************************************** ** + # Inverse of the operation of `forward_convtranspose2d` + rsize_inner = fp1.rsize + (kernel_size - 1) - output_padding + rsize0 = (rsize_inner - padding_input * 2 + (stride - 1)) / stride + + # *********************************************************************** ** + aff = fp1._aff + aff = aff * affine.Affine.scale(*stride) + _, dx, rx, _, ry, dy = aff.to_gdal() + if np.all(stride == 1): + assert np.all(aff.to_gdal() == fp1.gt), (aff.to_gdal(), fp1.gt) + + # *********************************************************************** ** + return Footprint( + gt=(tl0[0], dx, rx, tl0[1], ry, dy), + rsize=rsize0, + ) + # The end *********************************************************************************** ** # ******************************************************************************************* ** @@ -2259,3 +2685,24 @@ def _angle_between(a, b, c): (a - b) / np.linalg.norm(a - b), (c - b) / np.linalg.norm(c - b), )) / np.pi * 180. + +def _parse_conv2d_params(*args, allow_neg_padding=True): + for k, v in zip(['kernel_size', 'stride', 'padding', 'dilation', 'output_padding'], args): + v = np.asarray(v).flatten() + if v.size == 1: + v = np.asarray((v[0], v[0])) + if v.size != 2: # pragma: no cover + raise ValueError('{} should have size 1 or 2'.format(k)) + w = v.astype(int, copy=False) + if np.any(v != w): # pragma: no cover + raise ValueError('{} should be of type int'.format(k)) + if 'padding' not in k: # pragma: no cover + if np.any(v < 1): + raise ValueError('{} should be greater or equal to 1'.format(k)) + if 'padding' in k and not allow_neg_padding: # pragma: no cover + if np.any(v < 0): + raise ValueError('{} should be greater or equal to 0'.format(k)) + + # Flip all params to work with `xy` and `rsize` (instead of `yx` and `shape`) + v = np.flipud(v) + yield v diff --git a/buzzard/test/test_footprint_convs.py b/buzzard/test/test_footprint_convs.py new file mode 100644 index 0000000..6eff728 --- /dev/null +++ b/buzzard/test/test_footprint_convs.py @@ -0,0 +1,308 @@ +# pylint: disable=redefined-outer-name, unused-argument + +import itertools + +import pytest +import buzzard as buzz +import numpy as np + +try: + import torch +except ImportError: + pytest.skip('No pytorch', allow_module_level=True) + +@pytest.fixture(scope="module", autouse=True) +def env(): + np.set_printoptions(linewidth=400, threshold=np.iinfo('int64').max, suppress=True) + with buzz.Env(allow_complex_footprint=1): + yield + +def pytest_generate_tests(metafunc): + return _pytest_generate_tests(metafunc) + +@buzz.Env(allow_complex_footprint=1) +def _pytest_generate_tests(metafunc): + + # *********************************************************************** ** + scales = [ + (1, 1), + (1, -1), + (2, 3), + ] + rots = [ + 0, + 90, 180, -135, + ] + if 'output_padding' in metafunc.fixturenames: + rsizes = [ + (4, 5), + (6, 7), + ] + else: + rsizes = [ + (15, 16), + (17, 18), + ] + src_fp = buzz.Footprint(rsize=(1, 1), gt=(100, 1, 0, 100, 0, 1)) + confs = list(itertools.product(scales, rots, rsizes)) + fp0s = [] + + for (scalex, scaley), rot, rsize in confs: + fp = src_fp.dilate(rsize[0] * 3).intersection( + src_fp.dilate(rsize[0] * 3), + rotation=rot, + scale=src_fp.scale * [scalex, scaley] + ).clip(0, 0, *rsize) + assert np.allclose(fp.scale, (scalex, scaley)) + assert np.allclose((fp.angle + 360) % 360, (rot + 360) % 360) + assert np.all(fp.rsize == rsize) + fp0s.append(fp) + + # *********************************************************************** ** + kernel_sizes = np.asarray([ + (1, 2), + (3, 4), + ]) + + # *********************************************************************** ** + strides = np.asarray([ + (1, 2), + (3, 4), + ]) + + # *********************************************************************** ** + if 'output_padding' in metafunc.fixturenames: + paddings = np.asarray([ + (0, 1), + (2, 3), + ]) + else: + paddings = np.asarray([ + (0, 1), + (2, 3), + ]) + + # *********************************************************************** ** + dilations = np.asarray([ + (1, 1), + (2, 3), + ]) + + # *********************************************************************** ** + output_paddings = np.asarray([ + (0, 1), + (2, 3), + ]) + + # *********************************************************************** ** + if 'output_padding' in metafunc.fixturenames: + tests = list(itertools.product( + fp0s, kernel_sizes, strides, paddings, dilations, output_paddings, + )) + tests = [ + (fp0, kernel_size, stride, padding, dilation, output_padding) + for fp0, kernel_size, stride, padding, dilation, output_padding in tests + if np.all(output_padding < kernel_size) and np.all(output_padding < dilation) + ] + metafunc.parametrize( + argnames='fp0,kernel_size,stride,padding,dilation,output_padding', + argvalues=tests, + ) + else: + metafunc.parametrize( + argnames='fp0,kernel_size,stride,padding,dilation', + argvalues=list(itertools.product( + fp0s, kernel_sizes, strides, paddings, dilations, + )), + ) + +def test_conv2d(fp0, kernel_size, stride, padding, dilation): + # Test forward against meshgrid computed using pytorch ****************** ** + fp1 = fp0.forward_conv2d(kernel_size, stride, padding, dilation) + truth = meshgrid_of_forward_conv2d(fp0, kernel_size, stride, padding, dilation) + assert np.allclose(truth, fp1.meshgrid_spatial) + + # Test backward against forward ***************************************** ** + # (modulo small bottom-right overflows caused by stride and fp1.rsize) + fp0_bis = fp1.backward_conv2d(kernel_size, stride, padding, dilation) + + angle = (fp0.angle + 360) % 90 + if np.allclose(np.cos(angle * 4), 1): + # is orthogonal + assert np.all(fp0.gt == fp0_bis.gt) + else: + assert np.allclose(fp0.gt, fp0_bis.gt) + + shape_overflow = fp0.shape - fp0_bis.shape + assert np.all(shape_overflow >= 0) + assert np.all(shape_overflow < stride) + +def test_convtranspose2d(fp0, kernel_size, stride, padding, dilation, output_padding): + # Test forward against meshgrid computed with pytorch ******************* ** + # (modulo the meshgrid values at output_padding) + fp1 = fp0.forward_convtranspose2d(kernel_size, stride, padding, dilation, output_padding) + truth = meshgrid_of_forward_convtranspose2d( + fp0, kernel_size, stride, padding, dilation, output_padding, + ) + fromfp = np.asarray(fp1.meshgrid_spatial) + fromfp = fromfp[:, :fp1.shape[0] - output_padding[0], :fp1.shape[1] - output_padding[1]] + assert np.allclose(truth, fromfp) + + # Test backward against forward ***************************************** ** + fp0_bis = fp1.backward_convtranspose2d(kernel_size, stride, padding, dilation, output_padding) + + angle = (fp0.angle + 360) % 90 + if np.allclose(np.cos(angle * 4), 1): + # is orthogonal + assert fp0 == fp0_bis + else: + assert fp0.almost_equals(fp0_bis) + +# *********************************************************************************************** ** +# Utils ***************************************************************************************** ** +def meshgrid_of_forward_conv2d(fp, kernel_size, stride, padding, dilation): + fp_padded = fp.dilate(*np.flipud(padding)) + + c = torch.nn.Conv2d( + in_channels=2, out_channels=2, groups=2, bias=False, padding=0, + kernel_size=kernel_size, stride=stride.tolist(), dilation=dilation.tolist(), + ) + c.weight = torch.nn.Parameter( + torch.ones_like(c.weight) / np.prod(kernel_size), + requires_grad=False, + ) + + x = np.asarray(fp_padded.meshgrid_spatial).reshape(1, 2, *fp_padded.shape).astype('float32') + with torch.no_grad(): + y = c(torch.from_numpy(x)).numpy() + return y.reshape(2, *y.shape[2:]) + +def create_bilinear_kernel(stride, dtype=None): + # Parameters ************************************************************ ** + v = stride + v = np.asarray(v).reshape(-1) + if v.size == 1: + v = np.asarray((v, v)) + if v.size != 2: + raise ValueError('{} should have size 1 or 2'.format(k)) + w = v.astype(int, copy=False) + if np.any(v != w): + raise ValueError('{} should be of type int'.format(k)) + if np.any(v < 1): + raise ValueError('{} should be greater or equal to 1'.format(k)) + stride = v + + # Build kernel ********************************************************** ** + shape = (stride - 1) * 2 + 1 + center = stride - (shape % 2 + 1) / 2 + kernel = np.zeros(shape, dtype='float32') + for yx in np.ndindex(*shape): + val = 1 - np.abs((yx - center) / stride) + kernel[yx[0], yx[1]] = np.prod(val) + + # Assert correctness **************************************************** ** + indices = np.asarray(list(np.ndindex(*kernel.shape))) + for i in range(stride[0]): + for j in range(stride[1]): + mask = (indices % stride == (i, j)).all(axis=1) + facts = kernel.reshape(-1)[mask] + assert np.allclose(facts.sum(), 1) + + return kernel + +def inner_meshgrid_of_forward_convtranspose2d(fp, stride): + kernel = create_bilinear_kernel(stride) + kernel_shape = np.asarray(kernel.shape) + assert np.all(kernel_shape % 2 == 1) + padding_input = np.asarray((kernel_shape - 1) // 2) + padding_parameter = (kernel_shape - 1) - padding_input + + # Assert that the `padding_parameter` is good *************************** ** + c = torch.nn.ConvTranspose2d( + in_channels=1, out_channels=1, groups=1, bias=False, padding=padding_parameter.tolist(), + kernel_size=kernel.shape, stride=stride.tolist(), + ) + c.weight = torch.nn.Parameter( + torch.ones_like(c.weight) * torch.from_numpy(kernel), requires_grad=False + ) + x = np.ones(shape=(1, 1, *fp.shape), dtype='float32') + with torch.no_grad(): + y = c(torch.from_numpy(x)).numpy() + assert np.allclose(y, 1) + del x, y + + # Compute the meshgrid ************************************************** ** + c = torch.nn.ConvTranspose2d( + in_channels=2, out_channels=2, groups=2, bias=False, padding=padding_parameter.tolist(), + kernel_size=kernel.shape, stride=stride.tolist(), + ) + c.weight = torch.nn.Parameter( + torch.from_numpy(np.stack([kernel, kernel])[:, None, :, :]), requires_grad=False + ) + + x = np.asarray(fp.meshgrid_spatial).reshape(1, 2, *fp.shape).astype('float32') + with torch.no_grad(): + y = c(torch.from_numpy(x)).numpy() + return y[0] + +def meshgrid_of_forward_convtranspose2d(fp, kernel_size, stride, padding, dilation, output_padding): + dilation_parameter = dilation + kernel_size_parameter = kernel_size + padding_parameter = padding + del padding, dilation + + kernel_size = 1 + (kernel_size_parameter - 1) * dilation_parameter + padding_input = dilation_parameter * (kernel_size_parameter - 1) - padding_parameter + todilate_input = np.ceil(padding_input / stride).astype(int) + tocrop_inner_meshgrid = todilate_input * stride - padding_input + fp_padded = fp.dilate(*np.flipud(todilate_input)) + + # Create inner meshgrid ************************************************* ** + # First create a larger one and then crop what's needed + mg = inner_meshgrid_of_forward_convtranspose2d(fp_padded, stride) + mg = mg[ + :, + tocrop_inner_meshgrid[0]:mg.shape[1] - tocrop_inner_meshgrid[0], + tocrop_inner_meshgrid[1]:mg.shape[2] - tocrop_inner_meshgrid[1], + ] + + # Apply conv2d on inner meshgrid **************************************** ** + c = torch.nn.Conv2d( + in_channels=2, out_channels=2, groups=2, bias=False, padding=0, stride=1, dilation=1, + kernel_size=kernel_size, + ) + c.weight = torch.nn.Parameter( + torch.ones_like(c.weight) / np.prod(kernel_size), + requires_grad=False, + ) + + x = mg[None, :, :, :] + with torch.no_grad(): + y = c(torch.from_numpy(x)).numpy() + mg = y[0] + + # Check meshgrid shape ************************************************** ** + truth_shape = output_shape_of_conv2dtranspose( + fp, + kernel_size=kernel_size_parameter, stride=stride, + padding=padding_parameter, dilation=dilation_parameter, + output_padding=output_padding, + ) + found_shape = mg.shape[1:] + found_shape = tuple(found_shape + output_padding) + assert found_shape == truth_shape + + return mg + +def output_shape_of_conv2dtranspose(fp, **kwargs): + c = torch.nn.ConvTranspose2d( + in_channels=1, out_channels=1, groups=1, bias=False, + **{ + k: v.tolist() + for k, v in kwargs.items() + }, + ) + x = np.asarray(fp.meshgrid_spatial[0]).reshape(1, 1, *fp.shape).astype('float32') + with torch.no_grad(): + y = c(torch.from_numpy(x)).numpy() + return y.shape[2:] From 7dfad96b1418fc3efd9f52df36aa4326d588162a Mon Sep 17 00:00:00 2001 From: Ngoguey42 Date: Thu, 13 Aug 2020 11:27:16 +0200 Subject: [PATCH 2/2] Improve comments and documentation --- buzzard/_footprint.py | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/buzzard/_footprint.py b/buzzard/_footprint.py index 0108d20..12d8c06 100644 --- a/buzzard/_footprint.py +++ b/buzzard/_footprint.py @@ -2301,8 +2301,8 @@ def __hash__(self): def forward_conv2d(self, kernel_size, stride=1, padding=0, dilation=1): """Shift, scale and dilate the Footprint as if it went throught a 2d convolution kernel. - The arithmetic followed is the one from `pytorch`, but the arithmetics in other - deep-learning libraries are mostly the same. + The arithmetic followed is the one from `pytorch`, but other deep-learning libraries mostly + follow the same arithmetic. This function is a `many to one` mapping, two footprints with different `rsizes` can produce the same Footprint when `stride > 1`. @@ -2341,16 +2341,16 @@ def forward_conv2d(self, kernel_size, stride=1, padding=0, dilation=1): # *********************************************************************** ** # rf_rad: Receptive field radius (2,) # pxlrvec: Pixel Left-Right Vector (2,) - # pxtbvev: Pixel Top-Bottprint Vector (2,) + # pxtbvev: Pixel Top-Bottom Vector (2,) rf_rad = (kernel_size - 1) / 2 tl1 = ( fp0.tl - # A padding shift toward top-left + # Padding shifts toward top-left - fp0.pxlrvec * padding[0] - fp0.pxtbvec * padding[1] - # A convolution kernel shift toward bottom-left + # Kernel shifts toward bottom-right + fp0.pxlrvec * rf_rad[0] + fp0.pxtbvec * rf_rad[1] ) @@ -2389,8 +2389,8 @@ def backward_conv2d(self, kernel_size, stride=1, padding=0, dilation=1): """Shift, scale and dilate the Footprint as if it went backward throught a 2d convolution kernel. - The arithmetic followed is the one from `pytorch`, but the arithmetics in other - deep-learning libraries are mostly the same. + The arithmetic followed is the one from `pytorch`, but other deep-learning libraries mostly + follow the same arithmetic. This function is a `one to one` mapping, two different input footprints will produce two different output Footprints. It means that the `backward_conv2d` of a `forward_conv2d` may @@ -2433,10 +2433,8 @@ def backward_conv2d(self, kernel_size, stride=1, padding=0, dilation=1): rf_rad = (kernel_size - 1) / 2 tl0 = ( fp1.tl - + fp1.pxlrvec / stride[0] * padding[0] + fp1.pxtbvec / stride[1] * padding[1] - - fp1.pxlrvec / stride[0] * rf_rad[0] - fp1.pxtbvec / stride[1] * rf_rad[1] ) @@ -2463,13 +2461,13 @@ def forward_convtranspose2d(self, kernel_size, stride=1, padding=0, dilation=1, """Shift, scale and dilate the Footprint as if it went throught a 2d transposed convolution kernel. - The arithmetic followed is the one from `pytorch`, but the arithmetics in other - deep-learning libraries are mostly the same. + The arithmetic followed is the one from `pytorch`, but other deep-learning libraries mostly + follow the same arithmetic. A 2d transposed convolution has 4 internal steps: - 1. Apply stride (interleave the input pixels with zeroes) + 1. Apply stride (i.e. interleave the input pixels with zeroes) 2. Add padding - 3. Apply a 2d convolution stride:1, pad:0 + 3. Apply a 2d convolution with stride=1 and pad=0 4. Add output-padding This function is a `one to one` mapping, two different input footprints will produce two @@ -2557,8 +2555,8 @@ def backward_convtranspose2d(self, kernel_size, stride=1, padding=0, dilation=1, """Shift, scale and dilate the Footprint as if it went backward throught a 2d transposed convolution kernel. - The arithmetic followed is the one from `pytorch`, but the arithmetics in other - deep-learning libraries are mostly the same. + The arithmetic followed is the one from `pytorch`, but other deep-learning libraries mostly + follow the same arithmetic. A 2d transposed convolution has 4 internal steps: 1. Apply stride (interleave the input pixels with zeroes)