From 5b2676e281ad295f35f64ac076a22a6aeacd83bc Mon Sep 17 00:00:00 2001 From: ct Date: Wed, 13 Feb 2019 11:02:17 +0800 Subject: [PATCH 1/4] Fix #7149 Datavec: ImageLoader.scalingIfNeed may missing channel scale when size is right --- .../org/datavec/image/loader/ImageLoader.java | 100 ++++++++++-------- .../datavec/image/loader/TestImageLoader.java | 45 +++++++- 2 files changed, 97 insertions(+), 48 deletions(-) diff --git a/datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/loader/ImageLoader.java b/datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/loader/ImageLoader.java index 4eb9771611f2..7715be83a27c 100644 --- a/datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/loader/ImageLoader.java +++ b/datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/loader/ImageLoader.java @@ -36,8 +36,8 @@ /** * Image loader for taking images * and converting them to matrices - * @author Adam Gibson * + * @author Adam Gibson */ public class ImageLoader extends BaseImageLoader { @@ -50,8 +50,8 @@ public class ImageLoader extends BaseImageLoader { registry.registerServiceProvider(new com.twelvemonkeys.imageio.plugins.jpeg.JPEGImageWriterSpi()); registry.registerServiceProvider(new com.twelvemonkeys.imageio.plugins.psd.PSDImageReaderSpi()); registry.registerServiceProvider(Arrays.asList(new com.twelvemonkeys.imageio.plugins.bmp.BMPImageReaderSpi(), - new com.twelvemonkeys.imageio.plugins.bmp.CURImageReaderSpi(), - new com.twelvemonkeys.imageio.plugins.bmp.ICOImageReaderSpi())); + new com.twelvemonkeys.imageio.plugins.bmp.CURImageReaderSpi(), + new com.twelvemonkeys.imageio.plugins.bmp.ICOImageReaderSpi())); } public ImageLoader() { @@ -61,9 +61,9 @@ public ImageLoader() { /** * Instantiate an image with the given * height and width + * * @param height the height to load* * @param width the width to load - */ public ImageLoader(long height, long width) { super(); @@ -75,8 +75,9 @@ public ImageLoader(long height, long width) { /** * Instantiate an image with the given * height and width - * @param height the height to load - * @param width the width to load + * + * @param height the height to load + * @param width the width to load * @param channels the number of channels for the image* */ public ImageLoader(long height, long width, long channels) { @@ -89,9 +90,10 @@ public ImageLoader(long height, long width, long channels) { /** * Instantiate an image with the given * height and width - * @param height the height to load - * @param width the width to load - * @param channels the number of channels for the image* + * + * @param height the height to load + * @param width the width to load + * @param channels the number of channels for the image* * @param centerCropIfNeeded to crop before rescaling and converting */ public ImageLoader(long height, long width, long channels, boolean centerCropIfNeeded) { @@ -121,6 +123,7 @@ public INDArray asRowVector(InputStream inputStream) throws IOException { /** * Convert an image in to a row vector + * * @param image the image to convert * @return the row vector based on a rastered * representation of the image @@ -140,8 +143,9 @@ public INDArray asRowVector(BufferedImage image) { /** * Changes the input stream in to an * bgr based raveled(flattened) vector + * * @param file the input stream to convert - * @return the raveled bgr values for this input stream + * @return the raveled bgr values for this input stream */ public INDArray toRaveledTensor(File file) { try { @@ -157,8 +161,9 @@ public INDArray toRaveledTensor(File file) { /** * Changes the input stream in to an * bgr based raveled(flattened) vector + * * @param is the input stream to convert - * @return the raveled bgr values for this input stream + * @return the raveled bgr values for this input stream */ public INDArray toRaveledTensor(InputStream is) { return toBgr(is).ravel(); @@ -167,6 +172,7 @@ public INDArray toRaveledTensor(InputStream is) { /** * Convert an image in to a raveled tensor of * the bgr values of the image + * * @param image the image to parse * @return the raveled tensor of bgr values */ @@ -211,7 +217,7 @@ public INDArray toBgr(InputStream inputStream) { } } - private org.datavec.image.data.Image toBgrImage(InputStream inputStream){ + private org.datavec.image.data.Image toBgrImage(InputStream inputStream) { try { BufferedImage image = ImageIO.read(inputStream); INDArray img = toBgr(image); @@ -237,6 +243,7 @@ public INDArray toBgr(BufferedImage image) { /** * Convert an image file * in to a matrix + * * @param f the file to convert * @return a 2d matrix of a rastered version of the image * @throws IOException @@ -247,6 +254,7 @@ public INDArray asMatrix(File f) throws IOException { /** * Convert an input stream to a matrix + * * @param inputStream the input stream to convert * @return the input stream to convert */ @@ -283,6 +291,7 @@ public org.datavec.image.data.Image asImageMatrix(InputStream inputStream) throw /** * Convert an BufferedImage to a matrix + * * @param image the BufferedImage to convert * @return the input stream to convert */ @@ -297,7 +306,7 @@ public INDArray asMatrix(BufferedImage image) { for (int i = 0; i < h; i++) { for (int j = 0; j < w; j++) { - ret.putScalar(new int[] {i, j}, image.getRGB(j, i)); + ret.putScalar(new int[]{i, j}, image.getRGB(j, i)); } } return ret; @@ -307,8 +316,8 @@ public INDArray asMatrix(BufferedImage image) { /** * Slices up an image in to a mini batch. * - * @param f the file to load from - * @param numMiniBatches the number of images in a mini batch + * @param f the file to load from + * @param numMiniBatches the number of images in a mini batch * @param numRowsPerSlice the number of rows for each image * @return a tensor representing one image as a mini batch */ @@ -327,6 +336,7 @@ public int[] flattenedImageFromFile(File f) throws IOException { /** * Load a rastered image from file + * * @param file the file to load * @return the rastered image * @throws IOException @@ -339,6 +349,7 @@ public int[][] fromFile(File file) throws IOException { /** * Load a rastered image from file + * * @param file the file to load * @return the rastered image * @throws IOException @@ -349,9 +360,9 @@ public int[][][] fromFileMultipleChannels(File file) throws IOException { int w = image.getWidth(), h = image.getHeight(); int bands = image.getSampleModel().getNumBands(); - int[][][] ret = new int[(int)Math.min(channels, Integer.MAX_VALUE)] - [(int)Math.min(h, Integer.MAX_VALUE)] - [(int)Math.min(w, Integer.MAX_VALUE)]; + int[][][] ret = new int[(int) Math.min(channels, Integer.MAX_VALUE)] + [(int) Math.min(h, Integer.MAX_VALUE)] + [(int) Math.min(w, Integer.MAX_VALUE)]; byte[] pixels = ((DataBufferByte) image.getRaster().getDataBuffer()).getData(); for (int i = 0; i < h; i++) { @@ -359,7 +370,7 @@ public int[][][] fromFileMultipleChannels(File file) throws IOException { for (int k = 0; k < channels; k++) { if (k >= bands) break; - ret[k][i][j] = pixels[(int)Math.min(channels * w * i + channels * j + k, Integer.MAX_VALUE)]; + ret[k][i][j] = pixels[(int) Math.min(channels * w * i + channels * j + k, Integer.MAX_VALUE)]; } } } @@ -368,6 +379,7 @@ public int[][][] fromFileMultipleChannels(File file) throws IOException { /** * Convert a matrix in to a buffereed image + * * @param matrix the * @return {@link java.awt.image.BufferedImage} */ @@ -393,7 +405,8 @@ private static int[] rasterData(INDArray matrix) { /** * Convert the given image to an rgb image - * @param arr the array to use + * + * @param arr the array to use * @param image the image to set */ public void toBufferedImageRGB(INDArray arr, BufferedImage image) { @@ -416,12 +429,12 @@ public void toBufferedImageRGB(INDArray arr, BufferedImage image) { /** * Converts a given Image into a BufferedImage * - * @param img The Image to be converted + * @param img The Image to be converted * @param type The color model of BufferedImage * @return The converted BufferedImage */ public static BufferedImage toBufferedImage(Image img, int type) { - if (img instanceof BufferedImage) { + if (img instanceof BufferedImage && ((BufferedImage) img).getType() == type) { return (BufferedImage) img; } @@ -463,7 +476,7 @@ protected INDArray toINDArrayBGR(BufferedImage image) { int bands = image.getSampleModel().getNumBands(); byte[] pixels = ((DataBufferByte) image.getRaster().getDataBuffer()).getData(); - int[] shape = new int[] {height, width, bands}; + int[] shape = new int[]{height, width, bands}; INDArray ret2 = Nd4j.create(1, pixels.length); for (int i = 0; i < ret2.length(); i++) { @@ -491,32 +504,29 @@ public BufferedImage centerCropIfNeeded(BufferedImage img) { } protected BufferedImage scalingIfNeed(BufferedImage image, boolean needAlpha) { - return scalingIfNeed(image, height, width, needAlpha); + return scalingIfNeed(image, height, width, channels, needAlpha); } - protected BufferedImage scalingIfNeed(BufferedImage image, long dstHeight, long dstWidth, boolean needAlpha) { + protected static BufferedImage scalingIfNeed(BufferedImage image, long dstHeight, long dstWidth, long dstImageType, boolean needAlpha) { + Image scaled; + // Scale width and height first if necessary if (dstHeight > 0 && dstWidth > 0 && (image.getHeight() != dstHeight || image.getWidth() != dstWidth)) { - Image scaled = image.getScaledInstance((int) dstWidth, (int) dstHeight, Image.SCALE_SMOOTH); - - if (needAlpha && image.getColorModel().hasAlpha() && channels == BufferedImage.TYPE_4BYTE_ABGR) { - return toBufferedImage(scaled, BufferedImage.TYPE_4BYTE_ABGR); - } else { - if (channels == BufferedImage.TYPE_BYTE_GRAY) - return toBufferedImage(scaled, BufferedImage.TYPE_BYTE_GRAY); - else - return toBufferedImage(scaled, BufferedImage.TYPE_3BYTE_BGR); - } + scaled = image.getScaledInstance((int) dstWidth, (int) dstHeight, Image.SCALE_SMOOTH); } else { - if (image.getType() == BufferedImage.TYPE_4BYTE_ABGR || image.getType() == BufferedImage.TYPE_3BYTE_BGR) { - return image; - } else if (needAlpha && image.getColorModel().hasAlpha() && channels == BufferedImage.TYPE_4BYTE_ABGR) { - return toBufferedImage(image, BufferedImage.TYPE_4BYTE_ABGR); - } else { - if (channels == BufferedImage.TYPE_BYTE_GRAY) - return toBufferedImage(image, BufferedImage.TYPE_BYTE_GRAY); - else - return toBufferedImage(image, BufferedImage.TYPE_3BYTE_BGR); - } + scaled = image; + } + + // Transfer imageType if necessary and transfer to BufferedImage. + if (scaled instanceof BufferedImage && ((BufferedImage) scaled).getType() == dstImageType) { + return (BufferedImage) scaled; + } + if (needAlpha && image.getColorModel().hasAlpha() && dstImageType == BufferedImage.TYPE_4BYTE_ABGR) { + return toBufferedImage(scaled, BufferedImage.TYPE_4BYTE_ABGR); + } else { + if (dstImageType == BufferedImage.TYPE_BYTE_GRAY) + return toBufferedImage(scaled, BufferedImage.TYPE_BYTE_GRAY); + else + return toBufferedImage(scaled, BufferedImage.TYPE_3BYTE_BGR); } } diff --git a/datavec/datavec-data/datavec-data-image/src/test/java/org/datavec/image/loader/TestImageLoader.java b/datavec/datavec-data/datavec-data-image/src/test/java/org/datavec/image/loader/TestImageLoader.java index b8ed1b3a4b2a..0dc4636b316b 100644 --- a/datavec/datavec-data/datavec-data-image/src/test/java/org/datavec/image/loader/TestImageLoader.java +++ b/datavec/datavec-data/datavec-data-image/src/test/java/org/datavec/image/loader/TestImageLoader.java @@ -128,6 +128,27 @@ public void testScalingIfNeed() throws Exception { } + @Test + public void testScalingIfNeed_suitable_size_diff_channel() { + int width1 = 60, height1 = 110, channel1 = BufferedImage.TYPE_BYTE_GRAY; + BufferedImage img1 = makeRandomBufferedImage(true, width1, height1); + ImageLoader loader1 = new ImageLoader(height1, width1, channel1); + BufferedImage scaled1 = loader1.scalingIfNeed(img1, false); + assertEquals(width1, scaled1.getWidth()); + assertEquals(height1, scaled1.getHeight()); + assertEquals(channel1, scaled1.getType()); + assertEquals(1, scaled1.getSampleModel().getNumBands()); + + int width2 = 70, height2 = 120, channel2 = BufferedImage.TYPE_BYTE_GRAY; + BufferedImage img2 = makeRandomBufferedImage(false, width2, height2); + ImageLoader loader2 = new ImageLoader(height2, width2, channel2); + BufferedImage scaled2 = loader2.scalingIfNeed(img2, false); + assertEquals(width2, scaled2.getWidth()); + assertEquals(height2, scaled2.getHeight()); + assertEquals(channel2, scaled2.getType()); + assertEquals(1, scaled2.getSampleModel().getNumBands()); + } + @Test public void testToBufferedImageRGB() { BufferedImage img = makeRandomBufferedImage(false); @@ -150,9 +171,17 @@ public void testToBufferedImageRGB() { } - private BufferedImage makeRandomBufferedImage(boolean alpha) { - int w = rng.nextInt() % 100 + 100; - int h = rng.nextInt() % 100 + 100; + /** + * Generate a Random BufferedImage with specified width and height + * + * @param alpha Is image alpha + * @param width Proposed width + * @param height Proposed height + * @return Generated BufferedImage + */ + private BufferedImage makeRandomBufferedImage(boolean alpha, int width, int height) { + int w = width > 0 ? width : (rng.nextInt() % 100 + 100); + int h = height > 0 ? height : (rng.nextInt() % 100 + 100); int type = alpha ? BufferedImage.TYPE_4BYTE_ABGR : BufferedImage.TYPE_3BYTE_BGR; BufferedImage img = new BufferedImage(w, h, type); for (int i = 0; i < h; ++i) { @@ -167,4 +196,14 @@ private BufferedImage makeRandomBufferedImage(boolean alpha) { } return img; } + + /** + * Generate a Random BufferedImage with random width and height + * + * @param alpha Is image alpha + * @return Generated BufferedImage + */ + private BufferedImage makeRandomBufferedImage(boolean alpha) { + return makeRandomBufferedImage(alpha, -1, -1); + } } From 6ca698018b48b40fad849d4aa63967ce3e1a7325 Mon Sep 17 00:00:00 2001 From: ct Date: Wed, 13 Feb 2019 14:43:09 +0800 Subject: [PATCH 2/4] Fixed the compile error on toBufferedImageRGB --- .../src/main/java/org/datavec/image/loader/ImageLoader.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/loader/ImageLoader.java b/datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/loader/ImageLoader.java index 7715be83a27c..3a9461b5fc96 100644 --- a/datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/loader/ImageLoader.java +++ b/datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/loader/ImageLoader.java @@ -413,7 +413,7 @@ public void toBufferedImageRGB(INDArray arr, BufferedImage image) { if (arr.rank() < 3) throw new IllegalArgumentException("Arr must be 3d"); - image = scalingIfNeed(image, arr.size(-2), arr.size(-1), true); + image = scalingIfNeed(image, arr.size(-2), arr.size(-1), image.getType(), true); for (int i = 0; i < image.getHeight(); i++) { for (int j = 0; j < image.getWidth(); j++) { int r = arr.slice(2).getInt(i, j); From c70912ed76192e9b7670f5d112aeb307d7c9b283 Mon Sep 17 00:00:00 2001 From: ct Date: Fri, 15 Feb 2019 15:50:11 +0800 Subject: [PATCH 3/4] Fixed the Codacy Issues --- .../datavec/image/loader/TestImageLoader.java | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/datavec/datavec-data/datavec-data-image/src/test/java/org/datavec/image/loader/TestImageLoader.java b/datavec/datavec-data/datavec-data-image/src/test/java/org/datavec/image/loader/TestImageLoader.java index 0dc4636b316b..1683980f0acb 100644 --- a/datavec/datavec-data/datavec-data-image/src/test/java/org/datavec/image/loader/TestImageLoader.java +++ b/datavec/datavec-data/datavec-data-image/src/test/java/org/datavec/image/loader/TestImageLoader.java @@ -129,8 +129,10 @@ public void testScalingIfNeed() throws Exception { } @Test - public void testScalingIfNeed_suitable_size_diff_channel() { - int width1 = 60, height1 = 110, channel1 = BufferedImage.TYPE_BYTE_GRAY; + public void testScalingIfNeedWhenSuitableSizeButDiffChannel() { + int width1 = 60; + int height1 = 110; + int channel1 = BufferedImage.TYPE_BYTE_GRAY; BufferedImage img1 = makeRandomBufferedImage(true, width1, height1); ImageLoader loader1 = new ImageLoader(height1, width1, channel1); BufferedImage scaled1 = loader1.scalingIfNeed(img1, false); @@ -139,7 +141,9 @@ public void testScalingIfNeed_suitable_size_diff_channel() { assertEquals(channel1, scaled1.getType()); assertEquals(1, scaled1.getSampleModel().getNumBands()); - int width2 = 70, height2 = 120, channel2 = BufferedImage.TYPE_BYTE_GRAY; + int width2 = 70; + int height2 = 120; + int channel2 = BufferedImage.TYPE_BYTE_GRAY; BufferedImage img2 = makeRandomBufferedImage(false, width2, height2); ImageLoader loader2 = new ImageLoader(height2, width2, channel2); BufferedImage scaled2 = loader2.scalingIfNeed(img2, false); @@ -180,12 +184,10 @@ public void testToBufferedImageRGB() { * @return Generated BufferedImage */ private BufferedImage makeRandomBufferedImage(boolean alpha, int width, int height) { - int w = width > 0 ? width : (rng.nextInt() % 100 + 100); - int h = height > 0 ? height : (rng.nextInt() % 100 + 100); int type = alpha ? BufferedImage.TYPE_4BYTE_ABGR : BufferedImage.TYPE_3BYTE_BGR; - BufferedImage img = new BufferedImage(w, h, type); - for (int i = 0; i < h; ++i) { - for (int j = 0; j < w; ++j) { + BufferedImage img = new BufferedImage(width, height, type); + for (int i = 0; i < height; ++i) { + for (int j = 0; j < width; ++j) { int a = (alpha ? rng.nextInt() : 1) & 0xff; int r = rng.nextInt() & 0xff; int g = rng.nextInt() & 0xff; @@ -204,6 +206,6 @@ private BufferedImage makeRandomBufferedImage(boolean alpha, int width, int heig * @return Generated BufferedImage */ private BufferedImage makeRandomBufferedImage(boolean alpha) { - return makeRandomBufferedImage(alpha, -1, -1); + return makeRandomBufferedImage(alpha, rng.nextInt() % 100 + 100, rng.nextInt() % 100 + 100); } } From bfc7b58c7f2a8c37064b8006825914227b40bbb3 Mon Sep 17 00:00:00 2001 From: Samuel Audet Date: Tue, 19 Feb 2019 11:48:15 +0900 Subject: [PATCH 4/4] Remove static from scalingIfNeed() --- .../src/main/java/org/datavec/image/loader/ImageLoader.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/loader/ImageLoader.java b/datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/loader/ImageLoader.java index 3a9461b5fc96..27c0c019decb 100644 --- a/datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/loader/ImageLoader.java +++ b/datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/loader/ImageLoader.java @@ -507,7 +507,7 @@ protected BufferedImage scalingIfNeed(BufferedImage image, boolean needAlpha) { return scalingIfNeed(image, height, width, channels, needAlpha); } - protected static BufferedImage scalingIfNeed(BufferedImage image, long dstHeight, long dstWidth, long dstImageType, boolean needAlpha) { + protected BufferedImage scalingIfNeed(BufferedImage image, long dstHeight, long dstWidth, long dstImageType, boolean needAlpha) { Image scaled; // Scale width and height first if necessary if (dstHeight > 0 && dstWidth > 0 && (image.getHeight() != dstHeight || image.getWidth() != dstWidth)) {