From e1e6d735e27606f3c5477820d390760637a0108a Mon Sep 17 00:00:00 2001 From: Pranav Toggi Date: Thu, 8 Feb 2024 18:21:16 -0500 Subject: [PATCH 1/6] Update default behavior of RS_NormalizeAll --- .../sedona/common/raster/MapAlgebra.java | 24 +++++++++++-------- .../sedona/common/raster/MapAlgebraTest.java | 20 ++++++++++++---- docs/api/sql/Raster-operators.md | 12 ++++++---- 3 files changed, 36 insertions(+), 20 deletions(-) diff --git a/common/src/main/java/org/apache/sedona/common/raster/MapAlgebra.java b/common/src/main/java/org/apache/sedona/common/raster/MapAlgebra.java index bc73c82d1d..f0ba29ca8e 100644 --- a/common/src/main/java/org/apache/sedona/common/raster/MapAlgebra.java +++ b/common/src/main/java/org/apache/sedona/common/raster/MapAlgebra.java @@ -443,23 +443,23 @@ public static double[] normalize(double[] bandValues) { } public static GridCoverage2D normalizeAll(GridCoverage2D rasterGeom) { - return normalizeAll(rasterGeom, 0d, 255d, null, null, null, true); + return normalizeAll(rasterGeom, 0d, 255d, true, null, null, null); } public static GridCoverage2D normalizeAll(GridCoverage2D rasterGeom, double minLim, double maxLim) { - return normalizeAll(rasterGeom, minLim, maxLim, null, null, null, true); + return normalizeAll(rasterGeom, minLim, maxLim, true, null, null, null); } - public static GridCoverage2D normalizeAll(GridCoverage2D rasterGeom, double minLim, double maxLim, double noDataValue) { - return normalizeAll(rasterGeom, minLim, maxLim, noDataValue, null, null, true); + public static GridCoverage2D normalizeAll(GridCoverage2D rasterGeom, double minLim, double maxLim, boolean normalizeAcrossBands) { + return normalizeAll(rasterGeom, minLim, maxLim, normalizeAcrossBands, null, null, null); } - public static GridCoverage2D normalizeAll(GridCoverage2D rasterGeom, double minLim, double maxLim, Double noDataValue, boolean normalizeAcrossBands) { - return normalizeAll(rasterGeom, minLim, maxLim, noDataValue, null, null, normalizeAcrossBands); + public static GridCoverage2D normalizeAll(GridCoverage2D rasterGeom, double minLim, double maxLim, boolean normalizeAcrossBands, Double noDataValue) { + return normalizeAll(rasterGeom, minLim, maxLim, normalizeAcrossBands, noDataValue, null, null); } public static GridCoverage2D normalizeAll(GridCoverage2D rasterGeom, double minLim, double maxLim, Double noDataValue, Double minValue, Double maxValue) { - return normalizeAll(rasterGeom, minLim, maxLim, noDataValue, minValue, maxValue, true); + return normalizeAll(rasterGeom, minLim, maxLim, true, noDataValue, minValue, maxValue); } /** @@ -467,13 +467,13 @@ public static GridCoverage2D normalizeAll(GridCoverage2D rasterGeom, double minL * @param rasterGeom Raster to be normalized * @param minLim Lower limit of normalization range * @param maxLim Upper limit of normalization range + * @param normalizeAcrossBands flag to determine the normalization method * @param noDataValue NoDataValue used in raster * @param minValue Minimum value in raster * @param maxValue Maximum value in raster - * @param normalizeAcrossBands flag to determine the normalization method * @return a raster with all values in all bands normalized between minLim and maxLim */ - public static GridCoverage2D normalizeAll(GridCoverage2D rasterGeom, double minLim, double maxLim, Double noDataValue, Double minValue, Double maxValue, boolean normalizeAcrossBands) { + public static GridCoverage2D normalizeAll(GridCoverage2D rasterGeom, double minLim, double maxLim, boolean normalizeAcrossBands, Double noDataValue, Double minValue, Double maxValue) { if (minLim > maxLim) { throw new IllegalArgumentException("minLim cannot be greater than maxLim"); } @@ -491,6 +491,10 @@ public static GridCoverage2D normalizeAll(GridCoverage2D rasterGeom, double minL Arrays.fill(minValues, Double.MAX_VALUE); Arrays.fill(maxValues, -Double.MAX_VALUE); + // Trigger safe mode if noDataValue is null - noDataValue is set to maxLim and data values are normalized to range [minLim, maxLim-1]. + // This is done to prevent setting valid data as noDataValue. + double safetyTrigger = (noDataValue == null) ? 1 : 0; + // Compute global min and max values across all bands if necessary and not provided if (minValue == null || maxValue == null) { for (int bandIndex = 0; bandIndex < numBands; bandIndex++) { @@ -530,7 +534,7 @@ public static GridCoverage2D normalizeAll(GridCoverage2D rasterGeom, double minL } else { for (int i = 0; i < bandValues.length; i++) { if (bandValues[i] != bandNoDataValue) { - double normalizedValue = minLim + ((bandValues[i] - currentMin) * (maxLim - minLim)) / (currentMax - currentMin); + double normalizedValue = minLim + ((bandValues[i] - currentMin) * (maxLim - safetyTrigger - minLim)) / (currentMax - currentMin); bandValues[i] = castRasterDataType(normalizedValue, rasterDataType); } else { bandValues[i] = noDataValue; diff --git a/common/src/test/java/org/apache/sedona/common/raster/MapAlgebraTest.java b/common/src/test/java/org/apache/sedona/common/raster/MapAlgebraTest.java index 5fc21549e0..354eafda57 100644 --- a/common/src/test/java/org/apache/sedona/common/raster/MapAlgebraTest.java +++ b/common/src/test/java/org/apache/sedona/common/raster/MapAlgebraTest.java @@ -362,13 +362,15 @@ public void testNormalizeAll() throws FactoryException { raster3 = RasterBandEditors.setBandNoDataValue(raster3, 1, 16.0); raster3 = RasterBandEditors.setBandNoDataValue(raster3, 2, 1.0); - GridCoverage2D normalizedRaster1 = MapAlgebra.normalizeAll(raster1, 0, 255, -9999.0, false); - GridCoverage2D normalizedRaster2 = MapAlgebra.normalizeAll(raster1, 256d, 511d, -9999.0, false); + GridCoverage2D normalizedRaster1 = MapAlgebra.normalizeAll(raster1, 0, 255, false, -9999.0); + GridCoverage2D normalizedRaster2 = MapAlgebra.normalizeAll(raster1, 256d, 511d, false, -9999.0); GridCoverage2D normalizedRaster3 = MapAlgebra.normalizeAll(raster2); - GridCoverage2D normalizedRaster4 = MapAlgebra.normalizeAll(raster3, 0, 255, 95.0); - GridCoverage2D normalizedRaster5 = MapAlgebra.normalizeAll(raster4, 0, 255); + GridCoverage2D normalizedRaster4 = MapAlgebra.normalizeAll(raster3, 0, 255, true, 95.0); + GridCoverage2D normalizedRaster5 = MapAlgebra.normalizeAll(raster4, 0, 255, true, 255.0); GridCoverage2D normalizedRaster6 = MapAlgebra.normalizeAll(raster5, 0.0, 255.0, -9999.0, 0.0, 30.0); - GridCoverage2D normalizedRaster7 = MapAlgebra.normalizeAll(raster5, 0, 255, -9999.0, false); + GridCoverage2D normalizedRaster7 = MapAlgebra.normalizeAll(raster5, 0, 255, false, -9999.0); + GridCoverage2D normalizedRaster8 = MapAlgebra.normalizeAll(raster3, 0, 255); + GridCoverage2D normalizedRaster9 = MapAlgebra.normalizeAll(raster3, 0, 255, false); double[] expected1 = {0.0, 17.0, 34.0, 51.0, 68.0, 85.0, 102.0, 119.0, 136.0, 153.0, 170.0, 187.0, 204.0, 221.0, 238.0, 255.0}; double[] expected2 = {256.0, 273.0, 290.0, 307.0, 324.0, 341.0, 358.0, 375.0, 392.0, 409.0, 426.0, 443.0, 460.0, 477.0, 494.0, 511.0}; @@ -376,6 +378,10 @@ public void testNormalizeAll() throws FactoryException { double[] expected4 = {0.0, 17.0, 34.0, 51.0, 68.0, 85.0, 102.0, 119.0, 136.0, 153.0, 170.0, 187.0, 204.0, 221.0, 238.0, 95.0}; double[] expected5 = {95.0, 17.0, 34.0, 51.0, 68.0, 85.0, 102.0, 119.0, 136.0, 153.0, 170.0, 187.0, 204.0, 221.0, 238.0, 255.0}; double[] expected6 = {0.0, 18.214285714285715, 36.42857142857143, 54.642857142857146, 72.85714285714286, 91.07142857142857, 109.28571428571429, 127.5, 145.71428571428572, 163.92857142857142, 182.14285714285714, 200.35714285714286, 218.57142857142858, 236.78571428571428, 255.0, 255.0}; + double[] expected7 = {0.0, 16.0, 33.0, 50.0, 67.0, 84.0, 101.0, 118.0, 135.0, 152.0, 169.0, 186.0, 203.0, 220.0, 237.0, 255.0}; + double[] expected8 = {255.0, 16.0, 33.0, 50.0, 67.0, 84.0, 101.0, 118.0, 135.0, 152.0, 169.0, 186.0, 203.0, 220.0, 237.0, 254.0}; + double[] expected9 = {0.0, 18.0, 36.0, 54.0, 72.0, 90.0, 108.0, 127.0, 145.0, 163.0, 181.0, 199.0, 217.0, 235.0, 254.0, 255.0}; + double[] expected10 = {255.0, 0.0, 18.0, 36.0, 54.0, 72.0, 90.0, 108.0, 127.0, 145.0, 163.0, 181.0, 199.0, 217.0, 235.0, 254.0}; // Step 3: Validate the results for each band for (int band = 1; band <= 2; band++) { @@ -402,6 +408,10 @@ public void testNormalizeAll() throws FactoryException { assertEquals(Arrays.toString(expected3), Arrays.toString(MapAlgebra.bandAsArray(normalizedRaster3, 1))); assertEquals(Arrays.toString(expected4), Arrays.toString(MapAlgebra.bandAsArray(normalizedRaster4, 1))); assertEquals(Arrays.toString(expected5), Arrays.toString(MapAlgebra.bandAsArray(normalizedRaster4, 2))); + assertEquals(Arrays.toString(expected7), Arrays.toString(MapAlgebra.bandAsArray(normalizedRaster8, 1))); + assertEquals(Arrays.toString(expected8), Arrays.toString(MapAlgebra.bandAsArray(normalizedRaster8, 2))); + assertEquals(Arrays.toString(expected9), Arrays.toString(MapAlgebra.bandAsArray(normalizedRaster9, 1))); + assertEquals(Arrays.toString(expected10), Arrays.toString(MapAlgebra.bandAsArray(normalizedRaster9, 2))); } @Test diff --git a/docs/api/sql/Raster-operators.md b/docs/api/sql/Raster-operators.md index 5d3837cb4f..6bf36ac89e 100644 --- a/docs/api/sql/Raster-operators.md +++ b/docs/api/sql/Raster-operators.md @@ -2482,14 +2482,16 @@ SELECT RS_Normalize(band) ### RS_NormalizeAll -Introduction: Normalizes values in all bands of a raster between a given normalization range. The function maintains the data type of the raster values by ensuring that the normalized values are cast back to the original data type of each band in the raster. By default, the values are normalized to range [0, 255]. RS_NormalizeAll can take upto 6 of the following arguments. +Introduction: Normalizes values in all bands of a raster between a given normalization range. The function maintains the data type of the raster values by ensuring that the normalized values are cast back to the original data type of each band in the raster. By default, the values are normalized to range [0, 255]. RS_NormalizeAll can take upto 7 of the following arguments. - `raster`: The raster to be normalized. - `minLim` and `maxLim` (Optional): The lower and upper limits of the normalization range. By default, normalization range is set to [0, 255]. -- `noDataValue` (Optional): Defines the value to be used for missing or invalid data in raster bands. By default, noDataValue is set to `maxLim`. +- `noDataValue` (Optional): Defines the value to be used for missing or invalid data in raster bands. By default, noDataValue is set to `maxLim` and Safety mode is triggered. - `minValue` and `maxValue` (Optional): Optionally, specific minimum and maximum values of the input raster can be provided. If not provided, these values are computed from the raster data. - `normalizeAcrossBands` (Optional): A boolean flag to determine the normalization method. If set to true (default), normalization is performed across all bands based on global min and max values. If false, each band is normalized individually based on its own min and max values. +A safety mode is triggered when `noDataValue` is null, setting it to `maxLim` and normalizing data values to the range [minLim, maxLim-1]. This is to avoid replacing valid data that might coincide with the new `noDataValue`. + !!! Warning Using a noDataValue that falls within the normalization range can lead to loss of valid data. If any data value within a raster band matches the specified noDataValue, it will be replaced and cannot be distinguished or recovered later. Exercise caution in selecting a noDataValue to avoid unintentional data alteration. @@ -2501,16 +2503,16 @@ RS_NormalizeAll (raster: Raster)` RS_NormalizeAll (raster: Raster, minLim: Double, maxLim: Double) ``` ``` -RS_NormalizeAll (raster: Raster, minLim: Double, maxLim: Double, noDataValue: Double) +RS_NormalizeAll (raster: Raster, minLim: Double, maxLim: Double, normalizeAcrossBands: Boolean) ``` ``` -RS_NormalizeAll (raster: Raster, minLim: Double, maxLim: Double, noDataValue: Double, normalizeAcrossBands: Boolean) +RS_NormalizeAll (raster: Raster, minLim: Double, maxLim: Double, normalizeAcrossBands: Boolean, noDataValue: Double) ``` ``` RS_NormalizeAll (raster: Raster, minLim: Double, maxLim: Double, noDataValue: Double, minValue: Double, maxValue: Double) ``` ``` -RS_NormalizeAll (raster: Raster, minLim: Double, maxLim: Double, noDataValue: Double, minValue: Double, maxValue: Double, normalizeAcrossBands: Boolean) +RS_NormalizeAll (raster: Raster, minLim: Double, maxLim: Double, normalizeAcrossBands: Boolean, noDataValue: Double, minValue: Double, maxValue: Double ) ``` Since: `v1.6.0` From 72147e021a2cad86a4b9174a5da96e01d53618e1 Mon Sep 17 00:00:00 2001 From: Pranav Toggi Date: Thu, 8 Feb 2024 19:04:10 -0500 Subject: [PATCH 2/6] Update Scala test --- .../test/scala/org/apache/sedona/sql/rasteralgebraTest.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spark/common/src/test/scala/org/apache/sedona/sql/rasteralgebraTest.scala b/spark/common/src/test/scala/org/apache/sedona/sql/rasteralgebraTest.scala index 288e14afd0..2312147f89 100644 --- a/spark/common/src/test/scala/org/apache/sedona/sql/rasteralgebraTest.scala +++ b/spark/common/src/test/scala/org/apache/sedona/sql/rasteralgebraTest.scala @@ -220,7 +220,7 @@ class rasteralgebraTest extends TestBaseScala with BeforeAndAfter with GivenWhen var df = sparkSession.read.format("binaryFile").load(resourceFolder + "raster/test1.tiff") df = df.selectExpr("RS_FromGeoTiff(content) as raster") val result1 = df.selectExpr("RS_NormalizeAll(raster, 0, 255) as normalized").first().get(0) - val result2 = df.selectExpr("RS_NormalizeAll(raster, 0, 255, 0) as normalized").first().get(0) + val result2 = df.selectExpr("RS_NormalizeAll(raster, 0, 255, false) as normalized").first().get(0) assert(result1.isInstanceOf[GridCoverage2D]) assert(result2.isInstanceOf[GridCoverage2D]) } From 16582ef35e3057e6a3b0134775b318ca8ae0f3a8 Mon Sep 17 00:00:00 2001 From: Pranav Toggi Date: Thu, 8 Feb 2024 19:32:05 -0500 Subject: [PATCH 3/6] Fix typo in test --- .../java/org/apache/sedona/common/raster/MapAlgebraTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/src/test/java/org/apache/sedona/common/raster/MapAlgebraTest.java b/common/src/test/java/org/apache/sedona/common/raster/MapAlgebraTest.java index 354eafda57..d7d58385df 100644 --- a/common/src/test/java/org/apache/sedona/common/raster/MapAlgebraTest.java +++ b/common/src/test/java/org/apache/sedona/common/raster/MapAlgebraTest.java @@ -438,7 +438,7 @@ private void testNormalizeAll2(int width, int height, String pixelType) throws F // Check the normalized values and data type double[] normalizedBandValues = MapAlgebra.bandAsArray(normalizedRaster, 1); for (int i = 0; i < bandValues.length; i++) { - double expected = (bandValues[i] - 0) * (255 - 0) / (99 - 0); + double expected = (bandValues[i] - 0) * (254 - 0) / (99 - 0); double actual = normalizedBandValues[i]; switch (normalizedRaster.getRenderedImage().getSampleModel().getDataType()) { case DataBuffer.TYPE_BYTE: From 1c0360f295ff3edbc9544d03027f6de499171e96 Mon Sep 17 00:00:00 2001 From: Pranav Toggi Date: Fri, 9 Feb 2024 01:25:44 -0500 Subject: [PATCH 4/6] Add tests for noDataValue set to null; Update docs --- .../apache/sedona/common/raster/MapAlgebraTest.java | 12 +++++++++--- docs/api/sql/Raster-operators.md | 2 +- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/common/src/test/java/org/apache/sedona/common/raster/MapAlgebraTest.java b/common/src/test/java/org/apache/sedona/common/raster/MapAlgebraTest.java index d7d58385df..59e1854fbd 100644 --- a/common/src/test/java/org/apache/sedona/common/raster/MapAlgebraTest.java +++ b/common/src/test/java/org/apache/sedona/common/raster/MapAlgebraTest.java @@ -370,7 +370,9 @@ public void testNormalizeAll() throws FactoryException { GridCoverage2D normalizedRaster6 = MapAlgebra.normalizeAll(raster5, 0.0, 255.0, -9999.0, 0.0, 30.0); GridCoverage2D normalizedRaster7 = MapAlgebra.normalizeAll(raster5, 0, 255, false, -9999.0); GridCoverage2D normalizedRaster8 = MapAlgebra.normalizeAll(raster3, 0, 255); - GridCoverage2D normalizedRaster9 = MapAlgebra.normalizeAll(raster3, 0, 255, false); + GridCoverage2D normalizedRaster9 = MapAlgebra.normalizeAll(raster3, 0, 255, true, null); + GridCoverage2D normalizedRaster10 = MapAlgebra.normalizeAll(raster3, 0, 255, false); + GridCoverage2D normalizedRaster11 = MapAlgebra.normalizeAll(raster3, 0, 255, false, null); double[] expected1 = {0.0, 17.0, 34.0, 51.0, 68.0, 85.0, 102.0, 119.0, 136.0, 153.0, 170.0, 187.0, 204.0, 221.0, 238.0, 255.0}; double[] expected2 = {256.0, 273.0, 290.0, 307.0, 324.0, 341.0, 358.0, 375.0, 392.0, 409.0, 426.0, 443.0, 460.0, 477.0, 494.0, 511.0}; @@ -410,8 +412,12 @@ public void testNormalizeAll() throws FactoryException { assertEquals(Arrays.toString(expected5), Arrays.toString(MapAlgebra.bandAsArray(normalizedRaster4, 2))); assertEquals(Arrays.toString(expected7), Arrays.toString(MapAlgebra.bandAsArray(normalizedRaster8, 1))); assertEquals(Arrays.toString(expected8), Arrays.toString(MapAlgebra.bandAsArray(normalizedRaster8, 2))); - assertEquals(Arrays.toString(expected9), Arrays.toString(MapAlgebra.bandAsArray(normalizedRaster9, 1))); - assertEquals(Arrays.toString(expected10), Arrays.toString(MapAlgebra.bandAsArray(normalizedRaster9, 2))); + assertEquals(Arrays.toString(expected7), Arrays.toString(MapAlgebra.bandAsArray(normalizedRaster9, 1))); + assertEquals(Arrays.toString(expected8), Arrays.toString(MapAlgebra.bandAsArray(normalizedRaster9, 2))); + assertEquals(Arrays.toString(expected9), Arrays.toString(MapAlgebra.bandAsArray(normalizedRaster10, 1))); + assertEquals(Arrays.toString(expected10), Arrays.toString(MapAlgebra.bandAsArray(normalizedRaster10, 2))); + assertEquals(Arrays.toString(expected9), Arrays.toString(MapAlgebra.bandAsArray(normalizedRaster11, 1))); + assertEquals(Arrays.toString(expected10), Arrays.toString(MapAlgebra.bandAsArray(normalizedRaster11, 2))); } @Test diff --git a/docs/api/sql/Raster-operators.md b/docs/api/sql/Raster-operators.md index 6bf36ac89e..7b134617aa 100644 --- a/docs/api/sql/Raster-operators.md +++ b/docs/api/sql/Raster-operators.md @@ -2490,7 +2490,7 @@ Introduction: Normalizes values in all bands of a raster between a given normali - `minValue` and `maxValue` (Optional): Optionally, specific minimum and maximum values of the input raster can be provided. If not provided, these values are computed from the raster data. - `normalizeAcrossBands` (Optional): A boolean flag to determine the normalization method. If set to true (default), normalization is performed across all bands based on global min and max values. If false, each band is normalized individually based on its own min and max values. -A safety mode is triggered when `noDataValue` is null, setting it to `maxLim` and normalizing data values to the range [minLim, maxLim-1]. This is to avoid replacing valid data that might coincide with the new `noDataValue`. +A safety mode is triggered when `noDataValue` is not given or set to null. This sets `noDataValue` to `maxLim` and normalizes valid data values to the range [minLim, maxLim-1]. This is to avoid replacing valid data that might coincide with the new `noDataValue`. !!! Warning Using a noDataValue that falls within the normalization range can lead to loss of valid data. If any data value within a raster band matches the specified noDataValue, it will be replaced and cannot be distinguished or recovered later. Exercise caution in selecting a noDataValue to avoid unintentional data alteration. From 7ba7ee4974b3e468f39ec7ac6c49beb4d053c348 Mon Sep 17 00:00:00 2001 From: Pranav Toggi Date: Fri, 9 Feb 2024 14:23:19 -0500 Subject: [PATCH 5/6] Move RS_NormalizeAll to Raster Editors; Update docs --- .../sedona/common/raster/MapAlgebra.java | 126 ---------------- .../sedona/common/raster/RasterEditors.java | 132 +++++++++++++++++ .../sedona/common/raster/MapAlgebraTest.java | 134 ------------------ .../common/raster/RasterEditorsTest.java | 127 +++++++++++++++++ docs/api/sql/Raster-operators.md | 86 +++++------ .../expressions/raster/MapAlgebra.scala | 8 -- .../expressions/raster/RasterEditors.scala | 8 ++ .../apache/sedona/sql/rasteralgebraTest.scala | 8 +- 8 files changed, 316 insertions(+), 313 deletions(-) diff --git a/common/src/main/java/org/apache/sedona/common/raster/MapAlgebra.java b/common/src/main/java/org/apache/sedona/common/raster/MapAlgebra.java index f0ba29ca8e..564077a36e 100644 --- a/common/src/main/java/org/apache/sedona/common/raster/MapAlgebra.java +++ b/common/src/main/java/org/apache/sedona/common/raster/MapAlgebra.java @@ -442,132 +442,6 @@ public static double[] normalize(double[] bandValues) { return bandValues; } - public static GridCoverage2D normalizeAll(GridCoverage2D rasterGeom) { - return normalizeAll(rasterGeom, 0d, 255d, true, null, null, null); - } - - public static GridCoverage2D normalizeAll(GridCoverage2D rasterGeom, double minLim, double maxLim) { - return normalizeAll(rasterGeom, minLim, maxLim, true, null, null, null); - } - - public static GridCoverage2D normalizeAll(GridCoverage2D rasterGeom, double minLim, double maxLim, boolean normalizeAcrossBands) { - return normalizeAll(rasterGeom, minLim, maxLim, normalizeAcrossBands, null, null, null); - } - - public static GridCoverage2D normalizeAll(GridCoverage2D rasterGeom, double minLim, double maxLim, boolean normalizeAcrossBands, Double noDataValue) { - return normalizeAll(rasterGeom, minLim, maxLim, normalizeAcrossBands, noDataValue, null, null); - } - - public static GridCoverage2D normalizeAll(GridCoverage2D rasterGeom, double minLim, double maxLim, Double noDataValue, Double minValue, Double maxValue) { - return normalizeAll(rasterGeom, minLim, maxLim, true, noDataValue, minValue, maxValue); - } - - /** - * - * @param rasterGeom Raster to be normalized - * @param minLim Lower limit of normalization range - * @param maxLim Upper limit of normalization range - * @param normalizeAcrossBands flag to determine the normalization method - * @param noDataValue NoDataValue used in raster - * @param minValue Minimum value in raster - * @param maxValue Maximum value in raster - * @return a raster with all values in all bands normalized between minLim and maxLim - */ - public static GridCoverage2D normalizeAll(GridCoverage2D rasterGeom, double minLim, double maxLim, boolean normalizeAcrossBands, Double noDataValue, Double minValue, Double maxValue) { - if (minLim > maxLim) { - throw new IllegalArgumentException("minLim cannot be greater than maxLim"); - } - - int numBands = rasterGeom.getNumSampleDimensions(); - RenderedImage renderedImage = rasterGeom.getRenderedImage(); - int rasterDataType = renderedImage.getSampleModel().getDataType(); - - double globalMin = minValue != null ? minValue : Double.MAX_VALUE; - double globalMax = maxValue != null ? maxValue : -Double.MAX_VALUE; - - // Initialize arrays to store band-wise min and max values - double[] minValues = new double[numBands]; - double[] maxValues = new double[numBands]; - Arrays.fill(minValues, Double.MAX_VALUE); - Arrays.fill(maxValues, -Double.MAX_VALUE); - - // Trigger safe mode if noDataValue is null - noDataValue is set to maxLim and data values are normalized to range [minLim, maxLim-1]. - // This is done to prevent setting valid data as noDataValue. - double safetyTrigger = (noDataValue == null) ? 1 : 0; - - // Compute global min and max values across all bands if necessary and not provided - if (minValue == null || maxValue == null) { - for (int bandIndex = 0; bandIndex < numBands; bandIndex++) { - double[] bandValues = bandAsArray(rasterGeom, bandIndex + 1); - double bandNoDataValue = RasterUtils.getNoDataValue(rasterGeom.getSampleDimension(bandIndex)); - - if (noDataValue == null) { - noDataValue = maxLim; - } - - for (double val : bandValues) { - if (val != bandNoDataValue) { - if (normalizeAcrossBands) { - globalMin = Math.min(globalMin, val); - globalMax = Math.max(globalMax, val); - } else { - minValues[bandIndex] = Math.min(minValues[bandIndex], val); - maxValues[bandIndex] = Math.max(maxValues[bandIndex], val); - } - } - } - } - } else { - globalMin = minValue; - globalMax = maxValue; - } - - // Normalize each band - for (int bandIndex = 0; bandIndex < numBands; bandIndex++) { - double[] bandValues = bandAsArray(rasterGeom, bandIndex + 1); - double bandNoDataValue = RasterUtils.getNoDataValue(rasterGeom.getSampleDimension(bandIndex)); - double currentMin = normalizeAcrossBands ? globalMin : (minValue != null ? minValue : minValues[bandIndex]); - double currentMax = normalizeAcrossBands ? globalMax : (maxValue != null ? maxValue : maxValues[bandIndex]); - - if (Double.compare(currentMax, currentMin) == 0) { - Arrays.fill(bandValues, minLim); - } else { - for (int i = 0; i < bandValues.length; i++) { - if (bandValues[i] != bandNoDataValue) { - double normalizedValue = minLim + ((bandValues[i] - currentMin) * (maxLim - safetyTrigger - minLim)) / (currentMax - currentMin); - bandValues[i] = castRasterDataType(normalizedValue, rasterDataType); - } else { - bandValues[i] = noDataValue; - } - } - } - - // Update the raster with the normalized band and noDataValue - rasterGeom = addBandFromArray(rasterGeom, bandValues, bandIndex+1); - rasterGeom = RasterBandEditors.setBandNoDataValue(rasterGeom, bandIndex+1, noDataValue); - } - - return rasterGeom; - } - - private static double castRasterDataType(double value, int dataType) { - switch (dataType) { - case DataBuffer.TYPE_BYTE: - return (byte) value; - case DataBuffer.TYPE_SHORT: - return (short) value; - case DataBuffer.TYPE_INT: - return (int) value; - case DataBuffer.TYPE_USHORT: - return (char) value; - case DataBuffer.TYPE_FLOAT: - return (float) value; - case DataBuffer.TYPE_DOUBLE: - default: - return value; - } - } - /** * @param band1 band values * @param band2 band values diff --git a/common/src/main/java/org/apache/sedona/common/raster/RasterEditors.java b/common/src/main/java/org/apache/sedona/common/raster/RasterEditors.java index 926df696a1..27b77f529b 100644 --- a/common/src/main/java/org/apache/sedona/common/raster/RasterEditors.java +++ b/common/src/main/java/org/apache/sedona/common/raster/RasterEditors.java @@ -40,9 +40,15 @@ import javax.media.jai.Interpolation; import java.awt.geom.Point2D; +import java.awt.image.DataBuffer; +import java.awt.image.RenderedImage; +import java.util.Arrays; import java.util.Map; import java.util.Objects; +import static org.apache.sedona.common.raster.MapAlgebra.addBandFromArray; +import static org.apache.sedona.common.raster.MapAlgebra.bandAsArray; + public class RasterEditors { public static GridCoverage2D setSrid(GridCoverage2D raster, int srid) @@ -228,4 +234,130 @@ private static boolean noConfigChange(int oldWidth, int oldHeight, double oldUpp return ((approximateEquals(originalScaleX, newScaleX)) && (approximateEquals(originalScaleY, newScaleY)) && (approximateEquals(oldUpperX, newUpperX)) && (approximateEquals(oldUpperY, newUpperY))); } + public static GridCoverage2D normalizeAll(GridCoverage2D rasterGeom) { + return normalizeAll(rasterGeom, 0d, 255d, true, null, null, null); + } + + public static GridCoverage2D normalizeAll(GridCoverage2D rasterGeom, double minLim, double maxLim) { + return normalizeAll(rasterGeom, minLim, maxLim, true, null, null, null); + } + + public static GridCoverage2D normalizeAll(GridCoverage2D rasterGeom, double minLim, double maxLim, boolean normalizeAcrossBands) { + return normalizeAll(rasterGeom, minLim, maxLim, normalizeAcrossBands, null, null, null); + } + + public static GridCoverage2D normalizeAll(GridCoverage2D rasterGeom, double minLim, double maxLim, boolean normalizeAcrossBands, Double noDataValue) { + return normalizeAll(rasterGeom, minLim, maxLim, normalizeAcrossBands, noDataValue, null, null); + } + + public static GridCoverage2D normalizeAll(GridCoverage2D rasterGeom, double minLim, double maxLim, Double noDataValue, Double minValue, Double maxValue) { + return normalizeAll(rasterGeom, minLim, maxLim, true, noDataValue, minValue, maxValue); + } + + /** + * + * @param rasterGeom Raster to be normalized + * @param minLim Lower limit of normalization range + * @param maxLim Upper limit of normalization range + * @param normalizeAcrossBands flag to determine the normalization method + * @param noDataValue NoDataValue used in raster + * @param minValue Minimum value in raster + * @param maxValue Maximum value in raster + * @return a raster with all values in all bands normalized between minLim and maxLim + */ + public static GridCoverage2D normalizeAll(GridCoverage2D rasterGeom, double minLim, double maxLim, boolean normalizeAcrossBands, Double noDataValue, Double minValue, Double maxValue) { + if (minLim > maxLim) { + throw new IllegalArgumentException("minLim cannot be greater than maxLim"); + } + + int numBands = rasterGeom.getNumSampleDimensions(); + RenderedImage renderedImage = rasterGeom.getRenderedImage(); + int rasterDataType = renderedImage.getSampleModel().getDataType(); + + double globalMin = minValue != null ? minValue : Double.MAX_VALUE; + double globalMax = maxValue != null ? maxValue : -Double.MAX_VALUE; + + // Initialize arrays to store band-wise min and max values + double[] minValues = new double[numBands]; + double[] maxValues = new double[numBands]; + Arrays.fill(minValues, Double.MAX_VALUE); + Arrays.fill(maxValues, -Double.MAX_VALUE); + + // Trigger safe mode if noDataValue is null - noDataValue is set to maxLim and data values are normalized to range [minLim, maxLim-1]. + // This is done to prevent setting valid data as noDataValue. + double safetyTrigger = (noDataValue == null) ? 1 : 0; + + // Compute global min and max values across all bands if necessary and not provided + if (minValue == null || maxValue == null) { + for (int bandIndex = 0; bandIndex < numBands; bandIndex++) { + double[] bandValues = bandAsArray(rasterGeom, bandIndex + 1); + double bandNoDataValue = RasterUtils.getNoDataValue(rasterGeom.getSampleDimension(bandIndex)); + + if (noDataValue == null) { + noDataValue = maxLim; + } + + for (double val : bandValues) { + if (val != bandNoDataValue) { + if (normalizeAcrossBands) { + globalMin = Math.min(globalMin, val); + globalMax = Math.max(globalMax, val); + } else { + minValues[bandIndex] = Math.min(minValues[bandIndex], val); + maxValues[bandIndex] = Math.max(maxValues[bandIndex], val); + } + } + } + } + } else { + globalMin = minValue; + globalMax = maxValue; + } + + // Normalize each band + for (int bandIndex = 0; bandIndex < numBands; bandIndex++) { + double[] bandValues = bandAsArray(rasterGeom, bandIndex + 1); + double bandNoDataValue = RasterUtils.getNoDataValue(rasterGeom.getSampleDimension(bandIndex)); + double currentMin = normalizeAcrossBands ? globalMin : (minValue != null ? minValue : minValues[bandIndex]); + double currentMax = normalizeAcrossBands ? globalMax : (maxValue != null ? maxValue : maxValues[bandIndex]); + + if (Double.compare(currentMax, currentMin) == 0) { + Arrays.fill(bandValues, minLim); + } else { + for (int i = 0; i < bandValues.length; i++) { + if (bandValues[i] != bandNoDataValue) { + double normalizedValue = minLim + ((bandValues[i] - currentMin) * (maxLim - safetyTrigger - minLim)) / (currentMax - currentMin); + bandValues[i] = castRasterDataType(normalizedValue, rasterDataType); + } else { + bandValues[i] = noDataValue; + } + } + } + + // Update the raster with the normalized band and noDataValue + rasterGeom = addBandFromArray(rasterGeom, bandValues, bandIndex+1); + rasterGeom = RasterBandEditors.setBandNoDataValue(rasterGeom, bandIndex+1, noDataValue); + } + + return rasterGeom; + } + + private static double castRasterDataType(double value, int dataType) { + switch (dataType) { + case DataBuffer.TYPE_BYTE: + return (byte) value; + case DataBuffer.TYPE_SHORT: + return (short) value; + case DataBuffer.TYPE_INT: + return (int) value; + case DataBuffer.TYPE_USHORT: + return (char) value; + case DataBuffer.TYPE_FLOAT: + return (float) value; + case DataBuffer.TYPE_DOUBLE: + default: + return value; + } + } + } diff --git a/common/src/test/java/org/apache/sedona/common/raster/MapAlgebraTest.java b/common/src/test/java/org/apache/sedona/common/raster/MapAlgebraTest.java index 59e1854fbd..7bf17044c8 100644 --- a/common/src/test/java/org/apache/sedona/common/raster/MapAlgebraTest.java +++ b/common/src/test/java/org/apache/sedona/common/raster/MapAlgebraTest.java @@ -25,8 +25,6 @@ import org.opengis.referencing.FactoryException; import java.awt.image.DataBuffer; -import java.lang.reflect.Array; -import java.util.Arrays; import java.util.Random; import static org.junit.Assert.*; @@ -333,138 +331,6 @@ public void testNormalize() { assertArrayEquals(expected2, actual4, 0.1d); } - @Test - public void testNormalizeAll() throws FactoryException { - GridCoverage2D raster1 = RasterConstructors.makeEmptyRaster(2, 4, 4, 0, 0, 1); - GridCoverage2D raster2 = RasterConstructors.makeEmptyRaster(2, 4, 4, 0, 0, 1); - GridCoverage2D raster3 = RasterConstructors.makeEmptyRaster(2, "I", 4, 4, 0, 0, 1); - GridCoverage2D raster4 = RasterConstructors.makeEmptyRaster(2, 4, 4, 0, 0, 1); - GridCoverage2D raster5 = RasterConstructors.makeEmptyRaster(2, 4, 4, 0, 0, 1); - - for (int band = 1; band <= 2; band++) { - double[] bandValues1 = new double[4 * 4]; - double[] bandValues2 = new double[4 * 4]; - double[] bandValues3 = {1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16}; - double[] bandValues4 = {1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,0}; - double[] bandValues5 = new double[4 * 4]; - for (int i = 0; i < bandValues1.length; i++) { - bandValues1[i] = (i) * band; - bandValues2[i] = (1) * (band-1); - bandValues5[i] = i + ((band-1)*15); - } - raster1 = MapAlgebra.addBandFromArray(raster1, bandValues1, band); - raster2 = MapAlgebra.addBandFromArray(raster2, bandValues2, band); - raster3 = MapAlgebra.addBandFromArray(raster3, bandValues3, band); - raster4 = MapAlgebra.addBandFromArray(raster4, bandValues4, band); - raster4 = RasterBandEditors.setBandNoDataValue(raster4, band, 0.0); - raster5 = MapAlgebra.addBandFromArray(raster5, bandValues5, band); - } - raster3 = RasterBandEditors.setBandNoDataValue(raster3, 1, 16.0); - raster3 = RasterBandEditors.setBandNoDataValue(raster3, 2, 1.0); - - GridCoverage2D normalizedRaster1 = MapAlgebra.normalizeAll(raster1, 0, 255, false, -9999.0); - GridCoverage2D normalizedRaster2 = MapAlgebra.normalizeAll(raster1, 256d, 511d, false, -9999.0); - GridCoverage2D normalizedRaster3 = MapAlgebra.normalizeAll(raster2); - GridCoverage2D normalizedRaster4 = MapAlgebra.normalizeAll(raster3, 0, 255, true, 95.0); - GridCoverage2D normalizedRaster5 = MapAlgebra.normalizeAll(raster4, 0, 255, true, 255.0); - GridCoverage2D normalizedRaster6 = MapAlgebra.normalizeAll(raster5, 0.0, 255.0, -9999.0, 0.0, 30.0); - GridCoverage2D normalizedRaster7 = MapAlgebra.normalizeAll(raster5, 0, 255, false, -9999.0); - GridCoverage2D normalizedRaster8 = MapAlgebra.normalizeAll(raster3, 0, 255); - GridCoverage2D normalizedRaster9 = MapAlgebra.normalizeAll(raster3, 0, 255, true, null); - GridCoverage2D normalizedRaster10 = MapAlgebra.normalizeAll(raster3, 0, 255, false); - GridCoverage2D normalizedRaster11 = MapAlgebra.normalizeAll(raster3, 0, 255, false, null); - - double[] expected1 = {0.0, 17.0, 34.0, 51.0, 68.0, 85.0, 102.0, 119.0, 136.0, 153.0, 170.0, 187.0, 204.0, 221.0, 238.0, 255.0}; - double[] expected2 = {256.0, 273.0, 290.0, 307.0, 324.0, 341.0, 358.0, 375.0, 392.0, 409.0, 426.0, 443.0, 460.0, 477.0, 494.0, 511.0}; - double[] expected3 = {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}; - double[] expected4 = {0.0, 17.0, 34.0, 51.0, 68.0, 85.0, 102.0, 119.0, 136.0, 153.0, 170.0, 187.0, 204.0, 221.0, 238.0, 95.0}; - double[] expected5 = {95.0, 17.0, 34.0, 51.0, 68.0, 85.0, 102.0, 119.0, 136.0, 153.0, 170.0, 187.0, 204.0, 221.0, 238.0, 255.0}; - double[] expected6 = {0.0, 18.214285714285715, 36.42857142857143, 54.642857142857146, 72.85714285714286, 91.07142857142857, 109.28571428571429, 127.5, 145.71428571428572, 163.92857142857142, 182.14285714285714, 200.35714285714286, 218.57142857142858, 236.78571428571428, 255.0, 255.0}; - double[] expected7 = {0.0, 16.0, 33.0, 50.0, 67.0, 84.0, 101.0, 118.0, 135.0, 152.0, 169.0, 186.0, 203.0, 220.0, 237.0, 255.0}; - double[] expected8 = {255.0, 16.0, 33.0, 50.0, 67.0, 84.0, 101.0, 118.0, 135.0, 152.0, 169.0, 186.0, 203.0, 220.0, 237.0, 254.0}; - double[] expected9 = {0.0, 18.0, 36.0, 54.0, 72.0, 90.0, 108.0, 127.0, 145.0, 163.0, 181.0, 199.0, 217.0, 235.0, 254.0, 255.0}; - double[] expected10 = {255.0, 0.0, 18.0, 36.0, 54.0, 72.0, 90.0, 108.0, 127.0, 145.0, 163.0, 181.0, 199.0, 217.0, 235.0, 254.0}; - - // Step 3: Validate the results for each band - for (int band = 1; band <= 2; band++) { - double[] normalizedBand1 = MapAlgebra.bandAsArray(normalizedRaster1, band); - double[] normalizedBand2 = MapAlgebra.bandAsArray(normalizedRaster2, band); - double[] normalizedBand5 = MapAlgebra.bandAsArray(normalizedRaster5, band); - double[] normalizedBand6 = MapAlgebra.bandAsArray(normalizedRaster6, band); - double[] normalizedBand7 = MapAlgebra.bandAsArray(normalizedRaster7, band); - double normalizedMin6 = Arrays.stream(normalizedBand6).min().getAsDouble(); - double normalizedMax6 = Arrays.stream(normalizedBand6).max().getAsDouble(); - - assertEquals(Arrays.toString(expected1), Arrays.toString(normalizedBand1)); - assertEquals(Arrays.toString(expected2), Arrays.toString(normalizedBand2)); - assertEquals(Arrays.toString(expected6), Arrays.toString(normalizedBand5)); - assertEquals(Arrays.toString(expected1), Arrays.toString(normalizedBand7)); - - assertEquals(0+((band-1)*127.5), normalizedMin6, 0.01d); - assertEquals(127.5+((band-1)*127.5), normalizedMax6, 0.01d); - } - - assertEquals(95.0, RasterUtils.getNoDataValue(normalizedRaster4.getSampleDimension(0)), 0.01d); - assertEquals(95.0, RasterUtils.getNoDataValue(normalizedRaster4.getSampleDimension(1)), 0.01d); - - assertEquals(Arrays.toString(expected3), Arrays.toString(MapAlgebra.bandAsArray(normalizedRaster3, 1))); - assertEquals(Arrays.toString(expected4), Arrays.toString(MapAlgebra.bandAsArray(normalizedRaster4, 1))); - assertEquals(Arrays.toString(expected5), Arrays.toString(MapAlgebra.bandAsArray(normalizedRaster4, 2))); - assertEquals(Arrays.toString(expected7), Arrays.toString(MapAlgebra.bandAsArray(normalizedRaster8, 1))); - assertEquals(Arrays.toString(expected8), Arrays.toString(MapAlgebra.bandAsArray(normalizedRaster8, 2))); - assertEquals(Arrays.toString(expected7), Arrays.toString(MapAlgebra.bandAsArray(normalizedRaster9, 1))); - assertEquals(Arrays.toString(expected8), Arrays.toString(MapAlgebra.bandAsArray(normalizedRaster9, 2))); - assertEquals(Arrays.toString(expected9), Arrays.toString(MapAlgebra.bandAsArray(normalizedRaster10, 1))); - assertEquals(Arrays.toString(expected10), Arrays.toString(MapAlgebra.bandAsArray(normalizedRaster10, 2))); - assertEquals(Arrays.toString(expected9), Arrays.toString(MapAlgebra.bandAsArray(normalizedRaster11, 1))); - assertEquals(Arrays.toString(expected10), Arrays.toString(MapAlgebra.bandAsArray(normalizedRaster11, 2))); - } - - @Test - public void testNormalizeAll2() throws FactoryException { - String[] pixelTypes = {"B", "I", "S", "US", "F", "D"}; // Byte, Integer, Short, Unsigned Short, Float, Double - for (String pixelType : pixelTypes) { - testNormalizeAll2(10, 10, pixelType); - } - } - - private void testNormalizeAll2(int width, int height, String pixelType) throws FactoryException { - // Create an empty raster with the specified pixel type - GridCoverage2D raster = RasterConstructors.makeEmptyRaster(1, pixelType, width, height, 10, 20, 1); - - // Fill raster - double[] bandValues = new double[width * height]; - for (int i = 0; i < bandValues.length; i++) { - bandValues[i] = i; - } - raster = MapAlgebra.addBandFromArray(raster, bandValues, 1); - - GridCoverage2D normalizedRaster = MapAlgebra.normalizeAll(raster, 0, 255); - - // Check the normalized values and data type - double[] normalizedBandValues = MapAlgebra.bandAsArray(normalizedRaster, 1); - for (int i = 0; i < bandValues.length; i++) { - double expected = (bandValues[i] - 0) * (254 - 0) / (99 - 0); - double actual = normalizedBandValues[i]; - switch (normalizedRaster.getRenderedImage().getSampleModel().getDataType()) { - case DataBuffer.TYPE_BYTE: - case DataBuffer.TYPE_SHORT: - case DataBuffer.TYPE_USHORT: - case DataBuffer.TYPE_INT: - assertEquals((int) expected, (int) actual); - break; - default: - assertEquals(expected, actual, 0.01); - } - } - - // Assert the data type remains as expected - int resultDataType = normalizedRaster.getRenderedImage().getSampleModel().getDataType(); - int expectedDataType = RasterUtils.getDataTypeCode(pixelType); - assertEquals(expectedDataType, resultDataType); - } - - @Test public void testNormalizedDifference() { double[] band1 = new double[] {960, 1067, 107, 20, 1868}; diff --git a/common/src/test/java/org/apache/sedona/common/raster/RasterEditorsTest.java b/common/src/test/java/org/apache/sedona/common/raster/RasterEditorsTest.java index 649e0cb8d7..2c3b99a4f4 100644 --- a/common/src/test/java/org/apache/sedona/common/raster/RasterEditorsTest.java +++ b/common/src/test/java/org/apache/sedona/common/raster/RasterEditorsTest.java @@ -18,11 +18,13 @@ */ package org.apache.sedona.common.raster; +import org.apache.sedona.common.utils.RasterUtils; import org.geotools.coverage.grid.GridCoverage2D; import org.junit.Test; import org.opengis.referencing.FactoryException; import org.opengis.referencing.operation.TransformException; +import java.awt.image.DataBuffer; import java.io.IOException; import java.util.Arrays; @@ -266,4 +268,129 @@ public void testResampleRefRasterDiffSRID() throws FactoryException { assertThrows("Provided input raster and reference raster have different SRIDs", IllegalArgumentException.class, () -> RasterEditors.resample(raster, refRaster, false, null)); } + @Test + public void testNormalizeAll() throws FactoryException { + GridCoverage2D raster1 = RasterConstructors.makeEmptyRaster(2, 4, 4, 0, 0, 1); + GridCoverage2D raster2 = RasterConstructors.makeEmptyRaster(2, 4, 4, 0, 0, 1); + GridCoverage2D raster3 = RasterConstructors.makeEmptyRaster(2, "I", 4, 4, 0, 0, 1); + GridCoverage2D raster4 = RasterConstructors.makeEmptyRaster(2, 4, 4, 0, 0, 1); + GridCoverage2D raster5 = RasterConstructors.makeEmptyRaster(2, 4, 4, 0, 0, 1); + + for (int band = 1; band <= 2; band++) { + double[] bandValues1 = new double[4 * 4]; + double[] bandValues2 = new double[4 * 4]; + double[] bandValues3 = {1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16}; + double[] bandValues4 = {1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,0}; + double[] bandValues5 = new double[4 * 4]; + for (int i = 0; i < bandValues1.length; i++) { + bandValues1[i] = (i) * band; + bandValues2[i] = (1) * (band-1); + bandValues5[i] = i + ((band-1)*15); + } + raster1 = MapAlgebra.addBandFromArray(raster1, bandValues1, band); + raster2 = MapAlgebra.addBandFromArray(raster2, bandValues2, band); + raster3 = MapAlgebra.addBandFromArray(raster3, bandValues3, band); + raster4 = MapAlgebra.addBandFromArray(raster4, bandValues4, band); + raster4 = RasterBandEditors.setBandNoDataValue(raster4, band, 0.0); + raster5 = MapAlgebra.addBandFromArray(raster5, bandValues5, band); + } + raster3 = RasterBandEditors.setBandNoDataValue(raster3, 1, 16.0); + raster3 = RasterBandEditors.setBandNoDataValue(raster3, 2, 1.0); + + GridCoverage2D normalizedRaster1 = RasterEditors.normalizeAll(raster1, 0, 255, false, -9999.0); + GridCoverage2D normalizedRaster2 = RasterEditors.normalizeAll(raster1, 256d, 511d, false, -9999.0); + GridCoverage2D normalizedRaster3 = RasterEditors.normalizeAll(raster2); + GridCoverage2D normalizedRaster4 = RasterEditors.normalizeAll(raster3, 0, 255, true, 95.0); + GridCoverage2D normalizedRaster5 = RasterEditors.normalizeAll(raster4, 0, 255, true, 255.0); + GridCoverage2D normalizedRaster6 = RasterEditors.normalizeAll(raster5, 0.0, 255.0, -9999.0, 0.0, 30.0); + GridCoverage2D normalizedRaster7 = RasterEditors.normalizeAll(raster5, 0, 255, false, -9999.0); + GridCoverage2D normalizedRaster8 = RasterEditors.normalizeAll(raster3, 0, 255); + GridCoverage2D normalizedRaster10 = RasterEditors.normalizeAll(raster3, 0, 255, false); + + double[] expected1 = {0.0, 17.0, 34.0, 51.0, 68.0, 85.0, 102.0, 119.0, 136.0, 153.0, 170.0, 187.0, 204.0, 221.0, 238.0, 255.0}; + double[] expected2 = {256.0, 273.0, 290.0, 307.0, 324.0, 341.0, 358.0, 375.0, 392.0, 409.0, 426.0, 443.0, 460.0, 477.0, 494.0, 511.0}; + double[] expected3 = {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}; + double[] expected4 = {0.0, 17.0, 34.0, 51.0, 68.0, 85.0, 102.0, 119.0, 136.0, 153.0, 170.0, 187.0, 204.0, 221.0, 238.0, 95.0}; + double[] expected5 = {95.0, 17.0, 34.0, 51.0, 68.0, 85.0, 102.0, 119.0, 136.0, 153.0, 170.0, 187.0, 204.0, 221.0, 238.0, 255.0}; + double[] expected6 = {0.0, 18.214285714285715, 36.42857142857143, 54.642857142857146, 72.85714285714286, 91.07142857142857, 109.28571428571429, 127.5, 145.71428571428572, 163.92857142857142, 182.14285714285714, 200.35714285714286, 218.57142857142858, 236.78571428571428, 255.0, 255.0}; + double[] expected7 = {0.0, 16.0, 33.0, 50.0, 67.0, 84.0, 101.0, 118.0, 135.0, 152.0, 169.0, 186.0, 203.0, 220.0, 237.0, 255.0}; + double[] expected8 = {255.0, 16.0, 33.0, 50.0, 67.0, 84.0, 101.0, 118.0, 135.0, 152.0, 169.0, 186.0, 203.0, 220.0, 237.0, 254.0}; + double[] expected9 = {0.0, 18.0, 36.0, 54.0, 72.0, 90.0, 108.0, 127.0, 145.0, 163.0, 181.0, 199.0, 217.0, 235.0, 254.0, 255.0}; + double[] expected10 = {255.0, 0.0, 18.0, 36.0, 54.0, 72.0, 90.0, 108.0, 127.0, 145.0, 163.0, 181.0, 199.0, 217.0, 235.0, 254.0}; + + // Step 3: Validate the results for each band + for (int band = 1; band <= 2; band++) { + double[] normalizedBand1 = MapAlgebra.bandAsArray(normalizedRaster1, band); + double[] normalizedBand2 = MapAlgebra.bandAsArray(normalizedRaster2, band); + double[] normalizedBand5 = MapAlgebra.bandAsArray(normalizedRaster5, band); + double[] normalizedBand6 = MapAlgebra.bandAsArray(normalizedRaster6, band); + double[] normalizedBand7 = MapAlgebra.bandAsArray(normalizedRaster7, band); + double normalizedMin6 = Arrays.stream(normalizedBand6).min().getAsDouble(); + double normalizedMax6 = Arrays.stream(normalizedBand6).max().getAsDouble(); + + assertEquals(Arrays.toString(expected1), Arrays.toString(normalizedBand1)); + assertEquals(Arrays.toString(expected2), Arrays.toString(normalizedBand2)); + assertEquals(Arrays.toString(expected6), Arrays.toString(normalizedBand5)); + assertEquals(Arrays.toString(expected1), Arrays.toString(normalizedBand7)); + + assertEquals(0+((band-1)*127.5), normalizedMin6, 0.01d); + assertEquals(127.5+((band-1)*127.5), normalizedMax6, 0.01d); + } + + assertEquals(95.0, RasterUtils.getNoDataValue(normalizedRaster4.getSampleDimension(0)), 0.01d); + assertEquals(95.0, RasterUtils.getNoDataValue(normalizedRaster4.getSampleDimension(1)), 0.01d); + + assertEquals(Arrays.toString(expected3), Arrays.toString(MapAlgebra.bandAsArray(normalizedRaster3, 1))); + assertEquals(Arrays.toString(expected4), Arrays.toString(MapAlgebra.bandAsArray(normalizedRaster4, 1))); + assertEquals(Arrays.toString(expected5), Arrays.toString(MapAlgebra.bandAsArray(normalizedRaster4, 2))); + assertEquals(Arrays.toString(expected7), Arrays.toString(MapAlgebra.bandAsArray(normalizedRaster8, 1))); + assertEquals(Arrays.toString(expected8), Arrays.toString(MapAlgebra.bandAsArray(normalizedRaster8, 2))); + assertEquals(Arrays.toString(expected9), Arrays.toString(MapAlgebra.bandAsArray(normalizedRaster10, 1))); + assertEquals(Arrays.toString(expected10), Arrays.toString(MapAlgebra.bandAsArray(normalizedRaster10, 2))); + } + + @Test + public void testNormalizeAll2() throws FactoryException { + String[] pixelTypes = {"B", "I", "S", "US", "F", "D"}; // Byte, Integer, Short, Unsigned Short, Float, Double + for (String pixelType : pixelTypes) { + testNormalizeAll2(10, 10, pixelType); + } + } + + private void testNormalizeAll2(int width, int height, String pixelType) throws FactoryException { + // Create an empty raster with the specified pixel type + GridCoverage2D raster = RasterConstructors.makeEmptyRaster(1, pixelType, width, height, 10, 20, 1); + + // Fill raster + double[] bandValues = new double[width * height]; + for (int i = 0; i < bandValues.length; i++) { + bandValues[i] = i; + } + raster = MapAlgebra.addBandFromArray(raster, bandValues, 1); + + GridCoverage2D normalizedRaster = RasterEditors.normalizeAll(raster, 0, 255); + + // Check the normalized values and data type + double[] normalizedBandValues = MapAlgebra.bandAsArray(normalizedRaster, 1); + for (int i = 0; i < bandValues.length; i++) { + double expected = (bandValues[i] - 0) * (254 - 0) / (99 - 0); + double actual = normalizedBandValues[i]; + switch (normalizedRaster.getRenderedImage().getSampleModel().getDataType()) { + case DataBuffer.TYPE_BYTE: + case DataBuffer.TYPE_SHORT: + case DataBuffer.TYPE_USHORT: + case DataBuffer.TYPE_INT: + assertEquals((int) expected, (int) actual); + break; + default: + assertEquals(expected, actual, 0.01); + } + } + + // Assert the data type remains as expected + int resultDataType = normalizedRaster.getRenderedImage().getSampleModel().getDataType(); + int expectedDataType = RasterUtils.getDataTypeCode(pixelType); + assertEquals(expectedDataType, resultDataType); + } + } diff --git a/docs/api/sql/Raster-operators.md b/docs/api/sql/Raster-operators.md index 7b134617aa..0cdab05e54 100644 --- a/docs/api/sql/Raster-operators.md +++ b/docs/api/sql/Raster-operators.md @@ -1434,6 +1434,49 @@ Output: [-1.3095817809482181E7, 4021262.7487925636, 512.0, 517.0, 72.32861272132695, -72.32861272132695, 0.0, 0.0, 3857.0, 1.0] ``` +### RS_NormalizeAll + +Introduction: Normalizes values in all bands of a raster between a given normalization range. The function maintains the data type of the raster values by ensuring that the normalized values are cast back to the original data type of each band in the raster. By default, the values are normalized to range [0, 255]. RS_NormalizeAll can take upto 7 of the following arguments. + +- `raster`: The raster to be normalized. +- `minLim` and `maxLim` (Optional): The lower and upper limits of the normalization range. By default, normalization range is set to [0, 255]. +- `noDataValue` (Optional): Defines the value to be used for missing or invalid data in raster bands. By default, noDataValue is set to `maxLim` and Safety mode is triggered. +- `minValue` and `maxValue` (Optional): Optionally, specific minimum and maximum values of the input raster can be provided. If not provided, these values are computed from the raster data. +- `normalizeAcrossBands` (Optional): A boolean flag to determine the normalization method. If set to true (default), normalization is performed across all bands based on global min and max values. If false, each band is normalized individually based on its own min and max values. + +A safety mode is triggered when `noDataValue` is not given. This sets `noDataValue` to `maxLim` and normalizes valid data values to the range [minLim, maxLim-1]. This is to avoid replacing valid data that might coincide with the new `noDataValue`. + +!!! Warning +Using a noDataValue that falls within the normalization range can lead to loss of valid data. If any data value within a raster band matches the specified noDataValue, it will be replaced and cannot be distinguished or recovered later. Exercise caution in selecting a noDataValue to avoid unintentional data alteration. + +Formats: +``` +RS_NormalizeAll (raster: Raster)` +``` +``` +RS_NormalizeAll (raster: Raster, minLim: Double, maxLim: Double) +``` +``` +RS_NormalizeAll (raster: Raster, minLim: Double, maxLim: Double, normalizeAcrossBands: Boolean) +``` +``` +RS_NormalizeAll (raster: Raster, minLim: Double, maxLim: Double, normalizeAcrossBands: Boolean, noDataValue: Double) +``` +``` +RS_NormalizeAll (raster: Raster, minLim: Double, maxLim: Double, noDataValue: Double, minValue: Double, maxValue: Double) +``` +``` +RS_NormalizeAll (raster: Raster, minLim: Double, maxLim: Double, normalizeAcrossBands: Boolean, noDataValue: Double, minValue: Double, maxValue: Double ) +``` + +Since: `v1.6.0` + +Spark SQL Example: + +```sql +SELECT RS_NormalizeAll(raster, 0, 1) +``` + ### RS_NumBands Introduction: Returns the number of the bands in the raster. @@ -2480,49 +2523,6 @@ Spark SQL Example: SELECT RS_Normalize(band) ``` -### RS_NormalizeAll - -Introduction: Normalizes values in all bands of a raster between a given normalization range. The function maintains the data type of the raster values by ensuring that the normalized values are cast back to the original data type of each band in the raster. By default, the values are normalized to range [0, 255]. RS_NormalizeAll can take upto 7 of the following arguments. - -- `raster`: The raster to be normalized. -- `minLim` and `maxLim` (Optional): The lower and upper limits of the normalization range. By default, normalization range is set to [0, 255]. -- `noDataValue` (Optional): Defines the value to be used for missing or invalid data in raster bands. By default, noDataValue is set to `maxLim` and Safety mode is triggered. -- `minValue` and `maxValue` (Optional): Optionally, specific minimum and maximum values of the input raster can be provided. If not provided, these values are computed from the raster data. -- `normalizeAcrossBands` (Optional): A boolean flag to determine the normalization method. If set to true (default), normalization is performed across all bands based on global min and max values. If false, each band is normalized individually based on its own min and max values. - -A safety mode is triggered when `noDataValue` is not given or set to null. This sets `noDataValue` to `maxLim` and normalizes valid data values to the range [minLim, maxLim-1]. This is to avoid replacing valid data that might coincide with the new `noDataValue`. - -!!! Warning - Using a noDataValue that falls within the normalization range can lead to loss of valid data. If any data value within a raster band matches the specified noDataValue, it will be replaced and cannot be distinguished or recovered later. Exercise caution in selecting a noDataValue to avoid unintentional data alteration. - -Formats: -``` -RS_NormalizeAll (raster: Raster)` -``` -``` -RS_NormalizeAll (raster: Raster, minLim: Double, maxLim: Double) -``` -``` -RS_NormalizeAll (raster: Raster, minLim: Double, maxLim: Double, normalizeAcrossBands: Boolean) -``` -``` -RS_NormalizeAll (raster: Raster, minLim: Double, maxLim: Double, normalizeAcrossBands: Boolean, noDataValue: Double) -``` -``` -RS_NormalizeAll (raster: Raster, minLim: Double, maxLim: Double, noDataValue: Double, minValue: Double, maxValue: Double) -``` -``` -RS_NormalizeAll (raster: Raster, minLim: Double, maxLim: Double, normalizeAcrossBands: Boolean, noDataValue: Double, minValue: Double, maxValue: Double ) -``` - -Since: `v1.6.0` - -Spark SQL Example: - -```sql -SELECT RS_NormalizeAll(raster, 0, 1) -``` - ### RS_NormalizedDifference Introduction: Returns Normalized Difference between two bands(band2 and band1) in a Geotiff image(example: NDVI, NDBI) diff --git a/spark/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/raster/MapAlgebra.scala b/spark/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/raster/MapAlgebra.scala index f1d12b80f2..bd30844021 100644 --- a/spark/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/raster/MapAlgebra.scala +++ b/spark/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/raster/MapAlgebra.scala @@ -179,14 +179,6 @@ case class RS_Normalize(inputExpressions: Seq[Expression]) extends InferredExpre } } -case class RS_NormalizeAll(inputExpressions: Seq[Expression]) extends InferredExpression( - inferrableFunction1(MapAlgebra.normalizeAll), inferrableFunction3(MapAlgebra.normalizeAll), inferrableFunction4(MapAlgebra.normalizeAll), inferrableFunction5(MapAlgebra.normalizeAll), inferrableFunction6(MapAlgebra.normalizeAll), inferrableFunction7(MapAlgebra.normalizeAll) -) { - protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]) = { - copy(inputExpressions = newChildren) - } -} - case class RS_AddBandFromArray(inputExpressions: Seq[Expression]) extends InferredExpression(nullTolerantInferrableFunction3(MapAlgebra.addBandFromArray), nullTolerantInferrableFunction4(MapAlgebra.addBandFromArray), inferrableFunction2(MapAlgebra.addBandFromArray)) { protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]) = { diff --git a/spark/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/raster/RasterEditors.scala b/spark/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/raster/RasterEditors.scala index 3b13e03101..5ccf34ccb5 100644 --- a/spark/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/raster/RasterEditors.scala +++ b/spark/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/raster/RasterEditors.scala @@ -44,3 +44,11 @@ case class RS_Resample(inputExpressions: Seq[Expression]) extends InferredExpres copy(inputExpressions = newChildren) } } + +case class RS_NormalizeAll(inputExpressions: Seq[Expression]) extends InferredExpression( + inferrableFunction1(RasterEditors.normalizeAll), inferrableFunction3(RasterEditors.normalizeAll), inferrableFunction4(RasterEditors.normalizeAll), inferrableFunction5(RasterEditors.normalizeAll), inferrableFunction6(RasterEditors.normalizeAll), inferrableFunction7(RasterEditors.normalizeAll) +) { + protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]) = { + copy(inputExpressions = newChildren) + } +} diff --git a/spark/common/src/test/scala/org/apache/sedona/sql/rasteralgebraTest.scala b/spark/common/src/test/scala/org/apache/sedona/sql/rasteralgebraTest.scala index 2312147f89..0fbf674830 100644 --- a/spark/common/src/test/scala/org/apache/sedona/sql/rasteralgebraTest.scala +++ b/spark/common/src/test/scala/org/apache/sedona/sql/rasteralgebraTest.scala @@ -219,10 +219,14 @@ class rasteralgebraTest extends TestBaseScala with BeforeAndAfter with GivenWhen it("should pass RS_NormalizeAll") { var df = sparkSession.read.format("binaryFile").load(resourceFolder + "raster/test1.tiff") df = df.selectExpr("RS_FromGeoTiff(content) as raster") - val result1 = df.selectExpr("RS_NormalizeAll(raster, 0, 255) as normalized").first().get(0) - val result2 = df.selectExpr("RS_NormalizeAll(raster, 0, 255, false) as normalized").first().get(0) + val result1 = df.selectExpr("RS_NormalizeAll(raster) as normalized").first().get(0) + val result2 = df.selectExpr("RS_NormalizeAll(raster, 0, 255) as normalized").first().get(0) + val result3 = df.selectExpr("RS_NormalizeAll(raster, 0, 255, false) as normalized").first().get(0) + val result4 = df.selectExpr("RS_NormalizeAll(raster, 0, 255, false, 255) as normalized").first().get(0) assert(result1.isInstanceOf[GridCoverage2D]) assert(result2.isInstanceOf[GridCoverage2D]) + assert(result3.isInstanceOf[GridCoverage2D]) + assert(result4.isInstanceOf[GridCoverage2D]) } it("should pass RS_Array") { From bc2989d1f0aeba135c7f725099003e97b93134d5 Mon Sep 17 00:00:00 2001 From: Pranav Toggi Date: Fri, 9 Feb 2024 17:06:04 -0500 Subject: [PATCH 6/6] Fix indentation --- docs/api/sql/Raster-operators.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/api/sql/Raster-operators.md b/docs/api/sql/Raster-operators.md index 0cdab05e54..42d0a5fa7d 100644 --- a/docs/api/sql/Raster-operators.md +++ b/docs/api/sql/Raster-operators.md @@ -1440,18 +1440,18 @@ Introduction: Normalizes values in all bands of a raster between a given normali - `raster`: The raster to be normalized. - `minLim` and `maxLim` (Optional): The lower and upper limits of the normalization range. By default, normalization range is set to [0, 255]. +- `normalizeAcrossBands` (Optional): A boolean flag to determine the normalization method. If set to true (default), normalization is performed across all bands based on global min and max values. If false, each band is normalized individually based on its own min and max values. - `noDataValue` (Optional): Defines the value to be used for missing or invalid data in raster bands. By default, noDataValue is set to `maxLim` and Safety mode is triggered. - `minValue` and `maxValue` (Optional): Optionally, specific minimum and maximum values of the input raster can be provided. If not provided, these values are computed from the raster data. -- `normalizeAcrossBands` (Optional): A boolean flag to determine the normalization method. If set to true (default), normalization is performed across all bands based on global min and max values. If false, each band is normalized individually based on its own min and max values. -A safety mode is triggered when `noDataValue` is not given. This sets `noDataValue` to `maxLim` and normalizes valid data values to the range [minLim, maxLim-1]. This is to avoid replacing valid data that might coincide with the new `noDataValue`. +A Safety mode is triggered when `noDataValue` is not given. This sets `noDataValue` to `maxLim` and normalizes valid data values to the range [minLim, maxLim-1]. This is to avoid replacing valid data that might coincide with the new `noDataValue`. !!! Warning -Using a noDataValue that falls within the normalization range can lead to loss of valid data. If any data value within a raster band matches the specified noDataValue, it will be replaced and cannot be distinguished or recovered later. Exercise caution in selecting a noDataValue to avoid unintentional data alteration. + Using a noDataValue that falls within the normalization range can lead to loss of valid data. If any data value within a raster band matches the specified noDataValue, it will be replaced and cannot be distinguished or recovered later. Exercise caution in selecting a noDataValue to avoid unintentional data alteration. Formats: ``` -RS_NormalizeAll (raster: Raster)` +RS_NormalizeAll (raster: Raster) ``` ``` RS_NormalizeAll (raster: Raster, minLim: Double, maxLim: Double)