Merge 3f9c10d into ef81a52

analytics/mapreduce/src/main/java/mil/nga/giat/geowave/analytic/mapreduce/kde/KDEJobRunner.java

@@ -519,7 +519,7 @@ protected Class<?> getJob2OutputValueClass() {
 	}
 
 	protected Class<? extends Reducer<?, ?, ?, ?>> getJob2Reducer() {
-		return AccumuloKDEReducer.class;
+		return KDEReducer.class;
 	}
 
 	protected Class<? extends Partitioner<?, ?>> getJob2Partitioner() {
@@ -560,11 +560,11 @@ protected void setupJob2Output(
 			throws Exception {
 		final WritableDataAdapter<?> adapter = RasterUtils.createDataAdapterTypeDouble(
 				coverageName,
-				AccumuloKDEReducer.NUM_BANDS,
+				KDEReducer.NUM_BANDS,
 				TILE_SIZE,
-				AccumuloKDEReducer.MINS_PER_BAND,
-				AccumuloKDEReducer.MAXES_PER_BAND,
-				AccumuloKDEReducer.NAME_PER_BAND,
+				KDEReducer.MINS_PER_BAND,
+				KDEReducer.MAXES_PER_BAND,
+				KDEReducer.NAME_PER_BAND,
 				null);
 		setup(
 				statsReducer,

analytics/mapreduce/src/main/java/mil/nga/giat/geowave/analytic/mapreduce/kde/AccumuloKDEReducer.java → analytics/mapreduce/src/main/java/mil/nga/giat/geowave/analytic/mapreduce/kde/KDEReducer.java

@@ -25,11 +25,12 @@
 import mil.nga.giat.geowave.analytic.mapreduce.kde.GaussianFilter.ValueRange;
 import mil.nga.giat.geowave.core.geotime.ingest.SpatialDimensionalityTypeProvider;
 import mil.nga.giat.geowave.core.index.ByteArrayId;
+import mil.nga.giat.geowave.core.index.FloatCompareUtils;
 import mil.nga.giat.geowave.core.store.index.PrimaryIndex;
 import mil.nga.giat.geowave.mapreduce.JobContextIndexStore;
 import mil.nga.giat.geowave.mapreduce.output.GeoWaveOutputKey;
 
-public class AccumuloKDEReducer extends
+public class KDEReducer extends
 		Reducer<DoubleWritable, LongWritable, GeoWaveOutputKey, GridCoverage>
 {
 	private static final class TileInfo
@@ -101,6 +102,8 @@ public boolean equals(
 		}
 	}
 
+	private static final double WEIGHT_EPSILON = 2.22E-14;
+
 	public static final int NUM_BANDS = 3;
 	protected static final String[] NAME_PER_BAND = new String[] {
 		"Weight",
@@ -133,6 +136,8 @@ public boolean equals(
 	protected List<ByteArrayId> indexList;
 	protected ValueRange[] valueRangePerDimension;
 	protected String crsCode;
+	protected double prevValue = -1;
+	protected double prevPct = 0;
 
 	@Override
 	protected void reduce(
@@ -152,14 +157,32 @@ protected void reduce(
 			final double normalizedValue = value / max;
 			// for consistency give all cells with matching weight the same
 			// percentile
-			final double percentile = (currentKey + 1.0) / totalKeys;
+			// because we are using a DoubleWritable as the key, the ordering
+			// isn't always completely reproducible as Double equals does not
+			// take into account an epsilon, but we can make it reproducible by
+			// doing a comparison with the previous value using an appropriate
+			// epsilon
+			final double percentile;
+			if (FloatCompareUtils.checkDoublesEqual(
+					prevValue,
+					value,
+					WEIGHT_EPSILON)) {
+				percentile = prevPct;
+			}
+			else {
+				percentile = (currentKey + 1.0) / totalKeys;
+				prevPct = percentile;
+				prevValue = value;
+			}
+
 			// calculate weights for this key
 			for (final LongWritable v : values) {
 				final long cellIndex = v.get() / numLevels;
 				final TileInfo tileInfo = fromCellIndexToTileInfo(cellIndex);
 				final WritableRaster raster = RasterUtils.createRasterTypeDouble(
 						NUM_BANDS,
 						KDEJobRunner.TILE_SIZE);
+
 				raster.setSample(
 						tileInfo.x,
 						tileInfo.y,

core/index/src/main/java/mil/nga/giat/geowave/core/index/FloatCompareUtils.java

@@ -29,6 +29,25 @@ public class FloatCompareUtils
 	public static boolean checkDoublesEqual(
 			double x,
 			double y ) {
+		return checkDoublesEqual(
+				x,
+				y,
+				COMP_EPSILON);
+	}
+
+	/**
+	 * The == operator is not reliable for doubles, so we are using this method
+	 * to check if two doubles are equal
+	 * 
+	 * @param x
+	 * @param y
+	 * @param epsilon
+	 * @return true if the double are equal, false if they are not
+	 */
+	public static boolean checkDoublesEqual(
+			double x,
+			double y,
+			double epsilon ) {
 		boolean xNeg = false;
 		boolean yNeg = false;
 		double diff = (Math.abs(x) - Math.abs(y));
@@ -39,7 +58,7 @@ public static boolean checkDoublesEqual(
 		if (y < 0.0) {
 			yNeg = true;
 		}
-		return (diff <= COMP_EPSILON && diff >= -COMP_EPSILON && xNeg == yNeg);
+		return (diff <= epsilon && diff >= -epsilon && xNeg == yNeg);
 	}
 
 }

core/ingest/src/main/java/mil/nga/giat/geowave/core/ingest/operations/ListPluginsCommand.java

@@ -37,22 +37,23 @@ public class ListPluginsCommand extends
 	@Override
 	public void execute(
 			final OperationParams params ) {
-		JCommander.getConsole().println(computeResults(params));
+		JCommander.getConsole().println(
+				computeResults(params));
 	}
 
 	@Override
 	public String computeResults(
 			final OperationParams params ) {
 		StringBuilder builder = new StringBuilder();
-	
+
 		builder.append("Available index types currently registered as plugins:\n");
 		for (final Entry<String, DimensionalityTypeProviderSpi> pluginProviderEntry : DimensionalityTypeRegistry
 				.getRegisteredDimensionalityTypes()
 				.entrySet()) {
 			final DimensionalityTypeProviderSpi pluginProvider = pluginProviderEntry.getValue();
 			final String desc = pluginProvider.getDimensionalityTypeDescription() == null ? "no description"
 					: pluginProvider.getDimensionalityTypeDescription();
-			
+
 			builder.append(String.format(
 					"%n  %s:%n    %s%n",
 					pluginProviderEntry.getKey(),
@@ -71,7 +72,7 @@ public String computeResults(
 					pluginProviderEntry.getKey(),
 					desc));
 		}
-		
+
 		builder.append("\nAvailable datastores currently registered:\n");
 		final Map<String, StoreFactoryFamilySpi> dataStoreFactories = GeoWaveStoreFinder
 				.getRegisteredStoreFactoryFamilies();
@@ -84,7 +85,7 @@ public String computeResults(
 					dataStoreFactory.getType(),
 					desc));
 		}
-		
+
 		return builder.toString();
 	}
 }

core/store/src/main/java/mil/nga/giat/geowave/core/store/cli/config/AddIndexCommand.java

@@ -36,7 +36,7 @@
 @GeowaveOperation(name = "addindex", parentOperation = ConfigSection.class)
 @Parameters(commandDescription = "Configure an index for usage in GeoWave")
 public class AddIndexCommand extends
-	ServiceEnabledCommand<String>
+		ServiceEnabledCommand<String>
 {
 	private final static Logger LOGGER = LoggerFactory.getLogger(AddIndexCommand.class);
 
@@ -56,18 +56,18 @@ public class AddIndexCommand extends
 	private String type;
 
 	private IndexPluginOptions pluginOptions = new IndexPluginOptions();
-	
+
 	@ParametersDelegate
 	private final BasicIndexOptions basicIndexOptions = new BasicIndexOptions();
-	
-	@ParametersDelegate 
+
+	@ParametersDelegate
 	DimensionalityTypeOptions opts;
 
 	@Override
 	public boolean prepare(
 			OperationParams params ) {
 		super.prepare(params);
-		
+
 		// Load SPI options for the given type into pluginOptions.
 		if (type != null) {
 			pluginOptions.selectPlugin(type);
@@ -101,20 +101,20 @@ public boolean prepare(
 				}
 			}
 		}
-	
+
 		return true;
 	}
 
 	@Override
 	public void execute(
 			OperationParams params ) {
-		computeResults(params);	
+		computeResults(params);
 	}
-	
+
 	@Override
 	public String computeResults(
-			OperationParams params){
-		
+			OperationParams params ) {
+
 		// Ensure that a name is chosen.
 		if (parameters.size() != 1) {
 			throw new ParameterException(
@@ -148,7 +148,7 @@ public String computeResults(
 		ConfigOptions.writeProperties(
 				getGeoWaveConfigFile(params),
 				existingProps);
-		
+
 		StringBuilder builder = new StringBuilder();
 		for (Object key : existingProps.keySet()) {
 			String[] split = key.toString().split(

core/store/src/main/java/mil/nga/giat/geowave/core/store/cli/config/AddStoreCommand.java

@@ -35,7 +35,7 @@
 @GeowaveOperation(name = "addstore", parentOperation = ConfigSection.class)
 @Parameters(commandDescription = "Create a store within Geowave")
 public class AddStoreCommand extends
-	ServiceEnabledCommand<String>
+		ServiceEnabledCommand<String>
 {
 
 	private final static Logger LOGGER = LoggerFactory.getLogger(AddStoreCommand.class);
@@ -58,23 +58,23 @@ public class AddStoreCommand extends
 	private String storeType;
 
 	private DataStorePluginOptions pluginOptions = new DataStorePluginOptions();
-	
+
 	@ParametersDelegate
 	private StoreFactoryOptions requiredOptions;
 
 	@Override
 	public boolean prepare(
 			OperationParams params ) {
 		super.prepare(params);
-		
+
 		// Load SPI options for the given type into pluginOptions.
 		if (storeType != null) {
 			pluginOptions.selectPlugin(storeType);
 			requiredOptions = pluginOptions.getFactoryOptions();
 		}
-		else {	
+		else {
 			Properties existingProps = getGeoWaveConfigProperties(params);
-			
+
 			// Try to load the 'default' options.
 			String defaultStore = existingProps.getProperty(DataStorePluginOptions.DEFAULT_PROPERTY_NAMESPACE);
 
@@ -100,7 +100,7 @@ public boolean prepare(
 				}
 			}
 		}
-		
+
 		return true;
 	}
 
@@ -109,9 +109,10 @@ public void execute(
 			OperationParams params ) {
 		computeResults(params);
 	}
-	
+
 	@Override
-	public String computeResults(OperationParams params) {
+	public String computeResults(
+			OperationParams params ) {
 		Properties existingProps = getGeoWaveConfigProperties(params);
 
 		// Ensure that a name is chosen.
@@ -152,7 +153,7 @@ public String computeResults(OperationParams params) {
 				existingProps,
 				pluginOptions.getFactoryOptions().getClass(),
 				getNamespace() + "." + DataStorePluginOptions.OPTS);
-		
+
 		StringBuilder builder = new StringBuilder();
 		for (Object key : existingProps.keySet()) {
 			String[] split = key.toString().split(

services/rest/src/main/java/mil/nga/giat/geowave/service/rest/GeoWaveOperationFinder.java

@@ -32,13 +32,14 @@ public ServerResource create(
 			return new GeoWaveOperationServiceWrapper<>(
 					operation.getClass().newInstance(),
 					defaultConfigFile);
-		} catch (InstantiationException | IllegalAccessException e) {
+		}
+		catch (InstantiationException | IllegalAccessException e) {
 			getLogger().log(
 					Level.SEVERE,
 					"Unable to instantiate Service Resource",
 					e);
 			return null;
-			
+
 		}
 	}
 

services/rest/src/main/java/mil/nga/giat/geowave/service/rest/GeoWaveOperationServiceWrapper.java

@@ -191,23 +191,24 @@ private void injectParameters(
 			Class<?> type = f.getType();
 			Field field = f.getField();
 			final String strValue = (String) requestParameters.getString(f.getName());
-			
+
 			if (field.isAnnotationPresent(Parameter.class)) {
 				Class<? extends IStringConverter<?>> converter = field.getAnnotation(
 						Parameter.class).converter();
-				if(converter != null) {
+				if (converter != null) {
 					if (converter != NoConverter.class && strValue != null) {
 						try {
 							objValue = converter.newInstance().convert(
-								strValue);
+									strValue);
 						}
-						catch(InstantiationException e) {
-							LOGGER.warn("Cannot convert parameter since converter does not have zero argument constructor");	
+						catch (InstantiationException e) {
+							LOGGER
+									.warn("Cannot convert parameter since converter does not have zero argument constructor");
 						}
 					}
 				}
 			}
-			
+
 			if (objValue == null) {
 				if (List.class.isAssignableFrom(type)) {
 					objValue = requestParameters.getList(f.getName());

services/rest/src/main/java/mil/nga/giat/geowave/service/rest/field/AbstractMainParam.java

@@ -74,7 +74,7 @@ public void setValue(
 
 	abstract protected String valueToString(
 			T value );
-	
+
 	public Field getField() {
 		return this.listMainParamField;
 	}

services/rest/src/main/java/mil/nga/giat/geowave/service/rest/field/ParameterRestField.java

@@ -36,7 +36,7 @@ public String getDescription() {
 	public boolean isRequired() {
 		return parameter.required();
 	}
-	
+
 	public Field getField() {
 		return this.field;
 	}

services/rest/src/main/java/mil/nga/giat/geowave/service/rest/field/ParameterRestFieldValue.java

@@ -33,7 +33,7 @@ public void setValue(
 				instance,
 				value);
 	}
-	
+
 	public Field getField() {
 		return super.getField();
 	}

services/rest/src/main/java/mil/nga/giat/geowave/service/rest/field/RestFieldValue.java

@@ -9,6 +9,6 @@ public void setValue(
 			T value )
 			throws IllegalArgumentException,
 			IllegalAccessException;
-	
+
 	public Field getField();
 }