GEOWAVE-1018 and hbase serverside operations

.gitignore

@@ -13,3 +13,4 @@ bin
 docker-root
 generated
 .metadata
+

.travis.yml

@@ -1,3 +1,4 @@
+dist: precise
 language: java
 jdk:
   - oraclejdk8
@@ -6,16 +7,19 @@ env:
     # The first line of the matrix just builds docs (only on master) and runs unit tests
     # The rest skip the unit tests and run ITs only
     - NAME='UT and Publish Docs with Latest ASF Versions' MAVEN_PROFILES='""' BUILD_DOCS=true   IT_ONLY=false
-    - NAME='Accumulo IT on Latest ASF Versions' MAVEN_PROFILES='accumulo-it' BUILD_DOCS=false  IT_ONLY=true
-    #- NAME='HBase IT on Latest ASF Versions' MAVEN_PROFILES='hbase-it' BUILD_DOCS=false  IT_ONLY=true
-    #- NAME='Cassandra IT on Latest ASF Versions' MAVEN_PROFILES='cassandra-it' BUILD_DOCS=false  IT_ONLY=true
-    #- NAME='DynamoDB IT on Latest ASF Versions' MAVEN_PROFILES='dynamodb-it' BUILD_DOCS=false  IT_ONLY=true
-    #- NAME='Bigtable IT on Latest ASF Versions' MAVEN_PROFILES='bigtable-it' BUILD_DOCS=false  IT_ONLY=true
-    - NAME='Accumulo IT on Latest CDH Versions' MAVEN_PROFILES='accumulo-it,cloudera' BUILD_DOCS=false  IT_ONLY=true
-    - NAME='Accumulo IT on Latest HDP Versions' MAVEN_PROFILES='accumulo-it,hortonworks' BUILD_DOCS=false  IT_ONLY=true
-    - NAME='Accumulo IT on Older ASF and GeoTools Versions' MAVEN_PROFILES='accumulo-it,compatibility' BUILD_DOCS=false  IT_ONLY=true	 
-    #- NAME='HBase IT on Latest CDH Versions'  MAVEN_PROFILES='hbase-it,cloudera' BUILD_DOCS=false  IT_ONLY=true
-    #- NAME='HBase IT on Latest HDP Versions'  MAVEN_PROFILES='hbase-it,hortonworks' BUILD_DOCS=false  IT_ONLY=true
+    - NAME='Accumulo Client IT on Latest ASF Versions' MAVEN_PROFILES='accumulo-it-client' BUILD_DOCS=false  IT_ONLY=true
+    - NAME='Accumulo Server IT on Latest ASF Versions' MAVEN_PROFILES='accumulo-it-server' BUILD_DOCS=false  IT_ONLY=true
+    - NAME='HBase Client IT on Latest ASF Versions' MAVEN_PROFILES='hbase-it-client' BUILD_DOCS=false  IT_ONLY=true
+    - NAME='HBase Server IT on Latest ASF Versions' MAVEN_PROFILES='hbase-it-server' BUILD_DOCS=false  IT_ONLY=true
+    - NAME='Cassandra IT on Latest ASF Versions' MAVEN_PROFILES='cassandra-it' BUILD_DOCS=false  IT_ONLY=true
+    - NAME='DynamoDB IT on Latest ASF Versions' MAVEN_PROFILES='dynamodb-it' BUILD_DOCS=false  IT_ONLY=true
+    - NAME='Bigtable IT on Latest ASF Versions' MAVEN_PROFILES='bigtable-it' BUILD_DOCS=false  IT_ONLY=true
+    - NAME='Accumulo Client IT on Older ASF and GeoTools Versions' MAVEN_PROFILES='accumulo-it-client,compatibility' BUILD_DOCS=false  IT_ONLY=true
+    - NAME='HBase Server IT on Older ASF and GeoTools Versions' MAVEN_PROFILES='hbase-it-server,compatibility' BUILD_DOCS=false  IT_ONLY=true	
+    - NAME='Accumulo Server IT on Latest CDH Versions' MAVEN_PROFILES='accumulo-it-server,cloudera' BUILD_DOCS=false  IT_ONLY=true
+    - NAME='Accumulo Server IT on Latest HDP Versions' MAVEN_PROFILES='accumulo-it-server,hortonworks' BUILD_DOCS=false  IT_ONLY=true 
+    - NAME='HBase Server IT on Latest CDH Versions'  MAVEN_PROFILES='hbase-it-server,cloudera' BUILD_DOCS=false  IT_ONLY=true
+    - NAME='HBase Server IT on Latest HDP Versions'  MAVEN_PROFILES='hbase-it-server,hortonworks' BUILD_DOCS=false  IT_ONLY=true
   global:
     - secure: C3gdJmFavZxvsRVqCLpRUAaisNV3MhWbSYf4Wjs2SBLC/AIEwN0vDWHOuYiUExuXR1kApBVdNffuCX7bt/5siewHgmiVUGyY1xytxDdv+PAKZihlfYZRGattyg3XUbEFwyjr7c74639Y1w45NPUoBMUT4SKHZkfqD+FDU47OHIM=
 
@@ -30,11 +34,11 @@ install:
   - travis_wait 30 .utility/build-docs.sh
 script:
   # This script uses the IT_ONLY flag to determine whether to run unit tests using verify, or ITs only using failsafe
-  - travis_wait 50 .utility/run-tests.sh
+  - .utility/run-tests.sh
 before_install:
   # This must match the dev-resources version from the parent pom:
   - export DEV_RESOURCES_VERSION=1.2
-  - export MAVEN_OPTS="-XX:CompressedClassSpaceSize=256m -XX:+UseSerialGC -Xmx2048m -XX:MaxMetaspaceSize=256m -Dorg.slf4j.simpleLogger.defaultLogLevel=warn"
+  - export MAVEN_OPTS="-XX:CompressedClassSpaceSize=256m -XX:+UseSerialGC -Xmx2g -XX:MaxMetaspaceSize=512m -Dorg.slf4j.simpleLogger.defaultLogLevel=warn"
   - chmod +x .utility/pull-dev-resources.sh
   - chmod +x .utility/run-tests.sh
   - chmod +x .utility/build-docs.sh

.utility/publish-docs.sh

@@ -2,6 +2,7 @@
 
 if [ "$TRAVIS_REPO_SLUG" == "locationtech/geowave" ] && [ "$BUILD_DOCS" == "true" ] && [ "$TRAVIS_PULL_REQUEST" == "false" ] && [ "$TRAVIS_BRANCH" == "master" ]; then  
   echo -e "Generating changelog...\n"
+  export CHANGELOG_GITHUB_TOKEN=$GH_TOKEN
   gem install github_changelog_generator
   github_changelog_generator
   pandoc -f markdown -t html -s -c stylesheets/changelog.css CHANGELOG.md > changelog.html

CONTRIBUTING.md

@@ -0,0 +1,8 @@
+Before your contribution can be accepted by the project, you need to create an Eclipse Foundation 
+account and electronically sign the Eclipse Contributor Agreement (ECA).
+
+- http://www.eclipse.org/legal/ECA.php
+
+For more information on contributing to GeoWave, please see our developer guide here:
+
+- http://locationtech.github.io/geowave/devguide.html#contributions

NOTICE

@@ -1,6 +1,9 @@
 Copyright 2013-2016 RadiantBlue Technologies
 Copyright 2016-2017 DigitalGlobe, Inc.
 Copyright 2013-2017 Booz Allen Hamilton
+Copyright 2010 Lars Francke – RE:  OsmAvro.avsc
+Copyright 2013-2106 Commonwealth Computer Research, Inc. RE:  XZOrderSFC.java
+Copyright 2014 GeoSolutions RE:  WarpNearestOpImage.java
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

README.md

@@ -4,9 +4,9 @@
 
 ## About  
 
-| Continuous Integration | Code Coverage | Static Analysis | Chat |            
+| Continuous Integration | Code Coverage | License | Chat |            
 |:------------------:|:-------------:|:---------------:|:----:|
-| <a href="https://travis-ci.org/locationtech/geowave/branches"><img alt="Travis-CI test status" src="https://travis-ci.org/locationtech/geowave.svg?branch=master"/></a> | <a href='https://coveralls.io/r/locationtech/?branch=master'><img src='https://coveralls.io/repos/locationtech/geowave/badge.svg?branch=master' alt='Coverage Status' /></a> | <a href="https://scan.coverity.com/projects/3371"><img alt="Coverity Scan Build Status" src="https://scan.coverity.com/projects/3371/badge.svg"/></a> | [![Join the chat at https://gitter.im/locationtech/geowave](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/locationtech/geowave?utm_source=badge&utm_medium=badge&utm_content=badge) |  
+| <a href="https://travis-ci.org/locationtech/geowave/branches"><img alt="Travis-CI test status" src="https://travis-ci.org/locationtech/geowave.svg?branch=master"/></a> | <a href='https://coveralls.io/r/locationtech/?branch=master'><img src='https://coveralls.io/repos/locationtech/geowave/badge.svg?branch=master' alt='Coverage Status' /></a> | [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) | [![Join the chat at https://gitter.im/locationtech/geowave](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/locationtech/geowave?utm_source=badge&utm_medium=badge&utm_content=badge) |  
 
 GeoWave is an open source set of software that:
 
@@ -24,17 +24,18 @@ Basically, GeoWave is working to bridge geospatial software with distributed com
 ## The Docs
 * Check out our [GeoWave io page](http://locationtech.github.io/geowave/) page for detailed documentation.
 * A [changelog is available](http://locationtech.github.io/geowave/changelog.html) which details the changes and features for each of our [github releases](https://github.com/locationtech/geowave/releases)
+* The underlying principles employed in GeoWave are outlined in recent academic publications to include largely the background theory in [Advances in Spatial and Temporal Databases 2017](https://link.springer.com/chapter/10.1007/978-3-319-64367-0_6) and a derivative, more applied paper in [FOSS4G Conference Proceedings 2017](http://scholarworks.umass.edu/cgi/viewcontent.cgi?article=1027&context=foss4g).
 
 ## The Software
 * We have a [RPM repository](http://locationtech.github.io/geowave/packages.html)
-  * See [Documentation: Installation from RPM](http://locationtech.github.io/geowave/userguide.html#installation-from-rpm) for more info.
+  * See [Documentation: Installation from RPM](http://locationtech.github.io/geowave/devguide.html#installation-from-rpm) for more info.
   * Deb packages if enough people request them
-* We have [Maven artifact repositories](http://locationtech.github.io/geowave/userguide.html#maven-repositories-2) (indexes not enabled, but it works in a maven repo fragment)
+* We have [Maven artifact repositories](http://locationtech.github.io/geowave/devguide.html#maven-pom-fragments) (indexes not enabled, but it works in a maven repo fragment)
   * Releases: http://geowave-maven.s3-website-us-east-1.amazonaws.com/release
   * Snapshots: http://geowave-maven.s3-website-us-east-1.amazonaws.com/snapshot (nightly)
 * We have a [vagrant dev environment](https://github.com/locationtech/geowave-vagrant)
 * We have a development all in one RPM package: "geowave-cdh5-single-host"
-* And you can always [build from source](http://locationtech.github.io/geowave/userguide.html#installation-from-source)
+* And you can always [build from source](http://locationtech.github.io/geowave/devguide.html#development-setup)
 
 
 ## Community
@@ -58,7 +59,7 @@ We work to maintain a N and N-1 tested and supported version pace for the follow
 
 | Geoserver | Geotools | Accumulo | HBase | Hadoop | PDAL | Mapnik | Java |
 |:---------:|:--------:|:--------:|:-----:|:------:|:----:|:------:|:----:|
-| 2.10.x | 16.x | 1.6.x,1.7.x | 1.1.x,1.2.x,1.3.x | 2.x | 0.9.9 |  master (pull request pending) | Java8 |
+| 2.12.x | 18.x | 1.7.x,1.8.x | 1.1.x,1.2.x,1.3.x | 2.x | 0.9.9 |  3.x | Java8 |
 
 * [Apache Maven](http://maven.apache.org/) 3.x or greater is required for building
 * [Java Advanced Imaging](http://download.java.net/media/jai/builds/release/1_1_3/INSTALL.html) and [Java Image I/O](http://download.java.net/media/jai-imageio/builds/release/1.1/INSTALL-jai_imageio.html) should both be installed on Geoserver for GeoWave versions 0.9.2.1 and below (licensing prohibits us redistributing)
@@ -83,7 +84,4 @@ GeoWave was developed at the National Geospatial-Intelligence Agency (NGA) in co
 All pull request contributions to this project will be released under the Apache 2.0 or compatible license.
 Software source code previously released under an open source license and then modified by NGA staff is considered a "joint work" (see 17 USC § 101); it is partially copyrighted, partially public domain, and as a whole is protected by the copyrights of the non-government authors and must be released according to the terms of the original open source license.
 
-## Everything else
-Check out our talk at the [Accumulo Summit](http://accumulosummit.com/program/talks/geowave-geospatial-and-geotemporal-data-storage-and-retrieval-in-accumulo/).
-
 Did I mention our [documentation!](http://locationtech.github.io/geowave/)

analytics/api/pom.xml

@@ -4,7 +4,7 @@
 		<artifactId>geowave-analytic-parent</artifactId>
 		<groupId>mil.nga.giat</groupId>
 		<relativePath>../</relativePath>
-		<version>0.9.5-SNAPSHOT</version>
+		<version>0.9.7-SNAPSHOT</version>
 	</parent>
 	<artifactId>geowave-analytic-api</artifactId>
 	<name>GeoWave Analytics API</name>

analytics/api/src/main/java/mil/nga/giat/geowave/analytic/AnalyticFeature.java

@@ -13,6 +13,8 @@
 import java.util.List;
 
 import mil.nga.giat.geowave.adapter.vector.FeatureDataAdapter;
+import mil.nga.giat.geowave.core.geotime.ingest.SpatialDimensionalityTypeProvider;
+import mil.nga.giat.geowave.core.geotime.ingest.SpatialOptions;
 
 import org.geotools.feature.simple.SimpleFeatureBuilder;
 import org.geotools.feature.simple.SimpleFeatureTypeBuilder;
@@ -124,8 +126,12 @@ public static FeatureDataAdapter createFeatureAdapter(
 						extraDim,
 						Double.class);
 			}
-			return new FeatureDataAdapter(
+			FeatureDataAdapter adapter = new FeatureDataAdapter(
 					builder.buildFeatureType());
+			// TODO any consumers of this method will not be able to utilize
+			// custom CRS
+			adapter.init(new SpatialDimensionalityTypeProvider().createPrimaryIndex(new SpatialOptions()));
+			return adapter;
 		}
 		catch (final Exception e) {
 			LOGGER.warn(

analytics/api/src/main/java/mil/nga/giat/geowave/analytic/AnalyticPersistableRegistry.java

@@ -0,0 +1,38 @@
+/*******************************************************************************
+ * Copyright (c) 2013-2017 Contributors to the Eclipse Foundation
+ * 
+ * See the NOTICE file distributed with this work for additional
+ * information regarding copyright ownership.
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Apache License,
+ * Version 2.0 which accompanies this distribution and is available at
+ * http://www.apache.org/licenses/LICENSE-2.0.txt
+ ******************************************************************************/
+package mil.nga.giat.geowave.analytic;
+
+import mil.nga.giat.geowave.analytic.clustering.DistortionGroupManagement.BatchIdFilter;
+import mil.nga.giat.geowave.analytic.clustering.DistortionGroupManagement.DistortionDataAdapter;
+import mil.nga.giat.geowave.analytic.store.PersistableStore;
+import mil.nga.giat.geowave.core.index.persist.PersistableRegistrySpi;
+
+public class AnalyticPersistableRegistry implements
+		PersistableRegistrySpi
+{
+	@Override
+	public PersistableIdAndConstructor[] getSupportedPersistables() {
+		return new PersistableIdAndConstructor[] {
+			new PersistableIdAndConstructor(
+					(short) 700,
+					GeoObjectDimensionValues::new),
+			new PersistableIdAndConstructor(
+					(short) 701,
+					BatchIdFilter::new),
+			new PersistableIdAndConstructor(
+					(short) 702,
+					DistortionDataAdapter::new),
+			new PersistableIdAndConstructor(
+					(short) 703,
+					PersistableStore::new)
+		};
+	}
+}

analytics/api/src/main/java/mil/nga/giat/geowave/analytic/GeoObjectDimensionValues.java

@@ -12,7 +12,7 @@
 
 import java.nio.ByteBuffer;
 
-import mil.nga.giat.geowave.core.index.Persistable;
+import mil.nga.giat.geowave.core.index.persist.Persistable;
 
 /**
  * 
@@ -154,5 +154,4 @@ public void fromBinary(
 		}
 
 	}
-
 }

analytics/api/src/main/java/mil/nga/giat/geowave/analytic/GeometryDataSetGenerator.java

@@ -45,6 +45,7 @@
 import mil.nga.giat.geowave.analytic.distance.CoordinateCircleDistanceFn;
 import mil.nga.giat.geowave.analytic.distance.DistanceFn;
 import mil.nga.giat.geowave.core.geotime.ingest.SpatialDimensionalityTypeProvider;
+import mil.nga.giat.geowave.core.geotime.ingest.SpatialOptions;
 import mil.nga.giat.geowave.core.store.DataStore;
 import mil.nga.giat.geowave.core.store.IndexWriter;
 import mil.nga.giat.geowave.core.store.index.PrimaryIndex;
@@ -178,10 +179,11 @@ public void writeToGeoWave(
 			final DataStore dataStore,
 			final List<SimpleFeature> featureData )
 			throws IOException {
-		final PrimaryIndex index = new SpatialDimensionalityTypeProvider().createPrimaryIndex();
+		final PrimaryIndex index = new SpatialDimensionalityTypeProvider().createPrimaryIndex(new SpatialOptions());
 		final FeatureDataAdapter adapter = new FeatureDataAdapter(
 				featureData.get(
 						0).getFeatureType());
+		adapter.init(index);
 		final SimpleFeatureBuilder featureBuilder = new SimpleFeatureBuilder(
 				featureData.get(
 						0).getFeatureType());

analytics/api/src/main/java/mil/nga/giat/geowave/analytic/PropertyManagement.java

@@ -10,11 +10,6 @@
  ******************************************************************************/
 package mil.nga.giat.geowave.analytic;
 
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.ObjectInputStream;
-import java.io.ObjectOutputStream;
-import java.io.OutputStream;
 import java.io.Serializable;
 import java.io.UnsupportedEncodingException;
 import java.util.ArrayList;
@@ -25,8 +20,8 @@
 import mil.nga.giat.geowave.analytic.param.ParameterEnum;
 import mil.nga.giat.geowave.core.geotime.store.query.SpatialQuery;
 import mil.nga.giat.geowave.core.index.ByteArrayUtils;
-import mil.nga.giat.geowave.core.index.Persistable;
-import mil.nga.giat.geowave.core.index.PersistenceUtils;
+import mil.nga.giat.geowave.core.index.persist.Persistable;
+import mil.nga.giat.geowave.core.index.persist.PersistenceUtils;
 import mil.nga.giat.geowave.core.index.sfc.data.NumericRange;
 import mil.nga.giat.geowave.core.store.query.DistributableQuery;
 import mil.nga.giat.geowave.core.store.query.QueryOptions;
@@ -663,15 +658,12 @@ private static byte[] toBytes(
 	}
 
 	private static Persistable fromBytes(
-			final byte[] data,
-			final Class<? extends Persistable> expectedType )
+			final byte[] data )
 			throws InstantiationException,
 			IllegalAccessException,
 			ClassNotFoundException,
 			UnsupportedEncodingException {
-		return PersistenceUtils.fromBinary(
-				data,
-				expectedType);
+		return PersistenceUtils.fromBinary(data);
 	}
 
 	private Object validate(
@@ -780,9 +772,7 @@ public DistributableQuery convert(
 				final Serializable ob )
 				throws Exception {
 			if (ob instanceof byte[]) {
-				return (DistributableQuery) PropertyManagement.fromBytes(
-						(byte[]) ob,
-						DistributableQuery.class);
+				return (DistributableQuery) PropertyManagement.fromBytes((byte[]) ob);
 			}
 			final PrecisionModel precision = new PrecisionModel();
 			final GeometryFactory geometryFactory = new GeometryFactory(
@@ -829,9 +819,7 @@ public QueryOptions convert(
 				final Serializable ob )
 				throws Exception {
 			if (ob instanceof byte[]) {
-				return (QueryOptions) PropertyManagement.fromBytes(
-						(byte[]) ob,
-						QueryOptions.class);
+				return (QueryOptions) PropertyManagement.fromBytes((byte[]) ob);
 			}
 			else if (ob instanceof QueryOptions) {
 				return (QueryOptions) ob;
@@ -977,9 +965,7 @@ public Persistable convert(
 				final Serializable ob )
 				throws Exception {
 			if (ob instanceof byte[]) {
-				return fromBytes(
-						(byte[]) ob,
-						Persistable.class);
+				return fromBytes((byte[]) ob);
 			}
 			throw new IllegalArgumentException(
 					String.format(

analytics/api/src/main/java/mil/nga/giat/geowave/analytic/clustering/CentroidManagerGeoWave.java

@@ -65,6 +65,7 @@
 import mil.nga.giat.geowave.analytic.param.StoreParameters;
 import mil.nga.giat.geowave.analytic.store.PersistableStore;
 import mil.nga.giat.geowave.core.geotime.ingest.SpatialDimensionalityTypeProvider;
+import mil.nga.giat.geowave.core.geotime.ingest.SpatialOptions;
 import mil.nga.giat.geowave.core.index.ByteArrayId;
 import mil.nga.giat.geowave.core.store.CloseableIterator;
 import mil.nga.giat.geowave.core.store.DataStore;
@@ -248,7 +249,8 @@ private void init(
 
 		final String indexId = scopedJob.getString(
 				CentroidParameters.Centroid.INDEX_ID,
-				new SpatialDimensionalityTypeProvider().createPrimaryIndex().getId().getString());
+				new SpatialDimensionalityTypeProvider().createPrimaryIndex(
+						new SpatialOptions()).getId().getString());
 		final PersistableStore store = (PersistableStore) StoreParameters.StoreParam.INPUT_STORE.getHelper().getValue(
 				context,
 				scope,
@@ -607,7 +609,7 @@ SimpleFeature toSimpleFeature(
 				AnalyticItemWrapper<T> item );
 	}
 
-	private static FeatureDataAdapter createFeatureAdapter(
+	private static SimpleFeatureType createFeatureType(
 			final SimpleFeatureType featureType,
 			final Class<? extends Geometry> shapeClass ) {
 		try {
@@ -627,8 +629,7 @@ private static FeatureDataAdapter createFeatureAdapter(
 							attr.getType().getBinding());
 				}
 			}
-			return new FeatureDataAdapter(
-					builder.buildFeatureType());
+			return builder.buildFeatureType();
 		}
 		catch (final Exception e) {
 			LOGGER.warn(
@@ -659,14 +660,14 @@ private class SimpleFeatureConverter implements
 			ToSimpleFeatureConverter<T>
 	{
 
-		final FeatureDataAdapter adapter;
+		final SimpleFeatureType type;
 		final Object[] defaults;
 		final Class<? extends Geometry> shapeClass;
 
 		public SimpleFeatureConverter(
 				final FeatureDataAdapter adapter,
 				final Class<? extends Geometry> shapeClass ) {
-			this.adapter = createFeatureAdapter(
+			type = createFeatureType(
 					adapter.getFeatureType(),
 					shapeClass);
 			int p = 0;
@@ -680,14 +681,14 @@ public SimpleFeatureConverter(
 
 		@Override
 		public SimpleFeatureType getFeatureType() {
-			return adapter.getFeatureType();
+			return type;
 		}
 
 		@Override
 		public SimpleFeature toSimpleFeature(
 				final AnalyticItemWrapper<T> item ) {
 			final SimpleFeature newFeature = SimpleFeatureBuilder.build(
-					adapter.getFeatureType(),
+					type,
 					defaults,
 					item.getID());
 			int i = 0;