Skip to content

Commit

Permalink
Add options to gpx ingest to simplify geometry, and exclude tracks th…
Browse files Browse the repository at this point in the history
…at are too complex or cover too much area. (#1345)
  • Loading branch information
JWileczek authored and rfecher committed Jun 18, 2018
1 parent dc886e6 commit b58e6fa
Show file tree
Hide file tree
Showing 11 changed files with 242 additions and 27 deletions.
Expand Up @@ -29,6 +29,7 @@ private AbstractSimpleFeatureIngestPlugin<I> getInstance(
myInstance.setFilterProvider(myOptions.getCqlFilterOptionProvider());
myInstance.setTypeNameProvider(myOptions.getTypeNameOptionProvider());
myInstance.setSerializationFormatProvider(myOptions.getSerializationFormatOptionProvider());
myInstance.setGeometrySimpOptionProvider(myOptions.getGeometrySimpOptionProvider());
return myInstance;
}

Expand Down
Expand Up @@ -10,7 +10,6 @@
******************************************************************************/
package mil.nga.giat.geowave.adapter.vector.ingest;

import java.io.File;
import java.net.URL;
import java.nio.ByteBuffer;
import java.util.ArrayList;
Expand All @@ -24,6 +23,7 @@

import com.google.common.base.Predicate;
import com.google.common.collect.Iterators;
import com.vividsolutions.jts.geom.Geometry;

import mil.nga.giat.geowave.adapter.vector.AvroFeatureDataAdapter;
import mil.nga.giat.geowave.adapter.vector.FeatureDataAdapter;
Expand All @@ -50,6 +50,7 @@ abstract public class AbstractSimpleFeatureIngestPlugin<I> implements
protected CQLFilterOptionProvider filterOptionProvider = new CQLFilterOptionProvider();
protected FeatureSerializationOptionProvider serializationFormatOptionProvider = new FeatureSerializationOptionProvider();
protected TypeNameOptionProvider typeNameProvider = new TypeNameOptionProvider();
protected GeometrySimpOptionProvider simpOptionProvider = new GeometrySimpOptionProvider();

public void setFilterProvider(
final CQLFilterOptionProvider filterOptionProvider ) {
Expand All @@ -66,17 +67,28 @@ public void setTypeNameProvider(
this.typeNameProvider = typeNameProvider;
}

public void setGeometrySimpOptionProvider(
final GeometrySimpOptionProvider geometryProvider ) {
this.simpOptionProvider = geometryProvider;
}

@Override
public byte[] toBinary() {
final byte[] filterBinary = filterOptionProvider.toBinary();
final byte[] typeNameBinary = typeNameProvider.toBinary();
final ByteBuffer buf = ByteBuffer.allocate(filterBinary.length + typeNameBinary.length + 4);
final byte[] simpBinary = simpOptionProvider.toBinary();
final byte[] backingBuffer = new byte[filterBinary.length + typeNameBinary.length + simpBinary.length
+ (Integer.BYTES * 2)];
final ByteBuffer buf = ByteBuffer.wrap(backingBuffer);
buf.putInt(filterBinary.length);
buf.put(filterBinary);
buf.putInt(typeNameBinary.length);
buf.put(typeNameBinary);
buf.put(simpBinary);

return ArrayUtils.addAll(
serializationFormatOptionProvider.toBinary(),
buf.array());
backingBuffer);
}

@Override
Expand All @@ -95,10 +107,16 @@ public void fromBinary(
final ByteBuffer buf = ByteBuffer.wrap(otherBytes);
final int filterBinaryLength = buf.getInt();
final byte[] filterBinary = new byte[filterBinaryLength];
final byte[] typeNameBinary = new byte[otherBytes.length - filterBinaryLength - 4];
buf.get(filterBinary);

final int typeNameBinaryLength = buf.getInt();
final byte[] typeNameBinary = new byte[typeNameBinaryLength];
buf.get(typeNameBinary);

final byte[] geometrySimpBinary = new byte[otherBytes.length - filterBinary.length - typeNameBinary.length
- (Integer.BYTES * 2)];
buf.get(geometrySimpBinary);

serializationFormatOptionProvider = new FeatureSerializationOptionProvider();
serializationFormatOptionProvider.fromBinary(kryoBytes);

Expand All @@ -107,6 +125,9 @@ public void fromBinary(

typeNameProvider = new TypeNameOptionProvider();
typeNameProvider.fromBinary(typeNameBinary);

simpOptionProvider = new GeometrySimpOptionProvider();
simpOptionProvider.fromBinary(geometrySimpBinary);
}

protected WritableDataAdapter<SimpleFeature> newAdapter(
Expand Down Expand Up @@ -176,6 +197,13 @@ protected CloseableIterator<GeoWaveData<SimpleFeature>> wrapIteratorWithFilters(
else {
internalTypeNameProvider = null;
}
final GeometrySimpOptionProvider internalSimpOptionProvider;
if ((simpOptionProvider != null)) {
internalSimpOptionProvider = simpOptionProvider;
}
else {
internalSimpOptionProvider = null;
}
if ((internalFilterProvider != null) || (internalTypeNameProvider != null)) {
final Iterator<GeoWaveData<SimpleFeature>> it = Iterators.filter(
geowaveData,
Expand All @@ -190,6 +218,16 @@ public boolean apply(
if ((internalFilterProvider != null) && !internalFilterProvider.evaluate(input.getValue())) {
return false;
}
if ((internalSimpOptionProvider != null)) {
Geometry simpGeom = internalSimpOptionProvider.simplifyGeometry((Geometry) input
.getValue()
.getDefaultGeometry());
if (!internalSimpOptionProvider.filterGeometry(simpGeom)) {
return false;
}
input.getValue().setDefaultGeometry(
simpGeom);
}
return true;
}
});
Expand Down
@@ -0,0 +1,72 @@
package mil.nga.giat.geowave.adapter.vector.ingest;

import java.nio.ByteBuffer;

import com.beust.jcommander.Parameter;
import com.vividsolutions.jts.geom.Geometry;
import com.vividsolutions.jts.simplify.DouglasPeuckerSimplifier;

import mil.nga.giat.geowave.core.index.ByteArrayUtils;
import mil.nga.giat.geowave.core.index.persist.Persistable;
import mil.nga.giat.geowave.core.store.adapter.statistics.histogram.ByteUtils;

public class GeometrySimpOptionProvider implements
Persistable
{
@Parameter(names = "--maxVertices", description = "Maximum number of vertices to allow for the feature. Features with over this vertice count will be discarded.")
private int maxVertices = Integer.MAX_VALUE;

@Parameter(names = "--minSimpVertices", description = "Minimum vertex count to qualify for geometry simplification.")
private int simpVertMin = Integer.MAX_VALUE;

@Parameter(names = "--tolerance", description = "Maximum error tolerance in geometry simplification. Should range from 0.0 to 1.0 (i.e. .1 = 10%)")
private double tolerance = 0.02;

public Geometry simplifyGeometry(
Geometry geom ) {
if (geom.getCoordinates().length > this.simpVertMin) {
return DouglasPeuckerSimplifier.simplify(
geom,
this.tolerance);
}
return geom;
}

public boolean filterGeometry(
Geometry geom ) {
return (geom.getCoordinates().length < this.maxVertices && !geom.isEmpty() && geom.isValid());
}

@Override
public byte[] toBinary() {
final byte[] backingBuffer = new byte[Integer.BYTES * 2 + Double.BYTES];
ByteBuffer buf = ByteBuffer.wrap(backingBuffer);
buf.putInt(
maxVertices).putInt(
simpVertMin).putDouble(
tolerance);
return backingBuffer;
}

@Override
public void fromBinary(
byte[] bytes ) {
ByteBuffer buf = ByteBuffer.wrap(bytes);
maxVertices = buf.getInt();
simpVertMin = buf.getInt();
tolerance = buf.getDouble();
}

public int getMaxVertices() {
return maxVertices;
}

public int getSimpLimit() {
return simpVertMin;
}

public double getTolerance() {
return tolerance;
}

}
Expand Up @@ -30,11 +30,23 @@ public class SimpleFeatureIngestOptions implements
@ParametersDelegate
private FeatureSerializationOptionProvider serializationFormatOptionProvider = new FeatureSerializationOptionProvider();

@ParametersDelegate
private GeometrySimpOptionProvider simpOptionProvider = new GeometrySimpOptionProvider();

@ParametersDelegate
private Object pluginOptions = null;

public SimpleFeatureIngestOptions() {}

public GeometrySimpOptionProvider getGeometrySimpOptionProvider() {
return simpOptionProvider;
}

public void setGeometrySimpOptionProvider(
final GeometrySimpOptionProvider simpOptionProvider ) {
this.simpOptionProvider = simpOptionProvider;
}

public CQLFilterOptionProvider getCqlFilterOptionProvider() {
return cqlFilterOptionProvider;
}
Expand Down
Expand Up @@ -36,15 +36,20 @@
import mil.nga.giat.geowave.core.index.StringUtils;
import mil.nga.giat.geowave.core.ingest.GeoWaveData;
import mil.nga.giat.geowave.core.store.CloseableIterator;
import com.vividsolutions.jts.simplify.DouglasPeuckerSimplifier;
import com.vividsolutions.jts.simplify.TopologyPreservingSimplifier;

import org.apache.commons.io.IOUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.geotools.feature.simple.SimpleFeatureBuilder;
import org.opengis.feature.simple.SimpleFeature;
import org.opengis.feature.simple.SimpleFeatureType;
import org.opengis.geometry.BoundingBox;

import com.vividsolutions.jts.geom.Coordinate;
import com.vividsolutions.jts.geom.Geometry;
import com.vividsolutions.jts.geom.LineString;

/**
* Consumes a GPX file. The consumer is an iterator, parsing the input stream
Expand Down Expand Up @@ -93,12 +98,12 @@ public class GPXConsumer implements
final String globalVisibility;
final Map<String, Map<String, String>> additionalData;
final boolean uniqueWayPoints;
final double maxLength;

final XMLInputFactory inputFactory = XMLInputFactory.newInstance();

final Stack<GPXDataElement> currentElementStack = new Stack<GPXDataElement>();
final GPXDataElement top = new GPXDataElement(
"gpx");
GPXDataElement top = null;

static final NumberFormat LatLongFormat = new DecimalFormat(
"0000000000");
Expand Down Expand Up @@ -129,14 +134,19 @@ public GPXConsumer(
final String inputID,
final Map<String, Map<String, String>> additionalData,
final boolean uniqueWayPoints,
final String globalVisibility ) {
final String globalVisibility,
final double maxLength ) {
super();
this.fileStream = fileStream;
this.primaryIndexIds = primaryIndexIds;
this.inputID = inputID != null ? inputID : "";
this.uniqueWayPoints = uniqueWayPoints;
this.additionalData = additionalData;
this.globalVisibility = globalVisibility;
this.maxLength = maxLength;
this.top = new GPXDataElement(
"gpx",
this.maxLength);
pointBuilder = new SimpleFeatureBuilder(
pointType);
waypointBuilder = new SimpleFeatureBuilder(
Expand Down Expand Up @@ -237,7 +247,8 @@ private GeoWaveData<SimpleFeature> getNext()
node,
currentElement)) {
final GPXDataElement newElement = new GPXDataElement(
event.asStartElement().getName().getLocalPart());
event.asStartElement().getName().getLocalPart(),
this.maxLength);
currentElement.addChild(newElement);
currentElement = newElement;
currentElementStack.push(currentElement);
Expand Down Expand Up @@ -498,11 +509,20 @@ private static class GPXDataElement
long id = 0;
int childIdCounter = 0;

double maxLineLength = Double.MAX_VALUE;

public GPXDataElement(
final String myElType ) {
elementType = myElType;
}

public GPXDataElement(
final String myElType,
final double maxLength ) {
elementType = myElType;
maxLineLength = maxLength;
}

@Override
public String toString() {
return elementType;
Expand Down Expand Up @@ -754,25 +774,27 @@ public boolean build(

final List<Coordinate> childSequence = buildCoordinates();

if (childSequence.size() == 0) {
int childCoordCount = childSequence.size();
if (childCoordCount <= 1) {
return false;
}

if (childSequence.size() > 1) {
builder.set(
"geometry",
GeometryUtils.GEOMETRY_FACTORY.createLineString(childSequence
.toArray(new Coordinate[childSequence.size()])));
}
else {
builder.set(
"geometry",
GeometryUtils.GEOMETRY_FACTORY.createPoint(childSequence.get(0)));
LineString geom = GeometryUtils.GEOMETRY_FACTORY.createLineString(childSequence
.toArray(new Coordinate[childSequence.size()]));

// Filter gpx track based on maxExtent
if (geom.isEmpty() || geom.getEnvelopeInternal().maxExtent() > this.maxLineLength) {
return false;
}

builder.set(
"geometry",
geom);

setAttribute(
builder,
"NumberPoints",
Long.valueOf(childSequence.size()));
Long.valueOf(childCoordCount));

final Long minTime = getStartTime();
if (minTime != null) {
Expand Down
Expand Up @@ -11,7 +11,12 @@
package mil.nga.giat.geowave.format.gpx;

import mil.nga.giat.geowave.adapter.vector.ingest.AbstractSimpleFeatureIngestPlugin;
import mil.nga.giat.geowave.adapter.vector.ingest.GeometrySimpOptionProvider;
import mil.nga.giat.geowave.adapter.vector.ingest.SimpleFeatureIngestOptions;
import mil.nga.giat.geowave.core.ingest.spi.IngestFormatOptionProvider;

import com.beust.jcommander.ParametersDelegate;

import mil.nga.giat.geowave.adapter.vector.ingest.AbstractSimpleFeatureIngestFormat;

/**
Expand All @@ -22,10 +27,14 @@
public class GpxIngestFormat extends
AbstractSimpleFeatureIngestFormat<GpxTrack>
{
private final MaxExtentOptProvider extentOptProvider = new MaxExtentOptProvider();

@Override
protected AbstractSimpleFeatureIngestPlugin<GpxTrack> newPluginInstance(
IngestFormatOptionProvider options ) {
return new GpxIngestPlugin();
GpxIngestPlugin plugin = new GpxIngestPlugin();
plugin.setExtentOptionProvider(extentOptProvider);
return plugin;
}

@Override
Expand All @@ -38,4 +47,9 @@ public String getIngestFormatDescription() {
return "xml files adhering to the schema of gps exchange format";
}

@Override
protected Object internalGetIngestFormatOptionProviders() {
return extentOptProvider;
}

}

0 comments on commit b58e6fa

Please sign in to comment.