Skip to content

Commit

Permalink
Add 'mvt' field type format to geo fields (#75367) (#75771)
Browse files Browse the repository at this point in the history
In this commit we extend the Fields API formats for geo data in order to produce vector tiles features directly
on the data nodes. That helps the vector tile API to reduce the size of the data that needs to pull in order to
create the answer.
  • Loading branch information
iverase committed Jul 28, 2021
1 parent 8ad7307 commit 977f8fc
Show file tree
Hide file tree
Showing 21 changed files with 464 additions and 118 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/

package org.elasticsearch.common.geo;

import org.elasticsearch.geometry.Geometry;
import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils;

import java.util.List;
import java.util.Locale;
import java.util.function.Function;

/**
* Output formatters for geo fields. Adds support for vector tiles.
*/
public class GeoFormatterFactory {

@FunctionalInterface
public interface VectorTileEngine<T> {
/**
* Returns a formatter for a specific tile.
*/
Function<List<T>, List<Object>> getFormatter(int z, int x, int y, int extent);
}

private static final String MVT = "mvt";

/**
* Returns a formatter by name
*/
public static <T> Function<List<T>, List<Object>> getFormatter(String format, Function<T, Geometry> toGeometry,
VectorTileEngine<T> mvt) {
final int start = format.indexOf('(');
if (start == -1) {
return GeometryFormatterFactory.getFormatter(format, toGeometry);
}
final String formatName = format.substring(0, start);
if (MVT.equals(formatName) == false) {
throw new IllegalArgumentException("Invalid format: " + formatName);
}
final String param = format.substring(start + 1, format.length() - 1);
// we expect either z/x/y or z/x/y@extent
final String[] parts = param.split("@", 3);
if (parts.length > 2) {
throw new IllegalArgumentException(
"Invalid mvt formatter parameter [" + param + "]. Must have the form \"zoom/x/y\" or \"zoom/x/y@extent\"."
);
}
final int extent = parts.length == 2 ? Integer.parseInt(parts[1]) : 4096;
final String[] tileBits = parts[0].split("/", 4);
if (tileBits.length != 3) {
throw new IllegalArgumentException(
"Invalid tile string [" + parts[0] + "]. Must be three integers in a form \"zoom/x/y\"."
);
}
final int z = GeoTileUtils.checkPrecisionRange(Integer.parseInt(tileBits[0]));
final int tiles = 1 << z;
final int x = Integer.parseInt(tileBits[1]);
final int y = Integer.parseInt(tileBits[2]);
if (x < 0 || y < 0 || x >= tiles || y >= tiles) {
throw new IllegalArgumentException(String.format(Locale.ROOT, "Zoom/X/Y combination is not valid: %d/%d/%d", z, x, y));
}
return mvt.getFormatter(z, x, y, extent);
}
}
Original file line number Diff line number Diff line change
@@ -1,26 +1,26 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/

package org.elasticsearch.xpack.vectortile.feature;
package org.elasticsearch.common.geo;

import org.apache.lucene.util.BitUtil;
import org.elasticsearch.common.geo.SphericalMercatorUtils;
import org.elasticsearch.common.io.stream.BytesStreamOutput;
import org.elasticsearch.geometry.Point;
import org.elasticsearch.geometry.Rectangle;
import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils;

import java.io.IOException;
import java.io.UncheckedIOException;
import java.util.Comparator;
import java.util.List;

/**
* Similar to {@link FeatureFactory} but only supports points and rectangles. It is just
* more efficient for those shapes and it does not use external dependencies.
* Transforms points and rectangles objects in WGS84 into mvt features.
*/
public class SimpleFeatureFactory {

Expand Down Expand Up @@ -64,11 +64,11 @@ public byte[] point(double lon, double lat) throws IOException {
/**
* Returns a {@code byte[]} containing the mvt representation of the provided points
*/
public byte[] points(List<Point> multiPoint) throws IOException {
multiPoint.sort(Comparator.comparingDouble(Point::getLon).thenComparingDouble(Point::getLat));
public byte[] points(List<GeoPoint> multiPoint) {
multiPoint.sort(Comparator.comparingDouble(GeoPoint::getLon).thenComparingDouble(GeoPoint::getLat));
final int[] commands = new int[2 * multiPoint.size() + 1];
int pos = 1, prevLon = 0, prevLat = 0, numPoints = 0;
for (Point point : multiPoint) {
for (GeoPoint point : multiPoint) {
final int posLon = lon(point.getLon());
if (posLon > extent || posLon < 0) {
continue;
Expand All @@ -90,7 +90,11 @@ public byte[] points(List<Point> multiPoint) throws IOException {
return EMPTY;
}
commands[0] = encodeCommand(MOVETO, numPoints);
return writeCommands(commands, 1, pos);
try {
return writeCommands(commands, 1, pos);
} catch (IOException ioe) {
throw new UncheckedIOException(ioe);
}
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,13 @@
import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.common.CheckedBiFunction;
import org.elasticsearch.common.Explicit;
import org.elasticsearch.common.geo.GeoFormatterFactory;
import org.elasticsearch.common.geo.GeoPoint;
import org.elasticsearch.common.geo.GeoShapeUtils;
import org.elasticsearch.common.geo.GeoUtils;
import org.elasticsearch.common.geo.GeometryFormatterFactory;
import org.elasticsearch.common.geo.ShapeRelation;
import org.elasticsearch.common.geo.SimpleFeatureFactory;
import org.elasticsearch.common.unit.DistanceUnit;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.support.MapXContentParser;
Expand Down Expand Up @@ -243,7 +245,11 @@ public String typeName() {

@Override
protected Function<List<GeoPoint>, List<Object>> getFormatter(String format) {
return GeometryFormatterFactory.getFormatter(format, p -> new Point(p.lon(), p.lat()));
return GeoFormatterFactory.getFormatter(format, p -> new Point(p.getLon(), p.getLat()),
(z, x, y, extent) -> {
final SimpleFeatureFactory featureFactory = new SimpleFeatureFactory(z, x, y, extent);
return points -> org.elasticsearch.core.List.of(featureFactory.points(points));
});
}

@Override
Expand Down
Original file line number Diff line number Diff line change
@@ -1,14 +1,14 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/

package org.elasticsearch.xpack.vectortile.feature;
package org.elasticsearch.common.geo;

import org.apache.lucene.geo.GeoTestUtil;
import org.elasticsearch.geometry.Point;
import org.elasticsearch.geometry.Rectangle;
import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils;
import org.elasticsearch.test.ESTestCase;
Expand Down Expand Up @@ -48,7 +48,7 @@ public void testPoint() throws IOException {
}
}

public void testMultiPoint() throws IOException {
public void testMultiPoint() {
int z = randomIntBetween(3, 10);
int x = randomIntBetween(0, (1 << z) - 1);
int y = randomIntBetween(0, (1 << z) - 1);
Expand All @@ -57,20 +57,20 @@ public void testMultiPoint() throws IOException {
Rectangle rectangle = GeoTileUtils.toBoundingBox(x, y, z);
int numPoints = randomIntBetween(2, 10);
{
List<Point> points = new ArrayList<>();
List<GeoPoint> points = new ArrayList<>();
double lat = randomValueOtherThanMany((l) -> rectangle.getMinY() >= l || rectangle.getMaxY() <= l, GeoTestUtil::nextLatitude);
double lon = randomValueOtherThanMany((l) -> rectangle.getMinX() >= l || rectangle.getMaxX() <= l, GeoTestUtil::nextLongitude);
points.add(new Point(lon, lat));
points.add(new GeoPoint(lat, lon));
for (int i = 0; i < numPoints - 1; i++) {
points.add(new Point(GeoTestUtil.nextLongitude(), GeoTestUtil.nextLatitude()));
points.add(new GeoPoint(GeoTestUtil.nextLatitude(), GeoTestUtil.nextLongitude()));
}
assertThat(builder.points(points).length, Matchers.greaterThan(0));
}
{
int xNew = randomValueOtherThanMany(v -> Math.abs(v - x) < 2, () -> randomIntBetween(0, (1 << z) - 1));
int yNew = randomValueOtherThanMany(v -> Math.abs(v - y) < 2, () -> randomIntBetween(0, (1 << z) - 1));
Rectangle rectangleNew = GeoTileUtils.toBoundingBox(xNew, yNew, z);
List<Point> points = new ArrayList<>();
List<GeoPoint> points = new ArrayList<>();
for (int i = 0; i < numPoints; i++) {
double lat = randomValueOtherThanMany(
(l) -> rectangleNew.getMinY() >= l || rectangleNew.getMaxY() <= l,
Expand All @@ -80,7 +80,7 @@ public void testMultiPoint() throws IOException {
(l) -> rectangleNew.getMinX() >= l || rectangleNew.getMaxX() <= l,
GeoTestUtil::nextLongitude
);
points.add(new Point(lon, lat));
points.add(new GeoPoint(lat, lon));
}
assertThat(builder.points(points).length, Matchers.equalTo(0));
}
Expand All @@ -95,24 +95,24 @@ public void testPointsMethodConsistency() throws IOException {
Rectangle rectangle = GeoTileUtils.toBoundingBox(x, y, z);
int extraPoints = randomIntBetween(1, 10);
{
List<Point> points = new ArrayList<>();
List<GeoPoint> points = new ArrayList<>();
double lat = randomValueOtherThanMany((l) -> rectangle.getMinY() > l || rectangle.getMaxY() < l, GeoTestUtil::nextLatitude);
double lon = randomValueOtherThanMany((l) -> rectangle.getMinX() > l || rectangle.getMaxX() < l, GeoTestUtil::nextLongitude);
points.add(new Point(lon, lat));
points.add(new GeoPoint(lat, lon));
assertArrayEquals(builder.points(points), builder.point(lon, lat));
for (int i = 0; i < extraPoints; i++) {
points.add(new Point(lon, lat));
points.add(new GeoPoint(lat, lon));
}
assertArrayEquals(builder.points(points), builder.point(lon, lat));
}
{
List<Point> points = new ArrayList<>();
List<GeoPoint> points = new ArrayList<>();
double lat = randomValueOtherThanMany((l) -> rectangle.getMinY() <= l && rectangle.getMaxY() >= l, GeoTestUtil::nextLatitude);
double lon = randomValueOtherThanMany((l) -> rectangle.getMinX() <= l && rectangle.getMaxX() >= l, GeoTestUtil::nextLongitude);
points.add(new Point(lon, lat));
points.add(new GeoPoint(lat, lon));
assertArrayEquals(builder.points(points), builder.point(lon, lat));
for (int i = 0; i < extraPoints; i++) {
points.add(new Point(lon, lat));
points.add(new GeoPoint(lat, lon));
}
assertArrayEquals(builder.points(points), builder.point(lon, lat));
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,18 @@

package org.elasticsearch.index.mapper;

import org.apache.lucene.geo.GeoTestUtil;
import org.elasticsearch.common.geo.GeoPoint;
import org.elasticsearch.common.geo.SimpleFeatureFactory;
import org.elasticsearch.script.ScriptCompiler;
import org.hamcrest.Matchers;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

public class GeoPointFieldTypeTests extends FieldTypeTestCase {
Expand Down Expand Up @@ -60,4 +66,36 @@ public void testFetchSourceValue() throws IOException {
sourceValue = "malformed";
assertEquals(Collections.emptyList(), fetchSourceValue(mapper, sourceValue, null));
}

public void testFetchVectorTile() throws IOException {
MappedFieldType mapper
= new GeoPointFieldMapper.Builder("field", ScriptCompiler.NONE, false).build(new ContentPath()).fieldType();
final int z = randomIntBetween(1, 10);
int x = randomIntBetween(0, (1 << z) - 1);
int y = randomIntBetween(0, (1 << z) - 1);
final SimpleFeatureFactory featureFactory;
final String mvtString;
if (randomBoolean()) {
int extent = randomIntBetween(1 << 8, 1 << 14);
mvtString = "mvt(" + z + "/" + x + "/" + y + "@" + extent + ")";
featureFactory = new SimpleFeatureFactory(z, x, y, extent);
} else {
mvtString = "mvt(" + z + "/" + x + "/" + y + ")";
featureFactory = new SimpleFeatureFactory(z, x, y, 4096);
}
List<GeoPoint> geoPoints = new ArrayList<>();
List<List<Double>> values = new ArrayList<>();
for (int i = 0; i < randomIntBetween(1, 10); i++) {
final double lat = GeoTestUtil.nextLatitude();
final double lon = GeoTestUtil.nextLongitude();
List<?> sourceValue = fetchSourceValue(mapper, org.elasticsearch.core.List.of(lon, lat), mvtString);
assertThat(sourceValue.size(), Matchers.equalTo(1));
assertThat(sourceValue.get(0), Matchers.equalTo(featureFactory.point(lon, lat)));
geoPoints.add(new GeoPoint(lat, lon));
values.add(org.elasticsearch.core.List.of(lon, lat));
}
List<?> sourceValue = fetchSourceValue(mapper, values, mvtString);
assertThat(sourceValue.size(), Matchers.equalTo(1));
assertThat(sourceValue.get(0), Matchers.equalTo(featureFactory.points(geoPoints)));
}
}
1 change: 1 addition & 0 deletions x-pack/plugin/spatial/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ esplugin {
dependencies {
compileOnly project(path: xpackModule('core'))
testImplementation(testArtifact(project(xpackModule('core'))))
testImplementation project(path: xpackModule('vector-tile'))
yamlRestTestImplementation(testArtifact(project(xpackModule('core'))))
api project(path: ':modules:geo')
restTestConfig project(path: ':modules:geo', configuration: 'restTests')
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
*/
package org.elasticsearch.xpack.spatial;

import org.apache.lucene.util.SetOnce;
import org.elasticsearch.action.ActionRequest;
import org.elasticsearch.action.ActionResponse;
import org.elasticsearch.common.inject.Module;
Expand All @@ -16,6 +17,7 @@
import org.elasticsearch.license.LicenseUtils;
import org.elasticsearch.license.XPackLicenseState;
import org.elasticsearch.plugins.ActionPlugin;
import org.elasticsearch.plugins.ExtensiblePlugin;
import org.elasticsearch.plugins.IngestPlugin;
import org.elasticsearch.plugins.MapperPlugin;
import org.elasticsearch.plugins.SearchPlugin;
Expand Down Expand Up @@ -60,7 +62,7 @@

import static java.util.Collections.singletonList;

public class SpatialPlugin extends GeoPlugin implements MapperPlugin, ActionPlugin, SearchPlugin, IngestPlugin {
public class SpatialPlugin extends GeoPlugin implements MapperPlugin, ActionPlugin, SearchPlugin, IngestPlugin, ExtensiblePlugin {
private final SpatialUsage usage = new SpatialUsage();

public Collection<Module> createGuiceModules() {
Expand All @@ -74,6 +76,9 @@ protected XPackLicenseState getLicenseState() {
return XPackPlugin.getSharedLicenseState();
}

// register the vector tile factory from a different module
private final SetOnce<VectorTileExtension> vectorTileExtension = new SetOnce<>();

@Override
public List<ActionPlugin.ActionHandler<? extends ActionRequest, ? extends ActionResponse>> getActions() {
return singletonList(new ActionPlugin.ActionHandler<>(SpatialStatsAction.INSTANCE, SpatialStatsTransportAction.class));
Expand All @@ -84,7 +89,8 @@ public Map<String, Mapper.TypeParser> getMappers() {
Map<String, Mapper.TypeParser> mappers = new HashMap<>(super.getMappers());
mappers.put(ShapeFieldMapper.CONTENT_TYPE, ShapeFieldMapper.PARSER);
mappers.put(PointFieldMapper.CONTENT_TYPE, PointFieldMapper.PARSER);
mappers.put(GeoShapeWithDocValuesFieldMapper.CONTENT_TYPE, GeoShapeWithDocValuesFieldMapper.PARSER);
mappers.put(GeoShapeWithDocValuesFieldMapper.CONTENT_TYPE,
new GeoShapeWithDocValuesFieldMapper.TypeParser(vectorTileExtension.get()));
return Collections.unmodifiableMap(mappers);
}

Expand Down Expand Up @@ -205,4 +211,10 @@ private <T> ContextParser<String, T> checkLicense(ContextParser<String, T> realP
return realParser.parse(parser, name);
};
}

@Override
public void loadExtensions(ExtensionLoader loader) {
// we only expect one vector tile extension that comes from the vector tile module.
loader.loadExtensions(VectorTileExtension.class).forEach(vectorTileExtension::set);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/

package org.elasticsearch.xpack.spatial;

import org.elasticsearch.common.geo.GeoFormatterFactory;
import org.elasticsearch.geometry.Geometry;


public interface VectorTileExtension {
/**
* Get the vector tile engine. This is called when user ask for the MVT format on the field API.
* We are only expecting one instance of a vector tile engine coming from the vector tile module.
*/
GeoFormatterFactory.VectorTileEngine<Geometry> getVectorTileEngine();
}

0 comments on commit 977f8fc

Please sign in to comment.