From 3cb50604971cc9cd670045c0fcaa53b87ff37c91 Mon Sep 17 00:00:00 2001 From: Andreas Hocevar Date: Mon, 28 Aug 2017 19:05:10 +0200 Subject: [PATCH] Parse PBF directly in ol.format.MVT This change also fixes a previous issue, where MultiPolygon geometries for ol.Feature features were never created. --- doc/errors/index.md | 4 + package.json | 6 - src/ol/format/mvt.js | 367 +++++++++++++++++++++----------- src/ol/source/vectortile.js | 1 - test/spec/ol/format/mvt.test.js | 126 ++++++++--- 5 files changed, 347 insertions(+), 157 deletions(-) diff --git a/doc/errors/index.md b/doc/errors/index.md index e6b14708a81..4d7770b5096 100644 --- a/doc/errors/index.md +++ b/doc/errors/index.md @@ -224,3 +224,7 @@ At least 2 conditions are required. ### 58 Duplicate item added to a unique collection. For example, it may be that you tried to add the same layer to a map twice. Check for calls to `map.addLayer()` or other places where the map's layer collection is modified. + +### 59 + +Invalid command found in the PBF. This indicates that the loaded vector tile may be corrupt. diff --git a/package.json b/package.json index 238e492a3c9..324aa2d40f0 100644 --- a/package.json +++ b/package.json @@ -50,7 +50,6 @@ "rollup-plugin-commonjs": "^8.0.2", "rollup-plugin-node-resolve": "^3.0.0", "temp": "0.8.3", - "@mapbox/vector-tile": "1.3.0", "walk": "2.3.9" }, "devDependencies": { @@ -148,11 +147,6 @@ { "module": "pixelworks", "import": "Processor" - }, - { - "module": "@mapbox/vector-tile", - "name": "vectortile", - "import": "VectorTile" } ] } diff --git a/src/ol/format/mvt.js b/src/ol/format/mvt.js index 350754c568b..655a73de9d3 100644 --- a/src/ol/format/mvt.js +++ b/src/ol/format/mvt.js @@ -3,8 +3,8 @@ goog.provide('ol.format.MVT'); goog.require('ol'); +goog.require('ol.asserts'); goog.require('ol.ext.PBF'); -goog.require('ol.ext.vectortile.VectorTile'); goog.require('ol.format.Feature'); goog.require('ol.format.FormatType'); goog.require('ol.geom.GeometryLayout'); @@ -12,8 +12,10 @@ goog.require('ol.geom.GeometryType'); goog.require('ol.geom.LineString'); goog.require('ol.geom.MultiLineString'); goog.require('ol.geom.MultiPoint'); +goog.require('ol.geom.MultiPolygon'); goog.require('ol.geom.Point'); goog.require('ol.geom.Polygon'); +goog.require('ol.geom.flat.orient'); goog.require('ol.proj.Projection'); goog.require('ol.proj.Units'); goog.require('ol.render.Feature'); @@ -80,81 +82,269 @@ ol.inherits(ol.format.MVT, ol.format.Feature); /** - * @inheritDoc - * @api + * Reader callbacks for parsing the PBF. + * @type {Object.} */ -ol.format.MVT.prototype.getLastExtent = function() { - return this.extent_; +ol.format.MVT.pbfReaders_ = { + layers: function(tag, layers, pbf) { + if (tag === 3) { + var layer = { + keys: [], + values: [], + features: [] + }; + var end = pbf.readVarint() + pbf.pos; + pbf.readFields(ol.format.MVT.pbfReaders_.layer, layer, end); + layer.length = layer.features.length; + if (layer.length) { + layers[layer.name] = layer; + } + } + }, + layer: function(tag, layer, pbf) { + if (tag === 15) { + layer.version = pbf.readVarint(); + } else if (tag === 1) { + layer.name = pbf.readString(); + } else if (tag === 5) { + layer.extent = pbf.readVarint(); + } else if (tag === 2) { + layer.features.push(pbf.pos); + } else if (tag === 3) { + layer.keys.push(pbf.readString()); + } else if (tag === 4) { + var value = null; + var end = pbf.readVarint() + pbf.pos; + while (pbf.pos < end) { + tag = pbf.readVarint() >> 3; + value = tag === 1 ? pbf.readString() : + tag === 2 ? pbf.readFloat() : + tag === 3 ? pbf.readDouble() : + tag === 4 ? pbf.readVarint64() : + tag === 5 ? pbf.readVarint() : + tag === 6 ? pbf.readSVarint() : + tag === 7 ? pbf.readBoolean() : null; + } + layer.values.push(value); + } + }, + feature: function(tag, feature, pbf) { + if (tag == 1) { + feature.id = pbf.readVarint(); + } else if (tag == 2) { + var end = pbf.readVarint() + pbf.pos; + while (pbf.pos < end) { + var key = feature.layer.keys[pbf.readVarint()]; + var value = feature.layer.values[pbf.readVarint()]; + feature.properties[key] = value; + } + } else if (tag == 3) { + feature.type = pbf.readVarint(); + } else if (tag == 4) { + feature.geometry = pbf.pos; + } + } }; /** - * @inheritDoc + * Read a raw feature from the pbf offset stored at index `i` in the raw layer. + * @suppress {missingProperties} + * @private + * @param {ol.ext.PBF} pbf PBF. + * @param {Object} layer Raw layer. + * @param {number} i Index of the feature in the raw layer's `features` array. + * @return {Object} Raw feature. */ -ol.format.MVT.prototype.getType = function() { - return ol.format.FormatType.ARRAY_BUFFER; +ol.format.MVT.readRawFeature_ = function(pbf, layer, i) { + pbf.pos = layer.features[i]; + var end = pbf.readVarint() + pbf.pos; + + var feature = { + layer: layer, + type: 0, + properties: {} + }; + pbf.readFields(ol.format.MVT.pbfReaders_.feature, feature, end); + return feature; }; /** + * Read the raw geometry from the pbf offset stored in a raw feature's geometry + * proeprty. + * @suppress {missingProperties} * @private - * @param {Object} rawFeature Raw Mapbox feature. - * @param {string} layer Layer. - * @param {olx.format.ReadOptions=} opt_options Read options. - * @return {ol.Feature} Feature. + * @param {ol.ext.PBF} pbf PBF. + * @param {Object} feature Raw feature. + * @param {Array.} flatCoordinates Array to store flat coordinates in. + * @param {Array.} ends Array to store ends in. */ -ol.format.MVT.prototype.readFeature_ = function( - rawFeature, layer, opt_options) { - var feature = new this.featureClass_(); - var id = rawFeature.id; - var values = rawFeature.properties; - values[this.layerName_] = layer; - if (this.geometryName_) { - feature.setGeometryName(this.geometryName_); +ol.format.MVT.readRawGeometry_ = function(pbf, feature, flatCoordinates, ends) { + pbf.pos = feature.geometry; + + var end = pbf.readVarint() + pbf.pos; + var cmd = 1; + var length = 0; + var x = 0; + var y = 0; + var coordsLen = 0; + var currentEnd = 0; + + while (pbf.pos < end) { + if (!length) { + var cmdLen = pbf.readVarint(); + cmd = cmdLen & 0x7; + length = cmdLen >> 3; + } + + length--; + + if (cmd === 1 || cmd === 2) { + x += pbf.readSVarint(); + y += pbf.readSVarint(); + + if (cmd === 1) { // moveTo + if (coordsLen > currentEnd) { + ends.push(coordsLen); + currentEnd = coordsLen; + } + } + + flatCoordinates.push(x, y); + coordsLen += 2; + + } else if (cmd === 7) { + + if (coordsLen > currentEnd) { + // close polygon + flatCoordinates.push( + flatCoordinates[currentEnd], flatCoordinates[currentEnd + 1]); + coordsLen += 2; + } + + } else { + ol.asserts.assert(false, 59); // Invalid command found in the PBF + } } - var geometry = ol.format.Feature.transformWithOptions( - ol.format.MVT.readGeometry_(rawFeature), false, - this.adaptOptions(opt_options)); - feature.setGeometry(geometry); - feature.setId(id); - feature.setProperties(values); - return feature; + + if (coordsLen > currentEnd) { + ends.push(coordsLen); + currentEnd = coordsLen; + } + }; /** + * @suppress {missingProperties} * @private - * @param {Object} rawFeature Raw Mapbox feature. - * @param {string} layer Layer. - * @return {ol.render.Feature} Feature. + * @param {number} type The raw feature's geometry type + * @param {number} numEnds Number of ends of the flat coordinates of the + * geometry. + * @return {ol.geom.GeometryType} The geometry type. */ -ol.format.MVT.prototype.readRenderFeature_ = function(rawFeature, layer) { - var coords = rawFeature.loadGeometry(); - var ends = []; - var flatCoordinates = []; - ol.format.MVT.calculateFlatCoordinates_(coords, flatCoordinates, ends); - - var type = rawFeature.type; +ol.format.MVT.getGeometryType_ = function(type, numEnds) { /** @type {ol.geom.GeometryType} */ var geometryType; if (type === 1) { - geometryType = coords.length === 1 ? + geometryType = numEnds === 1 ? ol.geom.GeometryType.POINT : ol.geom.GeometryType.MULTI_POINT; } else if (type === 2) { - if (coords.length === 1) { - geometryType = ol.geom.GeometryType.LINE_STRING; - } else { - geometryType = ol.geom.GeometryType.MULTI_LINE_STRING; - } + geometryType = numEnds === 1 ? + ol.geom.GeometryType.LINE_STRING : + ol.geom.GeometryType.MULTI_LINE_STRING; } else if (type === 3) { geometryType = ol.geom.GeometryType.POLYGON; + // MultiPolygon not relevant for rendering - winding order determines + // outer rings of polygons. } + return geometryType; +}; - var values = rawFeature.properties; - values[this.layerName_] = layer; +/** + * @private + * @param {ol.ext.PBF} pbf PBF + * @param {Object} rawFeature Raw Mapbox feature. + * @param {olx.format.ReadOptions=} opt_options Read options. + * @return {ol.Feature|ol.render.Feature} Feature. + */ +ol.format.MVT.prototype.createFeature_ = function(pbf, rawFeature, opt_options) { + var type = rawFeature.type; + if (type === 0) { + return null; + } + + var feature; var id = rawFeature.id; + var values = rawFeature.properties; + values[this.layerName_] = rawFeature.layer.name; - return new this.featureClass_(geometryType, flatCoordinates, ends, values, id); + var flatCoordinates = []; + var ends = []; + ol.format.MVT.readRawGeometry_(pbf, rawFeature, flatCoordinates, ends); + + var geometryType = ol.format.MVT.getGeometryType_(type, ends.length); + + if (this.featureClass_ === ol.render.Feature) { + feature = new this.featureClass_(geometryType, flatCoordinates, ends, values, id); + } else { + var geom; + if (geometryType == ol.geom.GeometryType.POLYGON) { + var endss = []; + var offset = 0; + var prevEndIndex = 0; + for (var i = 0, ii = ends.length; i < ii; ++i) { + var end = ends[i]; + if (!ol.geom.flat.orient.linearRingIsClockwise(flatCoordinates, offset, end, 2)) { + endss.push(ends.slice(prevEndIndex, i)); + prevEndIndex = i; + } + offset = end; + } + if (endss.length > 1) { + ends = endss; + geom = new ol.geom.MultiPolygon(null); + } else { + geom = new ol.geom.Polygon(null); + } + } else { + geom = geometryType === ol.geom.GeometryType.POINT ? new ol.geom.Point(null) : + geometryType === ol.geom.GeometryType.LINE_STRING ? new ol.geom.LineString(null) : + geometryType === ol.geom.GeometryType.POLYGON ? new ol.geom.Polygon(null) : + geometryType === ol.geom.GeometryType.MULTI_POINT ? new ol.geom.MultiPoint (null) : + geometryType === ol.geom.GeometryType.MULTI_LINE_STRING ? new ol.geom.MultiLineString(null) : + null; + } + geom.setFlatCoordinates(ol.geom.GeometryLayout.XY, flatCoordinates, ends); + feature = new this.featureClass_(); + if (this.geometryName_) { + feature.setGeometryName(this.geometryName_); + } + var geometry = ol.format.Feature.transformWithOptions(geom, false, this.adaptOptions(opt_options)); + feature.setGeometry(geometry); + feature.setId(id); + feature.setProperties(values); + } + + return feature; +}; + + +/** + * @inheritDoc + * @api + */ +ol.format.MVT.prototype.getLastExtent = function() { + return this.extent_; +}; + + +/** + * @inheritDoc + */ +ol.format.MVT.prototype.getType = function() { + return ol.format.FormatType.ARRAY_BUFFER; }; @@ -166,27 +356,22 @@ ol.format.MVT.prototype.readFeatures = function(source, opt_options) { var layers = this.layers_; var pbf = new ol.ext.PBF(/** @type {ArrayBuffer} */ (source)); - var tile = new ol.ext.vectortile.VectorTile(pbf); + var pbfLayers = pbf.readFields(ol.format.MVT.pbfReaders_.layers, {}); + /** @type {Array.} */ var features = []; - var featureClass = this.featureClass_; - var layer, feature; - for (var name in tile.layers) { + var pbfLayer; + for (var name in pbfLayers) { if (layers && layers.indexOf(name) == -1) { continue; } - layer = tile.layers[name]; + pbfLayer = pbfLayers[name]; var rawFeature; - for (var i = 0, ii = layer.length; i < ii; ++i) { - rawFeature = layer.feature(i); - if (featureClass === ol.render.Feature) { - feature = this.readRenderFeature_(rawFeature, name); - } else { - feature = this.readFeature_(rawFeature, name, opt_options); - } - features.push(feature); + for (var i = 0, ii = pbfLayer.length; i < ii; ++i) { + rawFeature = ol.format.MVT.readRawFeature_(pbf, pbfLayer, i); + features.push(this.createFeature_(pbf, rawFeature)); } - this.extent_ = layer ? [0, 0, layer.extent, layer.extent] : null; + this.extent_ = pbfLayer ? [0, 0, pbfLayer.extent, pbfLayer.extent] : null; } return features; @@ -212,68 +397,6 @@ ol.format.MVT.prototype.setLayers = function(layers) { }; -/** - * @private - * @param {Object} coords Raw feature coordinates. - * @param {Array.} flatCoordinates Flat coordinates to be populated by - * this function. - * @param {Array.} ends Ends to be populated by this function. - */ -ol.format.MVT.calculateFlatCoordinates_ = function( - coords, flatCoordinates, ends) { - var end = 0; - for (var i = 0, ii = coords.length; i < ii; ++i) { - var line = coords[i]; - var j, jj; - for (j = 0, jj = line.length; j < jj; ++j) { - var coord = line[j]; - // Non-tilespace coords can be calculated here when a TileGrid and - // TileCoord are known. - flatCoordinates.push(coord.x, coord.y); - } - end += 2 * j; - ends.push(end); - } -}; - - -/** - * @private - * @param {Object} rawFeature Raw Mapbox feature. - * @return {ol.geom.Geometry} Geometry. - */ -ol.format.MVT.readGeometry_ = function(rawFeature) { - var type = rawFeature.type; - if (type === 0) { - return null; - } - - var coords = rawFeature.loadGeometry(); - var ends = []; - var flatCoordinates = []; - ol.format.MVT.calculateFlatCoordinates_(coords, flatCoordinates, ends); - - var geom; - if (type === 1) { - geom = coords.length === 1 ? - new ol.geom.Point(null) : new ol.geom.MultiPoint(null); - } else if (type === 2) { - if (coords.length === 1) { - geom = new ol.geom.LineString(null); - } else { - geom = new ol.geom.MultiLineString(null); - } - } else if (type === 3) { - geom = new ol.geom.Polygon(null); - } - - geom.setFlatCoordinates(ol.geom.GeometryLayout.XY, flatCoordinates, - ends); - - return geom; -}; - - /** * Not implemented. * @override diff --git a/src/ol/source/vectortile.js b/src/ol/source/vectortile.js index ef2b5331d19..95f7e770df7 100644 --- a/src/ol/source/vectortile.js +++ b/src/ol/source/vectortile.js @@ -4,7 +4,6 @@ goog.require('ol'); goog.require('ol.TileState'); goog.require('ol.VectorImageTile'); goog.require('ol.VectorTile'); -goog.require('ol.proj'); goog.require('ol.size'); goog.require('ol.tilegrid'); goog.require('ol.source.UrlTile'); diff --git a/test/spec/ol/format/mvt.test.js b/test/spec/ol/format/mvt.test.js index 64dd238d6b9..66b4f147978 100644 --- a/test/spec/ol/format/mvt.test.js +++ b/test/spec/ol/format/mvt.test.js @@ -1,11 +1,11 @@ goog.require('ol.Feature'); -goog.require('ol.ext.PBF'); -goog.require('ol.ext.vectortile.VectorTile'); goog.require('ol.extent'); goog.require('ol.format.MVT'); goog.require('ol.geom.Point'); +goog.require('ol.geom.Polygon'); +goog.require('ol.geom.MultiPolygon'); goog.require('ol.render.Feature'); where('ArrayBuffer.isView').describe('ol.format.MVT', function() { @@ -41,36 +41,23 @@ where('ArrayBuffer.isView').describe('ol.format.MVT', function() { featureClass: ol.Feature, layers: ['poi_label'] }); - var pbf = new ol.ext.PBF(data); - var tile = new ol.ext.vectortile.VectorTile(pbf); - var geometry, rawGeometry; + var geometry; - rawGeometry = tile.layers['poi_label'].feature(0).loadGeometry(); - geometry = format.readFeatures(data)[0] - .getGeometry(); + geometry = format.readFeatures(data)[0].getGeometry(); expect(geometry.getType()).to.be('Point'); - expect(geometry.getCoordinates()) - .to.eql([rawGeometry[0][0].x, rawGeometry[0][0].y]); + expect(geometry.getCoordinates()).to.eql([-1210, 2681]); - rawGeometry = tile.layers['water'].feature(0).loadGeometry(); format.setLayers(['water']); - geometry = format.readFeatures(data)[0] - .getGeometry(); + geometry = format.readFeatures(data)[0].getGeometry(); expect(geometry.getType()).to.be('Polygon'); - expect(rawGeometry[0].length) - .to.equal(geometry.getCoordinates()[0].length); - expect(geometry.getCoordinates()[0][0]) - .to.eql([rawGeometry[0][0].x, rawGeometry[0][0].y]); + expect(geometry.getCoordinates()[0].length).to.be(10); + expect(geometry.getCoordinates()[0][0]).to.eql([1007, 2302]); - rawGeometry = tile.layers['barrier_line'].feature(0).loadGeometry(); format.setLayers(['barrier_line']); - geometry = format.readFeatures(data)[0] - .getGeometry(); + geometry = format.readFeatures(data)[0].getGeometry(); expect(geometry.getType()).to.be('MultiLineString'); - expect(rawGeometry[1].length) - .to.equal(geometry.getCoordinates()[1].length); - expect(geometry.getCoordinates()[1][0]) - .to.eql([rawGeometry[1][0].x, rawGeometry[1][0].y]); + expect(geometry.getCoordinates()[1].length).to.be(6); + expect(geometry.getCoordinates()[1][0]).to.eql([4160, 3489]); }); it('parses id property', function() { @@ -102,7 +89,7 @@ where('ArrayBuffer.isView').describe('ol.format.MVT', function() { describe('ol.format.MVT', function() { - describe('#readFeature_', function() { + describe('#createFeature_', function() { it('accepts a geometryName', function() { var format = new ol.format.MVT({ featureClass: ol.Feature, @@ -114,16 +101,99 @@ describe('ol.format.MVT', function() { geometry: 'foo' }, type: 1, - loadGeometry: function() { - return [[0, 0]]; + layer: { + name: 'layer1' } }; - var feature = format.readFeature_(rawFeature, 'mapbox'); + var readRawGeometry_ = ol.format.MVT.readRawGeometry_; + ol.format.MVT.readRawGeometry_ = function({}, rawFeature, flatCoordinates, ends) { + flatCoordinates.push(0, 0); + ends.push(2); + }; + var feature = format.createFeature_({}, rawFeature); + ol.format.MVT.readRawGeometry_ = readRawGeometry_; var geometry = feature.getGeometry(); expect(geometry).to.be.a(ol.geom.Point); expect(feature.get('myGeom')).to.equal(geometry); expect(feature.get('geometry')).to.be('foo'); }); + + it('detects a Polygon', function() { + var format = new ol.format.MVT({ + featureClass: ol.Feature + }); + var rawFeature = { + type: 3, + properties: {}, + layer: { + name: 'layer1' + } + }; + var readRawGeometry_ = ol.format.MVT.readRawGeometry_; + ol.format.MVT.readRawGeometry_ = function({}, rawFeature, flatCoordinates, ends) { + flatCoordinates.push(0, 0, 3, 0, 3, 3, 3, 0, 0, 0); + flatCoordinates.push(1, 1, 1, 2, 2, 2, 2, 1, 1, 1); + ends.push(10, 20); + }; + var feature = format.createFeature_({}, rawFeature); + ol.format.MVT.readRawGeometry_ = readRawGeometry_; + var geometry = feature.getGeometry(); + expect(geometry).to.be.a(ol.geom.Polygon); + }); + + it('detects a MultiPolygon', function() { + var format = new ol.format.MVT({ + featureClass: ol.Feature + }); + var rawFeature = { + type: 3, + properties: {}, + layer: { + name: 'layer1' + } + }; + var readRawGeometry_ = ol.format.MVT.readRawGeometry_; + ol.format.MVT.readRawGeometry_ = function({}, rawFeature, flatCoordinates, ends) { + flatCoordinates.push(0, 0, 1, 0, 1, 1, 1, 0, 0, 0); + flatCoordinates.push(1, 1, 2, 1, 2, 2, 2, 1, 1, 1); + ends.push(10, 20); + }; + var feature = format.createFeature_({}, rawFeature); + ol.format.MVT.readRawGeometry_ = readRawGeometry_; + var geometry = feature.getGeometry(); + expect(geometry).to.be.a(ol.geom.MultiPolygon); + }); + + it('creates ol.render.Feature instances', function() { + var format = new ol.format.MVT(); + var rawFeature = { + type: 3, + properties: { + foo: 'bar' + }, + layer: { + name: 'layer1' + } + }; + var readRawGeometry_ = ol.format.MVT.readRawGeometry_; + var createdFlatCoordinates; + var createdEnds; + ol.format.MVT.readRawGeometry_ = function({}, rawFeature, flatCoordinates, ends) { + flatCoordinates.push(0, 0, 1, 0, 1, 1, 1, 0, 0, 0); + flatCoordinates.push(1, 1, 2, 1, 2, 2, 2, 1, 1, 1); + createdFlatCoordinates = flatCoordinates; + ends.push(10, 20); + createdEnds = ends; + }; + var feature = format.createFeature_({}, rawFeature); + ol.format.MVT.readRawGeometry_ = readRawGeometry_; + expect(feature).to.be.a(ol.render.Feature); + expect(feature.getType()).to.be('Polygon'); + expect(feature.getFlatCoordinates()).to.equal(createdFlatCoordinates); + expect(feature.getEnds()).to.equal(createdEnds); + expect(feature.get('foo')).to.be('bar'); + }); + }); });