From b779fa7ad9375f29289dc3a730b43983b61627e4 Mon Sep 17 00:00:00 2001 From: Roman Deev Date: Sat, 26 Apr 2025 17:57:25 +0300 Subject: [PATCH 1/3] #80 make data preprocessing more error-tolerant. Download incompleted ways --- src/building.js | 54 ++++++++++++++++++++++++++------ src/buildingpart.js | 6 ++-- src/extras/BuildingShapeUtils.js | 4 +-- src/multibuildingpart.js | 13 +++++--- 4 files changed, 59 insertions(+), 18 deletions(-) diff --git a/src/building.js b/src/building.js index 50a22c2..fb8906a 100644 --- a/src/building.js +++ b/src/building.js @@ -43,7 +43,8 @@ class Building { const nodelist = Building.buildNodeList(xmlData); const extents = Building.getExtents(id, xmlData, nodelist); const innerData = await Building.getInnerData(...extents); - return new Building(id, innerData); + const [augmentedNodelist, augmentedWays] = await Building.buildAugmentedData(innerData) + return new Building(id, innerData, augmentedNodelist, augmentedWays); } /** @@ -52,9 +53,11 @@ class Building { * @param {string} id - the unique XML id of the object. * @param {string} FullXmlData - XML data. */ - constructor(id, FullXmlData) { + constructor(id, FullXmlData, augmentedNodelist, augmentedWays) { this.id = id; this.fullXmlData = new window.DOMParser().parseFromString(FullXmlData, 'text/xml'); + this.augmentedNodelist = augmentedNodelist; + this.augmentedWays = augmentedWays; const outerElementXml = this.fullXmlData.getElementById(id); if (outerElementXml.tagName.toLowerCase() === 'way') { this.type = 'way'; @@ -68,17 +71,17 @@ class Building { this.setHome(); this.repositionNodes(); if (this.type === 'way') { - this.outerElement = new BuildingPart(id, this.fullXmlData, this.nodelist); + this.outerElement = new BuildingPart(id, this.fullXmlData, this.nodelist, this.augmentedNodelist, this.augmentedWays); } else if (this.type === 'multipolygon') { - this.outerElement = new MultiBuildingPart(id, this.fullXmlData, this.nodelist); + this.outerElement = new MultiBuildingPart(id, this.fullXmlData, this.nodelist, this.augmentedNodelist, this.augmentedWays); } else { const outlineRef = outerElementXml.querySelector('member[role="outline"]').getAttribute('ref'); const outline = this.fullXmlData.getElementById(outlineRef); const outlineType = outline.tagName.toLowerCase(); if (outlineType === 'way') { - this.outerElement = new BuildingPart(id, this.fullXmlData, this.nodelist); + this.outerElement = new BuildingPart(id, this.fullXmlData, this.nodelist, this.augmentedNodelist, this.augmentedWays); } else { - this.outerElement = new MultiBuildingPart(outlineRef, this.fullXmlData, this.nodelist); + this.outerElement = new MultiBuildingPart(outlineRef, this.fullXmlData, this.nodelist, this.augmentedNodelist, this.augmentedWays); } } this.addParts(); @@ -121,6 +124,34 @@ class Building { return nodeList; } + + /** + * @param {DOM.Element} fullXmlData - OSM XML with nodes + * @return {Promise<({}|*)[]>} + */ + static async buildAugmentedData(fullXmlData) { + const xmlData = new DOMParser().parseFromString(fullXmlData, 'text/xml'); + const completedWays = new Set(Array.from(xmlData.getElementsByTagName('way')).map(i => i.getAttribute('id'))); + const memberWays = xmlData.querySelectorAll('member[type="way"]'); + const nodeList = {}; + const waysList = {}; + await Promise.all(Array.from(memberWays).map(async currentWay => { + const wayID = currentWay.getAttribute('ref'); + if (completedWays.has(wayID)) { + return + } + printError('Additional downloading way ' + wayID); + const wayData = new DOMParser().parseFromString(await Building.getWayData(wayID), 'text/xml'); + printError(`Way ${wayID} was downloaded`); + waysList[wayID] = wayData.querySelector('way'); + wayData.querySelectorAll('node').forEach(i => { + nodeList[i.getAttribute('id')] = [i.getAttribute('lon'), i.getAttribute('lat')]; + }); + })) + return [nodeList, waysList]; + } + + /** * convert all the longitude latitude values * to meters from the home point. @@ -129,6 +160,9 @@ class Building { for (const key in this.nodelist) { this.nodelist[key] = BuildingShapeUtils.repositionPoint(this.nodelist[key], this.home); } + for (const key in this.augmentedNodelist) { + this.augmentedNodelist[key] = BuildingShapeUtils.repositionPoint(this.augmentedNodelist[key], this.home); + } } /** @@ -158,9 +192,9 @@ class Building { const ref = parts[i].getAttribute('ref'); const part = this.fullXmlData.getElementById(ref); if (part.tagName.toLowerCase() === 'way') { - this.parts.push(new BuildingPart(ref, this.fullXmlData, this.nodelist, this.outerElement.options)); + this.parts.push(new BuildingPart(ref, this.fullXmlData, this.nodelist, this.augmentedNodelist, this.augmentedWays, this.outerElement.options)); } else { - this.parts.push(new MultiBuildingPart(ref, this.fullXmlData, this.nodelist, this.outerElement.options)); + this.parts.push(new MultiBuildingPart(ref, this.fullXmlData, this.nodelist, this.augmentedNodelist, this.augmentedWays, this.outerElement.options)); } } } else { @@ -169,7 +203,7 @@ class Building { for (let j = 0; j < parts.length; j++) { if (parts[j].querySelector('[k="building:part"]')) { const id = parts[j].getAttribute('id'); - this.parts.push(new BuildingPart(id, this.fullXmlData, this.nodelist, this.outerElement.options)); + this.parts.push(new BuildingPart(id, this.fullXmlData, this.nodelist, this.augmentedNodelist, this.augmentedWays, this.outerElement.options)); } } // Filter all relations @@ -177,7 +211,7 @@ class Building { for (let i = 0; i < parts.length; i++) { if (parts[i].querySelector('[k="building:part"]')) { const id = parts[i].getAttribute('id'); - this.parts.push(new MultiBuildingPart(id, this.fullXmlData, this.nodelist, this.outerElement.options)); + this.parts.push(new MultiBuildingPart(id, this.fullXmlData, this.nodelist, this.augmentedNodelist, this.augmentedWays, this.outerElement.options)); } } } diff --git a/src/buildingpart.js b/src/buildingpart.js index e61a4f1..d9a6c63 100644 --- a/src/buildingpart.js +++ b/src/buildingpart.js @@ -68,7 +68,7 @@ class BuildingPart { * @param {[[number, number],...]} nodelist - Cartesian coordinates of each node keyed by node refID * @param {object} options - default values for the building part. */ - constructor(id, fullXmlData, nodelist, defaultOptions = {}) { + constructor(id, fullXmlData, nodelist, augmentedNodelist, augmentedWays, defaultOptions = {}) { this.options = this.blankOptions; if (Object.keys(defaultOptions).length === 0) { defaultOptions = this.blankOptions; @@ -78,6 +78,8 @@ class BuildingPart { this.id = id; this.way = fullXmlData.getElementById(id); this.nodelist = nodelist; + this.augmentedNodelist = augmentedNodelist; + this.augmentedWays = augmentedWays; this.shape = this.buildShape(); this.setOptions(); } @@ -89,7 +91,7 @@ class BuildingPart { */ buildShape() { this.type = 'way'; - return BuildingShapeUtils.createShape(this.way, this.nodelist); + return BuildingShapeUtils.createShape(this.way, this.nodelist, this.augmentedNodelist); } /** diff --git a/src/extras/BuildingShapeUtils.js b/src/extras/BuildingShapeUtils.js index 6c5ae8d..223c359 100644 --- a/src/extras/BuildingShapeUtils.js +++ b/src/extras/BuildingShapeUtils.js @@ -13,7 +13,7 @@ class BuildingShapeUtils extends ShapeUtils { * * @return {THREE.Shape} shape - the shape */ - static createShape(way, nodelist) { + static createShape(way, nodelist, augmentedNodelist = {}) { // Initialize objects const shape = new Shape(); var ref; @@ -25,7 +25,7 @@ class BuildingShapeUtils extends ShapeUtils { // Get the coordinates of all the nodes and add them to the shape outline. for (let i = 0; i < elements.length; i++) { ref = elements[i].getAttribute('ref'); - node = nodelist[ref]; + node = nodelist[ref] ?? augmentedNodelist[ref]; // The first node requires a differnet function call. if (i === 0) { shape.moveTo(parseFloat(node[0]), parseFloat(node[1])); diff --git a/src/multibuildingpart.js b/src/multibuildingpart.js index d8c885d..4cc4aeb 100644 --- a/src/multibuildingpart.js +++ b/src/multibuildingpart.js @@ -17,19 +17,24 @@ class MultiBuildingPart extends BuildingPart { const innerMembers = this.way.querySelectorAll('member[role="inner"]'); const outerMembers = this.way.querySelectorAll('member[role="outer"]'); const innerShapes = []; - var shapes = []; + let shapes = []; for (let i = 0; i < innerMembers.length; i++) { const way = this.fullXmlData.getElementById(innerMembers[i].getAttribute('ref')); - innerShapes.push(BuildingShapeUtils.createShape(way, this.nodelist)); + innerShapes.push(BuildingShapeUtils.createShape(way, this.nodelist, this.augmentedNodelist)); } const ways = []; for (let j = 0; j < outerMembers.length; j++) { const way = this.fullXmlData.getElementById(outerMembers[j].getAttribute('ref')); - ways.push(way); + if (way) { + ways.push(way); + } else { + printError(`Missing way ${outerMembers[j].getAttribute('ref')} for relation ${this.id}`); + ways.push(this.augmentedWays[outerMembers[j].getAttribute('ref')]) + } } const closedWays = BuildingShapeUtils.combineWays(ways); for (let k = 0; k < closedWays.length; k++) { - const shape = BuildingShapeUtils.createShape(closedWays[k], this.nodelist); + const shape = BuildingShapeUtils.createShape(closedWays[k], this.nodelist, this.augmentedNodelist); shape.holes.push(...innerShapes); shapes.push(shape); } From 69f83b65d7cd74ad170d851915842671b6784c90 Mon Sep 17 00:00:00 2001 From: Roman Deev Date: Sat, 26 Apr 2025 20:45:51 +0300 Subject: [PATCH 2/3] skip non-way members --- src/building.js | 16 ++++++++++------ src/buildingpart.js | 6 ++++-- src/extras/BuildingShapeUtils.js | 1 + src/multibuildingpart.js | 23 +++++++++++++++-------- 4 files changed, 30 insertions(+), 16 deletions(-) diff --git a/src/building.js b/src/building.js index fb8906a..4a54c8f 100644 --- a/src/building.js +++ b/src/building.js @@ -43,7 +43,7 @@ class Building { const nodelist = Building.buildNodeList(xmlData); const extents = Building.getExtents(id, xmlData, nodelist); const innerData = await Building.getInnerData(...extents); - const [augmentedNodelist, augmentedWays] = await Building.buildAugmentedData(innerData) + const [augmentedNodelist, augmentedWays] = await Building.buildAugmentedData(innerData); return new Building(id, innerData, augmentedNodelist, augmentedWays); } @@ -138,16 +138,16 @@ class Building { await Promise.all(Array.from(memberWays).map(async currentWay => { const wayID = currentWay.getAttribute('ref'); if (completedWays.has(wayID)) { - return + return; } - printError('Additional downloading way ' + wayID); + window.printError('Additional downloading way ' + wayID); const wayData = new DOMParser().parseFromString(await Building.getWayData(wayID), 'text/xml'); - printError(`Way ${wayID} was downloaded`); + window.printError(`Way ${wayID} was downloaded`); waysList[wayID] = wayData.querySelector('way'); wayData.querySelectorAll('node').forEach(i => { nodeList[i.getAttribute('id')] = [i.getAttribute('lon'), i.getAttribute('lat')]; }); - })) + })); return [nodeList, waysList]; } @@ -211,7 +211,11 @@ class Building { for (let i = 0; i < parts.length; i++) { if (parts[i].querySelector('[k="building:part"]')) { const id = parts[i].getAttribute('id'); - this.parts.push(new MultiBuildingPart(id, this.fullXmlData, this.nodelist, this.augmentedNodelist, this.augmentedWays, this.outerElement.options)); + try { + this.parts.push(new MultiBuildingPart(id, this.fullXmlData, this.nodelist, this.augmentedNodelist, this.augmentedWays, this.outerElement.options)); + } catch (e) { + window.printError(e); + } } } } diff --git a/src/buildingpart.js b/src/buildingpart.js index d9a6c63..d388c21 100644 --- a/src/buildingpart.js +++ b/src/buildingpart.js @@ -66,9 +66,11 @@ class BuildingPart { * @param {number} id - the OSM id of the way or multipolygon. * @param {XMLDocument} fullXmlData - XML for the region. * @param {[[number, number],...]} nodelist - Cartesian coordinates of each node keyed by node refID - * @param {object} options - default values for the building part. + * @param augmentedNodelist - list of nodes outside bbox + * @param augmentedWays - list of ways outside bbox + * @param {object} defaultOptions - default values for the building part. */ - constructor(id, fullXmlData, nodelist, augmentedNodelist, augmentedWays, defaultOptions = {}) { + constructor(id, fullXmlData, nodelist, augmentedNodelist = {}, augmentedWays = {}, defaultOptions = {}) { this.options = this.blankOptions; if (Object.keys(defaultOptions).length === 0) { defaultOptions = this.blankOptions; diff --git a/src/extras/BuildingShapeUtils.js b/src/extras/BuildingShapeUtils.js index 223c359..22741e5 100644 --- a/src/extras/BuildingShapeUtils.js +++ b/src/extras/BuildingShapeUtils.js @@ -10,6 +10,7 @@ class BuildingShapeUtils extends ShapeUtils { * * @param {DOM.Element} way - OSM XML way element. * @param {[number, number]} nodelist - list of all nodes + * @param augmentedNodelist - list of nodes outside bbox * * @return {THREE.Shape} shape - the shape */ diff --git a/src/multibuildingpart.js b/src/multibuildingpart.js index 4cc4aeb..f10dddc 100644 --- a/src/multibuildingpart.js +++ b/src/multibuildingpart.js @@ -14,22 +14,29 @@ class MultiBuildingPart extends BuildingPart { */ buildShape() { this.type = 'multipolygon'; - const innerMembers = this.way.querySelectorAll('member[role="inner"]'); - const outerMembers = this.way.querySelectorAll('member[role="outer"]'); + const innerMembers = this.way.querySelectorAll('member[role="inner"][type="way"]'); + const outerMembers = this.way.querySelectorAll('member[role="outer"][type="way"]'); const innerShapes = []; - let shapes = []; + const shapes = []; for (let i = 0; i < innerMembers.length; i++) { - const way = this.fullXmlData.getElementById(innerMembers[i].getAttribute('ref')); - innerShapes.push(BuildingShapeUtils.createShape(way, this.nodelist, this.augmentedNodelist)); + const wayID = innerMembers[i].getAttribute('ref'); + const way = this.fullXmlData.getElementById(wayID); + if (way) { + innerShapes.push(BuildingShapeUtils.createShape(way, this.nodelist, this.augmentedNodelist)); + } else { + window.printError(`Missing way ${wayID} for relation ${this.id} for inner members`); + innerShapes.push(BuildingShapeUtils.createShape(this.augmentedWays[wayID], this.nodelist, this.augmentedNodelist)); + } } const ways = []; for (let j = 0; j < outerMembers.length; j++) { - const way = this.fullXmlData.getElementById(outerMembers[j].getAttribute('ref')); + const wayID = outerMembers[j].getAttribute('ref'); + const way = this.fullXmlData.getElementById(wayID); if (way) { ways.push(way); } else { - printError(`Missing way ${outerMembers[j].getAttribute('ref')} for relation ${this.id}`); - ways.push(this.augmentedWays[outerMembers[j].getAttribute('ref')]) + window.printError(`Missing way ${wayID} for relation ${this.id}`); + ways.push(this.augmentedWays[wayID]); } } const closedWays = BuildingShapeUtils.combineWays(ways); From e19bbd63fc5a1ae76edb05e0260d046a4c0f5f2f Mon Sep 17 00:00:00 2001 From: Roman Deev Date: Sat, 26 Apr 2025 21:18:31 +0300 Subject: [PATCH 3/3] Update test data --- test/multipolygon.test.js | 2 +- test/split_way_multipolygon.test.js | 6 +++--- test/split_way_multipolygon_reverse.test.js | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/test/multipolygon.test.js b/test/multipolygon.test.js index 1125ce4..2fac8e8 100644 --- a/test/multipolygon.test.js +++ b/test/multipolygon.test.js @@ -16,7 +16,7 @@ const data = ` - + diff --git a/test/split_way_multipolygon.test.js b/test/split_way_multipolygon.test.js index b133f72..ffa0681 100644 --- a/test/split_way_multipolygon.test.js +++ b/test/split_way_multipolygon.test.js @@ -19,9 +19,9 @@ const data = ` - - - + + + diff --git a/test/split_way_multipolygon_reverse.test.js b/test/split_way_multipolygon_reverse.test.js index 4350ebe..d1bc3c2 100644 --- a/test/split_way_multipolygon_reverse.test.js +++ b/test/split_way_multipolygon_reverse.test.js @@ -19,9 +19,9 @@ const data = ` - - - + + +