ZES: Rework highway data download, refactor property enhancement

Rework the way we add the `FMC:length` to the GeoJSON: Before, we did it by  iterating over all ways to add the length based on a huge array.

Now we have the length as part of the properties right from overpass, BUT we need to manually create our GeoJSON, since osmtogeojson does not support the output of overpass convert.

(For some reasons, we don't need to cut the results to a bbox anymore after switching to convert.)

Related refactorings:
- Add the custom property links (OSM + Mapillary) up front when we create our main GeoJson
- Rename the custom category property
- Disable the addLengthFromOverpassStats method/file since we don't use it anymore

ZESPlus/AreaOfInterest/transpose.ts

@@ -1,4 +1,4 @@
-import { overpassJsonToGeoJson } from "../utils/overpassJsonToGeoJson"
+import { overpassJsonToGeoJson } from "../utils/overpassToGeoJson/overpassJsonToGeoJson"
 import { FeatureCollection } from "../utils/types"
 import { areaKeys } from "./areas.constant"
 

ZESPlus/Highways-PrepareData/1-download-highways/download.ts

@@ -4,27 +4,24 @@ import { overpassDownloadJson } from "../../utils/overpassDownloadJson"
 //  south west north east
 //  south-west -- north-east
 //  https://dev.overpass-api.de/overpass-doc/en/full_data/bbox.html
-// Cut to BBox
-//  This prevents large relations to blow up the data
-//  It will likely cause false data for the length of way at the edges of the BBox. Which is OK.
-//  Docs https://dev.overpass-api.de/overpass-doc/en/criteria/union.html
 // Filter most of the tags that are filtered in 'filterGenerallyIrrelevantTags.ts'
 //  This is a performance optimization.
 //  Docs https://dev.overpass-api.de/overpass-doc/en/criteria/per_tag.html
 //  Docs https://wiki.openstreetmap.org/wiki/Overpass_API/Overpass_QL#Key.2Fvalue_matches_regular_expression_.28.7E.22key_regex.22.7E.22value_regex.22.29
+// Convert adds 'length' property:
+//  Based on https://github.com/drolbr/Overpass-API/issues/237#issuecomment-927285088I
+//  This return a special json format, that requires the overpassCoposeToGeoJson.ts to convert to GeoJSON.
 const overpassQuery = `
 [out:json][timeout:25];
 (
   way
   [highway~"^(bridleway|cycleway|footway|living_street|motorway|motorway_link|path|pedestrian|primary|primary_link|residential|secondary|secondary_link|service|steps|tertiary|tertiary_link|track|trunk|trunk_link|unclassified)$"]
   [access!~"^(private|no|destination)$"]
   [service!~"^(driveway|parking_aisle)$"]
-  (52.2587,13.4529,52.4646,13.7730)
-  ;
+  (52.2587,13.4529,52.4646,13.7730);
 );
-out geom(52.2587,13.4529,52.4646,13.7730);
-make stat total_length=sum(length()),way_id_length=set(type() + "/" + id() + ":" + length());
-out;
+convert result ::=::,::geom= geom(),::id=id(),"__type"=type(),"FMC:length"=length();
+out geom;
 `
 
 overpassDownloadJson({

ZESPlus/Highways-PrepareData/2-transpose-highways/transpose.ts

@@ -1,9 +1,7 @@
 import de9im from "de9im"
 import fs from "fs"
-import {
-  AreaCallbackProps,
-  overpassJsonToGeoJson,
-} from "../../utils/overpassJsonToGeoJson"
+import { overpassComposeToGeoJson } from "../../utils/overpassToGeoJson/overpassComposeToGeoJson"
+import { AreaCallbackProps } from "../../utils/overpassToGeoJson/types"
 
 const enhanceDataWitAreaInformation: AreaCallbackProps = (areaKey, geoJson) => {
   console.time(`enhanceData(): Ergänze FMC:Gebiet:${areaKey}=True|False`)
@@ -26,7 +24,7 @@ const enhanceDataWitAreaInformation: AreaCallbackProps = (areaKey, geoJson) => {
   console.timeEnd(`enhanceData(): Ergänze FMC:Gebiet:${areaKey}=True|False`)
 }
 
-overpassJsonToGeoJson({
+overpassComposeToGeoJson({
   readFile:
     "./ZESPlus/Highways-PrepareData/1-download-highways/osmRawHighways.json",
   outputFolder: "./ZESPlus/Highways-PrepareData/2-transpose-highways/",

ZESPlus/utils/addFilterNameProperty.ts

@@ -0,0 +1,10 @@
+import { Feature } from "./types"
+
+export const addCategoryProperty = (
+  features: Feature[],
+  filterMethodName: string
+) => {
+  features.forEach((feature) => {
+    feature.properties["FMC:category"] = filterMethodName
+  })
+}

ZESPlus/utils/filterAndWrite.ts

@@ -1,4 +1,4 @@
-import { addCustomProperties } from "./addCustomProperties"
+import { addCategoryProperty } from "./addFilterNameProperty"
 import { collectFilteredHighways } from "./collectFilteredHighways"
 import { Feature, FeatureCollection } from "./types"
 import { writeGeoJson } from "./writeGeoJson"
@@ -14,7 +14,7 @@ export const filterAndWrite = (
   // Filter Data
   const filteredData = allHighways.features.filter(filterMethod)
 
-  addCustomProperties(filteredData, filterMethod.name)
+  addCategoryProperty(filteredData, filterMethod.name)
 
   collectFilteredHighways(filteredData, filterMethod.name)
 

ZESPlus/utils/overpassToGeoJson/overpassComposeToGeoJson.ts

@@ -0,0 +1,68 @@
+import fs from "fs"
+import { areaKeys } from "../../AreaOfInterest/areas.constant"
+import { addLinkProperties } from "../transpose/addLinkProperties"
+import { processingDateProperty } from "../transpose/addProcessingDateProperty"
+import { FeatureCollection } from "../types"
+import { writeFile } from "../writeFile"
+import { OverpassToGeoJson } from "./types"
+
+export const overpassComposeToGeoJson = ({
+  readFile,
+  outputFolder,
+  fileNamePart,
+  filterCallback,
+  areaCallback,
+}: OverpassToGeoJson) => {
+  fs.readFile(readFile, "utf8", (err, _data) => {
+    if (err) {
+      console.error(err)
+      return
+    }
+    console.log(
+      "overpassComposeToGeoJson()",
+      "Parse, Transform, Enhance and Write file"
+    )
+    console.time(`overpassComposeToGeoJson(): Transform, enhance, write`)
+
+    const rawJsonData = JSON.parse(_data)
+    const geoJsonData = {
+      type: "FeatureCollection",
+      generator: rawJsonData.generator,
+      copyright: rawJsonData.osm3s.copyright,
+      timestamp: rawJsonData.osm3s.timestamp_osm_base,
+      features: rawJsonData.elements.map((element) => {
+        // `__type` is a helper that we add in `1-download-highways/download.ts` > `compose``
+        // to re-generate the full GeoJSON osmId. Afterwards, we can clean it up.
+        const osmId = `${element.tags.__type}/${element.id}`
+        delete element.tags.__type
+
+        return {
+          type: "Feature",
+          properties: {
+            "@id": osmId,
+            ...element.tags,
+            ...processingDateProperty,
+          },
+          geometry: element.geometry,
+          id: osmId,
+        }
+      }),
+    } as FeatureCollection
+
+    // addProcessingDateProperty(geoJsonData.features) // Done via spread operator above
+    addLinkProperties(geoJsonData.features)
+    filterCallback && filterCallback(geoJsonData)
+    areaCallback &&
+      areaKeys.forEach((areaKey) => areaCallback(areaKey, geoJsonData))
+
+    writeFile({
+      dataString: JSON.stringify(geoJsonData, null, 2),
+      dataLength: geoJsonData.features.length,
+      outputFolder: outputFolder,
+      fileNamePart: fileNamePart,
+      format: "geojson",
+    })
+
+    console.timeEnd(`overpassComposeToGeoJson(): Transform, enhance, write`)
+  })
+}

ZESPlus/utils/overpassToGeoJson/overpassJsonToGeoJson.ts

@@ -0,0 +1,44 @@
+import fs from "fs"
+import osmtogeojson from "osmtogeojson"
+import { areaKeys } from "../../AreaOfInterest/areas.constant"
+import { addLinkProperties } from "../transpose/addLinkProperties"
+import { addProcessingDateProperty } from "../transpose/addProcessingDateProperty"
+import { FeatureCollection } from "../types"
+import { writeFile } from "../writeFile"
+import { OverpassToGeoJson } from "./types"
+
+export const overpassJsonToGeoJson = ({
+  readFile,
+  outputFolder,
+  fileNamePart,
+  filterCallback,
+  areaCallback,
+}: OverpassToGeoJson) => {
+  fs.readFile(readFile, "utf8", (err, _data) => {
+    if (err) {
+      console.error(err)
+      return
+    }
+    console.log(
+      "overpassJsonToGeoJson():",
+      "Parse, Transform, Enhance and Write file"
+    )
+
+    const rawJsonData = JSON.parse(_data)
+    const geoJsonData = osmtogeojson(rawJsonData) as FeatureCollection
+
+    addProcessingDateProperty(geoJsonData.features)
+    addLinkProperties(geoJsonData.features)
+    filterCallback && filterCallback(geoJsonData)
+    areaCallback &&
+      areaKeys.forEach((areaKey) => areaCallback(areaKey, geoJsonData))
+
+    writeFile({
+      dataString: JSON.stringify(geoJsonData, null, 2),
+      dataLength: geoJsonData.features.length,
+      outputFolder: outputFolder,
+      fileNamePart: fileNamePart,
+      format: "geojson",
+    })
+  })
+}

ZESPlus/utils/overpassToGeoJson/types.ts

@@ -0,0 +1,14 @@
+import { FeatureCollection } from "../types"
+
+export type AreaCallbackProps = (
+  areaKey: string,
+  geoJson: FeatureCollection
+) => void
+
+export type OverpassToGeoJson = {
+  readFile: string
+  outputFolder: string
+  fileNamePart: string
+  filterCallback?: (geoJson: FeatureCollection) => void
+  areaCallback?: AreaCallbackProps
+}

ZESPlus/utils/addCustomProperties.ts → ZESPlus/utils/transpose/addLinkProperties.ts

@@ -1,15 +1,9 @@
-import { Feature } from "./types"
+import { Feature } from "../types"
 
-/**
- * Add custom `FMC:*` helper properties.
- */
-export const addCustomProperties = (
-  features: Feature[],
-  filterMethodName: string
-) => {
-  features.forEach((feature) => {
-    feature.properties["FMC:category"] = filterMethodName
+export const addLinkProperties = (features: Feature[]) => {
+  console.time("addCustomLinkProperties()")
 
+  features.forEach((feature) => {
     feature.properties[
       "FMC:linkToOsmWebsite"
     ] = `https://www.openstreetmap.org/${feature.id}`
@@ -22,4 +16,6 @@ export const addCustomProperties = (
       "FMC:linkToMapillary"
     ] = `https://www.mapillary.com/app/?lat=${latLonInMiddle[1]}&lng=${latLonInMiddle[0]}&z=16&focus=map`
   })
+
+  console.timeEnd("addCustomLinkProperties()")
 }

ZESPlus/utils/transpose/addProcessingDateProperty.ts

@@ -0,0 +1,20 @@
+import { Feature } from "../types"
+
+const processingDateCreatedAt = new Date()
+  .toISOString()
+  .slice(0, 16)
+  .replace("T", " ")
+
+export const processingDateProperty = {
+  "FMC:DataCreatedAndProcessedAt": `${processingDateCreatedAt}`,
+}
+
+export const addProcessingDateProperty = (features: Feature[]) => {
+  console.time("addProcessingDateProperty()")
+
+  features.forEach((feature) => {
+    feature.properties = { ...feature.properties, ...processingDateProperty }
+  })
+
+  console.timeEnd("addProcessingDateProperty()")
+}