GMLCity Importer

backend/src/BIE.DataPipeline/DbHelper.cs

@@ -12,7 +12,7 @@ internal sealed class DbHelper
         private readonly StringBuilder mStringBuilder;
 
         private int mCount;
-        private const int MaxCount = 900;
+        private const int MaxCount = 1;//900;
 
 
         public DbHelper()
@@ -276,7 +276,24 @@ Location GEOMETRY
 END";
             }
 
-            var query = $@"
+            if (description.source.data_format == "CITYGML")
+            {
+                return $@"
+IF NOT EXISTS (SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = '{description.table_name}')
+BEGIN
+    CREATE TABLE {description.table_name} (
+        Id INT PRIMARY KEY IDENTITY(1,1),
+        Location GEOGRAPHY,
+        XmlData XML,
+        GroundHeight FLOAT,
+        DistrictKey VARCHAR(255),
+        CheckDate DATE,
+        GroundArea FLOAT,
+    );
+END";
+            }
+
+                var query = $@"
 IF NOT EXISTS (SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = '{description.table_name}')
 BEGIN CREATE TABLE {description.table_name} (";
 

backend/src/BIE.DataPipeline/Import/CityGmlImporter.cs

@@ -0,0 +1,297 @@
+using NetTopologySuite.Geometries;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using System.Xml;
+using System.Xml.Linq;
+using ProjNet.CoordinateSystems;
+using ProjNet.CoordinateSystems.Transformations;
+using ProjNet.IO.CoordinateSystems;
+using NetTopologySuite.Algorithm;
+using System.Globalization;
+
+namespace BIE.DataPipeline.Import
+{
+    class CityGmlImporter : IImporter
+    {
+        private DataSourceDescription description;
+        private int buildingIndex = 0;
+        private XmlNodeList buildingNodes;
+        private XmlNamespaceManager nsmgr;
+        private CultureInfo culture = new CultureInfo("en-US");
+
+        private readonly ICoordinateTransformation? mTransformation;
+
+        public CityGmlImporter(DataSourceDescription description)
+        {
+            // Define the source and target coordinate systems
+            var utmZone32 = CoordinateSystemWktReader
+                .Parse("PROJCS[\"WGS 84 / UTM zone 32N\",GEOGCS[\"WGS 84\",DATUM[\"WGS_1984\",SPHEROID[\"WGS" +
+                       " 84\",6378137,298.257223563,AUTHORITY[\"EPSG\",\"7030\"]],AUTHORITY[\"EPSG\",\"6326\"]]" +
+                       ",PRIMEM[\"Greenwich\",0,AUTHORITY[\"EPSG\",\"8901\"]],UNIT[\"degree\",0.0174532925199433," +
+                       "AUTHORITY[\"EPSG\",\"9122\"]],AUTHORITY[\"EPSG\",\"4326\"]],PROJECTION" +
+                       "[\"Transverse_Mercator\"],PARAMETER[\"latitude_of_origin\",0],PARAMETER" +
+                       "[\"central_meridian\",9],PARAMETER[\"scale_factor\",0.9996],PARAMETER[\"false_easting\"" +
+                       ",500000],PARAMETER[\"false_northing\",0],UNIT[\"metre\",1,AUTHORITY[\"EPSG\",\"9001\"]]" +
+                       ",AUTHORITY[\"EPSG\",\"32632\"]]");
+            var wgs84 = GeographicCoordinateSystem.WGS84;
+            // Create coordinate transformation
+            mTransformation =
+                new CoordinateTransformationFactory().CreateFromCoordinateSystems((CoordinateSystem)utmZone32, wgs84);
+            this.description = description;
+            this.buildingNodes = ReadBuildings();
+        }
+
+        public XmlNodeList ReadBuildings()
+        {
+            XmlDocument doc = new XmlDocument();
+            this.nsmgr = new XmlNamespaceManager(doc.NameTable);
+            nsmgr.AddNamespace("bldg", "http://www.opengis.net/citygml/building/1.0");
+            nsmgr.AddNamespace("tex", "http://www.opengis.net/citygml/texturedsurface/1.0");
+            nsmgr.AddNamespace("sch", "http://www.ascc.net/xml/schematron");
+            nsmgr.AddNamespace("veg", "http://www.opengis.net/citygml/vegetation/1.0");
+            nsmgr.AddNamespace("xlink", "http://www.w3.org/1999/xlink");
+            nsmgr.AddNamespace("gml", "http://www.opengis.net/gml");
+            nsmgr.AddNamespace("tran", "http://www.opengis.net/citygml/transportation/1.0");
+            nsmgr.AddNamespace("grp", "http://www.opengis.net/citygml/cityobjectgroup/1.0");
+            nsmgr.AddNamespace("base", "http://www.citygml.org/citygml/profiles/base/1.0");
+            nsmgr.AddNamespace("wtr", "http://www.opengis.net/citygml/waterbody/1.0");
+            nsmgr.AddNamespace("dem", "http://www.opengis.net/citygml/relief/1.0");
+            nsmgr.AddNamespace("gen", "http://www.opengis.net/citygml/generics/1.0");
+            nsmgr.AddNamespace("app", "http://www.opengis.net/citygml/appearance/1.0");
+            nsmgr.AddNamespace("frn", "http://www.opengis.net/citygml/cityfurniture/1.0");
+            nsmgr.AddNamespace("smil20", "http://www.w3.org/2001/SMIL20/");
+            nsmgr.AddNamespace("xsi", "http://www.w3.org/2001/XMLSchema-instance");
+            nsmgr.AddNamespace("smil20lang", "http://www.w3.org/2001/SMIL20/Language");
+            nsmgr.AddNamespace("xAL", "urn:oasis:names:tc:ciq:xsdschema:xAL:2.0");
+            nsmgr.AddNamespace(string.Empty, "http://www.opengis.net/citygml/1.0"); // Default namespace
+            nsmgr.AddNamespace("gml", "http://www.opengis.net/gml");
+            doc.Load(description.source.location);
+
+
+
+            XmlNodeList buildingNodes = doc.SelectNodes("//bldg:Building", nsmgr);
+            buildingIndex = 0;
+            Console.WriteLine(buildingNodes.Count + " building in this file");
+            return buildingNodes;
+        }
+
+        /// <summary>
+        /// 
+        /// </summary>
+        /// <param name="nextLine"></param>
+        /// <returns>SQL Polygon, xml data</returns>
+        public bool ReadLine(out string nextLine)
+        {
+            if(this.buildingIndex < buildingNodes.Count)
+            {
+                XmlNode buildingNode = buildingNodes[this.buildingIndex];
+                XmlNode groundSurfaceNode = buildingNode.SelectSingleNode(".//bldg:GroundSurface", this.nsmgr); 
+                if(groundSurfaceNode == null)
+                {
+                    Console.WriteLine("No Ground surface found");
+                    this.buildingIndex++;
+                    nextLine = "";
+                    return true;
+                }
+                XmlNode positionNode = groundSurfaceNode.SelectSingleNode(".//gml:posList", this.nsmgr);
+                if (positionNode == null)
+                {
+                    Console.WriteLine("The ground surface has no position");
+                    this.buildingIndex++;
+                    nextLine = "";
+                    return true;
+                }
+
+                Geometry geometry = UtmCoordinatesToGeometry(positionNode.InnerText);
+
+                float groundHeight = GetBuildingGroundHeight(buildingNode);
+                string districtKey = GetBuildingDistrictKey(buildingNode);
+                string checkDate = GetBuildingCheckDate(buildingNode);
+                float groundArea = GetBuildingGroundArea(buildingNode);
+
+                nextLine = $"geography::STGeomFromText('{geometry.AsText()}', 4326)";
+                nextLine += string.Format(",'{0}'", buildingNode.InnerXml);
+                nextLine += string.Format(",'{0}'", groundHeight.ToString(culture));
+                nextLine += string.Format(",'{0}'", districtKey);
+                nextLine += string.Format(",'{0}'", checkDate);
+                nextLine += string.Format(",{0}", groundArea.ToString(culture));
+
+                this.buildingIndex++;
+                return true;
+            }
+            else
+            {
+                nextLine = "";
+                return false;
+            }
+        }
+
+        private Geometry UtmCoordinatesToGeometry(string utmCoordinates)
+        {
+            //Console.WriteLine(utmCoordinates);
+            //Parse Coordinate string
+            var coordPairs = new List<(double Easting, double Northing)>();
+            var coords = utmCoordinates.Split(' ');
+
+            for (int i = 0; i < coords.Length; i += 3)
+            {
+                if (double.TryParse(coords[i], out double easting) && double.TryParse(coords[i + 1], out double northing))
+                {
+                    coordPairs.Add((easting, northing));
+                }
+            }
+
+            //convert to wgs84
+            var wgs84Coordinates = new List<(double Lon, double Lat)>();
+
+            var csFactory = new CoordinateSystemFactory();
+            var utmZone32 = CoordinateSystemWktReader
+            .Parse("PROJCS[\"WGS 84 / UTM zone 32N\",GEOGCS[\"WGS 84\",DATUM[\"WGS_1984\",SPHEROID[\"WGS" +
+                   " 84\",6378137,298.257223563,AUTHORITY[\"EPSG\",\"7030\"]],AUTHORITY[\"EPSG\",\"6326\"]]" +
+                   ",PRIMEM[\"Greenwich\",0,AUTHORITY[\"EPSG\",\"8901\"]],UNIT[\"degree\",0.0174532925199433," +
+                   "AUTHORITY[\"EPSG\",\"9122\"]],AUTHORITY[\"EPSG\",\"4326\"]],PROJECTION" +
+                   "[\"Transverse_Mercator\"],PARAMETER[\"latitude_of_origin\",0],PARAMETER" +
+                   "[\"central_meridian\",9],PARAMETER[\"scale_factor\",0.9996],PARAMETER[\"false_easting\"" +
+                   ",500000],PARAMETER[\"false_northing\",0],UNIT[\"metre\",1,AUTHORITY[\"EPSG\",\"9001\"]]" +
+                   ",AUTHORITY[\"EPSG\",\"32632\"]]");
+            var utm32 = csFactory.CreateFromWkt(utmZone32.ToString());
+            var wgs84 = GeographicCoordinateSystem.WGS84;
+
+            var transform = new CoordinateTransformationFactory().CreateFromCoordinateSystems(utm32, wgs84).MathTransform;
+
+            foreach (var utmCoord in coordPairs)
+            {
+                double[] utmPoint = { utmCoord.Easting, utmCoord.Northing };
+                double[] wgs84Point = transform.Transform(utmPoint);
+                wgs84Coordinates.Add((wgs84Point[0], wgs84Point[1]));
+            }
+
+            //Convert to polygon
+            if (wgs84Coordinates.Count < 4 || !wgs84Coordinates[0].Equals(wgs84Coordinates[wgs84Coordinates.Count - 1]))
+            {
+                throw new ArgumentException("The coordinates must form a closed ring with at least 4 points.");
+            }
+
+            var coordinateArray = new Coordinate[wgs84Coordinates.Count];
+            for (int i = 0; i < wgs84Coordinates.Count; i++)
+            {
+                coordinateArray[i] = new Coordinate(wgs84Coordinates[i].Lon, wgs84Coordinates[i].Lat);
+            }
+
+            var geometryFactory = new GeometryFactory();
+            var linearRing = new LinearRing(coordinateArray);
+            return new Polygon(linearRing);
+        }
+
+        private Geometry ConvertUtmToLatLong(Geometry polygon)
+        {
+            // Convert UTM coordinates to latitude and longitude
+            foreach (Coordinate coordinate in polygon.Coordinates)
+            {
+                double utmEasting = coordinate.X;
+                double utmNorthing = coordinate.Y;
+                double[] utmPoint = { utmEasting, utmNorthing };
+                double[] wgs84Point = mTransformation!.MathTransform.Transform(utmPoint);
+
+                // Extract latitude and longitude
+                double latitude = wgs84Point[1];
+                double longitude = wgs84Point[0];
+                coordinate.X = latitude;
+                coordinate.Y = longitude;
+            }
+
+            return polygon;
+        }
+
+        private float GetBuildingGroundHeight(XmlNode buildingNode)
+        {
+            XmlNode groundHeightNode = buildingNode.SelectSingleNode(".//gen:stringAttribute[@name='DatenquelleBodenhoehe']/gen:value", this.nsmgr);
+
+            if(groundHeightNode == null)
+            {
+                Console.WriteLine("No ground height node");
+                return -1;
+            }
+
+            float result = 0;
+            if(!float.TryParse(groundHeightNode.InnerText, NumberStyles.Any, culture, out result))
+            {
+                Console.WriteLine("Unable to get ground height");
+                return -1;
+            }
+
+            return result;
+        }
+
+        private string GetBuildingDistrictKey(XmlNode buildingNode)
+        {
+            XmlNode distrcitKeyNode = buildingNode.SelectSingleNode(".//gen:stringAttribute[@name='Gemeindeschluessel']/gen:value", this.nsmgr);
+
+            if (distrcitKeyNode == null)
+            {
+                Console.WriteLine("No district key node");
+                return "";
+            }
+
+            return distrcitKeyNode.InnerText;
+        }
+
+        private string GetBuildingCheckDate(XmlNode buildingNode)
+        {
+            XmlNode checkDateNode = buildingNode.SelectSingleNode(".//gen:stringAttribute[@name='Grundrissaktualitaet']/gen:value", this.nsmgr);
+
+            if (checkDateNode == null)
+            {
+                Console.WriteLine("No change date node");
+                return "";
+            }
+
+            return checkDateNode.InnerText;
+        }
+
+        private float GetBuildingGroundArea(XmlNode buildingNode)
+        {
+            XmlNodeList groundSurfaceNodes = buildingNode.SelectNodes(".//bldg:GroundSurface", this.nsmgr);
+
+            if (groundSurfaceNodes == null)
+            {
+                Console.WriteLine("No nodes");
+                return -1;
+            }
+
+            if(groundSurfaceNodes.Count > 1)
+            {
+                Console.WriteLine("multi nodes " + groundSurfaceNodes.Count);
+            }
+
+            float res = 0;
+            foreach(XmlNode groundSurface in groundSurfaceNodes)
+            {
+                res += ParseArea(groundSurface);
+            }
+            return res;
+        }
+
+        private float ParseArea(XmlNode node)
+        {
+            XmlNode areaNode = node.SelectSingleNode(".//gen:stringAttribute[@name='Flaeche']/gen:value", this.nsmgr);
+            if(areaNode == null)
+            {
+                Console.WriteLine("No area found");
+                return 0;
+            }
+
+            float result = 0;
+            if (!float.TryParse(areaNode.InnerText, NumberStyles.Any, culture, out result))
+            {
+                Console.WriteLine("Unable to get area");
+                return 0;
+            }
+
+            return result;
+        }
+    }
+}

backend/src/BIE.DataPipeline/Program.cs

@@ -2,8 +2,8 @@
 using BIE.DataPipeline.Import;
 using BIE.DataPipeline.Metadata;
 using Mono.Options;
-
-// Setup the command line options
+using System.Xml;
+// setup command line options.
 var tableInsertBehaviour = InsertBehaviour.none;
 var filename = HandleCliArguments();
 
@@ -70,6 +70,10 @@
             importer = new ShapeImporter(description);
             dbHelper.SetInfo(description.table_name, "Location");
             break;
+        case "CITYGML":
+            importer = new CityGmlImporter(description);
+            dbHelper.SetInfo(description.table_name, "Location, XmlData, GroundHeight, DistrictKey, CheckDate, GroundArea");
+            break;
 
         default:
             Console.WriteLine($"Unknown or missing data format: {description.source.data_format}");

backend/src/BIE.DataPipeline/yaml/building_models.yaml

@@ -0,0 +1,16 @@
+# describe the source
+source:
+  # link | filepath
+  type: URL
+  location: https://download1.bayernwolke.de/a/lod2/citygml/652_5496.gml
+  data_format: CITYGML
+options:
+  # skip lines at the beginning
+  skip_lines: 0
+  # discard any rows that have null values
+  discard_null_rows: false
+  # how to deal with existing table. Options: ignore, replace, skip (default).
+  if_table_exists: replace
+table_name: building_models
+
+table_cols:

backend/src/BIE.DataPipeline/yaml/building_models_fauTechFak.yaml

@@ -0,0 +1,16 @@
+# describe the source
+source:
+  # link | filepath
+  type: 
+  location: https://download1.bayernwolke.de/a/lod2/citygml/646_5492.gml
+  data_format: CITYGML
+options:
+  # skip lines at the beginning
+  skip_lines: 0
+  # discard any rows that have null values
+  discard_null_rows: false
+  # how to deal with existing table. Options: ignore, replace, skip (default).
+  if_table_exists: replace
+table_name: building_models
+
+table_cols: