Fix mangled tags and other things (#5482)

* Added more code to untangle tags and values refs: #5476 * Added extra fixes refs: #5476
ngageoint · Nov 2, 2022 · 50c3992 · 50c3992
1 parent dae66ca
commit 50c3992
Show file tree

Hide file tree

Showing 4 changed files with 214 additions and 26 deletions.
diff --git a/conf/schema/landcover.json b/conf/schema/landcover.json
@@ -0,0 +1,14 @@
+[
+    {
+        "dataType": "enumeration",
+        "geometries": [ "area" ],
+        "influence": 1,
+        "name": "landcover",
+        "objectType": "tag"
+    },
+    {
+        "isA": "landcover",
+        "name": "landcover=*",
+        "objectType": "tag"
+    }
+]
diff --git a/conf/schema/schema.json b/conf/schema/schema.json
@@ -55,6 +55,7 @@
     { "import": "is_divided.json" },
     { "import": "is_in.json" },
     { "import": "junction.json" },
+    { "import": "landcover.json" },
     { "import": "landuse.json" },
     { "import": "lanes.json" },
     { "import": "leisure.json" },

diff --git a/translations/checkTranslations.js b/translations/checkTranslations.js
@@ -255,7 +255,8 @@ var startArea = '<osm version="0.6" upload="true" generator="hootenanny">\
 var endPoint = '</node></osm>';
 var endLine = '</way></osm>'; // NOTE: This is also for Areas as well
 
-// Test a translated feature
+
+// Test translating OGR to and from OSM
 function testTranslated(schema,featureCode,tagList,geomList = ['Point','Line','Area'])
 {
   console.log('---------------');
@@ -328,7 +329,80 @@ function testTranslated(schema,featureCode,tagList,geomList = ['Point','Line','A
 }; // End testTranslated
 
 
-// Test a translated feature
+// Test translating from OGR to OSM
+function testToOSM(schema,featureCode,tagList,geomList = ['Point','Line','Area'])
+{
+  console.log('---------------');
+  var osmFeatures = {};
+
+  osmFeatures.Point = startPoint + '<tag k="F_CODE" v="' + featureCode + '"/>';
+  for (var tag in tagList) { osmFeatures.Point += "<tag k='" + tag + "' v='" + tagList[tag] + "'/>" }
+  osmFeatures.Point += endPoint;
+
+  osmFeatures.Line = startLine + '<tag k="F_CODE" v="' + featureCode + '"/>';
+  for (var tag in tagList) { osmFeatures.Line += "<tag k='" + tag + "' v='" + tagList[tag] + "'/>" }
+  osmFeatures.Line += endLine;
+
+  osmFeatures.Area = startArea + '<tag k="F_CODE" v="' + featureCode + '"/>';
+  for (var tag in tagList) { osmFeatures.Area += "<tag k='" + tag + "' v='" + tagList[tag] + "'/>" }
+  osmFeatures.Area += endLine;
+
+  for (var geom of geomList)
+  {
+    console.log('' + schema + '  ' + featureCode + '  ' + geom);
+
+    // Raw input
+    var inputJson = makeJson(osmFeatures[geom]);
+    console.log('Raw: ' + JSON.stringify(inputJson,Object.keys(inputJson).sort()));
+
+    var toOsm = ogrToOsm(osmFeatures[geom],schema);
+    var osmJson = makeJson(toOsm);
+
+    if (Object.keys(osmJson).length > 0)
+    {
+      // console.log(osmJson);
+      console.log('OSM: ' + JSON.stringify(osmJson,Object.keys(osmJson).sort()));
+    }
+    else
+    {
+      console.log('  ## No OSM tags for ' + featureCode + ' ##');
+    }
+
+    var toOgr = osmToOgr(toOsm,schema);
+    var ogrJson = makeJson(toOgr);
+
+    if (Object.keys(ogrJson).length > 0)
+    {
+      // console.log(ogrJson);
+      console.log('Ogr: ' + JSON.stringify(ogrJson,Object.keys(ogrJson).sort()));
+    }
+    else
+    {
+      console.log('  ## No Ogr tags for ' + featureCode + ' ##');
+    }
+
+    // Sanity Check
+    var backToOsm = ogrToOsm(toOgr,schema);
+    var secondJson = makeJson(backToOsm);
+    // console.log(secondJson);
+    console.log('OSM: ' + JSON.stringify(secondJson,Object.keys(secondJson).sort()));
+
+    if (JSON.stringify(osmJson,Object.keys(osmJson).sort()) !== JSON.stringify(secondJson,Object.keys(secondJson).sort()))
+    {
+      console.log('  ## Not Same OSM Tags: ' + ogrJson.F_CODE + ' vs ' + featureCode);
+    }
+
+    if (ogrJson.F_CODE !== featureCode)
+    {
+      console.log('  ## Not Same F_CODE: ' + ogrJson.F_CODE + ' vs ' + featureCode);
+    }
+
+    console.log('-----');
+  } // End geom
+}; // End testToOSM
+
+
+// Test translating OSM to and from OGR
 function testOSM(schema,tagList,geomList = ['Point','Line','Area'])
 {
   console.log('---------------');
@@ -396,6 +470,49 @@ function testOSM(schema,tagList,geomList = ['Point','Line','Area'])
   } // End geom
 }; // End testOSM
 
+// Test translating from OSM to OGR
+function testToOGR(schema,tagList,geomList = ['Point','Line','Area'])
+{
+  console.log('---------------');
+
+  var osmFeatures = {};
+
+  osmFeatures.Point = startPoint;
+  for (var tag in tagList) { osmFeatures.Point += "<tag k='" + tag + "' v='" + tagList[tag] + "'/>" }
+  osmFeatures.Point += endPoint;
+
+  osmFeatures.Line = startLine;
+  for (var tag in tagList) { osmFeatures.Line += "<tag k='" + tag + "' v='" + tagList[tag] + "'/>" }
+  osmFeatures.Line += endLine;
+
+  osmFeatures.Area = startArea;
+  for (var tag in tagList) { osmFeatures.Area += "<tag k='" + tag + "' v='" + tagList[tag] + "'/>" }
+  osmFeatures.Area += endLine;
+
+  for (var geom of geomList)
+  {
+    console.log('' + schema + '  ' + geom + ':');
+
+    // Raw input
+    var inputJson = makeJson(osmFeatures[geom]);
+    console.log('Raw: ' + JSON.stringify(inputJson,Object.keys(inputJson).sort()));
+
+    var toOgr = osmToOgr(osmFeatures[geom],schema);
+    var ogrJson = makeJson(toOgr);
+
+    if (Object.keys(ogrJson).length > 0)
+    {
+      // console.log(ogrJson);
+      console.log('Ogr: ' + JSON.stringify(ogrJson,Object.keys(ogrJson).sort()));
+    }
+    else
+    {
+      console.log('  ## No Ogr tags for ' + featureCode + ' ##');
+    }
+    console.log('-----');
+  } // End geom
+}; // End testToOGR
+
 
 // Go through a schema and test each feature
 function testSchema(schemaMap)
@@ -510,6 +627,8 @@ if (typeof exports !== 'undefined') {
     exports.testF_CODE = testF_CODE;
     exports.testTranslated = testTranslated;
     exports.testOSM = testOSM;
+    exports.testToOSM = testToOSM;
+    exports.testToOGR = testToOGR;
     exports.testSchema = testSchema;
     exports.dumpValues = dumpValues;
     exports.dumpSchema = dumpSchema;

diff --git a/translations/translate.js b/translations/translate.js
@@ -1768,39 +1768,93 @@ translate = {
   unpackOtherTags: function(tags)
   {
     // Sanity check
-    if (tags.other_tags)
-    {
+    if (!tags.other_tags) return;
 
-      var string = tags['other_tags'].toString();
-      delete tags.other_tags;
+    var rString = tags['other_tags'].toString();
+    delete tags.other_tags; // Cleanup
 
-      // NOTE: This is not my code.I found it when looking for how to parse hstore format from StackOverflow.
-      // I have tweaked it a bit
+    // NOTE: This is not my code.I found it when looking for how to parse hstore format from StackOverflow.
+    // I have tweaked it a bit
 
-      //using [\s\S] to match any character, including line feed and carriage return,
-      var r = /(["])(?:\\\1|\\\\|[\s\S])*?\1|NULL/g,
-          matches = string.match(r);
+    //using [\s\S] to match any character, including line feed and carriage return,
+    var r = /(["])(?:\\\1|\\\\|[\s\S])*?\1|NULL/g,
+        matches = rString.match(r);
 
-      var clean = function (value) {
-                value = value.replace(/^\"|\"$/g, ""); // Remove leading double quotes
-                value = value.replace(/\\"/g, "\""); // Unescape quotes
-                value = value.replace(/\\\\/g,"\\"); //Unescape backslashes
-                value = value.replace(/''/g,"'"); //Unescape single quotes
+    var clean = function (value) {
+              value = value.replace(/^\"|\"$/g, ""); // Remove leading double quotes
+              value = value.replace(/\\"/g, "\""); // Unescape quotes
+              value = value.replace(/\\\\/g,"\\"); //Unescape backslashes
+              value = value.replace(/''/g,"'"); //Unescape single quotes
 
-                return value;
-            };
+              return value;
+          };
 
-        if(matches) {
-          for (var i = 0, l = matches.length; i < l; i+= 2) {
-            if (matches[i] && matches[i + 1]) {
-              var key = clean(matches[i]);
-              var value = matches[i + 1];
+    if(matches) {
+      // Debug
+      // print('Num Matches = ' + matches.length);
+      var gotTag = false;
+      for (var i = 0, l = matches.length; i < l; i+= 2) {
+        if (matches[i] && matches[i + 1]) {
+          var key = clean(matches[i]);
+          var value = matches[i + 1];
 
-              // Taking the chance that this may stomp on an existing tag value.
-              tags[key] = value=="NULL"?null:clean(value);
-            }
+          // Debug
+          // print('matches key: ' + key + '  value: ' + value);
+
+          // Taking the chance that this may stomp on an existing tag value.
+          tags[key] = value=="NULL"?null:clean(value);
+          gotTag = true;
+        }
+      }
+      if (gotTag) return;
+    }
+
+    // If we get to here then the above code couldn't parse it
+    // First, try to sort out the other_tags tag
+    var otList = rString.replace(/\"/g,'').replace(/\\/g,'').split('=>');
+
+    if (otList.length > 1) // Sanity check. We should have an even number
+    {
+      for (var i = 0, iLen = otList.length; i < iLen; i=i+2)
+      {
+        tags[otList[i]] = otList[i+1];
+      }
+    }
+
+    // Now go looking for fragmented tags
+    for (var i in tags)
+    {
+      if (i.indexOf('=>') > -1) // Got one
+      {
+        tstr = i.toString();
+        if (tstr.charAt(0) !== '"') tstr = '"' + i;
+        if (tstr.substr(-1) !== '"') tstr += '"';
+
+        // Clean up
+        tstr = tstr.replace(/\\\"/g,'"').replace(/""/g,'\"').replace(/", "/g,'\",\"');
+
+        // Add the last value to the end
+        tstr = tstr + '=>' + '"' + tags[i] + '"';
+
+        // Convert to JSON
+        tstr = tstr.replace(/=>/g,':');
+
+        try
+        {
+          var tObj = JSON.parse('{' + tstr + '}');
+          for (var j in tObj)
+          {
+            tags[j] = tObj[j];
           }
+
+          // It unpacked so we can delete the ugly tag
+          delete tags[i];
         }
+        catch (error)
+        {
+          hoot.logWarn('Unable to unpack other_tags: ' + error);
+        }
+      }
     }
   }, // End unpackOtherTags