Allow YAML for human-edited metadata (YAMLKAN)

Netkan/CKAN-netkan.csproj

@@ -45,6 +45,7 @@
     <PackageReference Include="log4net" Version="2.0.10" />
     <PackageReference Include="Namotion.Reflection" Version="1.0.7" />
     <PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
+    <PackageReference Include="YamlDotNet" Version="9.1.0" />
     <PackageReference Include="NJsonSchema" Version="10.0.27" />
   </ItemGroup>
   <ItemGroup>
@@ -62,6 +63,7 @@
     <Compile Include="Constants.cs" />
     <Compile Include="Extensions\JObjectExtensions.cs" />
     <Compile Include="Extensions\VersionExtensions.cs" />
+    <Compile Include="Extensions\YamlExtensions.cs" />
     <Compile Include="Model\Metadata.cs" />
     <Compile Include="Model\RemoteRef.cs" />
     <Compile Include="Processors\Inflator.cs" />

Netkan/Extensions/YamlExtensions.cs

@@ -0,0 +1,90 @@
+using System.IO;
+using System.Linq;
+using log4net;
+using YamlDotNet.Core;
+using YamlDotNet.RepresentationModel;
+using Newtonsoft.Json.Linq;
+
+namespace CKAN.NetKAN.Extensions
+{
+    internal static class YamlExtensions
+    {
+        public static YamlMappingNode Parse(string input)
+        {
+            return Parse(new StringReader(input));
+        }
+
+        public static YamlMappingNode Parse(TextReader input)
+        {
+            var stream = new YamlStream();
+            stream.Load(input);
+            return stream.Documents.FirstOrDefault()?.RootNode as YamlMappingNode;
+        }
+
+        /// <summary>
+        /// Convert a YAML object to a JSON object
+        /// </summary>
+        /// <param name="yaml">The input object</param>
+        /// <returns>
+        /// A JObject representation of the input data
+        /// </returns>
+        public static JObject ToJObject(this YamlMappingNode yaml)
+        {
+            var jobj = new JObject();
+            foreach (var kvp in yaml)
+            {
+                switch (kvp.Value.NodeType)
+                {
+                    case YamlNodeType.Mapping:
+                        jobj.Add((string)kvp.Key, (kvp.Value as YamlMappingNode).ToJObject());
+                        break;
+                    case YamlNodeType.Sequence:
+                        jobj.Add((string)kvp.Key, (kvp.Value as YamlSequenceNode).ToJarray());
+                        break;
+                    case YamlNodeType.Scalar:
+                        jobj.Add((string)kvp.Key, (kvp.Value as YamlScalarNode).ToJValue());
+                        break;
+                }
+            }
+            return jobj;
+        }
+
+        private static JArray ToJarray(this YamlSequenceNode yaml)
+        {
+            var jarr = new JArray();
+            foreach (var elt in yaml)
+            {
+                switch (elt.NodeType)
+                {
+                    case YamlNodeType.Mapping:
+                        jarr.Add((elt as YamlMappingNode).ToJObject());
+                        break;
+                    case YamlNodeType.Sequence:
+                        jarr.Add((elt as YamlSequenceNode).ToJarray());
+                        break;
+                    case YamlNodeType.Scalar:
+                        jarr.Add((elt as YamlScalarNode).ToJValue());
+                        break;
+                }
+            }
+            return jarr;
+        }
+
+        private static JValue ToJValue(this YamlScalarNode yaml)
+        {
+            switch (yaml.Value)
+            {
+                case "null":  return JValue.CreateNull();
+                case "true":  return new JValue(true);
+                case "false": return new JValue(false);
+                // Convert unquoted integers to int type
+                default:      return yaml.Style == ScalarStyle.Plain
+                                     && int.TryParse(yaml.Value, out int intVal)
+                                         ? new JValue(intVal)
+                                         : new JValue(yaml.Value);
+            }
+        }
+
+        private static readonly ILog log = LogManager.GetLogger(typeof(YamlExtensions));
+    }
+}

Netkan/Model/Metadata.cs

@@ -2,6 +2,8 @@
 using System.Linq;
 using CKAN.Versioning;
 using Newtonsoft.Json.Linq;
+using YamlDotNet.RepresentationModel;
+using CKAN.NetKAN.Extensions;
 
 namespace CKAN.NetKAN.Model
 {
@@ -117,6 +119,10 @@ public Metadata(JObject json)
             }
         }
 
+        public Metadata(YamlMappingNode yaml) : this(yaml?.ToJObject())
+        {
+        }
+
         public string[] Licenses
         {
             get

Netkan/Processors/QueueHandler.cs

@@ -16,6 +16,7 @@
 using CKAN.Versioning;
 using CKAN.NetKAN.Transformers;
 using CKAN.NetKAN.Model;
+using CKAN.NetKAN.Extensions;
 
 namespace CKAN.NetKAN.Processors
 {
@@ -127,7 +128,7 @@ private void handleMessages(string url, int howMany, int timeoutMinutes)
         private IEnumerable<SendMessageBatchRequestEntry> Inflate(Message msg)
         {
             log.DebugFormat("Metadata returned: {0}", msg.Body);
-            var netkan = new Metadata(JObject.Parse(msg.Body));
+            var netkan = new Metadata(YamlExtensions.Parse(msg.Body));
 
             int releases = 1;
             MessageAttributeValue releasesAttr;

Netkan/Program.cs

@@ -15,6 +15,7 @@
 using CKAN.NetKAN.Model;
 using CKAN.NetKAN.Processors;
 using CKAN.NetKAN.Transformers;
+using CKAN.NetKAN.Extensions;
 
 namespace CKAN.NetKAN
 {
@@ -174,7 +175,7 @@ private static Metadata ReadNetkan()
                 Log.WarnFormat("Input is not a .netkan file");
             }
 
-            return new Metadata(JObject.Parse(File.ReadAllText(Options.File)));
+            return new Metadata(YamlExtensions.Parse(File.OpenText(Options.File)));
         }
 
         internal static string CkanFileName(Metadata metadata)

Netkan/Services/ModuleService.cs

@@ -8,6 +8,7 @@
 using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
 
+using CKAN.NetKAN.Extensions;
 using CKAN.Versioning;
 using CKAN.Extensions;
 using CKAN.NetKAN.Sources.Avc;
@@ -200,10 +201,7 @@ private static JObject DeserializeFromStream(Stream stream)
         {
             using (var sr = new StreamReader(stream))
             {
-                using (var jsonTextReader = new JsonTextReader(sr))
-                {
-                    return (JObject)JToken.ReadFrom(jsonTextReader);
-                }
+                return YamlExtensions.Parse(sr).ToJObject();
             }
         }
 

Netkan/Transformers/MetaNetkanTransformer.cs

@@ -2,6 +2,7 @@
 using System.Collections.Generic;
 using log4net;
 using Newtonsoft.Json.Linq;
+
 using CKAN.Versioning;
 using CKAN.NetKAN.Extensions;
 using CKAN.NetKAN.Model;
@@ -51,7 +52,7 @@ public IEnumerable<Metadata> Transform(Metadata metadata, TransformOptions opts)
 
                 Log.DebugFormat("Target netkan:{0}{1}", Environment.NewLine, targetFileText);
 
-                var targetJson = JObject.Parse(targetFileText);
+                var targetJson = YamlExtensions.Parse(targetFileText).ToJObject();
                 var targetMetadata = new Metadata(targetJson);
 
                 if (targetMetadata.Kref == null || targetMetadata.Kref.Source != "netkan")

Spec.md

@@ -688,6 +688,23 @@ NetKAN is the name the tool which is used to automatically generate CKAN files f
 consumes `.netkan` files to produce `.ckan` files. `.netkan` files are a *strict superset* of `.ckan` files. Every
 `.ckan` file is a valid `.netkan` file but not vice versa. NetKAN uses the following fields to produce `.ckan` files.
 
+##### YAML Option
+
+A `.netkan` file may be in either JSON or YAML format. All examples shown below assume JSON, but the YAML equivalents will work the same way.
+
+Note that `#` is the comment character in YAML, so even if you choose YAML syntax, you still can't omit the quotes around a value that includes `#`, such as typical values of `$kref` and `$vref`:
+
+```yaml
+$kref: "#/ckan/spacedock/1234"
+$vref: "#/ckan/ksp-avc"
+```
+
+##### Internal `.ckan` files
+
+If a module's download contains a file with a `.ckan` extension, this file will be parsed and its contents added to the module's metadata. This can be a convenient way to handle metadata values that can change from one version to the next, such as dependencies.
+
+An internal `.ckan` file may be in either JSON or YAML format.
+
 ##### `$kref`
 
 The `$kref` field indicates that data should be filled in from an external service provider. The following `$kref`
@@ -857,6 +874,8 @@ The remote `.netkan` file is downloaded and used as if it were the original. `.n
 reference are known as *recursive netkans* or *metanetkans*. They are primarily used so that mod authors can provide
 authoritative metadata.
 
+A metanetkan may be in either JSON or YAML format.
+
 The following conditions apply:
 - A metanekan may not reference another metanetkan, otherwise an error is produced.
 - Any fields specified in the metanetkan will override any fields in the target netkan file.

Tests/NetKAN/Extensions/YamlExtensionsTests.cs

@@ -0,0 +1,140 @@
+using System.Linq;
+using NUnit.Framework;
+using YamlDotNet.RepresentationModel;
+using Newtonsoft.Json.Linq;
+
+using CKAN.NetKAN.Extensions;
+
+namespace Tests.NetKAN.Extensions
+{
+    [TestFixture]
+    public sealed class YamlExtensionsTests
+    {
+        [Test]
+        public void Parse_ValidInput_Works()
+        {
+            // Arrange
+            string input = string.Join("\r\n", new string[]
+            {
+                "spec_version: v1.4",
+                "identifier: Astrogator",
+                "$kref: \"#/ckan/github/HebaruSan/Astrogator\"",
+                "$vref: \"#/ckan/ksp-avc\"",
+                "license: GPL-3.0",
+                "tags:",
+                "    - plugin",
+                "    - information",
+                "    - control",
+                "resources:",
+                "    homepage: https://forum.kerbalspaceprogram.com/index.php?/topic/155998-*",
+                "    bugtracker: https://github.com/HebaruSan/Astrogator/issues",
+                "    repository: https://github.com/HebaruSan/Astrogator",
+                "recommends:",
+                "    - name: ModuleManager",
+                "    - name: LoadingTipsPlus",
+            });
+
+            // Act
+            YamlMappingNode yaml = YamlExtensions.Parse(input);
+
+            // Assert
+            Assert.AreEqual("v1.4",                               (string)yaml["spec_version"]);
+            Assert.AreEqual("Astrogator",                         (string)yaml["identifier"]);
+            Assert.AreEqual("#/ckan/github/HebaruSan/Astrogator", (string)yaml["$kref"]);
+            Assert.AreEqual("#/ckan/ksp-avc",                     (string)yaml["$vref"]);
+            Assert.AreEqual("GPL-3.0",                            (string)yaml["license"]);
+
+            CollectionAssert.AreEqual(
+                new string[] { "plugin", "information", "control" },
+                (yaml["tags"] as YamlSequenceNode).Children.Select(yn => (string)yn)
+            );
+            Assert.AreEqual(
+                "https://forum.kerbalspaceprogram.com/index.php?/topic/155998-*",
+                (string)yaml["resources"]["homepage"]
+            );
+            Assert.AreEqual(
+                "https://github.com/HebaruSan/Astrogator/issues",
+                (string)yaml["resources"]["bugtracker"]
+            );
+            Assert.AreEqual(
+                "https://github.com/HebaruSan/Astrogator",
+                (string)yaml["resources"]["repository"]
+            );
+            Assert.AreEqual("ModuleManager",   (string)yaml["recommends"][0]["name"]);
+            Assert.AreEqual("LoadingTipsPlus", (string)yaml["recommends"][1]["name"]);
+        }
+
+        [Test]
+        public void ToJObject_ValidInput_Works()
+        {
+            // Arrange
+            var yaml = new YamlMappingNode()
+            {
+                { "spec_version", "v1.4" },
+                { "identifier",   "Astrogator" },
+                { "$kref",        "#/ckan/github/HebaruSan/Astrogator" },
+                { "$vref",        "#/ckan/ksp-avc" },
+                { "license",      "GPL-3.0" },
+                {
+                    "tags",
+                    new YamlSequenceNode(
+                        "plugin",
+                        "information",
+                        "control"
+                    )
+                },
+                {
+                    "resources",
+                    new YamlMappingNode()
+                    {
+                        { "homepage", "https://forum.kerbalspaceprogram.com/index.php?/topic/155998-*" },
+                        { "bugtracker", "https://github.com/HebaruSan/Astrogator/issues" },
+                        { "repository", "https://github.com/HebaruSan/Astrogator" },
+                    }
+                },
+                {
+                    "recommends",
+                    new YamlSequenceNode(
+                        new YamlMappingNode()
+                        {
+                            { "name", "ModuleManager"   }
+                        },
+                        new YamlMappingNode()
+                        {
+                            { "name", "LoadingTipsPlus" }
+                        }
+                    )
+                }
+            };
+
+            // Act
+            JObject json = yaml.ToJObject();
+
+            // Assert
+            Assert.AreEqual("v1.4",                               (string)json["spec_version"]);
+            Assert.AreEqual("Astrogator",                         (string)json["identifier"]);
+            Assert.AreEqual("#/ckan/github/HebaruSan/Astrogator", (string)json["$kref"]);
+            Assert.AreEqual("#/ckan/ksp-avc",                     (string)json["$vref"]);
+            Assert.AreEqual("GPL-3.0",                            (string)json["license"]);
+
+            CollectionAssert.AreEqual(
+                new string[] { "plugin", "information", "control" },
+                (json["tags"] as JArray).Select(elt => (string)elt)
+            );
+            Assert.AreEqual(
+                "https://forum.kerbalspaceprogram.com/index.php?/topic/155998-*",
+                (string)json["resources"]["homepage"]
+            );
+            Assert.AreEqual(
+                "https://github.com/HebaruSan/Astrogator/issues",
+                (string)json["resources"]["bugtracker"]
+            );
+            Assert.AreEqual(
+                "https://github.com/HebaruSan/Astrogator",
+                (string)json["resources"]["repository"]
+            );
+            Assert.AreEqual("ModuleManager",   (string)json["recommends"][0]["name"]);
+            Assert.AreEqual("LoadingTipsPlus", (string)json["recommends"][1]["name"]);
+        }
+    }
+}

Tests/Tests.csproj

@@ -46,6 +46,7 @@
     <PackageReference Include="log4net" Version="2.0.10" />
     <PackageReference Include="Moq" Version="4.14.5" />
     <PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
+    <PackageReference Include="YamlDotNet" Version="9.1.0" />
     <PackageReference Include="NUnit" Version="3.12.0" />
     <PackageReference Include="NUnit3TestAdapter" Version="3.16.1" />
   </ItemGroup>