Merge pull request #32 from alopezlago/alopezlago/public_v040

Merged PR 221: v0.4.0 - Read post question metadata, support multiple…

YetAnotherPacketParser/YetAnotherPacketParser/Ast/BonusNode.cs

@@ -5,20 +5,21 @@ namespace YetAnotherPacketParser.Ast
 {
     public class BonusNode
     {
-        public BonusNode(int number, FormattedText leadin, IEnumerable<BonusPartNode> parts, string? editorsNote)
+        public BonusNode(
+            int number, FormattedText leadin, IEnumerable<BonusPartNode> parts, string? metadata)
         {
-            this.EditorsNote = editorsNote;
             this.Leadin = leadin ?? throw new ArgumentNullException(nameof(leadin));
             this.Number = number;
             this.Parts = parts ?? throw new ArgumentNullException(nameof(parts));
+            this.Metadata = metadata;
         }
 
-        public string? EditorsNote { get; }
-
         public FormattedText Leadin { get; }
 
         public int Number { get; }
 
         public IEnumerable<BonusPartNode> Parts { get; }
+
+        public string? Metadata { get; }
     }
 }

YetAnotherPacketParser/YetAnotherPacketParser/Ast/TossupNode.cs

@@ -4,17 +4,17 @@ namespace YetAnotherPacketParser.Ast
 {
     public class TossupNode
     {
-        public TossupNode(int number, QuestionNode question, string? editorsNote = null)
+        public TossupNode(int number, QuestionNode question, string? metadata = null)
         {
-            this.EditorsNote = editorsNote;
             this.Number = number;
             this.Question = question ?? throw new ArgumentNullException(nameof(question));
+            this.Metadata = metadata;
         }
 
-        public string? EditorsNote { get; }
-
         public int Number { get; }
 
         public QuestionNode Question { get; }
+
+        public string? Metadata { get; }
     }
 }

YetAnotherPacketParser/YetAnotherPacketParser/Compiler/Html/HtmlCompiler.cs

@@ -48,6 +48,12 @@ private static void WriteTossup(TossupNode tossup, StringBuilder builder)
             builder.Append(tossup.Number);
             builder.Append(". ");
             WriteQuestion(tossup.Question, builder);
+            if (!string.IsNullOrEmpty(tossup.Metadata))
+            {
+                builder.Append(tossup.Metadata);
+                builder.Append("<br>");
+            }
+
             builder.Append("</p>");
         }
 
@@ -62,6 +68,13 @@ private static void WriteBonus(BonusNode bonus, StringBuilder builder)
             {
                 WriteBonusPart(bonusPart, builder);
             }
+
+            if (!string.IsNullOrEmpty(bonus.Metadata))
+            {
+                builder.Append(bonus.Metadata);
+                builder.Append("<br>");
+            }
+
             builder.Append("</p>");
         }
 

YetAnotherPacketParser/YetAnotherPacketParser/Compiler/Json/JsonBonusNode.cs

@@ -24,6 +24,7 @@ public JsonBonusNode(BonusNode bonusNode, bool includeSanitizedFields)
                 null;
             this.Values = new List<int>();
 
+            this.Metadata = bonusNode.Metadata;
             this.DifficultyModifiers = partNodes.Any(node => node.DifficultyModifier.HasValue) ? new List<char?>() : null;
 
             foreach (BonusPartNode partNode in partNodes)
@@ -42,6 +43,8 @@ public JsonBonusNode(BonusNode bonusNode, bool includeSanitizedFields)
 
         public string? Leadin_sanitized { get; }
 
+        public string? Metadata { get; }
+
         public ICollection<string> Answers { get; }
 
         public ICollection<string>? Answers_sanitized { get; }

YetAnotherPacketParser/YetAnotherPacketParser/Compiler/Json/JsonCompiler.cs

@@ -25,7 +25,7 @@ public async Task<string> CompileAsync(PacketNode packet)
             // The format that Jerry's parser uses for JSON (and that the reader expects as a result) is different
             // than the structure of the PacketNode, so transform it to a structure close to it (minus author and
             // packet fields)
-            JsonPacketNode sanitizedJsonPacket = new JsonPacketNode(sanitizedPacket, !this.Options.ModaqFormat);
+            JsonPacketNode sanitizedJsonPacket = new JsonPacketNode(sanitizedPacket, this.Options.ModaqFormat);
 
             using (Stream stream = new MemoryStream())
             {

YetAnotherPacketParser/YetAnotherPacketParser/Compiler/Json/JsonPacketNode.cs

@@ -14,21 +14,21 @@ public JsonPacketNode()
             this.Tossups = new List<JsonTossupNode>();
         }
 
-        public JsonPacketNode(PacketNode node, bool includeSanitizedFields) : this()
+        public JsonPacketNode(PacketNode node, bool modaqFormat) : this()
         {
             Verify.IsNotNull(node, nameof(node));
 
             foreach (TossupNode tossupNode in node.Tossups)
             {
-                this.Tossups.Add(new JsonTossupNode(tossupNode, includeSanitizedFields));
+                this.Tossups.Add(new JsonTossupNode(tossupNode, modaqFormat));
             }
 
             if (node.Bonuses != null)
             {
                 this.Bonuses = new List<JsonBonusNode>();
                 foreach (BonusNode bonusNode in node.Bonuses)
                 {
-                    this.Bonuses.Add(new JsonBonusNode(bonusNode, includeSanitizedFields));
+                    this.Bonuses.Add(new JsonBonusNode(bonusNode, modaqFormat));
                 }
             }
         }

YetAnotherPacketParser/YetAnotherPacketParser/Compiler/Json/JsonTossupNode.cs

@@ -4,25 +4,32 @@ namespace YetAnotherPacketParser.Compiler.Json
 {
     internal class JsonTossupNode
     {
-        public JsonTossupNode(TossupNode node, bool includeSanitizedFields)
+        public JsonTossupNode(TossupNode node, bool modaqFormat)
         {
-            this.Number = node.Number;
+            if (!modaqFormat)
+            {
+                this.Number = node.Number;
+            }
+
             this.Question = JsonTextFormatter.ToStringWithTags(node.Question.Question);
-            this.Question_sanitized = includeSanitizedFields ?
-                JsonTextFormatter.ToStringWithoutTags(node.Question.Question) :
-                null;
+            this.Question_sanitized = modaqFormat ?
+                null :
+                JsonTextFormatter.ToStringWithoutTags(node.Question.Question);
             this.Answer = JsonTextFormatter.ToStringWithTags(node.Question.Answer);
-            this.Answer_sanitized = includeSanitizedFields ?
-                JsonTextFormatter.ToStringWithoutTags(node.Question.Answer) :
-                null;
+            this.Answer_sanitized = modaqFormat ?
+                null :
+                JsonTextFormatter.ToStringWithoutTags(node.Question.Answer);
+            this.Metadata = node.Metadata;
         }
 
-        public int Number { get; }
+        public int? Number { get; }
 
         public string Question { get; }
 
         public string Answer { get; }
 
+        public string? Metadata { get; }
+
         // We name it _sanitized so the Json property name converter uses the right casing
         public string? Question_sanitized { get; }
 

YetAnotherPacketParser/YetAnotherPacketParser/Compiler/SanitizeHtmlTransformer.cs

@@ -55,11 +55,12 @@ private List<TossupNode> SanitizeTossups(IEnumerable<TossupNode> tossups)
         private TossupNode SanitizeTossup(TossupNode node)
         {
             QuestionNode sanitizedQuestion = this.SanitizeQuestion(node.Question);
-            string? sanitizedEditorNotes = node.EditorsNote == null ?
+            // We want to escape rather just Sanitize
+            string? sanitizedMetadata = node.Metadata == null ?
                 null :
-                this.Sanitizer.Sanitize(node.EditorsNote);
+                this.Sanitizer.Sanitize(node.Metadata.Replace("<", "&lt;").Replace(">", "&gt;"));
 
-            return new TossupNode(node.Number, sanitizedQuestion, sanitizedEditorNotes);
+            return new TossupNode(node.Number, sanitizedQuestion, sanitizedMetadata);
         }
 
         private List<BonusNode> SanitizeBonuses(IEnumerable<BonusNode> bonuses)
@@ -81,11 +82,11 @@ private BonusNode SanitizeBonus(BonusNode node)
             {
                 sanitizedBonusParts.Add(this.SanitizeBonusPart(bonusPart));
             }
-            string? sanitizedEditorNotes = node.EditorsNote != null ?
-                this.Sanitizer.Sanitize(node.EditorsNote) :
+            string? sanitizedMetadata = node.Metadata != null ?
+                this.Sanitizer.Sanitize(node.Metadata.Replace("<", "&lt;").Replace(">", "&gt;")) :
                 null;
 
-            return new BonusNode(node.Number, sanitizedLeadin, sanitizedBonusParts, sanitizedEditorNotes);
+            return new BonusNode(node.Number, sanitizedLeadin, sanitizedBonusParts, sanitizedMetadata);
         }
 
         private BonusPartNode SanitizeBonusPart(BonusPartNode node)

YetAnotherPacketParser/YetAnotherPacketParser/IResult.cs

@@ -9,7 +9,7 @@ public interface IResult<T>
         // If Success is true, ErrorMessages should throw
         IEnumerable<string> ErrorMessages { get; }
 
-        // If Sucess if false, ErrorMessage should throw
+        // If Sucess if false, Value should throw
         T Value { get; }
     }
 }

YetAnotherPacketParser/YetAnotherPacketParser/Lexer/DocxLexer.cs

@@ -251,6 +251,10 @@ private static List<ILine> GetLinesFromTextBlockLines(IEnumerable<TextBlockLine>
                     {
                         line = new BonusPartLine(formattedText.Substring(matchValue.Length), partValue.Value, difficultyModifier);
                     }
+                    else if (LexerClassifier.TextStartsWithPostQuestionMetadata(unformattedText))
+                    {
+                        line = new PostQuestionMetadataLine(formattedText);
+                    }
                     else
                     {
                         line = new Line(formattedText);

YetAnotherPacketParser/YetAnotherPacketParser/Lexer/LexerClassifier.cs

@@ -13,6 +13,8 @@ internal static class LexerClassifier
             "^\\s*(\\d+|tb|tie(breaker)?)\\s*\\.\\s*", RegexOptions.IgnoreCase | RegexOptions.Compiled);
         private static readonly Regex BonusPartValueRegex = new Regex(
             "^\\s*\\[\\s*(\\d)+\\s*[ehm]?\\s*\\]\\s*", RegexOptions.Compiled);
+        private static readonly Regex PostQuestionMetadataRegex = new Regex(
+            "^\\s*<(\\w|\\d|\\s|-|:|,)+(,(\\w|\\d|\\s|-|:|,)+)?>\\s*", RegexOptions.Compiled);
 
         public static bool TextStartsWithQuestionDigit(string text, out string matchValue, out int? number)
         {
@@ -83,5 +85,10 @@ public static bool TextStartsWithAnswer(string text, out string matchValue)
             partValue = value;
             return true;
         }
+
+        public static bool TextStartsWithPostQuestionMetadata(string text)
+        {
+            return PostQuestionMetadataRegex.Match(text).Success;
+        }
     }
 }

YetAnotherPacketParser/YetAnotherPacketParser/Lexer/LineType.cs

@@ -5,6 +5,7 @@ public enum LineType
         Unclassified,
         Answer,
         NumberedQuestion,
+        PostQuestionMetadata,
         BonusPart
     }
 }

YetAnotherPacketParser/YetAnotherPacketParser/Lexer/PostQuestionMetadataLine.cs

@@ -0,0 +1,16 @@
+namespace YetAnotherPacketParser.Lexer
+{
+    internal class PostQuestionMetadataLine : ILine
+    {
+        public PostQuestionMetadataLine(FormattedText text)
+        {
+            // TODO: Should we split it by ,? Should we take in two Formatted Texts, one for the first item, another for
+            // the second, or just take in an array of items?
+            this.Text = text;
+        }
+
+        public LineType Type => LineType.PostQuestionMetadata;
+
+        public FormattedText Text { get; }
+    }
+}

YetAnotherPacketParser/YetAnotherPacketParser/Parser/LinesParser.cs

@@ -81,6 +81,11 @@ public IResult<PacketNode> Parse(IEnumerable<ILine> lines)
             FormattedText formattedText = lines.Current.Text;
             IEnumerable<FormattedTextSegment> formattedTextSegments = formattedText.Segments;
 
+            if (lines.Current.Type == nextExpectedLineType)
+            {
+                return new SuccessResult<FormattedText>(formattedText);
+            }
+
             // The question number + question stays the same. We should do this in a loop (move next, is answer line, etc.)
             int linesChecked = 0;
             bool foundNextToken = false;
@@ -151,6 +156,46 @@ private static string GetFailureMessage(LinesEnumerator lines, string message)
             return Strings.ParseFailureMessage(message, lines.LineNumber, snippet.ToString());
         }
 
+        private static bool TryGetPostQuestionMetadata(LinesEnumerator lines, out PostQuestionMetadataLine? line)
+        {
+            line = null;
+            try
+            {
+                if (lines.Current == null)
+                {
+                    return false;
+                }
+            }
+            catch (InvalidOperationException)
+            {
+                return false;
+            }
+
+            do
+            {
+                switch (lines.Current.Type)
+                {
+                    // Post-question metadata normally follows an answer, though there could be some unclassified lines
+                    // in-between if there are some extra newlines between them.
+                    case LineType.Answer:
+                    case LineType.Unclassified:
+                        continue;
+                    case LineType.PostQuestionMetadata:
+                        if (lines.Current is PostQuestionMetadataLine metadataLine)
+                        {
+                            line = metadataLine;
+                            return true;
+                        }
+
+                        return false;
+                    default:
+                        return false;
+                }
+            } while (lines.MoveNext());
+
+            return false;
+        }
+
         private static bool TryGetNextQuestionLine(LinesEnumerator lines, out NumberedQuestionLine? line)
         {
             // Skip lines until we get to the next question
@@ -291,9 +336,13 @@ private static IResult<TossupNode> ParseTossup(LinesEnumerator lines, int tossup
                 return new FailureResult<TossupNode>(questionResult.ErrorMessages);
             }
 
-            // TODO: Support editor's notes. Would require changing the lexer, or having some look-behind so we can
-            // see if the previous line starts with an editor's note tag
-            return new SuccessResult<TossupNode>(new TossupNode(questionNumber, questionResult.Value));
+            string? metadata = null;
+            if (TryGetPostQuestionMetadata(lines, out PostQuestionMetadataLine? metadataLine) && metadataLine != null)
+            {
+                metadata = metadataLine.Text.UnformattedText;
+            }
+
+            return new SuccessResult<TossupNode>(new TossupNode(questionNumber, questionResult.Value, metadata));
         }
 
         private static IResult<BonusNode> ParseBonus(LinesEnumerator lines, int bonusNumber)
@@ -317,8 +366,15 @@ private static IResult<BonusNode> ParseBonus(LinesEnumerator lines, int bonusNum
                 return new FailureResult<BonusNode>(bonusPartsResult.ErrorMessages);
             }
 
+            // Metadata is always at the end of all of the bonus parts
+            string? metadata = null;
+            if (TryGetPostQuestionMetadata(lines, out PostQuestionMetadataLine? metadataLine) && metadataLine != null)
+            {
+                metadata = metadataLine.Text.UnformattedText;
+            }
+
             return new SuccessResult<BonusNode>(new BonusNode(
-                questionNumber, leadinResult.Value, bonusPartsResult.Value, null));
+                questionNumber, leadinResult.Value, bonusPartsResult.Value, metadata));
         }
 
         private static IResult<List<BonusPartNode>> ParseBonusParts(LinesEnumerator lines)
@@ -391,7 +447,7 @@ private static IResult<QuestionNode> ParseQuestion(LinesEnumerator lines, string
             FormattedText answer = lines.Current.Text;
 
             // TODO: Support editor's notes. Would require changing the lexer, or having some look-behind so we can
-            // see if the previous line starts with an editor's note tag
+            // see if the previous line starts with an editor's note tag.
             return new SuccessResult<QuestionNode>(new QuestionNode(questionResult.Value, answer));
         }
 

YetAnotherPacketParser/YetAnotherPacketParser/YetAnotherPacketParser.csproj

@@ -17,9 +17,9 @@
     <PackageTags>quizbowl packetparser quizbowlpacketparser</PackageTags>
     <PackageProjectUrl>https://github.com/alopezlago/YetAnotherPacketParser</PackageProjectUrl>
     <PackageLicenseFile>LICENSE.txt</PackageLicenseFile>
-    <AssemblyVersion>0.3.0.0</AssemblyVersion>
-    <FileVersion>0.3.0.0</FileVersion>
-    <Version>0.3.0.0</Version>
+    <AssemblyVersion>0.4.0.0</AssemblyVersion>
+    <FileVersion>0.4.0.0</FileVersion>
+    <Version>0.4.0.0</Version>
   </PropertyGroup>
 
   <ItemGroup>

YetAnotherPacketParser/YetAnotherPacketParserAzureFunction/YetAnotherPacketParserAzureFunction.csproj

@@ -7,9 +7,9 @@
     <AssemblyName>YetAnotherPacketParserAzureFunction</AssemblyName>
     <Authors>Alejandro Lopez-Lago</Authors>
     <Product>YAPP Azure Function</Product>
-    <AssemblyVersion>0.2.1.0</AssemblyVersion>
-    <FileVersion>0.2.1.0</FileVersion>
-    <Version>0.2.1.0</Version>
+    <AssemblyVersion>0.4.0.0</AssemblyVersion>
+    <FileVersion>0.4.0.0</FileVersion>
+    <Version>0.4.0.0</Version>
   </PropertyGroup>
   <ItemGroup>
     <PackageReference Include="Microsoft.ApplicationInsights.WorkerService" Version="2.18.0" />