diff --git a/YetAnotherPacketParser/YetAnotherPacketParser/Ast/BonusNode.cs b/YetAnotherPacketParser/YetAnotherPacketParser/Ast/BonusNode.cs index 3b16409..9e8b5c2 100644 --- a/YetAnotherPacketParser/YetAnotherPacketParser/Ast/BonusNode.cs +++ b/YetAnotherPacketParser/YetAnotherPacketParser/Ast/BonusNode.cs @@ -5,20 +5,21 @@ namespace YetAnotherPacketParser.Ast { public class BonusNode { - public BonusNode(int number, FormattedText leadin, IEnumerable parts, string? editorsNote) + public BonusNode( + int number, FormattedText leadin, IEnumerable parts, string? metadata) { - this.EditorsNote = editorsNote; this.Leadin = leadin ?? throw new ArgumentNullException(nameof(leadin)); this.Number = number; this.Parts = parts ?? throw new ArgumentNullException(nameof(parts)); + this.Metadata = metadata; } - public string? EditorsNote { get; } - public FormattedText Leadin { get; } public int Number { get; } public IEnumerable Parts { get; } + + public string? Metadata { get; } } } diff --git a/YetAnotherPacketParser/YetAnotherPacketParser/Ast/TossupNode.cs b/YetAnotherPacketParser/YetAnotherPacketParser/Ast/TossupNode.cs index 1439a07..e700624 100644 --- a/YetAnotherPacketParser/YetAnotherPacketParser/Ast/TossupNode.cs +++ b/YetAnotherPacketParser/YetAnotherPacketParser/Ast/TossupNode.cs @@ -4,17 +4,17 @@ namespace YetAnotherPacketParser.Ast { public class TossupNode { - public TossupNode(int number, QuestionNode question, string? editorsNote = null) + public TossupNode(int number, QuestionNode question, string? metadata = null) { - this.EditorsNote = editorsNote; this.Number = number; this.Question = question ?? throw new ArgumentNullException(nameof(question)); + this.Metadata = metadata; } - public string? EditorsNote { get; } - public int Number { get; } public QuestionNode Question { get; } + + public string? Metadata { get; } } } diff --git a/YetAnotherPacketParser/YetAnotherPacketParser/Compiler/Html/HtmlCompiler.cs b/YetAnotherPacketParser/YetAnotherPacketParser/Compiler/Html/HtmlCompiler.cs index 2001c31..f4757d9 100644 --- a/YetAnotherPacketParser/YetAnotherPacketParser/Compiler/Html/HtmlCompiler.cs +++ b/YetAnotherPacketParser/YetAnotherPacketParser/Compiler/Html/HtmlCompiler.cs @@ -48,6 +48,12 @@ private static void WriteTossup(TossupNode tossup, StringBuilder builder) builder.Append(tossup.Number); builder.Append(". "); WriteQuestion(tossup.Question, builder); + if (!string.IsNullOrEmpty(tossup.Metadata)) + { + builder.Append(tossup.Metadata); + builder.Append("
"); + } + builder.Append("

"); } @@ -62,6 +68,13 @@ private static void WriteBonus(BonusNode bonus, StringBuilder builder) { WriteBonusPart(bonusPart, builder); } + + if (!string.IsNullOrEmpty(bonus.Metadata)) + { + builder.Append(bonus.Metadata); + builder.Append("
"); + } + builder.Append("

"); } diff --git a/YetAnotherPacketParser/YetAnotherPacketParser/Compiler/Json/JsonBonusNode.cs b/YetAnotherPacketParser/YetAnotherPacketParser/Compiler/Json/JsonBonusNode.cs index f90302a..7b395c8 100644 --- a/YetAnotherPacketParser/YetAnotherPacketParser/Compiler/Json/JsonBonusNode.cs +++ b/YetAnotherPacketParser/YetAnotherPacketParser/Compiler/Json/JsonBonusNode.cs @@ -24,6 +24,7 @@ public JsonBonusNode(BonusNode bonusNode, bool includeSanitizedFields) null; this.Values = new List(); + this.Metadata = bonusNode.Metadata; this.DifficultyModifiers = partNodes.Any(node => node.DifficultyModifier.HasValue) ? new List() : null; foreach (BonusPartNode partNode in partNodes) @@ -42,6 +43,8 @@ public JsonBonusNode(BonusNode bonusNode, bool includeSanitizedFields) public string? Leadin_sanitized { get; } + public string? Metadata { get; } + public ICollection Answers { get; } public ICollection? Answers_sanitized { get; } diff --git a/YetAnotherPacketParser/YetAnotherPacketParser/Compiler/Json/JsonCompiler.cs b/YetAnotherPacketParser/YetAnotherPacketParser/Compiler/Json/JsonCompiler.cs index 3228a18..66f2403 100644 --- a/YetAnotherPacketParser/YetAnotherPacketParser/Compiler/Json/JsonCompiler.cs +++ b/YetAnotherPacketParser/YetAnotherPacketParser/Compiler/Json/JsonCompiler.cs @@ -25,7 +25,7 @@ public async Task CompileAsync(PacketNode packet) // The format that Jerry's parser uses for JSON (and that the reader expects as a result) is different // than the structure of the PacketNode, so transform it to a structure close to it (minus author and // packet fields) - JsonPacketNode sanitizedJsonPacket = new JsonPacketNode(sanitizedPacket, !this.Options.ModaqFormat); + JsonPacketNode sanitizedJsonPacket = new JsonPacketNode(sanitizedPacket, this.Options.ModaqFormat); using (Stream stream = new MemoryStream()) { diff --git a/YetAnotherPacketParser/YetAnotherPacketParser/Compiler/Json/JsonPacketNode.cs b/YetAnotherPacketParser/YetAnotherPacketParser/Compiler/Json/JsonPacketNode.cs index e3eeaf3..1d6d35e 100644 --- a/YetAnotherPacketParser/YetAnotherPacketParser/Compiler/Json/JsonPacketNode.cs +++ b/YetAnotherPacketParser/YetAnotherPacketParser/Compiler/Json/JsonPacketNode.cs @@ -14,13 +14,13 @@ public JsonPacketNode() this.Tossups = new List(); } - public JsonPacketNode(PacketNode node, bool includeSanitizedFields) : this() + public JsonPacketNode(PacketNode node, bool modaqFormat) : this() { Verify.IsNotNull(node, nameof(node)); foreach (TossupNode tossupNode in node.Tossups) { - this.Tossups.Add(new JsonTossupNode(tossupNode, includeSanitizedFields)); + this.Tossups.Add(new JsonTossupNode(tossupNode, modaqFormat)); } if (node.Bonuses != null) @@ -28,7 +28,7 @@ public JsonPacketNode(PacketNode node, bool includeSanitizedFields) : this() this.Bonuses = new List(); foreach (BonusNode bonusNode in node.Bonuses) { - this.Bonuses.Add(new JsonBonusNode(bonusNode, includeSanitizedFields)); + this.Bonuses.Add(new JsonBonusNode(bonusNode, modaqFormat)); } } } diff --git a/YetAnotherPacketParser/YetAnotherPacketParser/Compiler/Json/JsonTossupNode.cs b/YetAnotherPacketParser/YetAnotherPacketParser/Compiler/Json/JsonTossupNode.cs index 16ce7c6..9c9438e 100644 --- a/YetAnotherPacketParser/YetAnotherPacketParser/Compiler/Json/JsonTossupNode.cs +++ b/YetAnotherPacketParser/YetAnotherPacketParser/Compiler/Json/JsonTossupNode.cs @@ -4,25 +4,32 @@ namespace YetAnotherPacketParser.Compiler.Json { internal class JsonTossupNode { - public JsonTossupNode(TossupNode node, bool includeSanitizedFields) + public JsonTossupNode(TossupNode node, bool modaqFormat) { - this.Number = node.Number; + if (!modaqFormat) + { + this.Number = node.Number; + } + this.Question = JsonTextFormatter.ToStringWithTags(node.Question.Question); - this.Question_sanitized = includeSanitizedFields ? - JsonTextFormatter.ToStringWithoutTags(node.Question.Question) : - null; + this.Question_sanitized = modaqFormat ? + null : + JsonTextFormatter.ToStringWithoutTags(node.Question.Question); this.Answer = JsonTextFormatter.ToStringWithTags(node.Question.Answer); - this.Answer_sanitized = includeSanitizedFields ? - JsonTextFormatter.ToStringWithoutTags(node.Question.Answer) : - null; + this.Answer_sanitized = modaqFormat ? + null : + JsonTextFormatter.ToStringWithoutTags(node.Question.Answer); + this.Metadata = node.Metadata; } - public int Number { get; } + public int? Number { get; } public string Question { get; } public string Answer { get; } + public string? Metadata { get; } + // We name it _sanitized so the Json property name converter uses the right casing public string? Question_sanitized { get; } diff --git a/YetAnotherPacketParser/YetAnotherPacketParser/Compiler/SanitizeHtmlTransformer.cs b/YetAnotherPacketParser/YetAnotherPacketParser/Compiler/SanitizeHtmlTransformer.cs index dcf685d..6b033b4 100644 --- a/YetAnotherPacketParser/YetAnotherPacketParser/Compiler/SanitizeHtmlTransformer.cs +++ b/YetAnotherPacketParser/YetAnotherPacketParser/Compiler/SanitizeHtmlTransformer.cs @@ -55,11 +55,12 @@ private List SanitizeTossups(IEnumerable tossups) private TossupNode SanitizeTossup(TossupNode node) { QuestionNode sanitizedQuestion = this.SanitizeQuestion(node.Question); - string? sanitizedEditorNotes = node.EditorsNote == null ? + // We want to escape rather just Sanitize + string? sanitizedMetadata = node.Metadata == null ? null : - this.Sanitizer.Sanitize(node.EditorsNote); + this.Sanitizer.Sanitize(node.Metadata.Replace("<", "<").Replace(">", ">")); - return new TossupNode(node.Number, sanitizedQuestion, sanitizedEditorNotes); + return new TossupNode(node.Number, sanitizedQuestion, sanitizedMetadata); } private List SanitizeBonuses(IEnumerable bonuses) @@ -81,11 +82,11 @@ private BonusNode SanitizeBonus(BonusNode node) { sanitizedBonusParts.Add(this.SanitizeBonusPart(bonusPart)); } - string? sanitizedEditorNotes = node.EditorsNote != null ? - this.Sanitizer.Sanitize(node.EditorsNote) : + string? sanitizedMetadata = node.Metadata != null ? + this.Sanitizer.Sanitize(node.Metadata.Replace("<", "<").Replace(">", ">")) : null; - return new BonusNode(node.Number, sanitizedLeadin, sanitizedBonusParts, sanitizedEditorNotes); + return new BonusNode(node.Number, sanitizedLeadin, sanitizedBonusParts, sanitizedMetadata); } private BonusPartNode SanitizeBonusPart(BonusPartNode node) diff --git a/YetAnotherPacketParser/YetAnotherPacketParser/IResult.cs b/YetAnotherPacketParser/YetAnotherPacketParser/IResult.cs index 74db124..1868e86 100644 --- a/YetAnotherPacketParser/YetAnotherPacketParser/IResult.cs +++ b/YetAnotherPacketParser/YetAnotherPacketParser/IResult.cs @@ -9,7 +9,7 @@ public interface IResult // If Success is true, ErrorMessages should throw IEnumerable ErrorMessages { get; } - // If Sucess if false, ErrorMessage should throw + // If Sucess if false, Value should throw T Value { get; } } } \ No newline at end of file diff --git a/YetAnotherPacketParser/YetAnotherPacketParser/Lexer/DocxLexer.cs b/YetAnotherPacketParser/YetAnotherPacketParser/Lexer/DocxLexer.cs index 3831cc0..5b96cec 100644 --- a/YetAnotherPacketParser/YetAnotherPacketParser/Lexer/DocxLexer.cs +++ b/YetAnotherPacketParser/YetAnotherPacketParser/Lexer/DocxLexer.cs @@ -251,6 +251,10 @@ private static List GetLinesFromTextBlockLines(IEnumerable { line = new BonusPartLine(formattedText.Substring(matchValue.Length), partValue.Value, difficultyModifier); } + else if (LexerClassifier.TextStartsWithPostQuestionMetadata(unformattedText)) + { + line = new PostQuestionMetadataLine(formattedText); + } else { line = new Line(formattedText); diff --git a/YetAnotherPacketParser/YetAnotherPacketParser/Lexer/LexerClassifier.cs b/YetAnotherPacketParser/YetAnotherPacketParser/Lexer/LexerClassifier.cs index 7887c98..c515ca7 100644 --- a/YetAnotherPacketParser/YetAnotherPacketParser/Lexer/LexerClassifier.cs +++ b/YetAnotherPacketParser/YetAnotherPacketParser/Lexer/LexerClassifier.cs @@ -13,6 +13,8 @@ internal static class LexerClassifier "^\\s*(\\d+|tb|tie(breaker)?)\\s*\\.\\s*", RegexOptions.IgnoreCase | RegexOptions.Compiled); private static readonly Regex BonusPartValueRegex = new Regex( "^\\s*\\[\\s*(\\d)+\\s*[ehm]?\\s*\\]\\s*", RegexOptions.Compiled); + private static readonly Regex PostQuestionMetadataRegex = new Regex( + "^\\s*<(\\w|\\d|\\s|-|:|,)+(,(\\w|\\d|\\s|-|:|,)+)?>\\s*", RegexOptions.Compiled); public static bool TextStartsWithQuestionDigit(string text, out string matchValue, out int? number) { @@ -83,5 +85,10 @@ public static bool TextStartsWithAnswer(string text, out string matchValue) partValue = value; return true; } + + public static bool TextStartsWithPostQuestionMetadata(string text) + { + return PostQuestionMetadataRegex.Match(text).Success; + } } } diff --git a/YetAnotherPacketParser/YetAnotherPacketParser/Lexer/LineType.cs b/YetAnotherPacketParser/YetAnotherPacketParser/Lexer/LineType.cs index df0cb57..a3b0b46 100644 --- a/YetAnotherPacketParser/YetAnotherPacketParser/Lexer/LineType.cs +++ b/YetAnotherPacketParser/YetAnotherPacketParser/Lexer/LineType.cs @@ -5,6 +5,7 @@ public enum LineType Unclassified, Answer, NumberedQuestion, + PostQuestionMetadata, BonusPart } } diff --git a/YetAnotherPacketParser/YetAnotherPacketParser/Lexer/PostQuestionMetadataLine.cs b/YetAnotherPacketParser/YetAnotherPacketParser/Lexer/PostQuestionMetadataLine.cs new file mode 100644 index 0000000..6049128 --- /dev/null +++ b/YetAnotherPacketParser/YetAnotherPacketParser/Lexer/PostQuestionMetadataLine.cs @@ -0,0 +1,16 @@ +namespace YetAnotherPacketParser.Lexer +{ + internal class PostQuestionMetadataLine : ILine + { + public PostQuestionMetadataLine(FormattedText text) + { + // TODO: Should we split it by ,? Should we take in two Formatted Texts, one for the first item, another for + // the second, or just take in an array of items? + this.Text = text; + } + + public LineType Type => LineType.PostQuestionMetadata; + + public FormattedText Text { get; } + } +} diff --git a/YetAnotherPacketParser/YetAnotherPacketParser/Parser/LinesParser.cs b/YetAnotherPacketParser/YetAnotherPacketParser/Parser/LinesParser.cs index ab6a4ed..cd78508 100644 --- a/YetAnotherPacketParser/YetAnotherPacketParser/Parser/LinesParser.cs +++ b/YetAnotherPacketParser/YetAnotherPacketParser/Parser/LinesParser.cs @@ -81,6 +81,11 @@ public IResult Parse(IEnumerable lines) FormattedText formattedText = lines.Current.Text; IEnumerable formattedTextSegments = formattedText.Segments; + if (lines.Current.Type == nextExpectedLineType) + { + return new SuccessResult(formattedText); + } + // The question number + question stays the same. We should do this in a loop (move next, is answer line, etc.) int linesChecked = 0; bool foundNextToken = false; @@ -151,6 +156,46 @@ private static string GetFailureMessage(LinesEnumerator lines, string message) return Strings.ParseFailureMessage(message, lines.LineNumber, snippet.ToString()); } + private static bool TryGetPostQuestionMetadata(LinesEnumerator lines, out PostQuestionMetadataLine? line) + { + line = null; + try + { + if (lines.Current == null) + { + return false; + } + } + catch (InvalidOperationException) + { + return false; + } + + do + { + switch (lines.Current.Type) + { + // Post-question metadata normally follows an answer, though there could be some unclassified lines + // in-between if there are some extra newlines between them. + case LineType.Answer: + case LineType.Unclassified: + continue; + case LineType.PostQuestionMetadata: + if (lines.Current is PostQuestionMetadataLine metadataLine) + { + line = metadataLine; + return true; + } + + return false; + default: + return false; + } + } while (lines.MoveNext()); + + return false; + } + private static bool TryGetNextQuestionLine(LinesEnumerator lines, out NumberedQuestionLine? line) { // Skip lines until we get to the next question @@ -291,9 +336,13 @@ private static IResult ParseTossup(LinesEnumerator lines, int tossup return new FailureResult(questionResult.ErrorMessages); } - // TODO: Support editor's notes. Would require changing the lexer, or having some look-behind so we can - // see if the previous line starts with an editor's note tag - return new SuccessResult(new TossupNode(questionNumber, questionResult.Value)); + string? metadata = null; + if (TryGetPostQuestionMetadata(lines, out PostQuestionMetadataLine? metadataLine) && metadataLine != null) + { + metadata = metadataLine.Text.UnformattedText; + } + + return new SuccessResult(new TossupNode(questionNumber, questionResult.Value, metadata)); } private static IResult ParseBonus(LinesEnumerator lines, int bonusNumber) @@ -317,8 +366,15 @@ private static IResult ParseBonus(LinesEnumerator lines, int bonusNum return new FailureResult(bonusPartsResult.ErrorMessages); } + // Metadata is always at the end of all of the bonus parts + string? metadata = null; + if (TryGetPostQuestionMetadata(lines, out PostQuestionMetadataLine? metadataLine) && metadataLine != null) + { + metadata = metadataLine.Text.UnformattedText; + } + return new SuccessResult(new BonusNode( - questionNumber, leadinResult.Value, bonusPartsResult.Value, null)); + questionNumber, leadinResult.Value, bonusPartsResult.Value, metadata)); } private static IResult> ParseBonusParts(LinesEnumerator lines) @@ -391,7 +447,7 @@ private static IResult ParseQuestion(LinesEnumerator lines, string FormattedText answer = lines.Current.Text; // TODO: Support editor's notes. Would require changing the lexer, or having some look-behind so we can - // see if the previous line starts with an editor's note tag + // see if the previous line starts with an editor's note tag. return new SuccessResult(new QuestionNode(questionResult.Value, answer)); } diff --git a/YetAnotherPacketParser/YetAnotherPacketParser/YetAnotherPacketParser.csproj b/YetAnotherPacketParser/YetAnotherPacketParser/YetAnotherPacketParser.csproj index 6a187e9..8aa53de 100644 --- a/YetAnotherPacketParser/YetAnotherPacketParser/YetAnotherPacketParser.csproj +++ b/YetAnotherPacketParser/YetAnotherPacketParser/YetAnotherPacketParser.csproj @@ -17,9 +17,9 @@ quizbowl packetparser quizbowlpacketparser https://github.com/alopezlago/YetAnotherPacketParser LICENSE.txt - 0.3.0.0 - 0.3.0.0 - 0.3.0.0 + 0.4.0.0 + 0.4.0.0 + 0.4.0.0 diff --git a/YetAnotherPacketParser/YetAnotherPacketParserAzureFunction/YetAnotherPacketParserAzureFunction.csproj b/YetAnotherPacketParser/YetAnotherPacketParserAzureFunction/YetAnotherPacketParserAzureFunction.csproj index b171398..490a5af 100644 --- a/YetAnotherPacketParser/YetAnotherPacketParserAzureFunction/YetAnotherPacketParserAzureFunction.csproj +++ b/YetAnotherPacketParser/YetAnotherPacketParserAzureFunction/YetAnotherPacketParserAzureFunction.csproj @@ -7,9 +7,9 @@ YetAnotherPacketParserAzureFunction Alejandro Lopez-Lago YAPP Azure Function - 0.2.1.0 - 0.2.1.0 - 0.2.1.0 + 0.4.0.0 + 0.4.0.0 + 0.4.0.0 diff --git a/YetAnotherPacketParser/YetAnotherPacketParserAzureFunction/YetAnotherPacketParserParse.cs b/YetAnotherPacketParser/YetAnotherPacketParserAzureFunction/YetAnotherPacketParserParse.cs index af89213..306a9aa 100644 --- a/YetAnotherPacketParser/YetAnotherPacketParserAzureFunction/YetAnotherPacketParserParse.cs +++ b/YetAnotherPacketParser/YetAnotherPacketParserAzureFunction/YetAnotherPacketParserParse.cs @@ -3,6 +3,7 @@ using System.IO; using System.IO.Compression; using System.Linq; +using System.Text.Json; using System.Threading.Tasks; using System.Web.Http; using Microsoft.AspNetCore.Http; @@ -82,36 +83,43 @@ public static async Task Parse(HttpRequest request, ILogger log) return new OkObjectResult(compileResult.Result.Value); } - // We have a zip file. Return one. + if (TryGetStringValueFromQuery(request, "mergeMultiple", out string mergeMultipleValue) && + bool.TryParse(mergeMultipleValue, out bool mergeMultiple)) + { + log.LogInformation($"Merge multiple packets: {mergeMultiple}"); + } + else + { + mergeMultiple = false; + } + + // We have a zip file. Return one, or return a merged file if requested. // TODO: See if we can return a JSON result showing success, # successfully parsed packets, and the contents + // as a zip file and JSON array? bool outputFormatIsJson = options.OutputFormat == OutputFormat.Json; IEnumerable successResults = results.Where(result => result.Result.Success); using (MemoryStream memoryStream = new MemoryStream()) { - using (ZipArchive archive = new ZipArchive(memoryStream, ZipArchiveMode.Create)) + if (!mergeMultiple) { - foreach (ConvertResult compileResult in successResults) - { - string newFilename = outputFormatIsJson ? - compileResult.Filename.Replace(".docx", ".json") : - compileResult.Filename.Replace(".docx", ".html"); - ZipArchiveEntry entry = archive.CreateEntry(newFilename); - - // We can't do this asynchronously, because it complains about writing to the same ZipArchive stream - using (StreamWriter writer = new StreamWriter(entry.Open())) - { - writer.Write(compileResult.Result.Value); - } - } + WriteMultiplePacketsToZip(successResults, memoryStream, outputFormatIsJson); + } + else if (outputFormatIsJson) + { + WriteMultiplePacketsToJson(successResults, memoryStream); + } + else + { + WriteMultiplePacketsToHtml(successResults, memoryStream); + } - log.LogInformation($"Succesfully parsed {successResults.Count()} out of {results.Count()} packets"); - IEnumerable failedResults = results - .Where(result => !result.Result.Success) - .OrderBy(result => result.Filename); - foreach (ConvertResult compileResult in failedResults) - { - log.LogWarning($"{compileResult.Filename} failed to compile."); - } + log.LogInformation($"Succesfully parsed {successResults.Count()} out of {results.Count()} packets"); + IEnumerable failedResults = results + .Where(result => !result.Result.Success) + .OrderBy(result => result.Filename); + foreach (ConvertResult compileResult in failedResults) + { + log.LogWarning($"{compileResult.Filename} failed to compile."); } await memoryStream.FlushAsync(); @@ -250,5 +258,85 @@ private static bool TryGetStringValueFromQuery(HttpRequest request, string key, stringValue = null; return false; } + + // TODO: These were copied and modified from Program.cs. See if we can share the code. + private static void WriteMultiplePacketsToJson(IEnumerable packets, Stream stream) + { + IList jsonPackets = new List(); + foreach (ConvertResult compileResult in packets.OrderBy(packet => packet.Filename)) + { + jsonPackets.Add(new JsonPacket() + { + name = compileResult.Filename.Replace(".docx", string.Empty), + packet = JsonSerializer.Deserialize(compileResult.Result.Value) + }); + } + + string content = JsonSerializer.Serialize(jsonPackets); + + using (StreamWriter writer = new StreamWriter(stream)) + { + writer.Write(content); + } + } + + private static void WriteMultiplePacketsToHtml(IEnumerable packets, Stream stream) + { + IList htmlBodies = new List(); + foreach (ConvertResult compileResult in packets.OrderBy(packet => packet.Filename)) + { + string html = compileResult.Result.Value; + int bodyStartIndex = html.IndexOf("", StringComparison.OrdinalIgnoreCase); + int bodyEndIndex = html.LastIndexOf("", StringComparison.OrdinalIgnoreCase); + if (bodyStartIndex == -1 || bodyEndIndex == -1 || bodyStartIndex > bodyEndIndex) + { + // Skip, since the HTML was malformed + continue; + } + + // Skip past "" + bodyStartIndex += 6; + string htmlBody = $"

{compileResult.Filename.Replace(".docx", string.Empty)}

{html.Substring(bodyStartIndex, bodyEndIndex - bodyStartIndex)}"; + + htmlBodies.Add(htmlBody); + } + + string bundledHtml = $"{string.Join("
", htmlBodies)}"; + + using (StreamWriter writer = new StreamWriter(stream)) + { + writer.Write(bundledHtml); + } + } + + private static void WriteMultiplePacketsToZip( + IEnumerable packets, Stream stream, bool outputFormatIsJson) + { + using (ZipArchive archive = new ZipArchive(stream, ZipArchiveMode.Create)) + { + foreach (ConvertResult compileResult in packets) + { + string newFilename = outputFormatIsJson ? + compileResult.Filename.Replace(".docx", ".json") : + compileResult.Filename.Replace(".docx", ".html"); + ZipArchiveEntry entry = archive.CreateEntry(newFilename); + + // We can't do this asynchronously, because it complains about writing to the same ZipArchive stream + using (StreamWriter writer = new StreamWriter(entry.Open())) + { + writer.Write(compileResult.Result.Value); + } + } + } + } + + // TODO: See how to share this between the function and the command line + private class JsonPacket + { + // Lower-cased so that it appears lowercased in the JSON output + public string name { get; set; } + + public object packet { get; set; } + } } } diff --git a/YetAnotherPacketParser/YetAnotherPacketParserCommandLine/Options.cs b/YetAnotherPacketParser/YetAnotherPacketParserCommandLine/Options.cs index b44e4e0..827b348 100644 --- a/YetAnotherPacketParser/YetAnotherPacketParserCommandLine/Options.cs +++ b/YetAnotherPacketParser/YetAnotherPacketParserCommandLine/Options.cs @@ -32,5 +32,11 @@ public class CommandLineOptions [Option('v', "verbose", HelpText = "Verbose logging", Required = false, Default = false)] public bool Verbose { get; set; } + + [Option( + "mergeMultiple", + HelpText = "When parsing multiple packets from a zip file, return a JSON array or combined HTML file. If set to false, returns a zip file of individual packets. Defaults to false.", + Default = false)] + public bool MergeMultiplePackets { get; set; } } } diff --git a/YetAnotherPacketParser/YetAnotherPacketParserCommandLine/Program.cs b/YetAnotherPacketParser/YetAnotherPacketParserCommandLine/Program.cs index 4608d5b..d7146e3 100644 --- a/YetAnotherPacketParser/YetAnotherPacketParserCommandLine/Program.cs +++ b/YetAnotherPacketParser/YetAnotherPacketParserCommandLine/Program.cs @@ -3,6 +3,7 @@ using System.IO; using System.IO.Compression; using System.Linq; +using System.Text.Json; using System.Threading.Tasks; using CommandLine; using YetAnotherPacketParser; @@ -103,21 +104,18 @@ private static async Task RunAsync(CommandLineOptions options) File.Delete(options.Output); } - using (ZipArchive outputArchive = ZipFile.Open(options.Output, ZipArchiveMode.Create)) + // Choose between ZIP or combination + if (!options.MergeMultiplePackets) { - foreach (ConvertResult compileResult in successResults) - { - string newFilename = outputFormatIsJson ? - compileResult.Filename.Replace(".docx", ".json") : - compileResult.Filename.Replace(".docx", ".html"); - ZipArchiveEntry entry = outputArchive.CreateEntry(newFilename); - - // We can't do this asynchronously, because it complains about writing to the same ZipArchive stream - using (StreamWriter writer = new StreamWriter(entry.Open())) - { - writer.Write(compileResult.Result.Value); - } - } + WriteMultiplePacketsToZip(successResults, options, outputFormatIsJson); + } + else if (outputFormatIsJson) + { + WriteMultiplePacketsToJson(successResults, options); + } + else + { + WriteMultiplePacketsToHtml(successResults, options); } Console.WriteLine(); @@ -143,5 +141,86 @@ private static void Log(CommandLineOptions options, LogLevel logLevel, string me Console.WriteLine(message); } + + private static void WriteMultiplePacketsToZip( + IEnumerable packets, CommandLineOptions options, bool outputFormatIsJson) + { + using (ZipArchive outputArchive = ZipFile.Open(options.Output, ZipArchiveMode.Create)) + { + foreach (ConvertResult compileResult in packets) + { + string newFilename = outputFormatIsJson ? + compileResult.Filename.Replace(".docx", ".json") : + compileResult.Filename.Replace(".docx", ".html"); + ZipArchiveEntry entry = outputArchive.CreateEntry(newFilename); + + // We can't do this asynchronously, because it complains about writing to the same ZipArchive stream + using (StreamWriter writer = new StreamWriter(entry.Open())) + { + writer.Write(compileResult.Result.Value); + } + } + } + } + + private static void WriteMultiplePacketsToJson(IEnumerable packets, CommandLineOptions options) + { + IList jsonPackets = new List(); + foreach (ConvertResult compileResult in packets.OrderBy(packet => packet.Filename)) + { + jsonPackets.Add(new JsonPacket() + { + name = compileResult.Filename.Replace(".docx", string.Empty), + packet = JsonSerializer.Deserialize(compileResult.Result.Value) + }); + } + + string content = JsonSerializer.Serialize(jsonPackets); + + using (FileStream stream = new FileStream(options.Output, FileMode.OpenOrCreate, FileAccess.Write)) + using (StreamWriter writer = new StreamWriter(stream)) + { + writer.Write(content); + } + } + + private static void WriteMultiplePacketsToHtml(IEnumerable packets, CommandLineOptions options) + { + IList htmlBodies = new List(); + foreach (ConvertResult compileResult in packets.OrderBy(packet => packet.Filename)) + { + string html = compileResult.Result.Value; + int bodyStartIndex = html.IndexOf("", StringComparison.OrdinalIgnoreCase); + int bodyEndIndex = html.LastIndexOf("", StringComparison.OrdinalIgnoreCase); + if (bodyStartIndex == -1 || bodyEndIndex == -1 || bodyStartIndex > bodyEndIndex) + { + // Skip, since the HTML was malformed + continue; + } + + // Skip past "" + bodyStartIndex += 6; + string htmlBody = $"

{compileResult.Filename.Replace(".docx", string.Empty)}

{html.Substring(bodyStartIndex, bodyEndIndex - bodyStartIndex)}"; + + htmlBodies.Add(htmlBody); + } + + string bundledHtml = $"{string.Join("
", htmlBodies)}"; + + using (FileStream stream = new FileStream(options.Output, FileMode.OpenOrCreate, FileAccess.Write)) + using (StreamWriter writer = new StreamWriter(stream)) + { + writer.Write(bundledHtml); + } + } + + // TODO: See how to share this between the function and the command line + private class JsonPacket + { + // Lower-cased so that it appears lowercased in the JSON output + public string name { get; set; } + + public object packet { get; set; } + } } } diff --git a/YetAnotherPacketParser/YetAnotherPacketParserCommandLine/YetAnotherPacketParserCommandLine.csproj b/YetAnotherPacketParser/YetAnotherPacketParserCommandLine/YetAnotherPacketParserCommandLine.csproj index 55c22d8..82e6938 100644 --- a/YetAnotherPacketParser/YetAnotherPacketParserCommandLine/YetAnotherPacketParserCommandLine.csproj +++ b/YetAnotherPacketParser/YetAnotherPacketParserCommandLine/YetAnotherPacketParserCommandLine.csproj @@ -9,9 +9,9 @@ (c) 2020 Alejandro Lopez-Lago Yet Another Packet Parser parses quiz bowl packets and translates them to different formats YAPP - 0.3.0.0 - 0.3.0.0 - 0.3.0.0 + 0.4.0.0 + 0.4.0.0 + 0.4.0.0 diff --git a/YetAnotherPacketParser/YetAnotherPacketParserTests/JsonCompilerTests.cs b/YetAnotherPacketParser/YetAnotherPacketParserTests/JsonCompilerTests.cs index 5daf13b..7a4a4ca 100644 --- a/YetAnotherPacketParser/YetAnotherPacketParserTests/JsonCompilerTests.cs +++ b/YetAnotherPacketParser/YetAnotherPacketParserTests/JsonCompilerTests.cs @@ -20,7 +20,9 @@ public async Task CompileOneTossupPacket() new TossupNode[] { new TossupNode( - 1, new QuestionNode(CreateFormattedText(questionText), CreateFormattedText(answerText))) + 1, + new QuestionNode(CreateFormattedText(questionText), CreateFormattedText(answerText)), + "") }, Array.Empty()); @@ -34,7 +36,9 @@ public async Task CompileOneTossupPacket() Assert.IsFalse(string.IsNullOrEmpty(result), $"Compiled packet is empty. Packet: {result}"); Assert.IsTrue(result.Contains(questionText), $"Couldn't find question text in packet. Packet: {result}"); Assert.IsTrue(result.Contains(answerText), $"Couldn't find answer in packet. Packet: {result}"); - Assert.IsTrue(result.Contains("_sanitized"), "There should be sanitized fields"); + Assert.IsTrue(result.Contains("_sanitized"), "There should be sanitized fields. Packet: {result}"); + Assert.IsTrue(result.Contains("number"), "There should be a number field. Packet: {result}"); + Assert.IsTrue(result.Contains("metadata"), "There should be a metadata field. Packet: {result}"); // TODO: Verify the results as a JsonPacketNode by either using Json.Net or creating a JsonConverter } @@ -49,7 +53,9 @@ public async Task CompileOneTossupPacketWithModaqFormat() new TossupNode[] { new TossupNode( - 1, new QuestionNode(CreateFormattedText(questionText), CreateFormattedText(answerText))) + 1, + new QuestionNode(CreateFormattedText(questionText), CreateFormattedText(answerText)), + "") }, Array.Empty()); @@ -64,7 +70,9 @@ public async Task CompileOneTossupPacketWithModaqFormat() Assert.IsFalse(string.IsNullOrEmpty(result), $"Compiled packet is empty. Packet: {result}"); Assert.IsTrue(result.Contains(questionText), $"Couldn't find question text in packet. Packet: {result}"); Assert.IsTrue(result.Contains(answerText), $"Couldn't find answer in packet. Packet: {result}"); - Assert.IsFalse(result.Contains("_sanitized"), "There should be no sanitized fields"); + Assert.IsFalse(result.Contains("_sanitized"), "There should be no sanitized fields. Packet: {result}"); + Assert.IsFalse(result.Contains("number"), "There should be no number field. Packet: {result}"); + Assert.IsTrue(result.Contains("metadata"), "There should be a metadata field. Packet: {result}"); // TODO: Verify the results as a JsonPacketNode by either using Json.Net or creating a JsonConverter } diff --git a/YetAnotherPacketParser/YetAnotherPacketParserTests/LineParserTests.cs b/YetAnotherPacketParser/YetAnotherPacketParserTests/LineParserTests.cs index 911ef15..48a31fc 100644 --- a/YetAnotherPacketParser/YetAnotherPacketParserTests/LineParserTests.cs +++ b/YetAnotherPacketParser/YetAnotherPacketParserTests/LineParserTests.cs @@ -47,6 +47,37 @@ public void OneTossupPacketParses() Assert.AreEqual(number, tossup.Number, "Unexpected tossup number"); Assert.AreEqual(questionText, tossup.Question.Question.UnformattedText, "Unexpected question"); Assert.AreEqual(answer, tossup.Question.Answer.UnformattedText, "Unexpected answer"); + Assert.IsNull(tossup.Metadata, "There should be no metadata"); + } + + [TestMethod] + public void OneTossupWithPostQuestionMetadataPacketParses() + { + const int number = 1; + const string questionText = "This is my tossup"; + const string answer = "An answer"; + const string metadata = "Some metadata"; + + ILine[] lines = new ILine[] + { + CreateQuestionLine(number, questionText), + CreateAnswerLine(answer), + CreatePostQuestionMetadaLine(metadata) + }; + + LinesParser parser = new LinesParser(); + IResult packetResult = parser.Parse(lines); + Assert.IsTrue(packetResult.Success); + + PacketNode packet = packetResult.Value; + Assert.AreEqual(1, packet.Tossups.Count(), "Unexpected number of tossups"); + Assert.IsNull(packet.Bonuses?.Count(), "Bonuses should be null"); + + TossupNode tossup = packet.Tossups.First(); + Assert.AreEqual(number, tossup.Number, "Unexpected tossup number"); + Assert.AreEqual(questionText, tossup.Question.Question.UnformattedText, "Unexpected question"); + Assert.AreEqual(answer, tossup.Question.Answer.UnformattedText, "Unexpected answer"); + Assert.AreEqual(metadata, tossup.Metadata, "Unexpected metatadata"); } [TestMethod] @@ -134,6 +165,45 @@ public void TwoTossupsPacketParses() Assert.AreEqual(index + 1, tossup.Number, "Unexpected tossup number"); Assert.AreEqual(questions[index], tossup.Question.Question.UnformattedText, "Unexpected question"); Assert.AreEqual(answers[index], tossup.Question.Answer.UnformattedText, "Unexpected answer"); + Assert.IsNull(tossup.Metadata, "Metadata should be null"); + index++; + } + } + + [TestMethod] + public void TwoTossupsWithMetdataPacketParses() + { + string[] questions = new string[] { "This is my tossup", "Another tossup" }; + string[] answers = new string[] { "An answer", "Answer #2" }; + string[] metadata = new string[] { "", "" }; + + ILine[] lines = new ILine[] + { + CreateQuestionLine(1, questions[0]), + CreateAnswerLine(answers[0]), + CreatePostQuestionMetadaLine(metadata[0]), + new Line(CreateFormattedText(string.Empty)), + CreateQuestionLine(2, questions[1]), + CreateAnswerLine(answers[1]), + new Line(CreateFormattedText(string.Empty)), + CreatePostQuestionMetadaLine(metadata[1]) + }; + + LinesParser parser = new LinesParser(); + IResult packetResult = parser.Parse(lines); + Assert.IsTrue(packetResult.Success); + + PacketNode packet = packetResult.Value; + Assert.AreEqual(2, packet.Tossups.Count(), "Unexpected number of tossups"); + Assert.IsNull(packet.Bonuses?.Count(), "Bonuses should be null"); + + int index = 0; + foreach (TossupNode tossup in packet.Tossups) + { + Assert.AreEqual(index + 1, tossup.Number, "Unexpected tossup number"); + Assert.AreEqual(questions[index], tossup.Question.Question.UnformattedText, "Unexpected question"); + Assert.AreEqual(answers[index], tossup.Question.Answer.UnformattedText, "Unexpected answer"); + Assert.AreEqual(metadata[index], tossup.Metadata, "Unexpected metadata"); index++; } } @@ -296,6 +366,31 @@ public void BonusWithNoBonusPartsFails() Assert.AreEqual(1, packetResult.ErrorMessages.Count()); } + [TestMethod] + public void BonusWithMeatdataSucceeds() + { + const string metadata = "My metadata"; + ILine[] lines = new ILine[] + { + CreateQuestionLine(1, "Tossup"), + CreateAnswerLine("Answer"), + CreateQuestionLine(1, "Bonus leadin"), + CreatePartLine("Bonus part that is", 10), + CreateAnswerLine("The answer"), + CreatePostQuestionMetadaLine(metadata) + }; + + LinesParser parser = new LinesParser(); + IResult packetResult = parser.Parse(lines); + Assert.IsTrue(packetResult.Success); + Assert.IsNotNull(packetResult.Value.Bonuses); + Assert.AreEqual(1, packetResult.Value.Bonuses?.Count(), "Unexpected number of bonuses"); + + BonusNode bonus = packetResult.Value.Bonuses.First(); + Assert.AreEqual(1, bonus.Parts.Count(), "Unexpected number of parts"); + Assert.AreEqual(metadata, bonus.Metadata, "Unexpected metadata"); + } + [TestMethod] public void BonusPartWithDifficultyModifierSucceeds() { @@ -449,5 +544,10 @@ private static NumberedQuestionLine CreateQuestionLine(int number, string text) { return new NumberedQuestionLine(CreateFormattedText(text), number); } + + private static PostQuestionMetadataLine CreatePostQuestionMetadaLine(string metadata) + { + return new PostQuestionMetadataLine(CreateFormattedText(metadata)); + } } }