From 4a983aa3b59b540229c0c346944e9ac47b214760 Mon Sep 17 00:00:00 2001 From: Alejandro Lopez-Lago Date: Tue, 30 Nov 2021 05:59:00 +0000 Subject: [PATCH] Merged PR 226: v0.4.2 - .Net 6 support, minor perf fixes - Support .Net 6 as a build target - Minor performance and static analysis fixes - Bump version to 0.4.2 --- .../YetAnotherPacketParser.sln | 16 +- .../Compiler/Html/HtmlCompiler.cs | 8 +- .../Json/PascalCaseJsonNamingPolicy.cs | 7 +- .../Compiler/SanitizeHtmlTransformer.cs | 8 +- .../FormattedTextSegment.cs | 2 +- .../YetAnotherPacketParser/Lexer/DocxLexer.cs | 18 +- .../YetAnotherPacketParser/Lexer/HtmlLexer.cs | 22 +- .../MagicWordDetector.cs | 46 +-- .../YetAnotherPacketParser/PacketConverter.cs | 4 +- .../Parser/LinesParser.cs | 4 +- .../YetAnotherPacketParser.csproj | 17 +- .../.config/dotnet-tools.json | 12 + .../ErrorMessageResponse.cs | 12 + .../JsonPacketItem.cs | 19 ++ .../ParseProcessor.cs | 303 ++++++++++++++++++ .../YetAnotherPacketParserAPI/Program.cs | 63 ++++ .../Properties/launchSettings.json | 31 ++ .../YetAnotherPacketParserAPI.csproj | 18 ++ .../appsettings.Development.json | 8 + .../appsettings.json | 34 ++ ...YetAnotherPacketParserAzureFunction.csproj | 12 +- .../Program.cs | 28 +- .../YetAnotherPacketParserCommandLine.csproj | 11 +- .../YetAnotherPacketParserTests.csproj | 6 +- 24 files changed, 622 insertions(+), 87 deletions(-) create mode 100644 YetAnotherPacketParser/YetAnotherPacketParserAPI/.config/dotnet-tools.json create mode 100644 YetAnotherPacketParser/YetAnotherPacketParserAPI/ErrorMessageResponse.cs create mode 100644 YetAnotherPacketParser/YetAnotherPacketParserAPI/JsonPacketItem.cs create mode 100644 YetAnotherPacketParser/YetAnotherPacketParserAPI/ParseProcessor.cs create mode 100644 YetAnotherPacketParser/YetAnotherPacketParserAPI/Program.cs create mode 100644 YetAnotherPacketParser/YetAnotherPacketParserAPI/Properties/launchSettings.json create mode 100644 YetAnotherPacketParser/YetAnotherPacketParserAPI/YetAnotherPacketParserAPI.csproj create mode 100644 YetAnotherPacketParser/YetAnotherPacketParserAPI/appsettings.Development.json create mode 100644 YetAnotherPacketParser/YetAnotherPacketParserAPI/appsettings.json diff --git a/YetAnotherPacketParser/YetAnotherPacketParser.sln b/YetAnotherPacketParser/YetAnotherPacketParser.sln index 2f7ff63..b7bd22a 100644 --- a/YetAnotherPacketParser/YetAnotherPacketParser.sln +++ b/YetAnotherPacketParser/YetAnotherPacketParser.sln @@ -1,15 +1,17 @@  Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Version 16 -VisualStudioVersion = 16.0.30320.27 +# Visual Studio Version 17 +VisualStudioVersion = 17.0.31912.275 MinimumVisualStudioVersion = 10.0.40219.1 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "YetAnotherPacketParser", "YetAnotherPacketParser\YetAnotherPacketParser.csproj", "{99472784-A154-49F0-9E85-10F4A5300DE0}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "YetAnotherPacketParserCommandLine", "YetAnotherPacketParserCommandLine\YetAnotherPacketParserCommandLine.csproj", "{EC1C3290-BFAE-4E31-BF4E-A2A2D736DC6B}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "YetAnotherPacketParserCommandLine", "YetAnotherPacketParserCommandLine\YetAnotherPacketParserCommandLine.csproj", "{EC1C3290-BFAE-4E31-BF4E-A2A2D736DC6B}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "YetAnotherPacketParserTests", "YetAnotherPacketParserTests\YetAnotherPacketParserTests.csproj", "{811B7F2E-1E9C-486C-9B26-62B99C239CE0}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "YetAnotherPacketParserTests", "YetAnotherPacketParserTests\YetAnotherPacketParserTests.csproj", "{811B7F2E-1E9C-486C-9B26-62B99C239CE0}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "YetAnotherPacketParserAzureFunction", "YetAnotherPacketParserAzureFunction\YetAnotherPacketParserAzureFunction.csproj", "{B594E98C-9772-40F8-93BA-F641AE397140}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "YetAnotherPacketParserAzureFunction", "YetAnotherPacketParserAzureFunction\YetAnotherPacketParserAzureFunction.csproj", "{B594E98C-9772-40F8-93BA-F641AE397140}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "YetAnotherPacketParserAPI", "YetAnotherPacketParserAPI\YetAnotherPacketParserAPI.csproj", "{90AD57C5-763C-494E-AE92-F90BB0BB5A88}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -33,6 +35,10 @@ Global {B594E98C-9772-40F8-93BA-F641AE397140}.Debug|Any CPU.Build.0 = Debug|Any CPU {B594E98C-9772-40F8-93BA-F641AE397140}.Release|Any CPU.ActiveCfg = Release|Any CPU {B594E98C-9772-40F8-93BA-F641AE397140}.Release|Any CPU.Build.0 = Release|Any CPU + {90AD57C5-763C-494E-AE92-F90BB0BB5A88}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {90AD57C5-763C-494E-AE92-F90BB0BB5A88}.Debug|Any CPU.Build.0 = Debug|Any CPU + {90AD57C5-763C-494E-AE92-F90BB0BB5A88}.Release|Any CPU.ActiveCfg = Release|Any CPU + {90AD57C5-763C-494E-AE92-F90BB0BB5A88}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/YetAnotherPacketParser/YetAnotherPacketParser/Compiler/Html/HtmlCompiler.cs b/YetAnotherPacketParser/YetAnotherPacketParser/Compiler/Html/HtmlCompiler.cs index 3713232..e42d19a 100644 --- a/YetAnotherPacketParser/YetAnotherPacketParser/Compiler/Html/HtmlCompiler.cs +++ b/YetAnotherPacketParser/YetAnotherPacketParser/Compiler/Html/HtmlCompiler.cs @@ -50,7 +50,9 @@ private static void WriteTossup(TossupNode tossup, StringBuilder builder) WriteQuestion(tossup.Question, builder); if (!string.IsNullOrEmpty(tossup.Metadata)) { - builder.Append($"<{tossup.Metadata}>
"); + builder.Append("<"); + builder.Append(tossup.Metadata); + builder.Append(">
"); } builder.Append("

"); @@ -70,7 +72,9 @@ private static void WriteBonus(BonusNode bonus, StringBuilder builder) if (!string.IsNullOrEmpty(bonus.Metadata)) { - builder.Append($"<{bonus.Metadata}>
"); + builder.Append("<"); + builder.Append(bonus.Metadata); + builder.Append($">
"); } builder.Append("

"); diff --git a/YetAnotherPacketParser/YetAnotherPacketParser/Compiler/Json/PascalCaseJsonNamingPolicy.cs b/YetAnotherPacketParser/YetAnotherPacketParser/Compiler/Json/PascalCaseJsonNamingPolicy.cs index e90b8cf..d09c4e2 100644 --- a/YetAnotherPacketParser/YetAnotherPacketParser/Compiler/Json/PascalCaseJsonNamingPolicy.cs +++ b/YetAnotherPacketParser/YetAnotherPacketParser/Compiler/Json/PascalCaseJsonNamingPolicy.cs @@ -1,4 +1,5 @@ -using System.Diagnostics.CodeAnalysis; +using System; +using System.Diagnostics.CodeAnalysis; using System.Text.Json; namespace YetAnotherPacketParser.Compiler.Json @@ -14,7 +15,9 @@ public override string ConvertName(string name) Verify.IsNotNull(name, nameof(name)); // Names will not be null or empty - return name.Substring(0, 1).ToLowerInvariant() + name.Substring(1); + Span firstLetter = new Span(new char[1]); + name.AsSpan(0, 1).ToLowerInvariant(firstLetter); + return string.Concat(firstLetter, name.AsSpan(1)); } } } diff --git a/YetAnotherPacketParser/YetAnotherPacketParser/Compiler/SanitizeHtmlTransformer.cs b/YetAnotherPacketParser/YetAnotherPacketParser/Compiler/SanitizeHtmlTransformer.cs index 6b033b4..91bdca1 100644 --- a/YetAnotherPacketParser/YetAnotherPacketParser/Compiler/SanitizeHtmlTransformer.cs +++ b/YetAnotherPacketParser/YetAnotherPacketParser/Compiler/SanitizeHtmlTransformer.cs @@ -58,7 +58,9 @@ private TossupNode SanitizeTossup(TossupNode node) // We want to escape rather just Sanitize string? sanitizedMetadata = node.Metadata == null ? null : - this.Sanitizer.Sanitize(node.Metadata.Replace("<", "<").Replace(">", ">")); + this.Sanitizer.Sanitize(node.Metadata + .Replace("<", "<", StringComparison.Ordinal) + .Replace(">", ">", StringComparison.Ordinal)); return new TossupNode(node.Number, sanitizedQuestion, sanitizedMetadata); } @@ -83,7 +85,9 @@ private BonusNode SanitizeBonus(BonusNode node) sanitizedBonusParts.Add(this.SanitizeBonusPart(bonusPart)); } string? sanitizedMetadata = node.Metadata != null ? - this.Sanitizer.Sanitize(node.Metadata.Replace("<", "<").Replace(">", ">")) : + this.Sanitizer.Sanitize(node.Metadata + .Replace("<", "<", StringComparison.Ordinal) + .Replace(">", ">", StringComparison.Ordinal)) : null; return new BonusNode(node.Number, sanitizedLeadin, sanitizedBonusParts, sanitizedMetadata); diff --git a/YetAnotherPacketParser/YetAnotherPacketParser/FormattedTextSegment.cs b/YetAnotherPacketParser/YetAnotherPacketParser/FormattedTextSegment.cs index af0080d..64b0c0f 100644 --- a/YetAnotherPacketParser/YetAnotherPacketParser/FormattedTextSegment.cs +++ b/YetAnotherPacketParser/YetAnotherPacketParser/FormattedTextSegment.cs @@ -44,7 +44,7 @@ public override bool Equals(object? obj) public override int GetHashCode() { - return (this.Text?.GetHashCode() ?? 0) ^ + return (this.Text?.GetHashCode(StringComparison.Ordinal) ?? 0) ^ this.Bolded.GetHashCode() ^ (this.Italic.GetHashCode() << 1) ^ (this.Underlined.GetHashCode() << 2); diff --git a/YetAnotherPacketParser/YetAnotherPacketParser/Lexer/DocxLexer.cs b/YetAnotherPacketParser/YetAnotherPacketParser/Lexer/DocxLexer.cs index 5b96cec..8caa3bb 100644 --- a/YetAnotherPacketParser/YetAnotherPacketParser/Lexer/DocxLexer.cs +++ b/YetAnotherPacketParser/YetAnotherPacketParser/Lexer/DocxLexer.cs @@ -25,7 +25,7 @@ public class DocxLexer : ILexer /// Stream whose contents are a .docx Microsoft Word file /// If we were unable to open the stream, then the result is a FailureResult. Otherwise, it is a /// SuccessResult with a collection of lines from the document. - public Task>> GetLines(Stream stream) + public async Task>> GetLines(Stream stream) { Verify.IsNotNull(stream, nameof(stream)); @@ -39,32 +39,32 @@ public Task>> GetLines(Stream stream) { IResult> nullBodyLines = new FailureResult>( Strings.UnableToOpenDocx("Couldn't find the body of the document.")); - return Task.FromResult(nullBodyLines); + return nullBodyLines; } IResult> lines = new SuccessResult>(GetLinesFromBody(body)); - return Task.FromResult(lines); + return lines; } } catch (ArgumentNullException ex) { - Console.Error.WriteLine(ex); + await Console.Error.WriteLineAsync(ex.ToString()).ConfigureAwait(false); IResult> lines = new FailureResult>(Strings.UnexpectedNullValue); - return Task.FromResult(lines); + return lines; } catch (OpenXmlPackageException ex) { - Console.Error.WriteLine(ex); + await Console.Error.WriteLineAsync(ex.ToString()).ConfigureAwait(false); IResult> lines = new FailureResult>( Strings.UnableToOpenDocx(ex.Message)); - return Task.FromResult(lines); + return lines; } catch (FileFormatException ex) { - Console.Error.WriteLine(ex); + await Console.Error.WriteLineAsync(ex.ToString()).ConfigureAwait(false); IResult> lines = new FailureResult>( Strings.UnableToOpenDocx(ex.Message)); - return Task.FromResult(lines); + return lines; } } diff --git a/YetAnotherPacketParser/YetAnotherPacketParser/Lexer/HtmlLexer.cs b/YetAnotherPacketParser/YetAnotherPacketParser/Lexer/HtmlLexer.cs index 9313aab..630c92f 100644 --- a/YetAnotherPacketParser/YetAnotherPacketParser/Lexer/HtmlLexer.cs +++ b/YetAnotherPacketParser/YetAnotherPacketParser/Lexer/HtmlLexer.cs @@ -18,21 +18,27 @@ public async Task>> GetLines(Stream stream) // Should be surrounded by a try/catch, in case parsing fails try { - BrowsingContext context = new BrowsingContext(Configuration.Default); - IDocument document = await context.OpenAsync((request) => request.Content(stream)); - IHtmlElement? body = document.Body; - if (body == null) + IHtmlElement? body; + using (BrowsingContext context = new BrowsingContext(Configuration.Default)) { - return new FailureResult>(Strings.HtmlFileNeedsBodyElement); + IDocument document = await context.OpenAsync((request) => request.Content(stream)).ConfigureAwait(false); + body = document.Body; + if (body == null) + { + return new FailureResult>(Strings.HtmlFileNeedsBodyElement); + } } - IList textLines = this.GetTextLines(body); + IList textLines = GetTextLines(body); return ClassifyLines(textLines); } + // Unfortunately, we don't know what AngleSharp can throw, so we have to catch-all from here +#pragma warning disable CA1031 // Do not catch general exception types catch (Exception ex) +#pragma warning restore CA1031 // Do not catch general exception types { // This is bad form, but I'll try to narrow down the exceptions ltaer - Console.Error.WriteLine(ex); + await Console.Error.WriteLineAsync(ex.ToString()).ConfigureAwait(false); IResult> lines = new FailureResult>( Strings.UnableToOpenHtml(ex.Message)); return lines; @@ -40,7 +46,7 @@ public async Task>> GetLines(Stream stream) } // We get the root paragraphs, then get all of the lines included in the root paragraph - private IList GetTextLines(IHtmlElement body) + private static IList GetTextLines(IHtmlElement body) { IList formattedTexts = new List(); Formatting previousFormatting = new Formatting(); diff --git a/YetAnotherPacketParser/YetAnotherPacketParser/MagicWordDetector.cs b/YetAnotherPacketParser/YetAnotherPacketParser/MagicWordDetector.cs index a01e068..4e1e92f 100644 --- a/YetAnotherPacketParser/YetAnotherPacketParser/MagicWordDetector.cs +++ b/YetAnotherPacketParser/YetAnotherPacketParser/MagicWordDetector.cs @@ -18,7 +18,6 @@ internal static class MagicWordDetector zipSpannedMagicWords }; - // Returns a succesful result if it is a zip file. It returns a read-only stream public static async Task> IsZipFile(Stream stream) { @@ -34,7 +33,7 @@ internal static class MagicWordDetector } byte[] buffer = new byte[zipMagicWords.Length]; - await stream.ReadAsync(buffer, 0, zipMagicWords.Length); + await stream.ReadAsync(buffer, CancellationToken.None).ConfigureAwait(false); Stream peekableStream = new PeekableStream(stream, buffer); peekableStream.Position = 0; return new Tuple( @@ -95,7 +94,7 @@ public override int Read(byte[] buffer, int offset, int count) { if (offset + count > buffer.Length) { - throw new ArgumentOutOfRangeException(); + throw new ArgumentOutOfRangeException(nameof(offset)); } int index = offset; @@ -126,37 +125,32 @@ public override int Read(byte[] buffer, int offset, int count) } } - public override async Task ReadAsync( - byte[] buffer, int offset, int count, CancellationToken cancellationToken) + public override async ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = default) { cancellationToken.ThrowIfCancellationRequested(); - if (offset + count > buffer.Length) - { - throw new ArgumentOutOfRangeException(); - } - - int index = offset; - for (long i = this.position; i < this.peekBuffer.Length; i++) + int count = buffer.Length; + int index = 0; + for (long i = this.position; i < this.peekBuffer.Length && index < count; i++) { - buffer[index] = this.peekBuffer[i]; + buffer.Span[index] = this.peekBuffer[i]; index++; this.position++; } // truncation from long to int is safe if the long is less than int.MaxValue - int bytesFromBuffer = index - offset; - int bytesFromStream = count - bytesFromBuffer; + int bytesFromStream = count - index; if (bytesFromStream > 0) { - int readCount = await this.stream.ReadAsync(buffer, index, bytesFromStream); + int readCount = await this.stream.ReadAsync(buffer.Slice(index, bytesFromStream), cancellationToken) + .ConfigureAwait(false); if (this.CanSeek) { this.position = this.stream.Position; } - return readCount + bytesFromBuffer; + return readCount + index; } else { @@ -164,6 +158,19 @@ public override int Read(byte[] buffer, int offset, int count) } } + public override async Task ReadAsync( + byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + + if (offset + count > buffer.Length) + { + throw new ArgumentOutOfRangeException(nameof(offset)); + } + + return await this.ReadAsync(buffer.AsMemory().Slice(offset, count), cancellationToken); + } + public override long Seek(long offset, SeekOrigin origin) { long result = this.stream.Seek(offset, origin); @@ -192,9 +199,10 @@ public override void Close() this.stream.Close(); } - public override ValueTask DisposeAsync() + public override async ValueTask DisposeAsync() { - return this.stream.DisposeAsync(); + await base.DisposeAsync().ConfigureAwait(false); + await this.stream.DisposeAsync().ConfigureAwait(false); } } } diff --git a/YetAnotherPacketParser/YetAnotherPacketParser/PacketConverter.cs b/YetAnotherPacketParser/YetAnotherPacketParser/PacketConverter.cs index 05f90f9..a583d37 100644 --- a/YetAnotherPacketParser/YetAnotherPacketParser/PacketConverter.cs +++ b/YetAnotherPacketParser/YetAnotherPacketParser/PacketConverter.cs @@ -32,7 +32,7 @@ public static class PacketConverter Verify.IsNotNull(options, nameof(options)); Verify.IsNotNull(stream, nameof(stream)); - Tuple readStreamResult = await MagicWordDetector.IsZipFile(stream); + Tuple readStreamResult = await MagicWordDetector.IsZipFile(stream).ConfigureAwait(false); stream = readStreamResult.Item2; try @@ -42,7 +42,7 @@ public static class PacketConverter // Assume it's HTML for now, and refactor if we need to support more input formats return new ConvertResult[] { - await CompilePacketAsync(options.StreamName, stream, options, FileType.Html) + await CompilePacketAsync(options.StreamName, stream, options, FileType.Html).ConfigureAwait(false) }; } diff --git a/YetAnotherPacketParser/YetAnotherPacketParser/Parser/LinesParser.cs b/YetAnotherPacketParser/YetAnotherPacketParser/Parser/LinesParser.cs index 8276749..afb1aad 100644 --- a/YetAnotherPacketParser/YetAnotherPacketParser/Parser/LinesParser.cs +++ b/YetAnotherPacketParser/YetAnotherPacketParser/Parser/LinesParser.cs @@ -128,7 +128,7 @@ public IResult Parse(IEnumerable lines) string metadata = metadataLine.Text.UnformattedText; if (metadata.Length > 2) { - int metadataStart = metadata.IndexOf('<'); + int metadataStart = metadata.IndexOf('<', StringComparison.Ordinal); int metadataEnd = metadata.LastIndexOf('>'); if (metadataStart >= 0 && metadataStart < metadata.Length + 1 && metadataEnd > metadataStart) { @@ -163,7 +163,7 @@ private static string GetFailureMessage(LinesEnumerator lines, string message) } else { - snippet.Append(segment.Text.Substring(0, remainingLength)); + snippet.Append(segment.Text.AsSpan(0, remainingLength)); } remainingLength = checked(FailureSnippetCharacterLimit - snippet.Length); diff --git a/YetAnotherPacketParser/YetAnotherPacketParser/YetAnotherPacketParser.csproj b/YetAnotherPacketParser/YetAnotherPacketParser/YetAnotherPacketParser.csproj index 317cb66..b0e00c0 100644 --- a/YetAnotherPacketParser/YetAnotherPacketParser/YetAnotherPacketParser.csproj +++ b/YetAnotherPacketParser/YetAnotherPacketParser/YetAnotherPacketParser.csproj @@ -2,7 +2,7 @@ Library - netcoreapp3.1 + netcoreapp3.1;net6.0 enable YetAnotherPacketParser Yet Another Packet Parser parses .docx quiz bowl packets and translates them to different formats like JSON or HTML @@ -17,18 +17,17 @@ quizbowl packetparser quizbowlpacketparser https://github.com/alopezlago/YetAnotherPacketParser LICENSE.txt - 0.4.1.0 - 0.4.1.0 - 0.4.1.0 + 0.4.2.0 + 0.4.2.0 + 0.4.2.0 + true + Recommended + All - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - + diff --git a/YetAnotherPacketParser/YetAnotherPacketParserAPI/.config/dotnet-tools.json b/YetAnotherPacketParser/YetAnotherPacketParserAPI/.config/dotnet-tools.json new file mode 100644 index 0000000..3ba0fe6 --- /dev/null +++ b/YetAnotherPacketParser/YetAnotherPacketParserAPI/.config/dotnet-tools.json @@ -0,0 +1,12 @@ +{ + "version": 1, + "isRoot": true, + "tools": { + "dotnet-ef": { + "version": "6.0.0", + "commands": [ + "dotnet-ef" + ] + } + } +} \ No newline at end of file diff --git a/YetAnotherPacketParser/YetAnotherPacketParserAPI/ErrorMessageResponse.cs b/YetAnotherPacketParser/YetAnotherPacketParserAPI/ErrorMessageResponse.cs new file mode 100644 index 0000000..6c24bfd --- /dev/null +++ b/YetAnotherPacketParser/YetAnotherPacketParserAPI/ErrorMessageResponse.cs @@ -0,0 +1,12 @@ +namespace YetAnotherPacketParserAPI +{ + public class ErrorMessageResponse + { + public ErrorMessageResponse(string[] errorMessages) + { + this.ErrorMessages = errorMessages; + } + + public string[] ErrorMessages { get; } + } +} diff --git a/YetAnotherPacketParser/YetAnotherPacketParserAPI/JsonPacketItem.cs b/YetAnotherPacketParser/YetAnotherPacketParserAPI/JsonPacketItem.cs new file mode 100644 index 0000000..0745a8b --- /dev/null +++ b/YetAnotherPacketParser/YetAnotherPacketParserAPI/JsonPacketItem.cs @@ -0,0 +1,19 @@ +using System.Diagnostics.CodeAnalysis; + +namespace YetAnotherPacketParserAPI +{ + public class JsonPacketItem + { + public JsonPacketItem() + { + this.name = string.Empty; + this.packet = string.Empty; + } + + [SuppressMessage("Style", "IDE1006:Naming Styles", Justification = "Lower-cased so that it appears lowercased in the JSON output")] + public string name { get; init; } + + [SuppressMessage("Style", "IDE1006:Naming Styles", Justification = "Lower-cased so that it appears lowercased in the JSON output")] + public object packet { get; init; } + } +} diff --git a/YetAnotherPacketParser/YetAnotherPacketParserAPI/ParseProcessor.cs b/YetAnotherPacketParser/YetAnotherPacketParserAPI/ParseProcessor.cs new file mode 100644 index 0000000..3917e19 --- /dev/null +++ b/YetAnotherPacketParser/YetAnotherPacketParserAPI/ParseProcessor.cs @@ -0,0 +1,303 @@ +using System.IO.Compression; +using System.Text; +using System.Text.Json; +using Microsoft.Extensions.Primitives; +using YetAnotherPacketParser; + +namespace YetAnotherPacketParserAPI +{ + public class ParseProcessor + { + private const int MaximumPackets = 30; + private const int MaximumPacketSizeInBytes = 1 * 1024 * 1024; // 1 MB + + public static async Task Parse(HttpRequest request, ILogger log) + { + log.LogInformation("Parsing started"); + + if (request.Body == null) + { + log.LogError("Failed; called with no body"); + return GetBadRequest("Body is required"); + } + + IPacketConverterOptions options = GetOptions(request, log); + + // Unfortunately, the Zip libraries still use synchronous reads in some cases, so we have to copy our stream to one that doesn't block synchronous reads + using (Stream bodyStream = new MemoryStream()) + { + await request.Body.CopyToAsync(bodyStream); + await request.Body.FlushAsync(); + + IEnumerable results = await PacketConverter.ConvertPacketsAsync(bodyStream, options); + + int resultsCount = results.Count(); + if (resultsCount == 0) + { + return GetBadRequest("No packets found. Does the zip file have any .docx files?"); + } + else if (resultsCount == 1) + { + ConvertResult compileResult = results.First(); + if (!compileResult.Result.Success) + { + return GetBadRequest(compileResult.Result.ErrorMessages); + } + + // If it's JSON, we need to parse the JSON so the output isn't treated as a string. It's unfortunate + // we have to do this round-about way of returning a JSON result; I should research a cleaner way to + // do this that doesn't involve extra deserializaiton. + if (options.OutputFormat == OutputFormat.Json) + { + // Okay to deserialize object in this case since we fully control where the original string came from. + using (Stream stream = new MemoryStream(Encoding.UTF8.GetBytes(compileResult.Result.Value))) + { + return Results.Json(await JsonSerializer.DeserializeAsync(stream)); + } + } + + // TODO: Should we handle the 1-packet zip file case by returning a zip? + return Results.Ok(compileResult.Result.Value); + } + + if (TryGetStringValueFromQuery(request, "mergeMultiple", out string mergeMultipleValue) && + bool.TryParse(mergeMultipleValue, out bool mergeMultiple)) + { + log.LogInformation($"Merge multiple packets: {mergeMultiple}"); + } + else + { + mergeMultiple = false; + } + + // We have a zip file. Return one, or return a merged file if requested. + // TODO: See if we can return a JSON result showing success, # successfully parsed packets, and the contents + // as a zip file and JSON array? + bool outputFormatIsJson = options.OutputFormat == OutputFormat.Json; + IEnumerable successResults = results.Where(result => result.Result.Success); + using (MemoryStream memoryStream = new MemoryStream()) + { + if (!mergeMultiple) + { + WriteMultiplePacketsToZip(successResults, memoryStream, outputFormatIsJson); + } + else if (outputFormatIsJson) + { + WriteMultiplePacketsToJson(successResults, memoryStream); + } + else + { + WriteMultiplePacketsToHtml(successResults, memoryStream); + } + + log.LogInformation($"Succesfully parsed {successResults.Count()} out of {results.Count()} packets"); + IEnumerable failedResults = results + .Where(result => !result.Result.Success) + .OrderBy(result => result.Filename); + foreach (ConvertResult compileResult in failedResults) + { + log.LogWarning($"{compileResult.Filename} failed to compile."); + } + + await memoryStream.FlushAsync(); + return Results.Stream(memoryStream, fileDownloadName: options.StreamName); + } + } + } + + private static IResult GetBadRequest(string errorMessage) + { + return Results.BadRequest(new ErrorMessageResponse(new string[] { errorMessage })); + } + + private static IResult GetBadRequest(IEnumerable errorMessages) + { + return Results.BadRequest(new ErrorMessageResponse(errorMessages.ToArray())); + } + + private static IPacketConverterOptions GetOptions(HttpRequest request, ILogger log) + { + OutputFormat outputFormat; + if (TryGetStringValueFromQuery(request, "format", out string outputFormatString)) + { + log.LogInformation($"Parsed format: {outputFormatString}"); + switch (outputFormatString.ToUpperInvariant()) + { + case "HTML": + outputFormat = OutputFormat.Html; + break; + case "JSON": + outputFormat = OutputFormat.Json; + break; + default: + outputFormat = OutputFormat.Json; + log.LogWarning($"Unrecognized format: {outputFormatString}. Defaulting to JSON"); + break; + } + } + else + { + outputFormat = OutputFormat.Json; + log.LogInformation("Using the default format"); + } + + if (TryGetStringValueFromQuery(request, "prettyPrint", out string stringValue) && + bool.TryParse(stringValue, out bool prettyPrint)) + { + log.LogInformation($"Parsed prettyPrint: {prettyPrint}"); + } + else + { + prettyPrint = false; + log.LogInformation("Using the default pretty print setting"); + } + + if (TryGetStringValueFromQuery(request, "modaq", out string modaqValue) && + bool.TryParse(modaqValue, out bool modaqFormat)) + { + log.LogInformation($"Parsed MODAQ formatted: {modaqFormat}"); + } + else + { + modaqFormat = false; + } + + Action logMessage = (logLevel, message) => Log(log, logLevel, message); + switch (outputFormat) + { + case OutputFormat.Html: + return new HtmlPacketCompilerOptions() + { + StreamName = "Request", + MaximumPackets = MaximumPackets, + MaximumPacketSizeInBytes = MaximumPacketSizeInBytes, + Log = logMessage + }; + case OutputFormat.Json: + // default to JSON + return new JsonPacketCompilerOptions() + { + StreamName = "Request", + PrettyPrint = prettyPrint, + MaximumPackets = MaximumPackets, + MaximumPacketSizeInBytes = MaximumPacketSizeInBytes, + ModaqFormat = modaqFormat, + Log = logMessage + }; + default: + // Treat it as JSON and log an error + log.LogError($"Unrecognized OutputFormat: {outputFormat}"); + return new JsonPacketCompilerOptions() + { + StreamName = "Request", + PrettyPrint = prettyPrint, + MaximumPackets = MaximumPackets, + MaximumPacketSizeInBytes = MaximumPacketSizeInBytes, + Log = logMessage + }; + } + } + + private static void Log(ILogger logger, YetAnotherPacketParser.LogLevel logLevel, string message) + { + switch (logLevel) + { + case YetAnotherPacketParser.LogLevel.Informational: + logger.LogInformation(message); + break; + case YetAnotherPacketParser.LogLevel.Verbose: + logger.LogDebug(message); + break; + default: + logger.LogWarning($"Logged with unknown log level: {logLevel}"); + logger.LogDebug(message); + break; + } + } + + private static bool TryGetStringValueFromQuery(HttpRequest request, string key, out string stringValue) + { + if (request.Query.TryGetValue(key, out StringValues values) && values.Count > 0) + { + stringValue = values[0]; + return true; + } + + stringValue = string.Empty; + return false; + } + + // TODO: These were copied and modified from Program.cs. See if we can share the code. + private static void WriteMultiplePacketsToJson(IEnumerable packets, Stream stream) + { + IList jsonPackets = new List(); + foreach (ConvertResult compileResult in packets.OrderBy(packet => packet.Filename)) + { + jsonPackets.Add(new JsonPacketItem() + { + name = compileResult.Filename.Replace(".docx", string.Empty), +#pragma warning disable CS8601 // Possible null reference assignment. No idea where it thinks the null reference assignment is, since none of those values are null + packet = JsonSerializer.Deserialize(compileResult.Result.Value) +#pragma warning restore CS8601 // Possible null reference assignment. + }); + } + + string content = JsonSerializer.Serialize(jsonPackets); + + using (StreamWriter writer = new StreamWriter(stream)) + { + writer.Write(content); + } + } + + private static void WriteMultiplePacketsToHtml(IEnumerable packets, Stream stream) + { + IList htmlBodies = new List(); + foreach (ConvertResult compileResult in packets.OrderBy(packet => packet.Filename)) + { + string html = compileResult.Result.Value; + int bodyStartIndex = html.IndexOf("", StringComparison.OrdinalIgnoreCase); + int bodyEndIndex = html.LastIndexOf("", StringComparison.OrdinalIgnoreCase); + if (bodyStartIndex == -1 || bodyEndIndex == -1 || bodyStartIndex > bodyEndIndex) + { + // Skip, since the HTML was malformed + continue; + } + + // Skip past "" + bodyStartIndex += 6; + string htmlBody = $"

{compileResult.Filename.Replace(".docx", string.Empty)}

{html.Substring(bodyStartIndex, bodyEndIndex - bodyStartIndex)}"; + + htmlBodies.Add(htmlBody); + } + + string bundledHtml = $"{string.Join("
", htmlBodies)}"; + + using (StreamWriter writer = new StreamWriter(stream)) + { + writer.Write(bundledHtml); + } + } + + private static void WriteMultiplePacketsToZip( + IEnumerable packets, Stream stream, bool outputFormatIsJson) + { + using (ZipArchive archive = new ZipArchive(stream, ZipArchiveMode.Create)) + { + foreach (ConvertResult compileResult in packets) + { + string newFilename = outputFormatIsJson ? + compileResult.Filename.Replace(".docx", ".json") : + compileResult.Filename.Replace(".docx", ".html"); + ZipArchiveEntry entry = archive.CreateEntry(newFilename); + + // We can't do this asynchronously, because it complains about writing to the same ZipArchive stream + using (StreamWriter writer = new StreamWriter(entry.Open())) + { + writer.Write(compileResult.Result.Value); + } + } + } + } + } +} diff --git a/YetAnotherPacketParser/YetAnotherPacketParserAPI/Program.cs b/YetAnotherPacketParser/YetAnotherPacketParserAPI/Program.cs new file mode 100644 index 0000000..95798d8 --- /dev/null +++ b/YetAnotherPacketParser/YetAnotherPacketParserAPI/Program.cs @@ -0,0 +1,63 @@ +using AspNetCoreRateLimit; +using YetAnotherPacketParserAPI; + +const int MaximumRequestInBytes = 1 * 1024 * 1024; // 1 MB + +var builder = WebApplication.CreateBuilder(args); + +// Add services to the container. +// Learn more about configuring Swagger/OpenAPI at https://aka.ms/aspnetcore/swashbuckle +builder.Services.AddEndpointsApiExplorer(); +builder.Services.AddSwaggerGen(); + +builder.Services.AddCors(); + +// Needed for rate-limiting +builder.Services.AddOptions(); +builder.Services.AddMemoryCache(); + +// Load rate-limiting configuration from appsettings.json +builder.Services.Configure(builder.Configuration.GetSection("IpRateLimiting")); + +// inject counter and rules stores +builder.Services.AddInMemoryRateLimiting(); + +builder.Services.AddSingleton(); + +var app = builder.Build(); + +// Configure the HTTP request pipeline. +if (app.Environment.IsDevelopment()) +{ + app.UseSwagger(); + app.UseSwaggerUI(); +} + +app.UseHttpsRedirection(); +app.UseHsts(); +app.UseResponseCaching(); +app.UseIpRateLimiting(); + +app.UseCors(); + +// Add a basic GET method to test that the service is up +app.MapGet("/api/get", () => "1"); + +// For now, don't restrict who can call this API +app.MapPost("/api/parse", async (HttpContext context) => + { + // Be paranoid and reject requests larger than 1 MB + if (context.Request.ContentLength > MaximumRequestInBytes) + { + return Results.BadRequest($"Cannot parse requests greater than {MaximumRequestInBytes / 1024.0 / 1024} MB"); + } + + return await ParseProcessor.Parse(context.Request, app.Logger); + }) + .WithName("Parse") + .RequireCors(policy => policy.AllowAnyOrigin()) + .Produces(400) + .Produces(200) + .Produces(200); + +app.Run(); diff --git a/YetAnotherPacketParser/YetAnotherPacketParserAPI/Properties/launchSettings.json b/YetAnotherPacketParser/YetAnotherPacketParserAPI/Properties/launchSettings.json new file mode 100644 index 0000000..0cfc4c4 --- /dev/null +++ b/YetAnotherPacketParser/YetAnotherPacketParserAPI/Properties/launchSettings.json @@ -0,0 +1,31 @@ +{ + "$schema": "https://json.schemastore.org/launchsettings.json", + "iisSettings": { + "windowsAuthentication": false, + "anonymousAuthentication": true, + "iisExpress": { + "applicationUrl": "http://localhost:15766", + "sslPort": 44322 + } + }, + "profiles": { + "YetAnotherPacketParserAPI": { + "commandName": "Project", + "dotnetRunMessages": true, + "launchBrowser": true, + "launchUrl": "swagger", + "applicationUrl": "https://localhost:7068;http://localhost:5068", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + }, + "IIS Express": { + "commandName": "IISExpress", + "launchBrowser": true, + "launchUrl": "swagger", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + } + } +} diff --git a/YetAnotherPacketParser/YetAnotherPacketParserAPI/YetAnotherPacketParserAPI.csproj b/YetAnotherPacketParser/YetAnotherPacketParserAPI/YetAnotherPacketParserAPI.csproj new file mode 100644 index 0000000..87c55f4 --- /dev/null +++ b/YetAnotherPacketParser/YetAnotherPacketParserAPI/YetAnotherPacketParserAPI.csproj @@ -0,0 +1,18 @@ + + + + net6.0 + enable + enable + + + + + + + + + + + + diff --git a/YetAnotherPacketParser/YetAnotherPacketParserAPI/appsettings.Development.json b/YetAnotherPacketParser/YetAnotherPacketParserAPI/appsettings.Development.json new file mode 100644 index 0000000..0c208ae --- /dev/null +++ b/YetAnotherPacketParser/YetAnotherPacketParserAPI/appsettings.Development.json @@ -0,0 +1,8 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning" + } + } +} diff --git a/YetAnotherPacketParser/YetAnotherPacketParserAPI/appsettings.json b/YetAnotherPacketParser/YetAnotherPacketParserAPI/appsettings.json new file mode 100644 index 0000000..861e5a1 --- /dev/null +++ b/YetAnotherPacketParser/YetAnotherPacketParserAPI/appsettings.json @@ -0,0 +1,34 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning" + } + }, + "AllowedHosts": "*", + "IpRateLimiting": { + "EnableEndpointRateLimiting": false, + "StackBlockedRequests": true, + "RealIpHeader": "X-Real-IP", + "ClientIdHeader": "X-ClientId", + "HttpStatusCode": 429, + "IpWhitelist": [ "127.0.0.1", "::1/10" ], + "GeneralRules": [ + { + "Endpoint": "*", + "Period": "1s", + "Limit": 2 + }, + { + "Endpoint": "*", + "Period": "1h", + "Limit": 40 + }, + { + "Endpoint": "*", + "Period": "1d", + "Limit": 50 + } + ] + } +} diff --git a/YetAnotherPacketParser/YetAnotherPacketParserAzureFunction/YetAnotherPacketParserAzureFunction.csproj b/YetAnotherPacketParser/YetAnotherPacketParserAzureFunction/YetAnotherPacketParserAzureFunction.csproj index 4df5c3e..39ea8fc 100644 --- a/YetAnotherPacketParser/YetAnotherPacketParserAzureFunction/YetAnotherPacketParserAzureFunction.csproj +++ b/YetAnotherPacketParser/YetAnotherPacketParserAzureFunction/YetAnotherPacketParserAzureFunction.csproj @@ -1,19 +1,19 @@  - netcoreapp3.1 + netcoreapp3.1;net6.0 v3 Yet Another Packet Parser Azure Function parses quiz bowl packets and translates them to different formats (c) 2020 Alejandro Lopez-Lago YetAnotherPacketParserAzureFunction Alejandro Lopez-Lago YAPP Azure Function - 0.4.1.0 - 0.4.1.0 - 0.4.1.0 + 0.4.2.0 + 0.4.2.0 + 0.4.2.0 - - + + <_FunctionsSkipCleanOutput>true diff --git a/YetAnotherPacketParser/YetAnotherPacketParserCommandLine/Program.cs b/YetAnotherPacketParser/YetAnotherPacketParserCommandLine/Program.cs index d7146e3..1812769 100644 --- a/YetAnotherPacketParser/YetAnotherPacketParserCommandLine/Program.cs +++ b/YetAnotherPacketParser/YetAnotherPacketParserCommandLine/Program.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Globalization; using System.IO; using System.IO.Compression; using System.Linq; @@ -35,20 +36,20 @@ await Parser.Default.ParseArguments(args) private static async Task RunAsync(CommandLineOptions options) { - if (string.Equals(options.Input, options.Output)) + if (string.Equals(options.Input, options.Output, StringComparison.OrdinalIgnoreCase)) { - Console.Error.WriteLine("Input and output files must be different"); + await Console.Error.WriteLineAsync("Input and output files must be different").ConfigureAwait(true); return; } else if (!File.Exists(options.Input)) { - Console.Error.WriteLine($"File {options.Input} does not exist"); + await Console.Error.WriteLineAsync($"File {options.Input} does not exist").ConfigureAwait(true); return; } IPacketConverterOptions packetCompilerOptions; Action log = (logLevel, message) => Log(options, logLevel, message); - switch (options.OutputFormat.Trim().ToUpper()) + switch (options.OutputFormat.Trim().ToUpper(CultureInfo.CurrentCulture)) { case "JSON": packetCompilerOptions = new JsonPacketCompilerOptions() @@ -67,20 +68,20 @@ private static async Task RunAsync(CommandLineOptions options) }; break; default: - Console.Error.WriteLine("Invalid format. Valid formats: json, html"); + await Console.Error.WriteLineAsync("Invalid format. Valid formats: json, html").ConfigureAwait(true); return; } IEnumerable outputResults; using (FileStream fileStream = new FileStream(options.Input, FileMode.Open, FileAccess.Read, FileShare.Read)) { - outputResults = await PacketConverter.ConvertPacketsAsync(fileStream, packetCompilerOptions); + outputResults = await PacketConverter.ConvertPacketsAsync(fileStream, packetCompilerOptions).ConfigureAwait(true); } int resultsCount = outputResults.Count(); if (resultsCount == 0) { - Console.Error.WriteLine("No packets found"); + await Console.Error.WriteLineAsync("No packets found").ConfigureAwait(true); return; } else if (resultsCount == 1) @@ -88,7 +89,7 @@ private static async Task RunAsync(CommandLineOptions options) ConvertResult compileResult = outputResults.First(); if (!compileResult.Result.Success) { - Console.Error.WriteLine(compileResult.Result); + await Console.Error.WriteLineAsync(compileResult.Result.ToString()).ConfigureAwait(false); return; } @@ -125,7 +126,8 @@ private static async Task RunAsync(CommandLineOptions options) .OrderBy(result => result.Filename); foreach (ConvertResult compileResult in failedResults) { - Console.Error.WriteLine($"{compileResult.Filename} failed to compile. Error(s):\n {compileResult.Result}"); + await Console.Error.WriteLineAsync($"{compileResult.Filename} failed to compile. Error(s):\n {compileResult.Result}") + .ConfigureAwait(true); } } @@ -150,8 +152,8 @@ private static void Log(CommandLineOptions options, LogLevel logLevel, string me foreach (ConvertResult compileResult in packets) { string newFilename = outputFormatIsJson ? - compileResult.Filename.Replace(".docx", ".json") : - compileResult.Filename.Replace(".docx", ".html"); + compileResult.Filename.Replace(".docx", ".json", StringComparison.OrdinalIgnoreCase) : + compileResult.Filename.Replace(".docx", ".html", StringComparison.OrdinalIgnoreCase); ZipArchiveEntry entry = outputArchive.CreateEntry(newFilename); // We can't do this asynchronously, because it complains about writing to the same ZipArchive stream @@ -170,7 +172,7 @@ private static void WriteMultiplePacketsToJson(IEnumerable packet { jsonPackets.Add(new JsonPacket() { - name = compileResult.Filename.Replace(".docx", string.Empty), + name = compileResult.Filename.Replace(".docx", string.Empty, StringComparison.OrdinalIgnoreCase), packet = JsonSerializer.Deserialize(compileResult.Result.Value) }); } @@ -200,7 +202,7 @@ private static void WriteMultiplePacketsToHtml(IEnumerable packet // Skip past "" bodyStartIndex += 6; - string htmlBody = $"

{compileResult.Filename.Replace(".docx", string.Empty)}

{html.Substring(bodyStartIndex, bodyEndIndex - bodyStartIndex)}"; + string htmlBody = $"

{compileResult.Filename.Replace(".docx", string.Empty, StringComparison.OrdinalIgnoreCase)}

{html.Substring(bodyStartIndex, bodyEndIndex - bodyStartIndex)}"; htmlBodies.Add(htmlBody); } diff --git a/YetAnotherPacketParser/YetAnotherPacketParserCommandLine/YetAnotherPacketParserCommandLine.csproj b/YetAnotherPacketParser/YetAnotherPacketParserCommandLine/YetAnotherPacketParserCommandLine.csproj index 1b497bd..516655a 100644 --- a/YetAnotherPacketParser/YetAnotherPacketParserCommandLine/YetAnotherPacketParserCommandLine.csproj +++ b/YetAnotherPacketParser/YetAnotherPacketParserCommandLine/YetAnotherPacketParserCommandLine.csproj @@ -2,16 +2,19 @@ Exe - netcoreapp3.1 + netcoreapp3.1;net6.0 annotations YetAnotherPacketParserCommandLine Alejandro Lopez-Lago (c) 2020 Alejandro Lopez-Lago Yet Another Packet Parser parses quiz bowl packets and translates them to different formats YAPP - 0.4.1.0 - 0.4.1.0 - 0.4.1.0 + 0.4.2.0 + 0.4.2.0 + 0.4.2.0 + true + Recommended + All diff --git a/YetAnotherPacketParser/YetAnotherPacketParserTests/YetAnotherPacketParserTests.csproj b/YetAnotherPacketParser/YetAnotherPacketParserTests/YetAnotherPacketParserTests.csproj index 312ba93..20d96bc 100644 --- a/YetAnotherPacketParser/YetAnotherPacketParserTests/YetAnotherPacketParserTests.csproj +++ b/YetAnotherPacketParser/YetAnotherPacketParserTests/YetAnotherPacketParserTests.csproj @@ -1,7 +1,7 @@ - netcoreapp3.1 + netcoreapp3.1;net6.0 false @@ -10,8 +10,8 @@ - - + + all runtime; build; native; contentfiles; analyzers; buildtransitive