From 6cbd9cb866491ae610cdb8b2c60dc551e59da303 Mon Sep 17 00:00:00 2001 From: Shane32 Date: Sun, 27 Jun 2021 19:20:22 -0400 Subject: [PATCH 1/5] Progress --- src/GraphQLParser/LexerContext.cs | 202 +++++++++++++++++++++++++++++- src/GraphQLParser/Token.cs | 2 + src/GraphQLParser/TokenKind.cs | 14 ++- 3 files changed, 216 insertions(+), 2 deletions(-) diff --git a/src/GraphQLParser/LexerContext.cs b/src/GraphQLParser/LexerContext.cs index 837c38cf..e74ed989 100644 --- a/src/GraphQLParser/LexerContext.cs +++ b/src/GraphQLParser/LexerContext.cs @@ -47,7 +47,16 @@ public Token GetToken() return ReadNumber(); if (code == '"') - return ReadString(); + { + if (_currentIndex + 2 < _source.Length && _source.Span[_currentIndex + 1] == '"' && _source.Span[_currentIndex + 2] == '"') + { + return ReadBlockString(); + } + else + { + return ReadString(); + } + } return Throw_From_GetToken2(code); } @@ -172,6 +181,192 @@ private Token ReadComment() ); } + private Token ReadBlockString() + { + int start = _currentIndex += 2; + char code = NextCode(); + + Span buffer = stackalloc char[4096]; + StringBuilder? sb = null; + + int index = 0; + bool escape = false; //when the last character was \ + bool lastWasCr = false; + + while (_currentIndex < _source.Length) + { + if (code < 0x0020 && code != 0x0009 && code != 0x000A && code != 0x000D) + { + Throw_From_ReadBlockString1(code); + } + + //check for """ + if (code == '"' && _currentIndex + 2 < _source.Length && _source.Span[_currentIndex + 1] == '"' && _source.Span[_currentIndex + 2] == '"') + { + //if last character was \ then go ahead and write out the """, skipping the \ + if (escape) + { + escape = false; + } + else + { + //end of blockstring + break; + } + } + else if (escape) + { + //last character was \ so write the \ and then retry this character with escaped = false + code = '\\'; + _currentIndex--; + escape = false; + } + else if (code == '\\') + { + //this character is a \ so don't write anything yet, but check the next character + escape = true; + code = NextCode(); + lastWasCr = false; + continue; + } + else + { + escape = false; + } + + + if (!(lastWasCr && code == '\n')) + { + //write code + if (index < buffer.Length) + { + buffer[index++] = code == '\r' ? '\n' : code; + } + else // fallback to StringBuilder in case of buffer overflow + { + if (sb == null) + sb = new StringBuilder(buffer.Length * 2); + + for (int i = 0; i < buffer.Length; ++i) + sb.Append(buffer[i]); + + sb.Append(code == '\r' ? '\n' : code); + index = 0; + } + } + + lastWasCr = code == '\r'; + + code = NextCode(); + } + + if (_currentIndex >= _source.Length) + { + Throw_From_ReadString2(); + } + + if (sb != null) + { + for (int i = 0; i < index; ++i) + sb.Append(buffer[i]); + } + + //at this point, if sb != null, then sb has the whole string, otherwise buffer (of length index) has the whole string + //also, all line termination combinations have been replaced with LF + + ROM value; + if (sb != null) + { + var chars = new char[sb.Length]; + sb.CopyTo(0, chars, 0, sb.Length); + value = ProcessBuffer(chars, sb.Length); + } + else { + value = ProcessBuffer(buffer, index); + } + + return new Token + ( + TokenKind.BLOCKSTRING, + value, + start, + _currentIndex + 1 + ); + + ROM ProcessBuffer(Span buffer, int length) + { + //scan string to determine maximum valid commonIndent value, number of initial blank lines, and number of trailing blank lines + int commonIndent = int.MaxValue; + int line = 0; + int whitespace = 0; + bool allWhitespace = true; + int initialBlankLines = 1; + bool reachedCharacter = false; + int trailingBlankLines = 0; + for (int index = 0; index < length; index++) + { + code = buffer[index]; + if (code == '\n') + { + if (allWhitespace) + trailingBlankLines += 1; + if (line != 0 && !allWhitespace && whitespace < commonIndent) + commonIndent = whitespace; + line++; + whitespace = 0; + allWhitespace = true; + if (!reachedCharacter) + initialBlankLines++; + } + else if (code == ' ' || code == '\t') + { + if (allWhitespace) + commonIndent += 1; + } + else + { + allWhitespace = false; + reachedCharacter = true; + initialBlankLines--; + trailingBlankLines = 0; + } + } + if (allWhitespace) + trailingBlankLines += 1; + if (line != 0 && !allWhitespace && whitespace < commonIndent) + commonIndent = whitespace; + if (commonIndent == int.MaxValue) + commonIndent = 0; + int lines = line + 1; + int skipLinesAfter = lines - trailingBlankLines; + + //step through the input, skipping the initial blank lines and the trailing blank lines, and skipping the initial blank characters from the start of each line + Span output = length <= 4096 ? stackalloc char[length] : new char[length]; + int outputIndex = 0; + line = 0; + int col = 0; + for (int index = 0; index < length; index++) + { + code = buffer[index]; + if (code == '\n') + { + if (++line >= skipLinesAfter) + break; + col = 0; + if (line > initialBlankLines) + output[outputIndex++] = code; + } + else + { + if (line > 0 && col++ >= commonIndent) + output[outputIndex++] = code; + } + } + + return buffer.Slice(0, outputIndex).ToString(); + } + } + private Token ReadString() { int start = _currentIndex; @@ -245,6 +440,11 @@ private void Throw_From_ReadString2() throw new GraphQLSyntaxErrorException("Unterminated string.", _source, _currentIndex); } + private void Throw_From_ReadBlockString1(char code) + { + throw new GraphQLSyntaxErrorException($"Invalid character within BlockString: \\u{(int)code:D4}.", _source, _currentIndex); + } + // sets escaped only to true private char ReadCharacterFromString(char currentCharacter, ref bool escaped) { diff --git a/src/GraphQLParser/Token.cs b/src/GraphQLParser/Token.cs index 97fd7988..234e00ed 100644 --- a/src/GraphQLParser/Token.cs +++ b/src/GraphQLParser/Token.cs @@ -73,6 +73,7 @@ public Token(TokenKind kind, ROM value, int start, int end) TokenKind.INT => "Int", TokenKind.FLOAT => "Float", TokenKind.STRING => "String", + TokenKind.BLOCKSTRING => "BlockString", TokenKind.COMMENT => "#", TokenKind.UNKNOWN => "Unknown", _ => throw new NotSupportedException(kind.ToString()) @@ -83,6 +84,7 @@ public Token(TokenKind kind, ROM value, int start, int end) Kind == TokenKind.INT || Kind == TokenKind.FLOAT || Kind == TokenKind.STRING || + Kind == TokenKind.BLOCKSTRING || Kind == TokenKind.COMMENT || Kind == TokenKind.UNKNOWN; diff --git a/src/GraphQLParser/TokenKind.cs b/src/GraphQLParser/TokenKind.cs index 9c66b8ab..31f70c80 100644 --- a/src/GraphQLParser/TokenKind.cs +++ b/src/GraphQLParser/TokenKind.cs @@ -115,6 +115,18 @@ public enum TokenKind /// /// & /// - AMPERSAND = 21 + AMPERSAND = 21, + + /// + /// Block strings are sequences of characters wrapped in triple‐quotes ("""). White space, line terminators, + /// quote, and backslash characters may all be used unescaped to enable verbatim text. + /// Since block strings represent freeform text often used in indented positions, the string value semantics + /// of a block string excludes uniform indentation and blank initial and trailing lines. + /// Triple-quotes (""") may be escaped as \""" within the block string. + /// + /// + /// Line termination sequences (LF, CR, or CRLF) are always replaced with a line-feed (LF) character. + /// + BLOCKSTRING = 22, } } From 6758cb4215e24c3e605c68c3276ad0eb49c107aa Mon Sep 17 00:00:00 2001 From: Shane Krueger Date: Mon, 28 Jun 2021 01:12:15 -0400 Subject: [PATCH 2/5] Updates --- .../GraphQL-Parser.approved.txt | 1 + .../Validation/LexerValidationTests.cs | 35 +++++++++++++++++++ src/GraphQLParser/LexerContext.cs | 31 +++++++++------- 3 files changed, 55 insertions(+), 12 deletions(-) diff --git a/src/GraphQLParser.ApiTests/GraphQL-Parser.approved.txt b/src/GraphQLParser.ApiTests/GraphQL-Parser.approved.txt index 153cf2ec..4e8f9396 100644 --- a/src/GraphQLParser.ApiTests/GraphQL-Parser.approved.txt +++ b/src/GraphQLParser.ApiTests/GraphQL-Parser.approved.txt @@ -456,5 +456,6 @@ namespace GraphQLParser COMMENT = 19, UNKNOWN = 20, AMPERSAND = 21, + BLOCKSTRING = 22, } } \ No newline at end of file diff --git a/src/GraphQLParser.Tests/Validation/LexerValidationTests.cs b/src/GraphQLParser.Tests/Validation/LexerValidationTests.cs index f42cba49..c6ffe660 100644 --- a/src/GraphQLParser.Tests/Validation/LexerValidationTests.cs +++ b/src/GraphQLParser.Tests/Validation/LexerValidationTests.cs @@ -391,5 +391,40 @@ public void Lex_UnterminatedStringWithText_ThrowsExceptionWithCorrectMessage() exception.Line.ShouldBe(1); exception.Column.ShouldBe(14); } + + [Theory] + [InlineData("test", "test")] + [InlineData("te\\\"\"\"st", "te\"\"\"st")] + [InlineData("\ntest", "test")] + [InlineData("\r\ntest", "test")] + [InlineData(" \ntest", "test")] + [InlineData("\t\ntest", "test")] + [InlineData("\n\ntest", "test")] + [InlineData("test\nline2", "test\nline2")] + [InlineData("test\rline2", "test\nline2")] + [InlineData("test\r\nline2", "test\nline2")] + [InlineData("test\r\r\nline2", "test\n\nline2")] + [InlineData("test\r\n\nline2", "test\n\nline2")] + [InlineData("test\n", "test")] + [InlineData("test\n ", "test")] + [InlineData("test\n\t", "test")] + [InlineData("test\n\n", "test")] + [InlineData("test\n line2", "test\nline2")] + [InlineData("test\n\t\tline2", "test\nline2")] + [InlineData("test\n \tline2", "test\nline2")] + [InlineData(" test\n line2", " test\nline2")] + [InlineData(" test\n line2\n\t\tline3\n line4", " test\nline2\n\tline3\n line4")] + [InlineData(" test\n Hello,\n\n world!\n ", " test\nHello,\n\n world!")] + [InlineData(" \n Hello,\r\n\n world!\n ", "Hello,\n\n world!")] + [InlineData(" \n Hello,\r\n\n wor___ld!\n ", "Hello,\n\n wor___ld!")] + public void Lex_BlockString_Tests(string input, string expected) + { + input = input.Replace("___", new string('_', 9000)); + expected = expected.Replace("___", new string('_', 9000)); + input = "\"\"\"" + input + "\"\"\""; + var actual = input.Lex(); + actual.Kind.ShouldBe(TokenKind.BLOCKSTRING); + actual.Value.ToString().ShouldBe(expected); + } } } diff --git a/src/GraphQLParser/LexerContext.cs b/src/GraphQLParser/LexerContext.cs index e74ed989..ccd0fcf1 100644 --- a/src/GraphQLParser/LexerContext.cs +++ b/src/GraphQLParser/LexerContext.cs @@ -264,6 +264,7 @@ private Token ReadBlockString() { Throw_From_ReadString2(); } + _currentIndex += 2; if (sb != null) { @@ -279,10 +280,10 @@ private Token ReadBlockString() { var chars = new char[sb.Length]; sb.CopyTo(0, chars, 0, sb.Length); - value = ProcessBuffer(chars, sb.Length); + value = ProcessBuffer(chars); } else { - value = ProcessBuffer(buffer, index); + value = ProcessBuffer(buffer.Slice(0, index)); } return new Token @@ -293,7 +294,7 @@ private Token ReadBlockString() _currentIndex + 1 ); - ROM ProcessBuffer(Span buffer, int length) + ROM ProcessBuffer(Span buffer) { //scan string to determine maximum valid commonIndent value, number of initial blank lines, and number of trailing blank lines int commonIndent = int.MaxValue; @@ -303,7 +304,7 @@ ROM ProcessBuffer(Span buffer, int length) int initialBlankLines = 1; bool reachedCharacter = false; int trailingBlankLines = 0; - for (int index = 0; index < length; index++) + for (int index = 0; index < buffer.Length; index++) { code = buffer[index]; if (code == '\n') @@ -321,13 +322,14 @@ ROM ProcessBuffer(Span buffer, int length) else if (code == ' ' || code == '\t') { if (allWhitespace) - commonIndent += 1; + whitespace++; } else { allWhitespace = false; + if (!reachedCharacter) + initialBlankLines--; reachedCharacter = true; - initialBlankLines--; trailingBlankLines = 0; } } @@ -339,13 +341,18 @@ ROM ProcessBuffer(Span buffer, int length) commonIndent = 0; int lines = line + 1; int skipLinesAfter = lines - trailingBlankLines; - - //step through the input, skipping the initial blank lines and the trailing blank lines, and skipping the initial blank characters from the start of each line - Span output = length <= 4096 ? stackalloc char[length] : new char[length]; + //outputs: + // commonIndent + // initialBlankLines + // skipLinesAfter + + //step through the input, skipping the initial blank lines and the trailing blank lines, + //and skipping the initial blank characters from the start of each line + Span output = buffer.Length <= 4096 ? stackalloc char[buffer.Length] : new char[buffer.Length]; int outputIndex = 0; line = 0; int col = 0; - for (int index = 0; index < length; index++) + for (int index = 0; index < buffer.Length; index++) { code = buffer[index]; if (code == '\n') @@ -358,12 +365,12 @@ ROM ProcessBuffer(Span buffer, int length) } else { - if (line > 0 && col++ >= commonIndent) + if (line >= initialBlankLines && (line == 0 || col++ >= commonIndent)) output[outputIndex++] = code; } } - return buffer.Slice(0, outputIndex).ToString(); + return output.Slice(0, outputIndex).ToString(); } } From c6e916fbc81336dfaf7cfb5a2dfd971cdcb9c108 Mon Sep 17 00:00:00 2001 From: Shane32 Date: Mon, 28 Jun 2021 10:03:09 -0400 Subject: [PATCH 3/5] Updates --- .../GraphQL-Parser.approved.txt | 1 - src/GraphQLParser.Tests/LexerTests.cs | 41 ++++++ .../Validation/LexerValidationTests.cs | 71 +++++----- src/GraphQLParser/LexerContext.cs | 123 +++++++++--------- src/GraphQLParser/Token.cs | 2 - src/GraphQLParser/TokenKind.cs | 24 ++-- 6 files changed, 155 insertions(+), 107 deletions(-) diff --git a/src/GraphQLParser.ApiTests/GraphQL-Parser.approved.txt b/src/GraphQLParser.ApiTests/GraphQL-Parser.approved.txt index 4e8f9396..153cf2ec 100644 --- a/src/GraphQLParser.ApiTests/GraphQL-Parser.approved.txt +++ b/src/GraphQLParser.ApiTests/GraphQL-Parser.approved.txt @@ -456,6 +456,5 @@ namespace GraphQLParser COMMENT = 19, UNKNOWN = 20, AMPERSAND = 21, - BLOCKSTRING = 22, } } \ No newline at end of file diff --git a/src/GraphQLParser.Tests/LexerTests.cs b/src/GraphQLParser.Tests/LexerTests.cs index c100f21a..5169e7f1 100644 --- a/src/GraphQLParser.Tests/LexerTests.cs +++ b/src/GraphQLParser.Tests/LexerTests.cs @@ -974,6 +974,47 @@ public void Lex_WhiteSpaceStringToken_HasStringKind() token.Kind.ShouldBe(TokenKind.STRING); } + [Theory] + [InlineData("test", "test")] + [InlineData("te\\\"\"\"st", "te\"\"\"st")] + [InlineData("\ntest", "test")] + [InlineData("\r\ntest", "test")] + [InlineData(" \ntest", "test")] + [InlineData("\t\ntest", "test")] + [InlineData("\n\ntest", "test")] + [InlineData("test\nline2", "test\nline2")] + [InlineData("test\rline2", "test\nline2")] + [InlineData("test\r\nline2", "test\nline2")] + [InlineData("test\r\r\nline2", "test\n\nline2")] + [InlineData("test\r\n\nline2", "test\n\nline2")] + [InlineData("test\n", "test")] + [InlineData("test\n ", "test")] + [InlineData("test\n\t", "test")] + [InlineData("test\n\n", "test")] + [InlineData("test\n line2", "test\nline2")] + [InlineData("test\n\t\tline2", "test\nline2")] + [InlineData("test\n \tline2", "test\nline2")] + [InlineData(" test\nline2", " test\nline2")] + [InlineData(" test\n line2", " test\nline2")] + [InlineData("\n test\n line2", "test\nline2")] + [InlineData(" test\n line2\n\t\tline3\n line4", " test\nline2\n\tline3\n line4")] + [InlineData(" test\n Hello,\n\n world!\n ", " test\nHello,\n\n world!")] + [InlineData(" \n Hello,\r\n\n world!\n ", "Hello,\n\n world!")] + [InlineData(" \n Hello,\r\n\n wor___ld!\n ", "Hello,\n\n wor___ld!")] + [InlineData("\r\n Hello,\r\n World!\r\n\r\n Yours,\r\n GraphQL.\r\n ", "Hello,\n World!\n\nYours,\n GraphQL.")] + [InlineData("Test \\n escaping", "Test \\n escaping")] + [InlineData("Test \\u1234 escaping", "Test \\u1234 escaping")] + [InlineData("Test \\ escaping", "Test \\ escaping")] + public void Lex_BlockString_Tests(string input, string expected) + { + input = input.Replace("___", new string('_', 9000)); + expected = expected.Replace("___", new string('_', 9000)); + input = "\"\"\"" + input + "\"\"\""; + var actual = input.Lex(); + actual.Kind.ShouldBe(TokenKind.STRING); + actual.Value.ToString().ShouldBe(expected); + } + private static Token GetATPunctuationTokenLexer() { return "@".Lex(); diff --git a/src/GraphQLParser.Tests/Validation/LexerValidationTests.cs b/src/GraphQLParser.Tests/Validation/LexerValidationTests.cs index c6ffe660..98ea11a2 100644 --- a/src/GraphQLParser.Tests/Validation/LexerValidationTests.cs +++ b/src/GraphQLParser.Tests/Validation/LexerValidationTests.cs @@ -364,6 +364,20 @@ public void Lex_UnescapedControlChar_ThrowsExceptionWithCorrectMessage() exception.Column.ShouldBe(21); } + [Fact] + public void Lex_UnescapedControlChar_Blockstring_ThrowsExceptionWithCorrectMessage() + { + var exception = Should.Throw(() => "\"\"\"contains unescaped \u0007 control char".Lex()); + + exception.Message.ShouldBe( + "Syntax Error GraphQL (1:23) Invalid character within BlockString: \\u0007.\n" + + "1: \"\"\"contains unescaped \\u0007 control char\n" + + " ^\n"); + exception.Description.ShouldBe("Invalid character within BlockString: \\u0007."); + exception.Line.ShouldBe(1); + exception.Column.ShouldBe(23); + } + [Fact] public void Lex_UnterminatedString_ThrowsExceptionWithCorrectMessage() { @@ -392,39 +406,32 @@ public void Lex_UnterminatedStringWithText_ThrowsExceptionWithCorrectMessage() exception.Column.ShouldBe(14); } - [Theory] - [InlineData("test", "test")] - [InlineData("te\\\"\"\"st", "te\"\"\"st")] - [InlineData("\ntest", "test")] - [InlineData("\r\ntest", "test")] - [InlineData(" \ntest", "test")] - [InlineData("\t\ntest", "test")] - [InlineData("\n\ntest", "test")] - [InlineData("test\nline2", "test\nline2")] - [InlineData("test\rline2", "test\nline2")] - [InlineData("test\r\nline2", "test\nline2")] - [InlineData("test\r\r\nline2", "test\n\nline2")] - [InlineData("test\r\n\nline2", "test\n\nline2")] - [InlineData("test\n", "test")] - [InlineData("test\n ", "test")] - [InlineData("test\n\t", "test")] - [InlineData("test\n\n", "test")] - [InlineData("test\n line2", "test\nline2")] - [InlineData("test\n\t\tline2", "test\nline2")] - [InlineData("test\n \tline2", "test\nline2")] - [InlineData(" test\n line2", " test\nline2")] - [InlineData(" test\n line2\n\t\tline3\n line4", " test\nline2\n\tline3\n line4")] - [InlineData(" test\n Hello,\n\n world!\n ", " test\nHello,\n\n world!")] - [InlineData(" \n Hello,\r\n\n world!\n ", "Hello,\n\n world!")] - [InlineData(" \n Hello,\r\n\n wor___ld!\n ", "Hello,\n\n wor___ld!")] - public void Lex_BlockString_Tests(string input, string expected) + [Fact] + public void Lex_UnterminatedBlockString_ThrowsExceptionWithCorrectMessage() + { + var exception = Should.Throw(() => "\"\"\"".Lex()); + + exception.Message.ShouldBe( + "Syntax Error GraphQL (1:4) Unterminated string.\n" + + "1: \"\"\"\n" + + " ^\n"); + exception.Description.ShouldBe("Unterminated string."); + exception.Line.ShouldBe(1); + exception.Column.ShouldBe(4); + } + + [Fact] + public void Lex_UnterminatedBlockStringWithText_ThrowsExceptionWithCorrectMessage() { - input = input.Replace("___", new string('_', 9000)); - expected = expected.Replace("___", new string('_', 9000)); - input = "\"\"\"" + input + "\"\"\""; - var actual = input.Lex(); - actual.Kind.ShouldBe(TokenKind.BLOCKSTRING); - actual.Value.ToString().ShouldBe(expected); + var exception = Should.Throw(() => "\"\"\"no end triple-quote\"\"".Lex()); + + exception.Message.ShouldBe( + "Syntax Error GraphQL (1:25) Unterminated string.\n" + + "1: \"\"\"no end triple-quote\"\"\n" + + " ^\n"); + exception.Description.ShouldBe("Unterminated string."); + exception.Line.ShouldBe(1); + exception.Column.ShouldBe(25); } } } diff --git a/src/GraphQLParser/LexerContext.cs b/src/GraphQLParser/LexerContext.cs index ccd0fcf1..b764381b 100644 --- a/src/GraphQLParser/LexerContext.cs +++ b/src/GraphQLParser/LexerContext.cs @@ -288,88 +288,91 @@ private Token ReadBlockString() return new Token ( - TokenKind.BLOCKSTRING, + TokenKind.STRING, value, start, _currentIndex + 1 ); - ROM ProcessBuffer(Span buffer) + static ROM ProcessBuffer(Span buffer) { - //scan string to determine maximum valid commonIndent value, number of initial blank lines, and number of trailing blank lines + //scan string to determine maximum valid commonIndent value, + //number of initial blank lines, and number of trailing blank lines int commonIndent = int.MaxValue; - int line = 0; - int whitespace = 0; - bool allWhitespace = true; int initialBlankLines = 1; - bool reachedCharacter = false; - int trailingBlankLines = 0; - for (int index = 0; index < buffer.Length; index++) + int skipLinesAfter; //skip all text after line ###, as determined by the number of trailing blank lines { - code = buffer[index]; - if (code == '\n') - { - if (allWhitespace) - trailingBlankLines += 1; - if (line != 0 && !allWhitespace && whitespace < commonIndent) - commonIndent = whitespace; - line++; - whitespace = 0; - allWhitespace = true; - if (!reachedCharacter) - initialBlankLines++; - } - else if (code == ' ' || code == '\t') + int trailingBlankLines = 0; + int line = 0; + int whitespace = 0; + bool allWhitespace = true; + bool reachedCharacter = false; + for (int index = 0; index < buffer.Length; index++) { - if (allWhitespace) - whitespace++; - } - else - { - allWhitespace = false; - if (!reachedCharacter) - initialBlankLines--; - reachedCharacter = true; - trailingBlankLines = 0; + char code = buffer[index]; + if (code == '\n') + { + if (allWhitespace) + trailingBlankLines += 1; + if (line != 0 && !allWhitespace && whitespace < commonIndent) + commonIndent = whitespace; + line++; + whitespace = 0; + allWhitespace = true; + if (!reachedCharacter) + initialBlankLines++; + } + else if (code == ' ' || code == '\t') + { + if (allWhitespace) + whitespace++; + } + else + { + allWhitespace = false; + if (!reachedCharacter) + initialBlankLines--; + reachedCharacter = true; + trailingBlankLines = 0; + } } + if (allWhitespace) + trailingBlankLines += 1; + if (line != 0 && !allWhitespace && whitespace < commonIndent) + commonIndent = whitespace; + if (commonIndent == int.MaxValue) + commonIndent = 0; + int lines = line + 1; + skipLinesAfter = lines - trailingBlankLines; } - if (allWhitespace) - trailingBlankLines += 1; - if (line != 0 && !allWhitespace && whitespace < commonIndent) - commonIndent = whitespace; - if (commonIndent == int.MaxValue) - commonIndent = 0; - int lines = line + 1; - int skipLinesAfter = lines - trailingBlankLines; - //outputs: - // commonIndent - // initialBlankLines - // skipLinesAfter //step through the input, skipping the initial blank lines and the trailing blank lines, //and skipping the initial blank characters from the start of each line Span output = buffer.Length <= 4096 ? stackalloc char[buffer.Length] : new char[buffer.Length]; int outputIndex = 0; - line = 0; - int col = 0; - for (int index = 0; index < buffer.Length; index++) { - code = buffer[index]; - if (code == '\n') - { - if (++line >= skipLinesAfter) - break; - col = 0; - if (line > initialBlankLines) - output[outputIndex++] = code; - } - else + int line = 0; + int col = 0; + for (int index = 0; index < buffer.Length; index++) { - if (line >= initialBlankLines && (line == 0 || col++ >= commonIndent)) - output[outputIndex++] = code; + char code = buffer[index]; + if (code == '\n') + { + if (++line >= skipLinesAfter) + break; + col = 0; + if (line > initialBlankLines) + output[outputIndex++] = code; + } + else + { + if (line >= initialBlankLines && (line == 0 || col++ >= commonIndent)) + output[outputIndex++] = code; + } } } + //return the string value from the output buffer return output.Slice(0, outputIndex).ToString(); } } diff --git a/src/GraphQLParser/Token.cs b/src/GraphQLParser/Token.cs index 234e00ed..97fd7988 100644 --- a/src/GraphQLParser/Token.cs +++ b/src/GraphQLParser/Token.cs @@ -73,7 +73,6 @@ public Token(TokenKind kind, ROM value, int start, int end) TokenKind.INT => "Int", TokenKind.FLOAT => "Float", TokenKind.STRING => "String", - TokenKind.BLOCKSTRING => "BlockString", TokenKind.COMMENT => "#", TokenKind.UNKNOWN => "Unknown", _ => throw new NotSupportedException(kind.ToString()) @@ -84,7 +83,6 @@ public Token(TokenKind kind, ROM value, int start, int end) Kind == TokenKind.INT || Kind == TokenKind.FLOAT || Kind == TokenKind.STRING || - Kind == TokenKind.BLOCKSTRING || Kind == TokenKind.COMMENT || Kind == TokenKind.UNKNOWN; diff --git a/src/GraphQLParser/TokenKind.cs b/src/GraphQLParser/TokenKind.cs index 31f70c80..2f5951d1 100644 --- a/src/GraphQLParser/TokenKind.cs +++ b/src/GraphQLParser/TokenKind.cs @@ -93,9 +93,21 @@ public enum TokenKind FLOAT = 17, /// + /// A string value, encoded as either a 'string' or 'block string' + ///

/// Strings are sequences of characters wrapped in double‐quotes ("). (ex. "Hello World"). /// White space and other otherwise‐ignored characters are significant within a string value. + ///

+ /// Block strings are sequences of characters wrapped in triple‐quotes ("""). White space, line terminators, + /// quote, and backslash characters may all be used unescaped to enable verbatim text. + /// Since block strings represent freeform text often used in indented positions, the string value semantics + /// of a block string excludes uniform indentation and blank initial and trailing lines. + /// Triple-quotes (""") may be escaped as \""" within the block string. No other escape sequences may be used + /// within a block string. /// + /// + /// Within a block string, line termination sequences (LF, CR, or CRLF) are always replaced with a line-feed (LF) character. + /// STRING = 18, /// @@ -116,17 +128,5 @@ public enum TokenKind /// & /// AMPERSAND = 21, - - /// - /// Block strings are sequences of characters wrapped in triple‐quotes ("""). White space, line terminators, - /// quote, and backslash characters may all be used unescaped to enable verbatim text. - /// Since block strings represent freeform text often used in indented positions, the string value semantics - /// of a block string excludes uniform indentation and blank initial and trailing lines. - /// Triple-quotes (""") may be escaped as \""" within the block string. - /// - /// - /// Line termination sequences (LF, CR, or CRLF) are always replaced with a line-feed (LF) character. - /// - BLOCKSTRING = 22, } } From b5cf8fdb270dd5e60bb34a586a47402445dbc636 Mon Sep 17 00:00:00 2001 From: Shane32 Date: Mon, 28 Jun 2021 10:16:05 -0400 Subject: [PATCH 4/5] Bump preview version --- src/Directory.Build.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Directory.Build.props b/src/Directory.Build.props index 87f8028b..f82dbc2d 100644 --- a/src/Directory.Build.props +++ b/src/Directory.Build.props @@ -1,7 +1,7 @@ - 7.1.0-preview + 7.2.0-preview 8.0 true true From 66ed1f3c844d5a4fa33f8d99808f7dbcaf1b0f69 Mon Sep 17 00:00:00 2001 From: Shane32 Date: Mon, 28 Jun 2021 10:17:08 -0400 Subject: [PATCH 5/5] Update formatting --- src/GraphQLParser/LexerContext.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/GraphQLParser/LexerContext.cs b/src/GraphQLParser/LexerContext.cs index b764381b..7e42a343 100644 --- a/src/GraphQLParser/LexerContext.cs +++ b/src/GraphQLParser/LexerContext.cs @@ -282,7 +282,8 @@ private Token ReadBlockString() sb.CopyTo(0, chars, 0, sb.Length); value = ProcessBuffer(chars); } - else { + else + { value = ProcessBuffer(buffer.Slice(0, index)); }