Bleaphar/comment node investigation (#3492)

* Change for comment node

* Some refinements

* removal of offset

* fix for comments

* more precise testing

* Inclusion and refining of tests

src/Microsoft.DotNet.Interactive.Http.Parsing/Parsing/HttpRequestParser.cs

@@ -396,6 +396,18 @@ private void ConsumeCurrentTokenInto(HttpSyntaxNode node)
                 ConsumeCurrentTokenInto(node);
                 ConsumeCurrentTokenInto(node);
 
+                while (MoreTokens())
+                {
+                    if (CurrentToken?.Kind is not HttpTokenKind.NewLine)
+                    {
+                        ConsumeCurrentTokenInto(node);
+                    }
+                    else
+                    {
+                        break;
+                    }
+                }
+
                 return ParseTrailingWhitespace(node);
             }
 
@@ -462,6 +474,11 @@ private void ConsumeCurrentTokenInto(HttpSyntaxNode node)
                     }
                 }
 
+                if (!((CurrentToken?.Kind is HttpTokenKind.Word or HttpTokenKind.Punctuation) || (IsAtStartOfEmbeddedExpression())))
+                {
+                    break;
+                }
+
                 if (IsAtStartOfEmbeddedExpression())
                 {
                     node.Add(ParseEmbeddedExpression());
@@ -742,7 +759,7 @@ private IEnumerable<HttpCommentNode> ParseComments()
 
         private HttpCommentBodyNode? ParseCommentBody()
         {
-            if (!MoreTokens())
+            if (!MoreTokens() || CurrentToken is { Kind: HttpTokenKind.NewLine })
             {
                 return null;
             }
@@ -767,7 +784,7 @@ private IEnumerable<HttpCommentNode> ParseComments()
             {
                 var node = new HttpCommentStartNode(_sourceText, _syntaxTree);
                 ConsumeCurrentTokenInto(node);
-                return ParseTrailingWhitespace(node);
+                return ParseTrailingWhitespace(node, stopBeforeNewLine: true);
             }
 
             if (CurrentToken is { Kind: HttpTokenKind.Punctuation } and { Text: "/" } &&
@@ -777,7 +794,7 @@ private IEnumerable<HttpCommentNode> ParseComments()
 
                 ConsumeCurrentTokenInto(node);
                 ConsumeCurrentTokenInto(node);
-                return ParseTrailingWhitespace(node);
+                return ParseTrailingWhitespace(node, stopBeforeNewLine: true);
             }
 
             return null;

src/Microsoft.DotNet.Interactive.Http.Tests/HttpKernelTests.cs

@@ -313,7 +313,34 @@ public async Task binding_in_variable_value_is_valid()
         result.Events.Should().NotContainErrors();
         request.RequestUri.Should().Be($"https://httpbin.org");
 
+    }
+
+    [Fact]
+    public async Task binding_in_variable_value_with_request_separator()
+    {
+        HttpRequestMessage request = null;
+        var handler = new InterceptingHttpMessageHandler((message, _) =>
+        {
+            request = message;
+            var response = new HttpResponseMessage(HttpStatusCode.OK);
+            return Task.FromResult(response);
+        });
+        var client = new HttpClient(handler);
+        using var kernel = new HttpKernel(client: client);
+
+        using var _ = new AssertionScope();
+
+        var code = """
+            @host=https://microsoft.com
+
+            Post {{host}}
+            ###
+            """;
+
+        var result = await kernel.SendAsync(new SubmitCode(code));
 
+        result.Events.Should().NotContainErrors();
+        request.RequestUri.Should().Be($"https://microsoft.com");
     }
 
     [Fact]
@@ -1336,6 +1363,41 @@ public async Task can_bind_datetime_with_offset()
 
     }
 
+    [Fact]
+    public async Task can_bind_datetime_with_positive_offset_and_no_plus()
+    {
+        HttpRequestMessage request = null;
+        var handler = new InterceptingHttpMessageHandler((message, _) =>
+        {
+            request = message;
+            var response = new HttpResponseMessage(HttpStatusCode.OK);
+            return Task.FromResult(response);
+        });
+        var client = new HttpClient(handler);
+        using var kernel = new HttpKernel(client: client);
+
+        using var _ = new AssertionScope();
+
+        var code = """
+            POST https://api.example.com/comments
+
+            {
+                "custom_date" : "{{$datetime 'yyyy-MM-dd' 1 d}}"
+            }
+            """;
+
+        var result = await kernel.SendAsync(new SubmitCode(code));
+        result.Events.Should().NotContainErrors();
+
+        var offsetDate = DateTime.UtcNow.AddDays(1.0).ToString("yyyy-MM-dd");
+        var bodyAsString = await request.Content.ReadAsStringAsync();
+        var readDateSubstring = bodyAsString.Split(":").Last().Trim().Substring(1);
+        var readDateValue = readDateSubstring.Substring(0, readDateSubstring.IndexOf("\""));
+
+        readDateValue.Should().BeEquivalentTo(offsetDate);
+
+    }
+
     [Fact]
     public async Task can_bind_datetime_with_no_arguments()
     {

src/Microsoft.DotNet.Interactive.Http.Tests/ParserTests.Request.cs

@@ -135,6 +135,29 @@ public void request_with_forward_slash_at_beginning_of_nodes_works()
             bindingResult.Value.RequestUri.ToString().Should().Be("https://httpbin.org/anything");
         }
 
+        [Fact]
+        public void single_request_with_comment_and_request_separator_parsed_correctly()
+        {
+            var result = Parse(
+                """
+                #
+                @host=https://httpbin.org
+
+                Post {{host}}
+                ###
+                """);
+
+            var requestNode = result.SyntaxTree.RootNode.ChildNodes.Should().ContainSingle<HttpRequestNode>().Which;
+
+            var bindingResult = requestNode.TryGetHttpRequestMessage(node =>
+            {
+                return node.CreateBindingFailure(CreateDiagnosticInfo(""));
+            });
+
+            bindingResult.IsSuccessful.Should().BeTrue();
+            bindingResult.Value.RequestUri.ToString().Should().Be("https://httpbin.org/");
+        }
+
         [Fact]
         public void request_node_containing_method_and_url_and_no_variable_expressions_returns_HttpRequestMessage_with_specified_method()
         {

src/Microsoft.DotNet.Interactive.Http.Tests/ParserTests.RequestSeparator.cs

@@ -4,6 +4,7 @@
 using FluentAssertions;
 using Microsoft.DotNet.Interactive.Http.Parsing;
 using Microsoft.DotNet.Interactive.Http.Tests.Utility;
+using System.Linq;
 using Xunit;
 
 namespace Microsoft.DotNet.Interactive.Http.Tests;
@@ -28,5 +29,19 @@ public void request_separator_at_start_of_request_is_valid()
                   .Which.UrlNode.Text.Should().Be("https://example.com");
         }
 
+        [Fact]
+        public void tokens_on_request_separator_nodes_are_parsed_into_request_separator()
+        {
+            var result = Parse(
+                """
+                ### Slow Response (Json)
+                GET https://httpbin.org/anything?page=2&pageSize=10
+                """
+                );
+
+            var requestNodes = result.SyntaxTree.RootNode.ChildNodes
+                                    .Should().ContainSingle<HttpRequestSeparatorNode>()
+                                    .Which.Text.Should().Be("### Slow Response (Json)");
+        }
     }
 }

src/Microsoft.DotNet.Interactive.Http.Tests/ParserTests.Url.cs

@@ -160,5 +160,33 @@ public void Punctuation_at_url_start_produces_a_diagnostic()
                                    .Which.GetDiagnostics().Should().ContainSingle()
                                    .Which;
         }
+
+        [Fact]
+        public void Request_separator_following_embedded_expression_is_parsed_correctly()
+        {
+            var code = """
+                GET {{host}}
+                ###
+                """;
+
+            var result = Parse(code);
+
+            result.GetDiagnostics().Should().BeEmpty();
+
+            var children = result.SyntaxTree.RootNode.ChildNodes;
+
+            var requestNode = children.OfType<HttpRequestNode>();
+            var requestSeparator = children.OfType<HttpRequestSeparatorNode>();
+
+            requestNode.Should().HaveCount(1);
+            requestSeparator.Should().HaveCount(1);
+
+            children.Count().Should().Be(2);
+
+            requestNode.Single().Span.Should().BeLessThan(requestSeparator.Single().Span);
+
+
+
+        }
     }
 }