Parsing restructuring

src/Microsoft.DotNet.Interactive.Http.Parsing/Parsing/HttpRequestParser.cs

@@ -46,35 +46,38 @@ public HttpSyntaxTree Parse()
 
             while (MoreTokens())
             {
-                commentsToPrepend.AddRange(ParseComments());
-
-                if (ParseVariableDeclarations() is { } variableNodes)
+                while (GetNextSignificantToken() is not null)
                 {
-                    foreach (var variableNode in variableNodes)
+                    commentsToPrepend.AddRange(ParseComments());
+
+                    if (ParseVariableDeclarations() is { } variableNodes)
                     {
-                        foreach (var comment in commentsToPrepend)
+                        foreach (var variableNode in variableNodes)
                         {
-                            variableNode.Add(comment, addBefore: true);
+                            foreach (var comment in commentsToPrepend)
+                            {
+                                variableNode.Add(comment, addBefore: true);
+                            }
+                            commentsToPrepend.Clear();
+                            _syntaxTree.RootNode.Add(variableNode);
                         }
-                        commentsToPrepend.Clear();
-                        _syntaxTree.RootNode.Add(variableNode);
                     }
-                }
-
-                if (ParseRequest() is { } requestNode)
-                {
-                    foreach (var comment in commentsToPrepend)
+                    if (ParseRequestSeparator() is { } separatorNode)
                     {
-                        requestNode.Add(comment, addBefore: true);
+                        _syntaxTree.RootNode.Add(separatorNode);
+                    }
+                    if (ParseRequest() is { } requestNode)
+                    {
+                        foreach (var comment in commentsToPrepend)
+                        {
+                            requestNode.Add(comment, addBefore: true);
+                        }
+                        commentsToPrepend.Clear();
+                        _syntaxTree.RootNode.Add(requestNode);
                     }
-                    commentsToPrepend.Clear();
-                    _syntaxTree.RootNode.Add(requestNode);
                 }
 
-                if (ParseRequestSeparator() is { } separatorNode)
-                {
-                    _syntaxTree.RootNode.Add(separatorNode);
-                }
+                ConsumeCurrentTokenInto(_syntaxTree.RootNode);
             }
 
             foreach (var comment in commentsToPrepend)
@@ -309,6 +312,11 @@ private T ParseTrailingWhitespace<T>(T node, bool stopAfterNewLine = false, bool
                 return null;
             }
 
+            if (GetNextSignificantToken() is null)
+            {
+                return null;
+            }
+
             var requestNode = new HttpRequestNode(
                 _sourceText,
                 _syntaxTree);

src/Microsoft.DotNet.Interactive.Http.Tests/ParserTests.Comments.cs

@@ -3,6 +3,7 @@
 
 using System;
 using System.Collections.Generic;
+using System.Linq;
 using System.Management.Automation.Internal;
 using FluentAssertions;
 using Microsoft.DotNet.Interactive.Http.Parsing;
@@ -49,7 +50,7 @@ public void comment_without_text_at_end_of_request_is_parsed_correctly()
         [Fact]
         public void Comment_node_can_immediately_follow_headers()
         {
-             var code = """
+            var code = """
                 GET https://example.com
                 Accept: text/plain
                 # This is a comment
@@ -103,5 +104,23 @@ public void Comment_node_without_request_node_does_not_produce_diagnostics(strin
             result.GetDiagnostics().Should().BeEmpty();
 
         }
+
+        [Fact]
+        public void comment_after_request_separator_is_parsed_correctly()
+        {
+            var code = """
+                @MyRestaurantApi_HostAddress = https://localhost:7094
+
+                GET {{MyRestaurantApi_HostAddress}}/api/Contact
+
+                ###
+
+                # get a specific contact
+                """;
+
+            var result = Parse(code);
+
+            result.SyntaxTree.RootNode.ChildNodes.Last().Should().BeOfType<HttpCommentNode>().Which.Text.Should().Be("# get a specific contact");
+        }
     }
 }

src/Microsoft.DotNet.Interactive.Http.Tests/ParserTests.Lexer.cs

@@ -20,11 +20,9 @@ public void multiple_whitespaces_are_treated_as_a_single_token()
             var result = Parse("  \t  ");
 
             result.SyntaxTree.RootNode
-                  .ChildNodes.Should().ContainSingle<HttpRequestNode>().Which
                   .ChildTokens.First().Should().BeOfType<HttpSyntaxToken>();
 
             result.SyntaxTree.RootNode
-                  .ChildNodes.Should().ContainSingle<HttpRequestNode>().Which
                   .ChildTokens.Single().Text.Should().Be("  \t  ");
         }
 
@@ -33,7 +31,7 @@ public void multiple_newlines_are_parsed_into_different_tokens()
         {
             var result = Parse("\n\v\r\n\n");
 
-            result.SyntaxTree.RootNode.ChildNodes.Should().ContainSingle<HttpRequestNode>().Which
+            result.SyntaxTree.RootNode
                   .ChildTokens.Select(t => new { t.Text, t.Kind }).Should().BeEquivalentSequenceTo(
                       new { Text = "\n", Kind = HttpTokenKind.NewLine },
                       new { Text = "\v", Kind = HttpTokenKind.NewLine },
@@ -45,7 +43,7 @@ public void multiple_newlines_are_parsed_into_different_tokens()
         public void multiple_punctuations_are_parsed_into_different_tokens()
         {
             var result = Parse(".!?.:/");
-            
+
             var requestNode = result.SyntaxTree.RootNode.DescendantNodesAndTokens().Should().ContainSingle<HttpUrlNode>().Which;
             requestNode
                   .ChildTokens.Select(t => new { t.Text, t.Kind })

src/Microsoft.DotNet.Interactive.Http.Tests/ParserTests.RequestSeparator.cs

@@ -43,5 +43,34 @@ public void tokens_on_request_separator_nodes_are_parsed_into_request_separator(
                                     .Should().ContainSingle<HttpRequestSeparatorNode>()
                                     .Which.Text.Should().Be("### Slow Response (Json)");
         }
+
+        [Fact]
+        public void request_separator_nodes_after_various_nodes_are_parsed_correctly()
+        {
+            var result = Parse(
+                """
+                @MyRestaurantApi_HostAddress = https://localhost:7293
+
+                GET {{MyRestaurantApi_HostAddress}}/api/Contact
+                ###
+
+                @Somevar = hello
+
+                ###
+
+                PUT https://httpbin.org/anything
+                Content-Type: application/json
+
+                {
+                    "content": "content here",
+                    "message": {{Message}}
+                }
+
+                ###
+                """
+                );
+
+            result.SyntaxTree.RootNode.ChildNodes.OfType<HttpRequestSeparatorNode>().Count().Should().Be(3);
+        }
     }
 }

src/Microsoft.DotNet.Interactive.Http.Tests/ParserTests.Trivia.cs

@@ -26,17 +26,15 @@ public void it_can_parse_a_string_with_only_whitespace()
             var result = Parse(" \t ");
 
             result.SyntaxTree.RootNode
-                  .ChildNodes.Should().ContainSingle<HttpRequestNode>().Which
                   .ChildTokens.First().Text.Should().Be(" \t ");
         }
 
         [Fact]
-        public void it_can_parse_a_string_with_only_newlines()
+        public void string_with_only_newlines_is_parsed_into_root_node()
         {
             var result = Parse("\r\n\n\r\n");
 
             result.SyntaxTree.RootNode
-                  .ChildNodes.Should().ContainSingle<HttpRequestNode>().Which
                   .FullText.Should().Be("\r\n\n\r\n");
         }
     }

src/Microsoft.DotNet.Interactive.Http.Tests/ParserTests.Variables.cs

@@ -250,5 +250,21 @@ public void underscores_in_embedded_expressions_are_supported()
             variables.Should().Contain(n => n.Key == "host").Which.Value.Should().BeOfType<DeclaredVariable>().Which.Value.Should().Be("https://httpbin.org");
 
         }
+
+        [Fact]
+        public void spaces_after_variable_do_not_produce_diagnostics()
+        {
+            var result = Parse(
+                """
+                @host=https://httpbin.org
+
+
+
+
+
+                """);
+
+            result.SyntaxTree.RootNode.ChildNodes.Count().Should().Be(1);
+        }
     }
 }