From 3120b7df6933853ad171eba331c15f67d2d45db6 Mon Sep 17 00:00:00 2001
From: Luca Comellini <luca.comellini@nginx.com>
Date: Tue, 1 May 2018 15:27:56 -0700
Subject: [PATCH] Treat quotes inside a token like any other char

---
 crossplane/lexer.py                     | 9 ++-------
 tests/configs/quote-behavior/nginx.conf | 2 +-
 tests/test_lex.py                       | 2 +-
 3 files changed, 4 insertions(+), 9 deletions(-)

diff --git a/crossplane/lexer.py b/crossplane/lexer.py
index 4ee01f1..8a759ca 100644
--- a/crossplane/lexer.py
+++ b/crossplane/lexer.py
@@ -28,7 +28,6 @@ def _lex_file_object(file_obj):
     token = ''  # the token buffer
     token_line = 0  # the line the token starts on
     next_token_is_directive = True
-    quote_inside_token = False
 
     it = itertools.chain.from_iterable(file_obj)
     it = _iterescape(it)  # treat escaped characters differently
@@ -73,9 +72,10 @@ def _lex_file_object(file_obj):
 
         # if a quote is found, add the whole string to the token buffer
         if char in ('"', "'"):
+            # if a quote is inside a token, treat it like any other char
             if token:
                 token += char
-                quote_inside_token = True
+                continue
 
             quote = char
             char, line = next(it)
@@ -83,11 +83,6 @@ def _lex_file_object(file_obj):
                 token += quote if char == '\\' + quote else char
                 char, line = next(it)
 
-            if quote_inside_token:
-                token += quote
-                quote_inside_token = False
-                continue
-
             yield (token, token_line)
             if next_token_is_directive and token in EXTERNAL_LEXERS:
                 for custom_lexer_token in EXTERNAL_LEXERS[token](it, token):
diff --git a/tests/configs/quote-behavior/nginx.conf b/tests/configs/quote-behavior/nginx.conf
index b4d3960..5b0103c 100644
--- a/tests/configs/quote-behavior/nginx.conf
+++ b/tests/configs/quote-behavior/nginx.conf
@@ -1,2 +1,2 @@
 "outer-quote" "left"-quote right-"quote" inner"-"quote;
-"" ""left-empty right-empty"" inner""empty;
+"" ""left-empty right-empty"" inner""empty right-empty-single";
diff --git a/tests/test_lex.py b/tests/test_lex.py
index 9e85dcd..b459531 100644
--- a/tests/test_lex.py
+++ b/tests/test_lex.py
@@ -75,5 +75,5 @@ def test_quote_behavior():
     tokens = crossplane.lex(config)
     assert list(token for token, line in tokens) == [
         'outer-quote', 'left', '-quote', 'right-"quote"', 'inner"-"quote', ';',
-        '', '', 'left-empty', 'right-empty""', 'inner""empty', ';',
+        '', '', 'left-empty', 'right-empty""', 'inner""empty', 'right-empty-single"', ';',
     ]