Dialect Crumbs (#3625)

* Dialect Crumbs

* More mypy

* Update src/sqlfluff/core/parser/grammar/base.py

* Update src/sqlfluff/core/parser/grammar/base.py

Co-authored-by: Barry Hart <barrywhart@yahoo.com>

src/sqlfluff/core/parser/grammar/anyof.py

@@ -28,13 +28,16 @@ def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
 
     @cached_method_for_parse_context
-    def simple(self, parse_context: ParseContext) -> Optional[List[str]]:
+    def simple(
+        self, parse_context: ParseContext, crumbs: Optional[List[str]] = None
+    ) -> Optional[List[str]]:
         """Does this matcher support a uppercase hash matching route?
 
         AnyNumberOf does provide this, as long as *all* the elements *also* do.
         """
         simple_buff = [
-            opt.simple(parse_context=parse_context) for opt in self._elements
+            opt.simple(parse_context=parse_context, crumbs=crumbs)
+            for opt in self._elements
         ]
         if any(elem is None for elem in simple_buff):
             return None

src/sqlfluff/core/parser/grammar/base.py

@@ -53,14 +53,19 @@ def cached_method_for_parse_context(func):
     """
     cache_key = "__cache_" + func.__name__
 
-    def wrapped_method(self, parse_context: ParseContext):
-        """Cache the output of the method against a given parse context."""
+    def wrapped_method(self, parse_context: ParseContext, **kwargs):
+        """Cache the output of the method against a given parse context.
+
+        Note: kwargs are not taken into account in the caching, but
+        for the current use case of dependency loop debugging that's
+        ok.
+        """
         cache_tuple: Tuple = self.__dict__.get(cache_key, (None, None))
         # Do we currently have a cached value?
         if cache_tuple[0] == parse_context.uuid:
             return cache_tuple[1]
         # Generate a new value, cache it and return
-        result = func(self, parse_context=parse_context)
+        result = func(self, parse_context=parse_context, **kwargs)
         self.__dict__[cache_key] = (parse_context.uuid, result)
         return result
 
@@ -107,7 +112,7 @@ def _resolve_ref(elem):
 
     def __init__(
         self,
-        *args,
+        *args: Union[MatchableType, str],
         allow_gaps=True,
         optional=False,
         ephemeral_name=None,
@@ -116,7 +121,9 @@ def __init__(
 
         Args:
             *args: Any number of elements which because the subjects
-                of this grammar.
+                of this grammar. Optionally these elements may also be
+                string references to elements rather than the Matchable
+                elements themselves.
             allow_gaps (:obj:`bool`, optional): Does this instance of the
                 grammar allow gaps between the elements it matches? This
                 may be exhibited slightly differently in each grammar. See
@@ -174,7 +181,7 @@ def match(self, segments: Tuple["BaseSegment", ...], parse_context: ParseContext
         )  # pragma: no cover
 
     @cached_method_for_parse_context
-    def simple(self, parse_context: ParseContext) -> Optional[List[str]]:
+    def simple(self, parse_context: ParseContext, crumbs=None) -> Optional[List[str]]:
         """Does this matcher support a lowercase hash matching route?"""
         return None
 
@@ -795,27 +802,40 @@ class Ref(BaseGrammar):
     # and it also causes infinite recursion.
     allow_keyword_string_refs = False
 
-    def __init__(self, *args, **kwargs):
+    def __init__(self, *args: str, **kwargs):
         # Any patterns to _prevent_ a match.
         self.exclude = kwargs.pop("exclude", None)
         super().__init__(*args, **kwargs)
 
     @cached_method_for_parse_context
-    def simple(self, parse_context: ParseContext) -> Optional[List[str]]:
+    def simple(
+        self, parse_context: ParseContext, crumbs: Optional[Tuple[str]] = None
+    ) -> Optional[List[str]]:
         """Does this matcher support a uppercase hash matching route?
 
         A ref is simple, if the thing it references is simple.
         """
+        ref = self._get_ref()
+        if crumbs and ref in crumbs:  # pragma: no cover
+            loop = " -> ".join(crumbs)
+            raise RecursionError(f"Self referential grammar detected: {loop}")
         return self._get_elem(dialect=parse_context.dialect).simple(
-            parse_context=parse_context
+            parse_context=parse_context,
+            crumbs=(crumbs or ()) + (ref,),
         )
 
-    def _get_ref(self):
+    def _get_ref(self) -> str:
         """Get the name of the thing we're referencing."""
         # Unusually for a grammar we expect _elements to be a list of strings.
         # Notable ONE string for now.
         if len(self._elements) == 1:
             # We're good on length. Get the name of the reference
+            ref = self._elements[0]
+            if not isinstance(ref, str):  # pragma: no cover
+                raise ValueError(
+                    "Ref Grammar expects elements to be strings. "
+                    f"Found {ref!r} instead."
+                )
             return self._elements[0]
         else:  # pragma: no cover
             raise ValueError(

src/sqlfluff/core/parser/grammar/greedy.py

@@ -170,12 +170,12 @@ def __init__(self, target, *args, **kwargs):
         super().__init__(*args, **kwargs)
 
     @cached_method_for_parse_context
-    def simple(self, parse_context: ParseContext) -> Optional[List[str]]:
+    def simple(self, parse_context: ParseContext, crumbs=None) -> Optional[List[str]]:
         """Does this matcher support a uppercase hash matching route?
 
         `StartsWith` is simple, if the thing it starts with is also simple.
         """
-        return self.target.simple(parse_context=parse_context)
+        return self.target.simple(parse_context=parse_context, crumbs=crumbs)
 
     @match_wrapper()
     def match(self, segments, parse_context):

src/sqlfluff/core/parser/grammar/noncode.py

@@ -15,7 +15,7 @@
 class NonCodeMatcher(Matchable):
     """An object which behaves like a matcher to match non-code."""
 
-    def simple(self, parse_context: ParseContext) -> Optional[List[str]]:
+    def simple(self, parse_context: ParseContext, crumbs=None) -> Optional[List[str]]:
         """This element doesn't work with simple."""
         return None
 

src/sqlfluff/core/parser/grammar/sequence.py

@@ -30,15 +30,15 @@ class Sequence(BaseGrammar):
     test_env = getenv("SQLFLUFF_TESTENV", "")
 
     @cached_method_for_parse_context
-    def simple(self, parse_context: ParseContext) -> Optional[List[str]]:
+    def simple(self, parse_context: ParseContext, crumbs=None) -> Optional[List[str]]:
         """Does this matcher support a uppercase hash matching route?
 
         Sequence does provide this, as long as the *first* non-optional
         element does, *AND* and optional elements which preceded it also do.
         """
         simple_buff = []
         for opt in self._elements:
-            simple = opt.simple(parse_context=parse_context)
+            simple = opt.simple(parse_context=parse_context, crumbs=crumbs)
             if not simple:
                 return None
             simple_buff += simple
@@ -208,13 +208,13 @@ def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
 
     @cached_method_for_parse_context
-    def simple(self, parse_context: ParseContext) -> Optional[List[str]]:
+    def simple(self, parse_context: ParseContext, crumbs=None) -> Optional[List[str]]:
         """Does this matcher support a uppercase hash matching route?
 
         Bracketed does this easily, we just look for the bracket.
         """
         start_bracket, _, _ = self.get_bracket_from_dialect(parse_context)
-        return start_bracket.simple(parse_context=parse_context)
+        return start_bracket.simple(parse_context=parse_context, crumbs=crumbs)
 
     def get_bracket_from_dialect(self, parse_context):
         """Rehydrate the bracket segments in question."""

src/sqlfluff/core/parser/matchable.py

@@ -2,7 +2,7 @@
 
 import copy
 from abc import ABC, abstractmethod
-from typing import List, Optional, TYPE_CHECKING
+from typing import List, Optional, Tuple, TYPE_CHECKING
 
 
 if TYPE_CHECKING:  # pragma: no cover
@@ -18,8 +18,14 @@ def is_optional(self) -> bool:
         """Return whether this element is optional."""
 
     @abstractmethod
-    def simple(self, parse_context: "ParseContext") -> Optional[List[str]]:
-        """Try to obtain a simple response from the matcher."""
+    def simple(
+        self, parse_context: "ParseContext", crumbs: Optional[Tuple[str, ...]] = None
+    ) -> Optional[List[str]]:
+        """Try to obtain a simple response from the matcher.
+
+        NOTE: the crumbs kwarg is designed to be used by Ref to
+        detect recursion.
+        """
 
     @abstractmethod
     def match(self, segments: tuple, parse_context: "ParseContext") -> "MatchResult":

src/sqlfluff/core/parser/parsers.py

@@ -111,7 +111,7 @@ def __init__(
             **segment_kwargs,
         )
 
-    def simple(self, parse_context: "ParseContext") -> Optional[List[str]]:
+    def simple(self, parse_context: "ParseContext", crumbs=None) -> Optional[List[str]]:
         """Return simple options for this matcher.
 
         Because string matchers are not case sensitive we can
@@ -151,7 +151,7 @@ def __init__(
             **segment_kwargs,
         )
 
-    def simple(self, parse_context: "ParseContext") -> Optional[List[str]]:
+    def simple(self, parse_context: "ParseContext", crumbs=None) -> Optional[List[str]]:
         """Return simple options for this matcher.
 
         Because string matchers are not case sensitive we can
@@ -189,7 +189,7 @@ def __init__(
             **segment_kwargs,
         )
 
-    def simple(cls, parse_context: ParseContext) -> Optional[List[str]]:
+    def simple(cls, parse_context: ParseContext, crumbs=None) -> Optional[List[str]]:
         """Does this matcher support a uppercase hash matching route?
 
         NamedParser segment does NOT for now. We might need to later for efficiency.
@@ -240,7 +240,7 @@ def __init__(
             **segment_kwargs,
         )
 
-    def simple(cls, parse_context: ParseContext) -> Optional[List[str]]:
+    def simple(cls, parse_context: ParseContext, crumbs=None) -> Optional[List[str]]:
         """Does this matcher support a uppercase hash matching route?
 
         Regex segment does NOT for now. We might need to later for efficiency.

src/sqlfluff/core/parser/segments/base.py

@@ -602,15 +602,15 @@ def _position_segments(
     # ################ CLASS METHODS
 
     @classmethod
-    def simple(cls, parse_context: ParseContext) -> Optional[List[str]]:
+    def simple(cls, parse_context: ParseContext, crumbs=None) -> Optional[List[str]]:
         """Does this matcher support an uppercase hash matching route?
 
         This should be true if the MATCH grammar is simple. Most more
         complicated segments will be assumed to overwrite this method
         if they wish to be considered simple.
         """
         if cls.match_grammar:
-            return cls.match_grammar.simple(parse_context=parse_context)
+            return cls.match_grammar.simple(parse_context=parse_context, crumbs=crumbs)
         else:  # pragma: no cover TODO?
             # Other segments will either override this method, or aren't
             # simple.
@@ -1563,7 +1563,7 @@ def __init__(
         super().__init__(*args, **kwargs)
 
     @classmethod
-    def simple(cls, parse_context: ParseContext) -> Optional[List[str]]:
+    def simple(cls, parse_context: ParseContext, crumbs=None) -> Optional[List[str]]:
         """Simple methods for bracketed and the persitent brackets."""
         start_brackets = [
             start_bracket
@@ -1574,7 +1574,9 @@ def simple(cls, parse_context: ParseContext) -> Optional[List[str]]:
         ]
         start_simple = []
         for ref in start_brackets:
-            start_simple += parse_context.dialect.ref(ref).simple(parse_context)
+            start_simple += parse_context.dialect.ref(ref).simple(
+                parse_context, crumbs=crumbs
+            )
         return start_simple
 
     @classmethod

src/sqlfluff/core/parser/segments/meta.py

@@ -39,7 +39,7 @@ def match(cls, segments, parse_context):  # pragma: no cover
         )
 
     @classmethod
-    def simple(cls, parse_context: ParseContext) -> Optional[List[str]]:
+    def simple(cls, parse_context: ParseContext, crumbs=None) -> Optional[List[str]]:
         """Does this matcher support an uppercase hash matching route?
 
         This should be true if the MATCH grammar is simple. Most more