Deduplicate files before and during templating (#3629)

* Deduplicate in linter

* Deduplicate in templater

* Linting

* add more logging

* linting

* coverage

Co-authored-by: Barry Hart <barrywhart@yahoo.com>

plugins/sqlfluff-templater-dbt/sqlfluff_templater_dbt/templater.py

@@ -307,6 +307,12 @@ def sequence_files(
         for fname in fnames:
             if fname not in already_yielded:
                 yield fname
+                # Dedupe here so we don't yield twice
+                already_yielded.add(fname)
+            else:
+                templater_logger.debug(
+                    "- Skipping yield of previously sequenced file: %r", fname
+                )
 
     @large_file_check
     def process(self, *, fname, in_str=None, config=None, formatter=None):
@@ -450,6 +456,9 @@ def make_template(in_str):
             return old_from_string(*args, **kwargs)
 
         node = self._find_node(fname, config)
+        templater_logger.debug(
+            "_find_node for path %r returned object of type %s.", fname, type(node)
+        )
 
         save_ephemeral_nodes = dict(
             (k, v)

src/sqlfluff/core/linter/linter.py

@@ -1004,7 +1004,8 @@ def paths_from_path(
             return sorted(buffer)
 
         # Check the buffer for ignore items and normalise the rest.
-        filtered_buffer = []
+        # It's a set, so we can do natural deduplication.
+        filtered_buffer = set()
 
         for fpath in buffer:
             abs_fpath = os.path.abspath(fpath)
@@ -1031,9 +1032,18 @@ def paths_from_path(
                         )
                     break
             else:
-                filtered_buffer.append(os.path.normpath(fpath))
+                npath = os.path.normpath(fpath)
+                # For debugging, log if we already have the file.
+                if npath in filtered_buffer:
+                    linter_logger.debug(  # pragma: no cover
+                        "Developer Warning: Path crawler attempted to "
+                        "requeue the same file twice. %s is already in "
+                        "filtered buffer.",
+                        npath,
+                    )
+                filtered_buffer.add(npath)
 
-        # Return
+        # Return a sorted list
         return sorted(filtered_buffer)
 
     def lint_string_wrapped(