Adds experimental stdin support

src/ansiblelint/file_utils.py

@@ -3,10 +3,12 @@
 import logging
 import os
 import subprocess
+import sys
 from argparse import Namespace
 from collections import OrderedDict
 from contextlib import contextmanager
 from pathlib import Path
+from tempfile import NamedTemporaryFile
 from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Optional, Set, Union
 
 import wcmatch.pathlib
@@ -83,6 +85,9 @@ def kind_from_path(path: Path):
                 return k
     if path.is_dir():
         return "role"
+
+    if str(path) == '/dev/stdin':
+        return "playbook"
     raise RuntimeError("Unable to determine file type for %s" % pathex)
 
 
@@ -100,6 +105,10 @@ def __init__(
         kind: Optional[FileType] = None,
     ):
         """Create a Lintable instance."""
+        # Filename is effective file on disk, for stdin is a namedtempfile
+        self.filename: str = str(name)
+        self.dir: str = ""
+
         if isinstance(name, str):
             self.name = normpath(name)
             self.path = Path(self.name)
@@ -118,12 +127,24 @@ def __init__(
             if role.exists:
                 self.role = role.name
 
-        self.kind = kind or kind_from_path(self.path)
-        # We store absolute directory in dir
-        if self.kind == "role":
-            self.dir = str(self.path.resolve())
+        if str(self.path) in ['/dev/stdin', '-']:
+            self.file = NamedTemporaryFile(mode="w+", suffix="playbook.yml")
+            self.filename = self.file.name
+            self._content = sys.stdin.read()
+            self.file.write(self._content)
+            self.file.flush()
+            self.path = Path(self.file.name)
+            self.name = 'stdin'
+            self.kind = 'playbook'
+            self.dir = '/'
         else:
-            self.dir = str(self.path.parent.resolve())
+            self.kind = kind or kind_from_path(self.path)
+        # We store absolute directory in dir
+        if not self.dir:
+            if self.kind == "role":
+                self.dir = str(self.path.resolve())
+            else:
+                self.dir = str(self.path.parent.resolve())
 
     def __getitem__(self, item):
         """Provide compatibility subscriptable support."""

src/ansiblelint/rules/__init__.py

@@ -91,11 +91,11 @@ def matchtasks(self, file: Lintable) -> List[MatchError]:
         if not self.matchtask or file.kind == 'meta':
             return matches
 
-        yaml = ansiblelint.utils.parse_yaml_linenumbers(file.content, file.path)
+        yaml = ansiblelint.utils.parse_yaml_linenumbers(file)
         if not yaml:
             return matches
 
-        yaml = append_skipped_rules(yaml, file.content, file.kind)
+        yaml = append_skipped_rules(yaml, file)
 
         try:
             tasks = ansiblelint.utils.get_normalized_tasks(yaml, file)
@@ -138,7 +138,7 @@ def matchyaml(self, file: Lintable) -> List[MatchError]:
         if not self.matchplay:
             return matches
 
-        yaml = ansiblelint.utils.parse_yaml_linenumbers(file.content, file.path)
+        yaml = ansiblelint.utils.parse_yaml_linenumbers(file)
         # yaml returned can be an AnsibleUnicode (a string) when the yaml
         # file contains a single string. YAML spec allows this but we consider
         # this an fatal error.
@@ -150,9 +150,7 @@ def matchyaml(self, file: Lintable) -> List[MatchError]:
         if isinstance(yaml, dict):
             yaml = [yaml]
 
-        yaml = ansiblelint.skip_utils.append_skipped_rules(
-            yaml, file.content, file.kind
-        )
+        yaml = ansiblelint.skip_utils.append_skipped_rules(yaml, file)
 
         for play in yaml:
 

src/ansiblelint/runner.py

@@ -26,7 +26,7 @@ class LintResult:
     """Class that tracks result of linting."""
 
     matches: List[MatchError]
-    files: Set[str]
+    files: Set[Lintable]
 
 
 class Runner:
@@ -41,7 +41,7 @@ def __init__(
         skip_list: List[str] = [],
         exclude_paths: List[str] = [],
         verbosity: int = 0,
-        checked_files: Optional[Set[str]] = None
+        checked_files: Optional[Set[Lintable]] = None
     ) -> None:
         """Initialize a Runner instance."""
         self.rules = rules
@@ -127,7 +127,7 @@ def worker(lintable: Lintable) -> List[MatchError]:
             files = [value for n, value in enumerate(files) if value not in files[:n]]
 
             for file in self.lintables:
-                if str(file.path) in self.checked_files:
+                if file in self.checked_files:
                     continue
                 _logger.debug(
                     "Examining %s of type %s",
@@ -140,7 +140,7 @@ def worker(lintable: Lintable) -> List[MatchError]:
                 )
 
         # update list of checked files
-        self.checked_files.update([str(x.path) for x in self.lintables])
+        self.checked_files.update(self.lintables)
 
         # remove any matches made inside excluded files
         matches = list(
@@ -174,7 +174,7 @@ def _get_matches(rules: "RulesCollection", options: "Namespace") -> LintResult:
     lintables = ansiblelint.utils.get_lintables(options=options, args=options.lintables)
 
     matches = list()
-    checked_files: Set[str] = set()
+    checked_files: Set[Lintable] = set()
     runner = Runner(
         *lintables,
         rules=rules,
@@ -189,4 +189,12 @@ def _get_matches(rules: "RulesCollection", options: "Namespace") -> LintResult:
     # Assure we do not print duplicates and the order is consistent
     matches = sorted(set(matches))
 
+    # Convert reported filenames into human redable ones, so we hide the
+    # fact we used temporary files when processing input from stdin.
+    for match in matches:
+        for lintable in lintables:
+            if match.filename == lintable.filename:
+                match.filename = lintable.name
+                break
+
     return LintResult(matches=matches, files=checked_files)

src/ansiblelint/skip_utils.py

@@ -22,12 +22,17 @@
 import logging
 from functools import lru_cache
 from itertools import product
-from typing import Any, Generator, List, Optional, Sequence
+from typing import TYPE_CHECKING, Any, Generator, List, Sequence
 
 import ruamel.yaml
 
 from ansiblelint.config import used_old_tags
-from ansiblelint.constants import RENAMED_TAGS, FileType
+from ansiblelint.constants import RENAMED_TAGS
+from ansiblelint.file_utils import Lintable
+
+if TYPE_CHECKING:
+    from ansible.parsing.yaml.objects import AnsibleBaseYAMLObject
+
 
 _logger = logging.getLogger(__name__)
 
@@ -46,7 +51,7 @@ def get_rule_skips_from_line(line: str) -> List:
 
 
 def append_skipped_rules(
-    pyyaml_data: str, file_text: str, file_type: Optional[FileType] = None
+    pyyaml_data: "AnsibleBaseYAMLObject", lintable: Lintable
 ) -> Sequence:
     """Append 'skipped_rules' to individual tasks or single metadata block.
 
@@ -61,7 +66,7 @@ def append_skipped_rules(
     to individual tasks, or added to the single metadata block.
     """
     try:
-        yaml_skip = _append_skipped_rules(pyyaml_data, file_text, file_type)
+        yaml_skip = _append_skipped_rules(pyyaml_data, lintable)
     except RuntimeError:
         # Notify user of skip error, do not stop, do not change exit code
         _logger.error('Error trying to append skipped rules', exc_info=True)
@@ -82,21 +87,19 @@ def load_data(file_text: str) -> Any:
     return yaml.load(file_text)
 
 
-def _append_skipped_rules(
-    pyyaml_data: Sequence[Any], file_text: str, file_type: Optional[FileType] = None
-) -> Sequence:
+def _append_skipped_rules(pyyaml_data: Sequence[Any], lintable: Lintable) -> Sequence:
     # parse file text using 2nd parser library
-    ruamel_data = load_data(file_text)
+    ruamel_data = load_data(lintable.content)
 
-    if file_type == 'meta':
+    if lintable.kind == 'meta':
         pyyaml_data[0]['skipped_rules'] = _get_rule_skips_from_yaml(ruamel_data)
         return pyyaml_data
 
     # create list of blocks of tasks or nested tasks
-    if file_type in ('tasks', 'handlers'):
+    if lintable.kind in ('tasks', 'handlers'):
         ruamel_task_blocks = ruamel_data
         pyyaml_task_blocks = pyyaml_data
-    elif file_type in ('playbook', 'pre_tasks', 'post_tasks'):
+    elif lintable.kind in ('playbook', 'pre_tasks', 'post_tasks'):
         try:
             pyyaml_task_blocks = _get_task_blocks_from_playbook(pyyaml_data)
             ruamel_task_blocks = _get_task_blocks_from_playbook(ruamel_data)
@@ -105,10 +108,10 @@ def _append_skipped_rules(
             # assume it is a playbook, check needs to be added higher in the
             # call stack, and can remove this except
             return pyyaml_data
-    elif file_type in ['yaml', 'requirements', 'vars']:
+    elif lintable.kind in ['yaml', 'requirements', 'vars']:
         return pyyaml_data
     else:
-        raise RuntimeError('Unexpected file type: {}'.format(file_type))
+        raise RuntimeError('Unexpected file type: {}'.format(lintable.kind))
 
     # get tasks from blocks of tasks
     pyyaml_tasks = _get_tasks_from_blocks(pyyaml_task_blocks)

src/ansiblelint/utils.py

@@ -37,7 +37,7 @@
 from ansible.parsing.splitter import split_args
 from ansible.parsing.yaml.constructor import AnsibleConstructor
 from ansible.parsing.yaml.loader import AnsibleLoader
-from ansible.parsing.yaml.objects import AnsibleSequence
+from ansible.parsing.yaml.objects import AnsibleBaseYAMLObject, AnsibleSequence
 from ansible.plugins.loader import add_all_plugin_dirs
 from ansible.template import Templar
 
@@ -704,7 +704,7 @@ def get_normalized_tasks(yaml, file: Lintable) -> List[Dict[str, Any]]:
 
 
 @lru_cache(maxsize=128)
-def parse_yaml_linenumbers(data, filename):
+def parse_yaml_linenumbers(lintable: Lintable) -> AnsibleBaseYAMLObject:
     """Parse yaml as ansible.utils.parse_yaml but with linenumbers.
 
     The line numbers are stored in each node's LINE_NUMBER_KEY key.
@@ -723,19 +723,19 @@ def construct_mapping(node, deep=False):
             mapping[LINE_NUMBER_KEY] = node.__line__
         else:
             mapping[LINE_NUMBER_KEY] = mapping._line_number
-        mapping[FILENAME_KEY] = filename
+        mapping[FILENAME_KEY] = lintable.path
         return mapping
 
     try:
         kwargs = {}
         if 'vault_password' in inspect.getfullargspec(AnsibleLoader.__init__).args:
             kwargs['vault_password'] = DEFAULT_VAULT_PASSWORD
-        loader = AnsibleLoader(data, **kwargs)
+        loader = AnsibleLoader(lintable.content, **kwargs)
         loader.compose_node = compose_node
         loader.construct_mapping = construct_mapping
         data = loader.get_single_data()
     except (yaml.parser.ParserError, yaml.scanner.ScannerError) as e:
-        raise SystemExit("Failed to parse YAML in %s: %s" % (filename, str(e)))
+        raise SystemExit("Failed to parse YAML in %s: %s" % (lintable.path, str(e)))
     return data
 
 

test/TestRunner.py

@@ -99,7 +99,7 @@ def test_runner_with_directory(default_rules_collection, directory_name) -> None
 
 
 def test_files_not_scanned_twice(default_rules_collection) -> None:
-    checked_files: Set[str] = set()
+    checked_files: Set[Lintable] = set()
 
     filename = os.path.abspath('examples/playbooks/common-include-1.yml')
     runner = Runner(