WIP: Enable ansible-lint to auto-detect roles/playbooks

Do not review before merging:
    #620
    #621

When called without any arguments, ansible-lint will now try to look
for all playbooks and roles inside current git repository.

Unrecognized YAML files will be displayed in verbose mode but will not
be considered errors. This will allow users to enable ansible-lint
on any repository without being forced to alter the way is called or
its configuration each time a new playbooks/role is added.

Fixes: #613
Fixes: #564
Signed-off-by: Sorin Sbarnea <ssbarnea@redhat.com>

lib/ansiblelint/__init__.py

@@ -260,7 +260,7 @@ def run(self):
                 continue
             if playbook[1] == 'role':
                 continue
-            files.append({'path': playbook[0], 'type': playbook[1]})
+            files.append({'path': ansiblelint.utils.normpath(playbook[0]), 'type': playbook[1]})
         visited = set()
         while (visited != self.playbooks):
             for arg in self.playbooks - visited:
@@ -280,7 +280,9 @@ def run(self):
         files = [x for x in files if x['path'] not in self.checked_files]
         for file in files:
             if self.verbosity > 0:
-                print("Examining %s of type %s" % (file['path'], file['type']))
+                print("Examining %s of type %s" % (
+                    ansiblelint.utils.normpath(file['path']),
+                    file['type']))
             matches.extend(self.rules.run(file, tags=set(self.tags),
                            skip_list=self.skip_list))
         # update list of checked files

lib/ansiblelint/__main__.py

@@ -30,6 +30,7 @@
 import six
 from ansiblelint import default_rulesdir, RulesCollection, Runner
 from ansiblelint.version import __version__
+from ansiblelint.utils import get_playbooks_and_roles, normpath
 import yaml
 import os
 
@@ -51,7 +52,7 @@ def main():
 
     formatter = formatters.Formatter()
 
-    parser = optparse.OptionParser("%prog [options] playbook.yml [playbook2 ...]",
+    parser = optparse.OptionParser("%prog [options] [playbook.yml [playbook2 ...]]",
                                    version="%prog " + __version__)
 
     parser.add_option('-L', dest='listrules', default=False,
@@ -126,8 +127,9 @@ def main():
         if 'verbosity' in config:
             options.verbosity = options.verbosity + config['verbosity']
 
-        if 'exclude_paths' in config:
-            options.exclude_paths = options.exclude_paths + config['exclude_paths']
+        options.exclude_paths.extend(
+            config.get('exclude_paths', []), []),
+            )
 
         if 'rulesdir' in config:
             options.rulesdir = options.rulesdir + config['rulesdir']
@@ -147,9 +149,9 @@ def main():
     if options.parseable_severity:
         formatter = formatters.ParseableSeverityFormatter()
 
+    # no args triggers auto-detection mode
     if len(args) == 0 and not (options.listrules or options.listtags):
-        parser.print_help(file=sys.stderr)
-        return 1
+        args = get_playbooks_and_roles(options=options)
 
     if options.use_default_rules:
         rulesdirs = options.rulesdir + [default_rulesdir]
@@ -176,7 +178,7 @@ def main():
         skip.update(str(s).split(','))
     options.skip_list = frozenset(skip)
 
-    playbooks = set(args)
+    playbooks = sorted(set(args))
     matches = list()
     checked_files = set()
     for playbook in playbooks:
@@ -185,7 +187,7 @@ def main():
                         options.verbosity, checked_files)
         matches.extend(runner.run())
 
-    matches.sort(key=lambda x: (x.filename, x.linenumber, x.rule.id))
+    matches.sort(key=lambda x: (normpath(x.filename), x.linenumber, x.rule.id))
 
     for match in matches:
         print(formatter.format(match, options.colored))

lib/ansiblelint/formatters/__init__.py

@@ -3,6 +3,8 @@
 except ImportError:
     from ansible.utils import color
 
+from ansiblelint.utils import normpath
+
 
 class Formatter(object):
 
@@ -12,7 +14,7 @@ def format(self, match, colored=False):
             color.ANSIBLE_COLOR = True
             return formatstr.format(color.stringc(u"[{0}]".format(match.rule.id), 'bright red'),
                                     color.stringc(match.message, 'red'),
-                                    color.stringc(match.filename, 'blue'),
+                                    color.stringc(normpath(match.filename), 'blue'),
                                     color.stringc(str(match.linenumber), 'cyan'),
                                     color.stringc(u"{0}".format(match.line), 'purple'))
         else:
@@ -30,10 +32,10 @@ def format(self, match, colored=False):
         if colored:
             color.ANSIBLE_COLOR = True
             return formatstr.format(color.stringc(u"[{0}]".format(match.rule.id), 'bright red'),
-                                    color.stringc(match.filename, 'blue'),
+                                    color.stringc(normpath(match.filename), 'blue'),
                                     color.stringc(str(match.linenumber), 'cyan'))
         else:
-            return formatstr.format(match.rule.id, match.filename,
+            return formatstr.format(match.rule.id, normpath(match.filename),
                                     match.linenumber)
 
 
@@ -43,12 +45,12 @@ def format(self, match, colored=False):
         formatstr = u"{0}:{1}: [{2}] {3}"
         if colored:
             color.ANSIBLE_COLOR = True
-            return formatstr.format(color.stringc(match.filename, 'blue'),
+            return formatstr.format(color.stringc(normpath(match.filename), 'blue'),
                                     color.stringc(str(match.linenumber), 'cyan'),
                                     color.stringc(u"E{0}".format(match.rule.id), 'bright red'),
                                     color.stringc(u"{0}".format(match.message), 'red'))
         else:
-            return formatstr.format(match.filename,
+            return formatstr.format(normpath(match.filename),
                                     match.linenumber,
                                     "E" + match.rule.id,
                                     match.message)
@@ -59,7 +61,7 @@ class ParseableSeverityFormatter(object):
     def format(self, match, colored=False):
         formatstr = u"{0}:{1}: [{2}] [{3}] {4}"
 
-        filename = match.filename
+        filename = normpath(match.filename)
         linenumber = str(match.linenumber)
         rule_id = u"E{0}".format(match.rule.id)
         severity = match.rule.severity

lib/ansiblelint/utils.py

@@ -18,10 +18,17 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 
+from collections import OrderedDict
 import glob
 import imp
-import os
 from itertools import product
+import os
+try:
+    from pathlib import Path
+except ImportError:
+    from pathlib2 import Path
+import subprocess
+
 
 import six
 from ansible import constants
@@ -53,6 +60,7 @@
     from ansible.parsing.mod_args import ModuleArgsParser
     from ansible.parsing.yaml.constructor import AnsibleConstructor
     from ansible.parsing.yaml.loader import AnsibleLoader
+    from ansible.parsing.yaml.objects import AnsibleSequence
     from ansible.errors import AnsibleParserError
     ANSIBLE_VERSION = 2
 
@@ -732,3 +740,108 @@ def get_rule_skips_from_line(line):
         noqa_text = line.split('# noqa')[1]
         rule_id_list = noqa_text.split()
     return rule_id_list
+
+
+def is_playbook(filename):
+    """
+    Given a filename, it should return true if it looks like a playbook. The
+    function is not supposed to raise exceptions.
+    """
+    # we assume is a playbook if we loaded a sequence of dictionaries where
+    # at least one of these keys is present:
+    playbooks_keys = {
+        "gather_facts",
+        "hosts",
+        "import_playbook",
+        "post_tasks",
+        "pre_tasks",
+        "roles"
+        "tasks",
+    }
+
+    # makes it work with Path objects by converting them to strings
+    if not isinstance(filename, six.string_types):
+        filename = str(filename)
+
+    try:
+        f = parse_yaml_from_file(filename)
+        if isinstance(f, AnsibleSequence) and playbooks_keys.intersection(f[0].keys()):
+            return True
+    except Exception as e:
+        print(
+            "Warning: Failed to load %s with %s, assuming is not a playbook."
+            % (filename, e))
+    return False
+
+
+def normpath(path):
+    """
+    Normalize a path in order to provide a more consistent output. Currently
+    this means to generate a relative path but in the future we may want to
+    make this configurable.
+    """
+    return os.path.relpath(path)
+
+
+def get_playbooks_and_roles(options={}):
+
+    # git is preferred as it also considers .gitignore
+    files = OrderedDict.fromkeys(sorted(subprocess.check_output(
+        ["git", "ls-files", "*.yaml", "*.yml"],
+        universal_newlines=True).split()))
+
+    playbooks = []
+    role_dirs = []
+    role_internals = {
+        'defaults',
+        'files',
+        'handlers',
+        'meta',
+        'tasks',
+        'templates',
+        'vars',
+        }
+
+    # detect role in repository root:
+    if 'tasks/main.yml' in files:
+        role_dirs.append('.')
+
+    for p in map(Path, files):
+
+        if any(str(p).startswith(file_path) for file_path in options.exclude_paths):
+            continue
+        elif (next((i for i in p.parts if i.endswith('playbooks')), None)
+                or 'playbook' in p.parts[-1]):
+            playbooks.append(normpath(p))
+            continue
+
+        # ignore if any folder ends with _vars
+        if next((i for i in p.parts if i.endswith('_vars')), None):
+            continue
+        elif 'roles' in p.parts or '.' in role_dirs:
+            if 'tasks' in p.parts and p.parts[-1] == 'main.yaml':
+                role_dirs.append(str(p.parents[1]))
+            elif role_internals.intersection(p.parts):
+                continue
+            elif 'tests' in p.parts:
+                playbooks.append(normpath(p))
+        if 'molecule' in p.parts:
+            if p.parts[-1] != 'molecule.yml':
+                playbooks.append(normpath(p))
+            continue
+        # hidden files are clearly not playbooks, likely config files.
+        if p.parts[-1].startswith('.'):
+            continue
+
+        if is_playbook(p):
+            playbooks.append(normpath(p))
+            continue
+
+        if options.verbosity:
+            print('Unknown file type: %s' % normpath(p))
+
+    if options.verbosity:
+        print('Found roles: ' + ' '.join(role_dirs))
+        print('Found playbooks: ' + ' '.join(playbooks))
+
+    return role_dirs + playbooks

pyproject.toml

@@ -9,6 +9,7 @@ requires = [
   # machinery (`importlib.import_module`)
   # imports `__init__` first
   "ansible",
+  "pathlib2; python_version < '3.2'",
   "ruamel.yaml",
   "six",
 ]

setup.cfg

@@ -83,6 +83,7 @@ install_requires =
   ansible >= 2.7
   pyyaml
   six
+  pathlib2; python_version < "3.2"
   ruamel.yaml >= 0.15.34,<1; python_version < "3.7"
   ruamel.yaml >= 0.15.37,<1; python_version >= "3.7"
   # NOTE: per issue #509 0.15.34 included in debian backports

tox.ini

@@ -16,6 +16,7 @@ deps =
   ansible29: ansible>=2.9,<2.10
   ansibledevel: git+https://github.com/ansible/ansible.git
   ruamel.yaml==0.16.5  # NOTE: 0.15.34 has issues with py37
+  #pathlib2; python_version < '3.2'
   flake8
   pep8-naming
   pytest