Add gitfs and git_pillar fallback branch

doc/topics/tutorials/gitfs.rst

@@ -527,7 +527,7 @@ would only fetch branches and tags (the default).
 Global Remotes
 ==============
 
-.. versionadded:: 2018.3.0
+.. versionadded:: 2018.3.0 for all_saltenvs, neon for fallback
 
 The ``all_saltenvs`` per-remote configuration parameter overrides the logic
 Salt uses to map branches/tags to fileserver environments (i.e. saltenvs). This
@@ -539,7 +539,9 @@ allows a single branch/tag to appear in *all* GitFS saltenvs.
    environment defined via some other fileserver backend (e.g.
    :conf_master:`file_roots`).
 
-This is very useful in particular when working with :ref:`salt formulas
+The ``fallback`` global or per-remote configuration can also be used.
+
+Those are very useful in particular when working with :ref:`salt formulas
 <conventions-formula>`. Prior to the addition of this feature, it was necessary
 to push a branch/tag to the remote repo for each saltenv in which that formula
 was to be used. If the formula needed to be updated, this update would need to
@@ -555,6 +557,15 @@ single branch.
       - http://foo.com/quux.git:
         - all_saltenvs: anything
 
+If you want to also test working branches of the formula repository, use
+``fallback``:
+
+.. code-block:: yaml
+
+    gitfs_remotes:
+      - http://foo.com/quux.git:
+        - fallback: anything
+
 .. _gitfs-update-intervals:
 
 Update Intervals

salt/config/__init__.py

@@ -675,11 +675,11 @@ def _gather_buffer_space():
     's3fs_update_interval': int,
     'svnfs_update_interval': int,
 
-    # NOTE: git_pillar_base, git_pillar_branch, git_pillar_env, and
-    # git_pillar_root omitted here because their values could conceivably be
-    # loaded as non-string types, which is OK because git_pillar will normalize
-    # them to strings. But rather than include all the possible types they
-    # could be, we'll just skip type-checking.
+    # NOTE: git_pillar_base, git_pillar_fallback, git_pillar_branch,
+    # git_pillar_env, and git_pillar_root omitted here because their values
+    # could conceivably be loaded as non-string types, which is OK because
+    # git_pillar will normalize them to strings. But rather than include all the
+    # possible types they could be, we'll just skip type-checking.
     'git_pillar_ssl_verify': bool,
     'git_pillar_global_lock': bool,
     'git_pillar_user': six.string_types,
@@ -691,10 +691,10 @@ def _gather_buffer_space():
     'git_pillar_refspecs': list,
     'git_pillar_includes': bool,
     'git_pillar_verify_config': bool,
-    # NOTE: gitfs_base, gitfs_mountpoint, and gitfs_root omitted here because
-    # their values could conceivably be loaded as non-string types, which is OK
-    # because gitfs will normalize them to strings. But rather than include all
-    # the possible types they could be, we'll just skip type-checking.
+    # NOTE: gitfs_base, gitfs_fallback, gitfs_mountpoint, and gitfs_root omitted
+    # here because their values could conceivably be loaded as non-string types,
+    # which is OK because gitfs will normalize them to strings. But rather than
+    # include all the possible types they could be, we'll just skip type-checking.
     'gitfs_remotes': list,
     'gitfs_insecure_auth': bool,
     'gitfs_privkey': six.string_types,
@@ -1320,6 +1320,7 @@ def _gather_buffer_space():
     'svnfs_update_interval': DEFAULT_INTERVAL,
 
     'git_pillar_base': 'master',
+    'git_pillar_fallback': '',
     'git_pillar_branch': 'master',
     'git_pillar_env': '',
     'git_pillar_root': '',
@@ -1337,6 +1338,7 @@ def _gather_buffer_space():
     'gitfs_mountpoint': '',
     'gitfs_root': '',
     'gitfs_base': 'master',
+    'gitfs_fallback': '',
     'gitfs_user': '',
     'gitfs_password': '',
     'gitfs_insecure_auth': False,
@@ -1571,6 +1573,7 @@ def _gather_buffer_space():
     'svnfs_update_interval': DEFAULT_INTERVAL,
 
     'git_pillar_base': 'master',
+    'git_pillar_fallback': '',
     'git_pillar_branch': 'master',
     'git_pillar_env': '',
     'git_pillar_root': '',
@@ -1589,6 +1592,7 @@ def _gather_buffer_space():
     'gitfs_mountpoint': '',
     'gitfs_root': '',
     'gitfs_base': 'master',
+    'gitfs_fallback': '',
     'gitfs_user': '',
     'gitfs_password': '',
     'gitfs_insecure_auth': False,

salt/fileserver/gitfs.py

@@ -53,7 +53,7 @@
 import logging
 
 PER_REMOTE_OVERRIDES = (
-    'base', 'mountpoint', 'root', 'ssl_verify',
+    'base', 'fallback', 'mountpoint', 'root', 'ssl_verify',
     'saltenv_whitelist', 'saltenv_blacklist',
     'env_whitelist', 'env_blacklist', 'refspecs',
     'disable_saltenv_mapping', 'ref_types', 'update_interval',

salt/modules/virtualenv_mod.py

@@ -21,19 +21,26 @@
 from salt.exceptions import CommandExecutionError, SaltInvocationError
 from salt.ext.six import string_types
 
-KNOWN_BINARY_NAMES = frozenset([
-    'pyvenv-{0}.{1}'.format(*sys.version_info[:2]),
-    'virtualenv-{0}.{1}'.format(*sys.version_info[:2]),
-    'pyvenv{0}'.format(sys.version_info[0]),
-    'virtualenv{0}'.format(sys.version_info[0]),
-    'pyvenv',
-    'virtualenv',
-])
+if sys.version_info >= (3, 6):
+    KNOWN_BINARY_NAMES = frozenset([
+        'virtualenv-{0}.{1}'.format(*sys.version_info[:2]),
+        'virtualenv{0}'.format(sys.version_info[0]),
+        'virtualenv',
+    ])
+else:
+    KNOWN_BINARY_NAMES = frozenset([
+        'pyvenv-{0}.{1}'.format(*sys.version_info[:2]),
+        'virtualenv-{0}.{1}'.format(*sys.version_info[:2]),
+        'pyvenv{0}'.format(sys.version_info[0]),
+        'virtualenv{0}'.format(sys.version_info[0]),
+        'pyvenv',
+        'virtualenv',
+    ])
 
 log = logging.getLogger(__name__)
 
 __opts__ = {
-    'venv_bin': salt.utils.path.which_bin(KNOWN_BINARY_NAMES) or 'virtualenv'
+    'venv_bin': salt.utils.path.which_bin(KNOWN_BINARY_NAMES)
 }
 
 __pillar__ = {}
@@ -139,12 +146,14 @@ def create(path,
         salt '*' virtualenv.create /path/to/new/virtualenv
     '''
     if venv_bin is None:
+        venv_bin = __pillar__.get('venv_bin') or __opts__.get('venv_bin')
         # Beginning in 3.6, pyvenv has been deprecated
         # in favor of "python3 -m venv"
-        if sys.version_info >= (3, 6):
-            venv_bin = ['python3', '-m', 'venv']
-        else:
-            venv_bin = __pillar__.get('venv_bin') or __opts__.get('venv_bin')
+        if venv_bin is None:
+            if sys.version_info >= (3, 6):
+                venv_bin = ['python3', '-m', 'venv']
+            else:
+                venv_bin = 'virtualenv'
 
     if not isinstance(venv_bin, list):
         cmd = [venv_bin]

salt/pillar/git_pillar.py

@@ -350,6 +350,28 @@
         - __env__ https://mydomain.tld/pillar-appdata.git:
           - mountpoint: web/server/
 
+.. _git-pillar-fallback:
+
+fallback
+~~~~~~~~
+
+.. versionadded:: neon
+
+Setting ``fallback`` per-remote or global configuration parameter will map non-existing environments to a default branch. Example:
+
+.. code-block:: yaml
+
+    ext_pillar:
+      - git:
+        - __env__ https://mydomain.tld/top.git
+          - all_saltenvs: master
+        - __env__ https://mydomain.tld/pillar-nginx.git:
+          - mountpoint: web/server/
+          - fallback: master
+        - __env__ https://mydomain.tld/pillar-appdata.git:
+          - mountpoint: web/server/
+          - fallback: master
+
 '''
 from __future__ import absolute_import, print_function, unicode_literals
 
@@ -368,7 +390,7 @@
 # Import third party libs
 from salt.ext import six
 
-PER_REMOTE_OVERRIDES = ('base', 'env', 'root', 'ssl_verify', 'refspecs')
+PER_REMOTE_OVERRIDES = ('base', 'env', 'root', 'ssl_verify', 'refspecs', 'fallback')
 PER_REMOTE_ONLY = ('name', 'mountpoint', 'all_saltenvs')
 GLOBAL_ONLY = ('branch',)
 

salt/utils/gitfs.py

@@ -1042,6 +1042,21 @@ def get_tree(self, tgt_env):
                 if candidate is not None:
                     return candidate
 
+        if self.fallback:
+            for ref_type in self.ref_types:
+                try:
+                    func_name = 'get_tree_from_{0}'.format(ref_type)
+                    func = getattr(self, func_name)
+                except AttributeError:
+                    log.error(
+                        '%s class is missing function \'%s\'',
+                        self.__class__.__name__, func_name
+                    )
+                else:
+                    candidate = func(self.fallback)
+                    if candidate is not None:
+                        return candidate
+
         # No matches found
         return None
 
@@ -1145,12 +1160,17 @@ def checkout(self):
             head_sha = None
 
         # 'origin/' + tgt_ref ==> matches a branch head
-        # 'tags/' + tgt_ref + '@{commit}' ==> matches tag's commit
-        for rev_parse_target, checkout_ref in (
-                ('origin/' + tgt_ref, 'origin/' + tgt_ref),
-                ('tags/' + tgt_ref, 'tags/' + tgt_ref)):
+        # 'tags/' + tgt_ref ==> matches tag's commit
+        checkout_refs = [
+                ('origin/' + tgt_ref, False),
+                ('tags/' + tgt_ref, False)]
+        if self.fallback:
+            checkout_refs.extend([
+                ('origin/' + self.fallback, True),
+                ('tags/' + self.fallback, True)])
+        for checkout_ref, fallback in checkout_refs:
             try:
-                target_sha = self.repo.rev_parse(rev_parse_target).hexsha
+                target_sha = self.repo.rev_parse(checkout_ref).hexsha
             except Exception:
                 # ref does not exist
                 continue
@@ -1163,10 +1183,11 @@ def checkout(self):
                 with self.gen_lock(lock_type='checkout'):
                     self.repo.git.checkout(checkout_ref)
                     log.debug(
-                        '%s remote \'%s\' has been checked out to %s',
+                        '%s remote \'%s\' has been checked out to %s%s',
                         self.role,
                         self.id,
-                        checkout_ref
+                        checkout_ref,
+                        ' as fallback' if fallback else ''
                     )
             except GitLockError as exc:
                 if exc.errno == errno.EEXIST:
@@ -1498,6 +1519,11 @@ def _perform_checkout(checkout_ref, branch=True):
             return False
 
         try:
+            if remote_ref not in refs and tag_ref not in refs and self.fallback:
+                tgt_ref = self.fallback
+                local_ref = 'refs/heads/' + tgt_ref
+                remote_ref = 'refs/remotes/origin/' + tgt_ref
+                tag_ref = 'refs/tags/' + tgt_ref
             if remote_ref in refs:
                 # Get commit id for the remote ref
                 oid = self.peel(self.repo.lookup_reference(remote_ref)).id
@@ -2762,8 +2788,7 @@ def find_file(self, path, tgt_env='base', **kwargs):  # pylint: disable=W0613
         '''
         fnd = {'path': '',
                'rel': ''}
-        if os.path.isabs(path) or \
-                (not salt.utils.stringutils.is_hex(tgt_env) and tgt_env not in self.envs()):
+        if os.path.isabs(path):
             return fnd
 
         dest = salt.utils.path.join(self.cache_root, 'refs', tgt_env, path)
@@ -2797,6 +2822,8 @@ def find_file(self, path, tgt_env='base', **kwargs):  # pylint: disable=W0613
             if repo.mountpoint(tgt_env) \
                     and not path.startswith(repo.mountpoint(tgt_env) + os.sep):
                 continue
+            if not salt.utils.stringutils.is_hex(tgt_env) and tgt_env not in self.envs() and not repo.fallback:
+                continue
             repo_path = path[len(repo.mountpoint(tgt_env)):].lstrip(os.sep)
             if repo.root(tgt_env):
                 repo_path = salt.utils.path.join(repo.root(tgt_env), repo_path)
@@ -2954,9 +2981,10 @@ def _file_lists(self, load, form):
             return cache_match
         if refresh_cache:
             ret = {'files': set(), 'symlinks': {}, 'dirs': set()}
-            if salt.utils.stringutils.is_hex(load['saltenv']) \
-                    or load['saltenv'] in self.envs():
-                for repo in self.remotes:
+            for repo in self.remotes:
+                if salt.utils.stringutils.is_hex(load['saltenv']) \
+                        or load['saltenv'] in self.envs() \
+                        or repo.fallback:
                     repo_files, repo_symlinks = repo.file_list(load['saltenv'])
                     ret['files'].update(repo_files)
                     ret['symlinks'].update(repo_symlinks)

tests/integration/files/file/base/git_pillar/http/init.sls

@@ -2,6 +2,7 @@
 {%- set git_dir = pillar['git_pillar']['git_dir'] %}
 {%- set venv_dir = pillar['git_pillar']['venv_dir'] %}
 {%- set root_dir = pillar['git_pillar']['root_dir'] %}
+{%- set venv_bin = salt['cmd.retcode']('which virtualenv') %}
 
 {{ config_dir }}/nginx.conf:
   file.managed:
@@ -40,6 +41,9 @@
     - system_site_packages: False
     {#- Provide the real path for the python executable in case tests are running inside a virtualenv #}
     - python: {{ salt.runtests_helpers.get_python_executable() }}
+{%- if venv_bin == 0 %}
+    - venv_bin: virtualenv
+{%- endif %}
 
 uwsgi:
   pip.installed: