Add gitfs and git_pillar fallback branch

Add a fallback branch when no matching branch is found. Example:

```yaml
gitfs_remotes:
  - http://foo.com/states/top.git:
    - all_saltenvs: master
  - http://foo.com/states/apache.git:
    - fallback: master
  - http://foo.com/states/nginx.git:
    - fallback: master
  - http://foo.com/states/webapp.git:
    - fallback: develop
```

Where a state in `webapp` includes a state of `apache`.

With this change, one can test a feature branch of `webapp` without
needing to create the same branch in `apache`. Similarly, one can test a
feature branch of `apache` without needing to create the same branch in
`webapp`.

This is a forward port of pull request saltstack#51971 from Mathieu Parent.

Signed-off-by: Benjamin Drung <benjamin.drung@cloud.ionos.com>

doc/topics/tutorials/gitfs.rst

@@ -532,7 +532,7 @@ would only fetch branches and tags (the default).
 Global Remotes
 ==============
 
-.. versionadded:: 2018.3.0
+.. versionadded:: 2018.3.0 for all_saltenvs, sodium for fallback
 
 The ``all_saltenvs`` per-remote configuration parameter overrides the logic
 Salt uses to map branches/tags to fileserver environments (i.e. saltenvs). This
@@ -544,6 +544,8 @@ allows a single branch/tag to appear in *all* GitFS saltenvs.
    environment defined via some other fileserver backend (e.g.
    :conf_master:`file_roots`).
 
+The ``fallback`` global or per-remote configuration can also be used.
+
 This is very useful in particular when working with :ref:`salt formulas
 <conventions-formula>`. Prior to the addition of this feature, it was necessary
 to push a branch/tag to the remote repo for each saltenv in which that formula
@@ -560,6 +562,15 @@ single branch.
       - http://foo.com/quux.git:
         - all_saltenvs: anything
 
+If you want to also test working branches of the formula repository, use
+``fallback``:
+
+.. code-block:: yaml
+
+    gitfs_remotes:
+      - http://foo.com/quux.git:
+        - fallback: anything
+
 .. _gitfs-update-intervals:
 
 Update Intervals

salt/config/__init__.py

@@ -513,11 +513,11 @@ def _gather_buffer_space():
         "minionfs_update_interval": int,
         "s3fs_update_interval": int,
         "svnfs_update_interval": int,
-        # NOTE: git_pillar_base, git_pillar_branch, git_pillar_env, and
-        # git_pillar_root omitted here because their values could conceivably be
-        # loaded as non-string types, which is OK because git_pillar will normalize
-        # them to strings. But rather than include all the possible types they
-        # could be, we'll just skip type-checking.
+        # NOTE: git_pillar_base, git_pillar_fallback, git_pillar_branch,
+        # git_pillar_env, and git_pillar_root omitted here because their values
+        # could conceivably be loaded as non-string types, which is OK because
+        # git_pillar will normalize them to strings. But rather than include all the
+        # possible types they could be, we'll just skip type-checking.
         "git_pillar_ssl_verify": bool,
         "git_pillar_global_lock": bool,
         "git_pillar_user": six.string_types,
@@ -529,10 +529,10 @@ def _gather_buffer_space():
         "git_pillar_refspecs": list,
         "git_pillar_includes": bool,
         "git_pillar_verify_config": bool,
-        # NOTE: gitfs_base, gitfs_mountpoint, and gitfs_root omitted here because
-        # their values could conceivably be loaded as non-string types, which is OK
-        # because gitfs will normalize them to strings. But rather than include all
-        # the possible types they could be, we'll just skip type-checking.
+        # NOTE: gitfs_base, gitfs_fallback, gitfs_mountpoint, and gitfs_root omitted
+        # here because their values could conceivably be loaded as non-string types,
+        # which is OK because gitfs will normalize them to strings. But rather than
+        # include all the possible types they could be, we'll just skip type-checking.
         "gitfs_remotes": list,
         "gitfs_insecure_auth": bool,
         "gitfs_privkey": six.string_types,
@@ -1033,6 +1033,7 @@ def _gather_buffer_space():
         "git_pillar_base": "master",
         "git_pillar_branch": "master",
         "git_pillar_env": "",
+        "git_pillar_fallback": "",
         "git_pillar_root": "",
         "git_pillar_ssl_verify": True,
         "git_pillar_global_lock": True,
@@ -1048,6 +1049,7 @@ def _gather_buffer_space():
         "gitfs_mountpoint": "",
         "gitfs_root": "",
         "gitfs_base": "master",
+        "gitfs_fallback": "",
         "gitfs_user": "",
         "gitfs_password": "",
         "gitfs_insecure_auth": False,
@@ -1272,6 +1274,7 @@ def _gather_buffer_space():
         "git_pillar_base": "master",
         "git_pillar_branch": "master",
         "git_pillar_env": "",
+        "git_pillar_fallback": "",
         "git_pillar_root": "",
         "git_pillar_ssl_verify": True,
         "git_pillar_global_lock": True,
@@ -1288,6 +1291,7 @@ def _gather_buffer_space():
         "gitfs_mountpoint": "",
         "gitfs_root": "",
         "gitfs_base": "master",
+        "gitfs_fallback": "",
         "gitfs_user": "",
         "gitfs_password": "",
         "gitfs_insecure_auth": False,

salt/fileserver/gitfs.py

@@ -59,6 +59,7 @@
 
 PER_REMOTE_OVERRIDES = (
     "base",
+    "fallback",
     "mountpoint",
     "root",
     "ssl_verify",

salt/pillar/git_pillar.py

@@ -365,6 +365,28 @@
 
     git_pillar_update_interval: 120
 
+.. _git-pillar-fallback:
+
+fallback
+~~~~~~~~
+
+.. versionadded:: sodium
+
+Setting ``fallback`` per-remote or global configuration parameter will map non-existing environments to a default branch. Example:
+
+.. code-block:: yaml
+
+    ext_pillar:
+      - git:
+        - __env__ https://mydomain.tld/top.git
+          - all_saltenvs: master
+        - __env__ https://mydomain.tld/pillar-nginx.git:
+          - mountpoint: web/server/
+          - fallback: master
+        - __env__ https://mydomain.tld/pillar-appdata.git:
+          - mountpoint: web/server/
+          - fallback: master
+
 """
 from __future__ import absolute_import, print_function, unicode_literals
 
@@ -384,7 +406,7 @@
 from salt.ext import six
 from salt.pillar import Pillar
 
-PER_REMOTE_OVERRIDES = ("env", "root", "ssl_verify", "refspecs")
+PER_REMOTE_OVERRIDES = ("env", "root", "ssl_verify", "refspecs", "fallback")
 PER_REMOTE_ONLY = ("name", "mountpoint", "all_saltenvs")
 GLOBAL_ONLY = ("base", "branch")
 

salt/utils/gitfs.py

@@ -1073,6 +1073,22 @@ def get_tree(self, tgt_env):
                 if candidate is not None:
                     return candidate
 
+        if self.fallback:
+            for ref_type in self.ref_types:
+                try:
+                    func_name = "get_tree_from_{0}".format(ref_type)
+                    func = getattr(self, func_name)
+                except AttributeError:
+                    log.error(
+                        "%s class is missing function '%s'",
+                        self.__class__.__name__,
+                        func_name,
+                    )
+                else:
+                    candidate = func(self.fallback)
+                    if candidate is not None:
+                        return candidate
+
         # No matches found
         return None
 
@@ -1192,12 +1208,18 @@ def checkout(self):
 
         # 'origin/' + tgt_ref ==> matches a branch head
         # 'tags/' + tgt_ref + '@{commit}' ==> matches tag's commit
-        for rev_parse_target, checkout_ref in (
-            ("origin/" + tgt_ref, "origin/" + tgt_ref),
-            ("tags/" + tgt_ref, "tags/" + tgt_ref),
-        ):
+        checkout_refs = [
+            ("origin/" + tgt_ref, False),
+            ("tags/" + tgt_ref, False),
+        ]
+        if self.fallback:
+            checkout_refs += [
+                ("origin/" + self.fallback, True),
+                ("tags/" + self.fallback, True),
+            ]
+        for checkout_ref, fallback in checkout_refs:
             try:
-                target_sha = self.repo.rev_parse(rev_parse_target).hexsha
+                target_sha = self.repo.rev_parse(checkout_ref).hexsha
             except Exception:  # pylint: disable=broad-except
                 # ref does not exist
                 continue
@@ -1210,10 +1232,11 @@ def checkout(self):
                 with self.gen_lock(lock_type="checkout"):
                     self.repo.git.checkout(checkout_ref)
                     log.debug(
-                        "%s remote '%s' has been checked out to %s",
+                        "%s remote '%s' has been checked out to %s%s",
                         self.role,
                         self.id,
                         checkout_ref,
+                        " as fallback" if fallback else "",
                     )
             except GitLockError as exc:
                 if exc.errno == errno.EEXIST:
@@ -1559,6 +1582,11 @@ def _perform_checkout(checkout_ref, branch=True):
             return False
 
         try:
+            if remote_ref not in refs and tag_ref not in refs and self.fallback:
+                tgt_ref = self.fallback
+                local_ref = "refs/heads/" + tgt_ref
+                remote_ref = "refs/remotes/origin/" + tgt_ref
+                tag_ref = "refs/tags/" + tgt_ref
             if remote_ref in refs:
                 # Get commit id for the remote ref
                 oid = self.peel(self.repo.lookup_reference(remote_ref)).id
@@ -2890,9 +2918,7 @@ def find_file(self, path, tgt_env="base", **kwargs):  # pylint: disable=W0613
         and send the path to the newly cached file
         """
         fnd = {"path": "", "rel": ""}
-        if os.path.isabs(path) or (
-            not salt.utils.stringutils.is_hex(tgt_env) and tgt_env not in self.envs()
-        ):
+        if os.path.isabs(path):
             return fnd
 
         dest = salt.utils.path.join(self.cache_root, "refs", tgt_env, path)
@@ -2925,6 +2951,12 @@ def find_file(self, path, tgt_env="base", **kwargs):  # pylint: disable=W0613
                 repo.mountpoint(tgt_env) + os.sep
             ):
                 continue
+            if (
+                not salt.utils.stringutils.is_hex(tgt_env)
+                and tgt_env not in self.envs()
+                and not repo.fallback
+            ):
+                continue
             repo_path = path[len(repo.mountpoint(tgt_env)) :].lstrip(os.sep)
             if repo.root(tgt_env):
                 repo_path = salt.utils.path.join(repo.root(tgt_env), repo_path)
@@ -3082,11 +3114,12 @@ def _file_lists(self, load, form):
         if refresh_cache:
             log.trace("Start rebuilding gitfs file_list cache")
             ret = {"files": set(), "symlinks": {}, "dirs": set()}
-            if (
-                salt.utils.stringutils.is_hex(load["saltenv"])
-                or load["saltenv"] in self.envs()
-            ):
-                for repo in self.remotes:
+            for repo in self.remotes:
+                if (
+                    salt.utils.stringutils.is_hex(load["saltenv"])
+                    or load["saltenv"] in self.envs()
+                    or repo.fallback
+                ):
                     start = time.time()
                     repo_files, repo_symlinks = repo.file_list(load["saltenv"])
                     ret["files"].update(repo_files)