From 56e733d5f0780daf8644bfda7b48a1b543f80c48 Mon Sep 17 00:00:00 2001
From: Matt Clay <matt@mystile.com>
Date: Tue, 7 Apr 2020 02:29:28 -0700
Subject: [PATCH 1/6] Add env --list-files option.

---
 .../fragments/ansible-test-env-list-files.yml      |  2 ++
 test/lib/ansible_test/_internal/cli.py             |  4 ++++
 test/lib/ansible_test/_internal/env.py             | 14 +++++++++++++-
 3 files changed, 19 insertions(+), 1 deletion(-)
 create mode 100644 changelogs/fragments/ansible-test-env-list-files.yml

diff --git a/changelogs/fragments/ansible-test-env-list-files.yml b/changelogs/fragments/ansible-test-env-list-files.yml
new file mode 100644
index 00000000000000..aae86817a5e89f
--- /dev/null
+++ b/changelogs/fragments/ansible-test-env-list-files.yml
@@ -0,0 +1,2 @@
+minor_changes:
+    - ansible-test now has a ``--list-files`` option to list files using the ``env`` command.
diff --git a/test/lib/ansible_test/_internal/cli.py b/test/lib/ansible_test/_internal/cli.py
index 942ef905cc95e9..0c8345718b50fd 100644
--- a/test/lib/ansible_test/_internal/cli.py
+++ b/test/lib/ansible_test/_internal/cli.py
@@ -636,6 +636,10 @@ def key_value_type(value):  # type: (str) -> t.Tuple[str, str]
                      action='store_true',
                      help='dump environment to disk')
 
+    env.add_argument('--list-files',
+                     action='store_true',
+                     help='list files on stdout')
+
     # noinspection PyTypeChecker
     env.add_argument('--timeout',
                      type=int,
diff --git a/test/lib/ansible_test/_internal/env.py b/test/lib/ansible_test/_internal/env.py
index 31d61a8fafea55..f495981c16192d 100644
--- a/test/lib/ansible_test/_internal/env.py
+++ b/test/lib/ansible_test/_internal/env.py
@@ -31,6 +31,7 @@
 )
 
 from .util_common import (
+    data_context,
     write_json_test_results,
     ResultType,
 )
@@ -72,8 +73,9 @@ def __init__(self, args):
         self.show = args.show
         self.dump = args.dump
         self.timeout = args.timeout
+        self.list_files = args.list_files
 
-        if not self.show and not self.dump and self.timeout is None:
+        if not self.show and not self.dump and self.timeout is None and not self.list_files:
             # default to --show if no options were given
             self.show = True
 
@@ -83,6 +85,7 @@ def command_env(args):
     :type args: EnvConfig
     """
     show_dump_env(args)
+    list_files_env(args)
     set_timeout(args)
 
 
@@ -130,6 +133,15 @@ def show_dump_env(args):
         write_json_test_results(ResultType.BOT, 'data-environment.json', data)
 
 
+def list_files_env(args):  # type: (EnvConfig) -> None
+    """List files on stdout."""
+    if not args.list_files:
+        return
+
+    for path in data_context().content.all_files():
+        display.info(path)
+
+
 def set_timeout(args):
     """
     :type args: EnvConfig

From 0abba2ba43418d2c03ff81637eaa899344ebc0a4 Mon Sep 17 00:00:00 2001
From: Matt Clay <matt@mystile.com>
Date: Mon, 6 Apr 2020 21:12:01 -0700
Subject: [PATCH 2/6] Refactor ansible-test integration test.

This will make it easier to add additional tests without the main `runme.sh` file becoming too large.
---
 .../ansible-test/collection-tests/venv.sh     | 17 +++++++++++++
 .../integration/targets/ansible-test/runme.sh | 24 +++++++++----------
 2 files changed, 29 insertions(+), 12 deletions(-)
 create mode 100755 test/integration/targets/ansible-test/collection-tests/venv.sh

diff --git a/test/integration/targets/ansible-test/collection-tests/venv.sh b/test/integration/targets/ansible-test/collection-tests/venv.sh
new file mode 100755
index 00000000000000..45f41bcd85152b
--- /dev/null
+++ b/test/integration/targets/ansible-test/collection-tests/venv.sh
@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+set -eux -o pipefail
+
+cp -a "${TEST_DIR}/ansible_collections" "${WORK_DIR}"
+cd "${WORK_DIR}/ansible_collections/ns/col"
+
+# common args for all tests
+common=(--venv --python "${ANSIBLE_TEST_PYTHON_VERSION}" --color --truncate 0 "${@}")
+
+# prime the venv to work around issue with PyYAML detection in ansible-test
+ansible-test sanity "${common[@]}" --test ignores
+
+# tests
+ansible-test sanity "${common[@]}"
+ansible-test units "${common[@]}"
+ansible-test integration "${common[@]}"
diff --git a/test/integration/targets/ansible-test/runme.sh b/test/integration/targets/ansible-test/runme.sh
index d3c40f522d8680..7c956b4f1588bc 100755
--- a/test/integration/targets/ansible-test/runme.sh
+++ b/test/integration/targets/ansible-test/runme.sh
@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-set -eux -o pipefail
+set -eu -o pipefail
 
 # tests must be executed outside of the ansible source tree
 # otherwise ansible-test will test the ansible source instead of the test collection
@@ -9,16 +9,16 @@ tmp_dir=$(mktemp -d)
 
 trap 'rm -rf "${tmp_dir}"' EXIT
 
-cp -a ansible_collections "${tmp_dir}"
-cd "${tmp_dir}/ansible_collections/ns/col"
+export TEST_DIR
+export WORK_DIR
 
-# common args for all tests
-common=(--venv --python "${ANSIBLE_TEST_PYTHON_VERSION}" --color --truncate 0 "${@}")
+TEST_DIR="$PWD"
 
-# prime the venv to work around issue with PyYAML detection in ansible-test
-ansible-test sanity "${common[@]}" --test ignores
-
-# tests
-ansible-test sanity "${common[@]}"
-ansible-test units "${common[@]}"
-ansible-test integration "${common[@]}"
+for test in collection-tests/*.sh; do
+  WORK_DIR="${tmp_dir}/$(basename "${test}" ".sh")"
+  mkdir "${WORK_DIR}"
+  echo "**********************************************************************"
+  echo "TEST: ${test}: STARTING"
+  "${test}" "${@}" || (echo "TEST: ${test}: FAILED" && exit 1)
+  echo "TEST: ${test}: PASSED"
+done

From 86f1d125dc9997f0952106afab2b96249765ea75 Mon Sep 17 00:00:00 2001
From: Matt Clay <matt@mystile.com>
Date: Mon, 6 Apr 2020 21:18:31 -0700
Subject: [PATCH 3/6] Add tests for collection files tracked by git.

---
 .../git-at-collection-base.sh                 | 10 +++++
 .../git-at-collection-root.sh                 | 10 +++++
 .../collection-tests/git-common.bash          | 43 +++++++++++++++++++
 .../collection-tests/install-git.yml          |  5 +++
 4 files changed, 68 insertions(+)
 create mode 100755 test/integration/targets/ansible-test/collection-tests/git-at-collection-base.sh
 create mode 100755 test/integration/targets/ansible-test/collection-tests/git-at-collection-root.sh
 create mode 100755 test/integration/targets/ansible-test/collection-tests/git-common.bash
 create mode 100644 test/integration/targets/ansible-test/collection-tests/install-git.yml

diff --git a/test/integration/targets/ansible-test/collection-tests/git-at-collection-base.sh b/test/integration/targets/ansible-test/collection-tests/git-at-collection-base.sh
new file mode 100755
index 00000000000000..31ebfbbfc36613
--- /dev/null
+++ b/test/integration/targets/ansible-test/collection-tests/git-at-collection-base.sh
@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+set -eux -o pipefail
+
+export GIT_TOP_LEVEL SUBMODULE_DST
+
+GIT_TOP_LEVEL="${WORK_DIR}/super/ansible_collections/ns/col"
+SUBMODULE_DST="sub"
+
+source collection-tests/git-common.bash
diff --git a/test/integration/targets/ansible-test/collection-tests/git-at-collection-root.sh b/test/integration/targets/ansible-test/collection-tests/git-at-collection-root.sh
new file mode 100755
index 00000000000000..8af4387a3d0da8
--- /dev/null
+++ b/test/integration/targets/ansible-test/collection-tests/git-at-collection-root.sh
@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+set -eux -o pipefail
+
+export GIT_TOP_LEVEL SUBMODULE_DST
+
+GIT_TOP_LEVEL="${WORK_DIR}/super"
+SUBMODULE_DST="ansible_collections/ns/col/sub"
+
+source collection-tests/git-common.bash
diff --git a/test/integration/targets/ansible-test/collection-tests/git-common.bash b/test/integration/targets/ansible-test/collection-tests/git-common.bash
new file mode 100755
index 00000000000000..069b157c65fc18
--- /dev/null
+++ b/test/integration/targets/ansible-test/collection-tests/git-common.bash
@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+
+set -eux -o pipefail
+
+# make sure git is installed
+git --version || ansible-playbook collection-tests/install-git.yml -i ../../inventory "$@"
+
+# init sub project
+mkdir "${WORK_DIR}/sub"
+cd "${WORK_DIR}/sub"
+touch "README.md"
+git init
+git config user.name 'Ansible Test'
+git config user.email 'ansible-test@ansible.com'
+git add "README.md"
+git commit -m "Initial commit."
+
+# init super project
+rm -rf "${WORK_DIR}/super" # needed when re-creating in place
+mkdir "${WORK_DIR}/super"
+cp -a "${TEST_DIR}/ansible_collections" "${WORK_DIR}/super"
+cd "${GIT_TOP_LEVEL}"
+git init
+
+# add submodule
+git submodule add "${WORK_DIR}/sub" "${SUBMODULE_DST}"
+
+# prepare for tests
+expected="${WORK_DIR}/expected.txt"
+actual="${WORK_DIR}/actual.txt"
+cd "${WORK_DIR}/super/ansible_collections/ns/col"
+mkdir tests/.git
+touch tests/.git/keep.txt  # make sure ansible-test correctly ignores version control within collection subdirectories
+find . -type f ! -path '*/.git/*' ! -name .git | sed 's|^\./||' | sort >"${expected}"
+set -x
+
+# test at the collection base
+ansible-test env --list-files | sort >"${actual}"
+diff --unified "${expected}" "${actual}"
+
+# test at the submodule base
+(cd sub && ansible-test env --list-files | sort >"${actual}")
+diff --unified "${expected}" "${actual}"
diff --git a/test/integration/targets/ansible-test/collection-tests/install-git.yml b/test/integration/targets/ansible-test/collection-tests/install-git.yml
new file mode 100644
index 00000000000000..29adead7dedf71
--- /dev/null
+++ b/test/integration/targets/ansible-test/collection-tests/install-git.yml
@@ -0,0 +1,5 @@
+- hosts: localhost
+  tasks:
+    - name: Make sure git is installed
+      package:
+        name: git

From 2b4e31110a7cabd11db1404c34ed1a1b75fe2528 Mon Sep 17 00:00:00 2001
From: Matt Clay <matt@mystile.com>
Date: Tue, 7 Apr 2020 02:05:45 -0700
Subject: [PATCH 4/6] Fix ansible-test submodule usage on older git.

---
 .../ansible-test-git-submodule-top-level.yml  |  2 ++
 .../_internal/provider/source/git.py          | 22 ++++++++++++++++---
 2 files changed, 21 insertions(+), 3 deletions(-)
 create mode 100644 changelogs/fragments/ansible-test-git-submodule-top-level.yml

diff --git a/changelogs/fragments/ansible-test-git-submodule-top-level.yml b/changelogs/fragments/ansible-test-git-submodule-top-level.yml
new file mode 100644
index 00000000000000..6d29cc6ad5b3dd
--- /dev/null
+++ b/changelogs/fragments/ansible-test-git-submodule-top-level.yml
@@ -0,0 +1,2 @@
+bugfixes:
+    - ansible-test now supports submodules using older ``git`` versions which require querying status from the top level directory of the repo.
diff --git a/test/lib/ansible_test/_internal/provider/source/git.py b/test/lib/ansible_test/_internal/provider/source/git.py
index 081fbc44e2ccb8..5e7a2a8ef6628a 100644
--- a/test/lib/ansible_test/_internal/provider/source/git.py
+++ b/test/lib/ansible_test/_internal/provider/source/git.py
@@ -14,6 +14,10 @@
     to_bytes,
 )
 
+from ...util import (
+    SubprocessError,
+)
+
 from . import (
     SourceProvider,
 )
@@ -28,11 +32,23 @@ def is_content_root(path):  # type: (str) -> bool
 
     def get_paths(self, path):  # type: (str) -> t.List[str]
         """Return the list of available content paths under the given path."""
-        git = Git(path)
-
         paths = self.__get_paths(path)
 
-        submodule_paths = git.get_submodule_paths()
+        try:
+            submodule_paths = Git(path).get_submodule_paths()
+        except SubprocessError:
+            if path == self.root:
+                raise
+
+            # older versions of git require submodule commands to be executed from the top level of the working tree
+            # git version 2.18.1 (centos8) does not have this restriction
+            # git version 1.8.3.1 (centos7) does
+            # fall back to using the top level directory of the working tree only when needed
+            # this avoids penalizing newer git versions with a potentially slower analysis due to additional submodules
+            rel_path = os.path.relpath(path, self.root) + os.path.sep
+
+            submodule_paths = Git(self.root).get_submodule_paths()
+            submodule_paths = [os.path.relpath(p, rel_path) for p in submodule_paths if p.startswith(rel_path)]
 
         for submodule_path in submodule_paths:
             paths.extend(os.path.join(submodule_path, p) for p in self.__get_paths(os.path.join(path, submodule_path)))

From 1aa23dbb688932a62fb64b6ab48a2886d8624bcd Mon Sep 17 00:00:00 2001
From: Matt Clay <matt@mystile.com>
Date: Tue, 7 Apr 2020 02:57:45 -0700
Subject: [PATCH 5/6] Fix submodule directory detection as files.

---
 changelogs/fragments/ansible-test-submodule-dir-as-file.yml | 2 ++
 test/lib/ansible_test/_internal/provider/source/git.py      | 3 +++
 2 files changed, 5 insertions(+)
 create mode 100644 changelogs/fragments/ansible-test-submodule-dir-as-file.yml

diff --git a/changelogs/fragments/ansible-test-submodule-dir-as-file.yml b/changelogs/fragments/ansible-test-submodule-dir-as-file.yml
new file mode 100644
index 00000000000000..fef8b6c0676c8c
--- /dev/null
+++ b/changelogs/fragments/ansible-test-submodule-dir-as-file.yml
@@ -0,0 +1,2 @@
+minor_changes:
+    - ansible-test no longer detects ``git`` submodule directories as files.
diff --git a/test/lib/ansible_test/_internal/provider/source/git.py b/test/lib/ansible_test/_internal/provider/source/git.py
index 5e7a2a8ef6628a..0bf81a1cee3eb6 100644
--- a/test/lib/ansible_test/_internal/provider/source/git.py
+++ b/test/lib/ansible_test/_internal/provider/source/git.py
@@ -53,6 +53,9 @@ def get_paths(self, path):  # type: (str) -> t.List[str]
         for submodule_path in submodule_paths:
             paths.extend(os.path.join(submodule_path, p) for p in self.__get_paths(os.path.join(path, submodule_path)))
 
+        # git reports submodule directories as regular files
+        paths = [p for p in paths if p not in submodule_paths]
+
         return paths
 
     @staticmethod

From 8b2cb69f89dd23bb58adf46613eef4ea82944eb8 Mon Sep 17 00:00:00 2001
From: Matt Clay <matt@mystile.com>
Date: Tue, 7 Apr 2020 14:40:12 -0700
Subject: [PATCH 6/6] Improve handling of nested source control.

---
 .../ansible-test-nested-source-control.yml     |  2 ++
 test/lib/ansible_test/_internal/data.py        | 18 +++++-------------
 2 files changed, 7 insertions(+), 13 deletions(-)
 create mode 100644 changelogs/fragments/ansible-test-nested-source-control.yml

diff --git a/changelogs/fragments/ansible-test-nested-source-control.yml b/changelogs/fragments/ansible-test-nested-source-control.yml
new file mode 100644
index 00000000000000..85a347c53c114d
--- /dev/null
+++ b/changelogs/fragments/ansible-test-nested-source-control.yml
@@ -0,0 +1,2 @@
+bugfixes:
+  - ansible-test now ignores version control within subdirectories of collections. Previously this condition was an error.
diff --git a/test/lib/ansible_test/_internal/data.py b/test/lib/ansible_test/_internal/data.py
index 6f8cbf45cac090..38ae6d210e1232 100644
--- a/test/lib/ansible_test/_internal/data.py
+++ b/test/lib/ansible_test/_internal/data.py
@@ -40,15 +40,6 @@
 )
 
 
-class UnexpectedSourceRoot(ApplicationError):
-    """Exception generated when a source root is found below a layout root."""
-    def __init__(self, source_root, layout_root):  # type: (str, str) -> None
-        super(UnexpectedSourceRoot, self).__init__('Source root "%s" cannot be below layout root "%s".' % (source_root, layout_root))
-
-        self.source_root = source_root
-        self.layout_root = layout_root
-
-
 class DataContext:
     """Data context providing details about the current execution environment for ansible-test."""
     def __init__(self):
@@ -121,13 +112,14 @@ def __create_content_layout(layout_providers,  # type: t.List[t.Type[LayoutProvi
         layout_provider = find_path_provider(LayoutProvider, layout_providers, root, walk)
 
         try:
-            source_provider = find_path_provider(SourceProvider, source_providers, root, walk)
+            # Begin the search for the source provider at the layout provider root.
+            # This intentionally ignores version control within subdirectories of the layout root, a condition which was previously an error.
+            # Doing so allows support for older git versions for which it is difficult to distinguish between a super project and a sub project.
+            # It also provides a better user experience, since the solution for the user would effectively be the same -- to remove the nested version control.
+            source_provider = find_path_provider(SourceProvider, source_providers, layout_provider.root, walk)
         except ProviderNotFoundForPath:
             source_provider = UnversionedSource(layout_provider.root)
 
-        if source_provider.root != layout_provider.root and is_subdir(source_provider.root, layout_provider.root):
-            raise UnexpectedSourceRoot(source_provider.root, layout_provider.root)
-
         layout = layout_provider.create(layout_provider.root, source_provider.get_paths(layout_provider.root))
 
         return layout