Adding ability to compare git references to spack install

lib/spack/spack/fetch_strategy.py

@@ -49,10 +49,10 @@
 import spack.util.pattern as pattern
 import spack.util.url as url_util
 import spack.util.web as web_util
+import spack.version
 from spack.util.compression import decompressor_for, extension
 from spack.util.executable import CommandNotFoundError, which
 from spack.util.string import comma_and, quote
-from spack.version import Version, ver
 
 #: List of all fetch strategies, created by FetchStrategy metaclass.
 all_strategies = []
@@ -750,7 +750,7 @@ def __init__(self, **kwargs):
     @property
     def go_version(self):
         vstring = self.go('version', output=str).split(' ')[2]
-        return Version(vstring)
+        return spack.version.Version(vstring)
 
     @property
     def go(self):
@@ -843,7 +843,7 @@ def version_from_git(git_exe):
         """
         version_output = git_exe('--version', output=str)
         m = re.search(GitFetchStrategy.git_version_re, version_output)
-        return Version(m.group(1))
+        return spack.version.Version(m.group(1))
 
     @property
     def git(self):
@@ -852,7 +852,7 @@ def git(self):
 
             # Disable advice for a quieter fetch
             # https://github.com/git/git/blob/master/Documentation/RelNotes/1.7.2.txt
-            if self.git_version >= Version('1.7.2'):
+            if self.git_version >= spack.version.Version('1.7.2'):
                 self._git.add_default_arg('-c')
                 self._git.add_default_arg('advice.detachedHead=false')
 
@@ -895,44 +895,71 @@ def fetch(self):
             tty.debug('Already fetched {0}'.format(self.stage.source_path))
             return
 
+        self.clone(commit=self.commit, branch=self.branch, tag=self.tag)
+
+    def clone(self, dest=None, commit=None, branch=None, tag=None, bare=False):
+        """
+        Clone a repository to a path.
+
+        This method handles cloning from git, but does not require a stage.
+
+        Arguments:
+            dest (str or None): The path into which the code is cloned. If None,
+                requires a stage and uses the stage's source path.
+            commit (str or None): A commit to fetch from the remote. Only one of
+                commit, branch, and tag may be non-None.
+            branch (str or None): A branch to fetch from the remote.
+            tag (str or None): A tag to fetch from the remote.
+            bare (bool): Execute a "bare" git clone (--bare option to git)
+        """
+        # Default to spack source path
+        dest = dest or self.stage.source_path
         tty.debug('Cloning git repository: {0}'.format(self._repo_info()))
 
         git = self.git
-        if self.commit:
+        debug = spack.config.get('config:debug')
+
+        if bare:
+            # We don't need to worry about which commit/branch/tag is checked out
+            clone_args = ['clone', '--bare']
+            if not debug:
+                clone_args.append('--quiet')
+            clone_args.extend([self.url, dest])
+            git(*clone_args)
+        elif commit:
             # Need to do a regular clone and check out everything if
             # they asked for a particular commit.
-            debug = spack.config.get('config:debug')
-
             clone_args = ['clone', self.url]
             if not debug:
                 clone_args.insert(1, '--quiet')
             with temp_cwd():
                 git(*clone_args)
                 repo_name = get_single_file('.')
-                self.stage.srcdir = repo_name
-                shutil.move(repo_name, self.stage.source_path)
+                if self.stage:
+                    self.stage.srcdir = repo_name
+                shutil.move(repo_name, dest)
 
-            with working_dir(self.stage.source_path):
-                checkout_args = ['checkout', self.commit]
+            with working_dir(dest):
+                checkout_args = ['checkout', commit]
                 if not debug:
                     checkout_args.insert(1, '--quiet')
                 git(*checkout_args)
 
         else:
             # Can be more efficient if not checking out a specific commit.
             args = ['clone']
-            if not spack.config.get('config:debug'):
+            if not debug:
                 args.append('--quiet')
 
             # If we want a particular branch ask for it.
-            if self.branch:
-                args.extend(['--branch', self.branch])
-            elif self.tag and self.git_version >= ver('1.8.5.2'):
-                args.extend(['--branch', self.tag])
+            if branch:
+                args.extend(['--branch', branch])
+            elif tag and self.git_version >= spack.version.ver('1.8.5.2'):
+                args.extend(['--branch', tag])
 
             # Try to be efficient if we're using a new enough git.
             # This checks out only one branch's history
-            if self.git_version >= ver('1.7.10'):
+            if self.git_version >= spack.version.ver('1.7.10'):
                 if self.get_full_repo:
                     args.append('--no-single-branch')
                 else:
@@ -942,22 +969,23 @@ def fetch(self):
                 # Yet more efficiency: only download a 1-commit deep
                 # tree, if the in-use git and protocol permit it.
                 if (not self.get_full_repo) and \
-                   self.git_version >= ver('1.7.1') and \
+                   self.git_version >= spack.version.ver('1.7.1') and \
                    self.protocol_supports_shallow_clone():
                     args.extend(['--depth', '1'])
 
                 args.extend([self.url])
                 git(*args)
 
                 repo_name = get_single_file('.')
-                self.stage.srcdir = repo_name
-                shutil.move(repo_name, self.stage.source_path)
+                if self.stage:
+                    self.stage.srcdir = repo_name
+                shutil.move(repo_name, dest)
 
-            with working_dir(self.stage.source_path):
+            with working_dir(dest):
                 # For tags, be conservative and check them out AFTER
                 # cloning.  Later git versions can do this with clone
                 # --branch, but older ones fail.
-                if self.tag and self.git_version < ver('1.8.5.2'):
+                if tag and self.git_version < spack.version.ver('1.8.5.2'):
                     # pull --tags returns a "special" error code of 1 in
                     # older versions that we have to ignore.
                     # see: https://github.com/git/git/commit/19d122b
@@ -971,7 +999,7 @@ def fetch(self):
                     git(*co_args)
 
         if self.submodules_delete:
-            with working_dir(self.stage.source_path):
+            with working_dir(dest):
                 for submodule_to_delete in self.submodules_delete:
                     args = ['rm', submodule_to_delete]
                     if not spack.config.get('config:debug'):
@@ -980,7 +1008,7 @@ def fetch(self):
 
         # Init submodules if the user asked for them.
         if self.submodules:
-            with working_dir(self.stage.source_path):
+            with working_dir(dest):
                 args = ['submodule', 'update', '--init', '--recursive']
                 if not spack.config.get('config:debug'):
                     args.insert(1, '--quiet')
@@ -1502,8 +1530,15 @@ def for_package_version(pkg, version):
 
     check_pkg_attributes(pkg)
 
-    if not isinstance(version, Version):
-        version = Version(version)
+    if not isinstance(version, spack.version.Version):
+        version = spack.version.Version(version)
+
+    # if it's a commit, we must use a GitFetchStrategy
+    if version.is_commit and hasattr(pkg, "git"):
+        # Populate the version with comparisons to other commits
+        version.generate_commit_lookup(pkg)
+        fetcher = GitFetchStrategy(git=pkg.git, commit=str(version))
+        return fetcher
 
     # If it's not a known version, try to extrapolate one by URL
     if version not in pkg.versions:

lib/spack/spack/paths.py

@@ -56,6 +56,9 @@
 reports_path = os.path.join(user_config_path, "reports")
 monitor_path = os.path.join(reports_path, "monitor")
 
+# We cache repositories (git) in first, extracted metadata in second
+user_repos_cache_path = os.path.join(user_config_path, 'git_repos')
+
 opt_path        = os.path.join(prefix, "opt")
 etc_path        = os.path.join(prefix, "etc")
 system_etc_path = '/etc'

lib/spack/spack/spec.py

@@ -4699,6 +4699,15 @@ def do_parse(self):
         except spack.parse.ParseError as e:
             raise SpecParseError(e)
 
+        # Generate lookups for git-commit-based versions
+        for spec in specs:
+            # Cannot do lookups for versions in anonymous specs
+            # Only allow concrete versions using git for now
+            if spec.name and spec.versions.concrete and spec.version.is_commit:
+                pkg = spec.package
+                if hasattr(pkg, 'git'):
+                    spec.version.generate_commit_lookup(pkg)
+
         return specs
 
     def spec_from_file(self):

lib/spack/spack/test/cmd/install.py

@@ -21,6 +21,7 @@
 import spack.environment as ev
 import spack.hash_types as ht
 import spack.package
+import spack.util.executable
 from spack.error import SpackError
 from spack.main import SpackCommand
 from spack.spec import CompilerSpec, Spec
@@ -224,7 +225,7 @@ def test_install_overwrite(
 
 
 def test_install_overwrite_not_installed(
-        mock_packages, mock_archive, mock_fetch, config, install_mockery
+        mock_packages, mock_archive, mock_fetch, config, install_mockery,
 ):
     # Try to install a spec and then to reinstall it.
     spec = Spec('libdwarf')
@@ -236,6 +237,32 @@ def test_install_overwrite_not_installed(
     assert os.path.exists(spec.prefix)
 
 
+def test_install_commit(
+        mock_git_version_info, install_mockery, mock_packages, monkeypatch):
+    """
+    Test installing a git package from a commit.
+
+    This ensures Spack appropriately associates commit versions with their
+    packages in time to do version lookups. Details of version lookup tested elsewhere
+    """
+    repo_path, filename, commits = mock_git_version_info
+    monkeypatch.setattr(spack.package.PackageBase,
+                        'git', 'file://%s' % repo_path,
+                        raising=False)
+
+    commit = commits[-1]
+    spec = spack.spec.Spec('git-test-commit@%s' % commit)
+    spec.concretize()
+    spec.package.do_install()
+
+    # Ensure first commit file contents were written
+    installed = os.listdir(spec.prefix.bin)
+    assert filename in installed
+    with open(spec.prefix.bin.join(filename), 'r') as f:
+        content = f.read().strip()
+    assert content == '[]'  # contents are weird for another test
+
+
 def test_install_overwrite_multiple(
         mock_packages, mock_archive, mock_fetch, config, install_mockery
 ):

lib/spack/spack/test/conftest.py

@@ -61,6 +61,103 @@ def last_two_git_commits(scope='session'):
     yield regex.findall(git_log_out)
 
 
+def write_file(filename, contents):
+    with open(filename, 'w') as f:
+        f.write(contents)
+
+
+commit_counter = 0
+
+
+@pytest.fixture
+def mock_git_version_info(tmpdir, scope="function"):
+    """Create a mock git repo with known structure
+
+    The structure of commits in this repo is as follows::
+
+       | o fourth 1.x commit (1.2)
+       | o third 1.x commit
+       | |
+       o | fourth main commit (v2.0)
+       o | third main commit
+       | |
+       | o second 1.x commit (v1.1)
+       | o first 1.x commit
+       | /
+       |/
+       o second commit (v1.0)
+       o first commit
+
+    The repo consists of a single file, in which the Version._cmp representation
+    of each commit is expressed as a string.
+
+    Important attributes of the repo for test coverage are: multiple branches,
+    version tags on multiple branches, and version order is not equal to time
+    order or topological order.
+    """
+    git = spack.util.executable.which('git', required=True)
+    repo_path = str(tmpdir.mkdir('git_repo'))
+    filename = 'file.txt'
+
+    def commit(message):
+        global commit_counter
+        git('commit', '--date', '2020-01-%02d 12:0:00 +0300' % commit_counter,
+            '-am', message)
+        commit_counter += 1
+
+    with working_dir(repo_path):
+        git("init")
+
+        git('config', 'user.name', 'Spack')
+        git('config', 'user.email', 'spack@spack.io')
+
+        # Add two commits on main branch
+        write_file(filename, '[]')
+        git('add', filename)
+        commit('first commit')
+
+        # Get name of default branch (differs by git version)
+        main = git('rev-parse', '--abbrev-ref', 'HEAD', output=str, error=str).strip()
+
+        # Tag second commit as v1.0
+        write_file(filename, "[1, 0]")
+        commit('second commit')
+        git('tag', 'v1.0')
+
+        # Add two commits and a tag on 1.x branch
+        git('checkout', '-b', '1.x')
+        write_file(filename, "[1, 0, '', 1]")
+        commit('first 1.x commit')
+
+        write_file(filename, "[1, 1]")
+        commit('second 1.x commit')
+        git('tag', 'v1.1')
+
+        # Add two commits and a tag on main branch
+        git('checkout', main)
+        write_file(filename, "[1, 0, '', 1]")
+        commit('third main commit')
+        write_file(filename, "[2, 0]")
+        commit('fourth main commit')
+        git('tag', 'v2.0')
+
+        # Add two more commits on 1.x branch to ensure we aren't cheating by using time
+        git('checkout', '1.x')
+        write_file(filename, "[1, 1, '', 1]")
+        commit('third 1.x commit')
+        write_file(filename, "[1, 2]")
+        commit('fourth 1.x commit')
+        git('tag', '1.2')  # test robust parsing to different syntax, no v
+
+        # Get the commits in topo order
+        log = git('log', '--all', '--pretty=format:%H', '--topo-order',
+                  output=str, error=str)
+        commits = [c for c in log.split('\n') if c]
+
+    # Return the git directory to install, the filename used, and the commits
+    yield repo_path, filename, commits
+
+
 @pytest.fixture(autouse=True)
 def clear_recorded_monkeypatches():
     yield

lib/spack/spack/test/git_fetch.py

@@ -212,7 +212,7 @@ def test_get_full_repo(get_full_repo, git_version, mock_git_repository,
                 ncommits = len(commits)
 
         if get_full_repo:
-            assert(nbranches == 5)
+            assert(nbranches >= 5)
             assert(ncommits == 2)
         else:
             assert(nbranches == 2)