adding ability to compare git references to spack install

This works by way of adding a property, is_commit, to a version, and then given that a version
is a commit, we return a GitFetcherStrategy to use it. In additional, we instruct
the version command to generate a lookup of commits that keeps track of the order
(for comparing between commits) and the previous and next spack version (for comparison
of a commit with a spack version string). If the commit does not have known releases before
it, then the previous is None and we cannot determine the relationship and return False.
The same is true if the commit does not have any known releases after it, although it
is unlikely to hit this case unless the user is asking for a version that has been
released but not added to spack.

Signed-off-by: vsoch <vsoch@users.noreply.github.com>

lib/spack/spack/fetch_strategy.py

@@ -29,21 +29,22 @@
 import re
 import shutil
 import sys
-from typing import Optional, List  # novm
+from typing import List, Optional  # novm
 
-import llnl.util.tty as tty
 import six
 import six.moves.urllib.parse as urllib_parse
+
+import llnl.util.tty as tty
 import spack.config
 import spack.error
 import spack.util.crypto as crypto
 import spack.util.pattern as pattern
 import spack.util.url as url_util
 import spack.util.web as web_util
 from llnl.util.filesystem import (
-    working_dir, mkdirp, temp_rename, temp_cwd, get_single_file)
+    get_single_file, mkdirp, temp_cwd, temp_rename, working_dir)
 from spack.util.compression import decompressor_for, extension
-from spack.util.executable import which, CommandNotFoundError
+from spack.util.executable import CommandNotFoundError, which
 from spack.util.string import comma_and, quote
 from spack.version import Version, ver
 
@@ -889,10 +890,20 @@ def fetch(self):
             tty.debug('Already fetched {0}'.format(self.stage.source_path))
             return
 
+        self.clone(commit=self.commit)
+
+    def clone(self, dest=None, commit=None):
+        """
+        Clone a repository to a path.
+
+        This is the fetch logic, but does not require a stage.
+        """
+        # Default to spack source path
+        dest = dest or self.stage.source_path
         tty.debug('Cloning git repository: {0}'.format(self._repo_info()))
 
         git = self.git
-        if self.commit:
+        if commit:
             # Need to do a regular clone and check out everything if
             # they asked for a particular commit.
             debug = spack.config.get('config:debug')
@@ -903,11 +914,12 @@ def fetch(self):
             with temp_cwd():
                 git(*clone_args)
                 repo_name = get_single_file('.')
-                self.stage.srcdir = repo_name
-                shutil.move(repo_name, self.stage.source_path)
+                if self.stage:
+                    self.stage.srcdir = repo_name
+                shutil.move(repo_name, dest)
 
-            with working_dir(self.stage.source_path):
-                checkout_args = ['checkout', self.commit]
+            with working_dir(dest):
+                checkout_args = ['checkout', commit]
                 if not debug:
                     checkout_args.insert(1, '--quiet')
                 git(*checkout_args)
@@ -944,10 +956,11 @@ def fetch(self):
                 git(*args)
 
                 repo_name = get_single_file('.')
-                self.stage.srcdir = repo_name
-                shutil.move(repo_name, self.stage.source_path)
+                if self.stage:
+                    self.stage.srcdir = repo_name
+                shutil.move(repo_name, dest)
 
-            with working_dir(self.stage.source_path):
+            with working_dir(dest):
                 # For tags, be conservative and check them out AFTER
                 # cloning.  Later git versions can do this with clone
                 # --branch, but older ones fail.
@@ -965,7 +978,7 @@ def fetch(self):
                     git(*co_args)
 
         if self.submodules_delete:
-            with working_dir(self.stage.source_path):
+            with working_dir(dest):
                 for submodule_to_delete in self.submodules_delete:
                     args = ['rm', submodule_to_delete]
                     if not spack.config.get('config:debug'):
@@ -974,7 +987,7 @@ def fetch(self):
 
         # Init submodules if the user asked for them.
         if self.submodules:
-            with working_dir(self.stage.source_path):
+            with working_dir(dest):
                 args = ['submodule', 'update', '--init', '--recursive']
                 if not spack.config.get('config:debug'):
                     args.insert(1, '--quiet')
@@ -1498,6 +1511,24 @@ def for_package_version(pkg, version):
     if not isinstance(version, Version):
         version = Version(version)
 
+    # if it's a commit, we must use a GitURLFetcher
+    if version.is_commit:
+
+        # Figure out the url for the git fetcher
+        repo_regex = "http?s[:]//(github|gitlab)[.]com/(?P<repo>.+)/(?P<name>.+)/"
+        match = re.search(repo_regex, pkg.url)
+        if not match:
+            tty.die("Cannot derive repository from %s." % pkg.url)
+        service, repository, _ = match.groups()
+        repository = "https://%s.com/%s" % (service, repository)
+
+        # Create a GitFetchStrategy for the specific commit
+        fetcher = GitFetchStrategy(git=repository, commit=str(version))
+
+        # Populate the version with comparisons to other commits
+        version.generate_commit_lookup(fetcher, pkg.versions)
+        return fetcher
+
     # If it's not a known version, try to extrapolate one by URL
     if version not in pkg.versions:
         return _extrapolate(pkg, version)

lib/spack/spack/package.py

@@ -1329,6 +1329,7 @@ def do_fetch(self, mirror_only=False):
 
         checksum = spack.config.get('config:checksum')
         fetch = self.stage.managed_by_spack
+
         if checksum and fetch and self.version not in self.versions:
             tty.warn("There is no checksum on file to fetch %s safely." %
                      self.spec.cformat('{name}{@version}'))

lib/spack/spack/paths.py

@@ -10,6 +10,7 @@
 dependencies.
 """
 import os
+
 from llnl.util.filesystem import ancestor
 
 #: This file lives in $prefix/lib/spack/spack/__file__
@@ -56,6 +57,10 @@
 reports_path = os.path.join(user_config_path, "reports")
 monitor_path = os.path.join(reports_path, "monitor")
 
+# We cache repositories (git) in first, extracted metadata in second
+user_repos_cache_path = os.path.join(user_config_path, 'git_repos')
+user_repos_metadata_path = os.path.join(user_config_path, 'git_metadata')
+
 opt_path        = os.path.join(prefix, "opt")
 etc_path        = os.path.join(prefix, "etc")
 system_etc_path = '/etc'