Create option to build compilers as needed

.travis.yml

@@ -211,10 +211,12 @@ script:
 
 after_success:
   - ccache -s
-  - if [[ "$TEST_SUITE" == "unit" || "$TEST_SUITE" == "build" ]]; then
-        codecov --env PYTHON_VERSION
-                --required
-                --flags "${TEST_SUITE}${TRAVIS_OS_NAME}";
+  - if [[ "$TEST_SUITE" == "unit" ]]; then
+        if [[ "$PYTHON_VERSION == "2.7" || "$PYTHON_VERSION" == "3.6" ]]; then
+            codecov --env PYTHON_VERSION
+                    --required
+                    --flags "${TEST_SUITE}${TRAVIS_OS_NAME}";
+        fi
     fi
 
 #=============================================================================

etc/spack/defaults/config.yaml

@@ -66,6 +66,12 @@ config:
   verify_ssl: true
 
 
+  # If set to true, Spack will attempt to build any compiler on the spec
+  # that is not already available. If set to False, Spack will only use
+  # compilers already configured in compilers.yaml
+  install_missing_compilers: False
+
+
   # If set to true, Spack will always check checksums after downloading
   # archives. If false, Spack skips the checksum step.
   checksum: true

lib/spack/spack/compilers/__init__.py

@@ -28,6 +28,17 @@
 #: cache of compilers constructed from config data, keyed by config entry id.
 _compiler_cache = {}
 
+_compiler_to_pkg = {
+    'clang': 'llvm+clang'
+}
+
+
+def pkg_spec_for_compiler(cspec):
+    """Return the spec of the package that provides the compiler."""
+    spec_str = '%s@%s' % (_compiler_to_pkg.get(cspec.name, cspec.name),
+                          cspec.versions)
+    return spack.spec.Spec(spec_str)
+
 
 def _auto_compiler_spec(function):
     def converter(cspec_like, *args, **kwargs):
@@ -203,6 +214,17 @@ def find(compiler_spec, scope=None, init_config=True):
             if c.satisfies(compiler_spec)]
 
 
+@_auto_compiler_spec
+def find_by_arch(compiler_spec, arch_spec, scope=None, init_config=True):
+    """Return specs of available compilers that match the supplied
+       compiler spec.  Return an empty list if nothing found."""
+    return [c.spec for c in compilers_for_spec(compiler_spec,
+                                               arch_spec,
+                                               scope,
+                                               True,
+                                               init_config)]
+
+
 def all_compilers(scope=None):
     config = get_compiler_config(scope)
     compilers = list()

lib/spack/spack/concretize.py

@@ -28,6 +28,7 @@
 import spack.compilers
 import spack.architecture
 import spack.error
+from spack.config import config
 from spack.version import ver, Version, VersionList, VersionRange
 from spack.package_prefs import PackagePrefs, spec_externals, is_spec_buildable
 
@@ -47,7 +48,8 @@ class Concretizer(object):
     def __init__(self):
         # controls whether we check that compiler versions actually exist
         # during concretization. Used for testing and for mirror creation
-        self.check_for_compiler_existence = True
+        self.check_for_compiler_existence = not config.get(
+            'config:install_missing_compilers', False)
 
     @contextmanager
     def disable_compiler_existence_check(self):
@@ -56,6 +58,13 @@ def disable_compiler_existence_check(self):
         yield
         self.check_for_compiler_existence = saved
 
+    @contextmanager
+    def enable_compiler_existence_check(self):
+        saved = self.check_for_compiler_existence
+        self.check_for_compiler_existence = True
+        yield
+        self.check_for_compiler_existence = saved
+
     def _valid_virtuals_and_externals(self, spec):
         """Returns a list of candidate virtual dep providers and external
            packages that coiuld be used to concretize a spec.
@@ -303,36 +312,47 @@ def _proper_compiler_style(cspec, aspec):
         assert(other_spec)
 
         # Check if the compiler is already fully specified
-        if (other_compiler and other_compiler.concrete and
-                not self.check_for_compiler_existence):
-            spec.compiler = other_compiler.copy()
-            return True
-
-        all_compiler_specs = spack.compilers.all_compiler_specs()
-        if not all_compiler_specs:
-            # If compiler existence checking is disabled, then we would have
-            # exited by now if there were sufficient hints to form a full
-            # compiler spec. Therefore even if compiler existence checking is
-            # disabled, compilers must be available at this point because the
-            # available compilers are used to choose a compiler. If compiler
-            # existence checking is enabled then some compiler must exist in
-            # order to complete the spec.
-            raise spack.compilers.NoCompilersError()
-
-        if other_compiler in all_compiler_specs:
-            spec.compiler = other_compiler.copy()
-            if not _proper_compiler_style(spec.compiler, spec.architecture):
+        if other_compiler and other_compiler.concrete:
+            if (self.check_for_compiler_existence and not
+                    _proper_compiler_style(other_compiler, spec.architecture)):
                 _compiler_concretization_failure(
-                    spec.compiler, spec.architecture)
+                    other_compiler, spec.architecture)
+            spec.compiler = other_compiler
             return True
 
-        # Filter the compilers into a sorted list based on the compiler_order
-        # from spackconfig
-        compiler_list = all_compiler_specs if not other_compiler else \
-            spack.compilers.find(other_compiler)
-        if not compiler_list:
-            # No compiler with a satisfactory spec was found
-            raise UnavailableCompilerVersionError(other_compiler)
+        if other_compiler:
+            compiler_list = spack.compilers.find_by_arch(other_compiler,
+                                                         spec.architecture)
+            if not compiler_list and not self.check_for_compiler_existence:
+                # Concretize compiler spec versions as a package
+                cpkg_spec = spack.compilers.pkg_spec_for_compiler(
+                    other_compiler
+                )
+                self.concretize_version(cpkg_spec)
+                spec.compiler.versions = cpkg_spec.versions
+                return True
+            if not compiler_list:
+                # No compiler with a satisfactory spec was found
+                raise UnavailableCompilerVersionError(other_compiler)
+        else:
+            # We have no hints to go by, grab any compiler
+            compiler_list = spack.compilers.all_compiler_specs()
+            if not compiler_list:
+                # Spack has no compilers. Build the most preferred compiler
+                pref = PackagePrefs.order_for_package(spec.name, 'compiler')
+                if not pref:
+                    # If no preferences, just error out
+                    raise spack.compilers.NoCompilersError()
+
+                # Concretize the compiler version as a package and add to spec
+                cpkg_spec = spack.compilers.pkg_spec_for_compiler(
+                    pref[0]
+                )
+                self.concretize_version(cpkg_spec)
+                spec.compiler = spack.spec.CompilerSpec(
+                    cpkg_spec.name, cpkg_spec.versions
+                )
+                return True
 
         # By default, prefer later versions of compilers
         compiler_list = sorted(

lib/spack/spack/package.py

@@ -61,6 +61,7 @@
 from spack.version import Version
 from spack.package_prefs import get_package_dir_permissions, get_package_group
 
+
 """Allowed URL schemes for spack packages."""
 _ALLOWED_URL_SCHEMES = ["http", "https", "ftp", "file", "git"]
 
@@ -1410,6 +1411,34 @@ def do_install(self,
                     dirty=dirty,
                     **kwargs)
 
+        # Then, install the compiler if it is not already installed.
+        if install_deps:
+            compilers = spack.compilers.compilers_for_spec(
+                self.spec.compiler,
+                arch_spec=self.spec.architecture
+            )
+            if not compilers:
+                dep = spack.compilers.pkg_spec_for_compiler(self.spec.compiler)
+                dep.concretize()  # compiler specs don't have all info concrete
+                dep.package.do_install(
+                    install_deps=True,
+                    explicit=False,
+                    keep_prefix=keep_prefix,
+                    keep_stage=keep_stage,
+                    install_source=install_source,
+                    fake=fake,
+                    skip_patch=skip_patch,
+                    verbose=verbose,
+                    make_jobs=make_jobs,
+                    tests=tests,
+                    dirty=dirty,
+                    **kwargs
+                )
+                spack.compilers.add_compilers_to_config(
+                    spack.compilers.find_compilers(dep.prefix)
+                )
+
+        # Then, install the package proper
         tty.msg(colorize('@*{Installing} @*g{%s}' % self.name))
 
         if kwargs.get('use_cache', True):

lib/spack/spack/package_prefs.py

@@ -115,7 +115,7 @@ def _packages_config(cls):
         return cls._packages_config_cache
 
     @classmethod
-    def _order_for_package(cls, pkgname, component, vpkg=None, all=True):
+    def order_for_package(cls, pkgname, component, vpkg=None, all=True):
         """Given a package name, sort component (e.g, version, compiler, ...),
            and an optional vpkg, return the list from the packages config.
         """
@@ -151,7 +151,7 @@ def _specs_for_pkg(cls, pkgname, component, vpkg=None):
 
         specs = cls._spec_cache.get(key)
         if specs is None:
-            pkglist = cls._order_for_package(pkgname, component, vpkg)
+            pkglist = cls.order_for_package(pkgname, component, vpkg)
             spec_type = _spec_type(component)
             specs = [spec_type(s) for s in pkglist]
             cls._spec_cache[key] = specs
@@ -166,7 +166,7 @@ def clear_caches(cls):
     @classmethod
     def has_preferred_providers(cls, pkgname, vpkg):
         """Whether specific package has a preferred vpkg providers."""
-        return bool(cls._order_for_package(pkgname, 'providers', vpkg, False))
+        return bool(cls.order_for_package(pkgname, 'providers', vpkg, False))
 
     @classmethod
     def preferred_variants(cls, pkg_name):

lib/spack/spack/schema/config.py

@@ -41,6 +41,7 @@
             'source_cache': {'type': 'string'},
             'misc_cache': {'type': 'string'},
             'verify_ssl': {'type': 'boolean'},
+            'install_missing_compilers': {'type': 'boolean'},
             'debug': {'type': 'boolean'},
             'checksum': {'type': 'boolean'},
             'locks': {'type': 'boolean'},

lib/spack/spack/test/concretize.py

@@ -130,9 +130,11 @@ def test_concretize_with_restricted_virtual(self):
         concrete = check_concretize('mpileaks   ^mpich2@1.3.1:1.4')
         assert concrete['mpich2'].satisfies('mpich2@1.3.1:1.4')
 
-    def test_concretize_disable_compiler_existence_check(self):
-        with pytest.raises(spack.concretize.UnavailableCompilerVersionError):
-            check_concretize('dttop %gcc@100.100')
+    def test_concretize_enable_disable_compiler_existence_check(self):
+        with spack.concretize.concretizer.enable_compiler_existence_check():
+            with pytest.raises(
+                    spack.concretize.UnavailableCompilerVersionError):
+                check_concretize('dttop %gcc@100.100')
 
         with spack.concretize.concretizer.disable_compiler_existence_check():
             spec = check_concretize('dttop %gcc@100.100')
@@ -267,10 +269,13 @@ def test_concretize_two_virtuals_with_dual_provider_and_a_conflict(
         with pytest.raises(spack.spec.MultipleProviderError):
             s.concretize()
 
-    def test_no_matching_compiler_specs(self):
-        s = Spec('a %gcc@0.0.0')
-        with pytest.raises(spack.concretize.UnavailableCompilerVersionError):
-            s.concretize()
+    def test_no_matching_compiler_specs(self, mock_config):
+        # only relevant when not building compilers as needed
+        with spack.concretize.concretizer.enable_compiler_existence_check():
+            s = Spec('a %gcc@0.0.0')
+            with pytest.raises(
+                    spack.concretize.UnavailableCompilerVersionError):
+                s.concretize()
 
     def test_no_compilers_for_arch(self):
         s = Spec('a arch=linux-rhel0-x86_64')