From 59f89dd3bee0ee2b668554c537d1d18404108ec4 Mon Sep 17 00:00:00 2001
From: Matthew LeGendre <legendre1@llnl.gov>
Date: Fri, 29 May 2015 11:04:57 -0700
Subject: [PATCH 01/14] Allow long names in format string variables

---
 lib/spack/spack/spec.py | 71 ++++++++++++++++++++++++++++++++++++-----
 1 file changed, 63 insertions(+), 8 deletions(-)

diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py
index e1fbb84423403..3b5d16c7a750c 100644
--- a/lib/spack/spack/spec.py
+++ b/lib/spack/spack/spec.py
@@ -1503,14 +1503,28 @@ def format(self, format_string='$_$@$%@$+$=', **kwargs):
            in the format string.  The format strings you can provide are::
 
                $_   Package name
-               $@   Version
-               $%   Compiler
-               $%@  Compiler & compiler version
-               $+   Options
-               $=   Architecture
-               $#   7-char prefix of DAG hash
+               $@   Version with '@' prefix
+               $%   Compiler with '%' prefix
+               $%@  Compiler with '%' prefix & compiler version with '@' prefix
+               $+   Options 
+               $=   Architecture with '=' prefix
+               $#   7-char prefix of DAG hash with '-' prefix
                $$   $
 
+               You can also use full-string versions, which leave off the prefixes:
+
+               ${PACKAGE}       Package name
+               ${VERSION}       Version
+               ${COMPILER}      Full compiler string
+               ${COMPILERNAME}  Compiler name
+               ${COMPILERVER}   Compiler version
+               ${OPTIONS}       Options
+               ${ARCHITECTURE}  Architecture
+               ${SHA1}          Dependencies 8-char sha1 prefix
+
+               ${SPACK_ROOT}    The spack root directory
+               ${SPACK_INSTALL} The default spack install directory, ${SPACK_PREFIX}/opt
+
            Optionally you can provide a width, e.g. $20_ for a 20-wide name.
            Like printf, you can provide '-' for left justification, e.g.
            $-20_ for a left-justified name.
@@ -1526,7 +1540,8 @@ def format(self, format_string='$_$@$%@$+$=', **kwargs):
         color    = kwargs.get('color', False)
         length = len(format_string)
         out = StringIO()
-        escape = compiler = False
+        named = escape = compiler = False
+        named_str = fmt = ''
 
         def write(s, c):
             if color:
@@ -1566,9 +1581,12 @@ def write(s, c):
                 elif c == '#':
                     out.write('-' + fmt % (self.dag_hash(7)))
                 elif c == '$':
-                    if fmt != '':
+                    if fmt != '%s':
                         raise ValueError("Can't use format width with $$.")
                     out.write('$')
+                elif c == '{':
+                    named = True
+                    named_str = ''
                 escape = False
 
             elif compiler:
@@ -1582,6 +1600,43 @@ def write(s, c):
                     out.write(c)
                 compiler = False
 
+            elif named:
+                if not c == '}':
+                    if i == length - 1:
+                        raise ValueError("Error: unterminated ${ in format: '%s'"
+                                         % format_string)
+                    named_str += c
+                    continue;
+                if named_str == 'PACKAGE':
+                    write(fmt % self.name, '@')
+                if named_str == 'VERSION':
+                    if self.versions and self.versions != _any_version:
+                        write(fmt % str(self.versions), '@')
+                elif named_str == 'COMPILER':
+                    if self.compiler:
+                        write(fmt % self.compiler, '%')
+                elif named_str == 'COMPILERNAME':
+                    if self.compiler:
+                        write(fmt % self.compiler.name, '%')
+                elif named_str == 'COMPILERVER':
+                    if self.compiler:
+                        write(fmt % self.compiler.versions, '%')
+                elif named_str == 'OPTIONS':
+                    if self.variants:
+                        write(fmt % str(self.variants), '+')
+                elif named_str == 'ARCHITECTURE':
+                    if self.architecture:
+                        write(fmt % str(self.architecture), '=')
+                elif named_str == 'SHA1':
+                    if self.dependencies:
+                        out.write(fmt % str(self.dep_hash(8)))
+                elif named_str == 'SPACK_ROOT':
+                    out.write(fmt % spack.prefix)
+                elif named_str == 'SPACK_INSTALL':
+                    out.write(fmt % spack.install_path)
+
+                named = False
+
             elif c == '$':
                 escape = True
                 if i == length - 1:

From b5c597b31864826050162b358998e240761c5d7e Mon Sep 17 00:00:00 2001
From: Matthew LeGendre <legendre1@llnl.gov>
Date: Fri, 29 May 2015 12:19:57 -0700
Subject: [PATCH 02/14] Allow specs to be sorted based on preferred packages,
 versions, compilers, variants and dependencies.

---
 lib/spack/spack/__init__.py           |   8 ++
 lib/spack/spack/config.py             |  12 +-
 lib/spack/spack/preferred_packages.py | 172 ++++++++++++++++++++++++++
 lib/spack/spack/spec.py               |  34 +++++
 4 files changed, 222 insertions(+), 4 deletions(-)
 create mode 100644 lib/spack/spack/preferred_packages.py

diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py
index caa09eb6e0030..bd8478fb98e84 100644
--- a/lib/spack/spack/__init__.py
+++ b/lib/spack/spack/__init__.py
@@ -69,6 +69,14 @@
 from spack.directory_layout import YamlDirectoryLayout
 install_layout = YamlDirectoryLayout(install_path)
 
+#
+# This controls how packages are sorted when trying to choose
+# the most preferred package.  More preferred packages are sorted
+# first.
+#
+from spack.preferred_packages import PreferredPackages
+pkgsort = PreferredPackages()
+
 #
 # This controls how things are concretized in spack.
 # Replace it with a subclass if you want different
diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py
index 3e91958c2c20d..dbe225960a846 100644
--- a/lib/spack/spack/config.py
+++ b/lib/spack/spack/config.py
@@ -89,8 +89,8 @@
 import os
 import exceptions
 import sys
+import copy
 
-from external.ordereddict import OrderedDict
 from llnl.util.lang import memoized
 import spack.error
 
@@ -114,8 +114,7 @@ def __init__(self, n, f, m):
 
 _ConfigCategory('compilers', 'compilers.yaml', True)
 _ConfigCategory('mirrors', 'mirrors.yaml', True)
-_ConfigCategory('view', 'views.yaml', True)
-_ConfigCategory('order', 'orders.yaml', True)
+_ConfigCategory('preferred', 'preferred.yaml', True)
 
 """Names of scopes and their corresponding configuration files."""
 config_scopes = [('site', os.path.join(spack.etc_path, 'spack')),
@@ -156,7 +155,7 @@ def _merge_dicts(d1, d2):
     """Recursively merges two configuration trees, with entries
        in d2 taking precedence over d1"""
     if not d1:
-        return d2.copy()
+        return copy.copy(d2)
     if not d2:
         return d1
 
@@ -230,6 +229,11 @@ def get_mirror_config():
     return get_config('mirrors')
 
 
+def get_preferred_config():
+    """Get the preferred configuration from config files"""
+    return get_config('preferred')
+
+
 def get_config_scope_dirname(scope):
     """For a scope return the config directory"""
     global config_scopes
diff --git a/lib/spack/spack/preferred_packages.py b/lib/spack/spack/preferred_packages.py
new file mode 100644
index 0000000000000..248508fe809b7
--- /dev/null
+++ b/lib/spack/spack/preferred_packages.py
@@ -0,0 +1,172 @@
+##############################################################################
+# Copyright (c) 2013, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://scalability-llnl.github.io/spack
+# Please also see the LICENSE file for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License (as published by
+# the Free Software Foundation) version 2.1 dated February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+
+import spack
+from spack.version import *
+
+class PreferredPackages(object):
+    _default_order = {'compiler' : [ 'gcc', 'intel', 'clang', 'pgi', 'xlc' ] }, #Arbitrary, but consistent
+
+    def __init__(self):
+        self.preferred = spack.config.get_preferred_config()
+        self._spec_for_pkgname_cache = {}
+
+    #Given a package name, sort component (e.g, version, compiler, ...), and
+    # a second_key (used by providers), return the list 
+    def _order_for_package(self, pkgname, component, second_key):
+        pkglist = [pkgname]
+        pkglist.append('all')
+        for pkg in pkglist:
+            if not pkg in self.preferred:
+                continue
+            orders = self.preferred[pkg]
+            if not type(orders) is dict:
+                continue
+            if not component in orders:
+                continue
+            order = orders[component]
+            if type(order) is dict:
+                if not second_key in order:
+                    continue;
+                order = order[second_key]
+            if not type(order) is str:
+                tty.die('Expected version list in preferred config, but got %s' % str(order))
+            order_list = order.split(',')
+            return [s.strip() for s in order_list]
+        return []
+            
+
+    # A generic sorting function. Given a package name and sort 
+    # component, return less-than-0, 0, or greater-than-0 if 
+    # a is respectively less-than, equal to, or greater than b.
+    def _component_compare(self, pkgname, component, a, b, reverse_natural_compare, second_key):
+        orderlist = self._order_for_package(pkgname, component, second_key)
+        a_in_list = str(a) in orderlist
+        b_in_list = str(b) in orderlist
+        if a_in_list and not b_in_list:
+            return -1
+        elif b_in_list and not a_in_list:
+            return 1
+
+        cmp_a = None
+        cmp_b = None
+        reverse = None
+        if not a_in_list and not b_in_list:
+            cmp_a = a
+            cmp_b = b
+            reverse = -1 if reverse_natural_compare else 1
+        else:
+            cmp_a = orderlist.index(str(a))
+            cmp_b = orderlist.index(str(b))
+            reverse = 1
+        
+        if cmp_a < cmp_b:
+            return -1 * reverse
+        elif cmp_a > cmp_b:
+            return 1 * reverse
+        else:
+            return 0
+
+
+    # A sorting function for specs.  Similar to component_compare, but
+    # a and b are considered to match entries in the sorting list if they
+    # satisfy the list component.  
+    def _spec_compare(self, pkgname, component, a, b, reverse_natural_compare, second_key):
+        specs = self._spec_for_pkgname(pkgname, component, second_key)
+        a_index = None
+        b_index = None
+        reverse = -1 if reverse_natural_compare else 1
+        for i, cspec in enumerate(specs):
+            if a_index == None and cspec.satisfies(a):
+                a_index = i
+                if b_index:
+                    break
+            if b_index == None and cspec.satisfies(b):
+                b_index = i
+                if a_index:
+                    break
+
+        if   a_index != None and b_index == None: return -1
+        elif a_index == None and b_index != None: return 1
+        elif a_index != None and b_index == a_index: return -1 * cmp(a, b)
+        elif a_index != None and b_index != None and a_index != b_index: return cmp(a_index, b_index)
+        elif a < b: return 1 * reverse
+        elif b < a: return -1 * reverse
+        else: return 0
+
+
+    # Given a sort order specified by the pkgname/component/second_key, return
+    # a list of CompilerSpecs, VersionLists, or Specs for that sorting list.
+    def _spec_for_pkgname(self, pkgname, component, second_key):
+        key = (pkgname, component, second_key)
+        if not key in self._spec_for_pkgname_cache:
+            pkglist = self._order_for_package(pkgname, component, second_key)
+            if not pkglist: 
+                if component in self._default_order:
+                    pkglist = self._default_order[component]
+            if component == 'compiler':
+                self._spec_for_pkgname_cache[key] = [spack.spec.CompilerSpec(s) for s in pkglist]
+            elif component == 'version':
+                self._spec_for_pkgname_cache[key] = [VersionList(s) for s in pkglist]
+            else:
+                self._spec_for_pkgname_cache[key] = [spack.spec.Spec(s) for s in pkglist]
+        return self._spec_for_pkgname_cache[key]
+
+    
+    def provider_compare(self, pkgname, provider_str, a, b):
+        """Return less-than-0, 0, or greater than 0 if a is respecively less-than, equal-to, or 
+           greater-than b. A and b are possible implementations of provider_str.
+           One provider is less-than another if it is preferred over the other.
+           For example, provider_compare('scorep', 'mpi', 'mvapich', 'openmpi') would return -1 if
+           mvapich should be preferred over openmpi for scorep."""
+        return self._spec_compare(pkgname, 'providers', a, b, False, provider_str)
+
+
+    def version_compare(self, pkgname, a, b):
+        """Return less-than-0, 0, or greater than 0 if version a of pkgname is 
+           respecively less-than, equal-to, or greater-than version b of pkgname. 
+           One version is less-than another if it is preferred over the other."""
+        return self._spec_compare(pkgname, 'version', a, b, False, None)
+
+    
+    def variant_compare(self, pkgname, a, b):
+        """Return less-than-0, 0, or greater than 0 if variant a of pkgname is 
+           respecively less-than, equal-to, or greater-than variant b of pkgname. 
+           One variant is less-than another if it is preferred over the other."""
+        return self._component_compare(pkgname, 'variant', a, b, False, None)
+
+
+    def architecture_compare(self, pkgname, a, b):
+        """Return less-than-0, 0, or greater than 0 if architecture a of pkgname is 
+           respecively less-than, equal-to, or greater-than architecture b of pkgname. 
+           One architecture is less-than another if it is preferred over the other."""
+        return self._component_compare(pkgname, 'architecture', a, b, False, None)
+
+
+    def compiler_compare(self, pkgname, a, b):
+        """Return less-than-0, 0, or greater than 0 if compiler a of pkgname is 
+           respecively less-than, equal-to, or greater-than compiler b of pkgname. 
+           One compiler is less-than another if it is preferred over the other."""
+        return self._spec_compare(pkgname, 'compiler', a, b, False, None)
diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py
index 3b5d16c7a750c..83b1416e36d7f 100644
--- a/lib/spack/spack/spec.py
+++ b/lib/spack/spack/spec.py
@@ -1653,6 +1653,40 @@ def dep_string(self):
         return ''.join("^" + dep.format() for dep in self.sorted_deps())
 
 
+    def __cmp__(self, other):
+        #Package name sort order is not configurable, always goes alphabetical
+        if self.name != other.name:
+            return cmp(self.name, other.name)
+        
+        #Package version is second in compare order
+        pkgname = self.name
+        if self.versions != other.versions:
+            return spack.pkgsort.version_compare(pkgname,
+                         self.versions, other.versions)
+
+        #Compiler is third
+        if self.compiler != other.compiler:
+            return spack.pkgsort.compiler_compare(pkgname,
+                         self.compiler, other.compiler)
+
+        #Variants
+        if self.variants != other.variants:
+            return spack.pkgsort.variant_compare(pkgname,
+                         self.variants, other.variants)
+
+        #Architecture
+        if self.architecture != other.architecture:
+            return spack.pkgsort.architecture_compare(pkgname,
+                         self.architecture, other.architecture)
+
+        #Dependency is not configurable
+        if self.dep_hash() != other.dep_hash():
+            return -1 if self.dep_hash() < other.dep_hash() else 1
+
+        #Equal specs
+        return 0
+
+
     def __str__(self):
         return self.format() + self.dep_string()
 

From 8d7b7e5d5dadcec9b997b94d95898a4134e122b2 Mon Sep 17 00:00:00 2001
From: Matthew LeGendre <legendre1@llnl.gov>
Date: Fri, 29 May 2015 12:20:32 -0700
Subject: [PATCH 03/14] Use preferred package rules when concretize'ing specs

---
 lib/spack/spack/concretize.py | 68 +++++++++++++++++++----------------
 lib/spack/spack/spec.py       |  2 +-
 2 files changed, 38 insertions(+), 32 deletions(-)

diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py
index 66002492cb82e..0f258c909689e 100644
--- a/lib/spack/spack/concretize.py
+++ b/lib/spack/spack/concretize.py
@@ -38,6 +38,7 @@
 import spack.architecture
 import spack.error
 from spack.version import *
+from functools import partial
 
 
 
@@ -49,8 +50,8 @@ class DefaultConcretizer(object):
 
     def concretize_version(self, spec):
         """If the spec is already concrete, return.  Otherwise take
-           the most recent available version, and default to the package's
-           version if there are no avaialble versions.
+           the preferred version from spackconfig, and default to the package's
+           version if there are no available versions.
 
            TODO: In many cases we probably want to look for installed
                  versions of each package and use an installed version
@@ -68,12 +69,14 @@ def concretize_version(self, spec):
         # If there are known available versions, return the most recent
         # version that satisfies the spec
         pkg = spec.package
+        cmp_versions = partial(spack.pkgsort.version_compare, spec.name)
         valid_versions = sorted(
             [v for v in pkg.versions
-             if any(v.satisfies(sv) for sv in spec.versions)])
+             if any(v.satisfies(sv) for sv in spec.versions)],
+            cmp=cmp_versions)
 
         if valid_versions:
-            spec.versions = ver([valid_versions[-1]])
+            spec.versions = ver([valid_versions[0]])
         else:
             # We don't know of any SAFE versions that match the given
             # spec.  Grab the spec's versions and grab the highest
@@ -138,10 +141,10 @@ def concretize_compiler(self, spec):
         """If the spec already has a compiler, we're done.  If not, then take
            the compiler used for the nearest ancestor with a compiler
            spec and use that.  If the ancestor's compiler is not
-           concrete, then give it a valid version.  If there is no
-           ancestor with a compiler, use the system default compiler.
+           concrete, then used the preferred compiler as specified in 
+           spackconfig.
 
-           Intuition: Use the system default if no package that depends on
+           Intuition: Use the spackconfig default if no package that depends on
            this one has a strict compiler requirement.  Otherwise, try to
            build with the compiler that will be used by libraries that
            link to this one, to maximize compatibility.
@@ -153,40 +156,43 @@ def concretize_compiler(self, spec):
             spec.compiler in all_compilers):
             return False
 
-        try:
-            nearest = next(p for p in spec.traverse(direction='parents')
-                           if p.compiler is not None).compiler
-
-            if not nearest in all_compilers:
-                # Take the newest compiler that saisfies the spec
-                matches = sorted(spack.compilers.find(nearest))
-                if not matches:
-                    raise UnavailableCompilerVersionError(nearest)
-
-                # copy concrete version into nearest spec
-                nearest.versions = matches[-1].versions.copy()
-                assert(nearest.concrete)
-
-            spec.compiler = nearest.copy()
-
-        except StopIteration:
-            spec.compiler = spack.compilers.default_compiler().copy()
-
+        # Find the parent spec that has a compiler, or the root if none do
+        parent_spec = next(p for p in spec.traverse(direction='parents')
+                           if p.compiler is not None or not p.dependents)
+        parent_compiler = parent_spec.compiler
+        assert(parent_spec)
+        
+        # Check if the compiler is already fully specified
+        if parent_compiler in all_compilers:
+            spec.compiler = parent_compiler.copy()
+            return True
+            
+        # Filter the compilers into a sorted list based on the compiler_order from spackconfig
+        compiler_list = all_compilers if not parent_compiler else spack.compilers.find(parent_compiler)
+        cmp_compilers = partial(spack.pkgsort.compiler_compare, parent_spec.name)
+        matches = sorted(compiler_list, cmp=cmp_compilers)
+        if not matches:
+            raise UnavailableCompilerVersionError(parent_compiler)
+            
+        # copy concrete version into parent_compiler
+        spec.compiler = matches[0].copy()
+        assert(spec.compiler.concrete)
         return True  # things changed.
 
 
-    def choose_provider(self, spec, providers):
+    def choose_provider(self, package_spec, spec, providers):
         """This is invoked for virtual specs.  Given a spec with a virtual name,
            say "mpi", and a list of specs of possible providers of that spec,
            select a provider and return it.
         """
         assert(spec.virtual)
         assert(providers)
+        
+        provider_cmp = partial(spack.pkgsort.provider_compare, package_spec.name, spec.name)
+        sorted_providers = sorted(providers, cmp=provider_cmp)
+        first_key = sorted_providers[0]
 
-        index = spack.spec.index_specs(providers)
-        first_key = sorted(index.keys())[0]
-        latest_version = sorted(index[first_key])[-1]
-        return latest_version
+        return first_key
 
 
 class UnavailableCompilerVersionError(spack.error.SpackError):
diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py
index 83b1416e36d7f..41496b0e9d559 100644
--- a/lib/spack/spack/spec.py
+++ b/lib/spack/spack/spec.py
@@ -796,7 +796,7 @@ def _expand_virtual_packages(self):
 
             for spec in virtuals:
                 providers = spack.db.providers_for(spec)
-                concrete = spack.concretizer.choose_provider(spec, providers)
+                concrete = spack.concretizer.choose_provider(self, spec, providers)
                 concrete = concrete.copy()
                 spec._replace_with(concrete)
                 changed = True

From ee68a76a193890231d6df7fa7934d42e3708540b Mon Sep 17 00:00:00 2001
From: Matthew LeGendre <legendre1@llnl.gov>
Date: Fri, 29 May 2015 14:57:24 -0700
Subject: [PATCH 04/14] Bug fixes from testing spack preferred packages

---
 lib/spack/spack/preferred_packages.py    | 7 +++----
 var/spack/mock_packages/mpich/package.py | 1 +
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/lib/spack/spack/preferred_packages.py b/lib/spack/spack/preferred_packages.py
index 248508fe809b7..bc5271f693d8d 100644
--- a/lib/spack/spack/preferred_packages.py
+++ b/lib/spack/spack/preferred_packages.py
@@ -112,9 +112,8 @@ def _spec_compare(self, pkgname, component, a, b, reverse_natural_compare, secon
         elif a_index == None and b_index != None: return 1
         elif a_index != None and b_index == a_index: return -1 * cmp(a, b)
         elif a_index != None and b_index != None and a_index != b_index: return cmp(a_index, b_index)
-        elif a < b: return 1 * reverse
-        elif b < a: return -1 * reverse
-        else: return 0
+        else: return cmp(a, b) * reverse
+
 
 
     # Given a sort order specified by the pkgname/component/second_key, return
@@ -148,7 +147,7 @@ def version_compare(self, pkgname, a, b):
         """Return less-than-0, 0, or greater than 0 if version a of pkgname is 
            respecively less-than, equal-to, or greater-than version b of pkgname. 
            One version is less-than another if it is preferred over the other."""
-        return self._spec_compare(pkgname, 'version', a, b, False, None)
+        return self._spec_compare(pkgname, 'version', a, b, True, None)
 
     
     def variant_compare(self, pkgname, a, b):
diff --git a/var/spack/mock_packages/mpich/package.py b/var/spack/mock_packages/mpich/package.py
index f77d3efc5d1e2..e4110ad53012a 100644
--- a/var/spack/mock_packages/mpich/package.py
+++ b/var/spack/mock_packages/mpich/package.py
@@ -38,6 +38,7 @@ class Mpich(Package):
     version('3.0.2', 'foobarbaz')
     version('3.0.1', 'foobarbaz')
     version('3.0', 'foobarbaz')
+    version('1.0', 'foobarbas')
 
     provides('mpi@:3', when='@3:')
     provides('mpi@:1', when='@:1')

From 987cd9e78f99aa6ee6f3a48a4b8f556a68cb2965 Mon Sep 17 00:00:00 2001
From: Matthew LeGendre <legendre1@llnl.gov>
Date: Tue, 7 Jul 2015 16:27:13 -0700
Subject: [PATCH 05/14] Update docs for YAML configuration files and preferred
 concretization

---
 lib/spack/docs/basic_usage.rst        | 37 +++++++-----
 lib/spack/docs/mirrors.rst            |  9 ++-
 lib/spack/docs/packaging_guide.rst    | 66 ++++++++++++++++++++-
 lib/spack/docs/site_configuration.rst | 83 ---------------------------
 4 files changed, 91 insertions(+), 104 deletions(-)

diff --git a/lib/spack/docs/basic_usage.rst b/lib/spack/docs/basic_usage.rst
index 0578f0c8db672..5d5438220cc2f 100644
--- a/lib/spack/docs/basic_usage.rst
+++ b/lib/spack/docs/basic_usage.rst
@@ -357,7 +357,7 @@ Spack, you can simply run ``spack compiler add`` with the path to
 where the compiler is installed.  For example::
 
     $ spack compiler add /usr/local/tools/ic-13.0.079
-    ==> Added 1 new compiler to /Users/gamblin2/.spackconfig
+    ==> Added 1 new compiler to /Users/gamblin2/.spack/compilers.yaml
         intel@13.0.079
 
 Or you can run ``spack compiler add`` with no arguments to force
@@ -367,7 +367,7 @@ installed, but you know that new compilers have been added to your
 
     $ module load gcc-4.9.0
     $ spack compiler add
-    ==> Added 1 new compiler to /Users/gamblin2/.spackconfig
+    ==> Added 1 new compiler to /Users/gamblin2/.spack/compilers.yaml
         gcc@4.9.0
 
 This loads the environment module for gcc-4.9.0 to get it into the
@@ -398,27 +398,34 @@ Manual compiler configuration
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 If auto-detection fails, you can manually configure a compiler by
-editing your ``~/.spackconfig`` file.  You can do this by running
-``spack config edit``, which will open the file in your ``$EDITOR``.
+editing your ``~/.spack/compilers.yaml`` file.  You can do this by running
+``spack config edit compilers``, which will open the file in your ``$EDITOR``.
 
 Each compiler configuration in the file looks like this::
 
     ...
-    [compiler "intel@15.0.0"]
-        cc = /usr/local/bin/icc-15.0.024-beta
-        cxx = /usr/local/bin/icpc-15.0.024-beta
-        f77 = /usr/local/bin/ifort-15.0.024-beta
-        fc = /usr/local/bin/ifort-15.0.024-beta
-    ...
+    chaos_5_x86_64_ib:
+      ...
+      intel@15.0.0:
+          cc: /usr/local/bin/icc-15.0.024-beta
+          cxx: /usr/local/bin/icpc-15.0.024-beta
+          f77: /usr/local/bin/ifort-15.0.024-beta
+          fc: /usr/local/bin/ifort-15.0.024-beta
+      ...
+
+The chaos_5_x86_64_ib string is an architecture string, and multiple
+compilers can be listed underneath an architecture.  The architecture
+string may be replaced with the string 'all' to signify compilers that
+work on all architectures.
 
 For compilers, like ``clang``, that do not support Fortran, put
 ``None`` for ``f77`` and ``fc``::
 
-    [compiler "clang@3.3svn"]
-        cc = /usr/bin/clang
-        cxx = /usr/bin/clang++
-        f77 = None
-        fc = None
+    clang@3.3svn:
+        cc: /usr/bin/clang
+        cxx: /usr/bin/clang++
+        f77: None
+        fc: None
 
 Once you save the file, the configured compilers will show up in the
 list displayed by ``spack compilers``.
diff --git a/lib/spack/docs/mirrors.rst b/lib/spack/docs/mirrors.rst
index d732a3dd54fe9..7581a0e9ede7d 100644
--- a/lib/spack/docs/mirrors.rst
+++ b/lib/spack/docs/mirrors.rst
@@ -205,12 +205,11 @@ And, if you want to remove a mirror, just remove it by name::
 Mirror precedence
 ----------------------------
 
-Adding a mirror really just adds a section in ``~/.spackconfig``::
+Adding a mirror really just adds a section in ``~/.spack/mirrors.yaml``::
 
-   [mirror "local_filesystem"]
-       url = file:///Users/gamblin2/spack-mirror-2014-06-24
-   [mirror "remote_server"]
-       url = https://example.com/some/web-hosted/directory/spack-mirror-2014-06-24
+   mirrors:
+   - local_filesystem: file:///Users/gamblin2/spack-mirror-2014-06-24
+   - remote_server: https://example.com/some/web-hosted/directory/spack-mirror-2014-06-24
 
 If you want to change the order in which mirrors are searched for
 packages, you can edit this file and reorder the sections.  Spack will
diff --git a/lib/spack/docs/packaging_guide.rst b/lib/spack/docs/packaging_guide.rst
index 59ba63fa35527..5094f739c4542 100644
--- a/lib/spack/docs/packaging_guide.rst
+++ b/lib/spack/docs/packaging_guide.rst
@@ -632,7 +632,7 @@ Default
   revision instead.
 
 Revisions
-  Add ``hg`` and ``revision``parameters:
+  Add ``hg`` and ``revision`` parameters:
 
   .. code-block:: python
 
@@ -1524,6 +1524,70 @@ This is useful when you want to know exactly what Spack will do when
 you ask for a particular spec.
 
 
+``Concretization Policies``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+A user may have certain perferrences for how packages should
+be concretized on their system.  For example, one user may prefer packages
+built with OpenMPI and the Intel compiler.  Another user may prefer
+packages be built with MVAPICH and GCC.  
+
+Spack's ``preferred`` configuration can be used to set defaults for sites or users.
+Spack uses this configuration to make decisions about which compilers, package
+versions, depends_on, and variants it should prefer during concretization.
+
+The preferred configuration can be controlled by editing the 
+``~/.spack/preferred.yaml`` file for user configuations, or the
+
+
+Here's an example preferred.yaml file:
+
+.. code-block:: sh
+
+    preferred:
+      dyninst:
+        compiler: gcc@4.9
+        variants: +debug
+      gperftools:
+        version: 2.2, 2.4, 2.3
+      all:
+        compiler: gcc@4.4.7, gcc@4.6:, intel, clang, pgi
+        providers: 
+          mpi: mvapich, mpich, openmpi
+
+At a high level, this example is specifying how packages should be
+concretized.  The dyninst package should prefer using gcc 4.9 and 
+be built with debug options.  The gperftools package should prefer version
+2.2 over 2.4.  Every package on the system should prefer mvapich for
+its MPI and gcc 4.4.7 (except for Dyninst, which perfers gcc 4.9).  
+These options are used to fill in implicit defaults.  Any of them can be overwritten 
+on the command line if explicitly requested.
+
+Each preferred.yaml file begin with the string ``preferred:`` and 
+each subsequent entry is indented underneath it.  The next layer contains
+package names or the special string ``all`` (which applies to 
+every package).  Underneath each package name is 
+one or more components: ``compiler``, ``variants``, ``version``, 
+or ``providers``.  Each component has an ordered list of spec 
+``constraints``, with earlier entries in the list being prefered over
+latter entries.
+
+Sometimes a package installation may have constraints that forbid 
+the first concretization rule, in which case Spack will use the first
+legal concretization rule.  Going back to the example, if a user
+requests gperftools 2.3 or latter, then Spack will install version 2.4 
+as the 2.4 version of gperftools is preferred over 2.3.
+
+An explicit concretization rule in the preferred section will always 
+take preference over unlisted concretizations.  In the above example, 
+xlc isn't listed in the compiler list.  Every listed compiler from
+gcc to pgi will thus be preferred over the xlc compiler.
+
+The syntax for the ``providers`` section differs slightly from other
+concretization rules.  A provider lists a value that packages may
+``depend_on`` (e.g, mpi) and a list of rules for fulfilling that
+dependency.
+
 .. _install-method:
 
 Implementing the ``install`` method
diff --git a/lib/spack/docs/site_configuration.rst b/lib/spack/docs/site_configuration.rst
index b03df29573bef..1e6740a434fae 100644
--- a/lib/spack/docs/site_configuration.rst
+++ b/lib/spack/docs/site_configuration.rst
@@ -54,89 +54,6 @@ more elements to the list to indicate where your own site's temporary
 directory is.
 
 
-.. _concretization-policies:
-
-Concretization policies
-----------------------------
-
-When a user asks for a package like ``mpileaks`` to be installed,
-Spack has to make decisions like what version should be installed,
-what compiler to use, and how its dependencies should be configured.
-This process is called *concretization*, and it's covered in detail in
-:ref:`its own section <abstract-and-concrete>`.
-
-The default concretization policies are in the
-:py:mod:`spack.concretize` module, specifically in the
-:py:class:`spack.concretize.DefaultConcretizer` class.  These are the
-important methods used in the concretization process:
-
-* :py:meth:`concretize_version(self, spec) <spack.concretize.DefaultConcretizer.concretize_version>`
-* :py:meth:`concretize_architecture(self, spec) <spack.concretize.DefaultConcretizer.concretize_architecture>`
-* :py:meth:`concretize_compiler(self, spec) <spack.concretize.DefaultConcretizer.concretize_compiler>`
-* :py:meth:`choose_provider(self, spec, providers) <spack.concretize.DefaultConcretizer.choose_provider>`
-
-The first three take a :py:class:`Spec <spack.spec.Spec>` object and
-modify it by adding constraints for the version.  For example, if the
-input spec had a version range like `1.0:5.0.3`, then the
-``concretize_version`` method should set the spec's version to a
-*single* version in that range.  Likewise, ``concretize_architecture``
-selects an architecture when the input spec does not have one, and
-``concretize_compiler`` needs to set both a concrete compiler and a
-concrete compiler version.
-
-``choose_provider()`` affects how concrete implementations are chosen
-based on a virtual dependency spec.  The input spec is some virtual
-dependency and the ``providers`` index is a :py:class:`ProviderIndex
-<spack.packages.ProviderIndex>` object.  The ``ProviderIndex`` maps
-the virtual spec to specs for possible implementations, and
-``choose_provider()`` should simply choose one of these.  The
-``concretize_*`` methods will be called on the chosen implementation
-later, so there is no need to fully concretize the spec when returning
-it.
-
-The ``DefaultConcretizer`` is intended to provide sensible defaults
-for each policy, but there are certain choices that it can't know
-about.  For example, one site might prefer ``OpenMPI`` over ``MPICH``,
-or another might prefer an old version of some packages.  These types
-of special cases can be integrated with custom concretizers.
-
-Writing a custom concretizer
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-To write your own concretizer, you need only subclass
-``DefaultConcretizer`` and override the methods you want to change.
-For example, you might write a class like this to change *only* the
-``concretize_version()`` behavior:
-
-.. code-block:: python
-
-   from spack.concretize import DefaultConcretizer
-
-   class MyConcretizer(DefaultConcretizer):
-       def concretize_version(self, spec):
-           # implement custom logic here.
-
-Once you have written your custom concretizer, you can make Spack use
-it by editing ``globals.py``.  Find this part of the file:
-
-.. code-block:: python
-
-   #
-   # This controls how things are concretized in spack.
-   # Replace it with a subclass if you want different
-   # policies.
-   #
-   concretizer = DefaultConcretizer()
-
-Set concretizer to *your own* class instead of the default:
-
-.. code-block:: python
-
-   concretizer = MyConcretizer()
-
-The next time you run Spack, your changes should take effect.
-
-
 Profiling
 ~~~~~~~~~~~~~~~~~~~~~
 

From 53cde110b1d25f8fcf89d7c26ef02bb073cb6a29 Mon Sep 17 00:00:00 2001
From: Matthew LeGendre <legendre1@llnl.gov>
Date: Tue, 7 Jul 2015 16:27:46 -0700
Subject: [PATCH 06/14] Update Spack mirror command to match docs

---
 lib/spack/spack/cmd/mirror.py | 12 ++++++------
 lib/spack/spack/stage.py      |  5 ++---
 2 files changed, 8 insertions(+), 9 deletions(-)

diff --git a/lib/spack/spack/cmd/mirror.py b/lib/spack/spack/cmd/mirror.py
index 2356170a9acf4..baf64d30fc820 100644
--- a/lib/spack/spack/cmd/mirror.py
+++ b/lib/spack/spack/cmd/mirror.py
@@ -75,8 +75,8 @@ def mirror_add(args):
     if url.startswith('/'):
         url = 'file://' + url
 
-    mirror_dict = { args.name : url }
-    spack.config.add_to_mirror_config({ args.name : url })
+    newmirror = [ { args.name : url } ]
+    spack.config.add_to_mirror_config(newmirror)
 
 
 def mirror_remove(args):
@@ -90,15 +90,15 @@ def mirror_remove(args):
 
 def mirror_list(args):
     """Print out available mirrors to the console."""
-    sec_names = spack.config.get_mirror_config()
-    if not sec_names:
+    mirrors = spack.config.get_mirror_config()
+    if not mirrors:
         tty.msg("No mirrors configured.")
         return
 
-    max_len = max(len(s) for s in sec_names)
+    max_len = max(len(name) for name,path in mirrors)
     fmt = "%%-%ds%%s" % (max_len + 4)
 
-    for name, val in sec_names.iteritems():
+    for name, val in mirrors:
         print fmt % (name, val)
 
 
diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py
index 008c5f04298ba..c70c7a84a4eaa 100644
--- a/lib/spack/spack/stage.py
+++ b/lib/spack/spack/stage.py
@@ -26,6 +26,7 @@
 import re
 import shutil
 import tempfile
+import sys
 
 import llnl.util.tty as tty
 from llnl.util.filesystem import *
@@ -344,9 +345,7 @@ def destroy(self):
 
 def _get_mirrors():
     """Get mirrors from spack configuration."""
-    config = spack.config.get_mirror_config()
-    return [val for name, val in config.iteritems()]
-
+    return [path for name, path in spack.config.get_mirror_config()]
 
 
 def ensure_access(file=spack.stage_path):

From 53d70fff0121a05fb21c02363570f81573bbeffa Mon Sep 17 00:00:00 2001
From: Matthew LeGendre <legendre1@llnl.gov>
Date: Tue, 7 Jul 2015 16:28:51 -0700
Subject: [PATCH 07/14] Fix type error with YAML config when merging lists from
 different configs.

---
 lib/spack/spack/config.py | 33 ++++++++++++++++-----------------
 1 file changed, 16 insertions(+), 17 deletions(-)

diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py
index dbe225960a846..f3526b19fa3df 100644
--- a/lib/spack/spack/config.py
+++ b/lib/spack/spack/config.py
@@ -98,6 +98,7 @@
 from external.yaml.error import MarkedYAMLError
 import llnl.util.tty as tty
 from llnl.util.filesystem import mkdirp
+import copy
 
 _config_sections = {}
 class _ConfigCategory:
@@ -159,21 +160,19 @@ def _merge_dicts(d1, d2):
     if not d2:
         return d1
 
-    for key2, val2 in d2.iteritems():
-        if not key2 in d1:
-            d1[key2] = val2
-            continue
-        val1 = d1[key2]
-        if isinstance(val1, dict) and isinstance(val2, dict):
-            d1[key2] = _merge_dicts(val1, val2)
-            continue
-        if isinstance(val1, list) and isinstance(val2, list):
-            val1.extend(val2)
-            seen = set()
-            d1[key2] = [ x for x in val1 if not (x in seen or seen.add(x)) ]
-            continue
-        d1[key2] = val2
-    return d1
+    if (type(d1) is list) and (type(d2) is list):
+        d1.extend(d2)
+        return d1
+
+    if (type(d1) is dict) and (type(d2) is dict):
+        for key2, val2 in d2.iteritems():
+            if not key2 in d1:
+                d1[key2] = val2
+            else:
+                d1[key2] = _merge_dicts(d1[key2], val2)
+        return d1
+
+    return d2
 
 
 def get_config(category_name):
@@ -225,8 +224,8 @@ def get_compilers_config(arch=None):
 
 
 def get_mirror_config():
-    """Get the mirror configuration from config files"""
-    return get_config('mirrors')
+    """Get the mirror configuration from config files as a list of name/location tuples"""
+    return [x.items()[0] for x in get_config('mirrors')]
 
 
 def get_preferred_config():

From 650c9d4e36c6a58cf6bca0e6abd580ee54d8e175 Mon Sep 17 00:00:00 2001
From: Matthew LeGendre <legendre1@llnl.gov>
Date: Wed, 16 Sep 2015 10:56:11 -0700
Subject: [PATCH 08/14] Allow spack to build against external
 non-spack-installed packages.

---
 lib/spack/spack/__init__.py            |   6 ++
 lib/spack/spack/abi.py                 | 128 ++++++++++++++++++++++
 lib/spack/spack/concretize.py          | 140 ++++++++++++++++++++++---
 lib/spack/spack/config.py              |  59 ++++++++++-
 lib/spack/spack/directory_layout.py    |   8 ++
 lib/spack/spack/package.py             |   3 +
 lib/spack/spack/preferred_packages.py  |  10 +-
 lib/spack/spack/spec.py                |  58 +++++-----
 var/spack/packages/mpich/package.py    |   1 +
 var/spack/packages/mvapich2/package.py |  17 ++-
 10 files changed, 387 insertions(+), 43 deletions(-)
 create mode 100644 lib/spack/spack/abi.py

diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py
index bd8478fb98e84..5783005b5b860 100644
--- a/lib/spack/spack/__init__.py
+++ b/lib/spack/spack/__init__.py
@@ -77,6 +77,12 @@
 from spack.preferred_packages import PreferredPackages
 pkgsort = PreferredPackages()
 
+#
+# This tests ABI compatibility between packages
+#
+from spack.abi import ABI
+abi = ABI()
+
 #
 # This controls how things are concretized in spack.
 # Replace it with a subclass if you want different
diff --git a/lib/spack/spack/abi.py b/lib/spack/spack/abi.py
new file mode 100644
index 0000000000000..f0a997703cdce
--- /dev/null
+++ b/lib/spack/spack/abi.py
@@ -0,0 +1,128 @@
+##############################################################################
+# Copyright (c) 2015, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://scalability-llnl.github.io/spack
+# Please also see the LICENSE file for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License (as published by
+# the Free Software Foundation) version 2.1 dated February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+
+import os
+import spack
+import spack.spec
+from spack.spec import CompilerSpec
+from spack.util.executable import Executable, ProcessError
+from llnl.util.lang import memoized
+
+class ABI(object):
+    """This class provides methods to test ABI compatibility between specs.
+       The current implementation is rather rough and could be improved."""
+
+    def architecture_compatible(self, parent, child):
+        """Returns true iff the parent and child specs have ABI compatible architectures."""
+        return not parent.architecture or not child.architecture or parent.architecture == child.architecture
+
+
+    @memoized
+    def _gcc_get_libstdcxx_version(self, version):
+        """Returns gcc ABI compatibility info by getting the library version of
+           a compiler's libstdc++.so or libgcc_s.so"""
+        spec = CompilerSpec("gcc", version)
+        compilers = spack.compilers.compilers_for_spec(spec)
+        if not compilers:
+            return None
+        compiler = compilers[0]
+        rungcc = None
+        libname = None
+        output = None
+        if compiler.cxx:
+            rungcc = Executable(compiler.cxx)
+            libname = "libstdc++.so"
+        elif compiler.cc:
+            rungcc = Executable(compiler.cc)
+            libname = "libgcc_s.so"
+        else:
+            return None
+        try:
+            output = rungcc("--print-file-name=%s" % libname, return_output=True)
+        except ProcessError, e:
+            return None
+        if not output:
+            return None
+        libpath = os.readlink(output.strip())
+        if not libpath:
+            return None
+        return os.path.basename(libpath)
+        
+        
+    @memoized
+    def _gcc_compiler_compare(self, pversion, cversion):
+        """Returns true iff the gcc version pversion and cversion 
+          are ABI compatible."""
+        plib = self._gcc_get_libstdcxx_version(pversion)
+        clib = self._gcc_get_libstdcxx_version(cversion)
+        if not plib or not clib:
+            return False
+        return plib == clib
+
+
+    def _intel_compiler_compare(self, pversion, cversion):
+        """Returns true iff the intel version pversion and cversion
+           are ABI compatible"""
+
+        #Test major and minor versions.  Ignore build version.
+        if (len(pversion.version) < 2 or len(cversion.version) < 2):
+            return False
+        return (pversion.version[0] == cversion.version[0]) and \
+            (pversion.version[1] == cversion.version[1])
+        
+    
+    def compiler_compatible(self, parent, child, **kwargs):
+        """Returns true iff the compilers for parent and child specs are ABI compatible"""
+        if not parent.compiler or not child.compiler:
+            return True
+        
+        if parent.compiler.name != child.compiler.name:
+            #Different compiler families are assumed ABI incompatible
+            return False
+        
+        if kwargs.get('loose', False):
+            return True
+
+        for pversion in parent.compiler.versions:
+            for cversion in child.compiler.versions:
+                #For a few compilers use specialized comparisons.  Otherwise
+                # match on version match.
+                if pversion.satisfies(cversion):
+                    return True
+                elif parent.compiler.name == "gcc" and \
+                     self._gcc_compiler_compare(pversion, cversion):
+                    return True
+                elif parent.compiler.name == "intel" and \
+                     self._intel_compiler_compare(pversion, cversion):
+                    return True
+        return False
+
+    
+    def compatible(self, parent, child, **kwargs):
+        """Returns true iff a parent and child spec are ABI compatible"""
+        loosematch = kwargs.get('loose', False)
+        return self.architecture_compatible(parent, child) and \
+               self.compiler_compatible(parent, child, loose=loosematch)
+    
diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py
index 0f258c909689e..01ff163493b78 100644
--- a/lib/spack/spack/concretize.py
+++ b/lib/spack/spack/concretize.py
@@ -33,13 +33,16 @@
 TODO: make this customizable and allow users to configure
       concretization  policies.
 """
+import spack
 import spack.spec
 import spack.compilers
 import spack.architecture
 import spack.error
 from spack.version import *
 from functools import partial
-
+from spec import DependencyMap
+from itertools import chain
+from spack.config import *
 
 
 class DefaultConcretizer(object):
@@ -48,6 +51,107 @@ class DefaultConcretizer(object):
        default concretization strategies, or you can override all of them.
     """
 
+    def _find_other_spec(self, spec, condition):
+        """Searches the dag from spec in an intelligent order and looks
+           for a spec that matches a condition"""
+        dagiter = chain(spec.traverse(direction='parents'), spec.traverse(direction='children'))
+        found = next((x for x in dagiter if x is not spec and condition(x)), None)
+        if found:
+            return found
+        dagiter = chain(spec.traverse(direction='parents'), spec.traverse(direction='children'))
+        searched = list(dagiter)
+        found = next((x for x in spec.root.traverse() if x not in searched and x is not spec and condition(x)), None)
+        if found:
+            return found
+        if condition(spec):
+            return spec
+        return None
+                                   
+
+    def _valid_virtuals_and_externals(self, spec):
+        """Returns a list of spec/external-path pairs for both virtuals and externals
+           that can concretize this spec.""" 
+
+        # Get a list of candidate packages that could satisfy this spec
+        packages = []
+        if spec.virtual:
+            providers = spack.db.providers_for(spec)
+            if not providers:
+                raise UnsatisfiableProviderSpecError(providers[0], spec)
+            spec_w_preferred_providers = self._find_other_spec(spec, \
+                    lambda(x): spack.pkgsort.spec_has_preferred_provider(x.name, spec.name))
+            if not spec_w_preferred_providers:
+                spec_w_preferred_providers = spec
+            provider_cmp = partial(spack.pkgsort.provider_compare, spec_w_preferred_providers.name, spec.name)
+            packages = sorted(providers, cmp=provider_cmp)
+        else:
+            if not spec_externals(spec) or spec.external:
+                return None
+            packages = [spec]
+
+        # For each candidate package, if it has externals add those to the candidates
+        # if it's a nobuild, then only add the externals.
+        result = []
+        all_compilers = spack.compilers.all_compilers()
+        for pkg in packages:
+            externals = spec_externals(pkg)
+            buildable = not is_spec_nobuild(pkg)
+            if buildable:
+                result.append((pkg, None))
+            if externals:
+                sorted_externals = sorted(externals, cmp=lambda a,b: a[0].__cmp__(b[0]))
+                for external in sorted_externals:
+                    if external[0].satisfies(spec):
+                        result.append(external)
+        if not result:
+            raise NoBuildError(spec)
+        return result
+
+
+    def concretize_virtual_and_external(self, spec):
+        """From a list of candidate virtual and external packages, concretize to one that
+           is ABI compatible with the rest of the DAG."""
+        candidates = self._valid_virtuals_and_externals(spec)
+        if not candidates:
+            return False
+
+        #Find the another spec in the dag that has a compiler.  We'll use that
+        # spec to test compiler compatibility.
+        other_spec = self._find_other_spec(spec, lambda(x): x.compiler)
+        if not other_spec:
+            other_spec = spec.root
+
+        #Choose an ABI-compatible candidate, or the first match otherwise.
+        candidate = None
+        if other_spec:
+            candidate = next((c for c in candidates if spack.abi.compatible(c[0], other_spec)), None)
+            if not candidate:
+                #Try a looser ABI matching
+                candidate = next((c for c in candidates if spack.abi.compatible(c[0], other_spec, loose=True)), None)
+        if not candidate:
+            #Pick the first choice
+            candidate = candidates[0]
+        external = candidate[1]
+        candidate_spec = candidate[0]
+        
+        #Refine this spec to the candidate.
+        changed = False
+        if spec.virtual:
+            spec._replace_with(candidate_spec)
+            changed = True
+        if spec._dup(candidate_spec, deps=False, cleardeps=False):
+            changed = True
+        if not spec.external and external:
+            spec.external = external
+            changed = True
+        #If we're external then trim the dependencies
+        if external and spec.dependencies:
+            changed = True
+            spec.depencencies = DependencyMap()
+        
+        return changed
+        
+        
     def concretize_version(self, spec):
         """If the spec is already concrete, return.  Otherwise take
            the preferred version from spackconfig, and default to the package's
@@ -150,31 +254,32 @@ def concretize_compiler(self, spec):
            link to this one, to maximize compatibility.
         """
         all_compilers = spack.compilers.all_compilers()
-
+        
         if (spec.compiler and
             spec.compiler.concrete and
             spec.compiler in all_compilers):
             return False
 
-        # Find the parent spec that has a compiler, or the root if none do
-        parent_spec = next(p for p in spec.traverse(direction='parents')
-                           if p.compiler is not None or not p.dependents)
-        parent_compiler = parent_spec.compiler
-        assert(parent_spec)
+        #Find the another spec that has a compiler, or the root if none do
+        other_spec = self._find_other_spec(spec, lambda(x) : x.compiler)
+        if not other_spec:
+            other_spec = spec.root
+        other_compiler = other_spec.compiler
+        assert(other_spec)
         
         # Check if the compiler is already fully specified
-        if parent_compiler in all_compilers:
-            spec.compiler = parent_compiler.copy()
+        if other_compiler in all_compilers:
+            spec.compiler = other_compiler.copy()
             return True
             
         # Filter the compilers into a sorted list based on the compiler_order from spackconfig
-        compiler_list = all_compilers if not parent_compiler else spack.compilers.find(parent_compiler)
-        cmp_compilers = partial(spack.pkgsort.compiler_compare, parent_spec.name)
+        compiler_list = all_compilers if not other_compiler else spack.compilers.find(other_compiler)
+        cmp_compilers = partial(spack.pkgsort.compiler_compare, other_spec.name)
         matches = sorted(compiler_list, cmp=cmp_compilers)
         if not matches:
-            raise UnavailableCompilerVersionError(parent_compiler)
+            raise UnavailableCompilerVersionError(other_compiler)
             
-        # copy concrete version into parent_compiler
+        # copy concrete version into other_compiler
         spec.compiler = matches[0].copy()
         assert(spec.compiler.concrete)
         return True  # things changed.
@@ -210,3 +315,12 @@ class NoValidVersionError(spack.error.SpackError):
     def __init__(self, spec):
         super(NoValidVersionError, self).__init__(
             "There are no valid versions for %s that match '%s'" % (spec.name, spec.versions))
+
+
+class NoBuildError(spack.error.SpackError):
+    """Raised when a package is configured with the nobuild option, but
+       no satisfactory external versions can be found"""
+    def __init__(self, spec):
+        super(NoBuildError, self).__init__(
+            "The spec '%s' is configured as nobuild, and no matching external installs were found" % spec.name)
+       
diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py
index f3526b19fa3df..60577c45b3636 100644
--- a/lib/spack/spack/config.py
+++ b/lib/spack/spack/config.py
@@ -90,9 +90,12 @@
 import exceptions
 import sys
 import copy
-
-from llnl.util.lang import memoized
+import inspect
+import glob
+import imp
+import spack.spec
 import spack.error
+from llnl.util.lang import memoized
 
 from external import yaml
 from external.yaml.error import MarkedYAMLError
@@ -116,6 +119,9 @@ def __init__(self, n, f, m):
 _ConfigCategory('compilers', 'compilers.yaml', True)
 _ConfigCategory('mirrors', 'mirrors.yaml', True)
 _ConfigCategory('preferred', 'preferred.yaml', True)
+_ConfigCategory('view', 'views.yaml', True)
+_ConfigCategory('preferred', 'preferred.yaml', True)
+_ConfigCategory('packages', 'packages.yaml', True)
 
 """Names of scopes and their corresponding configuration files."""
 config_scopes = [('site', os.path.join(spack.etc_path, 'spack')),
@@ -233,6 +239,55 @@ def get_preferred_config():
     return get_config('preferred')
 
 
+@memoized
+def get_packages_config():
+    """Get the externals configuration from config files"""
+    package_config = get_config('packages')
+    if not package_config:
+        return {}
+    indexed_packages = {}
+    for p in package_config:
+        package_name = spack.spec.Spec(p.keys()[0]).name
+        if package_name not in indexed_packages:
+            indexed_packages[package_name] = []
+        indexed_packages[package_name].append({ spack.spec.Spec(key) : val for key, val in p.iteritems() })
+    return indexed_packages
+
+
+def is_spec_nobuild(spec):
+    """Return true if the spec pkgspec is configured as nobuild"""
+    allpkgs = get_packages_config()
+    name = spec.name
+    if not name in allpkgs:
+        return False
+    for itm in allpkgs[name]:
+        for pkg,conf in itm.iteritems():
+            if pkg.satisfies(spec):
+                if conf.get('nobuild', False):
+                    return True
+    return False
+
+
+def spec_externals(spec):
+    """Return a list of spec, directory pairs for each external location for spec"""
+    allpkgs = get_packages_config()
+    name = spec.name
+    spec_locations = []
+    
+    if not name in allpkgs:
+        return []
+    for itm in allpkgs[name]:
+        for pkg,conf in itm.iteritems():
+            if not pkg.satisfies(spec):
+                continue
+            path = conf.get('path', None)
+            if not path:
+                continue
+            spec_locations.append( (pkg, path) )
+    return spec_locations
+        
+    
+
 def get_config_scope_dirname(scope):
     """For a scope return the config directory"""
     global config_scopes
diff --git a/lib/spack/spack/directory_layout.py b/lib/spack/spack/directory_layout.py
index 85ecc1ce2b169..83e6eb566a7ad 100644
--- a/lib/spack/spack/directory_layout.py
+++ b/lib/spack/spack/directory_layout.py
@@ -187,6 +187,14 @@ def hidden_file_paths(self):
 
     def relative_path_for_spec(self, spec):
         _check_concrete(spec)
+        
+        if spec.external:
+            return spec.external
+
+        enabled_variants = (
+            '-' + v.name for v in spec.variants.values()
+            if v.enabled)
+
         dir_name = "%s-%s-%s" % (
             spec.name,
             spec.version,
diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py
index 61606d0590584..1e2f0378c8eee 100644
--- a/lib/spack/spack/package.py
+++ b/lib/spack/spack/package.py
@@ -752,6 +752,9 @@ def do_install(self,
         if not self.spec.concrete:
             raise ValueError("Can only install concrete packages.")
 
+        if self.spec.external:
+            return
+
         if os.path.exists(self.prefix):
             tty.msg("%s is already installed in %s." % (self.name, self.prefix))
             return
diff --git a/lib/spack/spack/preferred_packages.py b/lib/spack/spack/preferred_packages.py
index bc5271f693d8d..bc2a4ac234d93 100644
--- a/lib/spack/spack/preferred_packages.py
+++ b/lib/spack/spack/preferred_packages.py
@@ -35,9 +35,10 @@ def __init__(self):
 
     #Given a package name, sort component (e.g, version, compiler, ...), and
     # a second_key (used by providers), return the list 
-    def _order_for_package(self, pkgname, component, second_key):
+    def _order_for_package(self, pkgname, component, second_key, test_all=True):
         pkglist = [pkgname]
-        pkglist.append('all')
+        if test_all:
+            pkglist.append('all')
         for pkg in pkglist:
             if not pkg in self.preferred:
                 continue
@@ -143,6 +144,11 @@ def provider_compare(self, pkgname, provider_str, a, b):
         return self._spec_compare(pkgname, 'providers', a, b, False, provider_str)
 
 
+    def spec_has_preferred_provider(self, pkgname, provider_str):
+        """Return True iff the named package has a list of preferred provider"""
+        return bool(self._order_for_package(pkgname, 'providers', provider_str, False))
+
+
     def version_compare(self, pkgname, a, b):
         """Return less-than-0, 0, or greater than 0 if version a of pkgname is 
            respecively less-than, equal-to, or greater-than version b of pkgname. 
diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py
index 41496b0e9d559..6984b4a174a62 100644
--- a/lib/spack/spack/spec.py
+++ b/lib/spack/spack/spec.py
@@ -419,6 +419,7 @@ def __init__(self, spec_like, *dep_like, **kwargs):
         # package.py files for.
         self._normal = kwargs.get('normal', False)
         self._concrete = kwargs.get('concrete', False)
+        self.external = None
 
         # This allows users to construct a spec DAG with literals.
         # Note that given two specs a and b, Spec(a) copies a, but
@@ -426,7 +427,7 @@ def __init__(self, spec_like, *dep_like, **kwargs):
         for dep in dep_like:
             spec = dep if isinstance(dep, Spec) else Spec(dep)
             self._add_dependency(spec)
-
+        
 
     #
     # Private routines here are called by the parser when building a spec.
@@ -751,12 +752,11 @@ def _concretize_helper(self, presets=None, visited=None):
             # Concretize virtual dependencies last.  Because they're added
             # to presets below, their constraints will all be merged, but we'll
             # still need to select a concrete package later.
-            if not self.virtual:
-                changed |= any(
-                    (spack.concretizer.concretize_architecture(self),
-                     spack.concretizer.concretize_compiler(self),
-                     spack.concretizer.concretize_version(self),
-                     spack.concretizer.concretize_variants(self)))
+            changed |= any(
+                (spack.concretizer.concretize_architecture(self),
+                 spack.concretizer.concretize_compiler(self),
+                 spack.concretizer.concretize_version(self),
+                 spack.concretizer.concretize_variants(self)))
             presets[self.name] = self
 
         visited.add(self.name)
@@ -789,21 +789,18 @@ def _expand_virtual_packages(self):
               a problem.
         """
         changed = False
-        while True:
-            virtuals =[v for v in self.traverse() if v.virtual]
-            if not virtuals:
-                return changed
-
-            for spec in virtuals:
-                providers = spack.db.providers_for(spec)
-                concrete = spack.concretizer.choose_provider(self, spec, providers)
-                concrete = concrete.copy()
-                spec._replace_with(concrete)
-                changed = True
+        done = False
+        while not done:
+            done = True
+            for spec in list(self.traverse()):
+                if spack.concretizer.concretize_virtual_and_external(spec):
+                    done = False
+                    changed = True
 
-            # If there are duplicate providers or duplicate provider deps, this
-            # consolidates them and merge constraints.
-            changed |= self.normalize(force=True)
+        # If there are duplicate providers or duplicate provider deps, this
+        # consolidates them and merge constraints.
+        changed |= self.normalize(force=True)
+        return changed
 
 
     def concretize(self):
@@ -830,7 +827,6 @@ def concretize(self):
                        self._concretize_helper())
             changed = any(changes)
             force=True
-
         self._concrete = True
 
 
@@ -1346,15 +1342,26 @@ def _dup(self, other, **kwargs):
                Whether deps should be copied too.  Set to false to copy a
                spec but not its dependencies.
         """
+
+        # We don't count dependencies as changes here
+        changed = True
+        if hasattr(self, 'name'):
+            changed = (self.name != other.name and self.versions != other.versions and \
+                       self.architecture != other.architecture and self.compiler != other.compiler and \
+                       self.variants != other.variants and self._normal != other._normal and \
+                       self.concrete != other.concrete and self.external != other.external)
+                                       
         # Local node attributes get copied first.
         self.name = other.name
         self.versions = other.versions.copy()
         self.architecture = other.architecture
         self.compiler = other.compiler.copy() if other.compiler else None
-        self.dependents = DependencyMap()
-        self.dependencies = DependencyMap()
+        if kwargs.get('cleardeps', True):
+            self.dependents = DependencyMap()
+            self.dependencies = DependencyMap()
         self.variants = other.variants.copy()
         self.variants.spec = self
+        self.external = other.external
 
         # If we copy dependencies, preserve DAG structure in the new spec
         if kwargs.get('deps', True):
@@ -1372,6 +1379,8 @@ def _dup(self, other, **kwargs):
         # Since we preserved structure, we can copy _normal safely.
         self._normal = other._normal
         self._concrete = other._concrete
+        self.external = other.external
+        return changed
 
 
     def copy(self, **kwargs):
@@ -1796,6 +1805,7 @@ def spec(self):
         spec.variants = VariantMap(spec)
         spec.architecture = None
         spec.compiler = None
+        spec.external = None
         spec.dependents   = DependencyMap()
         spec.dependencies = DependencyMap()
 
diff --git a/var/spack/packages/mpich/package.py b/var/spack/packages/mpich/package.py
index b6b2dfde2142a..dfff22152dac3 100644
--- a/var/spack/packages/mpich/package.py
+++ b/var/spack/packages/mpich/package.py
@@ -45,6 +45,7 @@ def setup_dependent_environment(self, module, spec, dep_spec):
         os.environ['MPICH_F77'] = 'f77'
         os.environ['MPICH_F90'] = 'f90'
 
+        module.mpicc = join_path(self.prefix.bin, 'mpicc')
 
     def install(self, spec, prefix):
         config_args = ["--prefix=" + prefix,
diff --git a/var/spack/packages/mvapich2/package.py b/var/spack/packages/mvapich2/package.py
index ca0b1287c1ce4..93bce011b716b 100644
--- a/var/spack/packages/mvapich2/package.py
+++ b/var/spack/packages/mvapich2/package.py
@@ -11,10 +11,17 @@ class Mvapich2(Package):
 
     version('2.0', '9fbb68a4111a8b6338e476dc657388b4',
             url='http://mvapich.cse.ohio-state.edu/download/mvapich/mv2/mvapich2-2.0.tar.gz')
+    
+    version('2.1', '0095ceecb19bbb7fb262131cb9c2cdd6', 
+            url='http://mvapich.cse.ohio-state.edu/download/mvapich/mv2/mvapich2-2.1.tar.gz')
 
     provides('mpi@:2.2', when='@1.9')    # MVAPICH2-1.9 supports MPI 2.2
     provides('mpi@:3.0', when='@2.0')    # MVAPICH2-2.0 supports MPI 3.0
 
+    variant('psm', default=False, description="build with psm")
+
+    variant('pmi', default=False, description="build with pmi")
+    depends_on('pmgr_collective', when='+pmi')
 
     def install(self, spec, prefix):
         # we'll set different configure flags depending on our environment
@@ -80,7 +87,13 @@ def install(self, spec, prefix):
             configure_args.append("--with-device=ch3:psm")
         else:
             # throw this flag on IB systems
-            configure_args.append("--with-device=ch3:mrail", "--with-rdma=gen2")
+            configure_args.append("--with-device=ch3:mrail")
+            configure_args.append("--with-rdma=gen2")
+
+        if "+pmi" in spec:
+            configure_args.append("--with-pmi=pmgr_collective" % spec['pmgr_collective'].prefix)
+        else:
+            configure_args.append("--with-pmi=slurm")
 
         # TODO: shared-memory build
 
@@ -93,7 +106,7 @@ def install(self, spec, prefix):
             "--enable-f77", "--enable-fc", "--enable-cxx",
             "--enable-shared", "--enable-sharedlibs=gcc",
             "--enable-debuginfo",
-            "--with-pm=no", "--with-pmi=slurm",
+            "--with-pm=no",
             "--enable-romio", "--with-file-system=lustre+nfs+ufs",
             "--disable-mpe", "--without-mpe",
             "--disable-silent-rules",

From e4d2ba30b57618da388a1a990381d149b33d7aba Mon Sep 17 00:00:00 2001
From: Matthew LeGendre <legendre1@llnl.gov>
Date: Fri, 2 Oct 2015 11:00:41 -0700
Subject: [PATCH 09/14] Fix failure in spack.test.config.ConfigTest from
 incorrect compiler config merging

---
 lib/spack/spack/config.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py
index 60577c45b3636..712a2b78fc09e 100644
--- a/lib/spack/spack/config.py
+++ b/lib/spack/spack/config.py
@@ -174,8 +174,12 @@ def _merge_dicts(d1, d2):
         for key2, val2 in d2.iteritems():
             if not key2 in d1:
                 d1[key2] = val2
-            else:
+            elif type(d1[key2]) is dict and type(val2) is dict:
                 d1[key2] = _merge_dicts(d1[key2], val2)
+            elif (type(d1) is list) and (type(d2) is list):
+                d1.extend(d2)
+            else:
+                d1[key2] = val2
         return d1
 
     return d2
@@ -360,7 +364,7 @@ def add_to_mirror_config(addition_dict, scope=None):
 
 
 def add_to_compiler_config(addition_dict, scope=None, arch=None):
-    """Add compilerss to the configuration files"""
+    """Add compilers to the configuration files"""
     if not arch:
         arch = spack.architecture.sys_type()
     add_to_config('compilers', { arch : addition_dict }, scope)

From 18f0b24a7f21ec7b46510f45867386b7600bbc55 Mon Sep 17 00:00:00 2001
From: Matthew LeGendre <legendre1@llnl.gov>
Date: Mon, 5 Oct 2015 11:30:48 -0700
Subject: [PATCH 10/14] Add tests for spack external dependencies, plus fixes
 for issues found by those tests.

---
 lib/spack/spack/concretize.py                 |  9 +++--
 lib/spack/spack/spec.py                       |  2 +-
 lib/spack/spack/test/concretize.py            | 28 ++++++++++++++
 .../site_spackconfig/packages.yaml            | 13 +++++++
 .../mock_packages/externalprereq/package.py   | 34 +++++++++++++++++
 .../mock_packages/externaltest/package.py     | 37 +++++++++++++++++++
 .../mock_packages/externaltool/package.py     | 36 ++++++++++++++++++
 .../mock_packages/externalvirtual/package.py  | 37 +++++++++++++++++++
 8 files changed, 191 insertions(+), 5 deletions(-)
 create mode 100644 var/spack/mock_configs/site_spackconfig/packages.yaml
 create mode 100644 var/spack/mock_packages/externalprereq/package.py
 create mode 100644 var/spack/mock_packages/externaltest/package.py
 create mode 100644 var/spack/mock_packages/externaltool/package.py
 create mode 100644 var/spack/mock_packages/externalvirtual/package.py

diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py
index 01ff163493b78..c27a023136af2 100644
--- a/lib/spack/spack/concretize.py
+++ b/lib/spack/spack/concretize.py
@@ -85,8 +85,8 @@ def _valid_virtuals_and_externals(self, spec):
             provider_cmp = partial(spack.pkgsort.provider_compare, spec_w_preferred_providers.name, spec.name)
             packages = sorted(providers, cmp=provider_cmp)
         else:
-            if not spec_externals(spec) or spec.external:
-                return None
+            if spec.external:
+                return False
             packages = [spec]
 
         # For each candidate package, if it has externals add those to the candidates
@@ -129,7 +129,7 @@ def concretize_virtual_and_external(self, spec):
                 #Try a looser ABI matching
                 candidate = next((c for c in candidates if spack.abi.compatible(c[0], other_spec, loose=True)), None)
         if not candidate:
-            #Pick the first choice
+            #No ABI matches. Pick the top choice based on the orignal preferences.
             candidate = candidates[0]
         external = candidate[1]
         candidate_spec = candidate[0]
@@ -144,10 +144,11 @@ def concretize_virtual_and_external(self, spec):
         if not spec.external and external:
             spec.external = external
             changed = True
+
         #If we're external then trim the dependencies
         if external and spec.dependencies:
             changed = True
-            spec.depencencies = DependencyMap()
+            spec.dependencies = DependencyMap()
         
         return changed
         
diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py
index 6984b4a174a62..49b67cd361424 100644
--- a/lib/spack/spack/spec.py
+++ b/lib/spack/spack/spec.py
@@ -1022,7 +1022,7 @@ def _normalize_helper(self, visited, spec_deps, provider_index):
 
         # if we descend into a virtual spec, there's nothing more
         # to normalize.  Concretize will finish resolving it later.
-        if self.virtual:
+        if self.virtual or self.external:
             return False
 
         # Combine constraints from package deps with constraints from
diff --git a/lib/spack/spack/test/concretize.py b/lib/spack/spack/test/concretize.py
index b3a77d076a165..f81a2f5af8161 100644
--- a/lib/spack/spack/test/concretize.py
+++ b/lib/spack/spack/test/concretize.py
@@ -192,3 +192,31 @@ def test_compiler_inheritance(self):
         # TODO: not exactly the syntax I would like.
         self.assertTrue(spec['libdwarf'].compiler.satisfies('clang'))
         self.assertTrue(spec['libelf'].compiler.satisfies('clang'))
+
+
+    def test_external_package(self):
+        spec = Spec('externaltool')
+        spec.concretize()
+
+        self.assertEqual(spec['externaltool'].external, '/path/to/external_tool')
+        self.assertFalse('externalprereq' in spec)
+        self.assertTrue(spec['externaltool'].compiler.satisfies('gcc'))
+
+
+    def test_nobuild_package(self):
+        got_error = False
+        spec = Spec('externaltool%clang')
+        try:
+            spec.concretize()
+        except spack.concretize.NoBuildError:
+            got_error = True
+        self.assertTrue(got_error)
+
+
+    def test_external_and_virtual(self):
+        spec = Spec('externaltest')
+        spec.concretize()
+        self.assertTrue(spec['externaltool'].external, '/path/to/external_tool')
+        self.assertTrue(spec['stuff'].external, '/path/to/external_virtual_gcc')
+        self.assertTrue(spec['externaltool'].compiler.satisfies('gcc'))
+        self.assertTrue(spec['stuff'].compiler.satisfies('gcc'))
diff --git a/var/spack/mock_configs/site_spackconfig/packages.yaml b/var/spack/mock_configs/site_spackconfig/packages.yaml
new file mode 100644
index 0000000000000..eb52c6cf1137f
--- /dev/null
+++ b/var/spack/mock_configs/site_spackconfig/packages.yaml
@@ -0,0 +1,13 @@
+packages:
+  - externaltool:
+      nobuild: True
+  - externaltool@1.0%gcc@4.5.0:
+      path: /path/to/external_tool
+  - externalvirtual@2.0%clang@3.3:
+      path: /path/to/external_virtual_clang
+      nobuild: True
+  - externalvirtual@1.0%gcc@4.5.0:
+      path: /path/to/external_virtual_gcc
+      nobuild: True
+
+
diff --git a/var/spack/mock_packages/externalprereq/package.py b/var/spack/mock_packages/externalprereq/package.py
new file mode 100644
index 0000000000000..7d639256939d3
--- /dev/null
+++ b/var/spack/mock_packages/externalprereq/package.py
@@ -0,0 +1,34 @@
+##############################################################################
+# Copyright (c) 2013, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://scalability-llnl.github.io/spack
+# Please also see the LICENSE file for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License (as published by
+# the Free Software Foundation) version 2.1 dated February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+from spack import *
+
+class Externalprereq(Package):
+    homepage = "http://somewhere.com"
+    url      = "http://somewhere.com/prereq-1.0.tar.gz"
+
+    version('1.4', 'f1234567890abcdef1234567890abcde')
+
+    def install(self, spec, prefix):
+        pass
diff --git a/var/spack/mock_packages/externaltest/package.py b/var/spack/mock_packages/externaltest/package.py
new file mode 100644
index 0000000000000..c546922f876c6
--- /dev/null
+++ b/var/spack/mock_packages/externaltest/package.py
@@ -0,0 +1,37 @@
+##############################################################################
+# Copyright (c) 2013, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://scalability-llnl.github.io/spack
+# Please also see the LICENSE file for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License (as published by
+# the Free Software Foundation) version 2.1 dated February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+from spack import *
+
+class Externaltest(Package):
+    homepage = "http://somewhere.com"
+    url      = "http://somewhere.com/test-1.0.tar.gz"
+
+    version('1.0', '1234567890abcdef1234567890abcdef')
+
+    depends_on('stuff')
+    depends_on('externaltool')
+
+    def install(self, spec, prefix):
+        pass
diff --git a/var/spack/mock_packages/externaltool/package.py b/var/spack/mock_packages/externaltool/package.py
new file mode 100644
index 0000000000000..af902bd70eb20
--- /dev/null
+++ b/var/spack/mock_packages/externaltool/package.py
@@ -0,0 +1,36 @@
+##############################################################################
+# Copyright (c) 2013, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://scalability-llnl.github.io/spack
+# Please also see the LICENSE file for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License (as published by
+# the Free Software Foundation) version 2.1 dated February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+from spack import *
+
+class Externaltool(Package):
+    homepage = "http://somewhere.com"
+    url      = "http://somewhere.com/tool-1.0.tar.gz"
+
+    version('1.0', '1234567890abcdef1234567890abcdef')
+
+    depends_on('externalprereq')
+
+    def install(self, spec, prefix):
+        pass
diff --git a/var/spack/mock_packages/externalvirtual/package.py b/var/spack/mock_packages/externalvirtual/package.py
new file mode 100644
index 0000000000000..722c1e1c532d0
--- /dev/null
+++ b/var/spack/mock_packages/externalvirtual/package.py
@@ -0,0 +1,37 @@
+##############################################################################
+# Copyright (c) 2013, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://scalability-llnl.github.io/spack
+# Please also see the LICENSE file for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License (as published by
+# the Free Software Foundation) version 2.1 dated February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+from spack import *
+
+class Externalvirtual(Package):
+    homepage = "http://somewhere.com"
+    url      = "http://somewhere.com/stuff-1.0.tar.gz"
+
+    version('1.0', '1234567890abcdef1234567890abcdef')
+    version('2.0', '234567890abcdef1234567890abcdef1')
+
+    provides('stuff')
+
+    def install(self, spec, prefix):
+        pass

From fac4428766fb0a6b6cd357b654215f55df1220d4 Mon Sep 17 00:00:00 2001
From: Matthew LeGendre <legendre1@llnl.gov>
Date: Mon, 5 Oct 2015 14:04:33 -0700
Subject: [PATCH 11/14] Documentation for external packages.

---
 lib/spack/docs/site_configuration.rst | 73 +++++++++++++++++++++++++++
 1 file changed, 73 insertions(+)

diff --git a/lib/spack/docs/site_configuration.rst b/lib/spack/docs/site_configuration.rst
index 1e6740a434fae..a7211a9d95b88 100644
--- a/lib/spack/docs/site_configuration.rst
+++ b/lib/spack/docs/site_configuration.rst
@@ -54,6 +54,79 @@ more elements to the list to indicate where your own site's temporary
 directory is.
 
 
+External Packages
+~~~~~~~~~~~~~~~~~~~~~
+It's possible for Spack to use certain externally-installed 
+packages rather than always rebuilding packages. This may be desirable
+if machines ship with system packages, such as a customized MPI
+that should be used instead of Spack building its own MPI.
+
+External packages are configured through the ``packages.yaml`` file found
+in a Spack installation's ``etc/spack/`` or a user's ``~/.spack/``
+directory. Here's an example of an external configuration::
+
+.. code-block:: yaml
+
+  packages:
+    - openmpi@1.4.3%gcc@4.4.7=chaos_5_x86_64_ib:
+        path: /opt/openmpi-1.4.3
+    - openmpi@1.4.3%gcc@4.4.7=chaos_5_x86_64_ib+debug:
+        path: /opt/openmpi-1.4.3-debug
+    - openmpi@1.6.5%intel@10.1=chaos_5_x86_64_ib:
+        path: /opt/openmpi-1.6.5-intel
+
+This example lists three installations of OpenMPI, one built with gcc,
+one built with gcc and debug information, and another built with OpenMPI.
+If Spack is asked to build a package that uses one of these MPIs as a 
+dependency, it link the package to the pre-installed OpenMPI in
+the given directory.  
+
+Each ``packages.yaml`` should begin with a ``packages:`` token, followed
+by a list of package specs.  Specs in the ``packages.yaml`` have at most
+one ``path`` tag, which specifies the top-level directory where the 
+spec is installed.  
+
+Each spec should be as well-defined as reasonably possible.  If a 
+package lacks a spec component, such as missing a compiler or 
+package version, then Spack will guess the missing component based 
+on its most-favored packages, and it may guess incorrectly.
+
+All package versions and compilers listed in ``packages.yaml`` should 
+have entries in Spack's packages and compiler configuration, even
+the package and compiler may not actually be used.
+
+The packages configuration can tell Spack to use an external location
+for certain package versions, but it does not restrict Spack to using
+external packages.  In the above example, if an OpenMPI 1.8.4 became
+available Spack may choose to start building and linking with that version
+rather than continue using the pre-installed OpenMPI versions.
+
+To prevent this, the ``packages.yaml`` configuration also allows packages
+to be flagged as non-buildable.  The previous example could be modified to
+be::
+
+.. code-block:: yaml
+
+  packages:
+    - openmpi:
+        nobuild: True
+    - openmpi@1.4.3%gcc@4.4.7=chaos_5_x86_64_ib:
+        path: /opt/openmpi-1.4.3
+    - openmpi@1.4.3%gcc@4.4.7=chaos_5_x86_64_ib+debug:
+        path: /opt/openmpi-1.4.3-debug
+    - openmpi@1.6.5%intel@10.1=chaos_5_x86_64_ib:
+        path: /opt/openmpi-1.6.5-intel
+
+The addition of the ``nobuild`` flag tells Spack that it should never build
+its own version of OpenMPI, and it will instead always rely on a pre-built
+OpenMPI.  Similar to ``path``, ``nobuild`` is specified as a property under
+a spec and will prevent building of anything that satisfies that spec.
+
+The ``nobuild`` does not need to be paired with external packages.  
+It could also be used alone to forbid versions of packages that may be 
+buggy or otherwise undesirable.
+  
+
 Profiling
 ~~~~~~~~~~~~~~~~~~~~~
 

From 72d5cdf9abaee156d80dd027911a6cadbfebd7bb Mon Sep 17 00:00:00 2001
From: alalazo <massimiliano.culpo@googlemail.com>
Date: Wed, 20 Jan 2016 11:19:36 +0100
Subject: [PATCH 12/14] fixed two minor typos

---
 lib/spack/docs/site_configuration.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/spack/docs/site_configuration.rst b/lib/spack/docs/site_configuration.rst
index a7211a9d95b88..8ab4e89dfced0 100644
--- a/lib/spack/docs/site_configuration.rst
+++ b/lib/spack/docs/site_configuration.rst
@@ -76,9 +76,9 @@ directory. Here's an example of an external configuration::
         path: /opt/openmpi-1.6.5-intel
 
 This example lists three installations of OpenMPI, one built with gcc,
-one built with gcc and debug information, and another built with OpenMPI.
+one built with gcc and debug information, and another built with Intel.
 If Spack is asked to build a package that uses one of these MPIs as a 
-dependency, it link the package to the pre-installed OpenMPI in
+dependency, it links the package to the pre-installed OpenMPI in
 the given directory.  
 
 Each ``packages.yaml`` should begin with a ``packages:`` token, followed

From a384ad5b1270140d71110e46d39144a0f0e9081e Mon Sep 17 00:00:00 2001
From: Matthew LeGendre <legendre1@llnl.gov>
Date: Wed, 9 Mar 2016 16:11:33 -0800
Subject: [PATCH 13/14] Fix problem with pure integer arguments in preferred
 versions list (e.g, 2 instead of 2.7.3)

---
 lib/spack/spack/config.py             | 5 +++--
 lib/spack/spack/preferred_packages.py | 6 +++---
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py
index 807a898644d5e..95a988f7ffef2 100644
--- a/lib/spack/spack/config.py
+++ b/lib/spack/spack/config.py
@@ -214,7 +214,8 @@
                             'version': {
                                 'type' : 'array',
                                 'default' : [],
-                                'items' : { 'type' : 'string' } }, #version strings
+                                'items' : { 'anyOf' : [ { 'type' : 'string' },
+                                                        { 'type' : 'number'}]}}, #version strings
                             'compiler': {
                                 'type' : 'array',
                                 'default' : [],
@@ -573,7 +574,7 @@ def __init__(self, validation_error, data):
         # Try to get line number from erroneous instance and its parent
         instance_mark = getattr(validation_error.instance, '_start_mark', None)
         parent_mark = getattr(validation_error.parent, '_start_mark', None)
-        path = getattr(validation_error, 'path', None)
+        path = [str(s) for s in getattr(validation_error, 'path', None)]
 
         # Try really hard to get the parent (which sometimes is not
         # set) This digs it out of the validated structure if it's not
diff --git a/lib/spack/spack/preferred_packages.py b/lib/spack/spack/preferred_packages.py
index 2b0ba791b6b0c..eaea016a85a58 100644
--- a/lib/spack/spack/preferred_packages.py
+++ b/lib/spack/spack/preferred_packages.py
@@ -45,7 +45,7 @@ def _order_for_package(self, pkgname, component, second_key, test_all=True):
                 order = order.get(second_key, {})
             if not order:
                 continue
-            return [s.strip() for s in order]
+            return [str(s).strip() for s in order]
         return []
             
 
@@ -98,11 +98,11 @@ def _spec_compare(self, pkgname, component, a, b, reverse_natural_compare, secon
         b_index = None
         reverse = -1 if reverse_natural_compare else 1
         for i, cspec in enumerate(specs):
-            if a_index == None and cspec.satisfies(a):
+            if a_index == None and (cspec.satisfies(a) or a.satisfies(cspec)):
                 a_index = i
                 if b_index:
                     break
-            if b_index == None and cspec.satisfies(b):
+            if b_index == None and (cspec.satisfies(b) or b.satisfies(cspec)):
                 b_index = i
                 if a_index:
                     break

From 1f06dd40f7e65252568da23e9758bf5af02833eb Mon Sep 17 00:00:00 2001
From: Matthew LeGendre <legendre1@llnl.gov>
Date: Wed, 9 Mar 2016 16:11:53 -0800
Subject: [PATCH 14/14] Update documentation for new packages.yaml config
 format.

---
 lib/spack/docs/packaging_guide.rst    | 41 +++++++++---------
 lib/spack/docs/site_configuration.rst | 61 +++++++++++++--------------
 2 files changed, 49 insertions(+), 53 deletions(-)

diff --git a/lib/spack/docs/packaging_guide.rst b/lib/spack/docs/packaging_guide.rst
index f368d0a4fa81b..ef9fd89b62f9d 100644
--- a/lib/spack/docs/packaging_guide.rst
+++ b/lib/spack/docs/packaging_guide.rst
@@ -1561,50 +1561,49 @@ be concretized on their system.  For example, one user may prefer packages
 built with OpenMPI and the Intel compiler.  Another user may prefer
 packages be built with MVAPICH and GCC.  
 
-Spack's ``preferred`` configuration can be used to set defaults for sites or users.
-Spack uses this configuration to make decisions about which compilers, package
-versions, depends_on, and variants it should prefer during concretization.
+Spack can be configurated to prefer certain compilers, package
+versions, depends_on, and variants during concretization.
+The preferred configuration can be controlled via the 
+``~/.spack/packages.yaml`` file for user configuations, or the
+``etc/spack/packages.yaml`` site configuration.
 
-The preferred configuration can be controlled by editing the 
-``~/.spack/preferred.yaml`` file for user configuations, or the
 
-
-Here's an example preferred.yaml file:
+Here's an example packages.yaml file that sets preferred packages:
 
 .. code-block:: sh
 
-    preferred:
+    packages:
       dyninst:
-        compiler: gcc@4.9
+        compiler: [gcc@4.9]
         variants: +debug
       gperftools:
-        version: 2.2, 2.4, 2.3
+        version: [2.2, 2.4, 2.3]
       all:
-        compiler: gcc@4.4.7, gcc@4.6:, intel, clang, pgi
-        providers: 
-          mpi: mvapich, mpich, openmpi
+        compiler: [gcc@4.4.7, gcc@4.6:, intel, clang, pgi]
+        providers:
+          mpi: [mvapich, mpich, openmpi]
+          
 
 At a high level, this example is specifying how packages should be
 concretized.  The dyninst package should prefer using gcc 4.9 and 
 be built with debug options.  The gperftools package should prefer version
 2.2 over 2.4.  Every package on the system should prefer mvapich for
-its MPI and gcc 4.4.7 (except for Dyninst, which perfers gcc 4.9).  
+its MPI and gcc 4.4.7 (except for Dyninst, which overrides this by perfering gcc 4.9).  
 These options are used to fill in implicit defaults.  Any of them can be overwritten 
 on the command line if explicitly requested.
 
-Each preferred.yaml file begin with the string ``preferred:`` and 
-each subsequent entry is indented underneath it.  The next layer contains
-package names or the special string ``all`` (which applies to 
-every package).  Underneath each package name is 
+Each packages.yaml file begin with the string ``packages:`` and 
+package names are specified on the next level. The special string ``all``
+applies settings to each package. Underneath each package name is 
 one or more components: ``compiler``, ``variants``, ``version``, 
 or ``providers``.  Each component has an ordered list of spec 
 ``constraints``, with earlier entries in the list being prefered over
-latter entries.
+later entries.
 
 Sometimes a package installation may have constraints that forbid 
 the first concretization rule, in which case Spack will use the first
 legal concretization rule.  Going back to the example, if a user
-requests gperftools 2.3 or latter, then Spack will install version 2.4 
+requests gperftools 2.3 or later, then Spack will install version 2.4 
 as the 2.4 version of gperftools is preferred over 2.3.
 
 An explicit concretization rule in the preferred section will always 
@@ -1612,7 +1611,7 @@ take preference over unlisted concretizations.  In the above example,
 xlc isn't listed in the compiler list.  Every listed compiler from
 gcc to pgi will thus be preferred over the xlc compiler.
 
-The syntax for the ``providers`` section differs slightly from other
+The syntax for the ``provider`` section differs slightly from other
 concretization rules.  A provider lists a value that packages may
 ``depend_on`` (e.g, mpi) and a list of rules for fulfilling that
 dependency.
diff --git a/lib/spack/docs/site_configuration.rst b/lib/spack/docs/site_configuration.rst
index a7211a9d95b88..ebf043710674f 100644
--- a/lib/spack/docs/site_configuration.rst
+++ b/lib/spack/docs/site_configuration.rst
@@ -56,44 +56,43 @@ directory is.
 
 External Packages
 ~~~~~~~~~~~~~~~~~~~~~
-It's possible for Spack to use certain externally-installed 
-packages rather than always rebuilding packages. This may be desirable
+Spack can be configured to use externally-installed 
+packages rather than building its own packages. This may be desirable
 if machines ship with system packages, such as a customized MPI
 that should be used instead of Spack building its own MPI.
 
 External packages are configured through the ``packages.yaml`` file found
 in a Spack installation's ``etc/spack/`` or a user's ``~/.spack/``
-directory. Here's an example of an external configuration::
+directory. Here's an example of an external configuration:
 
 .. code-block:: yaml
 
-  packages:
-    - openmpi@1.4.3%gcc@4.4.7=chaos_5_x86_64_ib:
-        path: /opt/openmpi-1.4.3
-    - openmpi@1.4.3%gcc@4.4.7=chaos_5_x86_64_ib+debug:
-        path: /opt/openmpi-1.4.3-debug
-    - openmpi@1.6.5%intel@10.1=chaos_5_x86_64_ib:
-        path: /opt/openmpi-1.6.5-intel
+   packages:
+      openmpi:
+         paths:
+            openmpi@1.4.3%gcc@4.4.7=chaos_5_x86_64_ib: /opt/openmpi-1.4.3
+            openmpi@1.4.3%gcc@4.4.7=chaos_5_x86_64_ib+debug: /opt/openmpi-1.4.3-debug
+            openmpi@1.6.5%intel@10.1=chaos_5_x86_64_ib: /opt/openmpi-1.6.5-intel
 
 This example lists three installations of OpenMPI, one built with gcc,
 one built with gcc and debug information, and another built with OpenMPI.
 If Spack is asked to build a package that uses one of these MPIs as a 
-dependency, it link the package to the pre-installed OpenMPI in
-the given directory.  
-
-Each ``packages.yaml`` should begin with a ``packages:`` token, followed
-by a list of package specs.  Specs in the ``packages.yaml`` have at most
-one ``path`` tag, which specifies the top-level directory where the 
-spec is installed.  
-
-Each spec should be as well-defined as reasonably possible.  If a 
+dependency, it will use the the pre-installed OpenMPI in
+the given directory.  This example also specifies that Spack should never
+build its own OpenMPI via the ``nobuild: True`` option.
+
+Each ``packages.yaml`` begins with a ``packages:`` token, followed
+by a list of package names.  To specify externals, add a ``paths``
+token under the package name, which lists externals in a
+``spec : /path`` format.  Each spec should be as
+well-defined as reasonably possible.  If a 
 package lacks a spec component, such as missing a compiler or 
 package version, then Spack will guess the missing component based 
 on its most-favored packages, and it may guess incorrectly.
 
-All package versions and compilers listed in ``packages.yaml`` should 
+Each package version and compilers listed in an external should 
 have entries in Spack's packages and compiler configuration, even
-the package and compiler may not actually be used.
+though the package and compiler may not every be built.
 
 The packages configuration can tell Spack to use an external location
 for certain package versions, but it does not restrict Spack to using
@@ -103,27 +102,25 @@ rather than continue using the pre-installed OpenMPI versions.
 
 To prevent this, the ``packages.yaml`` configuration also allows packages
 to be flagged as non-buildable.  The previous example could be modified to
-be::
+be:
 
 .. code-block:: yaml
 
   packages:
-    - openmpi:
-        nobuild: True
-    - openmpi@1.4.3%gcc@4.4.7=chaos_5_x86_64_ib:
-        path: /opt/openmpi-1.4.3
-    - openmpi@1.4.3%gcc@4.4.7=chaos_5_x86_64_ib+debug:
-        path: /opt/openmpi-1.4.3-debug
-    - openmpi@1.6.5%intel@10.1=chaos_5_x86_64_ib:
-        path: /opt/openmpi-1.6.5-intel
+    openmpi:
+      paths:
+        openmpi@1.4.3%gcc@4.4.7=chaos_5_x86_64_ib: /opt/openmpi-1.4.3
+        openmpi@1.4.3%gcc@4.4.7=chaos_5_x86_64_ib+debug: /opt/openmpi-1.4.3-debug
+        openmpi@1.6.5%intel@10.1=chaos_5_x86_64_ib: /opt/openmpi-1.6.5-intel                
+    nobuild: True
 
 The addition of the ``nobuild`` flag tells Spack that it should never build
 its own version of OpenMPI, and it will instead always rely on a pre-built
 OpenMPI.  Similar to ``path``, ``nobuild`` is specified as a property under
-a spec and will prevent building of anything that satisfies that spec.
+a package name.
 
 The ``nobuild`` does not need to be paired with external packages.  
-It could also be used alone to forbid versions of packages that may be 
+It could also be used alone to forbid packages that may be 
 buggy or otherwise undesirable.