Skip to content

Commit

Permalink
Merge pull request #984 from epfl-scitas/module_file_explicit_load
Browse files Browse the repository at this point in the history 
module files : explicit load and mnemonic suffixes
  • Loading branch information
@tgamblin
tgamblin committed Jun 27, 2016
2 parents a1db306 + 670669e commit 73213ac
Show file tree
Hide file tree
Showing 3 changed files with 185 additions and 10 deletions.
2 changes: 2 additions & 0 deletions lib/spack/spack/config.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -307,6 +307,8 @@
'autoload': {'$ref': '#/definitions/dependency_selection'},
'prerequisites': {'$ref': '#/definitions/dependency_selection'},
'conflict': {'$ref': '#/definitions/array_of_strings'},
'load': {'$ref': '#/definitions/array_of_strings'},
'suffixes': {'$ref': '#/definitions/dictionary_of_strings'},
'environment': {
'type': 'object',
'default': {},
Expand Down
21 changes: 17 additions & 4 deletions lib/spack/spack/modules.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -285,11 +285,18 @@ def use_name(self):
naming_tokens = self.tokens
naming_scheme = self.naming_scheme
name = naming_scheme.format(**naming_tokens)
name += '-' + self.spec.dag_hash(
) # Always append the hash to make the module file unique
# Not everybody is working on linux...
parts = name.split('/')
name = join_path(*parts)
# Add optional suffixes based on constraints
configuration, _ = parse_config_options(self)
suffixes = [name]
for constraint, suffix in configuration.get('suffixes', {}).items():
if constraint in self.spec:
suffixes.append(suffix)
# Always append the hash to make the module file unique
suffixes.append(self.spec.dag_hash())
name = '-'.join(suffixes)
return name

@property
Expand Down Expand Up @@ -381,6 +388,8 @@ def write(self):
for x in filter_blacklisted(
module_configuration.pop('autoload', []), self.name):
module_file_content += self.autoload(x)
for x in module_configuration.pop('load', []):
module_file_content += self.autoload(x)
for x in filter_blacklisted(
module_configuration.pop('prerequisites', []), self.name):
module_file_content += self.prerequisite(x)
Expand All @@ -402,8 +411,12 @@ def module_specific_content(self, configuration):
return tuple()

def autoload(self, spec):
m = type(self)(spec)
return self.autoload_format.format(module_file=m.use_name)
if not isinstance(spec, str):
m = type(self)(spec)
module_file = m.use_name
else:
module_file = spec
return self.autoload_format.format(module_file=module_file)

def prerequisite(self, spec):
m = type(self)(spec)
Expand Down
172 changes: 166 additions & 6 deletions lib/spack/spack/test/modules.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@

import StringIO
import spack.modules
import unittest
from spack.test.mock_packages_test import MockPackagesTest

FILE_REGISTRY = collections.defaultdict(StringIO.StringIO)
Expand Down Expand Up @@ -67,23 +68,47 @@ def mock_open(filename, mode):
}
}

configuration_prerequisites_direct = {
'enable': ['tcl'],
'tcl': {
'all': {
'prerequisites': 'direct'
}
}
}

configuration_prerequisites_all = {
'enable': ['tcl'],
'tcl': {
'all': {
'prerequisites': 'all'
}
}
}

configuration_alter_environment = {
'enable': ['tcl'],
'tcl': {
'all': {
'filter': {'environment_blacklist': ['CMAKE_PREFIX_PATH']}
},
'platform=test target=x86_64': {
'environment': {'set': {'FOO': 'foo'},
'unset': ['BAR']}
'environment': {
'set': {'FOO': 'foo'},
'unset': ['BAR']
}
},
'platform=test target=x86_32': {
'load': ['foo/bar']
}
}
}

configuration_blacklist = {
'enable': ['tcl'],
'tcl': {
'blacklist': ['callpath'],
'whitelist': ['zmpi'],
'blacklist': ['callpath', 'mpi'],
'all': {
'autoload': 'direct'
}
Expand All @@ -100,8 +125,68 @@ def mock_open(filename, mode):
}
}

configuration_wrong_conflicts = {
'enable': ['tcl'],
'tcl': {
'naming_scheme': '{name}/{version}-{compiler.name}',
'all': {
'conflict': ['{name}/{compiler.name}']
}
}
}

configuration_suffix = {
'enable': ['tcl'],
'tcl': {
'mpileaks': {
'suffixes': {
'+debug': 'foo',
'~debug': 'bar'
}
}
}
}


class HelperFunctionsTests(unittest.TestCase):

def test_update_dictionary_extending_list(self):
target = {
'foo': {
'a': 1,
'b': 2,
'd': 4
},
'bar': [1, 2, 4],
'baz': 'foobar'
}
update = {
'foo': {
'c': 3,
},
'bar': [3],
'baz': 'foobaz',
'newkey': {
'd': 4
}
}
spack.modules.update_dictionary_extending_lists(target, update)
self.assertTrue(len(target) == 4)
self.assertTrue(len(target['foo']) == 4)
self.assertTrue(len(target['bar']) == 4)
self.assertEqual(target['baz'], 'foobaz')

def test_inspect_path(self):
env = spack.modules.inspect_path('/usr')
names = [item.name for item in env]
self.assertTrue('PATH' in names)
self.assertTrue('LIBRARY_PATH' in names)
self.assertTrue('LD_LIBRARY_PATH' in names)
self.assertTrue('CPATH' in names)


class TclTests(MockPackagesTest):

def setUp(self):
super(TclTests, self).setUp()
self.configuration_obj = spack.modules.CONFIGURATION
Expand All @@ -116,7 +201,6 @@ def tearDown(self):

def get_modulefile_content(self, spec):
spec.concretize()
print spec, '&&&&&'
generator = spack.modules.TclModule(spec)
generator.write()
content = FILE_REGISTRY[generator.file_name].split('\n')
Expand All @@ -127,6 +211,8 @@ def test_simple_case(self):
spec = spack.spec.Spec('mpich@3.0.4')
content = self.get_modulefile_content(spec)
self.assertTrue('module-whatis "mpich @3.0.4"' in content)
self.assertRaises(TypeError, spack.modules.dependencies,
spec, 'non-existing-tag')

def test_autoload(self):
spack.modules.CONFIGURATION = configuration_autoload_direct
Expand All @@ -141,11 +227,21 @@ def test_autoload(self):
self.assertEqual(len([x for x in content if 'is-loaded' in x]), 5)
self.assertEqual(len([x for x in content if 'module load ' in x]), 5)

def test_prerequisites(self):
spack.modules.CONFIGURATION = configuration_prerequisites_direct
spec = spack.spec.Spec('mpileaks arch=x86-linux')
content = self.get_modulefile_content(spec)
self.assertEqual(len([x for x in content if 'prereq' in x]), 2)

spack.modules.CONFIGURATION = configuration_prerequisites_all
spec = spack.spec.Spec('mpileaks arch=x86-linux')
content = self.get_modulefile_content(spec)
self.assertEqual(len([x for x in content if 'prereq' in x]), 5)

def test_alter_environment(self):
spack.modules.CONFIGURATION = configuration_alter_environment
spec = spack.spec.Spec('mpileaks platform=test target=x86_64')
content = self.get_modulefile_content(spec)
print content
self.assertEqual(
len([x
for x in content
Expand All @@ -156,21 +252,31 @@ def test_alter_environment(self):

spec = spack.spec.Spec('libdwarf %clang platform=test target=x86_32')
content = self.get_modulefile_content(spec)
print content
self.assertEqual(
len([x
for x in content
if x.startswith('prepend-path CMAKE_PREFIX_PATH')]), 0)
self.assertEqual(
len([x for x in content if 'setenv FOO "foo"' in x]), 0)
self.assertEqual(len([x for x in content if 'unsetenv BAR' in x]), 0)
self.assertEqual(
len([x for x in content if 'is-loaded foo/bar' in x]), 1)
self.assertEqual(
len([x for x in content if 'module load foo/bar' in x]), 1)

def test_blacklist(self):
spack.modules.CONFIGURATION = configuration_blacklist
spec = spack.spec.Spec('mpileaks')
content = self.get_modulefile_content(spec)
self.assertEqual(len([x for x in content if 'is-loaded' in x]), 1)
self.assertEqual(len([x for x in content if 'module load ' in x]), 1)
spec = spack.spec.Spec('callpath arch=x86-linux')
# Returns a StringIO instead of a string as no module file was written
self.assertRaises(AttributeError, self.get_modulefile_content, spec)
spec = spack.spec.Spec('zmpi arch=x86-linux')
content = self.get_modulefile_content(spec)
self.assertEqual(len([x for x in content if 'is-loaded' in x]), 1)
self.assertEqual(len([x for x in content if 'module load ' in x]), 1)

def test_conflicts(self):
spack.modules.CONFIGURATION = configuration_conflicts
Expand All @@ -182,3 +288,57 @@ def test_conflicts(self):
len([x for x in content if x == 'conflict mpileaks']), 1)
self.assertEqual(
len([x for x in content if x == 'conflict intel/14.0.1']), 1)

spack.modules.CONFIGURATION = configuration_wrong_conflicts
self.assertRaises(SystemExit, self.get_modulefile_content, spec)

def test_suffixes(self):
spack.modules.CONFIGURATION = configuration_suffix
spec = spack.spec.Spec('mpileaks+debug arch=x86-linux')
spec.concretize()
generator = spack.modules.TclModule(spec)
self.assertTrue('foo' in generator.use_name)

spec = spack.spec.Spec('mpileaks~debug arch=x86-linux')
spec.concretize()
generator = spack.modules.TclModule(spec)
self.assertTrue('bar' in generator.use_name)


configuration_dotkit = {
'enable': ['dotkit'],
'dotkit': {
'all': {
'prerequisites': 'direct'
}
}
}


class DotkitTests(MockPackagesTest):

def setUp(self):
super(DotkitTests, self).setUp()
self.configuration_obj = spack.modules.CONFIGURATION
spack.modules.open = mock_open
# Make sure that a non-mocked configuration will trigger an error
spack.modules.CONFIGURATION = None

def tearDown(self):
del spack.modules.open
spack.modules.CONFIGURATION = self.configuration_obj
super(DotkitTests, self).tearDown()

def get_modulefile_content(self, spec):
spec.concretize()
generator = spack.modules.Dotkit(spec)
generator.write()
content = FILE_REGISTRY[generator.file_name].split('\n')
return content

def test_dotkit(self):
spack.modules.CONFIGURATION = configuration_dotkit
spec = spack.spec.Spec('mpileaks arch=x86-linux')
content = self.get_modulefile_content(spec)
self.assertTrue('#c spack' in content)
self.assertTrue('#d mpileaks @2.3' in content)

0 comments on commit 73213ac

Please sign in to comment.