Merge pull request #879 from boegel/finish_custom_module_naming_schem…

…e_support

finish custom module naming scheme support

easybuild/framework/easyblock.py

@@ -51,10 +51,11 @@
 
 import easybuild.tools.environment as env
 from easybuild.tools import config, filetools
-from easybuild.framework.easyconfig.easyconfig import EasyConfig, ITERATE_OPTIONS, fetch_parameter_from_easyconfig_file
-from easybuild.framework.easyconfig.easyconfig import get_class_for, get_easyblock_class, get_module_path
-from easybuild.framework.easyconfig.easyconfig import resolve_template
-from easybuild.framework.easyconfig.tools import get_paths_for, resolve_dependencies
+from easybuild.framework.easyconfig.default import get_easyconfig_parameter_default
+from easybuild.framework.easyconfig.easyconfig import EasyConfig, ITERATE_OPTIONS
+from easybuild.framework.easyconfig.easyconfig import fetch_parameter_from_easyconfig_file, get_class_for
+from easybuild.framework.easyconfig.easyconfig import get_easyblock_class, get_module_path, resolve_template
+from easybuild.framework.easyconfig.tools import det_full_module_name, get_paths_for, resolve_dependencies
 from easybuild.framework.easyconfig.templates import TEMPLATE_NAMES_EASYBLOCK_RUN_STEP
 from easybuild.tools.build_details import get_build_stats
 from easybuild.tools.build_log import EasyBuildError, print_error, print_msg
@@ -67,7 +68,7 @@
 from easybuild.tools.filetools import write_file, compute_checksum, verify_checksum
 from easybuild.tools.jenkins import write_to_xml
 from easybuild.tools.module_generator import GENERAL_CLASS, ModuleGenerator
-from easybuild.tools.module_generator import det_full_module_name, det_devel_module_filename
+from easybuild.tools.module_generator import det_devel_module_filename
 from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version
 from easybuild.tools.modules import ROOT_ENV_VAR_NAME_PREFIX, VERSION_ENV_VAR_NAME_PREFIX, DEVEL_ENV_VAR_NAME_PREFIX
 from easybuild.tools.modules import get_software_root, modules_tool
@@ -777,10 +778,9 @@ def make_module_dep(self):
         builddeps = self.cfg.builddependencies()
         for dep in self.toolchain.dependencies:
             if not dep in builddeps:
-                dep_mod_name = det_full_module_name(dep)
-                self.log.debug("Adding %s as a module dependency" % dep_mod_name)
-                load += self.moduleGenerator.load_module(dep_mod_name, recursive_unload=self.recursive_mod_unload)
-                unload += self.moduleGenerator.unload_module(dep_mod_name)
+                self.log.debug("Adding %s as a module dependency" % dep['mod_name'])
+                load += self.moduleGenerator.load_module(dep['mod_name'], recursive_unload=self.recursive_mod_unload)
+                unload += self.moduleGenerator.unload_module(dep['mod_name'])
             else:
                 self.log.debug("Skipping build dependency %s" % str(dep))
 
@@ -1978,9 +1978,9 @@ def build_and_install_software(module, orig_environ):
                 # upload spec to central repository
                 currentbuildstats = app.cfg['buildstats']
                 repo = init_repository(get_repository(), get_repositorypath())
-                if 'originalSpec' in module:
+                if 'original_spec' in module:
                     block = det_full_ec_version(app.cfg) + ".block"
-                    repo.add_easyconfig(module['originalSpec'], app.name, block, buildstats, currentbuildstats)
+                    repo.add_easyconfig(module['original_spec'], app.name, block, buildstats, currentbuildstats)
                 repo.add_easyconfig(spec, app.name, det_full_ec_version(app.cfg), buildstats, currentbuildstats)
                 repo.commit("Built %s" % det_full_module_name(app.cfg))
                 del repo

easybuild/framework/easyconfig/easyconfig.py

@@ -39,21 +39,24 @@
 import difflib
 import os
 import re
-from vsc import fancylogger
+from vsc.utils import fancylogger
 from vsc.utils.missing import any, nub
 
 import easybuild.tools.environment as env
+from easybuild.tools.build_log import EasyBuildError
 from easybuild.tools.config import build_option
-from easybuild.tools.filetools import decode_class_name, encode_class_name, read_file, run_cmd
+from easybuild.tools.filetools import decode_class_name, encode_class_name, read_file
 from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version
+from easybuild.tools.module_generator import det_full_module_name as _det_full_module_name
 from easybuild.tools.modules import get_software_root_env_var_name, get_software_version_env_var_name
-from easybuild.tools.systemtools import check_os_dependency, get_shared_lib_ext
+from easybuild.tools.systemtools import check_os_dependency
 from easybuild.tools.toolchain import DUMMY_TOOLCHAIN_NAME, DUMMY_TOOLCHAIN_VERSION
 from easybuild.tools.toolchain.utilities import search_toolchain
 from easybuild.tools.utilities import remove_unwanted_chars
 from easybuild.framework.easyconfig import MANDATORY
 from easybuild.framework.easyconfig.default import DEFAULT_CONFIG, ALL_CATEGORIES, get_easyconfig_parameter_default
 from easybuild.framework.easyconfig.format.convert import Dependency
+from easybuild.framework.easyconfig.format.one import retrieve_blocks_in_spec
 from easybuild.framework.easyconfig.licenses import EASYCONFIG_LICENSES_DICT, License
 from easybuild.framework.easyconfig.parser import EasyConfigParser
 from easybuild.framework.easyconfig.templates import template_constant_dict
@@ -491,18 +494,19 @@ def _parse_dependency(self, dep):
         of these attributes, 'name' and 'version' are mandatory
 
         output dict contains these attributes:
-        ['name', 'version', 'versionsuffix', 'dummy', 'toolchain']
+        ['name', 'version', 'versionsuffix', 'dummy', 'toolchain', 'mod_name']
         """
         # convert tuple to string otherwise python might complain about the formatting
         self.log.debug("Parsing %s as a dependency" % str(dep))
 
         attr = ['name', 'version', 'versionsuffix', 'toolchain']
         dependency = {
-            'name': '',
+            'dummy': False,
+            'mod_name': None,  # module name
+            'name': '',  # software name
+            'toolchain': None,
             'version': '',
             'versionsuffix': '',
-            'toolchain': None,
-            'dummy': False,
         }
         if isinstance(dep, dict):
             dependency.update(dep)
@@ -558,6 +562,8 @@ def _parse_dependency(self, dep):
         if not dependency['version']:
             self.log.error("Dependency specified without version: %s" % dependency)
 
+        dependency['mod_name'] = det_full_module_name(dependency)
+
         return dependency
 
     def generate_template_values(self):
@@ -818,3 +824,127 @@ def resolve_template(value, tmpl_dict):
 
     return value
 
+
+def process_easyconfig(path, build_specs=None, validate=True):
+    """
+    Process easyconfig, returning some information for each block
+    @param path: path to easyconfig file
+    @param build_specs: dictionary specifying build specifications (e.g. version, toolchain, ...)
+    @param validate: whether or not to perform validation
+    """
+    blocks = retrieve_blocks_in_spec(path, build_option('only_blocks'))
+
+    easyconfigs = []
+    for spec in blocks:
+        # process for dependencies and real installversionname
+        _log.debug("Processing easyconfig %s" % spec)
+
+        # create easyconfig
+        try:
+            ec = EasyConfig(spec, build_specs=build_specs, validate=validate)
+        except EasyBuildError, err:
+            msg = "Failed to process easyconfig %s:\n%s" % (spec, err.msg)
+            _log.exception(msg)
+
+        name = ec['name']
+
+        # this app will appear as following module in the list
+        easyconfig = {
+            'ec': ec,
+            'spec': spec,
+            'module': det_full_module_name(ec),
+            'dependencies': [],
+            'builddependencies': [],
+        }
+        if len(blocks) > 1:
+            easyconfig['original_spec'] = path
+
+        # add build dependencies
+        for dep in ec.builddependencies():
+            _log.debug("Adding build dependency %s for app %s." % (dep, name))
+            easyconfig['builddependencies'].append(dep)
+
+        # add dependencies (including build dependencies)
+        for dep in ec.dependencies():
+            _log.debug("Adding dependency %s for app %s." % (dep, name))
+            easyconfig['dependencies'].append(dep)
+
+        # add toolchain as dependency too
+        if ec.toolchain.name != DUMMY_TOOLCHAIN_NAME:
+            dep = ec.toolchain.as_dict()
+            _log.debug("Adding toolchain %s as dependency for app %s." % (dep, name))
+            easyconfig['dependencies'].append(dep)
+
+        del ec
+
+        # this is used by the parallel builder
+        easyconfig['unresolved_deps'] = copy.deepcopy(easyconfig['dependencies'])
+
+        easyconfigs.append(easyconfig)
+
+    return easyconfigs
+
+
+def create_paths(path, name, version):
+    """
+    Returns all the paths where easyconfig could be located
+    <path> is the basepath
+    <name> should be a string
+    <version> can be a '*' if you use glob patterns, or an install version otherwise
+    """
+    cand_paths = [
+        (name, version),  # e.g. <path>/GCC/4.8.2.eb
+        (name, "%s-%s" % (name, version)),  # e.g. <path>/GCC/GCC-4.8.2.eb
+        (name.lower()[0], name, "%s-%s" % (name, version)),  # e.g. <path>/g/GCC/GCC-4.8.2.eb
+        ("%s-%s" % (name, version),),  # e.g. <path>/GCC-4.8.2.eb
+    ]
+    return ["%s.eb" % os.path.join(path, *cand_path) for cand_path in cand_paths]
+
+
+def robot_find_easyconfig(paths, name, version):
+    """
+    Find an easyconfig for module in path
+    """
+    if not isinstance(paths, (list, tuple)):
+        paths = [paths]
+    # candidate easyconfig paths
+    for path in paths:
+        easyconfigs_paths = create_paths(path, name, version)
+        for easyconfig_path in easyconfigs_paths:
+            _log.debug("Checking easyconfig path %s" % easyconfig_path)
+            if os.path.isfile(easyconfig_path):
+                _log.debug("Found easyconfig file for name %s, version %s at %s" % (name, version, easyconfig_path))
+                return os.path.abspath(easyconfig_path)
+
+    return None
+
+
+def det_full_module_name(ec, eb_ns=False):
+    """
+    Determine full module name following the currently active module naming scheme.
+
+    First try to pass 'parsed' easyconfig as supplied,
+    try and find a matching easyconfig file, parse it and supply it in case of a KeyError.
+    """
+    try:
+        mod_name = _det_full_module_name(ec, eb_ns=eb_ns)
+
+    except KeyError, err:
+        _log.debug("KeyError '%s' when determining module name for %s, trying fallback procedure..." % (err, ec))
+        # for dependencies, only name/version/versionsuffix/toolchain easyconfig parameters are available;
+        # when a key error occurs, try and find an easyconfig file to parse via the robot,
+        # and retry with the parsed easyconfig file (which will contains a full set of keys)
+        robot = build_option('robot_path')
+        eb_file = robot_find_easyconfig(robot, ec['name'], det_full_ec_version(ec))
+        if eb_file is None:
+            _log.error("Failed to find an easyconfig file when determining module name for: %s" % ec)
+        else:
+            parsed_ec = process_easyconfig(eb_file)
+            if len(parsed_ec) > 1:
+                _log.warning("More than one parsed easyconfig obtained from %s, only retaining first" % eb_file)
+            try:
+                mod_name = _det_full_module_name(parsed_ec[0]['ec'], eb_ns=eb_ns)
+            except KeyError, err:
+                _log.error("A KeyError '%s' occured when determining a module name for %s." % parsed_ec['ec'])
+
+    return mod_name

easybuild/framework/easyconfig/format/one.py

@@ -36,8 +36,11 @@
 import tempfile
 from vsc import fancylogger
 
+from easybuild.framework.easyconfig.format.format import FORMAT_DEFAULT_VERSION, get_format_version
 from easybuild.framework.easyconfig.format.pyheaderconfigobj import EasyConfigFormatConfigObj
 from easybuild.framework.easyconfig.format.version import EasyVersion
+from easybuild.tools.build_log import print_msg
+from easybuild.tools.filetools import write_file
 
 
 _log = fancylogger.getLogger('easyconfig.format.one', fname=False)
@@ -84,3 +87,91 @@ def parse(self, txt):
         Pre-process txt to extract header, docstring and pyheader, with non-indented section markers enforced.
         """
         super(FormatOneZero, self).parse(txt, strict_section_markers=True)
+
+
+def retrieve_blocks_in_spec(spec, only_blocks, silent=False):
+    """
+    Easyconfigs can contain blocks (headed by a [Title]-line)
+    which contain commands specific to that block. Commands in the beginning of the file
+    above any block headers are common and shared between each block.
+    """
+    reg_block = re.compile(r"^\s*\[([\w.-]+)\]\s*$", re.M)
+    reg_dep_block = re.compile(r"^\s*block\s*=(\s*.*?)\s*$", re.M)
+
+    spec_fn = os.path.basename(spec)
+    try:
+        txt = open(spec).read()
+    except IOError, err:
+        _log.error("Failed to read file %s: %s" % (spec, err))
+
+    # split into blocks using regex
+    pieces = reg_block.split(txt)
+    # the first block contains common statements
+    common = pieces.pop(0)
+
+    # determine version of easyconfig format
+    ec_format_version = get_format_version(txt)
+    if ec_format_version is None:
+        ec_format_version = FORMAT_DEFAULT_VERSION
+    _log.debug("retrieve_blocks_in_spec: derived easyconfig format version: %s" % ec_format_version)
+
+    # blocks in easyconfigs are only supported in format versions prior to 2.0
+    if pieces and ec_format_version < EasyVersion('2.0'):
+        # make a map of blocks
+        blocks = []
+        while pieces:
+            block_name = pieces.pop(0)
+            block_contents = pieces.pop(0)
+
+            if block_name in [b['name'] for b in blocks]:
+                msg = "Found block %s twice in %s." % (block_name, spec)
+                _log.error(msg)
+
+            block = {'name': block_name, 'contents': block_contents}
+
+            # dependency block
+            dep_block = reg_dep_block.search(block_contents)
+            if dep_block:
+                dependencies = eval(dep_block.group(1))
+                if type(dependencies) == list:
+                    block['dependencies'] = dependencies
+                else:
+                    block['dependencies'] = [dependencies]
+
+            blocks.append(block)
+
+        # make a new easyconfig for each block
+        # they will be processed in the same order as they are all described in the original file
+        specs = []
+        for block in blocks:
+            name = block['name']
+            if only_blocks and not (name in only_blocks):
+                print_msg("Skipping block %s-%s" % (spec_fn, name), silent=silent)
+                continue
+
+            (fd, block_path) = tempfile.mkstemp(prefix='easybuild-', suffix='%s-%s' % (spec_fn, name))
+            os.close(fd)
+
+            txt = common
+
+            if 'dependencies' in block:
+                for dep in block['dependencies']:
+                    if not dep in [b['name'] for b in blocks]:
+                        _log.error("Block %s depends on %s, but block was not found." % (name, dep))
+
+                    dep = [b for b in blocks if b['name'] == dep][0]
+                    txt += "\n# Dependency block %s" % (dep['name'])
+                    txt += dep['contents']
+
+            txt += "\n# Main block %s" % name
+            txt += block['contents']
+
+            write_file(block_path, txt)
+
+            specs.append(block_path)
+
+        _log.debug("Found %s block(s) in %s" % (len(specs), spec))
+        return specs
+    else:
+        # no blocks, one file
+        return [spec]

easybuild/framework/easyconfig/parser.py

@@ -30,14 +30,16 @@
 @author: Stijn De Weirdt (Ghent University)
 """
 import os
-
-from vsc import fancylogger
+from vsc.utils import fancylogger
 
 from easybuild.framework.easyconfig.format.format import FORMAT_DEFAULT_VERSION
 from easybuild.framework.easyconfig.format.format import get_format_version, get_format_version_classes
 from easybuild.tools.filetools import read_file, write_file
 
 
+_log = fancylogger.getLogger('easyconfig.parser', fname=False)
+
+
 class EasyConfigParser(object):
     """Read the easyconfig file, return a parsed config object
         Can contain references to multiple version and toolchain/toolchain versions