Merge pull request #827 from stdweird/typecheck_format2

Typecheck format2

easybuild/framework/easyconfig/format/convert.py

@@ -0,0 +1,97 @@
+# #
+# Copyright 2014-2014 Ghent University
+#
+# This file is part of EasyBuild,
+# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
+# with support of Ghent University (http://ugent.be/hpc),
+# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en),
+# the Hercules foundation (http://www.herculesstichting.be/in_English)
+# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
+#
+# http://github.com/hpcugent/easybuild
+#
+# EasyBuild is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation v2.
+#
+# EasyBuild is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with EasyBuild.  If not, see <http://www.gnu.org/licenses/>.
+# #
+
+"""
+This module implements easyconfig specific formats and their conversions.
+
+@author: Stijn De Weirdt (Ghent University)
+"""
+from easybuild.framework.easyconfig.format.version import VersionOperator, ToolchainVersionOperator
+from easybuild.tools.convert import Convert, DictOfStrings, ListOfStrings
+
+
+class Patch(DictOfStrings):
+    """Handle single patch"""
+    ALLOWED_KEYS = ['level', 'dest']
+    KEYLESS_ENTRIES = ['filename']  # filename as first element (also filename:some_path is supported)
+    # explicit definition of __str__ is required for unknown reason related to the way Wrapper is defined
+    __str__ = DictOfStrings.__str__
+
+    def _from_string(self, txt):
+        """Convert from string
+            # shorthand
+            filename;level:<int>;dest:<string> -> {'filename': filename, 'level': level, 'dest': dest}
+            # full dict notation
+            filename:filename;level:<int>;dest:<string> -> {'filename': filename, 'level': level, 'dest': dest}
+        """
+        res = DictOfStrings._from_string(self, txt)
+        if 'level' in res:
+            res['level'] = int(res['level'])
+        return res
+
+
+class Patches(ListOfStrings):
+    """Handle patches as list of Patch"""
+    # explicit definition of __str__ is required for unknown reason related to the way Wrapper is defined
+    __str__ = ListOfStrings.__str__
+
+    def _from_string(self, txt):
+        """Convert from comma-separated string"""
+        res = ListOfStrings._from_string(self, txt)
+        return [Patch(x) for x in res]
+
+
+class Dependency(Convert):
+    """Handle dependency"""
+    SEPARATOR_DEP = ';'
+    __wraps__ = dict
+
+    def _from_string(self, txt):
+        """Convert from string
+            versop_str;tc_versop_str -> {'versop': versop, 'tc_versop': tc_versop}
+        """
+        res = {}
+
+        items = self._split_string(txt, sep=self.SEPARATOR_DEP)
+        if len(items) < 1 or len(items) > 2:
+            msg = 'Dependency has at least one element (a version operator string), '
+            msg += 'and at most 2 (2nd element the toolchain version operator string). '
+            msg += 'Separator %s.' % self.SEPARATOR_DEP
+            raise ValueError(msg)
+
+        res['versop'] = VersionOperator(items[0])
+
+        if len(items) > 1:
+            res['tc_versop'] = ToolchainVersionOperator(items[1])
+
+        return res
+
+    def __str__(self):
+        """Return string"""
+        tmp = [str(self['versop'])]
+        if 'tc_versop' in self:
+            tmp.append(str(self['tc_versop']))
+
+        return self.SEPARATOR_DEP.join(tmp)

easybuild/framework/easyconfig/format/one.py

@@ -40,7 +40,7 @@
 from easybuild.framework.easyconfig.format.version import EasyVersion
 
 
-_log = fancylogger.getLogger('easyconfig.format.format', fname=False)
+_log = fancylogger.getLogger('easyconfig.format.one', fname=False)
 
 
 class FormatOneZero(EasyConfigFormatConfigObj):

easybuild/framework/easyconfig/format/version.py

@@ -72,6 +72,8 @@ class VersionOperator(object):
     # default operator when operator is undefined (but version is)
     DEFAULT_UNDEFINED_OPERATOR = OPERATOR_MAP['==']
 
+    DICT_SEPARATOR = ':'
+
     def __init__(self, versop_str=None, error_on_parse_failure=False):
         """
         Initialise VersionOperator instance.
@@ -85,6 +87,7 @@ def __init__(self, versop_str=None, error_on_parse_failure=False):
         self.version_str = None
         self.version = None
         self.operator = None
+        self.suffix = None
         self.regex = self.versop_regex()
 
         self.error_on_parse_failure = error_on_parse_failure
@@ -107,7 +110,7 @@ def __bool__(self):
     __nonzero__ = __bool__
 
     def is_valid(self):
-        """Check if this is a valid VersionOperator"""
+        """Check if this is a valid VersionOperator. Suffix can be anything."""
         return not(self.version is None or self.operator is None)
 
     def set(self, versop_str):
@@ -126,6 +129,7 @@ def set(self, versop_str):
     def test(self, test_version):
         """
         Convert argument to an EasyVersion instance if needed, and return self.operator(<argument>, self.version)
+            Versions only, no suffix.
         @param test_version: a version string or EasyVersion instance
         """
         # checks whether this VersionOperator instance is valid using __bool__ function
@@ -153,7 +157,11 @@ def __str__(self):
         else:
             operator = self.operator
         operator_str = self.REVERSE_OPERATOR_MAP[operator]
-        return ''.join(map(str, [operator_str, self.SEPARATOR, self.version]))
+
+        tmp = [operator_str, self.SEPARATOR, self.version]
+        if self.suffix is not None:
+            tmp.extend([self.SEPARATOR, self.suffix])
+        return ''.join(map(str, tmp))
 
     def get_version_str(self):
         """Return string representation of version (ignores operator)."""
@@ -169,7 +177,7 @@ def __eq__(self, versop):
             return False
         elif not isinstance(versop, self.__class__):
             self.log.error("Types don't match in comparison: %s, expected %s" % (type(versop), self.__class__))
-        return self.version == versop.version and self.operator == versop.operator
+        return self.version == versop.version and self.operator == versop.operator and self.suffix == versop.suffix
 
     def __ne__(self, versop):
         """Is self not equal to versop"""
@@ -191,10 +199,19 @@ def versop_regex(self, begin_end=True):
         # - operator_str part is optional
         # - version_str should start/end with any word character except separator
         # - minimal version_str length is 1
-        reg_text = r"(?:(?P<operator_str>%(ops)s)%(sep)s)?(?P<version_str>[^%(sep)s\W](?:\S*[^%(sep)s\W])?)" % {
+        # - optional extensions:
+        #    - suffix: the version suffix
+        reg_text_operator = r"(?:(?P<operator_str>%(ops)s)%(sep)s)?" % {
             'sep': self.SEPARATOR,
             'ops': '|'.join(operators),
         }
+        reg_text_version = r"(?P<version_str>[^%(sep)s\W](?:\S*[^%(sep)s\W])?)" % { 'sep': self.SEPARATOR }
+        reg_text_ext = r"(?:%(sep)s(?:suffix%(extsep)s(?P<suffix>[^%(sep)s]+)))?" % {
+            'sep': self.SEPARATOR,
+            'extsep': self.DICT_SEPARATOR,
+        }
+
+        reg_text = r"%s%s%s" % (reg_text_operator, reg_text_version, reg_text_ext)
         if begin_end:
             reg_text = r"^%s$" % reg_text
         reg = re.compile(reg_text)
@@ -277,6 +294,9 @@ def test_overlap_and_conflict(self, versop_other):
             '> 3' and '== 3' : no overlap
             '>= 3' and '== 3' : overlap, and conflict (boundary 3 is ambigous)
             '> 3' and '>= 3' : overlap, no conflict ('> 3' is more strict then '>= 3')
+            
+            # suffix
+            '> 2 suffix:-x1' > '> 1 suffix:-x2': suffix not equal, conflict
         """
         versop_msg = "this versop %s and versop_other %s" % (self, versop_other)
 
@@ -289,6 +309,8 @@ def test_overlap_and_conflict(self, versop_other):
         boundary_self_in_other = versop_other.test(self.version)
         boundary_other_in_self = self.test(versop_other.version)
 
+        suffix_allowed = self.suffix == versop_other.suffix
+
         same_family = False
         for fam in self.OPERATOR_FAMILIES:
             fam_op = [self.OPERATOR_MAP[x] for x in fam]
@@ -304,15 +326,15 @@ def test_overlap_and_conflict(self, versop_other):
             if same_boundary:
                 if op.xor(self_includes_boundary, other_includes_boundary):
                     self.log.debug("%s, one includes boundary and one is strict => overlap, no conflict" % msg)
-                    return True, False
+                    overlap_conflict = (True, False)
                 else:
                     # conflict
                     self.log.debug("%s, and both include the boundary => overlap and conflict" % msg)
-                    return True, True
+                    overlap_conflict = (True, True)
             else:
                 # conflict
                 self.log.debug("%s, and different boundaries => overlap and conflict" % msg)
-                return True, True
+                overlap_conflict = (True, True)
         else:
             # both boundaries not included in one other version expression
             # => never a conflict, only possible overlap
@@ -328,7 +350,14 @@ def test_overlap_and_conflict(self, versop_other):
                 # overlap if boundary of one is in other
                 overlap = boundary_self_in_other or boundary_other_in_self
             self.log.debug("No conflict between %s; %s overlap %s, no conflict" % (versop_msg, msg, overlap))
-            return overlap, False
+            overlap_conflict = (overlap, False)
+
+        if not suffix_allowed:
+            # always conflict
+            self.log.debug("Suffix for %s are not equal. Force conflict True." % versop_msg)
+            overlap_conflict = (overlap_conflict[0], True)
+
+        return overlap_conflict
 
     def __gt__(self, versop_other):
         """
@@ -345,6 +374,10 @@ def __gt__(self, versop_other):
             '== 4' > '> 3' : equality is more strict than inequality, but this order by boundaries
             '> 3' > '== 2' : there is no overlap, so just order the intervals according their boundaries
             '> 1' > '== 1' > '< 1' : no overlap, same boundaries, order by operator
+            suffix:
+                '> 2' > '> 1': both equal (both None), ordering like above
+                '> 2 suffix:-x1' > '> 1 suffix:-x1': both equal (both -x1), ordering like above
+                '> 2 suffix:-x1' > '> 1 suffix:-x2': not equal, conflict
         """
         overlap, conflict = self.test_overlap_and_conflict(versop_other)
         versop_msg = "this versop %s and versop_other %s" % (self, versop_other)
@@ -374,7 +407,9 @@ def __gt__(self, versop_other):
         return is_gt
 
     def _gt_safe(self, version_gt_op, versop_other):
-        """Conflict free comparsion by version first, and if versions are equal, by operator"""
+        """Conflict free comparsion by version first, and if versions are equal, by operator. 
+            Suffix are not considered.
+        """
         if len(self.ORDERED_OPERATORS) != len(self.OPERATOR_MAP):
             self.log.error('Inconsistency between ORDERED_OPERATORS and OPERATORS (lists are not of same length)')
 
@@ -626,7 +661,7 @@ def parse_sections(self, configobj, toparse=None, parent=None, depth=0):
             parsed = {}
         else:
             # parent specified, so not a top section
-            parsed = Section(parent=parent, depth=depth+1, main=configobj)
+            parsed = Section(parent=parent, depth=depth + 1, main=configobj)
 
         # start with full configobj initially, and then process subsections recursively
         if toparse is None:

easybuild/main.py

@@ -126,18 +126,23 @@ def main(testing_data=(None, None, None)):
         else:
             _log.error("No robot paths specified, and unable to determine easybuild-easyconfigs install path.")
 
+    # do not pass options.robot, it's not a list instance (and it shouldn't be modified)
+    robot_path = None
+    if options.robot:
+        robot_path = list(options.robot)
+
     # determine easybuild-easyconfigs package install path
-    easyconfigs_paths = get_paths_for("easyconfigs", robot_path=options.robot)
+    easyconfigs_paths = get_paths_for("easyconfigs", robot_path=robot_path)
     # keep track of paths for install easyconfigs, so we can obtain find specified easyconfigs
     easyconfigs_pkg_full_paths = easyconfigs_paths[:]
     if not easyconfigs_paths:
         _log.warning("Failed to determine install path for easybuild-easyconfigs package.")
 
     # specified robot paths are preferred over installed easyconfig files
-    if options.robot:
-        easyconfigs_paths = options.robot + easyconfigs_paths
-        options.robot.extend(easyconfigs_paths)
-        _log.info("Extended list of robot paths with paths for installed easyconfigs: %s" % options.robot)
+    if robot_path:
+        robot_path.extend(easyconfigs_paths)
+        easyconfigs_paths = robot_path[:]
+        _log.info("Extended list of robot paths with paths for installed easyconfigs: %s" % robot_path)
 
     # initialise the easybuild configuration
     config.init(options, eb_go.get_options_by_section('config'))
@@ -164,7 +169,7 @@ def main(testing_data=(None, None, None)):
         'regtest_online': options.regtest_online,
         'regtest_output_dir': options.regtest_output_dir,
         'retain_all_deps': retain_all_deps,
-        'robot_path': options.robot,
+        'robot_path': robot_path,
         'sequential': options.sequential,
         'silent': testing,
         'skip': options.skip,