sync with latest vsc-base version to fix log order

vsc/README.md

@@ -1,3 +1,3 @@
 Code from https://github.com/hpcugent/vsc-base
 
-based on a15bb01eb06b385144325a8d9be20184b1044259 (vsc-base v1.9.2)
+based on 95c2174a243874227dcc895d3e26c1b3b949ba22 (vsc-base v1.9.5)

vsc/utils/fancylogger.py

@@ -95,9 +95,10 @@
 
 # register new loglevelname
 logging.addLevelName(logging.CRITICAL * 2 + 1, 'APOCALYPTIC')
-# register EXCEPTION and FATAL alias
+# register QUIET, EXCEPTION and FATAL alias
 logging._levelNames['EXCEPTION'] = logging.ERROR
 logging._levelNames['FATAL'] = logging.CRITICAL
+logging._levelNames['QUIET'] = logging.WARNING
 
 
 # mpi rank support
@@ -677,3 +678,23 @@ def disableDefaultHandlers():
 def enableDefaultHandlers():
     """(re)Enable the default handlers on all fancyloggers"""
     _enable_disable_default_handlers(True)
+
+
+def getDetailsLogLevels(fancy=True):
+    """
+    Return list of (name,loglevelname) pairs of existing loggers
+
+    @param fancy: if True, returns only Fancylogger; if False, returns non-FancyLoggers, 
+                  anything else, return all loggers
+    """
+    func_map = {
+        True: getAllFancyloggers,
+        False: getAllNonFancyloggers,
+    }
+    func = func_map.get(fancy, getAllExistingLoggers)
+    res = []
+    for name, logger in func():
+        # PlaceHolder instances have no level attribute set
+        level_name = logging.getLevelName(getattr(logger, 'level', logging.NOTSET))
+        res.append((name, level_name))
+    return res

vsc/utils/generaloption.py

@@ -46,10 +46,11 @@
 from optparse import SUPPRESS_HELP as nohelp  # supported in optparse of python v2.4
 from optparse import _ as _gettext  # this is gettext normally
 from vsc.utils.dateandtime import date_parser, datetime_parser
-from vsc.utils.fancylogger import getLogger, setLogLevel
+from vsc.utils.fancylogger import getLogger, setLogLevel, getDetailsLogLevels
 from vsc.utils.missing import shell_quote, nub
 from vsc.utils.optcomplete import autocomplete, CompleterOption
 
+
 def set_columns(cols=None):
     """Set os.environ COLUMNS variable
         - only if it is not set already
@@ -126,6 +127,11 @@ class ExtOption(CompleterOption):
     TYPES = tuple(['strlist', 'strtuple'] + list(Option.TYPES))
     BOOLEAN_ACTIONS = ('store_true', 'store_false',) + EXTOPTION_LOG
 
+    def __init__(self, *args, **kwargs):
+        """Add logger to init"""
+        CompleterOption.__init__(self, *args, **kwargs)
+        self.log = getLogger(self.__class__.__name__)
+
     def _set_attrs(self, attrs):
         """overwrite _set_attrs to allow store_or callbacks"""
         Option._set_attrs(self, attrs)
@@ -184,9 +190,16 @@ def take_action(self, action, dest, opt, value, values, parser):
                     action = 'store_true'
 
             if orig_action in self.EXTOPTION_LOG and action == 'store_true':
-                setLogLevel(orig_action.split('_')[1][:-3].upper())
+                newloglevel = orig_action.split('_')[1][:-3].upper()
+                logstate = ", ".join(["(%s, %s)" % (n, l) for n, l in getDetailsLogLevels()])
+                self.log.debug("changing loglevel to %s, current state: %s" % (newloglevel, logstate))
+                setLogLevel(newloglevel)
+                self.log.debug("changed loglevel to %s, previous state: %s" % (newloglevel, logstate))
+                if hasattr(values, '_logaction_taken'):
+                    values._logaction_taken[dest] = True
 
             Option.take_action(self, action, dest, opt, value, values, parser)
+
         elif action in self.EXTOPTION_EXTRA_OPTIONS:
             if action == "extend":
                 # comma separated list convert in list
@@ -324,6 +337,7 @@ def __init__(self, *args, **kwargs):
         self.help_to_string = kwargs.pop('help_to_string', None)
         self.help_to_file = kwargs.pop('help_to_file', None)
         self.envvar_prefix = kwargs.pop('envvar_prefix', None)
+        self.process_env_options = kwargs.pop('process_env_options', True)
 
         # py2.4 epilog compatibilty with py2.7 / optparse 1.5.3
         self.epilog = kwargs.pop('epilog', None)
@@ -344,6 +358,9 @@ def __init__(self, *args, **kwargs):
             epilogtxt += ' e.g. option --someopt also supports --disable-someopt.'
             self.epilog.append(epilogtxt % {'disable': self.option_class.DISABLE})
 
+        self.environment_arguments = None
+        self.commandline_arguments = None
+
     def set_description_docstring(self):
         """Try to find the main docstring and add it if description is not None"""
         stack = inspect.stack()[-1]
@@ -400,12 +417,15 @@ def set_usage(self, usage):
     def get_default_values(self):
         """Introduce the ExtValues class with class constant
             - make it dynamic, otherwise the class constant is shared between multiple instances
-            - class constant is used to avoid _taken_action as option in the __dict__
+            - class constant is used to avoid _action_taken as option in the __dict__ 
+                - only works by using reference to object
+                - same for _logaction_taken
         """
         values = OptionParser.get_default_values(self)
 
         class ExtValues(self.VALUES_CLASS):
             _action_taken = {}
+            _logaction_taken = {}
 
         newvalues = ExtValues()
         newvalues.__dict__ = values.__dict__.copy()
@@ -524,9 +544,9 @@ def _add_help_option(self):
 
     def _get_args(self, args):
         """Prepend the options set through the environment"""
-        regular_args = OptionParser._get_args(self, args)
-        env_args = self.get_env_options()
-        return env_args + regular_args  # prepend the environment options as longopts
+        self.commandline_arguments = OptionParser._get_args(self, args)
+        self.get_env_options()
+        return self.environment_arguments + self.commandline_arguments  # prepend the environment options as longopts
 
     def get_env_options_prefix(self):
         """Return the prefix to use for options passed through the environment"""
@@ -537,7 +557,12 @@ def get_env_options_prefix(self):
 
     def get_env_options(self):
         """Retrieve options from the environment: prefix_longopt.upper()"""
-        env_long_opts = []
+        self.environment_arguments = []
+
+        if not self.process_env_options:
+            self.log.debug("Not processing environment for options")
+            return
+
         if self.envvar_prefix is None:
             self.get_env_options_prefix()
 
@@ -556,16 +581,16 @@ def get_env_options(self):
                 val = os.environ.get(env_opt_name, None)
                 if not val is None:
                     if opt.action in opt.TYPED_ACTIONS:  # not all typed actions are mandatory, but let's assume so
-                        env_long_opts.append("%s=%s" % (lo, val))
+                        self.environment_arguments.append("%s=%s" % (lo, val))
                     else:
                         # interpretation of values: 0/no/false means: don't set it
                         if not ("%s" % val).lower() in ("0", "no", "false",):
-                            env_long_opts.append("%s" % lo)
+                            self.environment_arguments.append("%s" % lo)
                 else:
                     self.log.debug("Environment variable %s is not set" % env_opt_name)
 
-        self.log.debug("Environment variable options with prefix %s: %s" % (self.envvar_prefix, env_long_opts))
-        return env_long_opts
+        self.log.debug("Environment variable options with prefix %s: %s" % (self.envvar_prefix, self.environment_arguments))
+        return self.environment_arguments
 
     def get_option_by_long_name(self, name):
         """Return the option matching the long option name"""
@@ -719,7 +744,7 @@ def _make_debug_options(self):
         self._logopts = {
             'debug': ("Enable debug log mode", None, "store_debuglog", False, 'd'),
             'info': ("Enable info log mode", None, "store_infolog", False),
-            'quiet': ("Enable info quiet/warning mode", None, "store_warninglog", False),
+            'quiet': ("Enable quiet/warning log mode", None, "store_warninglog", False),
         }
 
         descr = ['Debug and logging options', '']
@@ -970,6 +995,9 @@ def parseoptions(self, options_list=None):
             else:
                 sys.exit(err.code)
 
+        self.log.debug("parseoptions: options from environment %s" % (self.parser.environment_arguments))
+        self.log.debug("parseoptions: options from commandline %s" % (self.parser.commandline_arguments))
+
         # args should be empty, since everything is optional
         if len(self.args) > 1:
             self.log.debug("Found remaining args %s" % self.args)
@@ -1045,8 +1073,8 @@ def parseconfigfiles(self):
             if section not in cfg_sections_flat:
                 self.log.debug("parseconfigfiles: found section %s, adding to remainder" % section)
                 remainder = self.configfile_remainder.setdefault(section, {})
-                # parse te remaining options, sections starting with 'raw_' as their name will be considered raw sections
-
+                # parse te remaining options, sections starting with 'raw_'
+                # as their name will be considered raw sections
                 for opt, val in self.configfile_parser.items(section, raw=(section.startswith('raw_'))):
                     remainder[opt] = val
 
@@ -1075,7 +1103,17 @@ def parseconfigfiles(self):
 
                     configfile_options_default[opt_dest] = actual_option.default
 
-                    if actual_option.action in ExtOption.BOOLEAN_ACTIONS:
+                    # log actions require special care
+                    # if any log action was already taken before, it would precede the one from the configfile
+                    # however, multiple logactions in a configfile (or environment for that matter) have
+                    # undefined behaviour
+                    is_log_action = actual_option.action in ExtOption.EXTOPTION_LOG
+                    log_action_taken = getattr(self.options, '_logaction_taken', False)
+                    if is_log_action and log_action_taken:
+                        # value set through take_action. do not modify by configfile
+                        self.log.debug(('parseconfigfiles: log action %s (value %s) found,'
+                                        ' but log action already taken. Ignoring.') % (opt_dest, val))
+                    elif actual_option.action in ExtOption.BOOLEAN_ACTIONS:
                         try:
                             newval = self.configfile_parser.getboolean(section, opt)
                             self.log.debug(('parseconfigfiles: getboolean for option %s value %s '
@@ -1102,11 +1140,17 @@ def parseconfigfiles(self):
         # reparse
         self.log.debug('parseconfigfiles: going to parse options through cmdline %s' % configfile_cmdline)
         try:
+            # can't reprocress the environment, since we are not reporcessing the commandline either
+            self.parser.process_env_options = False
             (parsed_configfile_options, parsed_configfile_args) = self.parser.parse_args(configfile_cmdline)
+            self.parser.process_env_options = True
         except:
             self.log.raiseException('parseconfigfiles: failed to parse options through cmdline %s' %
                                     configfile_cmdline)
 
+        # re-report the options as parsed via parser
+        self.log.debug("parseconfigfiles: options from configfile %s" % (self.parser.commandline_arguments))
+
         if len(parsed_configfile_args) > 0:
             self.log.raiseException('parseconfigfiles: not all options were parsed: %s' % parsed_configfile_args)
 

vsc/utils/rest.py

@@ -65,7 +65,7 @@ class Client(object):
 
     USER_AGENT = 'vsc-rest-client'
 
-    def __init__(self, url, username=None, password=None, token=None, token_type='Token', user_agent=None):
+    def __init__(self, url, username=None, password=None, token=None, token_type='Token', user_agent=None, append_slash=False):
         """
         Create a Client object,
         this client can consume a REST api hosted at host/endpoint
@@ -78,6 +78,7 @@ def __init__(self, url, username=None, password=None, token=None, token_type='To
         self.auth_header = None
         self.username = username
         self.url = url
+        self.append_slash = append_slash
 
         if not user_agent:
             self.user_agent = self.USER_AGENT
@@ -103,6 +104,8 @@ def get(self, url, headers={}, **params):
         Do a http get request on the given url with given headers and parameters
         Parameters is a dictionary that will will be urlencoded
         """
+        if self.append_slash:
+            url += '/'
         url += self.urlencode(params)
         return self.request(self.GET, url, None, headers)
 
@@ -111,6 +114,8 @@ def head(self, url, headers={}, **params):
         Do a http head request on the given url with given headers and parameters
         Parameters is a dictionary that will will be urlencoded
         """
+        if self.append_slash:
+            url += '/'
         url += self.urlencode(params)
         return self.request(self.HEAD, url, None, headers)
 
@@ -119,6 +124,8 @@ def delete(self, url, headers={}, **params):
         Do a http delete request on the given url with given headers and parameters
         Parameters is a dictionary that will will be urlencoded
         """
+        if self.append_slash:
+            url += '/'
         url += self.urlencode(params)
         return self.request(self.DELETE, url, None, headers)
 
@@ -127,23 +134,32 @@ def post(self, url, body=None, headers={}, **params):
         Do a http post request on the given url with given body, headers and parameters
         Parameters is a dictionary that will will be urlencoded
         """
+        if self.append_slash:
+            url += '/'
         url += self.urlencode(params)
+        headers['Content-Type'] = 'application/json'
         return self.request(self.POST, url, json.dumps(body), headers)
 
     def put(self, url, body=None, headers={}, **params):
         """
         Do a http put request on the given url with given body, headers and parameters
         Parameters is a dictionary that will will be urlencoded
         """
+        if self.append_slash:
+            url += '/'
         url += self.urlencode(params)
+        headers['Content-Type'] = 'application/json'
         return self.request(self.PUT, url, json.dumps(body), headers)
 
     def patch(self, url, body=None, headers={}, **params):
         """
         Do a http patch request on the given url with given body, headers and parameters
         Parameters is a dictionary that will will be urlencoded
         """
+        if self.append_slash:
+            url += '/'
         url += self.urlencode(params)
+        headers['Content-Type'] = 'application/json'
         return self.request(self.PATCH, url, json.dumps(body), headers)
 
     def request(self, method, url, body, headers):