Remove the 'minimal' build parameter

Plug the resources initialization in Svc::get_resources(), so the service
build() callers don't have to care about minimal mode to optimize the build
time.

This patch also fixes the "print status" after purging the status.json cache
displaying no resources (and generating a new status.json we no resources).

lib/extconfig.py

@@ -558,7 +558,7 @@ def _handle_reference(self, ref, _section, _v, scope=False,
                 if _v != dref:
                     continue
                 try:
-                    res = self.get_resource(_section)
+                    res = self.resources_by_id[_section]
                     devs = getattr(res, dref)()
                     return list(devs)
                 except Exception as exc:

lib/node.py

@@ -547,7 +547,7 @@ def svcs_selector(self, selector):
             return list_services()
         if is_service(selector):
             return [selector]
-        self.build_services(minimal=True)
+        self.build_services()
         try:
             return self._svcs_selector(selector)
         finally:
@@ -739,18 +739,16 @@ def check_build_errors(svcnames, svcs, errors):
             msg += "\n".join(["- "+err for err in errors])
         raise ex.excError(msg)
 
-    def rebuild_services(self, svcnames, minimal):
+    def rebuild_services(self, svcnames):
         """
         Delete the list of Svc objects in the Node object and create a new one.
 
         Args:
           svcnames: add only Svc objects for services specified
-          minimal: include a minimal set of properties in the new Svc objects
         """
         del self.services
         self.services = None
-        self.build_services(svcnames=svcnames, minimal=minimal,
-                            node=self)
+        self.build_services(svcnames=svcnames, node=self)
 
     def close(self):
         """

lib/svc.py

@@ -448,6 +448,7 @@ def __init__(self, svcname=None, node=None, cf=None):
         )
         if cf:
             self.paths.cf = cf
+        self.resources_initialized = False
         self.resources_by_id = {}
         self.encap_resources = {}
         self.resourcesets_by_id = {}
@@ -562,6 +563,7 @@ def sched(self):
                 ),
             },
         )
+        return sched
 
     @lazy
     def ha(self):
@@ -864,9 +866,8 @@ def has_monitored_resources(self):
                 return True
         return False
 
-    def post_build(self):
+    def configure_sched_tasks(self):
         """
-        A method run after the service is done building.
         Add resource-dependent tasks to the scheduler.
         """
         try:
@@ -1256,11 +1257,25 @@ def setup_signal_handlers():
         signal.signal(signal.SIGINT, signal_handler)
         signal.signal(signal.SIGTERM, signal_handler)
 
+    def init_resources(self):
+        if self.resources_initialized:
+            return
+        if self.scale_target is not None:
+            # scalers can't have resources
+            return
+        from svcBuilder import add_resources
+        add_resources(self)
+        self.resources_initialized = True
+        #import traceback
+        #traceback.print_stack()
+        self.log.debug("resources initialized")
+
     def get_resource(self, rid, with_encap=False):
         """
         Return a resource object by id.
         Return None if the rid is not found.
         """
+        self.init_resources()
         if rid in self.resources_by_id:
             return self.resources_by_id[rid]
         if with_encap and rid in self.encap_resources:
@@ -1274,6 +1289,7 @@ def get_resources(self, _type=None, discard_disabled=True):
         <_type> can be:
           None: all resources are returned
         """
+        self.init_resources()
         if _type is None:
             return self.resources_by_id.values()
         if not isinstance(_type, (list, tuple)):
@@ -1540,17 +1556,21 @@ def print_status_data(self, from_resource_status_cache=False, mon_data=False, re
                 pass
 
         if not refresh:
-            data["running"] = self.get_running()
+            data["running"] = self.get_running(data["resources"].keys())
         return data
 
-    def get_running(self):
+    def get_running(self, rids=None):
         lockfile = os.path.join(rcEnv.paths.pathlock, self.svcname)
         running = []
         running += [self._get_running(lockfile).get("rid")]
         lockfile = os.path.join(rcEnv.paths.pathlock, self.svcname+".sync")
         running += [self._get_running(lockfile).get("rid")]
-        for task in self.get_resources("task"):
-            lockfile = os.path.join(task.var_d, "run.lock")
+        if rids is None:
+            rids = [r.rid for r in self.get_resources("task")]
+        else:
+            rids = [rid for rid in rids if rid.startswith("task")]
+        for rid in rids:
+            lockfile = os.path.join(self.var_d, rid, "run.lock")
             running += [self._get_running(lockfile).get("rid")]
         return [rid for rid in running if rid]
 
@@ -1709,8 +1729,6 @@ def write_status_data(self, data):
         return data
 
     def update_status_data(self):
-        if self.options.minimal:
-            return
         self.log.debug("update status dump")
         # print_status_data() with from_resource_status_cache=True does a status.json write
         self.print_status_data(from_resource_status_cache=True)
@@ -1752,10 +1770,10 @@ def print_resource_status(self):
                   "resources", file=sys.stderr)
             return 1
         for rid in self.action_rid:
-            if rid not in self.resources_by_id:
-                print("resource not found")
+            resource = self.get_resource(rid)
+            if resource is None:
+                print("resource %s not found" % rid)
                 continue
-            resource = self.resources_by_id[rid]
             print(rcStatus.colorize_status(str(resource.status(refresh=self.options.refresh))))
         return 0
 
@@ -3472,6 +3490,7 @@ def setup_environ(self, action=None):
             resource.setup_environ()
 
     def all_rids(self):
+        self.init_resources()
         return [rid for rid in self.resources_by_id if rid is not None] + \
                list(self.encap_resources.keys())
 
@@ -3675,7 +3694,6 @@ def options_to_rids(self, options, action):
                 raise ex.excAbortAction("no resource match the given --rid, --subset "
                                         "and --tags specifiers")
         else:
-            # no resources certainly mean the build was done with minimal=True
             # let the action go on. 'delete', for one, takes a --rid but does
             # not need resource initialization
             rids = rid
@@ -3693,6 +3711,8 @@ def action(self, action, options=None):
         except ex.excAbortAction as exc:
             self.log.info(exc)
             return 0
+        if self.options.cron and action in ("resource_monitor", "sync_all", "status") or action == "print_schedule":
+            self.configure_sched_tasks()
         try:
             return self._action(action, options=options)
         except lock.LOCK_EXCEPTIONS as exc:
@@ -3745,7 +3765,6 @@ def _action(self, action, options=None):
                            "action %s", action)
             self.purge_status_caches()
 
-        self.setup_environ(action=action)
         self.setup_signal_handlers()
         self.set_skip_resources(keeprid=self.action_rid, xtags=options.xtags)
         if action == "status" or \
@@ -3759,6 +3778,7 @@ def _action(self, action, options=None):
            options.dry_run:
             err = self.do_action(action, options)
         else:
+            self.setup_environ(action=action)
             err = self.do_logged_action(action, options)
         return err
 
@@ -3951,8 +3971,8 @@ def do_action(self, action, options):
 
         if action == "sync_all" and self.command_is_scoped():
             for rid in self.action_rid:
-                resource = self.resources_by_id[rid]
-                if not resource.type.startswith("sync"):
+                resource = self.get_resource(rid)
+                if not resource or not resource.type.startswith("sync"):
                     continue
                 try:
                     resource.reslock(action=action, suffix="sync")

lib/svcBuilder.py

@@ -162,7 +162,7 @@ def get_sync_args(svc, s):
 
     return kwargs
 
-def add_resources(svc, restype):
+def add_resources_group(svc, restype):
     for s in svc.config.sections():
         try:
             add_resource(svc, restype, s)
@@ -2049,7 +2049,7 @@ class Logger instance name: 'log'
     rcLogger.max_svcname_len = max_svcname_len
     rcLogger.initLogger(rcEnv.nodename)
 
-def build(name, minimal=False, svcconf=None, node=None):
+def build(name, svcconf=None, node=None):
     """build(name) is in charge of Svc creation
     it return None if service Name is not managed by local node
     else it return new Svc instance
@@ -2210,41 +2210,34 @@ def build(name, minimal=False, svcconf=None, node=None):
     except ex.OptNotFound as exc:
         svc.bwlimit = None
 
-    if minimal:
-        svc.options.minimal = True
-        return svc
-    if svc.scale_target is not None:
-        return svc
-
-    svc.options.minimal = False
+    return svc
 
+def add_resources(svc):
     #
     # instanciate resources
     #
-    add_resources(svc, 'container')
-    add_resources(svc, 'ip')
-    add_resources(svc, 'disk')
-    add_resources(svc, 'fs')
-    add_resources(svc, 'share')
-    add_resources(svc, 'app')
-    add_resources(svc, 'task')
+    add_resources_group(svc, 'container')
+    add_resources_group(svc, 'ip')
+    add_resources_group(svc, 'disk')
+    add_resources_group(svc, 'fs')
+    add_resources_group(svc, 'share')
+    add_resources_group(svc, 'app')
+    add_resources_group(svc, 'task')
 
     # deprecated, folded into "disk"
-    add_resources(svc, 'vdisk')
-    add_resources(svc, 'vmdg')
-    add_resources(svc, 'loop')
-    add_resources(svc, 'drbd')
-    add_resources(svc, 'vg')
-    add_resources(svc, 'pool')
-
-    add_resources(svc, 'sync')
+    add_resources_group(svc, 'vdisk')
+    add_resources_group(svc, 'vmdg')
+    add_resources_group(svc, 'loop')
+    add_resources_group(svc, 'drbd')
+    add_resources_group(svc, 'vg')
+    add_resources_group(svc, 'pool')
+
+    add_resources_group(svc, 'sync')
     add_mandatory_syncs(svc)
 
-    svc.post_build()
-    return svc
 
 def build_services(status=None, svcnames=None, create_instance=False,
-                   minimal=False, node=None):
+                   node=None):
     """
     Returns a list of all services of status matching the specified status.
     If no status is specified, returns all services.
@@ -2279,7 +2272,7 @@ def build_services(status=None, svcnames=None, create_instance=False,
 
     for name in svcnames:
         try:
-            svc = build(name, minimal=minimal, node=node)
+            svc = build(name, node=node)
         except (ex.excError, ex.excInitError, ValueError, rcConfigParser.ParsingError) as e:
             errors.append("%s: %s" % (name, str(e)))
             svclog = rcLogger.initLogger(rcEnv.nodename+"."+name, handlers=["file", "syslog"])

lib/svcmgr.py

@@ -41,33 +41,12 @@ def get_docker_argv(argv=None):
     argv = argv[:pos+1]
     return argv, docker_argv
 
-def get_minimal(action, options):
-    """
-    Return True if the services can be built with minimal parsing
-    """
-    if action == "ls" and not options.status:
-        return True
-    if action == "get" and not options.eval:
-        return True
-    if action == "print_status" and not options.refresh:
-        return True
-    if action == "edit_config":
-        return True
-    if action.startswith("print_config"):
-        return True
-    if action.startswith("json_config"):
-        return True
-    if action.startswith("collector_"):
-        return True
-    return False
-
 def get_build_kwargs(optparser, options, action):
     """
     Return the service build function keyword arguments, deduced from
     parsed command line options.
     """
     build_kwargs = {}
-    build_kwargs["minimal"] = get_minimal(action, options)
 
     if len(set(["svcnames", "status"]) & set(build_kwargs.keys())) == 0:
         if os.environ.get("OSVC_SERVICE_LINK"):
@@ -160,7 +139,7 @@ def do_svc_create(node, svcnames, action, options, build_kwargs):
 
     # force a refresh of node.svcs
     try:
-        node.rebuild_services(svcnames, build_kwargs["minimal"])
+        node.rebuild_services(svcnames)
     except ex.excError as exc:
         print(exc, file=sys.stderr)
         ret = 1
@@ -170,7 +149,7 @@ def do_svc_create(node, svcnames, action, options, build_kwargs):
         # setenv changed the service config file
         # we need to rebuild again
         try:
-            node.rebuild_services(svcnames, build_kwargs["minimal"])
+            node.rebuild_services(svcnames)
         except ex.excError as exc:
             print(exc, file=sys.stderr)
             ret = 1
@@ -208,7 +187,7 @@ def export_env_from_options(options):
 
 def _main(node, argv=None):
     """
-    Build the service list, full or minimal depending on the requested action.
+    Build the service list.
     Execute action-specific codepaths.
     """
     build_err = False