Use cc_* module meta defintion over hardcoded vars (SC-888) (#1385)

Previously, each cc_* module required a hardcoded 'distro' and
'frequency' variable to control the respective distro to run on and
run frequency. If undefined or invalid, we silently changed this to
a default value.

Instead, this commit will validate the MetaSchema definition and raise
an exception if invalid. This means every module MUST contain a
MetaSchema definition.

Additionally, the Modules class was moved out of stages.py into its
own module, along with some helper functions in config/__init__.py.

cloudinit/cloud.py

@@ -6,9 +6,13 @@
 
 import copy
 import os
+from typing import Optional
 
 from cloudinit import log as logging
+from cloudinit.distros import Distro
+from cloudinit.helpers import Paths, Runners
 from cloudinit.reporting import events
+from cloudinit.sources import DataSource
 
 LOG = logging.getLogger(__name__)
 
@@ -25,7 +29,15 @@
 
 
 class Cloud(object):
-    def __init__(self, datasource, paths, cfg, distro, runners, reporter=None):
+    def __init__(
+        self,
+        datasource: DataSource,
+        paths: Paths,
+        cfg: dict,
+        distro: Distro,
+        runners: Runners,
+        reporter: Optional[events.ReportEventStack] = None,
+    ):
         self.datasource = datasource
         self.paths = paths
         self.distro = distro

cloudinit/cmd/main.py

@@ -11,6 +11,9 @@
 #
 # This file is part of cloud-init. See LICENSE file for license information.
 
+# Skip isort on this file because of the patch that comes between imports
+# isort: skip_file
+
 import argparse
 import json
 import os
@@ -19,6 +22,7 @@
 import traceback
 
 from cloudinit import patcher
+from cloudinit.config.modules import Modules
 
 patcher.patch_logging()
 
@@ -105,7 +109,7 @@ def extract_fns(args):
     return fn_cfgs
 
 
-def run_module_section(mods, action_name, section):
+def run_module_section(mods: Modules, action_name, section):
     full_section_name = MOD_SECTION_TPL % (section)
     (which_ran, failures) = mods.run_section(full_section_name)
     total_attempted = len(which_ran) + len(failures)
@@ -484,7 +488,7 @@ def main_init(name, args):
     apply_reporting_cfg(init.cfg)
 
     # Stage 8 - re-read and apply relevant cloud-config to include user-data
-    mods = stages.Modules(init, extract_fns(args), reporter=args.reporter)
+    mods = Modules(init, extract_fns(args), reporter=args.reporter)
     # Stage 9
     try:
         outfmt_orig = outfmt
@@ -587,7 +591,7 @@ def main_modules(action_name, args):
             return [(msg)]
     _maybe_persist_instance_data(init)
     # Stage 3
-    mods = stages.Modules(init, extract_fns(args), reporter=args.reporter)
+    mods = Modules(init, extract_fns(args), reporter=args.reporter)
     # Stage 4
     try:
         LOG.debug("Closing stdin")
@@ -642,7 +646,7 @@ def main_single(name, args):
             return 1
     _maybe_persist_instance_data(init)
     # Stage 3
-    mods = stages.Modules(init, extract_fns(args), reporter=args.reporter)
+    mods = Modules(init, extract_fns(args), reporter=args.reporter)
     mod_args = args.module_args
     if mod_args:
         LOG.debug("Using passed in arguments %s", mod_args)

cloudinit/config/__init__.py

@@ -1,47 +0,0 @@
-# Copyright (C) 2008-2010 Canonical Ltd.
-# Copyright (C) 2012 Hewlett-Packard Development Company, L.P.
-#
-# Author: Chuck Short <chuck.short@canonical.com>
-# Author: Juerg Haefliger <juerg.haefliger@hp.com>
-#
-# This file is part of cloud-init. See LICENSE file for license information.
-
-from cloudinit import log as logging
-from cloudinit.settings import FREQUENCIES, PER_INSTANCE
-
-LOG = logging.getLogger(__name__)
-
-# This prefix is used to make it less
-# of a chance that when importing
-# we will not find something else with the same
-# name in the lookup path...
-MOD_PREFIX = "cc_"
-
-
-def form_module_name(name):
-    canon_name = name.replace("-", "_")
-    if canon_name.lower().endswith(".py"):
-        canon_name = canon_name[0 : (len(canon_name) - 3)]
-    canon_name = canon_name.strip()
-    if not canon_name:
-        return None
-    if not canon_name.startswith(MOD_PREFIX):
-        canon_name = "%s%s" % (MOD_PREFIX, canon_name)
-    return canon_name
-
-
-def fixup_module(mod, def_freq=PER_INSTANCE):
-    if not hasattr(mod, "frequency"):
-        setattr(mod, "frequency", def_freq)
-    else:
-        freq = mod.frequency
-        if freq and freq not in FREQUENCIES:
-            LOG.warning("Module %s has an unknown frequency %s", mod, freq)
-    if not hasattr(mod, "distros"):
-        setattr(mod, "distros", [])
-    if not hasattr(mod, "osfamilies"):
-        setattr(mod, "osfamilies", [])
-    return mod
-
-
-# vi: ts=4 expandtab

cloudinit/config/cc_refresh_rmc_and_interface.py

@@ -4,11 +4,18 @@
 #
 # This file is part of cloud-init. See LICENSE file for license information.
 
-"""
-Refresh IPv6 interface and RMC
-------------------------------
-**Summary:** Ensure Network Manager is not managing IPv6 interface
+"""Refresh IPv6 interface and RMC:
+Ensure Network Manager is not managing IPv6 interface"""
+
+import errno
 
+from cloudinit import log as logging
+from cloudinit import netinfo, subp, util
+from cloudinit.config.schema import MetaSchema
+from cloudinit.distros import ALL_DISTROS
+from cloudinit.settings import PER_ALWAYS
+
+MODULE_DESCRIPTION = """\
 This module is IBM PowerVM Hypervisor specific
 
 Reliable Scalable Cluster Technology (RSCT) is a set of software components
@@ -25,22 +32,20 @@
 - Refreshing RMC
 - Disabling NetworkManager from handling IPv6 interface, as IPv6 interface
   is used for communication between RMC daemon and PowerVM hypervisor.
-
-**Internal name:** ``cc_refresh_rmc_and_interface``
-
-**Module frequency:** always
-
-**Supported distros:** RHEL
-
 """
 
-import errno
-
-from cloudinit import log as logging
-from cloudinit import netinfo, subp, util
-from cloudinit.settings import PER_ALWAYS
-
-frequency = PER_ALWAYS
+meta: MetaSchema = {
+    "id": "cc_refresh_rmc_and_interface",
+    "name": "Refresh IPv6 Interface and RMC",
+    "title": "Ensure Network Manager is not managing IPv6 interface",
+    "description": MODULE_DESCRIPTION,
+    "distros": [ALL_DISTROS],
+    "frequency": PER_ALWAYS,
+    "examples": [],
+}
+
+# This module is undocumented in our schema docs
+__doc__ = ""
 
 LOG = logging.getLogger(__name__)
 # Ensure that /opt/rsct/bin has been added to standard PATH of the

cloudinit/config/cc_reset_rmc.py

@@ -3,13 +3,18 @@
 # Author: Aman Kumar Sinha <amansi26@in.ibm.com>
 #
 # This file is part of cloud-init. See LICENSE file for license information.
+"""Reset RMC: Reset rsct node id"""
 
 
-"""
-Reset RMC
-------------
-**Summary:** reset rsct node id
+import os
+
+from cloudinit import log as logging
+from cloudinit import subp, util
+from cloudinit.config.schema import MetaSchema
+from cloudinit.distros import ALL_DISTROS
+from cloudinit.settings import PER_INSTANCE
 
+MODULE_DESCRIPTION = """\
 Reset RMC module is IBM PowerVM Hypervisor specific
 
 Reliable Scalable Cluster Technology (RSCT) is a set of software components,
@@ -28,21 +33,20 @@
   In order to do so, it restarts RSCT service.
 
 Prerequisite of using this module is to install RSCT packages.
-
-**Internal name:** ``cc_reset_rmc``
-
-**Module frequency:** per instance
-
-**Supported distros:** rhel, sles and ubuntu
-
 """
-import os
-
-from cloudinit import log as logging
-from cloudinit import subp, util
-from cloudinit.settings import PER_INSTANCE
 
-frequency = PER_INSTANCE
+meta: MetaSchema = {
+    "id": "cc_reset_rmc",
+    "name": "Reset RMC",
+    "title": "reset rsct node id",
+    "description": MODULE_DESCRIPTION,
+    "distros": [ALL_DISTROS],
+    "frequency": PER_INSTANCE,
+    "examples": [],
+}
+
+# This module is undocumented in our schema docs
+__doc__ = ""
 
 # RMCCTRL is expected to be in system PATH (/opt/rsct/bin)
 # The symlink for RMCCTRL and RECFGCT are

cloudinit/config/cc_spacewalk.py

@@ -7,7 +7,6 @@
 from cloudinit.config.schema import MetaSchema, get_meta_doc
 from cloudinit.settings import PER_INSTANCE
 
-distros = ["redhat", "fedora"]
 MODULE_DESCRIPTION = """\
 This module installs spacewalk and applies basic configuration. If the
 ``spacewalk`` config key is present spacewalk will be installed. The server to
@@ -23,7 +22,7 @@
     "name": "Spacewalk",
     "title": "Install and configure spacewalk",
     "description": MODULE_DESCRIPTION,
-    "distros": distros,
+    "distros": ["rhel", "fedora"],
     "frequency": PER_INSTANCE,
     "examples": [
         dedent(

cloudinit/config/cc_write_files_deferred.py

@@ -2,31 +2,35 @@
 #
 # This file is part of cloud-init. See LICENSE file for license information.
 
-"""Defer writing certain files"""
+"""Write Files Deferred: Defer writing certain files"""
 
 from cloudinit import util
 from cloudinit.config.cc_write_files import DEFAULT_DEFER, write_files
-
-# meta is not used in this module, but it remains as code documentation
-#
-# id: cc_write_files_deferred'
-# name: 'Write Deferred Files
-# distros: ['all'],
-# frequency: PER_INSTANCE,
-# title:
-#    write certain files, whose creation as been deferred, during
-#    final stage
-# description:
-#    This module is based on `'Write Files' <write-files>`__, and
-#    will handle all files from the write_files list, that have been
-#    marked as deferred and thus are not being processed by the
-#    write-files module.
-#
-#    *Please note that his module is not exposed to the user through
-#    its own dedicated top-level directive.*
-
-# Not exposed, because related modules should document this behaviour
-__doc__ = None
+from cloudinit.config.schema import MetaSchema
+from cloudinit.distros import ALL_DISTROS
+from cloudinit.settings import PER_INSTANCE
+
+MODULE_DESCRIPTION = """\
+This module is based on `'Write Files' <write-files>`__, and
+will handle all files from the write_files list, that have been
+marked as deferred and thus are not being processed by the
+write-files module.
+
+*Please note that his module is not exposed to the user through
+its own dedicated top-level directive.*
+"""
+meta: MetaSchema = {
+    "id": "cc_write_files_deferred",
+    "name": "Write Files Deferred",
+    "title": "Defer writing certain files",
+    "description": __doc__,
+    "distros": [ALL_DISTROS],
+    "frequency": PER_INSTANCE,
+    "examples": [],
+}
+
+# This module is undocumented in our schema docs
+__doc__ = ""
 
 
 def handle(name, cfg, _cloud, log, _args):
@@ -44,6 +48,3 @@ def handle(name, cfg, _cloud, log, _args):
         )
         return
     write_files(name, filtered_files)
-
-
-# vi: ts=4 expandtab