Instrumentation revamp

Change-Id: Ic76900e63068a519b9667115af99f83a5de4880f
Signed-off-by: Nicolas Godinho <nicolas.godinho@ssi.gouv.fr>
Signed-off-by: Timothée Ravier <timothee.ravier@ssi.gouv.fr>

clipostoolkit/cosmk/instrumentation.py

@@ -1,14 +1,14 @@
 # SPDX-License-Identifier: LGPL-2.1-or-later
 # Copyright © 2017 ANSSI. All rights reserved.
 
-"""Module handling recipe instrumentation logic."""
+"""Module handling product instrumentation logic."""
 
 import os
 import re
 import sys
 from enum import Enum
 from typing import (Any, Dict, Iterable, MutableMapping, Optional, Text, Tuple,
-                    Union)
+                    Union, List)
 
 import schema
 import toml
@@ -20,68 +20,46 @@
 from .sourcetree import repo_root_path
 
 
-class InstrumentationLevel(Enum):
-    """Instrumentation level enumeration for the recipes"""
+class Instrumentation(object):
+    """Class describing the CLIP OS product (or derivative) instrumentation
+    properties requested by the developer in the "instrumentation.toml" drop-in
+    file at the root of the CLIP OS source tree."""
 
-    PRODUCTION = 0
-    DEVELOPMENT = 1
-    DEBUG = 2
-
-
-def instrumented_recipes() -> Dict[str, InstrumentationLevel]:
-    """Probes for existence of an "instrumentation.toml" drop-in file at the
-    root of the `repo` source tree, parses this file and return a
-    :py:data:`dict` of all the recipe identifiers (as keys) requested to be
-    instrumented with their instrumentation level as value.
-
-    :raise InstrumentationSpecificationError: in case of an incoherence in the
-        ``instrumentation.toml`` file
-
-    """
-
-    instru_filepath = os.path.join(repo_root_path(), "instrumentation.toml")
-
-    if not os.path.exists(instru_filepath):
-        return dict()
-
-    try:
-        with open(instru_filepath, "r") as instru_fp:
-            instru: MutableMapping[str, Any] = toml.load(instru_fp)
-    except:
-        raise InstrumentationSpecificationError(line(
-            """Cannot open or parse as TOML the "instrumentation.toml" file
-            placed at the root of the repo source tree."""))
-
-    instru_file_schema = schema.Schema({
-        schema.Optional(level.name.lower(), default=[]): [
-            schema.Regex(RECIPE_IDENTIFIER_RE.pattern)
-        ] for level in InstrumentationLevel
+    SCHEMA = schema.Schema({
+        "features": [schema.And(str, len)],
     })
 
-    try:
-        instru = instru_file_schema.validate(instru)
-    except schema.SchemaError as exc:
-        raise InstrumentationSpecificationError(line(
-            """The "instrumentation.toml" file has an unexpected data
-            structure. Reason: {!s}""").format(exc))
-
-    for level in InstrumentationLevel:
-        for recipe in instru[level.name.lower()]:
-            recipe_config_path = os.path.join(repo_root_path(), "products",
-                                              recipe, "recipe.toml")
-            if not os.path.exists(recipe_config_path):
-                raise InstrumentationSpecificationError(line(
-                    """The recipe {!r} is not a valid recipe or has no
-                    configuration file in the products
-                    folder.""").format(recipe))
-
-    recipes: Dict[str, InstrumentationLevel] = dict()
-    for level in InstrumentationLevel:
-        for recipe_id in instru[level.name.lower()]:
-            if recipe_id in recipes:
-                raise InstrumentationSpecificationError(line(
-                    """The recipe {!r} is specified more than once in the
-                    "instrumentation.toml" file.""").format(recipe_id))
-            recipes[recipe_id] = level
-
-    return recipes
+    INSTRUMENTATION_FILEPATH = os.path.join(repo_root_path(),
+                                            "instrumentation.toml")
+
+    def __init__(self) -> None:
+        self.__file_present = os.path.exists(self.INSTRUMENTATION_FILEPATH)
+        if not self.__file_present:
+            return
+
+        try:
+            with open(self.INSTRUMENTATION_FILEPATH, "r") as instru_fp:
+                instru_data: MutableMapping[str, Any] = toml.load(instru_fp)
+        except:
+            raise InstrumentationSpecificationError(line(
+                """Cannot open or parse as TOML the "instrumentation.toml" file
+                placed at the root of the repo source tree."""))
+
+        try:
+            self.__instru = self.SCHEMA.validate(instru_data)
+        except schema.SchemaError as exc:
+            raise InstrumentationSpecificationError(line(
+                """The "instrumentation.toml" file has an unexpected data
+                structure. Reason: {!s}""").format(exc))
+
+    @property
+    def defined(self) -> bool:
+        return bool(self.__file_present and self.__instru["features"])
+
+    @property
+    def features(self) -> Optional[List[str]]:
+        if self.defined:
+            features: List[str] = self.__instru['features']
+            return features
+        else:
+            return None

clipostoolkit/cosmk/product.py

@@ -13,7 +13,7 @@
 
 from .commons import PRODUCT_NAME_RE, line
 from .exceptions import ProductPropertiesError
-from .instrumentation import instrumented_recipes
+from .instrumentation import Instrumentation
 from .log import critical, debug, error, info, warn
 from .sourcetree import repo_root_path
 
@@ -57,16 +57,16 @@ def __init__(self, name: str) -> None:
 
     @property
     def version(self) -> str:
-        """Returns the version of this product as defined in its "properties.toml"
-        file."""
+        """Returns the version of this product as defined in its
+        "properties.toml" file."""
         return self.properties["version"]
 
     @property
     def tainted_version(self) -> str:
-        """Returns the version of this product as defined in its "properties.toml"
-        file but "tainted" with a build flag ``instrumentation`` if an
-        "instrumentation.toml" file is present at the root of the repo source
-        tree."""
+        """Returns the version of this product as defined in its
+        "properties.toml" file but "tainted" with a build flag
+        ``instrumentation`` if at least one instrumentation feature is
+        enabled."""
 
         version = semver.parse(self.version)
         def taint_version(tag: str) -> None:
@@ -75,10 +75,8 @@ def taint_version(tag: str) -> None:
             else:
                 version["build"] = tag
 
-        # Yes, even if the instrumentation does not affect any recipe composing
-        # the product, that product version will still be tainted as
-        # "instrumented".
-        if instrumented_recipes():
+        instru = Instrumentation()
+        if instru.features:
             taint_version("instrumented")
 
         return str(semver.VersionInfo(**version))

clipostoolkit/cosmk/recipe.py

@@ -15,7 +15,6 @@
 from .commons import (ENVVAR_FORMAT_RE, RECIPE_IDENTIFIER_RE, RECIPE_NAME_RE,
                       line)
 from .exceptions import RecipeConfigurationError
-from .instrumentation import InstrumentationLevel, instrumented_recipes
 from .log import critical, debug, error, info, warn
 from .product import Product
 from .sourcetree import repo_root_path
@@ -64,12 +63,6 @@ def __init__(self, identifier: str) -> None:
                 "Cannot parse or open \"recipe.toml\" configuration file.")
         self.config = self.validate_metaconfig(config)
 
-        # Recipe instrumentation level with fallback to production if not
-        # defined:
-        self.instrumentation_level: InstrumentationLevel = (
-            instrumented_recipes().get(self.identifier,
-                                       InstrumentationLevel.PRODUCTION))
-
         # initialize features for this recipe:
         self.features: Dict[str, 'features.RecipeFeature'] = {}
         self._featured_attrs: Dict[str, 'features.RecipeFeature'] = {}

clipostoolkit/cosmk/sdk.py

@@ -21,6 +21,7 @@
 from .exceptions import (CosmkError, SdkBootstrapError, SdkError,
                          SdkNotFoundError)
 from .fs import mksquashfs
+from .instrumentation import Instrumentation
 from .log import critical, debug, error, info, warn
 from .privileges import ElevatedPrivileges
 from .sourcetree import repo_root_path
@@ -267,8 +268,6 @@ def bootstrap(self,
                 "CURRENT_PRODUCT_TAINTED_VERSION":
                     self.recipe.product.tainted_version,
                 "CURRENT_RECIPE": self.recipe.name,
-                "CURRENT_RECIPE_INSTRUMENTATION_LEVEL":
-                    self.recipe.instrumentation_level.value,
                 "CURRENT_ACTION": "bootstrap",
                 "CURRENT_SDK_PRODUCT": self.recipe.product.name,
                 "CURRENT_SDK_RECIPE": self.recipe.name,
@@ -372,6 +371,13 @@ def session(self,
             raise SdkNotFoundError("could not found rootfs squashfs image "
                                    "file for this SDK")
 
+        # Serialize the instrumentation feature in a string to be passed as an
+        # environment variable:
+        instrumentation = Instrumentation()
+        serialized_instru_features = ''
+        if instrumentation.features:
+            serialized_instru_features = " ".join(instrumentation.features)
+
         sdk_container = Container(
             name="{sdkproduct}-{sdk}.{action}.{product}-{recipe}".format(
                 sdkproduct=self.recipe.product.name,
@@ -406,8 +412,7 @@ def session(self,
             "CURRENT_PRODUCT_TAINTED_VERSION":
                 action_targeted_recipe.product.tainted_version,
             "CURRENT_RECIPE": action_targeted_recipe.name,
-            "CURRENT_RECIPE_INSTRUMENTATION_LEVEL":
-                action_targeted_recipe.instrumentation_level.value,
+            "CURRENT_INSTRUMENTATION_FEATURES": serialized_instru_features,
             "CURRENT_ACTION": action_name,
             "CURRENT_SDK_PRODUCT": self.recipe.product.name,
             "CURRENT_SDK_RECIPE": self.recipe.name,

instrumentation.toml.example

@@ -3,44 +3,114 @@
 
 # This is an example of an "instrumentation.toml" file intended to be dropped
 # at the root of the CLIP OS source tree (i.e. the "repo" root).
-# When existing at the root of the CLIP OS source tree product repository, this
-# "drop-in" file changes to the behavior of the recipe actions (mainly the
-# "build", "image", "configure" and "bundle" actions) for providing
-# instrumentation to the selected recipes.
 #
-# Note 1: By default, all the recipes of the current product *NOT DECLARED* in
-# this file will be produced without instrumentation (i.e. as it would be in a
-# normal build intended for production use).
+# What is the purpose of this file?
+#   When existing at the root of the CLIP OS source tree product repository,
+#   this "drop-in" file changes the behavior of the product build actions (i.e.
+#   the "build", "image", "configure" and "bundle" actions of the concerned
+#   recipes) for providing the requied instrumentation feature to the built
+#   product.
 #
-# Note 2: If instrumentation is requested for at least one recipe, then the
-# product version will be tainted with an "instrumented" tag as a SemVer build
-# metadata (e.g. "X.Y.Z+instrumented").
+# Note 1:
+#   By default, if no instrumentation feature is enabled in this file, then the
+#   product will be built without instrumentation, as it would be in a normal
+#   build intended for production use.
 #
-# For a reference of all the effects of each instrumentation level per recipe,
-# please refer to the appropriate page of the CLIP OS product documentation:
-# https://docs.clip-os.org/clipos/development.html#effect-of-instrumentation-levels
-# The other products (i.e. CLIP OS derivatives) should have their own
-# documentation with their own instrumentation levels reference.
-
-# Recipes that shall be produced instrumented for use by a developer:
+# Note 2:
+#   If at least one instrumentation feature is specified in this file, then the
+#   product version will be tainted with an "instrumented" tag as a SemVer
+#   build metadata (e.g. "X.Y.Z+instrumented").
 #
-# For instance, a recipe instrumented in "development" mode may embed tools of
-# various nature (such as networking utilities, debuggers, text editors, etc.
-# depending on the needs of that recipe) to provide means of analyzing issues
-# or means to develop new features or enhance that recipe.
-development = [
-    "clipos/core",
-    "clipos/efiboot",
-    "clipos/qemu",
-]
+# For more information:
+#   For an in-depth reference of the effects of all instrumentation features
+#   proposed below, please refer to the appropriate page of the CLIP OS product
+#   documentation:
+#   <https://docs.clip-os.org/clipos/development.html#effect-of-instrumentation-levels>
+#   The other products (i.e. CLIP OS derivatives) may implement or enhance
+#   those product instrumentation features.
 
-# Recipes that shall be produced instrumented for debugging investigations:
-#
-# Usually, this instrumentation level includes the features provided by the
-# "development" instrumentation level but goes further as it may also force the
-# build of some consequent parts of the recipe to use a compilation mode with
-# debugging features (e.g. symbols) for further bug investigations.
-debug = [
+features = [
+    # Install additional software development tools in the Core, such as Bash,
+    # Vim, tmux, tree, strace, grep, ip, less, gdb, tcpdump, etc. (This is a
+    # non-exhautive list, for the complete list please refer to the
+    # "clipos-meta/clipos-core" ebuild):
+    "instrumented-core",
+
+    # Enable local and remote (SSH) root login without password
+    #
+    # Requires: "instrumented-core"
+    "passwordless-root-login",
+
+    # Configure SSH in order to allow a developer to log in as root (account
+    # must be enabled with the appropriate instrumentation feature) via SSH and
+    # without any password thanks to an installed SSH key pair (an SSH key pair
+    # will be generated in the cache directory at first usage of this
+    # instrumentation feature).
+    #
+    # Requires: "passwordless-root-login"
+    "allow-ssh-root-login",
+
+    # Configure the bootloader to provide handy features to developer (i.e.
+    # "Reboot into firmware") and a greater timeout before loading the default
+    # boot entry:
+    "dev-friendly-bootloader",
+
+    # Install additional software development tools in the initramfs, such as
+    # strace, ltrace, less, grep, gdb, etc. (This is a non-exhautive list, for
+    # the complete list please refer to the "clipos-meta/clipos-efiboot"
+    # ebuild):
+    #"instrumented-initramfs",
+
+    # Lower Linux kernel security parameters at runtime.
+    #"soften-kernel-configuration",
+
+    # Do not require a TPM 2.0 for the LUKS disk partition decryption.
+    # Therefore, in case of missing TPM 2.0, LUKS passphrase will be prompted
+    # on the active console (either tty or serial console ttyS0):
+    "initramfs-no-require-tpm",
+
+    # Do not lockout TPM (brute-force attack protection) when interacting with
+    # the TPM 2.0 (check out the TPM documentation for the "noDA" attribute and
+    # for "dictionary attack protections"):
+    "initramfs-no-tpm-lockout",
+
+    # Activate alterations to initramfs/efiboot packages intended to ease their
+    # debugging (e.g. debugging symbols):
+    #"debuggable-initramfs",
+
+    # Enable dracut breakpoints with interactive shell drop-outs:
+    #
+    # Important note:
+    #   This feature requires multiple interactions on the console (or serial
+    #   console for the QEMU target) to make the boot sequence proceed to the
+    #   final pivot_root(2).
+    #
+    # Requires: "instrumented-initramfs"
+    #"breakpointed-initramfs",
+
+    # Spawn a persistent root interactive shell on a serial console or an tty
+    # very early in the boot up sequence (right after the pivot_root(2) done by
+    # the initramfs) in order to ease systemd debugging tasks. See
+    # <https://freedesktop.org/wiki/Software/systemd/Debugging> for more
+    # information.
+    #
+    # Requires: "instrumented-core"
+    #"early-root-shell",
+
+    # Activates features for debugging purposes in the kernel (KALLSYMS, core
+    # dump production, etc.) with logging output sent on the serial port
+    # (ttyS0) early on boot up sequence.
+    #"debuggable-kernel",
+
+    # Sets parameters on the kernel command line to make systemd very verbose
+    # on the defined console output (ttyS0 serial port by default):
+    #"verbose-systemd",
+
+    # Configure systemd in order to install the systemd-coredump to allow
+    # kernel produce core dump files.
+    #
+    # Requires: "debuggable-kernel"
+    #"coredump-handler",
 ]
 
 # vim: set ts=4 sts=4 sw=4 et ft=toml: