Add config spec data model consumer

.azure-pipelines/templates/run-validations.yml

@@ -33,6 +33,11 @@ steps:
     ddev validate config
   displayName: 'Validate default configuration files'
 
+- script: |
+    echo "ddev validate models"
+    ddev validate models
+  displayName: 'Validate configuration data models'
+
 - script: |
     echo "ddev validate dashboards"
     ddev validate dashboards

.flake8

@@ -8,3 +8,11 @@ ignore = E203,E722,E741,W503,G200
 exclude = .eggs,.tox,build,compat.py,__init__.py,datadog_checks_dev/datadog_checks/dev/tooling/templates/*,*/datadog_checks/*/vendor/*
 max-line-length = 120
 enable-extensions=G
+per-file-ignores =
+    # potentially long literal strings
+    datadog_checks/*/config_models/deprecations.py: E501
+    # https://pydantic-docs.helpmanual.io/usage/validators/
+    # > validators are "class methods", so the first argument value they
+    #   receive is the UserModel class, not an instance of UserModel
+    datadog_checks/*/config_models/instance.py: B902
+    datadog_checks/*/config_models/shared.py: B902

datadog_checks_base/datadog_checks/base/checks/base.py

@@ -15,10 +15,11 @@
 from typing import TYPE_CHECKING, Any, AnyStr, Callable, Deque, Dict, List, Optional, Sequence, Tuple, Union
 
 import yaml
-from six import binary_type, iteritems, text_type
+from six import PY2, binary_type, iteritems, raise_from, text_type
 
 from ..config import is_affirmative
 from ..constants import ServiceCheck
+from ..errors import ConfigurationError
 from ..types import (
     AgentConfigType,
     Event,
@@ -63,6 +64,9 @@
 
     monkey_patch_pyyaml()
 
+if not PY2:
+    from pydantic import ValidationError
+
 if TYPE_CHECKING:
     import ssl
 
@@ -241,9 +245,16 @@ def __init__(self, *args, **kwargs):
         # Setup metric limits
         self.metric_limiter = self._get_metric_limiter(self.name, instance=self.instance)
 
+        # Lazily load and validate config
+        self.__config = None
+        self.__shared_config = None
+
         # Functions that will be called exactly once (if successful) before the first check run
         self.check_initializations = deque([self.send_config_metadata])  # type: Deque[Callable[[], None]]
 
+        if not PY2:
+            self.check_initializations.append(self.load_configuration_models)
+
     def _get_metric_limiter(self, name, instance=None):
         # type: (str, InstanceType) -> Optional[Limiter]
         limit = self._get_metric_limit(instance=instance)
@@ -364,6 +375,72 @@ def in_developer_mode(self):
         self._log_deprecation('in_developer_mode')
         return False
 
+    @property
+    def config(self):
+        return self.__config
+
+    @property
+    def shared_config(self):
+        return self.__shared_config
+
+    def load_configuration_models(self):
+        # 'datadog_checks.<PACKAGE>.<MODULE>...'
+        module_parts = self.__module__.split('.')
+        package_path = '{}.config_models'.format('.'.join(module_parts[:2]))
+
+        if self.__shared_config is None:
+            self.__shared_config = self.load_configuration_model(
+                package_path, 'SharedConfig', self._get_shared_config()
+            )
+
+        if self.__config is None:
+            self.__config = self.load_configuration_model(package_path, 'InstanceConfig', self._get_instance_config())
+
+    @staticmethod
+    def load_configuration_model(import_path, model_name, config):
+        try:
+            package = importlib.import_module(import_path)
+        except ModuleNotFoundError as e:  # type: ignore
+            # Don't fail if there are no models
+            if str(e) == 'No module named {!r}'.format(import_path):
+                return
+
+            raise
+
+        model = getattr(package, model_name, None)
+        if model is not None:
+            try:
+                config_model = model(**copy.deepcopy(config))
+            except ValidationError as e:
+                errors = e.errors()
+                num_errors = len(errors)
+                message_lines = [
+                    'Detected {} error{} while loading configuration model `{}`:'.format(
+                        num_errors, 's' if num_errors > 1 else '', model_name
+                    )
+                ]
+
+                for error in e.errors():
+                    message_lines.append(
+                        ' -> '.join(
+                            # Start array indexes at one for user-friendliness
+                            str(loc + 1) if isinstance(loc, int) else str(loc) for loc in error['loc']
+                        )
+                    )
+                    message_lines.append('  {}'.format(error['msg']))
+
+                raise_from(ConfigurationError('\n'.join(message_lines)), None)
+            else:
+                return config_model
+
+    def _get_instance_config(self):
+        # Any extra fields will be available during a config model's initial validation stage
+        return self.instance
+
+    def _get_shared_config(self):
+        # Any extra fields will be available during a config model's initial validation stage
+        return self.init_config
+
     def register_secret(self, secret):
         # type: (str) -> None
         """

datadog_checks_base/datadog_checks/base/data/agent_requirements.in

@@ -19,6 +19,7 @@ flup-py3==1.0.3; python_version > "3.0"
 flup==1.0.3.dev-20110405; python_version < "3.0"
 futures==3.3.0; python_version < "3.0"
 gearman==2.0.2; sys_platform != "win32" and python_version < "3.0"
+immutables==0.15; python_version > "3.0"
 in-toto==0.5.0
 ipaddress==1.0.22; python_version < "3.0"
 jaydebeapi==1.2.3
@@ -42,6 +43,7 @@ psutil==5.7.2
 psycopg2-binary==2.8.4
 pyasn1==0.4.6
 pycryptodomex==3.9.4
+pydantic==1.7.3; python_version > "3.0"
 pyhdb==0.3.4
 pyjwt==1.7.1
 pymongo==3.8.0

datadog_checks_base/datadog_checks/base/utils/models/__init__.py

@@ -0,0 +1,3 @@
+# (C) Datadog, Inc. 2021-present
+# All rights reserved
+# Licensed under a 3-clause BSD style license (see LICENSE)

datadog_checks_base/datadog_checks/base/utils/models/fields.py

@@ -0,0 +1,17 @@
+# (C) Datadog, Inc. 2021-present
+# All rights reserved
+# Licensed under a 3-clause BSD style license (see LICENSE)
+from pydantic.fields import SHAPE_MAPPING, SHAPE_SEQUENCE, SHAPE_SINGLETON
+
+
+def get_default_field_value(field, value):
+    if field.shape == SHAPE_MAPPING:
+        return {}
+    elif field.shape == SHAPE_SEQUENCE:
+        return []
+    elif field.shape == SHAPE_SINGLETON:
+        field_type = field.type_
+        if field_type in (float, int, str):
+            return field_type()
+
+    return value

datadog_checks_base/datadog_checks/base/utils/models/types.py

@@ -0,0 +1,15 @@
+# (C) Datadog, Inc. 2021-present
+# All rights reserved
+# Licensed under a 3-clause BSD style license (see LICENSE)
+from immutables import Map
+
+
+def make_immutable_check_config(obj):
+    # Only consider container types that can be de-serialized from YAML/JSON
+    if isinstance(obj, list):
+        return tuple(make_immutable_check_config(item) for item in obj)
+    elif isinstance(obj, dict):
+        # There are no ordering guarantees, see https://github.com/MagicStack/immutables/issues/57
+        return Map((k, make_immutable_check_config(v)) for k, v in obj.items())
+
+    return obj

datadog_checks_base/datadog_checks/base/utils/models/validation/__init__.py

@@ -0,0 +1,4 @@
+# (C) Datadog, Inc. 2021-present
+# All rights reserved
+# Licensed under a 3-clause BSD style license (see LICENSE)
+from . import core

datadog_checks_base/datadog_checks/base/utils/models/validation/core.py

@@ -0,0 +1,17 @@
+# (C) Datadog, Inc. 2021-present
+# All rights reserved
+# Licensed under a 3-clause BSD style license (see LICENSE)
+from ..types import make_immutable_check_config
+
+
+def initialize_config(values):
+    # This is what is returned by the initial root validator of each config model.
+    return values
+
+
+def finalize_config(values):
+    # This is what is returned by the final root validator of each config model. Note:
+    #
+    # 1. the final object must be a dict
+    # 2. we maintain the original order of keys
+    return {field: make_immutable_check_config(value) for field, value in values.items()}

datadog_checks_base/requirements.in

@@ -5,12 +5,14 @@ contextlib2==0.6.0; python_version < '3.0'
 cryptography==3.3.2
 ddtrace==0.32.2
 enum34==1.1.6; python_version < '3.0'
+immutables==0.15; python_version > '3.0'
 ipaddress==1.0.22; python_version < '3.0'
 kubernetes==8.0.1
 mmh3==2.5.1
 orjson==2.6.1; python_version > '3.0'
 prometheus-client==0.9.0
 protobuf==3.7.0
+pydantic==1.7.3; python_version > '3.0'
 pyjwt==1.7.1
 pysocks==1.7.0
 python-dateutil==2.8.0

datadog_checks_base/tests/models/__init__.py

@@ -0,0 +1,3 @@
+# (C) Datadog, Inc. 2021-present
+# All rights reserved
+# Licensed under a 3-clause BSD style license (see LICENSE)

datadog_checks_base/tests/models/test_types.py

@@ -0,0 +1,44 @@
+# (C) Datadog, Inc. 2021-present
+# All rights reserved
+# Licensed under a 3-clause BSD style license (see LICENSE)
+from ..utils import requires_py3
+
+pytestmark = [requires_py3]
+
+
+def test_make_immutable_check_config():
+    # TODO: move imports up top when we drop Python 2
+    from immutables import Map
+
+    from datadog_checks.base.utils.models.types import make_immutable_check_config
+
+    obj = make_immutable_check_config(
+        {
+            'string': 'foo',
+            'integer': 9000,
+            'float': 3.14,
+            'boolean': True,
+            'array': [{'key': 'foo'}, {'key': 'bar'}],
+            'mapping': {'foo': 'bar'},
+        }
+    )
+
+    assert isinstance(obj, Map)
+    assert len(obj) == 6
+    assert isinstance(obj['string'], str)
+    assert obj['string'] == 'foo'
+    assert isinstance(obj['integer'], int)
+    assert obj['integer'] == 9000
+    assert isinstance(obj['float'], float)
+    assert obj['float'] == 3.14
+    assert isinstance(obj['boolean'], bool)
+    assert obj['boolean'] is True
+    assert isinstance(obj['array'], tuple)
+    assert len(obj['array']) == 2
+    assert isinstance(obj['array'][0], Map)
+    assert obj['array'][0]['key'] == 'foo'
+    assert isinstance(obj['array'][1], Map)
+    assert obj['array'][1]['key'] == 'bar'
+    assert isinstance(obj['mapping'], Map)
+    assert len(obj['mapping']) == 1
+    assert obj['mapping']['foo'] == 'bar'

datadog_checks_base/tests/test_agent_check.py

@@ -9,13 +9,16 @@
 
 import mock
 import pytest
+from immutables import Map
 from six import PY3
 
 from datadog_checks.base import AgentCheck
 from datadog_checks.base import __version__ as base_package_version
 from datadog_checks.base import to_native_string
 from datadog_checks.base.checks.base import datadog_agent
 
+from .utils import requires_py3
+
 
 def test_instance():
     """
@@ -840,3 +843,31 @@ def check(self, _):
         check.run()
 
         assert check.initialize.call_count == 2
+
+
+@requires_py3
+def test_load_configuration_models(dd_run_check, mocker):
+    instance = {'endpoint': 'url', 'tags': ['foo:bar'], 'proxy': {'http': 'http://1.2.3.4:9000'}}
+    init_config = {'proxy': {'https': 'https://1.2.3.4:4242'}}
+    check = AgentCheck('test', init_config, [instance])
+    check.check_id = 'test:123'
+    check.check = lambda _: None
+
+    assert check.config is None
+    assert check.shared_config is None
+
+    config_model = {'endpoint': 'url', 'tags': ('foo:bar',), 'proxy': Map(http='http://1.2.3.4:9000')}
+    shared_config_model = {'proxy': Map(https='https://1.2.3.4:4242')}
+    package = mocker.MagicMock()
+    package.InstanceConfig = mocker.MagicMock(return_value=config_model)
+    package.SharedConfig = mocker.MagicMock(return_value=shared_config_model)
+    import_module = mocker.patch('importlib.import_module', return_value=package)
+
+    dd_run_check(check)
+
+    import_module.assert_called_once_with('datadog_checks.base.models')
+    package.InstanceConfig.assert_called_once_with(**config_model)
+    package.SharedConfig.assert_called_once_with(**shared_config_model)
+
+    assert check.config is config_model
+    assert check.shared_config is shared_config_model

datadog_checks_dev/datadog_checks/dev/tooling/commands/validate/__init__.py

@@ -17,6 +17,7 @@
 from .jmx_metrics import jmx_metrics
 from .manifest import manifest
 from .metadata import metadata
+from .models import models
 from .package import package
 from .readmes import readmes
 from .recommended_monitors import recommended_monitors
@@ -37,6 +38,7 @@
     imports,
     manifest,
     metadata,
+    models,
     package,
     readmes,
     recommended_monitors,