From e7a9043a5a2148f2e0eff6ee2cddb89ff1060f81 Mon Sep 17 00:00:00 2001 From: Ofek Lev Date: Wed, 10 Feb 2021 16:30:49 -0500 Subject: [PATCH] Add config spec data model consumer --- .../templates/run-validations.yml | 5 + .flake8 | 11 + .../datadog_checks/base/checks/base.py | 86 +++- .../base/data/agent_requirements.in | 2 + .../base/utils/models/__init__.py | 3 + .../base/utils/models/fields.py | 17 + .../datadog_checks/base/utils/models/types.py | 16 + .../base/utils/models/validation/__init__.py | 4 + .../base/utils/models/validation/core.py | 17 + .../base/utils/models/validation/helpers.py | 7 + .../base/utils/models/validation/utils.py | 27 ++ datadog_checks_base/requirements.in | 2 + datadog_checks_base/tests/models/__init__.py | 3 + .../tests/models/config_models/__init__.py | 18 + .../tests/models/config_models/defaults.py | 52 +++ .../models/config_models/deprecations.py | 11 + .../tests/models/config_models/instance.py | 64 +++ .../tests/models/config_models/shared.py | 48 +++ .../tests/models/config_models/validators.py | 3 + .../tests/models/data/spec.yaml | 78 ++++ .../tests/models/test_interface.py | 150 +++++++ .../tests/models/test_types.py | 44 ++ datadog_checks_base/tests/test_agent_check.py | 35 ++ datadog_checks_base/tests/test_log.py | 6 +- datadog_checks_base/tests/utils.py | 1 + .../datadog_checks/dev/plugin/tox.py | 2 + .../dev/tooling/commands/validate/__init__.py | 2 + .../dev/tooling/commands/validate/models.py | 174 ++++++++ .../datadog_checks/dev/tooling/create.py | 9 +- .../tooling/specs/configuration/constants.py | 44 ++ .../specs/configuration/consumers/__init__.py | 1 + .../specs/configuration/consumers/model.py | 384 ++++++++++++++++++ .../dev/tooling/specs/configuration/spec.py | 21 +- .../dev/tooling/specs/configuration/utils.py | 35 ++ .../datadog_checks/dev/tooling/specs/utils.py | 4 - .../{check_name}/config_models/__init__.py | 16 + .../{check_name}/config_models/defaults.py | 22 + .../{check_name}/config_models/instance.py | 43 ++ .../{check_name}/config_models/shared.py | 40 ++ .../{check_name}/config_models/validators.py | 1 + .../{check_name}/config_models/__init__.py | 16 + .../{check_name}/config_models/defaults.py | 98 +++++ .../{check_name}/config_models/instance.py | 61 +++ .../{check_name}/config_models/shared.py | 45 ++ .../{check_name}/config_models/validators.py | 1 + .../datadog_checks/dev/tooling/utils.py | 17 + datadog_checks_dev/setup.py | 1 + .../configuration/consumers/model/__init__.py | 3 + .../consumers/model/test_all_required.py | 97 +++++ .../consumers/model/test_array.py | 118 ++++++ .../consumers/model/test_both_models_basic.py | 172 ++++++++ .../consumers/model/test_common_validators.py | 152 +++++++ .../consumers/model/test_defaults.py | 178 ++++++++ .../consumers/model/test_nested_option.py | 148 +++++++ .../consumers/model/test_no_models.py | 24 ++ .../model/test_object_arbitrary_values.py | 117 ++++++ .../consumers/model/test_object_model.py | 133 ++++++ .../model/test_object_typed_values.py | 124 ++++++ .../consumers/model/test_only_shared.py | 110 +++++ .../model/test_option_name_normalization.py | 116 ++++++ .../consumers/model/test_union_types.py | 120 ++++++ .../configuration/consumers/test_example.py | 5 +- .../tests/tooling/configuration/utils.py | 10 +- docs/developer/.scripts/33_render_status.py | 25 ++ docs/developer/.snippets/links.txt | 1 + docs/developer/meta/config-specs.md | 13 + mypy.ini | 2 + 67 files changed, 3386 insertions(+), 29 deletions(-) create mode 100644 datadog_checks_base/datadog_checks/base/utils/models/__init__.py create mode 100644 datadog_checks_base/datadog_checks/base/utils/models/fields.py create mode 100644 datadog_checks_base/datadog_checks/base/utils/models/types.py create mode 100644 datadog_checks_base/datadog_checks/base/utils/models/validation/__init__.py create mode 100644 datadog_checks_base/datadog_checks/base/utils/models/validation/core.py create mode 100644 datadog_checks_base/datadog_checks/base/utils/models/validation/helpers.py create mode 100644 datadog_checks_base/datadog_checks/base/utils/models/validation/utils.py create mode 100644 datadog_checks_base/tests/models/__init__.py create mode 100644 datadog_checks_base/tests/models/config_models/__init__.py create mode 100644 datadog_checks_base/tests/models/config_models/defaults.py create mode 100644 datadog_checks_base/tests/models/config_models/deprecations.py create mode 100644 datadog_checks_base/tests/models/config_models/instance.py create mode 100644 datadog_checks_base/tests/models/config_models/shared.py create mode 100644 datadog_checks_base/tests/models/config_models/validators.py create mode 100644 datadog_checks_base/tests/models/data/spec.yaml create mode 100644 datadog_checks_base/tests/models/test_interface.py create mode 100644 datadog_checks_base/tests/models/test_types.py create mode 100644 datadog_checks_dev/datadog_checks/dev/tooling/commands/validate/models.py create mode 100644 datadog_checks_dev/datadog_checks/dev/tooling/specs/configuration/constants.py create mode 100644 datadog_checks_dev/datadog_checks/dev/tooling/specs/configuration/consumers/model.py create mode 100644 datadog_checks_dev/datadog_checks/dev/tooling/specs/configuration/utils.py create mode 100644 datadog_checks_dev/datadog_checks/dev/tooling/templates/integration/check/{check_name}/datadog_checks/{check_name}/config_models/__init__.py create mode 100644 datadog_checks_dev/datadog_checks/dev/tooling/templates/integration/check/{check_name}/datadog_checks/{check_name}/config_models/defaults.py create mode 100644 datadog_checks_dev/datadog_checks/dev/tooling/templates/integration/check/{check_name}/datadog_checks/{check_name}/config_models/instance.py create mode 100644 datadog_checks_dev/datadog_checks/dev/tooling/templates/integration/check/{check_name}/datadog_checks/{check_name}/config_models/shared.py create mode 100644 datadog_checks_dev/datadog_checks/dev/tooling/templates/integration/check/{check_name}/datadog_checks/{check_name}/config_models/validators.py create mode 100644 datadog_checks_dev/datadog_checks/dev/tooling/templates/integration/jmx/{check_name}/datadog_checks/{check_name}/config_models/__init__.py create mode 100644 datadog_checks_dev/datadog_checks/dev/tooling/templates/integration/jmx/{check_name}/datadog_checks/{check_name}/config_models/defaults.py create mode 100644 datadog_checks_dev/datadog_checks/dev/tooling/templates/integration/jmx/{check_name}/datadog_checks/{check_name}/config_models/instance.py create mode 100644 datadog_checks_dev/datadog_checks/dev/tooling/templates/integration/jmx/{check_name}/datadog_checks/{check_name}/config_models/shared.py create mode 100644 datadog_checks_dev/datadog_checks/dev/tooling/templates/integration/jmx/{check_name}/datadog_checks/{check_name}/config_models/validators.py create mode 100644 datadog_checks_dev/tests/tooling/configuration/consumers/model/__init__.py create mode 100644 datadog_checks_dev/tests/tooling/configuration/consumers/model/test_all_required.py create mode 100644 datadog_checks_dev/tests/tooling/configuration/consumers/model/test_array.py create mode 100644 datadog_checks_dev/tests/tooling/configuration/consumers/model/test_both_models_basic.py create mode 100644 datadog_checks_dev/tests/tooling/configuration/consumers/model/test_common_validators.py create mode 100644 datadog_checks_dev/tests/tooling/configuration/consumers/model/test_defaults.py create mode 100644 datadog_checks_dev/tests/tooling/configuration/consumers/model/test_nested_option.py create mode 100644 datadog_checks_dev/tests/tooling/configuration/consumers/model/test_no_models.py create mode 100644 datadog_checks_dev/tests/tooling/configuration/consumers/model/test_object_arbitrary_values.py create mode 100644 datadog_checks_dev/tests/tooling/configuration/consumers/model/test_object_model.py create mode 100644 datadog_checks_dev/tests/tooling/configuration/consumers/model/test_object_typed_values.py create mode 100644 datadog_checks_dev/tests/tooling/configuration/consumers/model/test_only_shared.py create mode 100644 datadog_checks_dev/tests/tooling/configuration/consumers/model/test_option_name_normalization.py create mode 100644 datadog_checks_dev/tests/tooling/configuration/consumers/model/test_union_types.py diff --git a/.azure-pipelines/templates/run-validations.yml b/.azure-pipelines/templates/run-validations.yml index c5a95bd6c3efc..f7a65cf841b8e 100644 --- a/.azure-pipelines/templates/run-validations.yml +++ b/.azure-pipelines/templates/run-validations.yml @@ -33,6 +33,11 @@ steps: ddev validate config displayName: 'Validate default configuration files' +- script: | + echo "ddev validate models" + ddev validate models + displayName: 'Validate configuration data models' + - script: | echo "ddev validate dashboards" ddev validate dashboards diff --git a/.flake8 b/.flake8 index 796ebd4423cc4..a05ec156817cc 100644 --- a/.flake8 +++ b/.flake8 @@ -8,3 +8,14 @@ ignore = E203,E722,E741,W503,G200 exclude = .eggs,.tox,build,compat.py,__init__.py,datadog_checks_dev/datadog_checks/dev/tooling/templates/*,*/datadog_checks/*/vendor/* max-line-length = 120 enable-extensions=G +per-file-ignores = + # potentially long literal strings + datadog_checks/*/config_models/deprecations.py: E501 + tests/models/config_models/deprecations.py: E501 + # https://pydantic-docs.helpmanual.io/usage/validators/ + # > validators are "class methods", so the first argument value they + # receive is the UserModel class, not an instance of UserModel + datadog_checks/*/config_models/instance.py: B902 + datadog_checks/*/config_models/shared.py: B902 + tests/models/config_models/instance.py: B902 + tests/models/config_models/shared.py: B902 diff --git a/datadog_checks_base/datadog_checks/base/checks/base.py b/datadog_checks_base/datadog_checks/base/checks/base.py index 95dcba34cac01..dd5deb1988f38 100644 --- a/datadog_checks_base/datadog_checks/base/checks/base.py +++ b/datadog_checks_base/datadog_checks/base/checks/base.py @@ -15,10 +15,11 @@ from typing import TYPE_CHECKING, Any, AnyStr, Callable, Deque, Dict, List, Optional, Sequence, Tuple, Union import yaml -from six import binary_type, iteritems, text_type +from six import PY2, binary_type, iteritems, raise_from, text_type from ..config import is_affirmative from ..constants import ServiceCheck +from ..errors import ConfigurationError from ..types import ( AgentConfigType, Event, @@ -63,6 +64,9 @@ monkey_patch_pyyaml() +if not PY2: + from pydantic import ValidationError + if TYPE_CHECKING: import ssl @@ -241,9 +245,16 @@ def __init__(self, *args, **kwargs): # Setup metric limits self.metric_limiter = self._get_metric_limiter(self.name, instance=self.instance) + # Lazily load and validate config + self._config_model_instance = None # type: Any + self._config_model_shared = None # type: Any + # Functions that will be called exactly once (if successful) before the first check run self.check_initializations = deque([self.send_config_metadata]) # type: Deque[Callable[[], None]] + if not PY2: + self.check_initializations.append(self.load_configuration_models) + def _get_metric_limiter(self, name, instance=None): # type: (str, InstanceType) -> Optional[Limiter] limit = self._get_metric_limit(instance=instance) @@ -364,6 +375,79 @@ def in_developer_mode(self): self._log_deprecation('in_developer_mode') return False + def load_configuration_models(self): + # 'datadog_checks.....' + module_parts = self.__module__.split('.') + package_path = '{}.config_models'.format('.'.join(module_parts[:2])) + + if self._config_model_shared is None: + raw_shared_config = self._get_config_model_initialization_data() + raw_shared_config.update(self._get_shared_config()) + + shared_config = self.load_configuration_model(package_path, 'SharedConfig', raw_shared_config) + if shared_config is not None: + self._config_model_shared = shared_config + + if self._config_model_instance is None: + raw_instance_config = self._get_config_model_initialization_data() + raw_instance_config.update(self._get_instance_config()) + + instance_config = self.load_configuration_model(package_path, 'InstanceConfig', raw_instance_config) + if instance_config is not None: + self._config_model_instance = instance_config + + @staticmethod + def load_configuration_model(import_path, model_name, config): + try: + package = importlib.import_module(import_path) + # TODO: remove the type ignore when we drop Python 2 + except ModuleNotFoundError as e: # type: ignore + # Don't fail if there are no models + if str(e).startswith('No module named '): + return + + raise + + model = getattr(package, model_name, None) + if model is not None: + try: + config_model = model(**config) + # TODO: remove the type ignore when we drop Python 2 + except ValidationError as e: # type: ignore + errors = e.errors() + num_errors = len(errors) + message_lines = [ + 'Detected {} error{} while loading configuration model `{}`:'.format( + num_errors, 's' if num_errors > 1 else '', model_name + ) + ] + + for error in errors: + message_lines.append( + ' -> '.join( + # Start array indexes at one for user-friendliness + str(loc + 1) if isinstance(loc, int) else str(loc) + for loc in error['loc'] + ) + ) + message_lines.append(' {}'.format(error['msg'])) + + raise_from(ConfigurationError('\n'.join(message_lines)), None) + else: + return config_model + + def _get_shared_config(self): + # Any extra fields will be available during a config model's initial validation stage + return copy.deepcopy(self.init_config) + + def _get_instance_config(self): + # Any extra fields will be available during a config model's initial validation stage + return copy.deepcopy(self.instance) + + def _get_config_model_initialization_data(self): + # Allow for advanced functionality during the initial root validation stage + return {'__data': {'logger': self.log, 'warning': self.warning}} + def register_secret(self, secret): # type: (str) -> None """ diff --git a/datadog_checks_base/datadog_checks/base/data/agent_requirements.in b/datadog_checks_base/datadog_checks/base/data/agent_requirements.in index c63346a77add2..5416ee4c93a14 100644 --- a/datadog_checks_base/datadog_checks/base/data/agent_requirements.in +++ b/datadog_checks_base/datadog_checks/base/data/agent_requirements.in @@ -20,6 +20,7 @@ flup-py3==1.0.3; python_version > "3.0" flup==1.0.3.dev-20110405; python_version < "3.0" futures==3.3.0; python_version < "3.0" gearman==2.0.2; sys_platform != "win32" and python_version < "3.0" +immutables==0.15; python_version > "3.0" in-toto==0.5.0 ipaddress==1.0.22; python_version < "3.0" jaydebeapi==1.2.3 @@ -43,6 +44,7 @@ psutil==5.7.2 psycopg2-binary==2.8.4 pyasn1==0.4.6 pycryptodomex==3.9.4 +pydantic==1.8; python_version > "3.0" pyhdb==0.3.4 pyjwt==1.7.1; python_version < "3.0" pyjwt==2.0.1; python_version > "3.0" diff --git a/datadog_checks_base/datadog_checks/base/utils/models/__init__.py b/datadog_checks_base/datadog_checks/base/utils/models/__init__.py new file mode 100644 index 0000000000000..9d0b0155542cb --- /dev/null +++ b/datadog_checks_base/datadog_checks/base/utils/models/__init__.py @@ -0,0 +1,3 @@ +# (C) Datadog, Inc. 2021-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) diff --git a/datadog_checks_base/datadog_checks/base/utils/models/fields.py b/datadog_checks_base/datadog_checks/base/utils/models/fields.py new file mode 100644 index 0000000000000..a35384eb34dc9 --- /dev/null +++ b/datadog_checks_base/datadog_checks/base/utils/models/fields.py @@ -0,0 +1,17 @@ +# (C) Datadog, Inc. 2021-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from pydantic.fields import SHAPE_MAPPING, SHAPE_SEQUENCE, SHAPE_SINGLETON + + +def get_default_field_value(field, value): + if field.shape == SHAPE_MAPPING: + return {} + elif field.shape == SHAPE_SEQUENCE: + return [] + elif field.shape == SHAPE_SINGLETON: + field_type = field.type_ + if field_type in (float, int, str): + return field_type() + + return value diff --git a/datadog_checks_base/datadog_checks/base/utils/models/types.py b/datadog_checks_base/datadog_checks/base/utils/models/types.py new file mode 100644 index 0000000000000..2e183d205c4ab --- /dev/null +++ b/datadog_checks_base/datadog_checks/base/utils/models/types.py @@ -0,0 +1,16 @@ +# (C) Datadog, Inc. 2021-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from collections.abc import Mapping, Sequence + +from immutables import Map + + +def make_immutable_check_config(obj): + if isinstance(obj, Sequence) and not isinstance(obj, str): + return tuple(make_immutable_check_config(item) for item in obj) + elif isinstance(obj, Mapping): + # There are no ordering guarantees, see https://github.com/MagicStack/immutables/issues/57 + return Map((k, make_immutable_check_config(v)) for k, v in obj.items()) + + return obj diff --git a/datadog_checks_base/datadog_checks/base/utils/models/validation/__init__.py b/datadog_checks_base/datadog_checks/base/utils/models/validation/__init__.py new file mode 100644 index 0000000000000..1de02c99dd465 --- /dev/null +++ b/datadog_checks_base/datadog_checks/base/utils/models/validation/__init__.py @@ -0,0 +1,4 @@ +# (C) Datadog, Inc. 2021-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from . import core, utils diff --git a/datadog_checks_base/datadog_checks/base/utils/models/validation/core.py b/datadog_checks_base/datadog_checks/base/utils/models/validation/core.py new file mode 100644 index 0000000000000..a5bd35c563f24 --- /dev/null +++ b/datadog_checks_base/datadog_checks/base/utils/models/validation/core.py @@ -0,0 +1,17 @@ +# (C) Datadog, Inc. 2021-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from ..types import make_immutable_check_config + + +def initialize_config(values, **kwargs): + # This is what is returned by the initial root validator of each config model. + return values + + +def finalize_config(values, **kwargs): + # This is what is returned by the final root validator of each config model. Note: + # + # 1. the final object must be a dict + # 2. we maintain the original order of keys + return {field: make_immutable_check_config(value) for field, value in values.items()} diff --git a/datadog_checks_base/datadog_checks/base/utils/models/validation/helpers.py b/datadog_checks_base/datadog_checks/base/utils/models/validation/helpers.py new file mode 100644 index 0000000000000..962eb42d7f848 --- /dev/null +++ b/datadog_checks_base/datadog_checks/base/utils/models/validation/helpers.py @@ -0,0 +1,7 @@ +# (C) Datadog, Inc. 2021-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + + +def get_initialization_data(values): + return values['__data'] diff --git a/datadog_checks_base/datadog_checks/base/utils/models/validation/utils.py b/datadog_checks_base/datadog_checks/base/utils/models/validation/utils.py new file mode 100644 index 0000000000000..e4d72394ebbd3 --- /dev/null +++ b/datadog_checks_base/datadog_checks/base/utils/models/validation/utils.py @@ -0,0 +1,27 @@ +# (C) Datadog, Inc. 2021-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from .helpers import get_initialization_data + + +def handle_deprecations(config_section, deprecations, values): + warning_method = get_initialization_data(values)['warning'] + + for option, data in deprecations.items(): + if option not in values: + continue + + message = f'Option `{option}` in `{config_section}` is deprecated ->\n' + + for key, info in data.items(): + key_part = f'{key}: ' + info_pad = ' ' * len(key_part) + message += key_part + + for i, line in enumerate(info.splitlines()): + if i > 0: + message += info_pad + + message += f'{line}\n' + + warning_method(message) diff --git a/datadog_checks_base/requirements.in b/datadog_checks_base/requirements.in index 27ea2066a84f6..1edccaa24fd2c 100644 --- a/datadog_checks_base/requirements.in +++ b/datadog_checks_base/requirements.in @@ -5,12 +5,14 @@ contextlib2==0.6.0; python_version < '3.0' cryptography==3.3.2 ddtrace==0.32.2 enum34==1.1.6; python_version < '3.0' +immutables==0.15; python_version > '3.0' ipaddress==1.0.22; python_version < '3.0' kubernetes==12.0.1 mmh3==2.5.1 orjson==2.6.1; python_version > '3.0' prometheus-client==0.9.0 protobuf==3.7.0 +pydantic==1.8; python_version > '3.0' pyjwt==1.7.1; python_version < '3.0' pyjwt==2.0.1; python_version > '3.0' pysocks==1.7.0 diff --git a/datadog_checks_base/tests/models/__init__.py b/datadog_checks_base/tests/models/__init__.py new file mode 100644 index 0000000000000..9d0b0155542cb --- /dev/null +++ b/datadog_checks_base/tests/models/__init__.py @@ -0,0 +1,3 @@ +# (C) Datadog, Inc. 2021-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) diff --git a/datadog_checks_base/tests/models/config_models/__init__.py b/datadog_checks_base/tests/models/config_models/__init__.py new file mode 100644 index 0000000000000..ba42dbdc7ffb0 --- /dev/null +++ b/datadog_checks_base/tests/models/config_models/__init__.py @@ -0,0 +1,18 @@ +# (C) Datadog, Inc. 2021-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from .instance import InstanceConfig +from .shared import SharedConfig + + +class ConfigMixin: + _config_model_instance: InstanceConfig + _config_model_shared: SharedConfig + + @property + def config(self) -> InstanceConfig: + return self._config_model_instance + + @property + def shared_config(self) -> SharedConfig: + return self._config_model_shared diff --git a/datadog_checks_base/tests/models/config_models/defaults.py b/datadog_checks_base/tests/models/config_models/defaults.py new file mode 100644 index 0000000000000..f25d83da6c688 --- /dev/null +++ b/datadog_checks_base/tests/models/config_models/defaults.py @@ -0,0 +1,52 @@ +# (C) Datadog, Inc. 2021-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from datadog_checks.base.utils.models.fields import get_default_field_value + + +def shared_deprecated(field, value): + return get_default_field_value(field, value) + + +def shared_timeout(field, value): + return get_default_field_value(field, value) + + +def instance_array(field, value): + return get_default_field_value(field, value) + + +def instance_deprecated(field, value): + return get_default_field_value(field, value) + + +def instance_flag(field, value): + return False + + +def instance_hyphenated_name(field, value): + return get_default_field_value(field, value) + + +def instance_mapping(field, value): + return get_default_field_value(field, value) + + +def instance_obj(field, value): + return get_default_field_value(field, value) + + +def instance_pass_(field, value): + return get_default_field_value(field, value) + + +def instance_pid(field, value): + return get_default_field_value(field, value) + + +def instance_text(field, value): + return get_default_field_value(field, value) + + +def instance_timeout(field, value): + return get_default_field_value(field, value) diff --git a/datadog_checks_base/tests/models/config_models/deprecations.py b/datadog_checks_base/tests/models/config_models/deprecations.py new file mode 100644 index 0000000000000..31120f1ad4c6d --- /dev/null +++ b/datadog_checks_base/tests/models/config_models/deprecations.py @@ -0,0 +1,11 @@ +# (C) Datadog, Inc. 2021-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + + +def shared(): + return {'deprecated': {'Release': '8.0.0', 'Migration': 'do this\nand that\n'}} + + +def instance(): + return {'deprecated': {'Release': '9.0.0', 'Migration': 'do this\nand that\n'}} diff --git a/datadog_checks_base/tests/models/config_models/instance.py b/datadog_checks_base/tests/models/config_models/instance.py new file mode 100644 index 0000000000000..fc78716747352 --- /dev/null +++ b/datadog_checks_base/tests/models/config_models/instance.py @@ -0,0 +1,64 @@ +# (C) Datadog, Inc. 2021-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from __future__ import annotations + +from typing import Any, Mapping, Optional, Sequence + +from pydantic import BaseModel, Field, root_validator, validator + +from datadog_checks.base.utils.functions import identity +from datadog_checks.base.utils.models import validation + +from . import defaults, deprecations, validators + + +class Obj(BaseModel): + class Config: + allow_mutation = False + + bar: Optional[Sequence[str]] + foo: bool + + +class InstanceConfig(BaseModel): + class Config: + allow_mutation = False + + array: Optional[Sequence[str]] + deprecated: Optional[str] + flag: Optional[bool] + hyphenated_name: Optional[str] = Field(None, alias='hyphenated-name') + mapping: Optional[Mapping[str, Any]] + obj: Optional[Obj] + pass_: Optional[str] = Field(None, alias='pass') + pid: Optional[int] + text: Optional[str] + timeout: Optional[float] + + @root_validator(pre=True) + def _handle_deprecations(cls, values): + validation.utils.handle_deprecations('instances', deprecations.instance(), values) + return values + + @root_validator(pre=True) + def _initial_validation(cls, values): + return validation.core.initialize_config(getattr(validators, 'initialize_instance', identity)(values)) + + @validator('*', pre=True, always=True) + def _ensure_defaults(cls, v, field): + if v is not None or field.required: + return v + + return getattr(defaults, f'instance_{field.name}')(field, v) + + @validator('*') + def _run_validations(cls, v, field): + if not v: + return v + + return getattr(validators, f'instance_{field.name}', identity)(v, field=field) + + @root_validator(pre=False) + def _final_validation(cls, values): + return validation.core.finalize_config(getattr(validators, 'finalize_instance', identity)(values)) diff --git a/datadog_checks_base/tests/models/config_models/shared.py b/datadog_checks_base/tests/models/config_models/shared.py new file mode 100644 index 0000000000000..a15d00d9c16d2 --- /dev/null +++ b/datadog_checks_base/tests/models/config_models/shared.py @@ -0,0 +1,48 @@ +# (C) Datadog, Inc. 2021-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from __future__ import annotations + +from typing import Optional + +from pydantic import BaseModel, root_validator, validator + +from datadog_checks.base.utils.functions import identity +from datadog_checks.base.utils.models import validation + +from . import defaults, deprecations, validators + + +class SharedConfig(BaseModel): + class Config: + allow_mutation = False + + deprecated: Optional[str] + timeout: Optional[float] + + @root_validator(pre=True) + def _handle_deprecations(cls, values): + validation.utils.handle_deprecations('init_config', deprecations.shared(), values) + return values + + @root_validator(pre=True) + def _initial_validation(cls, values): + return validation.core.initialize_config(getattr(validators, 'initialize_shared', identity)(values)) + + @validator('*', pre=True, always=True) + def _ensure_defaults(cls, v, field): + if v is not None or field.required: + return v + + return getattr(defaults, f'shared_{field.name}')(field, v) + + @validator('*') + def _run_validations(cls, v, field): + if not v: + return v + + return getattr(validators, f'shared_{field.name}', identity)(v, field=field) + + @root_validator(pre=False) + def _final_validation(cls, values): + return validation.core.finalize_config(getattr(validators, 'finalize_shared', identity)(values)) diff --git a/datadog_checks_base/tests/models/config_models/validators.py b/datadog_checks_base/tests/models/config_models/validators.py new file mode 100644 index 0000000000000..9d0b0155542cb --- /dev/null +++ b/datadog_checks_base/tests/models/config_models/validators.py @@ -0,0 +1,3 @@ +# (C) Datadog, Inc. 2021-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) diff --git a/datadog_checks_base/tests/models/data/spec.yaml b/datadog_checks_base/tests/models/data/spec.yaml new file mode 100644 index 0000000000000..b871c06944eaf --- /dev/null +++ b/datadog_checks_base/tests/models/data/spec.yaml @@ -0,0 +1,78 @@ +name: test +files: +- name: test.yaml + options: + - template: init_config + options: + - name: timeout + description: words + value: + type: number + - name: deprecated + description: words + deprecation: + Release: 8.0.0 + Migration: | + do this + and that + value: + type: string + - template: instances + options: + - name: text + description: words + value: + type: string + - name: flag + description: words + value: + type: boolean + example: false + - name: timeout + description: words + value: + type: number + - name: pid + description: words + value: + type: integer + - name: array + description: words + value: + type: array + items: + type: string + - name: mapping + description: words + value: + type: object + - name: obj + description: words + value: + type: object + required: + - foo + properties: + - name: foo + type: boolean + - name: bar + type: array + items: + type: string + - name: deprecated + description: words + deprecation: + Release: 9.0.0 + Migration: | + do this + and that + value: + type: string + - name: pass + description: words + value: + type: string + - name: hyphenated-name + description: words + value: + type: string diff --git a/datadog_checks_base/tests/models/test_interface.py b/datadog_checks_base/tests/models/test_interface.py new file mode 100644 index 0000000000000..4db94f6cc19c8 --- /dev/null +++ b/datadog_checks_base/tests/models/test_interface.py @@ -0,0 +1,150 @@ +# (C) Datadog, Inc. 2021-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import pytest +from six import PY3 + +from datadog_checks.base import AgentCheck + +from ..utils import requires_py3 + +if PY3: + from .config_models import ConfigMixin +else: + ConfigMixin = object + +pytestmark = [requires_py3] + + +class Check(AgentCheck, ConfigMixin): + def check(self, _): + pass + + +def test_defaults(dd_run_check): + # TODO: move imports up top when we drop Python 2 + from immutables import Map + + init_config = {} + instance = {} + + check = Check('test', init_config, [instance]) + check.check_id = 'test:123' + + dd_run_check(check) + + assert check.shared_config.deprecated == '' + + assert check.config.text == '' + assert check.config.flag is False + assert check.config.timeout == 0 and isinstance(check.config.timeout, float) + assert check.config.pid == 0 and isinstance(check.config.pid, int) + assert check.config.array == () + assert check.config.mapping == Map() + assert check.config.obj is None + + assert not check.warnings + + +def test_errors_shared_config(dd_run_check): + init_config = {'timeout': 'foo'} + instance = {} + + check = Check('test', init_config, [instance]) + check.check_id = 'test:123' + + with pytest.raises( + Exception, + match="""Detected 1 error while loading configuration model `SharedConfig`: +timeout + value is not a valid float""", + ): + dd_run_check(check, extract_message=True) + + +def test_errors_instance_config(dd_run_check): + init_config = {} + instance = {'timeout': 'foo', 'array': [[]], 'obj': {'bar': []}} + + check = Check('test', init_config, [instance]) + check.check_id = 'test:123' + + with pytest.raises( + Exception, + match="""Detected 3 errors while loading configuration model `InstanceConfig`: +array -> 1 + str type expected +obj -> foo + field required +timeout + value is not a valid float""", + ): + dd_run_check(check, extract_message=True) + + +@pytest.mark.parametrize( + 'value, result', + [ + (0, False), + (1, True), + (2, 'error'), + ('foo', 'error'), + ('yes', True), + ('no', False), + ('true', True), + ('false', False), + ], +) +def test_boolean_allowance(dd_run_check, value, result): + init_config = {} + instance = {'flag': value} + + check = Check('test', init_config, [instance]) + check.check_id = 'test:123' + + if result == 'error': + with pytest.raises(Exception): + dd_run_check(check) + else: + dd_run_check(check) + + assert check.config.flag is result + + +@pytest.mark.parametrize( + 'name, normalized_name', + [pytest.param('pass', 'pass_', id='keyword'), pytest.param('hyphenated-name', 'hyphenated_name', id='hyphenated')], +) +def test_name_edge_cases(dd_run_check, name, normalized_name): + init_config = {} + instance = {name: 'foo'} + + check = Check('test', init_config, [instance]) + check.check_id = 'test:123' + + dd_run_check(check) + + assert getattr(check.config, normalized_name) == 'foo' + + +def test_deprecations(dd_run_check): + init_config = {'deprecated': 'foo'} + instance = {'deprecated': 'foo'} + + check = Check('test', init_config, [instance]) + check.check_id = 'test:123' + + dd_run_check(check) + + assert check.warnings == [ + """Option `deprecated` in `init_config` is deprecated -> +Release: 8.0.0 +Migration: do this + and that +""", + """Option `deprecated` in `instances` is deprecated -> +Release: 9.0.0 +Migration: do this + and that +""", + ] diff --git a/datadog_checks_base/tests/models/test_types.py b/datadog_checks_base/tests/models/test_types.py new file mode 100644 index 0000000000000..1c07ecbf8d96c --- /dev/null +++ b/datadog_checks_base/tests/models/test_types.py @@ -0,0 +1,44 @@ +# (C) Datadog, Inc. 2021-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from ..utils import requires_py3 + +pytestmark = [requires_py3] + + +def test_make_immutable_check_config(): + # TODO: move imports up top when we drop Python 2 + from immutables import Map + + from datadog_checks.base.utils.models.types import make_immutable_check_config + + obj = make_immutable_check_config( + { + 'string': 'foo', + 'integer': 9000, + 'float': 3.14, + 'boolean': True, + 'array': [{'key': 'foo'}, {'key': 'bar'}], + 'mapping': {'foo': 'bar'}, + } + ) + + assert isinstance(obj, Map) + assert len(obj) == 6 + assert isinstance(obj['string'], str) + assert obj['string'] == 'foo' + assert isinstance(obj['integer'], int) + assert obj['integer'] == 9000 + assert isinstance(obj['float'], float) + assert obj['float'] == 3.14 + assert isinstance(obj['boolean'], bool) + assert obj['boolean'] is True + assert isinstance(obj['array'], tuple) + assert len(obj['array']) == 2 + assert isinstance(obj['array'][0], Map) + assert obj['array'][0]['key'] == 'foo' + assert isinstance(obj['array'][1], Map) + assert obj['array'][1]['key'] == 'bar' + assert isinstance(obj['mapping'], Map) + assert len(obj['mapping']) == 1 + assert obj['mapping']['foo'] == 'bar' diff --git a/datadog_checks_base/tests/test_agent_check.py b/datadog_checks_base/tests/test_agent_check.py index 964761afe6fa4..6e7a8a5f480b8 100644 --- a/datadog_checks_base/tests/test_agent_check.py +++ b/datadog_checks_base/tests/test_agent_check.py @@ -16,6 +16,8 @@ from datadog_checks.base import to_native_string from datadog_checks.base.checks.base import datadog_agent +from .utils import requires_py3 + def test_instance(): """ @@ -840,3 +842,36 @@ def check(self, _): check.run() assert check.initialize.call_count == 2 + + +@requires_py3 +def test_load_configuration_models(dd_run_check, mocker): + instance = {'endpoint': 'url', 'tags': ['foo:bar'], 'proxy': {'http': 'http://1.2.3.4:9000'}} + init_config = {'proxy': {'https': 'https://1.2.3.4:4242'}} + check = AgentCheck('test', init_config, [instance]) + check.check_id = 'test:123' + check.check = lambda _: None + + assert check._config_model_instance is None + assert check._config_model_shared is None + + instance_config = object() + shared_config = object() + package = mocker.MagicMock() + package.InstanceConfig = mocker.MagicMock(return_value=instance_config) + package.SharedConfig = mocker.MagicMock(return_value=shared_config) + import_module = mocker.patch('importlib.import_module', return_value=package) + + dd_run_check(check) + + instance_data = check._get_config_model_initialization_data() + instance_data.update(instance) + init_config_data = check._get_config_model_initialization_data() + init_config_data.update(init_config) + + import_module.assert_called_with('datadog_checks.base.config_models') + package.InstanceConfig.assert_called_once_with(**instance_data) + package.SharedConfig.assert_called_once_with(**init_config_data) + + assert check._config_model_instance is instance_config + assert check._config_model_shared is shared_config diff --git a/datadog_checks_base/tests/test_log.py b/datadog_checks_base/tests/test_log.py index 3ea59eaa71f94..eea9765fa3cdf 100644 --- a/datadog_checks_base/tests/test_log.py +++ b/datadog_checks_base/tests/test_log.py @@ -50,15 +50,15 @@ def do_something(self): class MyCheck(AgentCheck): def __init__(self, *args, **kwargs): super(MyCheck, self).__init__(*args, **kwargs) - self.config = FooConfig() + self._config = FooConfig() def check(self, _): - self.config.do_something() + self._config.do_something() check = MyCheck() check.check({}) - assert check.log is check.config.log + assert check.log is check._config.log assert "This is a warning" in caplog.text diff --git a/datadog_checks_base/tests/utils.py b/datadog_checks_base/tests/utils.py index bd2f43b11e6d7..a8990ac2e8a60 100644 --- a/datadog_checks_base/tests/utils.py +++ b/datadog_checks_base/tests/utils.py @@ -4,4 +4,5 @@ from six import PY2 requires_windows = pytest.mark.skipif(platform.system() != 'Windows', reason='Test only valid on Windows') +requires_py2 = pytest.mark.skipif(not PY2, reason='Test only available on Python 2') requires_py3 = pytest.mark.skipif(PY2, reason='Test only available on Python 3') diff --git a/datadog_checks_dev/datadog_checks/dev/plugin/tox.py b/datadog_checks_dev/datadog_checks/dev/plugin/tox.py index 123718406ae7e..6fddb3b3dcf5e 100644 --- a/datadog_checks_dev/datadog_checks/dev/plugin/tox.py +++ b/datadog_checks_dev/datadog_checks/dev/plugin/tox.py @@ -24,6 +24,7 @@ FLAKE8_BUGBEAR_DEP = 'flake8-bugbear==20.1.4' FLAKE8_LOGGING_FORMAT_DEP = 'flake8-logging-format==0.6.0' MYPY_DEP = 'mypy==0.770' +PYDANTIC_DEP = 'pydantic==1.8' # Keep in sync with: /datadog_checks_base/requirements.in @tox.hookimpl @@ -116,6 +117,7 @@ def add_style_checker(config, sections, make_envconfig, reader): FLAKE8_LOGGING_FORMAT_DEP, BLACK_DEP, ISORT_DEP, + PYDANTIC_DEP, ] commands = [ diff --git a/datadog_checks_dev/datadog_checks/dev/tooling/commands/validate/__init__.py b/datadog_checks_dev/datadog_checks/dev/tooling/commands/validate/__init__.py index 4e702113a1bda..8f7d1fd2c4145 100644 --- a/datadog_checks_dev/datadog_checks/dev/tooling/commands/validate/__init__.py +++ b/datadog_checks_dev/datadog_checks/dev/tooling/commands/validate/__init__.py @@ -17,6 +17,7 @@ from .jmx_metrics import jmx_metrics from .manifest import manifest from .metadata import metadata +from .models import models from .package import package from .readmes import readmes from .recommended_monitors import recommended_monitors @@ -37,6 +38,7 @@ imports, manifest, metadata, + models, package, readmes, recommended_monitors, diff --git a/datadog_checks_dev/datadog_checks/dev/tooling/commands/validate/models.py b/datadog_checks_dev/datadog_checks/dev/tooling/commands/validate/models.py new file mode 100644 index 0000000000000..6f620dbfbf7af --- /dev/null +++ b/datadog_checks_dev/datadog_checks/dev/tooling/commands/validate/models.py @@ -0,0 +1,174 @@ +# (C) Datadog, Inc. 2021-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import click + +from ....utils import ( + chdir, + dir_exists, + ensure_parent_dir_exists, + file_exists, + path_join, + read_file, + read_file_lines, + write_file_lines, +) +from ...constants import get_root +from ...specs.configuration import ConfigSpec +from ...specs.configuration.consumers import ModelConsumer +from ...utils import ( + complete_valid_checks, + get_config_spec, + get_license_header, + get_models_location, + get_valid_checks, + get_version_string, +) +from ..console import CONTEXT_SETTINGS, abort, echo_failure, echo_info, echo_success, echo_waiting + + +@click.command(context_settings=CONTEXT_SETTINGS, short_help='Validate configuration data models') +@click.argument('check', autocompletion=complete_valid_checks, required=False) +@click.option('--sync', '-s', is_flag=True, help='Generate data models based on specifications') +@click.option('--verbose', '-v', is_flag=True, help='Verbose mode') +@click.pass_context +def models(ctx, check, sync, verbose): + """Validate configuration data models.""" + root = get_root() + community_check = ctx.obj['repo_choice'] not in ('core', 'internal') + if check: + checks = [check] + else: + checks = {'datadog_checks_base'} + checks.update(get_valid_checks()) + checks = sorted(checks) + + specs_failed = {} + files_failed = {} + num_files = 0 + + license_header_lines = get_license_header().splitlines(True) + license_header_lines.append('\n') + + code_formatter = ModelConsumer.create_code_formatter() + + echo_waiting('Validating data models...') + for check in checks: + check_display_queue = [] + + if check == 'datadog_checks_base': + spec_path = path_join(root, 'datadog_checks_base', 'tests', 'models', 'data', 'spec.yaml') + source = 'test' + version = '0.0.1' + else: + spec_path = get_config_spec(check) + if not file_exists(spec_path): + continue + + source = check + version = get_version_string(check) + + spec_file = read_file(spec_path) + spec = ConfigSpec(spec_file, source=source, version=version) + spec.load() + + if spec.errors: + specs_failed[spec_path] = True + echo_info(f'{check}:') + for error in spec.errors: + echo_failure(error, indent=True) + continue + + if check == 'datadog_checks_base': + models_location = path_join(root, 'datadog_checks_base', 'tests', 'models', 'config_models') + else: + models_location = get_models_location(check) + + # TODO: Remove when all integrations have models + if not sync and not dir_exists(models_location): + continue + + model_consumer = ModelConsumer(spec.data, code_formatter) + + # So formatters see config files + with chdir(root): + model_definitions = model_consumer.render() + + model_files = model_definitions.get(f'{source}.yaml') + if not model_files: + continue + + for model_file, (contents, errors) in model_files.items(): + num_files += 1 + + model_file_path = path_join(models_location, model_file) + if errors: + files_failed[model_file_path] = True + for error in errors: + check_display_queue.append(lambda error=error, **kwargs: echo_failure(error, **kwargs)) + continue + + model_file_lines = contents.splitlines(True) + current_model_file_lines = [] + expected_model_file_lines = [] + if file_exists(model_file_path): + # No contents indicates a custom file + if not contents: + continue + + current_model_file_lines.extend(read_file_lines(model_file_path)) + + for line in current_model_file_lines: + if not line.startswith('#'): + break + + expected_model_file_lines.append(line) + + expected_model_file_lines.extend(model_file_lines) + else: + if not community_check: + expected_model_file_lines.extend(license_header_lines) + + expected_model_file_lines.extend(model_file_lines) + + if current_model_file_lines != expected_model_file_lines: + if sync: + echo_info(f'Writing data model file to `{model_file_path}`') + ensure_parent_dir_exists(model_file_path) + write_file_lines(model_file_path, expected_model_file_lines) + else: + files_failed[model_file_path] = True + check_display_queue.append( + lambda model_file=model_file, **kwargs: echo_failure( + f'File `{model_file}` is not in sync, run "ddev validate models -s"', **kwargs + ) + ) + + if check_display_queue or verbose: + echo_info(f'{check}:') + if verbose: + check_display_queue.append(lambda **kwargs: echo_info('Valid spec', **kwargs)) + for display in check_display_queue: + display(indent=True) + + specs_failed = len(specs_failed) + files_failed = len(files_failed) + files_passed = num_files - files_failed + + if specs_failed or files_failed: + click.echo() + + if specs_failed: + echo_failure(f'Specs with errors: {specs_failed}') + + if files_failed: + echo_failure(f'Files with errors: {files_failed}') + + if files_passed: + if specs_failed or files_failed: + echo_success(f'Files valid: {files_passed}') + else: + echo_success(f'All {num_files} data model files are in sync!') + + if specs_failed or files_failed: + abort() diff --git a/datadog_checks_dev/datadog_checks/dev/tooling/create.py b/datadog_checks_dev/datadog_checks/dev/tooling/create.py index 6ec623df2f37b..0e25994b340d5 100644 --- a/datadog_checks_dev/datadog_checks/dev/tooling/create.py +++ b/datadog_checks_dev/datadog_checks/dev/tooling/create.py @@ -2,7 +2,6 @@ # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) import os -from datetime import datetime from uuid import uuid4 from ..utils import ( @@ -15,7 +14,7 @@ write_file, write_file_binary, ) -from .utils import kebab_case_name, normalize_package_name +from .utils import get_license_header, kebab_case_name, normalize_package_name TEMPLATES_DIR = path_join(os.path.dirname(os.path.abspath(__file__)), 'templates', 'integration') BINARY_EXTENSIONS = ('.png',) @@ -58,11 +57,7 @@ def construct_template_fields(integration_name, repo_choice, **kwargs): 'The {integration_name} check is included in the [Datadog Agent][2] package.\n' 'No additional installation is needed on your server.'.format(integration_name=integration_name) ) - license_header = ( - '# (C) Datadog, Inc. {year}-present\n' - '# All rights reserved\n' - '# Licensed under a 3-clause BSD style license (see LICENSE)'.format(year=str(datetime.utcnow().year)) - ) + license_header = get_license_header() support_type = 'core' test_dev_dep = '-e ../datadog_checks_dev' tox_base_dep = '-e../datadog_checks_base[deps]' diff --git a/datadog_checks_dev/datadog_checks/dev/tooling/specs/configuration/constants.py b/datadog_checks_dev/datadog_checks/dev/tooling/specs/configuration/constants.py new file mode 100644 index 0000000000000..5b8442b5d50d4 --- /dev/null +++ b/datadog_checks_dev/datadog_checks/dev/tooling/specs/configuration/constants.py @@ -0,0 +1,44 @@ +# (C) Datadog, Inc. 2021-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +# https://swagger.io/docs/specification/data-models/data-types/ +OPENAPI_DATA_TYPES = { + 'array', + 'boolean', + 'integer', + 'number', + 'object', + 'string', +} + +# https://spec.openapis.org/oas/v3.0.3#properties +OPENAPI_SCHEMA_PROPERTIES = { + 'additionalProperties', + 'allOf', + 'anyOf', + 'default', + 'description', + 'enum', + 'exclusiveMaximum', + 'exclusiveMinimum', + 'format', + 'items', + 'maxItems', + 'maxLength', + 'maxProperties', + 'maximum', + 'minItems', + 'minLength', + 'minProperties', + 'minimum', + 'multipleOf', + 'not', + 'oneOf', + 'pattern', + 'properties', + 'required', + 'title', + 'type', + 'uniqueItems', +} diff --git a/datadog_checks_dev/datadog_checks/dev/tooling/specs/configuration/consumers/__init__.py b/datadog_checks_dev/datadog_checks/dev/tooling/specs/configuration/consumers/__init__.py index 0d5e18c66958b..32fbfe76e5750 100644 --- a/datadog_checks_dev/datadog_checks/dev/tooling/specs/configuration/consumers/__init__.py +++ b/datadog_checks_dev/datadog_checks/dev/tooling/specs/configuration/consumers/__init__.py @@ -2,3 +2,4 @@ # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) from .example import ExampleConsumer +from .model import ModelConsumer diff --git a/datadog_checks_dev/datadog_checks/dev/tooling/specs/configuration/consumers/model.py b/datadog_checks_dev/datadog_checks/dev/tooling/specs/configuration/consumers/model.py new file mode 100644 index 0000000000000..cbd2cac303075 --- /dev/null +++ b/datadog_checks_dev/datadog_checks/dev/tooling/specs/configuration/consumers/model.py @@ -0,0 +1,384 @@ +# (C) Datadog, Inc. 2021-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from collections import defaultdict +from keyword import iskeyword + +import yaml +from datamodel_code_generator.format import CodeFormatter, PythonVersion +from datamodel_code_generator.parser import LiteralType +from datamodel_code_generator.parser.openapi import OpenAPIParser + +from ..constants import OPENAPI_SCHEMA_PROPERTIES +from ..utils import sanitize_openapi_object_properties + +PYTHON_VERSION = PythonVersion.PY_38 + +# We don't need any self-documenting features +ALLOWED_TYPE_FIELDS = OPENAPI_SCHEMA_PROPERTIES - {'default', 'description', 'example', 'title'} + +# Singleton allowing `None` to be a valid default value +NO_DEFAULT = object() + + +def normalize_option_name(option_name): + # https://github.com/koxudaxi/datamodel-code-generator/blob/0.8.3/datamodel_code_generator/model/base.py#L82-L84 + if iskeyword(option_name): + option_name += '_' + + return option_name.replace('-', '_') + + +def example_looks_informative(example): + return '<' in example and '>' in example and example == example.upper() + + +def get_default_value(type_data): + if 'default' in type_data: + return type_data['default'] + elif 'type' not in type_data or type_data['type'] in ('array', 'object'): + return NO_DEFAULT + + example = type_data['example'] + if type_data['type'] == 'string': + if example_looks_informative(example): + return NO_DEFAULT + elif isinstance(example, str): + return NO_DEFAULT + + return example + + +def add_imports(model_file_lines, need_defaults, need_deprecations): + import_lines = [] + + for i, line in enumerate(model_file_lines): + if line.startswith('from '): + import_lines.append(i) + + # pydantic imports + final_import_line = import_lines[-1] + model_file_lines[final_import_line] += ', root_validator, validator' + + local_imports = ['validators'] + if need_defaults: + local_imports.append('defaults') + if need_deprecations: + local_imports.append('deprecations') + + local_imports_part = ', '.join(sorted(local_imports)) + + local_import_start_location = final_import_line + 1 + for line in reversed( + ( + '', + 'from datadog_checks.base.utils.functions import identity', + 'from datadog_checks.base.utils.models import validation', + '', + f'from . import {local_imports_part}', + ) + ): + model_file_lines.insert(local_import_start_location, line) + + +class ModelConsumer: + def __init__(self, spec, code_formatter=None): + self.spec = spec + self.code_formatter = code_formatter or self.create_code_formatter() + + def render(self): + files = {} + + for file in self.spec['files']: + # (, (, )) + model_files = {} + + model_data = [] + defaults_file_lines = [] + deprecation_data = defaultdict(dict) + defaults_file_needs_dynamic_values = False + defaults_file_needs_value_normalization = False + + for section in sorted(file['options'], key=lambda s: s['name']): + errors = [] + + section_name = section['name'] + if section_name == 'init_config': + model_id = 'shared' + model_file_name = f'{model_id}.py' + schema_name = 'SharedConfig' + elif section_name == 'instances': + model_id = 'instance' + model_file_name = f'{model_id}.py' + schema_name = 'InstanceConfig' + # Skip anything checks don't use directly + else: + continue + + model_data.append((model_id, schema_name)) + + # We want to create something like: + # + # paths: + # /instance: + # get: + # responses: + # '200': + # content: + # application/json: + # schema: + # $ref: '#/components/schemas/InstanceConfig' + # components: + # schemas: + # InstanceConfig: + # required: + # - endpoint + # properties: + # endpoint: + # ... + # timeout: + # ... + # ... + openapi_document = { + 'paths': { + f'/{model_id}': { + 'get': { + 'responses': { + '200': { + 'content': { + 'application/json': { + 'schema': {'$ref': f'#/components/schemas/{schema_name}'} + } + } + } + } + } + } + }, + 'components': {'schemas': {}}, + } + schema = openapi_document['components']['schemas'][schema_name] = {} + + options = schema['properties'] = {} + required_options = schema['required'] = [] + options_with_defaults = False + validator_data = [] + + for option in sorted(section['options'], key=lambda o: o['name']): + option_name = option['name'] + normalized_option_name = normalize_option_name(option_name) + + if 'value' in option: + type_data = option['value'] + # Some integrations (like `mysql`) have options that are grouped under a top-level option + elif 'options' in option: + nested_properties = [] + type_data = {'type': 'object', 'properties': nested_properties} + for nested_option in option['options']: + nested_type_data = nested_option['value'] + + # Remove fields that aren't part of the OpenAPI specification + for extra_field in set(nested_type_data) - ALLOWED_TYPE_FIELDS: + nested_type_data.pop(extra_field, None) + + nested_properties.append({'name': nested_option['name'], **nested_type_data}) + else: + errors.append(f'Option `{option_name}` must have a `value` or `options` attribute') + continue + + if option['deprecation']: + deprecation_data[model_id][option_name] = option['deprecation'] + + options[option_name] = type_data + + if option['required']: + required_options.append(option_name) + else: + options_with_defaults = True + defaults_file_lines.append('') + defaults_file_lines.append('') + defaults_file_lines.append(f'def {model_id}_{normalized_option_name}(field, value):') + + default_value = get_default_value(type_data) + if default_value is not NO_DEFAULT: + defaults_file_needs_value_normalization = True + defaults_file_lines.append(f' return {default_value!r}') + else: + defaults_file_needs_dynamic_values = True + defaults_file_lines.append(' return get_default_field_value(field, value)') + + validators = type_data.pop('validators', []) + if not isinstance(validators, list): + errors.append(f'Config spec property `{option_name}.value.validators` must be an array') + elif validators: + for i, import_path in enumerate(validators, 1): + if not isinstance(import_path, str): + errors.append( + f'Entry #{i} of config spec property `{option_name}.value.validators` ' + f'must be a string' + ) + break + else: + validator_data.append((normalized_option_name, validators)) + + # Remove fields that aren't part of the OpenAPI specification + for extra_field in set(type_data) - ALLOWED_TYPE_FIELDS: + type_data.pop(extra_field, None) + + sanitize_openapi_object_properties(type_data) + + try: + parser = OpenAPIParser( + yaml.safe_dump(openapi_document), + target_python_version=PythonVersion.PY_38, + enum_field_as_literal=LiteralType.All, + encoding='utf-8', + use_generic_container_types=True, + enable_faux_immutability=True, + # TODO: uncomment when the Agent upgrades Python to 3.9 + # use_standard_collections=True, + strip_default_none=True, + # https://github.com/koxudaxi/datamodel-code-generator/pull/173 + field_constraints=True, + ) + model_file_contents = parser.parse() + except Exception as e: + errors.append(f'Error parsing the OpenAPI schema `{schema_name}`: {e}') + model_files[model_file_name] = ('', errors) + continue + + model_file_lines = model_file_contents.splitlines() + add_imports(model_file_lines, options_with_defaults, len(deprecation_data)) + + if model_id in deprecation_data: + model_file_lines.append('') + model_file_lines.append(' @root_validator(pre=True)') + model_file_lines.append(' def _handle_deprecations(cls, values):') + model_file_lines.append( + f' validation.utils.handle_deprecations(' + f'{section_name!r}, deprecations.{model_id}(), values)' + ) + model_file_lines.append(' return values') + + model_file_lines.append('') + model_file_lines.append(' @root_validator(pre=True)') + model_file_lines.append(' def _initial_validation(cls, values):') + model_file_lines.append( + f" return validation.core.initialize_config(" + f"getattr(validators, 'initialize_{model_id}', identity)(values))" + ) + + model_file_lines.append('') + model_file_lines.append(" @validator('*', pre=True, always=True)") + model_file_lines.append(' def _ensure_defaults(cls, v, field):') + model_file_lines.append(' if v is not None or field.required:') + model_file_lines.append(' return v') + model_file_lines.append('') + model_file_lines.append(f" return getattr(defaults, f'{model_id}_{{field.name}}')(field, v)") + + model_file_lines.append('') + model_file_lines.append(" @validator('*')") + model_file_lines.append(' def _run_validations(cls, v, field):') + # TODO: remove conditional when there is a workaround: + # https://github.com/samuelcolvin/pydantic/issues/2376 + model_file_lines.append(' if not v:') + model_file_lines.append(' return v') + model_file_lines.append('') + model_file_lines.append( + f" return getattr(validators, f'{model_id}_{{field.name}}', identity)(v, field=field)" + ) + + for option_name, import_paths in validator_data: + for import_path in import_paths: + validator_name = import_path.replace('.', '_') + + model_file_lines.append('') + model_file_lines.append(f' @validator({option_name!r})') + model_file_lines.append(f' def _run_{option_name}_{validator_name}(cls, v, field):') + # TODO: remove conditional when there is a workaround: + # https://github.com/samuelcolvin/pydantic/issues/2376 + model_file_lines.append(' if not v:') + model_file_lines.append(' return v') + model_file_lines.append('') + model_file_lines.append(f' return validation.{import_path}(v, field=field)') + + model_file_lines.append('') + model_file_lines.append(' @root_validator(pre=False)') + model_file_lines.append(' def _final_validation(cls, values):') + model_file_lines.append( + f" return validation.core.finalize_config(" + f"getattr(validators, 'finalize_{model_id}', identity)(values))" + ) + + model_file_lines.append('') + model_file_contents = '\n'.join(model_file_lines) + if any(len(line) > 120 for line in model_file_lines): + model_file_contents = self.code_formatter.apply_black(model_file_contents) + + model_files[model_file_name] = (model_file_contents, errors) + + # Logs-only integrations + if not model_files: + continue + + if defaults_file_lines: + if defaults_file_needs_dynamic_values: + defaults_file_lines.insert( + 0, 'from datadog_checks.base.utils.models.fields import get_default_field_value' + ) + + defaults_file_lines.append('') + defaults_file_contents = '\n'.join(defaults_file_lines) + if defaults_file_needs_value_normalization: + defaults_file_contents = self.code_formatter.apply_black(defaults_file_contents) + + model_files['defaults.py'] = (defaults_file_contents, []) + + if deprecation_data: + file_needs_formatting = False + deprecations_file_lines = [] + for model_id, deprecations in deprecation_data.items(): + deprecations_file_lines.append('') + deprecations_file_lines.append('') + deprecations_file_lines.append(f'def {model_id}():') + deprecations_file_lines.append(f' return {deprecations!r}') + if len(deprecations_file_lines[-1]) > 120: + file_needs_formatting = True + + deprecations_file_lines.append('') + deprecations_file_contents = '\n'.join(deprecations_file_lines) + if file_needs_formatting: + deprecations_file_contents = self.code_formatter.apply_black(deprecations_file_contents) + + model_files['deprecations.py'] = (deprecations_file_contents, []) + + model_data.sort() + package_root_lines = [] + for model_id, schema_name in model_data: + package_root_lines.append(f'from .{model_id} import {schema_name}') + + package_root_lines.append('') + package_root_lines.append('') + package_root_lines.append('class ConfigMixin:') + for model_id, schema_name in model_data: + package_root_lines.append(f' _config_model_{model_id}: {schema_name}') + for model_id, schema_name in model_data: + property_name = 'config' if model_id == 'instance' else f'{model_id}_config' + package_root_lines.append('') + package_root_lines.append(' @property') + package_root_lines.append(f' def {property_name}(self) -> {schema_name}:') + package_root_lines.append(f' return self._config_model_{model_id}') + + package_root_lines.append('') + model_files['__init__.py'] = ('\n'.join(package_root_lines), []) + + # Custom + model_files['validators.py'] = ('', []) + + files[file['name']] = {file_name: model_files[file_name] for file_name in sorted(model_files)} + + return files + + @staticmethod + def create_code_formatter(): + return CodeFormatter(PYTHON_VERSION) diff --git a/datadog_checks_dev/datadog_checks/dev/tooling/specs/configuration/spec.py b/datadog_checks_dev/datadog_checks/dev/tooling/specs/configuration/spec.py index a0aeed193c35c..ba1cea9819d5e 100644 --- a/datadog_checks_dev/datadog_checks/dev/tooling/specs/configuration/spec.py +++ b/datadog_checks_dev/datadog_checks/dev/tooling/specs/configuration/spec.py @@ -1,7 +1,9 @@ # (C) Datadog, Inc. 2019-present # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) -from ..utils import default_option_example, normalize_source_name +from ..utils import normalize_source_name +from .constants import OPENAPI_DATA_TYPES +from .utils import default_option_example def spec_validator(spec, loader): @@ -355,16 +357,6 @@ def options_validator(options, loader, file_name, *sections): ) -VALID_TYPES = { - 'string', - 'integer', - 'number', - 'boolean', - 'array', - 'object', -} - - def value_validator(value, loader, file_name, sections_display, option_name, depth=0): if 'anyOf' in value: if 'type' in value: @@ -667,6 +659,11 @@ def value_validator(value, loader, file_name, sections_display, option_name, dep else: loader.errors.append( '{}, {}, {}{}: Unknown type `{}`, valid types are {}'.format( - loader.source, file_name, sections_display, option_name, value_type, ' | '.join(sorted(VALID_TYPES)) + loader.source, + file_name, + sections_display, + option_name, + value_type, + ' | '.join(sorted(OPENAPI_DATA_TYPES)), ) ) diff --git a/datadog_checks_dev/datadog_checks/dev/tooling/specs/configuration/utils.py b/datadog_checks_dev/datadog_checks/dev/tooling/specs/configuration/utils.py new file mode 100644 index 0000000000000..6acb8799fd6a3 --- /dev/null +++ b/datadog_checks_dev/datadog_checks/dev/tooling/specs/configuration/utils.py @@ -0,0 +1,35 @@ +# (C) Datadog, Inc. 2021-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + + +def default_option_example(option_name): + return f'<{option_name.upper()}>' + + +def sanitize_openapi_object_properties(value): + if 'anyOf' in value: + for data in value['anyOf']: + sanitize_openapi_object_properties(data) + + return + + value_type = value['type'] + if value_type == 'array': + sanitize_openapi_object_properties(value['items']) + elif value_type == 'object': + spec_properties = value.pop('properties') + properties = value['properties'] = {} + + # The config spec `properties` object modifier is not a map, but rather a list of maps with a + # required `name` attribute. This is so consumers will load objects consistently regardless of + # language guarantees regarding map key order. + for spec_prop in spec_properties: + name = spec_prop.pop('name') + properties[name] = spec_prop + sanitize_openapi_object_properties(spec_prop) + + if 'additionalProperties' in value: + additional_properties = value['additionalProperties'] + if isinstance(additional_properties, dict): + sanitize_openapi_object_properties(additional_properties) diff --git a/datadog_checks_dev/datadog_checks/dev/tooling/specs/utils.py b/datadog_checks_dev/datadog_checks/dev/tooling/specs/utils.py index ab51d8d231d2e..1603757692166 100644 --- a/datadog_checks_dev/datadog_checks/dev/tooling/specs/utils.py +++ b/datadog_checks_dev/datadog_checks/dev/tooling/specs/utils.py @@ -3,9 +3,5 @@ # Licensed under a 3-clause BSD style license (see LICENSE) -def default_option_example(option_name): - return f'<{option_name.upper()}>' - - def normalize_source_name(source_name): return source_name.lower().replace(' ', '_') diff --git a/datadog_checks_dev/datadog_checks/dev/tooling/templates/integration/check/{check_name}/datadog_checks/{check_name}/config_models/__init__.py b/datadog_checks_dev/datadog_checks/dev/tooling/templates/integration/check/{check_name}/datadog_checks/{check_name}/config_models/__init__.py new file mode 100644 index 0000000000000..d92fb862e84b0 --- /dev/null +++ b/datadog_checks_dev/datadog_checks/dev/tooling/templates/integration/check/{check_name}/datadog_checks/{check_name}/config_models/__init__.py @@ -0,0 +1,16 @@ +{license_header} +from .instance import InstanceConfig +from .shared import SharedConfig + + +class ConfigMixin: + _config_model_instance: InstanceConfig + _config_model_shared: SharedConfig + + @property + def config(self) -> InstanceConfig: + return self._config_model_instance + + @property + def shared_config(self) -> SharedConfig: + return self._config_model_shared diff --git a/datadog_checks_dev/datadog_checks/dev/tooling/templates/integration/check/{check_name}/datadog_checks/{check_name}/config_models/defaults.py b/datadog_checks_dev/datadog_checks/dev/tooling/templates/integration/check/{check_name}/datadog_checks/{check_name}/config_models/defaults.py new file mode 100644 index 0000000000000..4d209aa7f0d63 --- /dev/null +++ b/datadog_checks_dev/datadog_checks/dev/tooling/templates/integration/check/{check_name}/datadog_checks/{check_name}/config_models/defaults.py @@ -0,0 +1,22 @@ +{license_header} +from datadog_checks.base.utils.models.fields import get_default_field_value + + +def shared_service(field, value): + return get_default_field_value(field, value) + + +def instance_empty_default_hostname(field, value): + return False + + +def instance_min_collection_interval(field, value): + return 15 + + +def instance_service(field, value): + return get_default_field_value(field, value) + + +def instance_tags(field, value): + return get_default_field_value(field, value) diff --git a/datadog_checks_dev/datadog_checks/dev/tooling/templates/integration/check/{check_name}/datadog_checks/{check_name}/config_models/instance.py b/datadog_checks_dev/datadog_checks/dev/tooling/templates/integration/check/{check_name}/datadog_checks/{check_name}/config_models/instance.py new file mode 100644 index 0000000000000..53d6d742b3a4b --- /dev/null +++ b/datadog_checks_dev/datadog_checks/dev/tooling/templates/integration/check/{check_name}/datadog_checks/{check_name}/config_models/instance.py @@ -0,0 +1,43 @@ +{license_header} +from __future__ import annotations + +from typing import Optional, Sequence + +from pydantic import BaseModel, root_validator, validator + +from datadog_checks.base.utils.functions import identity +from datadog_checks.base.utils.models import validation + +from . import defaults, validators + + +class InstanceConfig(BaseModel): + class Config: + allow_mutation = False + + empty_default_hostname: Optional[bool] + min_collection_interval: Optional[float] + service: Optional[str] + tags: Optional[Sequence[str]] + + @root_validator(pre=True) + def _initial_validation(cls, values): + return validation.core.initialize_config(getattr(validators, 'initialize_instance', identity)(values)) + + @validator('*', pre=True, always=True) + def _ensure_defaults(cls, v, field): + if v is not None or field.required: + return v + + return getattr(defaults, f'instance_{{field.name}}')(field, v) + + @validator('*') + def _run_validations(cls, v, field): + if not v: + return v + + return getattr(validators, f'instance_{{field.name}}', identity)(v, field=field) + + @root_validator(pre=False) + def _final_validation(cls, values): + return validation.core.finalize_config(getattr(validators, 'finalize_instance', identity)(values)) diff --git a/datadog_checks_dev/datadog_checks/dev/tooling/templates/integration/check/{check_name}/datadog_checks/{check_name}/config_models/shared.py b/datadog_checks_dev/datadog_checks/dev/tooling/templates/integration/check/{check_name}/datadog_checks/{check_name}/config_models/shared.py new file mode 100644 index 0000000000000..82dd52936ff40 --- /dev/null +++ b/datadog_checks_dev/datadog_checks/dev/tooling/templates/integration/check/{check_name}/datadog_checks/{check_name}/config_models/shared.py @@ -0,0 +1,40 @@ +{license_header} +from __future__ import annotations + +from typing import Optional + +from pydantic import BaseModel, root_validator, validator + +from datadog_checks.base.utils.functions import identity +from datadog_checks.base.utils.models import validation + +from . import defaults, validators + + +class SharedConfig(BaseModel): + class Config: + allow_mutation = False + + service: Optional[str] + + @root_validator(pre=True) + def _initial_validation(cls, values): + return validation.core.initialize_config(getattr(validators, 'initialize_shared', identity)(values)) + + @validator('*', pre=True, always=True) + def _ensure_defaults(cls, v, field): + if v is not None or field.required: + return v + + return getattr(defaults, f'shared_{{field.name}}')(field, v) + + @validator('*') + def _run_validations(cls, v, field): + if not v: + return v + + return getattr(validators, f'shared_{{field.name}}', identity)(v, field=field) + + @root_validator(pre=False) + def _final_validation(cls, values): + return validation.core.finalize_config(getattr(validators, 'finalize_shared', identity)(values)) diff --git a/datadog_checks_dev/datadog_checks/dev/tooling/templates/integration/check/{check_name}/datadog_checks/{check_name}/config_models/validators.py b/datadog_checks_dev/datadog_checks/dev/tooling/templates/integration/check/{check_name}/datadog_checks/{check_name}/config_models/validators.py new file mode 100644 index 0000000000000..4ad5a0451cb35 --- /dev/null +++ b/datadog_checks_dev/datadog_checks/dev/tooling/templates/integration/check/{check_name}/datadog_checks/{check_name}/config_models/validators.py @@ -0,0 +1 @@ +{license_header} diff --git a/datadog_checks_dev/datadog_checks/dev/tooling/templates/integration/jmx/{check_name}/datadog_checks/{check_name}/config_models/__init__.py b/datadog_checks_dev/datadog_checks/dev/tooling/templates/integration/jmx/{check_name}/datadog_checks/{check_name}/config_models/__init__.py new file mode 100644 index 0000000000000..d92fb862e84b0 --- /dev/null +++ b/datadog_checks_dev/datadog_checks/dev/tooling/templates/integration/jmx/{check_name}/datadog_checks/{check_name}/config_models/__init__.py @@ -0,0 +1,16 @@ +{license_header} +from .instance import InstanceConfig +from .shared import SharedConfig + + +class ConfigMixin: + _config_model_instance: InstanceConfig + _config_model_shared: SharedConfig + + @property + def config(self) -> InstanceConfig: + return self._config_model_instance + + @property + def shared_config(self) -> SharedConfig: + return self._config_model_shared diff --git a/datadog_checks_dev/datadog_checks/dev/tooling/templates/integration/jmx/{check_name}/datadog_checks/{check_name}/config_models/defaults.py b/datadog_checks_dev/datadog_checks/dev/tooling/templates/integration/jmx/{check_name}/datadog_checks/{check_name}/config_models/defaults.py new file mode 100644 index 0000000000000..b8e2589d3dfb9 --- /dev/null +++ b/datadog_checks_dev/datadog_checks/dev/tooling/templates/integration/jmx/{check_name}/datadog_checks/{check_name}/config_models/defaults.py @@ -0,0 +1,98 @@ +{license_header} +from datadog_checks.base.utils.models.fields import get_default_field_value + + +def shared_conf(field, value): + return get_default_field_value(field, value) + + +def shared_new_gc_metrics(field, value): + return False + + +def shared_service(field, value): + return get_default_field_value(field, value) + + +def shared_service_check_prefix(field, value): + return get_default_field_value(field, value) + + +def instance_collect_default_jvm_metrics(field, value): + return True + + +def instance_empty_default_hostname(field, value): + return False + + +def instance_java_bin_path(field, value): + return get_default_field_value(field, value) + + +def instance_java_options(field, value): + return get_default_field_value(field, value) + + +def instance_jmx_url(field, value): + return get_default_field_value(field, value) + + +def instance_key_store_password(field, value): + return get_default_field_value(field, value) + + +def instance_key_store_path(field, value): + return get_default_field_value(field, value) + + +def instance_min_collection_interval(field, value): + return 15 + + +def instance_name(field, value): + return get_default_field_value(field, value) + + +def instance_password(field, value): + return get_default_field_value(field, value) + + +def instance_process_name_regex(field, value): + return get_default_field_value(field, value) + + +def instance_rmi_client_timeout(field, value): + return 15000 + + +def instance_rmi_connection_timeout(field, value): + return 20000 + + +def instance_rmi_registry_ssl(field, value): + return False + + +def instance_service(field, value): + return get_default_field_value(field, value) + + +def instance_tags(field, value): + return get_default_field_value(field, value) + + +def instance_tools_jar_path(field, value): + return get_default_field_value(field, value) + + +def instance_trust_store_password(field, value): + return get_default_field_value(field, value) + + +def instance_trust_store_path(field, value): + return get_default_field_value(field, value) + + +def instance_user(field, value): + return get_default_field_value(field, value) diff --git a/datadog_checks_dev/datadog_checks/dev/tooling/templates/integration/jmx/{check_name}/datadog_checks/{check_name}/config_models/instance.py b/datadog_checks_dev/datadog_checks/dev/tooling/templates/integration/jmx/{check_name}/datadog_checks/{check_name}/config_models/instance.py new file mode 100644 index 0000000000000..b9925dcc4677b --- /dev/null +++ b/datadog_checks_dev/datadog_checks/dev/tooling/templates/integration/jmx/{check_name}/datadog_checks/{check_name}/config_models/instance.py @@ -0,0 +1,61 @@ +{license_header} +from __future__ import annotations + +from typing import Optional, Sequence + +from pydantic import BaseModel, root_validator, validator + +from datadog_checks.base.utils.functions import identity +from datadog_checks.base.utils.models import validation + +from . import defaults, validators + + +class InstanceConfig(BaseModel): + class Config: + allow_mutation = False + + collect_default_jvm_metrics: Optional[bool] + empty_default_hostname: Optional[bool] + host: str + java_bin_path: Optional[str] + java_options: Optional[str] + jmx_url: Optional[str] + key_store_password: Optional[str] + key_store_path: Optional[str] + min_collection_interval: Optional[float] + name: Optional[str] + password: Optional[str] + port: int + process_name_regex: Optional[str] + rmi_client_timeout: Optional[float] + rmi_connection_timeout: Optional[float] + rmi_registry_ssl: Optional[bool] + service: Optional[str] + tags: Optional[Sequence[str]] + tools_jar_path: Optional[str] + trust_store_password: Optional[str] + trust_store_path: Optional[str] + user: Optional[str] + + @root_validator(pre=True) + def _initial_validation(cls, values): + return validation.core.initialize_config(getattr(validators, 'initialize_instance', identity)(values)) + + @validator('*', pre=True, always=True) + def _ensure_defaults(cls, v, field): + if v is not None or field.required: + return v + + return getattr(defaults, f'instance_{{field.name}}')(field, v) + + @validator('*') + def _run_validations(cls, v, field): + if not v: + return v + + return getattr(validators, f'instance_{{field.name}}', identity)(v, field=field) + + @root_validator(pre=False) + def _final_validation(cls, values): + return validation.core.finalize_config(getattr(validators, 'finalize_instance', identity)(values)) diff --git a/datadog_checks_dev/datadog_checks/dev/tooling/templates/integration/jmx/{check_name}/datadog_checks/{check_name}/config_models/shared.py b/datadog_checks_dev/datadog_checks/dev/tooling/templates/integration/jmx/{check_name}/datadog_checks/{check_name}/config_models/shared.py new file mode 100644 index 0000000000000..0218d2f79b316 --- /dev/null +++ b/datadog_checks_dev/datadog_checks/dev/tooling/templates/integration/jmx/{check_name}/datadog_checks/{check_name}/config_models/shared.py @@ -0,0 +1,45 @@ +{license_header} +from __future__ import annotations + +from typing import Any, Mapping, Optional, Sequence + +from pydantic import BaseModel, root_validator, validator + +from datadog_checks.base.utils.functions import identity +from datadog_checks.base.utils.models import validation + +from . import defaults, validators + + +class SharedConfig(BaseModel): + class Config: + allow_mutation = False + + collect_default_metrics: bool + conf: Optional[Sequence[Mapping[str, Any]]] + is_jmx: bool + new_gc_metrics: Optional[bool] + service: Optional[str] + service_check_prefix: Optional[str] + + @root_validator(pre=True) + def _initial_validation(cls, values): + return validation.core.initialize_config(getattr(validators, 'initialize_shared', identity)(values)) + + @validator('*', pre=True, always=True) + def _ensure_defaults(cls, v, field): + if v is not None or field.required: + return v + + return getattr(defaults, f'shared_{{field.name}}')(field, v) + + @validator('*') + def _run_validations(cls, v, field): + if not v: + return v + + return getattr(validators, f'shared_{{field.name}}', identity)(v, field=field) + + @root_validator(pre=False) + def _final_validation(cls, values): + return validation.core.finalize_config(getattr(validators, 'finalize_shared', identity)(values)) diff --git a/datadog_checks_dev/datadog_checks/dev/tooling/templates/integration/jmx/{check_name}/datadog_checks/{check_name}/config_models/validators.py b/datadog_checks_dev/datadog_checks/dev/tooling/templates/integration/jmx/{check_name}/datadog_checks/{check_name}/config_models/validators.py new file mode 100644 index 0000000000000..4ad5a0451cb35 --- /dev/null +++ b/datadog_checks_dev/datadog_checks/dev/tooling/templates/integration/jmx/{check_name}/datadog_checks/{check_name}/config_models/validators.py @@ -0,0 +1 @@ +{license_header} diff --git a/datadog_checks_dev/datadog_checks/dev/tooling/utils.py b/datadog_checks_dev/datadog_checks/dev/tooling/utils.py index e086d5cad9c39..1263c42b7c0e5 100644 --- a/datadog_checks_dev/datadog_checks/dev/tooling/utils.py +++ b/datadog_checks_dev/datadog_checks/dev/tooling/utils.py @@ -7,6 +7,7 @@ import os import re from ast import literal_eval +from datetime import datetime, timezone from json.decoder import JSONDecodeError import requests @@ -91,6 +92,14 @@ ) +def get_license_header(): + return ( + '# (C) Datadog, Inc. {year}-present\n' + '# All rights reserved\n' + '# Licensed under a 3-clause BSD style license (see LICENSE)'.format(year=str(datetime.now(timezone.utc).year)) + ) + + def format_commit_id(commit_id): if commit_id: if commit_id.isdigit(): @@ -344,6 +353,10 @@ def get_data_directory(check_name): return os.path.join(get_root(), check_name, 'datadog_checks', check_name, 'data') +def get_models_location(check_name): + return os.path.join(get_root(), check_name, 'datadog_checks', check_name, 'config_models') + + def get_check_directory(check_name): return os.path.join(get_root(), check_name, 'datadog_checks', check_name) @@ -644,6 +657,10 @@ def has_dashboard(check): return os.path.isdir(dashboards_path) and len(os.listdir(dashboards_path)) > 0 +def has_config_models(check): + return dir_exists(get_models_location(check)) + + def has_logs(check): config_file = get_config_file(check) if os.path.exists(config_file): diff --git a/datadog_checks_dev/setup.py b/datadog_checks_dev/setup.py index cba0c3633cade..3d01933e78961 100644 --- a/datadog_checks_dev/setup.py +++ b/datadog_checks_dev/setup.py @@ -73,6 +73,7 @@ 'atomicwrites', 'click>7', 'colorama', + 'datamodel-code-generator~=0.9.0; python_version > "3.0"', 'docker-compose>=1.25', 'in-toto>=0.4.2', 'jsonschema', diff --git a/datadog_checks_dev/tests/tooling/configuration/consumers/model/__init__.py b/datadog_checks_dev/tests/tooling/configuration/consumers/model/__init__.py new file mode 100644 index 0000000000000..9d0b0155542cb --- /dev/null +++ b/datadog_checks_dev/tests/tooling/configuration/consumers/model/__init__.py @@ -0,0 +1,3 @@ +# (C) Datadog, Inc. 2021-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) diff --git a/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_all_required.py b/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_all_required.py new file mode 100644 index 0000000000000..79e87406b049c --- /dev/null +++ b/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_all_required.py @@ -0,0 +1,97 @@ +# (C) Datadog, Inc. 2021-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import pytest + +from ...utils import get_model_consumer, normalize_yaml + +pytestmark = [pytest.mark.conf, pytest.mark.conf_consumer, pytest.mark.conf_consumer_model] + + +def test(): + consumer = get_model_consumer( + """ + name: test + version: 0.0.0 + files: + - name: test.yaml + options: + - template: instances + options: + - name: foo + required: true + description: words + value: + type: string + """ + ) + + model_definitions = consumer.render() + assert len(model_definitions) == 1 + + files = model_definitions['test.yaml'] + assert len(files) == 3 + + validators_contents, validators_errors = files['validators.py'] + assert not validators_errors + assert validators_contents == '' + + package_root_contents, package_root_errors = files['__init__.py'] + assert not package_root_errors + assert package_root_contents == normalize_yaml( + """ + from .instance import InstanceConfig + + + class ConfigMixin: + _config_model_instance: InstanceConfig + + @property + def config(self) -> InstanceConfig: + return self._config_model_instance + """ + ) + + instance_model_contents, instance_model_errors = files['instance.py'] + assert not instance_model_errors + assert instance_model_contents == normalize_yaml( + """ + from __future__ import annotations + + from pydantic import BaseModel, root_validator, validator + + from datadog_checks.base.utils.functions import identity + from datadog_checks.base.utils.models import validation + + from . import validators + + + class InstanceConfig(BaseModel): + class Config: + allow_mutation = False + + foo: str + + @root_validator(pre=True) + def _initial_validation(cls, values): + return validation.core.initialize_config(getattr(validators, 'initialize_instance', identity)(values)) + + @validator('*', pre=True, always=True) + def _ensure_defaults(cls, v, field): + if v is not None or field.required: + return v + + return getattr(defaults, f'instance_{field.name}')(field, v) + + @validator('*') + def _run_validations(cls, v, field): + if not v: + return v + + return getattr(validators, f'instance_{field.name}', identity)(v, field=field) + + @root_validator(pre=False) + def _final_validation(cls, values): + return validation.core.finalize_config(getattr(validators, 'finalize_instance', identity)(values)) + """ + ) diff --git a/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_array.py b/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_array.py new file mode 100644 index 0000000000000..3dde9c3237480 --- /dev/null +++ b/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_array.py @@ -0,0 +1,118 @@ +# (C) Datadog, Inc. 2021-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import pytest + +from ...utils import get_model_consumer, normalize_yaml + +pytestmark = [pytest.mark.conf, pytest.mark.conf_consumer, pytest.mark.conf_consumer_model] + + +def test(): + consumer = get_model_consumer( + """ + name: test + version: 0.0.0 + files: + - name: test.yaml + options: + - template: instances + options: + - name: foo + required: true + description: words + value: + type: string + - name: tags + description: words + value: + type: array + items: + type: string + """ + ) + + model_definitions = consumer.render() + assert len(model_definitions) == 1 + + files = model_definitions['test.yaml'] + assert len(files) == 4 + + validators_contents, validators_errors = files['validators.py'] + assert not validators_errors + assert validators_contents == '' + + package_root_contents, package_root_errors = files['__init__.py'] + assert not package_root_errors + assert package_root_contents == normalize_yaml( + """ + from .instance import InstanceConfig + + + class ConfigMixin: + _config_model_instance: InstanceConfig + + @property + def config(self) -> InstanceConfig: + return self._config_model_instance + """ + ) + + defaults_contents, defaults_errors = files['defaults.py'] + assert not defaults_errors + assert defaults_contents == normalize_yaml( + """ + from datadog_checks.base.utils.models.fields import get_default_field_value + + + def instance_tags(field, value): + return get_default_field_value(field, value) + """ + ) + + instance_model_contents, instance_model_errors = files['instance.py'] + assert not instance_model_errors + assert instance_model_contents == normalize_yaml( + """ + from __future__ import annotations + + from typing import Optional, Sequence + + from pydantic import BaseModel, root_validator, validator + + from datadog_checks.base.utils.functions import identity + from datadog_checks.base.utils.models import validation + + from . import defaults, validators + + + class InstanceConfig(BaseModel): + class Config: + allow_mutation = False + + foo: str + tags: Optional[Sequence[str]] + + @root_validator(pre=True) + def _initial_validation(cls, values): + return validation.core.initialize_config(getattr(validators, 'initialize_instance', identity)(values)) + + @validator('*', pre=True, always=True) + def _ensure_defaults(cls, v, field): + if v is not None or field.required: + return v + + return getattr(defaults, f'instance_{field.name}')(field, v) + + @validator('*') + def _run_validations(cls, v, field): + if not v: + return v + + return getattr(validators, f'instance_{field.name}', identity)(v, field=field) + + @root_validator(pre=False) + def _final_validation(cls, values): + return validation.core.finalize_config(getattr(validators, 'finalize_instance', identity)(values)) + """ + ) diff --git a/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_both_models_basic.py b/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_both_models_basic.py new file mode 100644 index 0000000000000..37d746114b46d --- /dev/null +++ b/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_both_models_basic.py @@ -0,0 +1,172 @@ +# (C) Datadog, Inc. 2021-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import pytest + +from ...utils import get_model_consumer, normalize_yaml + +pytestmark = [pytest.mark.conf, pytest.mark.conf_consumer, pytest.mark.conf_consumer_model] + + +def test(): + consumer = get_model_consumer( + """ + name: test + version: 0.0.0 + files: + - name: test.yaml + options: + - template: init_config + options: + - name: foo + description: words + value: + type: string + - template: instances + options: + - name: foo + description: words + value: + type: string + """ + ) + + model_definitions = consumer.render() + assert len(model_definitions) == 1 + + files = model_definitions['test.yaml'] + assert len(files) == 5 + + validators_contents, validators_errors = files['validators.py'] + assert not validators_errors + assert validators_contents == '' + + package_root_contents, package_root_errors = files['__init__.py'] + assert not package_root_errors + assert package_root_contents == normalize_yaml( + """ + from .instance import InstanceConfig + from .shared import SharedConfig + + + class ConfigMixin: + _config_model_instance: InstanceConfig + _config_model_shared: SharedConfig + + @property + def config(self) -> InstanceConfig: + return self._config_model_instance + + @property + def shared_config(self) -> SharedConfig: + return self._config_model_shared + """ + ) + + defaults_contents, defaults_errors = files['defaults.py'] + assert not defaults_errors + assert defaults_contents == normalize_yaml( + """ + from datadog_checks.base.utils.models.fields import get_default_field_value + + + def shared_foo(field, value): + return get_default_field_value(field, value) + + + def instance_foo(field, value): + return get_default_field_value(field, value) + """ + ) + + shared_model_contents, shared_model_errors = files['shared.py'] + assert not shared_model_errors + assert shared_model_contents == normalize_yaml( + """ + from __future__ import annotations + + from typing import Optional + + from pydantic import BaseModel, root_validator, validator + + from datadog_checks.base.utils.functions import identity + from datadog_checks.base.utils.models import validation + + from . import defaults, validators + + + class SharedConfig(BaseModel): + class Config: + allow_mutation = False + + foo: Optional[str] + + @root_validator(pre=True) + def _initial_validation(cls, values): + return validation.core.initialize_config(getattr(validators, 'initialize_shared', identity)(values)) + + @validator('*', pre=True, always=True) + def _ensure_defaults(cls, v, field): + if v is not None or field.required: + return v + + return getattr(defaults, f'shared_{field.name}')(field, v) + + @validator('*') + def _run_validations(cls, v, field): + if not v: + return v + + return getattr(validators, f'shared_{field.name}', identity)(v, field=field) + + @root_validator(pre=False) + def _final_validation(cls, values): + return validation.core.finalize_config(getattr(validators, 'finalize_shared', identity)(values)) + """ + ) + + instance_model_contents, instance_model_errors = files['instance.py'] + assert not instance_model_errors + assert instance_model_contents == normalize_yaml( + """ + from __future__ import annotations + + from typing import Optional + + from pydantic import BaseModel, root_validator, validator + + from datadog_checks.base.utils.functions import identity + from datadog_checks.base.utils.models import validation + + from . import defaults, validators + + + class InstanceConfig(BaseModel): + class Config: + allow_mutation = False + + foo: Optional[str] + + @root_validator(pre=True) + def _initial_validation(cls, values): + return validation.core.initialize_config(getattr(validators, 'initialize_instance', identity)(values)) + + @validator('*', pre=True, always=True) + def _ensure_defaults(cls, v, field): + if v is not None or field.required: + return v + + return getattr(defaults, f'instance_{field.name}')(field, v) + + @validator('*') + def _run_validations(cls, v, field): + if not v: + return v + + return getattr(validators, f'instance_{field.name}', identity)(v, field=field) + + @root_validator(pre=False) + def _final_validation(cls, values): + return validation.core.finalize_config(getattr(validators, 'finalize_instance', identity)(values)) + """ + ) diff --git a/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_common_validators.py b/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_common_validators.py new file mode 100644 index 0000000000000..3d57a4b49ce10 --- /dev/null +++ b/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_common_validators.py @@ -0,0 +1,152 @@ +# (C) Datadog, Inc. 2021-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import pytest + +from ...utils import get_model_consumer, normalize_yaml + +pytestmark = [pytest.mark.conf, pytest.mark.conf_consumer, pytest.mark.conf_consumer_model] + + +def test(): + consumer = get_model_consumer( + """ + name: test + version: 0.0.0 + files: + - name: test.yaml + options: + - template: instances + options: + - name: foo + required: true + description: words + value: + type: string + validators: + - pkg.subpkg2.validate2 + - pkg.subpkg2.validate1 + - name: tags + description: words + value: + type: array + items: + type: string + validators: + - pkg.subpkg1.validate2 + - pkg.subpkg1.validate1 + """ + ) + + model_definitions = consumer.render() + assert len(model_definitions) == 1 + + files = model_definitions['test.yaml'] + assert len(files) == 4 + + validators_contents, validators_errors = files['validators.py'] + assert not validators_errors + assert validators_contents == '' + + package_root_contents, package_root_errors = files['__init__.py'] + assert not package_root_errors + assert package_root_contents == normalize_yaml( + """ + from .instance import InstanceConfig + + + class ConfigMixin: + _config_model_instance: InstanceConfig + + @property + def config(self) -> InstanceConfig: + return self._config_model_instance + """ + ) + + defaults_contents, defaults_errors = files['defaults.py'] + assert not defaults_errors + assert defaults_contents == normalize_yaml( + """ + from datadog_checks.base.utils.models.fields import get_default_field_value + + + def instance_tags(field, value): + return get_default_field_value(field, value) + """ + ) + + instance_model_contents, instance_model_errors = files['instance.py'] + assert not instance_model_errors + assert instance_model_contents == normalize_yaml( + """ + from __future__ import annotations + + from typing import Optional, Sequence + + from pydantic import BaseModel, root_validator, validator + + from datadog_checks.base.utils.functions import identity + from datadog_checks.base.utils.models import validation + + from . import defaults, validators + + + class InstanceConfig(BaseModel): + class Config: + allow_mutation = False + + foo: str + tags: Optional[Sequence[str]] + + @root_validator(pre=True) + def _initial_validation(cls, values): + return validation.core.initialize_config(getattr(validators, 'initialize_instance', identity)(values)) + + @validator('*', pre=True, always=True) + def _ensure_defaults(cls, v, field): + if v is not None or field.required: + return v + + return getattr(defaults, f'instance_{field.name}')(field, v) + + @validator('*') + def _run_validations(cls, v, field): + if not v: + return v + + return getattr(validators, f'instance_{field.name}', identity)(v, field=field) + + @validator('foo') + def _run_foo_pkg_subpkg2_validate2(cls, v, field): + if not v: + return v + + return validation.pkg.subpkg2.validate2(v, field=field) + + @validator('foo') + def _run_foo_pkg_subpkg2_validate1(cls, v, field): + if not v: + return v + + return validation.pkg.subpkg2.validate1(v, field=field) + + @validator('tags') + def _run_tags_pkg_subpkg1_validate2(cls, v, field): + if not v: + return v + + return validation.pkg.subpkg1.validate2(v, field=field) + + @validator('tags') + def _run_tags_pkg_subpkg1_validate1(cls, v, field): + if not v: + return v + + return validation.pkg.subpkg1.validate1(v, field=field) + + @root_validator(pre=False) + def _final_validation(cls, values): + return validation.core.finalize_config(getattr(validators, 'finalize_instance', identity)(values)) + """ + ) diff --git a/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_defaults.py b/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_defaults.py new file mode 100644 index 0000000000000..71ccc5a141615 --- /dev/null +++ b/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_defaults.py @@ -0,0 +1,178 @@ +# (C) Datadog, Inc. 2021-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import pytest + +from ...utils import get_model_consumer, normalize_yaml + +pytestmark = [pytest.mark.conf, pytest.mark.conf_consumer, pytest.mark.conf_consumer_model] + + +def test(): + consumer = get_model_consumer( + """ + name: test + version: 0.0.0 + files: + - name: test.yaml + options: + - template: instances + options: + - name: foo + required: true + description: words + value: + type: string + - name: example + description: words + value: + example: bar + type: string + - name: default_precedence + description: words + value: + example: bar + default: baz + type: string + - name: example_ignored_array + description: words + value: + example: + - test + type: array + items: + type: string + - name: example_ignored_object + description: words + value: + example: + key: value + type: object + additionalProperties: true + - name: long_default_formatted + description: words + value: + default: + - ["01", "02", "03", "04", "05"] + - ["06", "07", "08", "09", "10"] + - ["11", "12", "13", "14", "15"] + - ["16", "17", "18", "19", "20"] + - ["21", "22", "23", "24", "25"] + type: array + items: + type: array + items: + type: string + """ + ) + + model_definitions = consumer.render() + assert len(model_definitions) == 1 + + files = model_definitions['test.yaml'] + assert len(files) == 4 + + validators_contents, validators_errors = files['validators.py'] + assert not validators_errors + assert validators_contents == '' + + package_root_contents, package_root_errors = files['__init__.py'] + assert not package_root_errors + assert package_root_contents == normalize_yaml( + """ + from .instance import InstanceConfig + + + class ConfigMixin: + _config_model_instance: InstanceConfig + + @property + def config(self) -> InstanceConfig: + return self._config_model_instance + """ + ) + + defaults_contents, defaults_errors = files['defaults.py'] + assert not defaults_errors + assert defaults_contents == normalize_yaml( + """ + from datadog_checks.base.utils.models.fields import get_default_field_value + + + def instance_default_precedence(field, value): + return 'baz' + + + def instance_example(field, value): + return 'bar' + + + def instance_example_ignored_array(field, value): + return get_default_field_value(field, value) + + + def instance_example_ignored_object(field, value): + return get_default_field_value(field, value) + + + def instance_long_default_formatted(field, value): + return [ + ['01', '02', '03', '04', '05'], + ['06', '07', '08', '09', '10'], + ['11', '12', '13', '14', '15'], + ['16', '17', '18', '19', '20'], + ['21', '22', '23', '24', '25'], + ] + """ + ) + + instance_model_contents, instance_model_errors = files['instance.py'] + assert not instance_model_errors + assert instance_model_contents == normalize_yaml( + """ + from __future__ import annotations + + from typing import Any, Mapping, Optional, Sequence + + from pydantic import BaseModel, root_validator, validator + + from datadog_checks.base.utils.functions import identity + from datadog_checks.base.utils.models import validation + + from . import defaults, validators + + + class InstanceConfig(BaseModel): + class Config: + allow_mutation = False + + default_precedence: Optional[str] + example: Optional[str] + example_ignored_array: Optional[Sequence[str]] + example_ignored_object: Optional[Mapping[str, Any]] + foo: str + long_default_formatted: Optional[Sequence[Sequence[str]]] + + @root_validator(pre=True) + def _initial_validation(cls, values): + return validation.core.initialize_config(getattr(validators, 'initialize_instance', identity)(values)) + + @validator('*', pre=True, always=True) + def _ensure_defaults(cls, v, field): + if v is not None or field.required: + return v + + return getattr(defaults, f'instance_{field.name}')(field, v) + + @validator('*') + def _run_validations(cls, v, field): + if not v: + return v + + return getattr(validators, f'instance_{field.name}', identity)(v, field=field) + + @root_validator(pre=False) + def _final_validation(cls, values): + return validation.core.finalize_config(getattr(validators, 'finalize_instance', identity)(values)) + """ + ) diff --git a/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_nested_option.py b/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_nested_option.py new file mode 100644 index 0000000000000..951360f71d03c --- /dev/null +++ b/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_nested_option.py @@ -0,0 +1,148 @@ +# (C) Datadog, Inc. 2021-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import pytest + +from ...utils import get_model_consumer, normalize_yaml + +pytestmark = [pytest.mark.conf, pytest.mark.conf_consumer, pytest.mark.conf_consumer_model] + + +def test(): + consumer = get_model_consumer( + """ + name: test + version: 0.0.0 + files: + - name: test.yaml + options: + - template: instances + options: + - name: foo + required: true + description: words + value: + type: string + - name: settings + description: words + options: + - name: setting1 + description: words + value: + type: string + - name: setting2 + description: words + value: + type: object + required: + - bar + properties: + - name: bar + type: string + - name: baz + type: array + items: + type: string + """ + ) + + model_definitions = consumer.render() + assert len(model_definitions) == 1 + + files = model_definitions['test.yaml'] + assert len(files) == 4 + + validators_contents, validators_errors = files['validators.py'] + assert not validators_errors + assert validators_contents == '' + + package_root_contents, package_root_errors = files['__init__.py'] + assert not package_root_errors + assert package_root_contents == normalize_yaml( + """ + from .instance import InstanceConfig + + + class ConfigMixin: + _config_model_instance: InstanceConfig + + @property + def config(self) -> InstanceConfig: + return self._config_model_instance + """ + ) + + defaults_contents, defaults_errors = files['defaults.py'] + assert not defaults_errors + assert defaults_contents == normalize_yaml( + """ + from datadog_checks.base.utils.models.fields import get_default_field_value + + + def instance_settings(field, value): + return get_default_field_value(field, value) + """ + ) + + instance_model_contents, instance_model_errors = files['instance.py'] + assert not instance_model_errors + assert instance_model_contents == normalize_yaml( + """ + from __future__ import annotations + + from typing import Optional, Sequence + + from pydantic import BaseModel, root_validator, validator + + from datadog_checks.base.utils.functions import identity + from datadog_checks.base.utils.models import validation + + from . import defaults, validators + + + class Setting2(BaseModel): + class Config: + allow_mutation = False + + bar: str + baz: Optional[Sequence[str]] + + + class Settings(BaseModel): + class Config: + allow_mutation = False + + setting1: Optional[str] + setting2: Optional[Setting2] + + + class InstanceConfig(BaseModel): + class Config: + allow_mutation = False + + foo: str + settings: Optional[Settings] + + @root_validator(pre=True) + def _initial_validation(cls, values): + return validation.core.initialize_config(getattr(validators, 'initialize_instance', identity)(values)) + + @validator('*', pre=True, always=True) + def _ensure_defaults(cls, v, field): + if v is not None or field.required: + return v + + return getattr(defaults, f'instance_{field.name}')(field, v) + + @validator('*') + def _run_validations(cls, v, field): + if not v: + return v + + return getattr(validators, f'instance_{field.name}', identity)(v, field=field) + + @root_validator(pre=False) + def _final_validation(cls, values): + return validation.core.finalize_config(getattr(validators, 'finalize_instance', identity)(values)) + """ + ) diff --git a/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_no_models.py b/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_no_models.py new file mode 100644 index 0000000000000..70fb85c3d1dec --- /dev/null +++ b/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_no_models.py @@ -0,0 +1,24 @@ +# (C) Datadog, Inc. 2021-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import pytest + +from ...utils import get_model_consumer + +pytestmark = [pytest.mark.conf, pytest.mark.conf_consumer, pytest.mark.conf_consumer_model] + + +def test(): + consumer = get_model_consumer( + """ + name: test + version: 0.0.0 + files: + - name: test.yaml + options: + - template: logs + """ + ) + + model_definitions = consumer.render() + assert not model_definitions diff --git a/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_object_arbitrary_values.py b/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_object_arbitrary_values.py new file mode 100644 index 0000000000000..8b84d5bb53ad9 --- /dev/null +++ b/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_object_arbitrary_values.py @@ -0,0 +1,117 @@ +# (C) Datadog, Inc. 2021-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import pytest + +from ...utils import get_model_consumer, normalize_yaml + +pytestmark = [pytest.mark.conf, pytest.mark.conf_consumer, pytest.mark.conf_consumer_model] + + +def test(): + consumer = get_model_consumer( + """ + name: test + version: 0.0.0 + files: + - name: test.yaml + options: + - template: instances + options: + - name: foo + required: true + description: words + value: + type: string + - name: obj + description: words + value: + type: object + additionalProperties: true + """ + ) + + model_definitions = consumer.render() + assert len(model_definitions) == 1 + + files = model_definitions['test.yaml'] + assert len(files) == 4 + + validators_contents, validators_errors = files['validators.py'] + assert not validators_errors + assert validators_contents == '' + + package_root_contents, package_root_errors = files['__init__.py'] + assert not package_root_errors + assert package_root_contents == normalize_yaml( + """ + from .instance import InstanceConfig + + + class ConfigMixin: + _config_model_instance: InstanceConfig + + @property + def config(self) -> InstanceConfig: + return self._config_model_instance + """ + ) + + defaults_contents, defaults_errors = files['defaults.py'] + assert not defaults_errors + assert defaults_contents == normalize_yaml( + """ + from datadog_checks.base.utils.models.fields import get_default_field_value + + + def instance_obj(field, value): + return get_default_field_value(field, value) + """ + ) + + instance_model_contents, instance_model_errors = files['instance.py'] + assert not instance_model_errors + assert instance_model_contents == normalize_yaml( + """ + from __future__ import annotations + + from typing import Any, Mapping, Optional + + from pydantic import BaseModel, root_validator, validator + + from datadog_checks.base.utils.functions import identity + from datadog_checks.base.utils.models import validation + + from . import defaults, validators + + + class InstanceConfig(BaseModel): + class Config: + allow_mutation = False + + foo: str + obj: Optional[Mapping[str, Any]] + + @root_validator(pre=True) + def _initial_validation(cls, values): + return validation.core.initialize_config(getattr(validators, 'initialize_instance', identity)(values)) + + @validator('*', pre=True, always=True) + def _ensure_defaults(cls, v, field): + if v is not None or field.required: + return v + + return getattr(defaults, f'instance_{field.name}')(field, v) + + @validator('*') + def _run_validations(cls, v, field): + if not v: + return v + + return getattr(validators, f'instance_{field.name}', identity)(v, field=field) + + @root_validator(pre=False) + def _final_validation(cls, values): + return validation.core.finalize_config(getattr(validators, 'finalize_instance', identity)(values)) + """ + ) diff --git a/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_object_model.py b/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_object_model.py new file mode 100644 index 0000000000000..c7656dca6ee9f --- /dev/null +++ b/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_object_model.py @@ -0,0 +1,133 @@ +# (C) Datadog, Inc. 2021-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import pytest + +from ...utils import get_model_consumer, normalize_yaml + +pytestmark = [pytest.mark.conf, pytest.mark.conf_consumer, pytest.mark.conf_consumer_model] + + +def test(): + consumer = get_model_consumer( + """ + name: test + version: 0.0.0 + files: + - name: test.yaml + options: + - template: instances + options: + - name: foo + required: true + description: words + value: + type: string + - name: obj + description: words + value: + type: object + required: + - bar + properties: + - name: bar + type: string + - name: baz + type: array + items: + type: string + """ + ) + + model_definitions = consumer.render() + assert len(model_definitions) == 1 + + files = model_definitions['test.yaml'] + assert len(files) == 4 + + validators_contents, validators_errors = files['validators.py'] + assert not validators_errors + assert validators_contents == '' + + package_root_contents, package_root_errors = files['__init__.py'] + assert not package_root_errors + assert package_root_contents == normalize_yaml( + """ + from .instance import InstanceConfig + + + class ConfigMixin: + _config_model_instance: InstanceConfig + + @property + def config(self) -> InstanceConfig: + return self._config_model_instance + """ + ) + + defaults_contents, defaults_errors = files['defaults.py'] + assert not defaults_errors + assert defaults_contents == normalize_yaml( + """ + from datadog_checks.base.utils.models.fields import get_default_field_value + + + def instance_obj(field, value): + return get_default_field_value(field, value) + """ + ) + + instance_model_contents, instance_model_errors = files['instance.py'] + assert not instance_model_errors + assert instance_model_contents == normalize_yaml( + """ + from __future__ import annotations + + from typing import Optional, Sequence + + from pydantic import BaseModel, root_validator, validator + + from datadog_checks.base.utils.functions import identity + from datadog_checks.base.utils.models import validation + + from . import defaults, validators + + + class Obj(BaseModel): + class Config: + allow_mutation = False + + bar: str + baz: Optional[Sequence[str]] + + + class InstanceConfig(BaseModel): + class Config: + allow_mutation = False + + foo: str + obj: Optional[Obj] + + @root_validator(pre=True) + def _initial_validation(cls, values): + return validation.core.initialize_config(getattr(validators, 'initialize_instance', identity)(values)) + + @validator('*', pre=True, always=True) + def _ensure_defaults(cls, v, field): + if v is not None or field.required: + return v + + return getattr(defaults, f'instance_{field.name}')(field, v) + + @validator('*') + def _run_validations(cls, v, field): + if not v: + return v + + return getattr(validators, f'instance_{field.name}', identity)(v, field=field) + + @root_validator(pre=False) + def _final_validation(cls, values): + return validation.core.finalize_config(getattr(validators, 'finalize_instance', identity)(values)) + """ + ) diff --git a/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_object_typed_values.py b/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_object_typed_values.py new file mode 100644 index 0000000000000..3b7f782bfab18 --- /dev/null +++ b/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_object_typed_values.py @@ -0,0 +1,124 @@ +# (C) Datadog, Inc. 2021-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import pytest + +from ...utils import get_model_consumer, normalize_yaml + +pytestmark = [pytest.mark.conf, pytest.mark.conf_consumer, pytest.mark.conf_consumer_model] + + +def test(): + consumer = get_model_consumer( + """ + name: test + version: 0.0.0 + files: + - name: test.yaml + options: + - template: instances + options: + - name: foo + required: true + description: words + value: + type: string + - name: obj + description: words + value: + type: object + additionalProperties: + type: array + items: + type: number + """ + ) + + model_definitions = consumer.render() + assert len(model_definitions) == 1 + + files = model_definitions['test.yaml'] + assert len(files) == 4 + + validators_contents, validators_errors = files['validators.py'] + assert not validators_errors + assert validators_contents == '' + + package_root_contents, package_root_errors = files['__init__.py'] + assert not package_root_errors + assert package_root_contents == normalize_yaml( + """ + from .instance import InstanceConfig + + + class ConfigMixin: + _config_model_instance: InstanceConfig + + @property + def config(self) -> InstanceConfig: + return self._config_model_instance + """ + ) + + defaults_contents, defaults_errors = files['defaults.py'] + assert not defaults_errors + assert defaults_contents == normalize_yaml( + """ + from datadog_checks.base.utils.models.fields import get_default_field_value + + + def instance_obj(field, value): + return get_default_field_value(field, value) + """ + ) + + instance_model_contents, instance_model_errors = files['instance.py'] + assert not instance_model_errors + assert instance_model_contents == normalize_yaml( + """ + from __future__ import annotations + + from typing import Mapping, Optional, Sequence + + from pydantic import BaseModel, root_validator, validator + + from datadog_checks.base.utils.functions import identity + from datadog_checks.base.utils.models import validation + + from . import defaults, validators + + + class Obj(BaseModel): + __root__: Sequence[float] + + + class InstanceConfig(BaseModel): + class Config: + allow_mutation = False + + foo: str + obj: Optional[Mapping[str, Obj]] + + @root_validator(pre=True) + def _initial_validation(cls, values): + return validation.core.initialize_config(getattr(validators, 'initialize_instance', identity)(values)) + + @validator('*', pre=True, always=True) + def _ensure_defaults(cls, v, field): + if v is not None or field.required: + return v + + return getattr(defaults, f'instance_{field.name}')(field, v) + + @validator('*') + def _run_validations(cls, v, field): + if not v: + return v + + return getattr(validators, f'instance_{field.name}', identity)(v, field=field) + + @root_validator(pre=False) + def _final_validation(cls, values): + return validation.core.finalize_config(getattr(validators, 'finalize_instance', identity)(values)) + """ + ) diff --git a/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_only_shared.py b/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_only_shared.py new file mode 100644 index 0000000000000..11b41c8c782d8 --- /dev/null +++ b/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_only_shared.py @@ -0,0 +1,110 @@ +# (C) Datadog, Inc. 2021-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import pytest + +from ...utils import get_model_consumer, normalize_yaml + +pytestmark = [pytest.mark.conf, pytest.mark.conf_consumer, pytest.mark.conf_consumer_model] + + +def test(): + consumer = get_model_consumer( + """ + name: test + version: 0.0.0 + files: + - name: test.yaml + options: + - template: init_config + options: + - name: foo + description: words + value: + type: string + """ + ) + + model_definitions = consumer.render() + assert len(model_definitions) == 1 + + files = model_definitions['test.yaml'] + assert len(files) == 4 + + validators_contents, validators_errors = files['validators.py'] + assert not validators_errors + assert validators_contents == '' + + package_root_contents, package_root_errors = files['__init__.py'] + assert not package_root_errors + assert package_root_contents == normalize_yaml( + """ + from .shared import SharedConfig + + + class ConfigMixin: + _config_model_shared: SharedConfig + + @property + def shared_config(self) -> SharedConfig: + return self._config_model_shared + """ + ) + + defaults_contents, defaults_errors = files['defaults.py'] + assert not defaults_errors + assert defaults_contents == normalize_yaml( + """ + from datadog_checks.base.utils.models.fields import get_default_field_value + + + def shared_foo(field, value): + return get_default_field_value(field, value) + """ + ) + + shared_model_contents, shared_model_errors = files['shared.py'] + assert not shared_model_errors + assert shared_model_contents == normalize_yaml( + """ + from __future__ import annotations + + from typing import Optional + + from pydantic import BaseModel, root_validator, validator + + from datadog_checks.base.utils.functions import identity + from datadog_checks.base.utils.models import validation + + from . import defaults, validators + + + class SharedConfig(BaseModel): + class Config: + allow_mutation = False + + foo: Optional[str] + + @root_validator(pre=True) + def _initial_validation(cls, values): + return validation.core.initialize_config(getattr(validators, 'initialize_shared', identity)(values)) + + @validator('*', pre=True, always=True) + def _ensure_defaults(cls, v, field): + if v is not None or field.required: + return v + + return getattr(defaults, f'shared_{field.name}')(field, v) + + @validator('*') + def _run_validations(cls, v, field): + if not v: + return v + + return getattr(validators, f'shared_{field.name}', identity)(v, field=field) + + @root_validator(pre=False) + def _final_validation(cls, values): + return validation.core.finalize_config(getattr(validators, 'finalize_shared', identity)(values)) + """ + ) diff --git a/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_option_name_normalization.py b/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_option_name_normalization.py new file mode 100644 index 0000000000000..06fb40531455d --- /dev/null +++ b/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_option_name_normalization.py @@ -0,0 +1,116 @@ +# (C) Datadog, Inc. 2021-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import pytest + +from ...utils import get_model_consumer, normalize_yaml + +pytestmark = [pytest.mark.conf, pytest.mark.conf_consumer, pytest.mark.conf_consumer_model] + + +def test(): + consumer = get_model_consumer( + """ + name: test + version: 0.0.0 + files: + - name: test.yaml + options: + - template: instances + options: + - name: foo + required: true + description: words + value: + type: string + - name: bar-baz + description: words + value: + type: string + """ + ) + + model_definitions = consumer.render() + assert len(model_definitions) == 1 + + files = model_definitions['test.yaml'] + assert len(files) == 4 + + validators_contents, validators_errors = files['validators.py'] + assert not validators_errors + assert validators_contents == '' + + package_root_contents, package_root_errors = files['__init__.py'] + assert not package_root_errors + assert package_root_contents == normalize_yaml( + """ + from .instance import InstanceConfig + + + class ConfigMixin: + _config_model_instance: InstanceConfig + + @property + def config(self) -> InstanceConfig: + return self._config_model_instance + """ + ) + + defaults_contents, defaults_errors = files['defaults.py'] + assert not defaults_errors + assert defaults_contents == normalize_yaml( + """ + from datadog_checks.base.utils.models.fields import get_default_field_value + + + def instance_bar_baz(field, value): + return get_default_field_value(field, value) + """ + ) + + instance_model_contents, instance_model_errors = files['instance.py'] + assert not instance_model_errors + assert instance_model_contents == normalize_yaml( + """ + from __future__ import annotations + + from typing import Optional + + from pydantic import BaseModel, Field, root_validator, validator + + from datadog_checks.base.utils.functions import identity + from datadog_checks.base.utils.models import validation + + from . import defaults, validators + + + class InstanceConfig(BaseModel): + class Config: + allow_mutation = False + + bar_baz: Optional[str] = Field(None, alias='bar-baz') + foo: str + + @root_validator(pre=True) + def _initial_validation(cls, values): + return validation.core.initialize_config(getattr(validators, 'initialize_instance', identity)(values)) + + @validator('*', pre=True, always=True) + def _ensure_defaults(cls, v, field): + if v is not None or field.required: + return v + + return getattr(defaults, f'instance_{field.name}')(field, v) + + @validator('*') + def _run_validations(cls, v, field): + if not v: + return v + + return getattr(validators, f'instance_{field.name}', identity)(v, field=field) + + @root_validator(pre=False) + def _final_validation(cls, values): + return validation.core.finalize_config(getattr(validators, 'finalize_instance', identity)(values)) + """ + ) diff --git a/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_union_types.py b/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_union_types.py new file mode 100644 index 0000000000000..4271787c6ad66 --- /dev/null +++ b/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_union_types.py @@ -0,0 +1,120 @@ +# (C) Datadog, Inc. 2021-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import pytest + +from ...utils import get_model_consumer, normalize_yaml + +pytestmark = [pytest.mark.conf, pytest.mark.conf_consumer, pytest.mark.conf_consumer_model] + + +def test(): + consumer = get_model_consumer( + """ + name: test + version: 0.0.0 + files: + - name: test.yaml + options: + - template: instances + options: + - name: foo + required: true + description: words + value: + type: string + - name: obj + description: words + value: + anyOf: + - type: string + - type: array + items: + type: string + """ + ) + + model_definitions = consumer.render() + assert len(model_definitions) == 1 + + files = model_definitions['test.yaml'] + assert len(files) == 4 + + validators_contents, validators_errors = files['validators.py'] + assert not validators_errors + assert validators_contents == '' + + package_root_contents, package_root_errors = files['__init__.py'] + assert not package_root_errors + assert package_root_contents == normalize_yaml( + """ + from .instance import InstanceConfig + + + class ConfigMixin: + _config_model_instance: InstanceConfig + + @property + def config(self) -> InstanceConfig: + return self._config_model_instance + """ + ) + + defaults_contents, defaults_errors = files['defaults.py'] + assert not defaults_errors + assert defaults_contents == normalize_yaml( + """ + from datadog_checks.base.utils.models.fields import get_default_field_value + + + def instance_obj(field, value): + return get_default_field_value(field, value) + """ + ) + + instance_model_contents, instance_model_errors = files['instance.py'] + assert not instance_model_errors + assert instance_model_contents == normalize_yaml( + """ + from __future__ import annotations + + from typing import Optional, Sequence, Union + + from pydantic import BaseModel, root_validator, validator + + from datadog_checks.base.utils.functions import identity + from datadog_checks.base.utils.models import validation + + from . import defaults, validators + + + class InstanceConfig(BaseModel): + class Config: + allow_mutation = False + + foo: str + obj: Optional[Union[str, Sequence[str]]] + + @root_validator(pre=True) + def _initial_validation(cls, values): + return validation.core.initialize_config(getattr(validators, 'initialize_instance', identity)(values)) + + @validator('*', pre=True, always=True) + def _ensure_defaults(cls, v, field): + if v is not None or field.required: + return v + + return getattr(defaults, f'instance_{field.name}')(field, v) + + @validator('*') + def _run_validations(cls, v, field): + if not v: + return v + + return getattr(validators, f'instance_{field.name}', identity)(v, field=field) + + @root_validator(pre=False) + def _final_validation(cls, values): + return validation.core.finalize_config(getattr(validators, 'finalize_instance', identity)(values)) + """ + ) diff --git a/datadog_checks_dev/tests/tooling/configuration/consumers/test_example.py b/datadog_checks_dev/tests/tooling/configuration/consumers/test_example.py index b5e1c54d89c3d..f6143f4b47c2b 100644 --- a/datadog_checks_dev/tests/tooling/configuration/consumers/test_example.py +++ b/datadog_checks_dev/tests/tooling/configuration/consumers/test_example.py @@ -8,7 +8,7 @@ from ..utils import get_example_consumer, normalize_yaml -pytestmark = [pytest.mark.conf, pytest.mark.conf_consumer] +pytestmark = [pytest.mark.conf, pytest.mark.conf_consumer, pytest.mark.conf_consumer_example] def test_option_no_section(): @@ -1490,6 +1490,9 @@ def test_option_multiple_types_nested(): anyOf: - type: string - type: object + properties: + - name: foo + type: string required: - foo """ diff --git a/datadog_checks_dev/tests/tooling/configuration/utils.py b/datadog_checks_dev/tests/tooling/configuration/utils.py index d3e252773a758..c5acee7ba3b2e 100644 --- a/datadog_checks_dev/tests/tooling/configuration/utils.py +++ b/datadog_checks_dev/tests/tooling/configuration/utils.py @@ -4,7 +4,7 @@ from textwrap import dedent from datadog_checks.dev.tooling.specs.configuration import ConfigSpec -from datadog_checks.dev.tooling.specs.configuration.consumers import ExampleConsumer +from datadog_checks.dev.tooling.specs.configuration.consumers import ExampleConsumer, ModelConsumer def get_spec(text, **kwargs): @@ -15,8 +15,16 @@ def get_spec(text, **kwargs): def get_example_consumer(text, **kwargs): spec = get_spec(text, **kwargs) spec.load() + assert not spec.errors return ExampleConsumer(spec.data) +def get_model_consumer(text, **kwargs): + spec = get_spec(text, **kwargs) + spec.load() + assert not spec.errors + return ModelConsumer(spec.data) + + def normalize_yaml(text): return dedent(text).lstrip() diff --git a/docs/developer/.scripts/33_render_status.py b/docs/developer/.scripts/33_render_status.py index 0fed84eaacc19..7e2fee91bb570 100644 --- a/docs/developer/.scripts/33_render_status.py +++ b/docs/developer/.scripts/33_render_status.py @@ -9,6 +9,7 @@ get_valid_integrations, has_logs, has_agent_8_check_signature, + has_config_models, has_dashboard, has_e2e, has_process_signature, @@ -35,6 +36,7 @@ def patch(lines): render_config_spec_progress, render_docs_spec_progress, render_e2e_progress, + render_config_validation_progress, render_metadata_progress, render_process_signatures_progress, render_check_signatures_progress, @@ -300,3 +302,26 @@ def render_recommended_monitors_progress(): lines[2] = f'[={formatted_percent}% "{formatted_percent}%"]' lines[4] = f'??? check "Completed {checks_with_rm}/{total_checks}"' return lines + + +def render_config_validation_progress(): + valid_checks = sorted(c for c in get_valid_checks() if os.path.isfile(get_default_config_spec(c))) + total_checks = len(valid_checks) + checks_with_config_validation = 0 + + lines = ['## Config validation', '', None, '', '??? check "Completed"'] + + for check in valid_checks: + if has_config_models(check): + status = 'X' + checks_with_config_validation += 1 + else: + status = ' ' + + lines.append(f' - [{status}] {check}') + + percent = checks_with_config_validation / total_checks * 100 + formatted_percent = f'{percent:.2f}' + lines[2] = f'[={formatted_percent}% "{formatted_percent}%"]' + lines[4] = f'??? check "Completed {checks_with_config_validation}/{total_checks}"' + return lines diff --git a/docs/developer/.snippets/links.txt b/docs/developer/.snippets/links.txt index 4fa505ee5c788..1208533516b5c 100644 --- a/docs/developer/.snippets/links.txt +++ b/docs/developer/.snippets/links.txt @@ -15,6 +15,7 @@ [click-github]: https://github.com/pallets/click [codecov-home]: https://codecov.io [config-spec-example-consumer]: https://github.com/DataDog/integrations-core/blob/master/datadog_checks_dev/datadog_checks/dev/tooling/specs/configuration/consumers/example.py +[config-spec-model-consumer]: https://github.com/DataDog/integrations-core/blob/master/datadog_checks_dev/datadog_checks/dev/tooling/specs/configuration/consumers/model.py [config-spec-producer]: https://github.com/DataDog/integrations-core/blob/master/datadog_checks_dev/datadog_checks/dev/tooling/specs/configuration/core.py [config-spec-template-init-config-http]: https://github.com/DataDog/integrations-core/blob/master/datadog_checks_dev/datadog_checks/dev/tooling/templates/configuration/init_config/http.yaml [config-spec-template-instances-http]: https://github.com/DataDog/integrations-core/blob/master/datadog_checks_dev/datadog_checks/dev/tooling/templates/configuration/instances/http.yaml diff --git a/docs/developer/meta/config-specs.md b/docs/developer/meta/config-specs.md index f8a870e750492..5a92cd1fd7235 100644 --- a/docs/developer/meta/config-specs.md +++ b/docs/developer/meta/config-specs.md @@ -140,6 +140,19 @@ It also respects a few extra fields under the `value` attribute of each option: Use the `--sync` flag of the [config validation command](../ddev/cli.md#ddev-validate-config) to render the example configuration files. +## Data model consumer + +The [model consumer][config-spec-model-consumer] uses each spec to render the [pydantic](https://github.com/samuelcolvin/pydantic) models +that checks use to validate and interface with configuration. The models are shipped with every Agent and individual Integration release. + +It respects an extra field under the `value` attribute of each option: + +- `default` - This is the default value that options will be set to, taking precedence over the `example`. + +### Usage + +Use the `--sync` flag of the [model validation command](../ddev/cli.md#ddev-validate-models) to render the data model files. + ## API ::: datadog_checks.dev.tooling.specs.configuration.ConfigSpec diff --git a/mypy.ini b/mypy.ini index e182ce615746e..062f271cf5efb 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1,6 +1,8 @@ ; See: https://mypy.readthedocs.io/en/stable/config_file.html [mypy] +plugins = pydantic.mypy + ; Follows imports and type-check imported modules. follow_imports = normal