diff --git a/.azure-pipelines/templates/run-validations.yml b/.azure-pipelines/templates/run-validations.yml index c5a95bd6c3efc..f7a65cf841b8e 100644 --- a/.azure-pipelines/templates/run-validations.yml +++ b/.azure-pipelines/templates/run-validations.yml @@ -33,6 +33,11 @@ steps: ddev validate config displayName: 'Validate default configuration files' +- script: | + echo "ddev validate models" + ddev validate models + displayName: 'Validate configuration data models' + - script: | echo "ddev validate dashboards" ddev validate dashboards diff --git a/.flake8 b/.flake8 index 796ebd4423cc4..77cf74eee27ce 100644 --- a/.flake8 +++ b/.flake8 @@ -8,3 +8,11 @@ ignore = E203,E722,E741,W503,G200 exclude = .eggs,.tox,build,compat.py,__init__.py,datadog_checks_dev/datadog_checks/dev/tooling/templates/*,*/datadog_checks/*/vendor/* max-line-length = 120 enable-extensions=G +per-file-ignores = + # potentially long literal strings + datadog_checks/*/config_models/deprecations.py: E501 + # https://pydantic-docs.helpmanual.io/usage/validators/ + # > validators are "class methods", so the first argument value they + # receive is the UserModel class, not an instance of UserModel + datadog_checks/*/config_models/instance.py: B902 + datadog_checks/*/config_models/shared.py: B902 diff --git a/datadog_checks_base/datadog_checks/base/checks/base.py b/datadog_checks_base/datadog_checks/base/checks/base.py index 9a540b331700d..d7d90f94feaa7 100644 --- a/datadog_checks_base/datadog_checks/base/checks/base.py +++ b/datadog_checks_base/datadog_checks/base/checks/base.py @@ -15,10 +15,11 @@ from typing import TYPE_CHECKING, Any, AnyStr, Callable, Deque, Dict, List, Optional, Sequence, Tuple, Union import yaml -from six import binary_type, iteritems, text_type +from six import PY2, binary_type, iteritems, raise_from, text_type from ..config import is_affirmative from ..constants import ServiceCheck +from ..errors import ConfigurationError from ..types import ( AgentConfigType, Event, @@ -63,6 +64,9 @@ monkey_patch_pyyaml() +if not PY2: + from pydantic import ValidationError + if TYPE_CHECKING: import ssl @@ -241,9 +245,16 @@ def __init__(self, *args, **kwargs): # Setup metric limits self.metric_limiter = self._get_metric_limiter(self.name, instance=self.instance) + # Lazily load and validate config + self.__config = None + self.__shared_config = None + # Functions that will be called exactly once (if successful) before the first check run self.check_initializations = deque([self.send_config_metadata]) # type: Deque[Callable[[], None]] + if not PY2: + self.check_initializations.append(self.load_configuration_models) + def _get_metric_limiter(self, name, instance=None): # type: (str, InstanceType) -> Optional[Limiter] limit = self._get_metric_limit(instance=instance) @@ -364,6 +375,72 @@ def in_developer_mode(self): self._log_deprecation('in_developer_mode') return False + @property + def config(self): + return self.__config + + @property + def shared_config(self): + return self.__shared_config + + def load_configuration_models(self): + # 'datadog_checks.....' + module_parts = self.__module__.split('.') + package_path = '{}.config_models'.format('.'.join(module_parts[:2])) + + if self.__shared_config is None: + self.__shared_config = self.load_configuration_model( + package_path, 'SharedConfig', self._get_shared_config() + ) + + if self.__config is None: + self.__config = self.load_configuration_model(package_path, 'InstanceConfig', self._get_instance_config()) + + @staticmethod + def load_configuration_model(import_path, model_name, config): + try: + package = importlib.import_module(import_path) + except ModuleNotFoundError as e: # type: ignore + # Don't fail if there are no models + if str(e) == 'No module named {!r}'.format(import_path): + return + + raise + + model = getattr(package, model_name, None) + if model is not None: + try: + config_model = model(**copy.deepcopy(config)) + except ValidationError as e: + errors = e.errors() + num_errors = len(errors) + message_lines = [ + 'Detected {} error{} while loading configuration model `{}`:'.format( + num_errors, 's' if num_errors > 1 else '', model_name + ) + ] + + for error in e.errors(): + message_lines.append( + ' -> '.join( + # Start array indexes at one for user-friendliness + str(loc + 1) if isinstance(loc, int) else str(loc) for loc in error['loc'] + ) + ) + message_lines.append(' {}'.format(error['msg'])) + + raise_from(ConfigurationError('\n'.join(message_lines)), None) + else: + return config_model + + def _get_instance_config(self): + # Any extra fields will be available during a config model's initial validation stage + return self.instance + + def _get_shared_config(self): + # Any extra fields will be available during a config model's initial validation stage + return self.init_config + def register_secret(self, secret): # type: (str) -> None """ diff --git a/datadog_checks_base/datadog_checks/base/data/agent_requirements.in b/datadog_checks_base/datadog_checks/base/data/agent_requirements.in index 15899f03d3a26..26a42799f12dc 100644 --- a/datadog_checks_base/datadog_checks/base/data/agent_requirements.in +++ b/datadog_checks_base/datadog_checks/base/data/agent_requirements.in @@ -19,6 +19,7 @@ flup-py3==1.0.3; python_version > "3.0" flup==1.0.3.dev-20110405; python_version < "3.0" futures==3.3.0; python_version < "3.0" gearman==2.0.2; sys_platform != "win32" and python_version < "3.0" +immutables==0.15; python_version > "3.0" in-toto==0.5.0 ipaddress==1.0.22; python_version < "3.0" jaydebeapi==1.2.3 @@ -42,6 +43,7 @@ psutil==5.7.2 psycopg2-binary==2.8.4 pyasn1==0.4.6 pycryptodomex==3.9.4 +pydantic==1.7.3; python_version > "3.0" pyhdb==0.3.4 pyjwt==1.7.1 pymongo==3.8.0 diff --git a/datadog_checks_base/datadog_checks/base/utils/models/__init__.py b/datadog_checks_base/datadog_checks/base/utils/models/__init__.py new file mode 100644 index 0000000000000..9d0b0155542cb --- /dev/null +++ b/datadog_checks_base/datadog_checks/base/utils/models/__init__.py @@ -0,0 +1,3 @@ +# (C) Datadog, Inc. 2021-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) diff --git a/datadog_checks_base/datadog_checks/base/utils/models/fields.py b/datadog_checks_base/datadog_checks/base/utils/models/fields.py new file mode 100644 index 0000000000000..a35384eb34dc9 --- /dev/null +++ b/datadog_checks_base/datadog_checks/base/utils/models/fields.py @@ -0,0 +1,17 @@ +# (C) Datadog, Inc. 2021-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from pydantic.fields import SHAPE_MAPPING, SHAPE_SEQUENCE, SHAPE_SINGLETON + + +def get_default_field_value(field, value): + if field.shape == SHAPE_MAPPING: + return {} + elif field.shape == SHAPE_SEQUENCE: + return [] + elif field.shape == SHAPE_SINGLETON: + field_type = field.type_ + if field_type in (float, int, str): + return field_type() + + return value diff --git a/datadog_checks_base/datadog_checks/base/utils/models/types.py b/datadog_checks_base/datadog_checks/base/utils/models/types.py new file mode 100644 index 0000000000000..f4a6c9078316f --- /dev/null +++ b/datadog_checks_base/datadog_checks/base/utils/models/types.py @@ -0,0 +1,15 @@ +# (C) Datadog, Inc. 2021-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from immutables import Map + + +def make_immutable_check_config(obj): + # Only consider container types that can be de-serialized from YAML/JSON + if isinstance(obj, list): + return tuple(make_immutable_check_config(item) for item in obj) + elif isinstance(obj, dict): + # There are no ordering guarantees, see https://github.com/MagicStack/immutables/issues/57 + return Map((k, make_immutable_check_config(v)) for k, v in obj.items()) + + return obj diff --git a/datadog_checks_base/datadog_checks/base/utils/models/validation/__init__.py b/datadog_checks_base/datadog_checks/base/utils/models/validation/__init__.py new file mode 100644 index 0000000000000..f63316854c136 --- /dev/null +++ b/datadog_checks_base/datadog_checks/base/utils/models/validation/__init__.py @@ -0,0 +1,4 @@ +# (C) Datadog, Inc. 2021-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from . import core diff --git a/datadog_checks_base/datadog_checks/base/utils/models/validation/core.py b/datadog_checks_base/datadog_checks/base/utils/models/validation/core.py new file mode 100644 index 0000000000000..03a2c673d2917 --- /dev/null +++ b/datadog_checks_base/datadog_checks/base/utils/models/validation/core.py @@ -0,0 +1,17 @@ +# (C) Datadog, Inc. 2021-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from ..types import make_immutable_check_config + + +def initialize_config(values): + # This is what is returned by the initial root validator of each config model. + return values + + +def finalize_config(values): + # This is what is returned by the final root validator of each config model. Note: + # + # 1. the final object must be a dict + # 2. we maintain the original order of keys + return {field: make_immutable_check_config(value) for field, value in values.items()} diff --git a/datadog_checks_base/requirements.in b/datadog_checks_base/requirements.in index d2b51f41d7a21..3a5ec3f9945de 100644 --- a/datadog_checks_base/requirements.in +++ b/datadog_checks_base/requirements.in @@ -5,12 +5,14 @@ contextlib2==0.6.0; python_version < '3.0' cryptography==3.3.2 ddtrace==0.32.2 enum34==1.1.6; python_version < '3.0' +immutables==0.15; python_version > '3.0' ipaddress==1.0.22; python_version < '3.0' kubernetes==8.0.1 mmh3==2.5.1 orjson==2.6.1; python_version > '3.0' prometheus-client==0.9.0 protobuf==3.7.0 +pydantic==1.7.3; python_version > '3.0' pyjwt==1.7.1 pysocks==1.7.0 python-dateutil==2.8.0 diff --git a/datadog_checks_base/tests/models/__init__.py b/datadog_checks_base/tests/models/__init__.py new file mode 100644 index 0000000000000..9d0b0155542cb --- /dev/null +++ b/datadog_checks_base/tests/models/__init__.py @@ -0,0 +1,3 @@ +# (C) Datadog, Inc. 2021-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) diff --git a/datadog_checks_base/tests/models/test_types.py b/datadog_checks_base/tests/models/test_types.py new file mode 100644 index 0000000000000..1c07ecbf8d96c --- /dev/null +++ b/datadog_checks_base/tests/models/test_types.py @@ -0,0 +1,44 @@ +# (C) Datadog, Inc. 2021-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from ..utils import requires_py3 + +pytestmark = [requires_py3] + + +def test_make_immutable_check_config(): + # TODO: move imports up top when we drop Python 2 + from immutables import Map + + from datadog_checks.base.utils.models.types import make_immutable_check_config + + obj = make_immutable_check_config( + { + 'string': 'foo', + 'integer': 9000, + 'float': 3.14, + 'boolean': True, + 'array': [{'key': 'foo'}, {'key': 'bar'}], + 'mapping': {'foo': 'bar'}, + } + ) + + assert isinstance(obj, Map) + assert len(obj) == 6 + assert isinstance(obj['string'], str) + assert obj['string'] == 'foo' + assert isinstance(obj['integer'], int) + assert obj['integer'] == 9000 + assert isinstance(obj['float'], float) + assert obj['float'] == 3.14 + assert isinstance(obj['boolean'], bool) + assert obj['boolean'] is True + assert isinstance(obj['array'], tuple) + assert len(obj['array']) == 2 + assert isinstance(obj['array'][0], Map) + assert obj['array'][0]['key'] == 'foo' + assert isinstance(obj['array'][1], Map) + assert obj['array'][1]['key'] == 'bar' + assert isinstance(obj['mapping'], Map) + assert len(obj['mapping']) == 1 + assert obj['mapping']['foo'] == 'bar' diff --git a/datadog_checks_base/tests/test_agent_check.py b/datadog_checks_base/tests/test_agent_check.py index 964761afe6fa4..3985d819c799b 100644 --- a/datadog_checks_base/tests/test_agent_check.py +++ b/datadog_checks_base/tests/test_agent_check.py @@ -9,6 +9,7 @@ import mock import pytest +from immutables import Map from six import PY3 from datadog_checks.base import AgentCheck @@ -16,6 +17,8 @@ from datadog_checks.base import to_native_string from datadog_checks.base.checks.base import datadog_agent +from .utils import requires_py3 + def test_instance(): """ @@ -840,3 +843,31 @@ def check(self, _): check.run() assert check.initialize.call_count == 2 + + +@requires_py3 +def test_load_configuration_models(dd_run_check, mocker): + instance = {'endpoint': 'url', 'tags': ['foo:bar'], 'proxy': {'http': 'http://1.2.3.4:9000'}} + init_config = {'proxy': {'https': 'https://1.2.3.4:4242'}} + check = AgentCheck('test', init_config, [instance]) + check.check_id = 'test:123' + check.check = lambda _: None + + assert check.config is None + assert check.shared_config is None + + config_model = {'endpoint': 'url', 'tags': ('foo:bar',), 'proxy': Map(http='http://1.2.3.4:9000')} + shared_config_model = {'proxy': Map(https='https://1.2.3.4:4242')} + package = mocker.MagicMock() + package.InstanceConfig = mocker.MagicMock(return_value=config_model) + package.SharedConfig = mocker.MagicMock(return_value=shared_config_model) + import_module = mocker.patch('importlib.import_module', return_value=package) + + dd_run_check(check) + + import_module.assert_called_once_with('datadog_checks.base.models') + package.InstanceConfig.assert_called_once_with(**config_model) + package.SharedConfig.assert_called_once_with(**shared_config_model) + + assert check.config is config_model + assert check.shared_config is shared_config_model diff --git a/datadog_checks_dev/datadog_checks/dev/tooling/commands/validate/__init__.py b/datadog_checks_dev/datadog_checks/dev/tooling/commands/validate/__init__.py index 4e702113a1bda..8f7d1fd2c4145 100644 --- a/datadog_checks_dev/datadog_checks/dev/tooling/commands/validate/__init__.py +++ b/datadog_checks_dev/datadog_checks/dev/tooling/commands/validate/__init__.py @@ -17,6 +17,7 @@ from .jmx_metrics import jmx_metrics from .manifest import manifest from .metadata import metadata +from .models import models from .package import package from .readmes import readmes from .recommended_monitors import recommended_monitors @@ -37,6 +38,7 @@ imports, manifest, metadata, + models, package, readmes, recommended_monitors, diff --git a/datadog_checks_dev/datadog_checks/dev/tooling/commands/validate/models.py b/datadog_checks_dev/datadog_checks/dev/tooling/commands/validate/models.py new file mode 100644 index 0000000000000..597d71b953e3b --- /dev/null +++ b/datadog_checks_dev/datadog_checks/dev/tooling/commands/validate/models.py @@ -0,0 +1,157 @@ +# (C) Datadog, Inc. 2021-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import click + +from ....utils import ( + chdir, + dir_exists, + ensure_parent_dir_exists, + file_exists, + get_parent_dir, + path_join, + read_file, + read_file_lines, + write_file_lines, +) +from ...constants import get_root +from ...specs.configuration import ConfigSpec +from ...specs.configuration.consumers import ModelConsumer +from ...utils import ( + complete_valid_checks, + get_config_spec, + get_license_header, + get_valid_checks, + get_version_file, + get_version_string, +) +from ..console import CONTEXT_SETTINGS, abort, echo_failure, echo_info, echo_success, echo_waiting + + +@click.command(context_settings=CONTEXT_SETTINGS, short_help='Validate configuration data models') +@click.argument('check', autocompletion=complete_valid_checks, required=False) +@click.option('--sync', '-s', is_flag=True, help='Generate data models based on specifications') +@click.option('--verbose', '-v', is_flag=True, help='Verbose mode') +@click.pass_context +def models(ctx, check, sync, verbose): + """Validate configuration data models.""" + root = get_root() + community_check = ctx.obj['repo_choice'] not in ('core', 'internal') + checks = [check] if check else sorted(get_valid_checks()) + + specs_failed = {} + files_failed = {} + num_files = 0 + + license_header_lines = get_license_header().splitlines(True) + license_header_lines.append('\n') + + code_formatter = ModelConsumer.create_code_formatter() + + echo_waiting('Validating data models...') + for check in checks: + check_display_queue = [] + + spec_path = get_config_spec(check) + if not file_exists(spec_path): + continue + + spec_file = read_file(spec_path) + spec = ConfigSpec(spec_file, source=check, version=get_version_string(check)) + spec.load() + + if spec.errors: + specs_failed[spec_path] = True + echo_info(f'{check}:') + for error in spec.errors: + echo_failure(error, indent=True) + continue + + models_location = path_join(get_parent_dir(get_version_file(check)), 'config_models') + # TODO: Remove when all integrations have models + if not sync and not dir_exists(models_location): + continue + model_consumer = ModelConsumer(spec.data, code_formatter) + + # So formatters see config files + with chdir(root): + model_definitions = model_consumer.render() + + model_files = model_definitions.get(f'{check}.yaml') + if not model_files: + continue + + for model_file, (contents, errors) in model_files.items(): + num_files += 1 + + model_file_path = path_join(models_location, model_file) + if errors: + files_failed[model_file_path] = True + for error in errors: + check_display_queue.append(lambda error=error, **kwargs: echo_failure(error, **kwargs)) + continue + + model_file_lines = contents.splitlines(True) + current_model_file_lines = [] + expected_model_file_lines = [] + if file_exists(model_file_path): + # No contents indicates a custom file + if not contents: + continue + + current_model_file_lines.extend(read_file_lines(model_file_path)) + + for line in current_model_file_lines: + if not line.startswith('#'): + break + + expected_model_file_lines.append(line) + + expected_model_file_lines.extend(model_file_lines) + else: + if not community_check: + expected_model_file_lines.extend(license_header_lines) + + expected_model_file_lines.extend(model_file_lines) + + if current_model_file_lines != expected_model_file_lines: + if sync: + echo_info(f'Writing data model file to `{model_file_path}`') + ensure_parent_dir_exists(model_file_path) + write_file_lines(model_file_path, expected_model_file_lines) + else: + files_failed[model_file_path] = True + check_display_queue.append( + lambda model_file=model_file, **kwargs: echo_failure( + f'File `{model_file}` is not in sync, run "ddev validate models -s"', **kwargs + ) + ) + + if check_display_queue or verbose: + echo_info(f'{check}:') + if verbose: + check_display_queue.append(lambda **kwargs: echo_info('Valid spec', **kwargs)) + for display in check_display_queue: + display(indent=True) + + specs_failed = len(specs_failed) + files_failed = len(files_failed) + files_passed = num_files - files_failed + + if specs_failed or files_failed: + click.echo() + + if specs_failed: + echo_failure(f'Specs with errors: {specs_failed}') + + if files_failed: + echo_failure(f'Files with errors: {files_failed}') + + if files_passed: + if specs_failed or files_failed: + echo_success(f'Files valid: {files_passed}') + else: + echo_success(f'All {num_files} data model files are in sync!') + + if specs_failed or files_failed: + abort() diff --git a/datadog_checks_dev/datadog_checks/dev/tooling/create.py b/datadog_checks_dev/datadog_checks/dev/tooling/create.py index 6e65a8fdf9230..9a3461cf0ba03 100644 --- a/datadog_checks_dev/datadog_checks/dev/tooling/create.py +++ b/datadog_checks_dev/datadog_checks/dev/tooling/create.py @@ -2,7 +2,6 @@ # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) import os -from datetime import datetime from uuid import uuid4 from ..utils import ( @@ -15,7 +14,7 @@ write_file, write_file_binary, ) -from .utils import kebab_case_name, normalize_package_name +from .utils import get_license_header, kebab_case_name, normalize_package_name TEMPLATES_DIR = path_join(os.path.dirname(os.path.abspath(__file__)), 'templates', 'integration') BINARY_EXTENSIONS = ('.png',) @@ -56,11 +55,7 @@ def construct_template_fields(integration_name, repo_choice, **kwargs): 'The {integration_name} check is included in the [Datadog Agent][2] package.\n' 'No additional installation is needed on your server.'.format(integration_name=integration_name) ) - license_header = ( - '# (C) Datadog, Inc. {year}-present\n' - '# All rights reserved\n' - '# Licensed under a 3-clause BSD style license (see LICENSE)'.format(year=str(datetime.utcnow().year)) - ) + license_header = get_license_header() support_type = 'core' test_dev_dep = '-e ../datadog_checks_dev' tox_base_dep = '-e../datadog_checks_base[deps]' diff --git a/datadog_checks_dev/datadog_checks/dev/tooling/specs/configuration/consumers/__init__.py b/datadog_checks_dev/datadog_checks/dev/tooling/specs/configuration/consumers/__init__.py index 0d5e18c66958b..32fbfe76e5750 100644 --- a/datadog_checks_dev/datadog_checks/dev/tooling/specs/configuration/consumers/__init__.py +++ b/datadog_checks_dev/datadog_checks/dev/tooling/specs/configuration/consumers/__init__.py @@ -2,3 +2,4 @@ # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) from .example import ExampleConsumer +from .model import ModelConsumer diff --git a/datadog_checks_dev/datadog_checks/dev/tooling/specs/configuration/consumers/model.py b/datadog_checks_dev/datadog_checks/dev/tooling/specs/configuration/consumers/model.py new file mode 100644 index 0000000000000..d2386af13a758 --- /dev/null +++ b/datadog_checks_dev/datadog_checks/dev/tooling/specs/configuration/consumers/model.py @@ -0,0 +1,314 @@ +# (C) Datadog, Inc. 2021-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from collections import defaultdict + +import yaml +from datamodel_code_generator.format import CodeFormatter, PythonVersion +from datamodel_code_generator.parser import LiteralType +from datamodel_code_generator.parser.openapi import OpenAPIParser + +from ..utils import sanitize_openapi_object_properties + +PYTHON_VERSION = PythonVersion.PY_38 +EXTRA_TYPE_FIELDS = ('compact_example', 'default', 'display_default', 'example') + +# Singleton allowing `None` to be a valid default value +NO_DEFAULT = object() + + +def normalize_option_name(option_name): + return option_name.replace('-', '_') + + +def example_looks_informative(example): + return '<' in example and '>' in example and example == example.upper() + + +def get_default_value(type_data): + if 'default' in type_data: + return type_data['default'] + elif 'type' not in type_data or type_data['type'] in ('array', 'object'): + return NO_DEFAULT + + example = type_data['example'] + if type_data['type'] == 'string': + if example_looks_informative(example): + return NO_DEFAULT + elif isinstance(example, str): + return NO_DEFAULT + + return example + + +def add_imports(model_file_lines, need_defaults): + import_lines = [] + + for i, line in enumerate(model_file_lines): + if line.startswith('from '): + import_lines.append(i) + + # pydantic imports + final_import_line = import_lines[-1] + model_file_lines[final_import_line] += ', root_validator, validator' + + local_import_start_location = final_import_line + 1 + for line in reversed( + ( + '', + 'from datadog_checks.base.utils.functions import identity', + 'from datadog_checks.base.utils.models import validation', + '', + 'from . import defaults, validators' if need_defaults else 'from . import validators', + ) + ): + model_file_lines.insert(local_import_start_location, line) + + +class ModelConsumer: + def __init__(self, spec, code_formatter=None): + self.spec = spec + self.code_formatter = code_formatter or self.create_code_formatter() + + def render(self): + files = {} + + for file in self.spec['files']: + # (, (, )) + model_files = {} + + root_imports = [] + defaults_file_lines = [] + deprecation_data = defaultdict(dict) + defaults_file_needs_dynamic_values = False + defaults_file_needs_value_normalization = False + + for section in sorted(file['options'], key=lambda s: s['name']): + errors = [] + + section_name = section['name'] + if section_name == 'init_config': + model_id = 'shared' + model_file_name = f'{model_id}.py' + schema_name = 'SharedConfig' + elif section_name == 'instances': + model_id = 'instance' + model_file_name = f'{model_id}.py' + schema_name = 'InstanceConfig' + # Skip anything checks don't use directly + else: + continue + + root_imports.append(f'from .{model_id} import {schema_name}') + + # We want to create something like: + # + # components: + # schemas: + # InstanceConfig: + # required: + # - endpoint + # properties: + # endpoint: + # ... + # timeout: + # ... + # ... + openapi_document = {'components': {'schemas': {}}} + schema = openapi_document['components']['schemas'][schema_name] = {} + + options = schema['properties'] = {} + required_options = schema['required'] = [] + options_with_defaults = False + validator_data = [] + + for option in sorted(section['options'], key=lambda o: o['name']): + option_name = option['name'] + normalized_option_name = normalize_option_name(option_name) + + if 'value' in option: + type_data = option['value'] + # Some integrations (like `mysql`) have options that are grouped under a top-level option + elif 'options' in option: + nested_properties = [] + type_data = {'type': 'object', 'properties': nested_properties} + for nested_option in option['options']: + nested_type_data = nested_option['value'] + + # Remove fields that aren't part of the OpenAPI specification + for extra_field in EXTRA_TYPE_FIELDS: + nested_type_data.pop(extra_field, None) + + nested_properties.append({'name': nested_option['name'], **nested_type_data}) + else: + errors.append(f'Option `{option_name}` must have a `value` or `options` attribute') + continue + + if option['deprecation']: + deprecation_data[model_id][option_name] = option['deprecation'] + + options[option_name] = type_data + + if option['required']: + required_options.append(option_name) + else: + options_with_defaults = True + defaults_file_lines.append('') + defaults_file_lines.append('') + defaults_file_lines.append(f'def {model_id}_{normalized_option_name}(field, value):') + + default_value = get_default_value(type_data) + if default_value is not NO_DEFAULT: + defaults_file_needs_value_normalization = True + defaults_file_lines.append(f' return {default_value!r}') + else: + defaults_file_needs_dynamic_values = True + defaults_file_lines.append(' return get_default_field_value(field, value)') + + validators = type_data.pop('validators', []) + if not isinstance(validators, list): + errors.append(f'Config spec property `{option_name}.value.validators` must be an array') + elif validators: + for i, import_path in enumerate(validators, 1): + if not isinstance(import_path, str): + errors.append( + f'Entry #{i} of config spec property `{option_name}.value.validators` ' + f'must be a string' + ) + break + else: + validator_data.append((normalized_option_name, validators)) + + # Remove fields that aren't part of the OpenAPI specification + for extra_field in EXTRA_TYPE_FIELDS: + type_data.pop(extra_field, None) + + sanitize_openapi_object_properties(type_data) + + try: + parser = OpenAPIParser( + yaml.safe_dump(openapi_document), + target_python_version=PythonVersion.PY_38, + enum_field_as_literal=LiteralType.All, + encoding='utf-8', + use_generic_container_types=True, + enable_faux_immutability=True, + # TODO: uncomment when the Agent upgrades Python to 3.9 + # use_standard_collections=True, + strip_default_none=True, + # https://github.com/koxudaxi/datamodel-code-generator/pull/173 + field_constraints=True, + ) + model_file_contents = parser.parse() + except Exception as e: + errors.append(f'Error parsing the OpenAPI schema `{schema_name}`: {e}') + model_files[model_file_name] = ('', errors) + continue + + model_file_lines = model_file_contents.splitlines() + add_imports(model_file_lines, options_with_defaults) + + model_file_lines.append('') + model_file_lines.append(' @root_validator(pre=True)') + model_file_lines.append(' def _initial_validation(cls, values):') + model_file_lines.append( + f" return validation.core.initialize_config(" + f"getattr(validators, 'initialize_{model_id}', identity)(values))" + ) + + model_file_lines.append('') + model_file_lines.append(" @validator('*', pre=True, always=True)") + model_file_lines.append(' def _ensure_defaults(cls, v, field):') + model_file_lines.append(' if v is not None or field.required:') + model_file_lines.append(' return v') + model_file_lines.append('') + model_file_lines.append(f" return getattr(defaults, f'{model_id}_{{field.name}}')(field, v)") + + model_file_lines.append('') + model_file_lines.append(" @validator('*')") + model_file_lines.append(' def _run_validations(cls, v, field):') + # TODO: remove conditional when there is a workaround: + # https://github.com/samuelcolvin/pydantic/issues/2376 + model_file_lines.append(' if not v:') + model_file_lines.append(' return v') + model_file_lines.append('') + model_file_lines.append( + f" return getattr(validators, f'{model_id}_{{field.name}}', identity)(v, field=field)" + ) + + for option_name, import_paths in validator_data: + for import_path in import_paths: + validator_name = import_path.replace('.', '_') + + model_file_lines.append('') + model_file_lines.append(f' @validator({option_name!r})') + model_file_lines.append(f' def _run_{option_name}_{validator_name}(cls, v, field):') + # TODO: remove conditional when there is a workaround: + # https://github.com/samuelcolvin/pydantic/issues/2376 + model_file_lines.append(' if not v:') + model_file_lines.append(' return v') + model_file_lines.append('') + model_file_lines.append(f' return validation.{import_path}(v, field=field)') + + model_file_lines.append('') + model_file_lines.append(' @root_validator(pre=False)') + model_file_lines.append(' def _final_validation(cls, values):') + model_file_lines.append( + f" return validation.core.finalize_config(" + f"getattr(validators, 'finalize_{model_id}', identity)(values))" + ) + + model_file_lines.append('') + model_file_contents = '\n'.join(model_file_lines) + if any(len(line) > 120 for line in model_file_lines): + model_file_contents = self.code_formatter.apply_black(model_file_contents) + + model_files[model_file_name] = (model_file_contents, errors) + + # Logs-only integrations + if not model_files: + continue + + if defaults_file_lines: + if defaults_file_needs_dynamic_values: + defaults_file_lines.insert( + 0, 'from datadog_checks.base.utils.models.fields import get_default_field_value' + ) + + defaults_file_lines.append('') + defaults_file_contents = '\n'.join(defaults_file_lines) + if defaults_file_needs_value_normalization: + defaults_file_contents = self.code_formatter.apply_black(defaults_file_contents) + + model_files['defaults.py'] = (defaults_file_contents, []) + + if deprecation_data: + file_needs_formatting = False + deprecations_file_lines = [] + for model_id, deprecations in deprecation_data.items(): + deprecations_file_lines.append('') + deprecations_file_lines.append('') + deprecations_file_lines.append(f'def {model_id}():') + deprecations_file_lines.append(f' return {deprecations!r}') + if len(deprecations_file_lines[-1]) > 120: + file_needs_formatting = True + + deprecations_file_lines.append('') + deprecations_file_contents = '\n'.join(deprecations_file_lines) + if file_needs_formatting: + deprecations_file_contents = self.code_formatter.apply_black(deprecations_file_contents) + + model_files['deprecations.py'] = (deprecations_file_contents, []) + + root_imports.sort() + root_imports.append('') + model_files['__init__.py'] = ('\n'.join(root_imports), []) + model_files['validators.py'] = ('', []) + + files[file['name']] = {file_name: model_files[file_name] for file_name in sorted(model_files)} + + return files + + @staticmethod + def create_code_formatter(): + return CodeFormatter(PYTHON_VERSION) diff --git a/datadog_checks_dev/datadog_checks/dev/tooling/specs/configuration/spec.py b/datadog_checks_dev/datadog_checks/dev/tooling/specs/configuration/spec.py index a0aeed193c35c..a06cc6ae98186 100644 --- a/datadog_checks_dev/datadog_checks/dev/tooling/specs/configuration/spec.py +++ b/datadog_checks_dev/datadog_checks/dev/tooling/specs/configuration/spec.py @@ -1,7 +1,8 @@ # (C) Datadog, Inc. 2019-present # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) -from ..utils import default_option_example, normalize_source_name +from ..utils import normalize_source_name +from .utils import default_option_example def spec_validator(spec, loader): diff --git a/datadog_checks_dev/datadog_checks/dev/tooling/specs/configuration/utils.py b/datadog_checks_dev/datadog_checks/dev/tooling/specs/configuration/utils.py new file mode 100644 index 0000000000000..6acb8799fd6a3 --- /dev/null +++ b/datadog_checks_dev/datadog_checks/dev/tooling/specs/configuration/utils.py @@ -0,0 +1,35 @@ +# (C) Datadog, Inc. 2021-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + + +def default_option_example(option_name): + return f'<{option_name.upper()}>' + + +def sanitize_openapi_object_properties(value): + if 'anyOf' in value: + for data in value['anyOf']: + sanitize_openapi_object_properties(data) + + return + + value_type = value['type'] + if value_type == 'array': + sanitize_openapi_object_properties(value['items']) + elif value_type == 'object': + spec_properties = value.pop('properties') + properties = value['properties'] = {} + + # The config spec `properties` object modifier is not a map, but rather a list of maps with a + # required `name` attribute. This is so consumers will load objects consistently regardless of + # language guarantees regarding map key order. + for spec_prop in spec_properties: + name = spec_prop.pop('name') + properties[name] = spec_prop + sanitize_openapi_object_properties(spec_prop) + + if 'additionalProperties' in value: + additional_properties = value['additionalProperties'] + if isinstance(additional_properties, dict): + sanitize_openapi_object_properties(additional_properties) diff --git a/datadog_checks_dev/datadog_checks/dev/tooling/specs/utils.py b/datadog_checks_dev/datadog_checks/dev/tooling/specs/utils.py index ab51d8d231d2e..1603757692166 100644 --- a/datadog_checks_dev/datadog_checks/dev/tooling/specs/utils.py +++ b/datadog_checks_dev/datadog_checks/dev/tooling/specs/utils.py @@ -3,9 +3,5 @@ # Licensed under a 3-clause BSD style license (see LICENSE) -def default_option_example(option_name): - return f'<{option_name.upper()}>' - - def normalize_source_name(source_name): return source_name.lower().replace(' ', '_') diff --git a/datadog_checks_dev/datadog_checks/dev/tooling/templates/integration/check/{check_name}/datadog_checks/{check_name}/config_models/__init__.py b/datadog_checks_dev/datadog_checks/dev/tooling/templates/integration/check/{check_name}/datadog_checks/{check_name}/config_models/__init__.py new file mode 100644 index 0000000000000..23f5fbd5a5999 --- /dev/null +++ b/datadog_checks_dev/datadog_checks/dev/tooling/templates/integration/check/{check_name}/datadog_checks/{check_name}/config_models/__init__.py @@ -0,0 +1,3 @@ +{license_header} +from .instance import InstanceConfig +from .shared import SharedConfig diff --git a/datadog_checks_dev/datadog_checks/dev/tooling/templates/integration/check/{check_name}/datadog_checks/{check_name}/config_models/defaults.py b/datadog_checks_dev/datadog_checks/dev/tooling/templates/integration/check/{check_name}/datadog_checks/{check_name}/config_models/defaults.py new file mode 100644 index 0000000000000..4d209aa7f0d63 --- /dev/null +++ b/datadog_checks_dev/datadog_checks/dev/tooling/templates/integration/check/{check_name}/datadog_checks/{check_name}/config_models/defaults.py @@ -0,0 +1,22 @@ +{license_header} +from datadog_checks.base.utils.models.fields import get_default_field_value + + +def shared_service(field, value): + return get_default_field_value(field, value) + + +def instance_empty_default_hostname(field, value): + return False + + +def instance_min_collection_interval(field, value): + return 15 + + +def instance_service(field, value): + return get_default_field_value(field, value) + + +def instance_tags(field, value): + return get_default_field_value(field, value) diff --git a/datadog_checks_dev/datadog_checks/dev/tooling/templates/integration/check/{check_name}/datadog_checks/{check_name}/config_models/instance.py b/datadog_checks_dev/datadog_checks/dev/tooling/templates/integration/check/{check_name}/datadog_checks/{check_name}/config_models/instance.py new file mode 100644 index 0000000000000..53d6d742b3a4b --- /dev/null +++ b/datadog_checks_dev/datadog_checks/dev/tooling/templates/integration/check/{check_name}/datadog_checks/{check_name}/config_models/instance.py @@ -0,0 +1,43 @@ +{license_header} +from __future__ import annotations + +from typing import Optional, Sequence + +from pydantic import BaseModel, root_validator, validator + +from datadog_checks.base.utils.functions import identity +from datadog_checks.base.utils.models import validation + +from . import defaults, validators + + +class InstanceConfig(BaseModel): + class Config: + allow_mutation = False + + empty_default_hostname: Optional[bool] + min_collection_interval: Optional[float] + service: Optional[str] + tags: Optional[Sequence[str]] + + @root_validator(pre=True) + def _initial_validation(cls, values): + return validation.core.initialize_config(getattr(validators, 'initialize_instance', identity)(values)) + + @validator('*', pre=True, always=True) + def _ensure_defaults(cls, v, field): + if v is not None or field.required: + return v + + return getattr(defaults, f'instance_{{field.name}}')(field, v) + + @validator('*') + def _run_validations(cls, v, field): + if not v: + return v + + return getattr(validators, f'instance_{{field.name}}', identity)(v, field=field) + + @root_validator(pre=False) + def _final_validation(cls, values): + return validation.core.finalize_config(getattr(validators, 'finalize_instance', identity)(values)) diff --git a/datadog_checks_dev/datadog_checks/dev/tooling/templates/integration/check/{check_name}/datadog_checks/{check_name}/config_models/shared.py b/datadog_checks_dev/datadog_checks/dev/tooling/templates/integration/check/{check_name}/datadog_checks/{check_name}/config_models/shared.py new file mode 100644 index 0000000000000..82dd52936ff40 --- /dev/null +++ b/datadog_checks_dev/datadog_checks/dev/tooling/templates/integration/check/{check_name}/datadog_checks/{check_name}/config_models/shared.py @@ -0,0 +1,40 @@ +{license_header} +from __future__ import annotations + +from typing import Optional + +from pydantic import BaseModel, root_validator, validator + +from datadog_checks.base.utils.functions import identity +from datadog_checks.base.utils.models import validation + +from . import defaults, validators + + +class SharedConfig(BaseModel): + class Config: + allow_mutation = False + + service: Optional[str] + + @root_validator(pre=True) + def _initial_validation(cls, values): + return validation.core.initialize_config(getattr(validators, 'initialize_shared', identity)(values)) + + @validator('*', pre=True, always=True) + def _ensure_defaults(cls, v, field): + if v is not None or field.required: + return v + + return getattr(defaults, f'shared_{{field.name}}')(field, v) + + @validator('*') + def _run_validations(cls, v, field): + if not v: + return v + + return getattr(validators, f'shared_{{field.name}}', identity)(v, field=field) + + @root_validator(pre=False) + def _final_validation(cls, values): + return validation.core.finalize_config(getattr(validators, 'finalize_shared', identity)(values)) diff --git a/datadog_checks_dev/datadog_checks/dev/tooling/templates/integration/check/{check_name}/datadog_checks/{check_name}/config_models/validators.py b/datadog_checks_dev/datadog_checks/dev/tooling/templates/integration/check/{check_name}/datadog_checks/{check_name}/config_models/validators.py new file mode 100644 index 0000000000000..4ad5a0451cb35 --- /dev/null +++ b/datadog_checks_dev/datadog_checks/dev/tooling/templates/integration/check/{check_name}/datadog_checks/{check_name}/config_models/validators.py @@ -0,0 +1 @@ +{license_header} diff --git a/datadog_checks_dev/datadog_checks/dev/tooling/templates/integration/jmx/{check_name}/datadog_checks/{check_name}/config_models/__init__.py b/datadog_checks_dev/datadog_checks/dev/tooling/templates/integration/jmx/{check_name}/datadog_checks/{check_name}/config_models/__init__.py new file mode 100644 index 0000000000000..23f5fbd5a5999 --- /dev/null +++ b/datadog_checks_dev/datadog_checks/dev/tooling/templates/integration/jmx/{check_name}/datadog_checks/{check_name}/config_models/__init__.py @@ -0,0 +1,3 @@ +{license_header} +from .instance import InstanceConfig +from .shared import SharedConfig diff --git a/datadog_checks_dev/datadog_checks/dev/tooling/templates/integration/jmx/{check_name}/datadog_checks/{check_name}/config_models/defaults.py b/datadog_checks_dev/datadog_checks/dev/tooling/templates/integration/jmx/{check_name}/datadog_checks/{check_name}/config_models/defaults.py new file mode 100644 index 0000000000000..b8e2589d3dfb9 --- /dev/null +++ b/datadog_checks_dev/datadog_checks/dev/tooling/templates/integration/jmx/{check_name}/datadog_checks/{check_name}/config_models/defaults.py @@ -0,0 +1,98 @@ +{license_header} +from datadog_checks.base.utils.models.fields import get_default_field_value + + +def shared_conf(field, value): + return get_default_field_value(field, value) + + +def shared_new_gc_metrics(field, value): + return False + + +def shared_service(field, value): + return get_default_field_value(field, value) + + +def shared_service_check_prefix(field, value): + return get_default_field_value(field, value) + + +def instance_collect_default_jvm_metrics(field, value): + return True + + +def instance_empty_default_hostname(field, value): + return False + + +def instance_java_bin_path(field, value): + return get_default_field_value(field, value) + + +def instance_java_options(field, value): + return get_default_field_value(field, value) + + +def instance_jmx_url(field, value): + return get_default_field_value(field, value) + + +def instance_key_store_password(field, value): + return get_default_field_value(field, value) + + +def instance_key_store_path(field, value): + return get_default_field_value(field, value) + + +def instance_min_collection_interval(field, value): + return 15 + + +def instance_name(field, value): + return get_default_field_value(field, value) + + +def instance_password(field, value): + return get_default_field_value(field, value) + + +def instance_process_name_regex(field, value): + return get_default_field_value(field, value) + + +def instance_rmi_client_timeout(field, value): + return 15000 + + +def instance_rmi_connection_timeout(field, value): + return 20000 + + +def instance_rmi_registry_ssl(field, value): + return False + + +def instance_service(field, value): + return get_default_field_value(field, value) + + +def instance_tags(field, value): + return get_default_field_value(field, value) + + +def instance_tools_jar_path(field, value): + return get_default_field_value(field, value) + + +def instance_trust_store_password(field, value): + return get_default_field_value(field, value) + + +def instance_trust_store_path(field, value): + return get_default_field_value(field, value) + + +def instance_user(field, value): + return get_default_field_value(field, value) diff --git a/datadog_checks_dev/datadog_checks/dev/tooling/templates/integration/jmx/{check_name}/datadog_checks/{check_name}/config_models/instance.py b/datadog_checks_dev/datadog_checks/dev/tooling/templates/integration/jmx/{check_name}/datadog_checks/{check_name}/config_models/instance.py new file mode 100644 index 0000000000000..b9925dcc4677b --- /dev/null +++ b/datadog_checks_dev/datadog_checks/dev/tooling/templates/integration/jmx/{check_name}/datadog_checks/{check_name}/config_models/instance.py @@ -0,0 +1,61 @@ +{license_header} +from __future__ import annotations + +from typing import Optional, Sequence + +from pydantic import BaseModel, root_validator, validator + +from datadog_checks.base.utils.functions import identity +from datadog_checks.base.utils.models import validation + +from . import defaults, validators + + +class InstanceConfig(BaseModel): + class Config: + allow_mutation = False + + collect_default_jvm_metrics: Optional[bool] + empty_default_hostname: Optional[bool] + host: str + java_bin_path: Optional[str] + java_options: Optional[str] + jmx_url: Optional[str] + key_store_password: Optional[str] + key_store_path: Optional[str] + min_collection_interval: Optional[float] + name: Optional[str] + password: Optional[str] + port: int + process_name_regex: Optional[str] + rmi_client_timeout: Optional[float] + rmi_connection_timeout: Optional[float] + rmi_registry_ssl: Optional[bool] + service: Optional[str] + tags: Optional[Sequence[str]] + tools_jar_path: Optional[str] + trust_store_password: Optional[str] + trust_store_path: Optional[str] + user: Optional[str] + + @root_validator(pre=True) + def _initial_validation(cls, values): + return validation.core.initialize_config(getattr(validators, 'initialize_instance', identity)(values)) + + @validator('*', pre=True, always=True) + def _ensure_defaults(cls, v, field): + if v is not None or field.required: + return v + + return getattr(defaults, f'instance_{{field.name}}')(field, v) + + @validator('*') + def _run_validations(cls, v, field): + if not v: + return v + + return getattr(validators, f'instance_{{field.name}}', identity)(v, field=field) + + @root_validator(pre=False) + def _final_validation(cls, values): + return validation.core.finalize_config(getattr(validators, 'finalize_instance', identity)(values)) diff --git a/datadog_checks_dev/datadog_checks/dev/tooling/templates/integration/jmx/{check_name}/datadog_checks/{check_name}/config_models/shared.py b/datadog_checks_dev/datadog_checks/dev/tooling/templates/integration/jmx/{check_name}/datadog_checks/{check_name}/config_models/shared.py new file mode 100644 index 0000000000000..0218d2f79b316 --- /dev/null +++ b/datadog_checks_dev/datadog_checks/dev/tooling/templates/integration/jmx/{check_name}/datadog_checks/{check_name}/config_models/shared.py @@ -0,0 +1,45 @@ +{license_header} +from __future__ import annotations + +from typing import Any, Mapping, Optional, Sequence + +from pydantic import BaseModel, root_validator, validator + +from datadog_checks.base.utils.functions import identity +from datadog_checks.base.utils.models import validation + +from . import defaults, validators + + +class SharedConfig(BaseModel): + class Config: + allow_mutation = False + + collect_default_metrics: bool + conf: Optional[Sequence[Mapping[str, Any]]] + is_jmx: bool + new_gc_metrics: Optional[bool] + service: Optional[str] + service_check_prefix: Optional[str] + + @root_validator(pre=True) + def _initial_validation(cls, values): + return validation.core.initialize_config(getattr(validators, 'initialize_shared', identity)(values)) + + @validator('*', pre=True, always=True) + def _ensure_defaults(cls, v, field): + if v is not None or field.required: + return v + + return getattr(defaults, f'shared_{{field.name}}')(field, v) + + @validator('*') + def _run_validations(cls, v, field): + if not v: + return v + + return getattr(validators, f'shared_{{field.name}}', identity)(v, field=field) + + @root_validator(pre=False) + def _final_validation(cls, values): + return validation.core.finalize_config(getattr(validators, 'finalize_shared', identity)(values)) diff --git a/datadog_checks_dev/datadog_checks/dev/tooling/templates/integration/jmx/{check_name}/datadog_checks/{check_name}/config_models/validators.py b/datadog_checks_dev/datadog_checks/dev/tooling/templates/integration/jmx/{check_name}/datadog_checks/{check_name}/config_models/validators.py new file mode 100644 index 0000000000000..4ad5a0451cb35 --- /dev/null +++ b/datadog_checks_dev/datadog_checks/dev/tooling/templates/integration/jmx/{check_name}/datadog_checks/{check_name}/config_models/validators.py @@ -0,0 +1 @@ +{license_header} diff --git a/datadog_checks_dev/datadog_checks/dev/tooling/utils.py b/datadog_checks_dev/datadog_checks/dev/tooling/utils.py index 327f1d4508586..da92a66cc423c 100644 --- a/datadog_checks_dev/datadog_checks/dev/tooling/utils.py +++ b/datadog_checks_dev/datadog_checks/dev/tooling/utils.py @@ -7,6 +7,7 @@ import os import re from ast import literal_eval +from datetime import datetime, timezone from json.decoder import JSONDecodeError import requests @@ -91,6 +92,14 @@ ) +def get_license_header(): + return ( + '# (C) Datadog, Inc. {year}-present\n' + '# All rights reserved\n' + '# Licensed under a 3-clause BSD style license (see LICENSE)'.format(year=str(datetime.now(timezone.utc).year)) + ) + + def format_commit_id(commit_id): if commit_id: if commit_id.isdigit(): diff --git a/datadog_checks_dev/setup.py b/datadog_checks_dev/setup.py index 21f1c491abc0e..2a6c3fbb01a1b 100644 --- a/datadog_checks_dev/setup.py +++ b/datadog_checks_dev/setup.py @@ -73,6 +73,7 @@ 'atomicwrites', 'click>7', 'colorama', + 'datamodel-code-generator~=0.7.3', 'docker-compose>=1.25', 'in-toto>=0.4.2', 'jsonschema', diff --git a/datadog_checks_dev/tests/tooling/configuration/consumers/model/__init__.py b/datadog_checks_dev/tests/tooling/configuration/consumers/model/__init__.py new file mode 100644 index 0000000000000..9d0b0155542cb --- /dev/null +++ b/datadog_checks_dev/tests/tooling/configuration/consumers/model/__init__.py @@ -0,0 +1,3 @@ +# (C) Datadog, Inc. 2021-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) diff --git a/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_all_required.py b/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_all_required.py new file mode 100644 index 0000000000000..eccfc0dff4398 --- /dev/null +++ b/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_all_required.py @@ -0,0 +1,89 @@ +# (C) Datadog, Inc. 2021-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import pytest + +from ...utils import get_model_consumer, normalize_yaml + +pytestmark = [pytest.mark.conf, pytest.mark.conf_consumer, pytest.mark.conf_consumer_model] + + +def test(): + consumer = get_model_consumer( + """ + name: test + version: 0.0.0 + files: + - name: test.yaml + options: + - template: instances + options: + - name: foo + required: true + description: words + value: + type: string + """ + ) + + model_definitions = consumer.render() + assert len(model_definitions) == 1 + + files = model_definitions['test.yaml'] + assert len(files) == 3 + + validators_contents, validators_errors = files['validators.py'] + assert not validators_errors + assert validators_contents == '' + + package_root_contents, package_root_errors = files['__init__.py'] + assert not package_root_errors + assert package_root_contents == normalize_yaml( + """ + from .instance import InstanceConfig + """ + ) + + instance_model_contents, instance_model_errors = files['instance.py'] + assert not instance_model_errors + assert instance_model_contents == normalize_yaml( + """ + from __future__ import annotations + + from pydantic import BaseModel, root_validator, validator + + from datadog_checks.base.utils.functions import identity + from datadog_checks.base.utils.models import validation + + from . import validators + + + class InstanceConfig(BaseModel): + class Config: + allow_mutation = False + + foo: str + + @root_validator(pre=True) + def _initial_validation(cls, values): + return validation.core.initialize_config(getattr(validators, 'initialize_instance', identity)(values)) + + @validator('*', pre=True, always=True) + def _ensure_defaults(cls, v, field): + if v is not None or field.required: + return v + + return getattr(defaults, f'instance_{field.name}')(field, v) + + @validator('*') + def _run_validations(cls, v, field): + if not v: + return v + + return getattr(validators, f'instance_{field.name}', identity)(v, field=field) + + @root_validator(pre=False) + def _final_validation(cls, values): + return validation.core.finalize_config(getattr(validators, 'finalize_instance', identity)(values)) + """ + ) diff --git a/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_array.py b/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_array.py new file mode 100644 index 0000000000000..7f7f19b962988 --- /dev/null +++ b/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_array.py @@ -0,0 +1,110 @@ +# (C) Datadog, Inc. 2021-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import pytest + +from ...utils import get_model_consumer, normalize_yaml + +pytestmark = [pytest.mark.conf, pytest.mark.conf_consumer, pytest.mark.conf_consumer_model] + + +def test(): + consumer = get_model_consumer( + """ + name: test + version: 0.0.0 + files: + - name: test.yaml + options: + - template: instances + options: + - name: foo + required: true + description: words + value: + type: string + - name: tags + description: words + value: + type: array + items: + type: string + """ + ) + + model_definitions = consumer.render() + assert len(model_definitions) == 1 + + files = model_definitions['test.yaml'] + assert len(files) == 4 + + validators_contents, validators_errors = files['validators.py'] + assert not validators_errors + assert validators_contents == '' + + package_root_contents, package_root_errors = files['__init__.py'] + assert not package_root_errors + assert package_root_contents == normalize_yaml( + """ + from .instance import InstanceConfig + """ + ) + + defaults_contents, defaults_errors = files['defaults.py'] + assert not defaults_errors + assert defaults_contents == normalize_yaml( + """ + from datadog_checks.base.utils.models.fields import get_default_field_value + + + def instance_tags(field, value): + return get_default_field_value(field, value) + """ + ) + + instance_model_contents, instance_model_errors = files['instance.py'] + assert not instance_model_errors + assert instance_model_contents == normalize_yaml( + """ + from __future__ import annotations + + from typing import Optional, Sequence + + from pydantic import BaseModel, root_validator, validator + + from datadog_checks.base.utils.functions import identity + from datadog_checks.base.utils.models import validation + + from . import defaults, validators + + + class InstanceConfig(BaseModel): + class Config: + allow_mutation = False + + foo: str + tags: Optional[Sequence[str]] + + @root_validator(pre=True) + def _initial_validation(cls, values): + return validation.core.initialize_config(getattr(validators, 'initialize_instance', identity)(values)) + + @validator('*', pre=True, always=True) + def _ensure_defaults(cls, v, field): + if v is not None or field.required: + return v + + return getattr(defaults, f'instance_{field.name}')(field, v) + + @validator('*') + def _run_validations(cls, v, field): + if not v: + return v + + return getattr(validators, f'instance_{field.name}', identity)(v, field=field) + + @root_validator(pre=False) + def _final_validation(cls, values): + return validation.core.finalize_config(getattr(validators, 'finalize_instance', identity)(values)) + """ + ) diff --git a/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_both_models_basic.py b/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_both_models_basic.py new file mode 100644 index 0000000000000..23dbc5621a3a0 --- /dev/null +++ b/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_both_models_basic.py @@ -0,0 +1,159 @@ +# (C) Datadog, Inc. 2021-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import pytest + +from ...utils import get_model_consumer, normalize_yaml + +pytestmark = [pytest.mark.conf, pytest.mark.conf_consumer, pytest.mark.conf_consumer_model] + + +def test(): + consumer = get_model_consumer( + """ + name: test + version: 0.0.0 + files: + - name: test.yaml + options: + - template: init_config + options: + - name: foo + description: words + value: + type: string + - template: instances + options: + - name: foo + description: words + value: + type: string + """ + ) + + model_definitions = consumer.render() + assert len(model_definitions) == 1 + + files = model_definitions['test.yaml'] + assert len(files) == 5 + + validators_contents, validators_errors = files['validators.py'] + assert not validators_errors + assert validators_contents == '' + + package_root_contents, package_root_errors = files['__init__.py'] + assert not package_root_errors + assert package_root_contents == normalize_yaml( + """ + from .instance import InstanceConfig + from .shared import SharedConfig + """ + ) + + defaults_contents, defaults_errors = files['defaults.py'] + assert not defaults_errors + assert defaults_contents == normalize_yaml( + """ + from datadog_checks.base.utils.models.fields import get_default_field_value + + + def shared_foo(field, value): + return get_default_field_value(field, value) + + + def instance_foo(field, value): + return get_default_field_value(field, value) + """ + ) + + shared_model_contents, shared_model_errors = files['shared.py'] + assert not shared_model_errors + assert shared_model_contents == normalize_yaml( + """ + from __future__ import annotations + + from typing import Optional + + from pydantic import BaseModel, root_validator, validator + + from datadog_checks.base.utils.functions import identity + from datadog_checks.base.utils.models import validation + + from . import defaults, validators + + + class SharedConfig(BaseModel): + class Config: + allow_mutation = False + + foo: Optional[str] + + @root_validator(pre=True) + def _initial_validation(cls, values): + return validation.core.initialize_config(getattr(validators, 'initialize_shared', identity)(values)) + + @validator('*', pre=True, always=True) + def _ensure_defaults(cls, v, field): + if v is not None or field.required: + return v + + return getattr(defaults, f'shared_{field.name}')(field, v) + + @validator('*') + def _run_validations(cls, v, field): + if not v: + return v + + return getattr(validators, f'shared_{field.name}', identity)(v, field=field) + + @root_validator(pre=False) + def _final_validation(cls, values): + return validation.core.finalize_config(getattr(validators, 'finalize_shared', identity)(values)) + """ + ) + + instance_model_contents, instance_model_errors = files['instance.py'] + assert not instance_model_errors + assert instance_model_contents == normalize_yaml( + """ + from __future__ import annotations + + from typing import Optional + + from pydantic import BaseModel, root_validator, validator + + from datadog_checks.base.utils.functions import identity + from datadog_checks.base.utils.models import validation + + from . import defaults, validators + + + class InstanceConfig(BaseModel): + class Config: + allow_mutation = False + + foo: Optional[str] + + @root_validator(pre=True) + def _initial_validation(cls, values): + return validation.core.initialize_config(getattr(validators, 'initialize_instance', identity)(values)) + + @validator('*', pre=True, always=True) + def _ensure_defaults(cls, v, field): + if v is not None or field.required: + return v + + return getattr(defaults, f'instance_{field.name}')(field, v) + + @validator('*') + def _run_validations(cls, v, field): + if not v: + return v + + return getattr(validators, f'instance_{field.name}', identity)(v, field=field) + + @root_validator(pre=False) + def _final_validation(cls, values): + return validation.core.finalize_config(getattr(validators, 'finalize_instance', identity)(values)) + """ + ) diff --git a/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_common_validators.py b/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_common_validators.py new file mode 100644 index 0000000000000..d05d87c8fef77 --- /dev/null +++ b/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_common_validators.py @@ -0,0 +1,144 @@ +# (C) Datadog, Inc. 2021-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import pytest + +from ...utils import get_model_consumer, normalize_yaml + +pytestmark = [pytest.mark.conf, pytest.mark.conf_consumer, pytest.mark.conf_consumer_model] + + +def test(): + consumer = get_model_consumer( + """ + name: test + version: 0.0.0 + files: + - name: test.yaml + options: + - template: instances + options: + - name: foo + required: true + description: words + value: + type: string + validators: + - pkg.subpkg2.validate2 + - pkg.subpkg2.validate1 + - name: tags + description: words + value: + type: array + items: + type: string + validators: + - pkg.subpkg1.validate2 + - pkg.subpkg1.validate1 + """ + ) + + model_definitions = consumer.render() + assert len(model_definitions) == 1 + + files = model_definitions['test.yaml'] + assert len(files) == 4 + + validators_contents, validators_errors = files['validators.py'] + assert not validators_errors + assert validators_contents == '' + + package_root_contents, package_root_errors = files['__init__.py'] + assert not package_root_errors + assert package_root_contents == normalize_yaml( + """ + from .instance import InstanceConfig + """ + ) + + defaults_contents, defaults_errors = files['defaults.py'] + assert not defaults_errors + assert defaults_contents == normalize_yaml( + """ + from datadog_checks.base.utils.models.fields import get_default_field_value + + + def instance_tags(field, value): + return get_default_field_value(field, value) + """ + ) + + instance_model_contents, instance_model_errors = files['instance.py'] + assert not instance_model_errors + assert instance_model_contents == normalize_yaml( + """ + from __future__ import annotations + + from typing import Optional, Sequence + + from pydantic import BaseModel, root_validator, validator + + from datadog_checks.base.utils.functions import identity + from datadog_checks.base.utils.models import validation + + from . import defaults, validators + + + class InstanceConfig(BaseModel): + class Config: + allow_mutation = False + + foo: str + tags: Optional[Sequence[str]] + + @root_validator(pre=True) + def _initial_validation(cls, values): + return validation.core.initialize_config(getattr(validators, 'initialize_instance', identity)(values)) + + @validator('*', pre=True, always=True) + def _ensure_defaults(cls, v, field): + if v is not None or field.required: + return v + + return getattr(defaults, f'instance_{field.name}')(field, v) + + @validator('*') + def _run_validations(cls, v, field): + if not v: + return v + + return getattr(validators, f'instance_{field.name}', identity)(v, field=field) + + @validator('foo') + def _run_foo_pkg_subpkg2_validate2(cls, v, field): + if not v: + return v + + return validation.pkg.subpkg2.validate2(v, field=field) + + @validator('foo') + def _run_foo_pkg_subpkg2_validate1(cls, v, field): + if not v: + return v + + return validation.pkg.subpkg2.validate1(v, field=field) + + @validator('tags') + def _run_tags_pkg_subpkg1_validate2(cls, v, field): + if not v: + return v + + return validation.pkg.subpkg1.validate2(v, field=field) + + @validator('tags') + def _run_tags_pkg_subpkg1_validate1(cls, v, field): + if not v: + return v + + return validation.pkg.subpkg1.validate1(v, field=field) + + @root_validator(pre=False) + def _final_validation(cls, values): + return validation.core.finalize_config(getattr(validators, 'finalize_instance', identity)(values)) + """ + ) diff --git a/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_defaults.py b/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_defaults.py new file mode 100644 index 0000000000000..aefc088789d72 --- /dev/null +++ b/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_defaults.py @@ -0,0 +1,170 @@ +# (C) Datadog, Inc. 2021-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import pytest + +from ...utils import get_model_consumer, normalize_yaml + +pytestmark = [pytest.mark.conf, pytest.mark.conf_consumer, pytest.mark.conf_consumer_model] + + +def test(): + consumer = get_model_consumer( + """ + name: test + version: 0.0.0 + files: + - name: test.yaml + options: + - template: instances + options: + - name: foo + required: true + description: words + value: + type: string + - name: example + description: words + value: + example: bar + type: string + - name: default_precedence + description: words + value: + example: bar + default: baz + type: string + - name: example_ignored_array + description: words + value: + example: + - test + type: array + items: + type: string + - name: example_ignored_object + description: words + value: + example: + key: value + type: object + additionalProperties: true + - name: long_default_formatted + description: words + value: + default: + - ["01", "02", "03", "04", "05"] + - ["06", "07", "08", "09", "10"] + - ["11", "12", "13", "14", "15"] + - ["16", "17", "18", "19", "20"] + - ["21", "22", "23", "24", "25"] + type: array + items: + type: array + items: + type: string + """ + ) + + model_definitions = consumer.render() + assert len(model_definitions) == 1 + + files = model_definitions['test.yaml'] + assert len(files) == 4 + + validators_contents, validators_errors = files['validators.py'] + assert not validators_errors + assert validators_contents == '' + + package_root_contents, package_root_errors = files['__init__.py'] + assert not package_root_errors + assert package_root_contents == normalize_yaml( + """ + from .instance import InstanceConfig + """ + ) + + defaults_contents, defaults_errors = files['defaults.py'] + assert not defaults_errors + assert defaults_contents == normalize_yaml( + """ + from datadog_checks.base.utils.models.fields import get_default_field_value + + + def instance_default_precedence(field, value): + return 'baz' + + + def instance_example(field, value): + return 'bar' + + + def instance_example_ignored_array(field, value): + return get_default_field_value(field, value) + + + def instance_example_ignored_object(field, value): + return get_default_field_value(field, value) + + + def instance_long_default_formatted(field, value): + return [ + ['01', '02', '03', '04', '05'], + ['06', '07', '08', '09', '10'], + ['11', '12', '13', '14', '15'], + ['16', '17', '18', '19', '20'], + ['21', '22', '23', '24', '25'], + ] + """ + ) + + instance_model_contents, instance_model_errors = files['instance.py'] + assert not instance_model_errors + assert instance_model_contents == normalize_yaml( + """ + from __future__ import annotations + + from typing import Any, Mapping, Optional, Sequence + + from pydantic import BaseModel, root_validator, validator + + from datadog_checks.base.utils.functions import identity + from datadog_checks.base.utils.models import validation + + from . import defaults, validators + + + class InstanceConfig(BaseModel): + class Config: + allow_mutation = False + + default_precedence: Optional[str] + example: Optional[str] + example_ignored_array: Optional[Sequence[str]] + example_ignored_object: Optional[Mapping[str, Any]] + foo: str + long_default_formatted: Optional[Sequence[Sequence[str]]] + + @root_validator(pre=True) + def _initial_validation(cls, values): + return validation.core.initialize_config(getattr(validators, 'initialize_instance', identity)(values)) + + @validator('*', pre=True, always=True) + def _ensure_defaults(cls, v, field): + if v is not None or field.required: + return v + + return getattr(defaults, f'instance_{field.name}')(field, v) + + @validator('*') + def _run_validations(cls, v, field): + if not v: + return v + + return getattr(validators, f'instance_{field.name}', identity)(v, field=field) + + @root_validator(pre=False) + def _final_validation(cls, values): + return validation.core.finalize_config(getattr(validators, 'finalize_instance', identity)(values)) + """ + ) diff --git a/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_nested_option.py b/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_nested_option.py new file mode 100644 index 0000000000000..889b57054b330 --- /dev/null +++ b/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_nested_option.py @@ -0,0 +1,140 @@ +# (C) Datadog, Inc. 2021-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import pytest + +from ...utils import get_model_consumer, normalize_yaml + +pytestmark = [pytest.mark.conf, pytest.mark.conf_consumer, pytest.mark.conf_consumer_model] + + +def test(): + consumer = get_model_consumer( + """ + name: test + version: 0.0.0 + files: + - name: test.yaml + options: + - template: instances + options: + - name: foo + required: true + description: words + value: + type: string + - name: settings + description: words + options: + - name: setting1 + description: words + value: + type: string + - name: setting2 + description: words + value: + type: object + required: + - bar + properties: + - name: bar + type: string + - name: baz + type: array + items: + type: string + """ + ) + + model_definitions = consumer.render() + assert len(model_definitions) == 1 + + files = model_definitions['test.yaml'] + assert len(files) == 4 + + validators_contents, validators_errors = files['validators.py'] + assert not validators_errors + assert validators_contents == '' + + package_root_contents, package_root_errors = files['__init__.py'] + assert not package_root_errors + assert package_root_contents == normalize_yaml( + """ + from .instance import InstanceConfig + """ + ) + + defaults_contents, defaults_errors = files['defaults.py'] + assert not defaults_errors + assert defaults_contents == normalize_yaml( + """ + from datadog_checks.base.utils.models.fields import get_default_field_value + + + def instance_settings(field, value): + return get_default_field_value(field, value) + """ + ) + + instance_model_contents, instance_model_errors = files['instance.py'] + assert not instance_model_errors + assert instance_model_contents == normalize_yaml( + """ + from __future__ import annotations + + from typing import Optional, Sequence + + from pydantic import BaseModel, root_validator, validator + + from datadog_checks.base.utils.functions import identity + from datadog_checks.base.utils.models import validation + + from . import defaults, validators + + + class Setting2(BaseModel): + class Config: + allow_mutation = False + + bar: str + baz: Optional[Sequence[str]] + + + class Settings(BaseModel): + class Config: + allow_mutation = False + + setting1: Optional[str] + setting2: Optional[Setting2] + + + class InstanceConfig(BaseModel): + class Config: + allow_mutation = False + + foo: str + settings: Optional[Settings] + + @root_validator(pre=True) + def _initial_validation(cls, values): + return validation.core.initialize_config(getattr(validators, 'initialize_instance', identity)(values)) + + @validator('*', pre=True, always=True) + def _ensure_defaults(cls, v, field): + if v is not None or field.required: + return v + + return getattr(defaults, f'instance_{field.name}')(field, v) + + @validator('*') + def _run_validations(cls, v, field): + if not v: + return v + + return getattr(validators, f'instance_{field.name}', identity)(v, field=field) + + @root_validator(pre=False) + def _final_validation(cls, values): + return validation.core.finalize_config(getattr(validators, 'finalize_instance', identity)(values)) + """ + ) diff --git a/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_no_models.py b/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_no_models.py new file mode 100644 index 0000000000000..70fb85c3d1dec --- /dev/null +++ b/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_no_models.py @@ -0,0 +1,24 @@ +# (C) Datadog, Inc. 2021-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import pytest + +from ...utils import get_model_consumer + +pytestmark = [pytest.mark.conf, pytest.mark.conf_consumer, pytest.mark.conf_consumer_model] + + +def test(): + consumer = get_model_consumer( + """ + name: test + version: 0.0.0 + files: + - name: test.yaml + options: + - template: logs + """ + ) + + model_definitions = consumer.render() + assert not model_definitions diff --git a/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_object_arbitrary_values.py b/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_object_arbitrary_values.py new file mode 100644 index 0000000000000..e40679b93a90e --- /dev/null +++ b/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_object_arbitrary_values.py @@ -0,0 +1,109 @@ +# (C) Datadog, Inc. 2021-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import pytest + +from ...utils import get_model_consumer, normalize_yaml + +pytestmark = [pytest.mark.conf, pytest.mark.conf_consumer, pytest.mark.conf_consumer_model] + + +def test(): + consumer = get_model_consumer( + """ + name: test + version: 0.0.0 + files: + - name: test.yaml + options: + - template: instances + options: + - name: foo + required: true + description: words + value: + type: string + - name: obj + description: words + value: + type: object + additionalProperties: true + """ + ) + + model_definitions = consumer.render() + assert len(model_definitions) == 1 + + files = model_definitions['test.yaml'] + assert len(files) == 4 + + validators_contents, validators_errors = files['validators.py'] + assert not validators_errors + assert validators_contents == '' + + package_root_contents, package_root_errors = files['__init__.py'] + assert not package_root_errors + assert package_root_contents == normalize_yaml( + """ + from .instance import InstanceConfig + """ + ) + + defaults_contents, defaults_errors = files['defaults.py'] + assert not defaults_errors + assert defaults_contents == normalize_yaml( + """ + from datadog_checks.base.utils.models.fields import get_default_field_value + + + def instance_obj(field, value): + return get_default_field_value(field, value) + """ + ) + + instance_model_contents, instance_model_errors = files['instance.py'] + assert not instance_model_errors + assert instance_model_contents == normalize_yaml( + """ + from __future__ import annotations + + from typing import Any, Mapping, Optional + + from pydantic import BaseModel, root_validator, validator + + from datadog_checks.base.utils.functions import identity + from datadog_checks.base.utils.models import validation + + from . import defaults, validators + + + class InstanceConfig(BaseModel): + class Config: + allow_mutation = False + + foo: str + obj: Optional[Mapping[str, Any]] + + @root_validator(pre=True) + def _initial_validation(cls, values): + return validation.core.initialize_config(getattr(validators, 'initialize_instance', identity)(values)) + + @validator('*', pre=True, always=True) + def _ensure_defaults(cls, v, field): + if v is not None or field.required: + return v + + return getattr(defaults, f'instance_{field.name}')(field, v) + + @validator('*') + def _run_validations(cls, v, field): + if not v: + return v + + return getattr(validators, f'instance_{field.name}', identity)(v, field=field) + + @root_validator(pre=False) + def _final_validation(cls, values): + return validation.core.finalize_config(getattr(validators, 'finalize_instance', identity)(values)) + """ + ) diff --git a/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_object_model.py b/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_object_model.py new file mode 100644 index 0000000000000..f3ee942a82c3a --- /dev/null +++ b/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_object_model.py @@ -0,0 +1,125 @@ +# (C) Datadog, Inc. 2021-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import pytest + +from ...utils import get_model_consumer, normalize_yaml + +pytestmark = [pytest.mark.conf, pytest.mark.conf_consumer, pytest.mark.conf_consumer_model] + + +def test(): + consumer = get_model_consumer( + """ + name: test + version: 0.0.0 + files: + - name: test.yaml + options: + - template: instances + options: + - name: foo + required: true + description: words + value: + type: string + - name: obj + description: words + value: + type: object + required: + - bar + properties: + - name: bar + type: string + - name: baz + type: array + items: + type: string + """ + ) + + model_definitions = consumer.render() + assert len(model_definitions) == 1 + + files = model_definitions['test.yaml'] + assert len(files) == 4 + + validators_contents, validators_errors = files['validators.py'] + assert not validators_errors + assert validators_contents == '' + + package_root_contents, package_root_errors = files['__init__.py'] + assert not package_root_errors + assert package_root_contents == normalize_yaml( + """ + from .instance import InstanceConfig + """ + ) + + defaults_contents, defaults_errors = files['defaults.py'] + assert not defaults_errors + assert defaults_contents == normalize_yaml( + """ + from datadog_checks.base.utils.models.fields import get_default_field_value + + + def instance_obj(field, value): + return get_default_field_value(field, value) + """ + ) + + instance_model_contents, instance_model_errors = files['instance.py'] + assert not instance_model_errors + assert instance_model_contents == normalize_yaml( + """ + from __future__ import annotations + + from typing import Optional, Sequence + + from pydantic import BaseModel, root_validator, validator + + from datadog_checks.base.utils.functions import identity + from datadog_checks.base.utils.models import validation + + from . import defaults, validators + + + class Obj(BaseModel): + class Config: + allow_mutation = False + + bar: str + baz: Optional[Sequence[str]] + + + class InstanceConfig(BaseModel): + class Config: + allow_mutation = False + + foo: str + obj: Optional[Obj] + + @root_validator(pre=True) + def _initial_validation(cls, values): + return validation.core.initialize_config(getattr(validators, 'initialize_instance', identity)(values)) + + @validator('*', pre=True, always=True) + def _ensure_defaults(cls, v, field): + if v is not None or field.required: + return v + + return getattr(defaults, f'instance_{field.name}')(field, v) + + @validator('*') + def _run_validations(cls, v, field): + if not v: + return v + + return getattr(validators, f'instance_{field.name}', identity)(v, field=field) + + @root_validator(pre=False) + def _final_validation(cls, values): + return validation.core.finalize_config(getattr(validators, 'finalize_instance', identity)(values)) + """ + ) diff --git a/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_object_typed_values.py b/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_object_typed_values.py new file mode 100644 index 0000000000000..2995a1f2160df --- /dev/null +++ b/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_object_typed_values.py @@ -0,0 +1,116 @@ +# (C) Datadog, Inc. 2021-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import pytest + +from ...utils import get_model_consumer, normalize_yaml + +pytestmark = [pytest.mark.conf, pytest.mark.conf_consumer, pytest.mark.conf_consumer_model] + + +def test(): + consumer = get_model_consumer( + """ + name: test + version: 0.0.0 + files: + - name: test.yaml + options: + - template: instances + options: + - name: foo + required: true + description: words + value: + type: string + - name: obj + description: words + value: + type: object + additionalProperties: + type: array + items: + type: number + """ + ) + + model_definitions = consumer.render() + assert len(model_definitions) == 1 + + files = model_definitions['test.yaml'] + assert len(files) == 4 + + validators_contents, validators_errors = files['validators.py'] + assert not validators_errors + assert validators_contents == '' + + package_root_contents, package_root_errors = files['__init__.py'] + assert not package_root_errors + assert package_root_contents == normalize_yaml( + """ + from .instance import InstanceConfig + """ + ) + + defaults_contents, defaults_errors = files['defaults.py'] + assert not defaults_errors + assert defaults_contents == normalize_yaml( + """ + from datadog_checks.base.utils.models.fields import get_default_field_value + + + def instance_obj(field, value): + return get_default_field_value(field, value) + """ + ) + + instance_model_contents, instance_model_errors = files['instance.py'] + assert not instance_model_errors + assert instance_model_contents == normalize_yaml( + """ + from __future__ import annotations + + from typing import Mapping, Optional, Sequence + + from pydantic import BaseModel, root_validator, validator + + from datadog_checks.base.utils.functions import identity + from datadog_checks.base.utils.models import validation + + from . import defaults, validators + + + class Obj(BaseModel): + __root__: Sequence[float] + + + class InstanceConfig(BaseModel): + class Config: + allow_mutation = False + + foo: str + obj: Optional[Mapping[str, Obj]] + + @root_validator(pre=True) + def _initial_validation(cls, values): + return validation.core.initialize_config(getattr(validators, 'initialize_instance', identity)(values)) + + @validator('*', pre=True, always=True) + def _ensure_defaults(cls, v, field): + if v is not None or field.required: + return v + + return getattr(defaults, f'instance_{field.name}')(field, v) + + @validator('*') + def _run_validations(cls, v, field): + if not v: + return v + + return getattr(validators, f'instance_{field.name}', identity)(v, field=field) + + @root_validator(pre=False) + def _final_validation(cls, values): + return validation.core.finalize_config(getattr(validators, 'finalize_instance', identity)(values)) + """ + ) diff --git a/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_only_shared.py b/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_only_shared.py new file mode 100644 index 0000000000000..16b522f717dc7 --- /dev/null +++ b/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_only_shared.py @@ -0,0 +1,102 @@ +# (C) Datadog, Inc. 2021-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import pytest + +from ...utils import get_model_consumer, normalize_yaml + +pytestmark = [pytest.mark.conf, pytest.mark.conf_consumer, pytest.mark.conf_consumer_model] + + +def test(): + consumer = get_model_consumer( + """ + name: test + version: 0.0.0 + files: + - name: test.yaml + options: + - template: init_config + options: + - name: foo + description: words + value: + type: string + """ + ) + + model_definitions = consumer.render() + assert len(model_definitions) == 1 + + files = model_definitions['test.yaml'] + assert len(files) == 4 + + validators_contents, validators_errors = files['validators.py'] + assert not validators_errors + assert validators_contents == '' + + package_root_contents, package_root_errors = files['__init__.py'] + assert not package_root_errors + assert package_root_contents == normalize_yaml( + """ + from .shared import SharedConfig + """ + ) + + defaults_contents, defaults_errors = files['defaults.py'] + assert not defaults_errors + assert defaults_contents == normalize_yaml( + """ + from datadog_checks.base.utils.models.fields import get_default_field_value + + + def shared_foo(field, value): + return get_default_field_value(field, value) + """ + ) + + shared_model_contents, shared_model_errors = files['shared.py'] + assert not shared_model_errors + assert shared_model_contents == normalize_yaml( + """ + from __future__ import annotations + + from typing import Optional + + from pydantic import BaseModel, root_validator, validator + + from datadog_checks.base.utils.functions import identity + from datadog_checks.base.utils.models import validation + + from . import defaults, validators + + + class SharedConfig(BaseModel): + class Config: + allow_mutation = False + + foo: Optional[str] + + @root_validator(pre=True) + def _initial_validation(cls, values): + return validation.core.initialize_config(getattr(validators, 'initialize_shared', identity)(values)) + + @validator('*', pre=True, always=True) + def _ensure_defaults(cls, v, field): + if v is not None or field.required: + return v + + return getattr(defaults, f'shared_{field.name}')(field, v) + + @validator('*') + def _run_validations(cls, v, field): + if not v: + return v + + return getattr(validators, f'shared_{field.name}', identity)(v, field=field) + + @root_validator(pre=False) + def _final_validation(cls, values): + return validation.core.finalize_config(getattr(validators, 'finalize_shared', identity)(values)) + """ + ) diff --git a/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_option_name_normalization.py b/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_option_name_normalization.py new file mode 100644 index 0000000000000..2fd72b24eade6 --- /dev/null +++ b/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_option_name_normalization.py @@ -0,0 +1,108 @@ +# (C) Datadog, Inc. 2021-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import pytest + +from ...utils import get_model_consumer, normalize_yaml + +pytestmark = [pytest.mark.conf, pytest.mark.conf_consumer, pytest.mark.conf_consumer_model] + + +def test(): + consumer = get_model_consumer( + """ + name: test + version: 0.0.0 + files: + - name: test.yaml + options: + - template: instances + options: + - name: foo + required: true + description: words + value: + type: string + - name: bar-baz + description: words + value: + type: string + """ + ) + + model_definitions = consumer.render() + assert len(model_definitions) == 1 + + files = model_definitions['test.yaml'] + assert len(files) == 4 + + validators_contents, validators_errors = files['validators.py'] + assert not validators_errors + assert validators_contents == '' + + package_root_contents, package_root_errors = files['__init__.py'] + assert not package_root_errors + assert package_root_contents == normalize_yaml( + """ + from .instance import InstanceConfig + """ + ) + + defaults_contents, defaults_errors = files['defaults.py'] + assert not defaults_errors + assert defaults_contents == normalize_yaml( + """ + from datadog_checks.base.utils.models.fields import get_default_field_value + + + def instance_bar_baz(field, value): + return get_default_field_value(field, value) + """ + ) + + instance_model_contents, instance_model_errors = files['instance.py'] + assert not instance_model_errors + assert instance_model_contents == normalize_yaml( + """ + from __future__ import annotations + + from typing import Optional + + from pydantic import BaseModel, Field, root_validator, validator + + from datadog_checks.base.utils.functions import identity + from datadog_checks.base.utils.models import validation + + from . import defaults, validators + + + class InstanceConfig(BaseModel): + class Config: + allow_mutation = False + + bar_baz: Optional[str] = Field(None, alias='bar-baz') + foo: str + + @root_validator(pre=True) + def _initial_validation(cls, values): + return validation.core.initialize_config(getattr(validators, 'initialize_instance', identity)(values)) + + @validator('*', pre=True, always=True) + def _ensure_defaults(cls, v, field): + if v is not None or field.required: + return v + + return getattr(defaults, f'instance_{field.name}')(field, v) + + @validator('*') + def _run_validations(cls, v, field): + if not v: + return v + + return getattr(validators, f'instance_{field.name}', identity)(v, field=field) + + @root_validator(pre=False) + def _final_validation(cls, values): + return validation.core.finalize_config(getattr(validators, 'finalize_instance', identity)(values)) + """ + ) diff --git a/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_union_types.py b/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_union_types.py new file mode 100644 index 0000000000000..bf65edcdf4bd1 --- /dev/null +++ b/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_union_types.py @@ -0,0 +1,112 @@ +# (C) Datadog, Inc. 2021-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import pytest + +from ...utils import get_model_consumer, normalize_yaml + +pytestmark = [pytest.mark.conf, pytest.mark.conf_consumer, pytest.mark.conf_consumer_model] + + +def test(): + consumer = get_model_consumer( + """ + name: test + version: 0.0.0 + files: + - name: test.yaml + options: + - template: instances + options: + - name: foo + required: true + description: words + value: + type: string + - name: obj + description: words + value: + anyOf: + - type: string + - type: array + items: + type: string + """ + ) + + model_definitions = consumer.render() + assert len(model_definitions) == 1 + + files = model_definitions['test.yaml'] + assert len(files) == 4 + + validators_contents, validators_errors = files['validators.py'] + assert not validators_errors + assert validators_contents == '' + + package_root_contents, package_root_errors = files['__init__.py'] + assert not package_root_errors + assert package_root_contents == normalize_yaml( + """ + from .instance import InstanceConfig + """ + ) + + defaults_contents, defaults_errors = files['defaults.py'] + assert not defaults_errors + assert defaults_contents == normalize_yaml( + """ + from datadog_checks.base.utils.models.fields import get_default_field_value + + + def instance_obj(field, value): + return get_default_field_value(field, value) + """ + ) + + instance_model_contents, instance_model_errors = files['instance.py'] + assert not instance_model_errors + assert instance_model_contents == normalize_yaml( + """ + from __future__ import annotations + + from typing import Optional, Sequence, Union + + from pydantic import BaseModel, root_validator, validator + + from datadog_checks.base.utils.functions import identity + from datadog_checks.base.utils.models import validation + + from . import defaults, validators + + + class InstanceConfig(BaseModel): + class Config: + allow_mutation = False + + foo: str + obj: Optional[Union[str, Sequence[str]]] + + @root_validator(pre=True) + def _initial_validation(cls, values): + return validation.core.initialize_config(getattr(validators, 'initialize_instance', identity)(values)) + + @validator('*', pre=True, always=True) + def _ensure_defaults(cls, v, field): + if v is not None or field.required: + return v + + return getattr(defaults, f'instance_{field.name}')(field, v) + + @validator('*') + def _run_validations(cls, v, field): + if not v: + return v + + return getattr(validators, f'instance_{field.name}', identity)(v, field=field) + + @root_validator(pre=False) + def _final_validation(cls, values): + return validation.core.finalize_config(getattr(validators, 'finalize_instance', identity)(values)) + """ + ) diff --git a/datadog_checks_dev/tests/tooling/configuration/consumers/test_example.py b/datadog_checks_dev/tests/tooling/configuration/consumers/test_example.py index b5e1c54d89c3d..f6143f4b47c2b 100644 --- a/datadog_checks_dev/tests/tooling/configuration/consumers/test_example.py +++ b/datadog_checks_dev/tests/tooling/configuration/consumers/test_example.py @@ -8,7 +8,7 @@ from ..utils import get_example_consumer, normalize_yaml -pytestmark = [pytest.mark.conf, pytest.mark.conf_consumer] +pytestmark = [pytest.mark.conf, pytest.mark.conf_consumer, pytest.mark.conf_consumer_example] def test_option_no_section(): @@ -1490,6 +1490,9 @@ def test_option_multiple_types_nested(): anyOf: - type: string - type: object + properties: + - name: foo + type: string required: - foo """ diff --git a/datadog_checks_dev/tests/tooling/configuration/utils.py b/datadog_checks_dev/tests/tooling/configuration/utils.py index d3e252773a758..c5acee7ba3b2e 100644 --- a/datadog_checks_dev/tests/tooling/configuration/utils.py +++ b/datadog_checks_dev/tests/tooling/configuration/utils.py @@ -4,7 +4,7 @@ from textwrap import dedent from datadog_checks.dev.tooling.specs.configuration import ConfigSpec -from datadog_checks.dev.tooling.specs.configuration.consumers import ExampleConsumer +from datadog_checks.dev.tooling.specs.configuration.consumers import ExampleConsumer, ModelConsumer def get_spec(text, **kwargs): @@ -15,8 +15,16 @@ def get_spec(text, **kwargs): def get_example_consumer(text, **kwargs): spec = get_spec(text, **kwargs) spec.load() + assert not spec.errors return ExampleConsumer(spec.data) +def get_model_consumer(text, **kwargs): + spec = get_spec(text, **kwargs) + spec.load() + assert not spec.errors + return ModelConsumer(spec.data) + + def normalize_yaml(text): return dedent(text).lstrip()