-
Notifications
You must be signed in to change notification settings - Fork 1.4k
/
config.py
237 lines (200 loc) · 9.76 KB
/
config.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
# (C) Datadog, Inc. 2018-present
# All rights reserved
# Licensed under a 3-clause BSD style license (see LICENSE)
import difflib
import click
import yaml
from ....fs import basepath, file_exists, path_join, read_file, write_file
from ...config_validator.validator import validate_config
from ...config_validator.validator_errors import SEVERITY_ERROR, SEVERITY_WARNING
from ...configuration import ConfigSpec
from ...configuration.consumers import ExampleConsumer
from ...manifest_utils import Manifest
from ...testing import process_checks_option
from ...utils import complete_valid_checks, get_config_files, get_data_directory, get_version_string
from ..console import (
CONTEXT_SETTINGS,
abort,
annotate_error,
echo_debug,
echo_failure,
echo_info,
echo_success,
echo_waiting,
echo_warning,
)
FILE_INDENT = ' ' * 8
IGNORE_DEFAULT_INSTANCE = {'ceph', 'dotnetclr', 'gunicorn', 'marathon', 'pgbouncer', 'process', 'supervisord'}
TEMPLATES = ['default', 'openmetrics_legacy', 'openmetrics', 'jmx']
@click.command(context_settings=CONTEXT_SETTINGS, short_help='Validate default configuration files')
@click.argument('check', shell_complete=complete_valid_checks, required=False)
@click.option('--sync', '-s', is_flag=True, help='Generate example configuration files based on specifications')
@click.option('--verbose', '-v', is_flag=True, help='Verbose mode')
@click.pass_context
def config(ctx, check, sync, verbose):
"""Validate default configuration files.
If `check` is specified, only the check will be validated, if check value is 'changed' will only apply to changed
checks, an 'all' or empty `check` value will validate all README files.
"""
repo_choice = ctx.obj['repo_choice']
if repo_choice == 'agent':
checks = ['agent']
else:
checks = process_checks_option(check, source='valid_checks', extend_changed=True)
files_failed = {}
files_warned = {}
file_counter = []
echo_waiting(f'Validating default configuration files for {len(checks)} checks...')
for check in checks:
check_display_queue = []
manifest = Manifest.load_manifest(check)
if not manifest:
echo_debug(f"Skipping validation for check: {check}; can't process manifest")
continue
spec_file_path = manifest.get_config_spec()
if not file_exists(spec_file_path):
validate_config_legacy(check, check_display_queue, files_failed, files_warned, file_counter)
if verbose:
check_display_queue.append(lambda: echo_warning('No spec found', indent=True))
if check_display_queue:
echo_info(f'{check}:')
for display in check_display_queue:
display()
continue
file_counter.append(None)
# source is the default file name
if check == 'agent':
display_name = 'Datadog Agent'
source = 'datadog'
version = None
else:
display_name = manifest.get_display_name()
source = check
version = get_version_string(check)
spec_file_content = read_file(spec_file_path)
default_temp = validate_default_template(spec_file_content)
spec = ConfigSpec(spec_file_content, source=source, version=version)
spec.load()
if not default_temp:
message = "Missing default template in init_config or instances section"
check_display_queue.append(lambda **kwargs: echo_failure(message))
annotate_error(spec_file_path, message)
if spec.errors:
files_failed[spec_file_path] = True
for error in spec.errors:
check_display_queue.append(lambda error=error, **kwargs: echo_failure(error, **kwargs))
else:
if spec.data['name'] != display_name:
files_failed[spec_file_path] = True
message = f"Spec name `{spec.data['name']}` should be `{display_name}`"
check_display_queue.append(lambda **kwargs: echo_failure(message, **kwargs))
annotate_error(spec_file_path, message)
example_location = get_data_directory(check)
example_consumer = ExampleConsumer(spec.data)
for example_file, (contents, errors) in example_consumer.render().items():
file_counter.append(None)
example_file_path = path_join(example_location, example_file)
if errors:
files_failed[example_file_path] = True
for error in errors:
check_display_queue.append(lambda error=error, **kwargs: echo_failure(error, **kwargs))
else:
if not file_exists(example_file_path) or read_file(example_file_path) != contents:
if sync:
echo_info(f"Writing config file to `{example_file_path}`")
write_file(example_file_path, contents)
else:
files_failed[example_file_path] = True
message = f'File `{example_file}` is not in sync, run "ddev validate config {check} -s"'
if file_exists(example_file_path):
example_file = read_file(example_file_path)
for diff_line in difflib.context_diff(
example_file.splitlines(), contents.splitlines(), "current", "expected"
):
message += f'\n{diff_line}'
check_display_queue.append(
lambda example_file=example_file, **kwargs: echo_failure(message, **kwargs)
)
annotate_error(example_file_path, message)
if check_display_queue or verbose:
echo_info(f'{check}:')
if verbose:
check_display_queue.append(lambda **kwargs: echo_info('Valid spec', **kwargs))
for display in check_display_queue:
display(indent=True)
num_files = len(file_counter)
files_failed = len(files_failed)
files_warned = len(files_warned)
files_passed = num_files - (files_failed + files_warned)
if files_failed or files_warned:
click.echo()
if files_failed:
echo_failure(f'Files with errors: {files_failed}')
if files_warned:
echo_warning(f'Files with warnings: {files_warned}')
if files_passed:
if files_failed or files_warned:
echo_success(f'Files valid: {files_passed}')
else:
echo_success(f'All {num_files} configuration files are valid!')
if files_failed:
abort()
def validate_default_template(spec_file):
init_config_default = False
instances_default = False
if 'template: init_config' not in spec_file or 'template: instances' not in spec_file:
# This config spec does not have init_config or instances
return True
for line in spec_file.split('\n'):
if any("init_config/{}".format(template) in line for template in TEMPLATES):
init_config_default = True
if any("instances/{}".format(template) in line for template in TEMPLATES):
instances_default = True
if instances_default and init_config_default:
return True
return False
def validate_config_legacy(check, check_display_queue, files_failed, files_warned, file_counter):
config_files = get_config_files(check)
for config_file in config_files:
file_counter.append(None)
file_name = basepath(config_file)
try:
file_data = read_file(config_file)
config_data = yaml.safe_load(file_data)
except Exception as e:
files_failed[config_file] = True
# We must convert to text here to free Exception object before it goes out of scope
error = str(e)
check_display_queue.append(lambda: echo_info(f'{file_name}:', indent=True))
check_display_queue.append(lambda: echo_failure('Invalid YAML -', indent=FILE_INDENT))
check_display_queue.append(lambda: echo_info(error, indent=FILE_INDENT * 2))
continue
file_display_queue = []
errors = validate_config(file_data)
for err in errors:
err_msg = str(err)
if err.severity == SEVERITY_ERROR:
file_display_queue.append(lambda x=err_msg: echo_failure(x, indent=FILE_INDENT))
files_failed[config_file] = True
elif err.severity == SEVERITY_WARNING:
file_display_queue.append(lambda x=err_msg: echo_warning(x, indent=FILE_INDENT))
files_warned[config_file] = True
else:
file_display_queue.append(lambda x=err_msg: echo_info(x, indent=FILE_INDENT))
# Verify there is an `instances` section
if 'instances' not in config_data:
files_failed[config_file] = True
message = 'Missing `instances` section'
file_display_queue.append(lambda: echo_failure(message, indent=FILE_INDENT))
annotate_error(file_name, message)
# Verify there is a default instance
else:
instances = config_data['instances']
if check not in IGNORE_DEFAULT_INSTANCE and not isinstance(instances, list):
files_failed[config_file] = True
message = 'No default instance'
file_display_queue.append(lambda: echo_failure(message, indent=FILE_INDENT))
annotate_error(file_name, message)
if file_display_queue:
check_display_queue.append(lambda x=file_name: echo_info(f'{x}:', indent=True))
check_display_queue.extend(file_display_queue)