-
Notifications
You must be signed in to change notification settings - Fork 1.4k
/
test.py
120 lines (107 loc) · 4.57 KB
/
test.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
# (C) Datadog, Inc. 2019-present
# All rights reserved
# Licensed under a 3-clause BSD style license (see LICENSE)
import click
from .... import EnvVars
from ...e2e import create_interface, get_configured_envs
from ...e2e.agent import DEFAULT_PYTHON_VERSION
from ...testing import complete_active_checks, get_tox_envs
from ..console import CONTEXT_SETTINGS, DEBUG_OUTPUT, echo_info, echo_warning
from ..test import test as test_command
from .start import start
from .stop import stop
@click.command(context_settings=CONTEXT_SETTINGS, short_help='Test an environment')
@click.argument('checks', shell_complete=complete_active_checks, nargs=-1)
@click.option(
'--agent',
'-a',
help=(
'The agent build to use e.g. a Docker image like `datadog/agent:latest`. You can '
'also use the name of an agent defined in the `agents` configuration section.'
),
)
@click.option(
'--python',
'-py',
type=click.INT,
help=f'The version of Python to use. Defaults to {DEFAULT_PYTHON_VERSION} if no tox Python is specified.',
)
@click.option('--dev/--prod', default=None, help='Whether to use the latest version of a check or what is shipped')
@click.option('--base', is_flag=True, help='Whether to use the latest version of the base check or what is shipped')
@click.option(
'--env-vars',
'-e',
multiple=True,
help=(
'ENV Variable that should be passed to the Agent container. '
'Ex: -e DD_URL=app.datadoghq.com -e DD_API_KEY=123456'
),
)
@click.option('--new-env', '-ne', is_flag=True, help='Execute setup and tear down actions')
@click.option('--profile-memory', '-pm', is_flag=True, help='Whether to collect metrics about memory usage')
@click.option('--junit', '-j', 'junit', is_flag=True, help='Generate junit reports')
@click.option('--ddtrace', is_flag=True, help='Run tests using dd-trace-py')
@click.option('--filter', '-k', 'test_filter', help='Only run tests matching given substring expression')
@click.option('--changed', is_flag=True, help='Only test changed checks')
@click.pass_context
def test(
ctx, checks, agent, python, dev, base, env_vars, new_env, profile_memory, junit, ddtrace, test_filter, changed
):
"""Test an environment."""
check_envs = get_tox_envs(checks, e2e_tests_only=True, changed_only=changed)
tests_ran = False
# If no checks or a subset of checks are specified it means we're testing what has changed compared
# to master, probably on CI rather than during local development. In this case,
# ensure environments and Agents are spun up and down.
if not checks or changed:
new_env = True
# Default to testing the local development version.
if dev is None:
dev = True
if profile_memory and not new_env:
echo_warning('Ignoring --profile-memory, to utilize that you must also select --new-env')
for check, envs in check_envs:
if not envs:
echo_warning(
f'No end-to-end environments found for `{check}`. Make sure to add them to the {check}/tox.ini file.'
)
continue
config_envs = get_configured_envs(check)
# For performance reasons we're generating what to test on the fly and therefore
# need a way to tell if anything ran since we don't know anything upfront.
tests_ran = True
for env in envs:
if new_env:
ctx.invoke(
start,
check=check,
env=env,
agent=agent,
python=python,
dev=dev,
base=base,
env_vars=env_vars,
profile_memory=profile_memory,
)
elif env not in config_envs:
continue
environment = create_interface(check, env)
persisted_env_vars = environment.metadata.get('env_vars', {})
try:
with EnvVars(persisted_env_vars):
ctx.invoke(
test_command,
checks=[f'{check}:{env}'],
debug=DEBUG_OUTPUT,
e2e=True,
passenv=' '.join(persisted_env_vars) if persisted_env_vars else None,
junit=junit,
ddtrace=ddtrace,
test_filter=test_filter,
latest=env == 'latest',
)
finally:
if new_env:
ctx.invoke(stop, check=check, env=env)
if not tests_ran:
echo_info('Nothing to test!')