From 48ec712abe02f62368ee940796651593c4acbdd5 Mon Sep 17 00:00:00 2001 From: David Goffredo Date: Wed, 16 Aug 2023 19:39:54 -0400 Subject: [PATCH 1/2] tolerate more docker-compose quirks --- test/cases/formats.py | 16 ++++++++-------- test/cases/orchestration.py | 29 +++++++++++++++++++---------- 2 files changed, 27 insertions(+), 18 deletions(-) diff --git a/test/cases/formats.py b/test/cases/formats.py index 180d9340..43037e22 100644 --- a/test/cases/formats.py +++ b/test/cases/formats.py @@ -34,7 +34,7 @@ def parse_docker_compose_up_line(line): }) # begin_create_container: {container} - begin_create_container = r'(Rec|C)reating (?P\S+)\s*\.\.\.\s*' + begin_create_container = r'\s*(Rec|C)reating (?P\S+)\s*\.\.\.\s*' match = try_match(begin_create_container, line) if match is not None: return ('begin_create_container', { @@ -53,21 +53,21 @@ def parse_docker_compose_up_line(line): # begin_create_container and finish_create_container. # begin_create_container: {container} - match = try_match(r'Container (?P\S+)\s+Creating\s*', line) + match = try_match(r'\s*Container (?P\S+)\s+Creating\s*', line) if match is not None: return ('begin_create_container', { 'container': match.groupdict()['container'] }) # finish_create_container: {container} - match = try_match(r'Container (?P\S+)\s+Created\s*', line) + match = try_match(r'\s*Container (?P\S+)\s+Created\s*', line) if match is not None: return ('finish_create_container', { 'container': match.groupdict()['container'] }) # attach_to_logs: {'containers': [container, ...]} - match = try_match(r'Attaching to (?P\S+(, \S+)*\s*)', line) + match = try_match(r'\s*Attaching to (?P\S+(, \S+)*\s*)', line) if match is not None: return ('attach_to_logs', { 'containers': [ @@ -77,7 +77,7 @@ def parse_docker_compose_up_line(line): }) # image_build_success: {image} - match = try_match(r'Successfully built (?P\S+)\s*', line) + match = try_match(r'\s*Successfully built (?P\S+)\s*', line) if match is not None: return ('image_build_success', {'image': match.groupdict()['image']}) @@ -85,11 +85,11 @@ def parse_docker_compose_up_line(line): def parse_docker_compose_down_line(line): - match = try_match(r'Removing network (?P\S+)\n', line) + match = try_match(r'\s*Removing network (?P\S+)\n', line) if match is not None: return ('remove_network', {'network': match.groupdict()['network']}) - begin_stop_container = r'Stopping (?P\S+)\s*\.\.\.\s*' + begin_stop_container = r'\s*Stopping (?P\S+)\s*\.\.\.\s*' match = try_match(begin_stop_container, line) if match is not None: return ('begin_stop_container', { @@ -102,7 +102,7 @@ def parse_docker_compose_down_line(line): 'container': match.groupdict()['container'] }) - begin_remove_container = r'Removing (?P\S+)\s*\.\.\.\s*' + begin_remove_container = r'\s*Removing (?P\S+)\s*\.\.\.\s*' match = try_match(begin_remove_container, line) if match is not None: return ('begin_remove_container', { diff --git a/test/cases/orchestration.py b/test/cases/orchestration.py index cc01a569..0868cca8 100644 --- a/test/cases/orchestration.py +++ b/test/cases/orchestration.py @@ -165,12 +165,15 @@ def nginx_worker_pids(nginx_container, verbose_output): if re.match(r'\s*nginx: worker process', cmd)) -def with_retries(max_attempts, thunk): +def docker_compose_ps_with_retries(max_attempts, service): assert max_attempts > 0 while True: try: - return thunk() - except BaseException: + result = docker_compose_ps(service) + if result == '': + raise Exception(f'docker_compose_ps({json.dumps(service)}) returned an empty string') + return result + except Exception: max_attempts -= 1 if max_attempts == 0: raise @@ -240,7 +243,8 @@ def docker_compose_up(on_ready, logs, verbose_file): print('Not all services are ready. Going to wait', poll_seconds, 'seconds', - file=verbose_file) + file=verbose_file, + flush=True) time.sleep(poll_seconds) on_ready({'containers': containers}) elif kind == 'finish_create_container': @@ -250,12 +254,17 @@ def docker_compose_up(on_ready, logs, verbose_file): # Consult `docker-compose ps` for the service's container ID. # Since we're handling a finish_create_container event, you'd # think that the container would be available now. However, - # with CircleCI's remote docker setup, there's a race here - # where docker-compose does not yet know which container - # corresponds to `service` (even though it just told us that - # the container was created). So, we retry a few times. - containers[service] = with_retries( - 5, lambda: docker_compose_ps(service)) + # there's a race here where docker-compose does not yet know + # which container corresponds to `service` (even though it just + # told us that the container was created). + # + # Additionally, some `docker-compose ps` implementations exit + # with a nonzero (error) status when there is no container ID + # to report, while others exit with status zero (success) and + # don't print any output. So, we retry (up to a limit) until + # `docker-compose ps` exits with status zero and produces + # output. + containers[service] = docker_compose_ps_with_retries(max_attempts=100, service=service) elif kind == 'service_log': # Got a line of logging from some service. Push it onto the # appropriate queue for consumption by tests. From bcd80889f3c3f2bc83855d04cf7b72ed1a7061fc Mon Sep 17 00:00:00 2001 From: David Goffredo Date: Wed, 16 Aug 2023 19:54:09 -0400 Subject: [PATCH 2/2] =?UTF-8?q?docker-compose=20=20=E2=86=92=20=20docker?= =?UTF-8?q?=20compose?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- test/.gitignore | 1 - test/README.md | 6 ++-- test/bin/README.md | 2 +- test/bin/run | 2 +- test/bin/run_parallel | 4 +-- test/cases/README.md | 20 ++++++------ test/cases/case.py | 4 +-- test/cases/formats.py | 6 ++-- test/cases/orchestration.py | 54 +++++++++++++++----------------- test/docker-compose.yml | 10 +++--- test/services/client/README.md | 6 ++-- test/services/client/curljson.sh | 2 +- test/services/nginx/Dockerfile | 2 +- test/services/nginx/README.md | 2 +- 14 files changed, 59 insertions(+), 62 deletions(-) diff --git a/test/.gitignore b/test/.gitignore index 33ee72c9..66c57506 100644 --- a/test/.gitignore +++ b/test/.gitignore @@ -1,3 +1,2 @@ __pycache__/ -docker-compose-verbose.log test.times diff --git a/test/README.md b/test/README.md index 952b41dd..2d870b05 100644 --- a/test/README.md +++ b/test/README.md @@ -1,7 +1,7 @@ Tests ===== These are integration tests. They're written in python, and use -`docker-compose` to orchestrate an instance of nginx containing the module +`docker compose` to orchestrate an instance of nginx containing the module under test, and other services reverse proxied by nginx. See the readme file in [cases/](cases/) for usage information. @@ -11,8 +11,8 @@ Files - [bin/](bin/) contains scripts for running and developing the tests. Notably, [bin/run](bin/run) runs the tests. - [cases/](cases/) contains the actual python test cases that run tests against - the `docker-compose` setup. -- [services/](services/) contains the dockerfiles for the `docker-compose` + the `docker compose` setup. +- [services/](services/) contains the dockerfiles for the `docker compose` services, and other data relevant to the services. - [docker-compose.yaml](docker-compose.yml) defines the services used by the tests. diff --git a/test/bin/README.md b/test/bin/README.md index 1eaeb777..f86b56a6 100644 --- a/test/bin/README.md +++ b/test/bin/README.md @@ -8,4 +8,4 @@ These are scripts that are useful when working with the integration tests. unittest`, so for example you can run a subset of tests, or increase the verbosity of logging. - [run_parallel](run_parallel) is a wrapper around [run](run) that executes - each test Python module in its own docker-compose "project," all in parallel. + each test Python module in its own docker compose "project," all in parallel. diff --git a/test/bin/run b/test/bin/run index da3f53ad..b36c2dea 100755 --- a/test/bin/run +++ b/test/bin/run @@ -9,5 +9,5 @@ export BASE_IMAGE export NGINX_MODULES_PATH="${NGINX_MODULES_PATH:-/usr/lib/nginx/modules}" export NGINX_CONF_PATH="${NGINX_CONF_PATH:-/etc/nginx/nginx.conf}" -docker-compose build --parallel +docker compose build --parallel python3 -m unittest "$@" diff --git a/test/bin/run_parallel b/test/bin/run_parallel index c2b89555..fccc09fa 100755 --- a/test/bin/run_parallel +++ b/test/bin/run_parallel @@ -9,7 +9,7 @@ if [ "$BASE_IMAGE" = '' ]; then fi export NGINX_MODULES_PATH="${NGINX_MODULES_PATH:-/usr/lib/nginx/modules}" -docker-compose build --parallel +docker compose build --parallel scratch=$(mktemp -d) mkfifo "$scratch/pipe" @@ -28,7 +28,7 @@ else fi while read -r i tests; do - echo "docker-compose project \"test$i\" will run the following: $tests" + echo "docker compose project \"test$i\" will run the following: $tests" # shellcheck disable=SC2086 COMPOSE_PROJECT_NAME="test$i" python3 -m unittest "$@" $tests & done <"$scratch/pipe" diff --git a/test/cases/README.md b/test/cases/README.md index 6f34e3ac..a7e05252 100644 --- a/test/cases/README.md +++ b/test/cases/README.md @@ -1,9 +1,9 @@ These are the actual integration tests. The tests run as python `unittest` test cases. They share an instance of -`class Orchestration`, which encapsulates the `docker-compose` services -session by running `docker-compose up` before any tests begin and by running -`docker-compose down` after all tests have completed. +`class Orchestration`, which encapsulates the `docker compose` services +session by running `docker compose up` before any tests begin and by running +`docker compose down` after all tests have completed. Usage ----- @@ -46,9 +46,9 @@ relevant. For example, to run only the test $ test/bin/run cases.configuration.test_configuration ``` -To see very detailed output, tail the `logs/docker-compose-verbose.log` file. +To see very detailed output, tail the `logs/test.log` file. ```console -$ touch logs/docker-compose-verbose.log && tail -f logs/docker-compose-verbose.log & +$ touch logs/test.log && tail -f logs/test.log & $ test/bin/run ``` @@ -77,16 +77,16 @@ The `*.py` directly in this directory are common code shared by the tests. - [case.py](case.py) is a wrapper around `unittest.TestCase` that provides an attribute `.orch` of type `Orchestration`. Test cases can use this to share - a single scoped session of `docker-compose` services. + a single scoped session of `docker compose` services. - [formats.py](formats.py) contains parsing functions for the output of - `docker-compose up`, `docker-compose down`, and the JSON-formatted + `docker compose up`, `docker compose down`, and the JSON-formatted traces logged by the agent service. - [lazy_singleton.py](lazy_singleton.py) defines a generic singleton class, which is then used to define a single instance of `Orchestration`, the - `docker-compose` wrapper. + `docker compose` wrapper. - [orchestration.py](orchestration.py) defines a `class Orchestration` that - manages a thread that runs and consumes the output of `docker-compose up`, - and has methods for performing operations on the `docker-compose` setup, + manages a thread that runs and consumes the output of `docker compose up`, + and has methods for performing operations on the `docker compose` setup, e.g. - sending a request to nginx, - retrieving the logs of a service, diff --git a/test/cases/case.py b/test/cases/case.py index cd7eea20..520b0643 100644 --- a/test/cases/case.py +++ b/test/cases/case.py @@ -31,8 +31,8 @@ def tearDown(self): # `startTestRun` and `stopTestRun` are injected into the `unittest` module so # that test suites that span multiple modules share a scoped instance of -# `Orchestration`, i.e. `docker-compose up` happens before any tests run, -# and `docker-compose down` happens after all tests are finished. +# `Orchestration`, i.e. `docker compose up` happens before any tests run, +# and `docker compose down` happens after all tests are finished. # # See . global_orch_context = None diff --git a/test/cases/formats.py b/test/cases/formats.py index 43037e22..56295abd 100644 --- a/test/cases/formats.py +++ b/test/cases/formats.py @@ -1,4 +1,4 @@ -"""interpret the output of `docker-compose` commands""" +"""interpret the output of `docker compose` commands""" import json import re @@ -13,7 +13,7 @@ def parse_docker_compose_up_line(line): match = try_match( r'(?P\S+)(?P[_-])\d+\s*\| (?P.*)\n', line) if match is not None: - # Some docker-compose setups (versions?) prefix the service name by the + # Some docker compose setups (versions?) prefix the service name by the # project name, while others don't. The parts of the name are # delimited by either underscores or hyphens, and we don't use service # names with either delimiter in them, so we can pull apart the name @@ -48,7 +48,7 @@ def parse_docker_compose_up_line(line): 'container': match.groupdict()['container'] }) - # Different docker-compose setups (versions?) produce different output + # Different docker compose setups (versions?) produce different output # when a container is creating/created. Here are the other flavors of # begin_create_container and finish_create_container. diff --git a/test/cases/orchestration.py b/test/cases/orchestration.py index 0868cca8..b1c23aca 100644 --- a/test/cases/orchestration.py +++ b/test/cases/orchestration.py @@ -1,4 +1,4 @@ -"""Service orchestration (docker-compose) facilities for testing""" +"""Service orchestration (docker compose) facilities for testing""" from . import formats from .lazy_singleton import LazySingleton @@ -30,13 +30,11 @@ def quit_signal_handler(signum, frame): signal.signal(signal.SIGQUIT, quit_signal_handler) # Since we override the environment variables of child processes, -# `subprocess.Popen` (and its derivatives) need to know exactly where -# the "docker-compose" executable is, since it won't find it in the passed-in -# env's PATH. -docker_compose_command_path = shutil.which('docker-compose') +# `subprocess.Popen` (and its derivatives) need to know exactly where the +# "docker" executable is, since it won't find it in the passed-in env's PATH. docker_command_path = shutil.which('docker') -# If we want to always pass some flags to `docker-compose` or to `docker`, put +# If we want to always pass some flags to `docker compose` or to `docker`, put # them here. For example, "--tls". However, TLS behavior can be specified in # environment variables. docker_compose_flags = [] @@ -49,15 +47,15 @@ def quit_signal_handler(signum, frame): def docker_compose_command(*args): - return [docker_compose_command_path, *docker_compose_flags, *args] + return [docker_command_path, 'compose', *docker_compose_flags, *args] def docker_command(*args): return [docker_command_path, *docker_flags, *args] -# docker-compose (at least the version running on my laptop) invokes `docker` -# unqualified, and so when we run `docker-compose` commands, we have to do it +# docker compose (at least the version running on my laptop) invokes `docker` +# unqualified, and so when we run `docker compose` commands, we have to do it # in an environment where `docker_command_path` is in the PATH. # See `child_env`. docker_bin = str(Path(docker_command_path).parent) @@ -95,7 +93,7 @@ def child_env(parent_env=None): def to_service_name(container_name): # test_foo_bar_1 -> foo_bar # - # Note that the "test_" prefix is the docker-compose project name, which + # Note that the "test_" prefix is the docker compose project name, which # you can override using the COMPOSE_PROJECT_NAME environment # variable. We use that environment variable to hard-code the name to # "test". Otherwise, it defaults to the basename of the directory. Right @@ -103,9 +101,9 @@ def to_service_name(container_name): # to be able to break this. See mention of COMPOSE_PROJECT_NAME in # `child_env()`. # - # When I run docker-compose locally on my machine, the parts of the + # When I run docker compose locally on my machine, the parts of the # container name are separated by underscore ("_"), while when I run - # docker-compose in CircleCI, hyphen ("-") is used. Go with whichever is + # docker compose in CircleCI, hyphen ("-") is used. Go with whichever is # being used. if '_' in container_name and '-' in container_name: raise Exception( @@ -141,7 +139,7 @@ def docker_top(container, verbose_output): fields = ('pid', 'cmd') command = docker_command('top', container, '-o', ','.join(fields)) - with print_duration('Consuming docker-compose top PIDs', verbose_output): + with print_duration('Consuming docker compose top PIDs', verbose_output): with subprocess.Popen(command, stdout=subprocess.PIPE, env=child_env(), @@ -251,18 +249,18 @@ def docker_compose_up(on_ready, logs, verbose_file): # Started a container. Add its container ID to `containers`. container_name = fields['container'] service = to_service_name(container_name) - # Consult `docker-compose ps` for the service's container ID. + # Consult `docker compose ps` for the service's container ID. # Since we're handling a finish_create_container event, you'd # think that the container would be available now. However, - # there's a race here where docker-compose does not yet know + # there's a race here where docker compose does not yet know # which container corresponds to `service` (even though it just # told us that the container was created). # - # Additionally, some `docker-compose ps` implementations exit + # Additionally, some `docker compose ps` implementations exit # with a nonzero (error) status when there is no container ID # to report, while others exit with status zero (success) and # don't print any output. So, we retry (up to a limit) until - # `docker-compose ps` exits with status zero and produces + # `docker compose ps` exits with status zero and produces # output. containers[service] = docker_compose_ps_with_retries(max_attempts=100, service=service) elif kind == 'service_log': @@ -300,7 +298,7 @@ def header_args(): yield '--header' yield f'{name}: {value}' - # "curljson.sh" is a script that lives in the "client" docker-compose + # "curljson.sh" is a script that lives in the "client" docker compose # service. It's a wrapper around "curl" that outputs a JSON object of # information on the first line, and outputs a JSON string of the response # body on the second line. See the documentation of the "json" format for @@ -328,12 +326,12 @@ def header_args(): class Orchestration: - """A handle for a `docker-compose` session. + """A handle for a `docker compose` session. - `up()` runs `docker-compose up` and spawns a thread that consumes its + `up()` runs `docker compose up` and spawns a thread that consumes its output. - `down()` runs `docker-compose down`. + `down()` runs `docker compose down`. Other methods perform integration test specific actions. @@ -351,24 +349,24 @@ def __init__(self): file=self.verbose) # Properties (all private) - # - `up_thread` is the `threading.Thread` running `docker-compose up`. + # - `up_thread` is the `threading.Thread` running `docker compose up`. # - `logs` is a `dict` that maps service name to a `queue.SimpleQueue` of log # lines. # - `containers` is a `dict` {service: container ID} that, per service, # maps to the Docker container ID in which the service is running. # - `services` is a `list` of service names as defined in the - # `docker-compose` config. + # `docker compose` config. # - `verbose` is a file-like object to which verbose logging is written. def up(self): """Start service orchestration. - Run `docker-compose up` to bring up the orchestrated services. Begin + Run `docker compose up` to bring up the orchestrated services. Begin parsing their logs on a separate thread. """ # Before we bring things up, first clean up any detritus left over from # previous runs. Failing to do so can create problems later when we - # ask docker-compose which container a service is running in. + # ask docker compose which container a service is running in. command = docker_compose_command('down', '--remove-orphans') subprocess.run(command, stdin=subprocess.DEVNULL, @@ -392,7 +390,7 @@ def up(self): def down(self): """Stop service orchestration. - Run `docker-compose down` to bring down the orchestrated services. + Run `docker compose down` to bring down the orchestrated services. Join the log-parsing thread. """ command = docker_compose_command('down', '--remove-orphans') @@ -449,7 +447,7 @@ def sync_service(self, service): Send a "sync" request to the specified `service`, and wait for the corresponding log messages to appear in the - docker-compose log. This way, we know that whatever we have done + docker compose log. This way, we know that whatever we have done previously has already appeared in the log. log_lines = orch.sync_service('agent') @@ -534,7 +532,7 @@ def reload_nginx(self, wait_for_workers_to_terminate=True): """Send a "reload" signal to nginx. If `wait_for_workers_to_terminate` is true, then poll - `docker-compose ps` until the workers associated with nginx's previous + `docker compose ps` until the workers associated with nginx's previous cycle have terminated. """ with print_duration('Reloading nginx', self.verbose): diff --git a/test/docker-compose.yml b/test/docker-compose.yml index f668356f..4cd27440 100644 --- a/test/docker-compose.yml +++ b/test/docker-compose.yml @@ -23,7 +23,7 @@ services: # `agent` is a mock trace agent. It listens on port 8126, accepts msgpack, # decodes the resulting traces, and prints them to standard output as JSON. # The tests can inspect traces sent to the agent (e.g. from the nginx module) - # by looking at `agent` log lines in the output of `docker-compose up`. + # by looking at `agent` log lines in the output of `docker compose up`. agent: image: nginx-datadog-test-services-agent build: @@ -82,12 +82,12 @@ services: - agent # `client` is a container that exists solely so that the test runner can - # `docker-compose exec` into it to run command line tools such as `curl` and + # `docker compose exec` into it to run command line tools such as `curl` and # `grpcurl`. This way, the test runner's network doesn't need access to the - # network created by docker-compose. The test runner interacts with the - # docker-compose services via `docker-compose` and `docker` commands only. + # network created by docker compose. The test runner interacts with the + # docker compose services via `docker compose` and `docker` commands only. # This became necessary when these tests were integrated into CircleCI -- - # CircleCI's container-based remote-docker docker-compose setup does not + # CircleCI's container-based remote-docker docker compose setup does not # allow a host port binding. client: image: nginx-datadog-test-services-client diff --git a/test/services/client/README.md b/test/services/client/README.md index c9fb64df..f7e3a5fa 100644 --- a/test/services/client/README.md +++ b/test/services/client/README.md @@ -2,11 +2,11 @@ This directory contains the build instructions for the "client" service from [docker-compose.yml](../../docker-compose.yml). It contains command line tools that the test runner will use via -`docker-compose exec`, such as `grpcurl` and `curl`. The test runner uses +`docker compose exec`, such as `grpcurl` and `curl`. The test runner uses these in-compose-container command line tools instead of using the network -directly, so that the test runner's network and the docker-compose network need +directly, so that the test runner's network and the docker compose network need not be bridged. We want to be able to run the tests in contexts where host -port binding is not allowed (such as CircleCI's in-docker docker-compose +port binding is not allowed (such as CircleCI's in-docker docker compose offering). `curljson.sh` is a wrapper around curl whose output is easy to parse. diff --git a/test/services/client/curljson.sh b/test/services/client/curljson.sh index 949c1b78..0d891592 100755 --- a/test/services/client/curljson.sh +++ b/test/services/client/curljson.sh @@ -12,7 +12,7 @@ # a JSON string, i.e. double quoted and with escape sequences. # # The intention is that the test driver invoke this script using -# `docker-compose exec`. The output format is chosen to be easy to parse in +# `docker compose exec`. The output format is chosen to be easy to parse in # Python. tmpdir=$(mktemp -d) diff --git a/test/services/nginx/Dockerfile b/test/services/nginx/Dockerfile index f8dcfaa1..4468317c 100644 --- a/test/services/nginx/Dockerfile +++ b/test/services/nginx/Dockerfile @@ -9,7 +9,7 @@ RUN mkdir -p /usr/share/nginx/html COPY index.html /usr/share/nginx/html # Install the `kill` command (`procps` package on Debian), so that one-off -# nginx instances that are started via `docker-compose exec` can be signaled to +# nginx instances that are started via `docker compose exec` can be signaled to # gracefully shutdown. COPY ./install_tools.sh /tmp/ RUN /tmp/install_tools.sh diff --git a/test/services/nginx/README.md b/test/services/nginx/README.md index 37351188..991cae13 100644 --- a/test/services/nginx/README.md +++ b/test/services/nginx/README.md @@ -1,5 +1,5 @@ This directory contains the build instructions for the "nginx" service from [docker-compose.yml](../../docker-compose.yml). It's based on a specified nginx docker image, and then has the Datadog module installation added to it, -along with some utilities that the integration tests `docker-compose exec` +along with some utilities that the integration tests `docker compose exec` within the container (such as `kill`). \ No newline at end of file