upload backcompat artifacts to bk (#8184)

integration_tests/test_suites/backcompat-test-suite/README.md

@@ -2,6 +2,25 @@
 
 This test suite ensures that the branch Dagster code can successfully communicate cross-process with older Dagster code.
 
+## Looking at test artifacts in BuildKite
+In buildkite the backcompat test suite uploads the docker logs of all of the containers it spins up during the test. If you
+see these tests failing, looking at the logs should be your first step in debugging.
+
+Logs from the following containers should get uploaded:
+* dagit
+* docker_daemon
+* dagster_grpc_server
+* docker_postgresql
+
+To download the logs, go to the Artifacts tab in the buildkite test.
+
+If for some reason you don't see logs for one of the containers, there might be some helpful information
+in the test logs. If you download those logs you can search for some stdout and see if any of the following occurred:
+* if you search for `container log dump failed with stdout` you will find the stdout and stderr for the command
+`docker logs <container>` if the command failed.
+* if you search for `Buildkite artifact added with stdout` you will get the stdout and stderr for the command to upload
+artifacts to buildkite.
+
 ## Running tests locally
 
 In order to run, the `EARLIEST_TESTED_RELEASE` environment variable needs to be set.

integration_tests/test_suites/backcompat-test-suite/tests/test_backcompat.py

@@ -1,8 +1,12 @@
+# pylint: disable=print-call
+
 import os
 import subprocess
 import time
 from contextlib import contextmanager
+from pathlib import Path
 
+import docker
 import packaging
 import pytest
 import requests
@@ -74,7 +78,60 @@ def release_test_map(request, dagster_most_recent_release):
 @contextmanager
 def docker_service_up(docker_compose_file, build_args=None):
     if IS_BUILDKITE:
-        yield  # buildkite pipeline handles the service
+        try:
+            yield  # buildkite pipeline handles the service
+        finally:
+
+            # collect logs from the containers and upload to buildkite
+            client = docker.client.from_env()
+            containers = client.containers.list()
+
+            current_test = os.environ.get("PYTEST_CURRENT_TEST").split(":")[-1].split(" ")[0]
+            logs_dir = f".docker_logs/{current_test}"
+
+            # delete any existing logs
+            p = subprocess.Popen(["rm", "-rf", "{dir}".format(dir=logs_dir)])
+            p.communicate()
+            assert p.returncode == 0
+
+            Path(logs_dir).mkdir(parents=True, exist_ok=True)
+
+            for c in containers:
+                with open(
+                    "{dir}/{container}-logs.txt".format(dir=logs_dir, container=c.name),
+                    "w",
+                    encoding="utf8",
+                ) as log:
+                    p = subprocess.Popen(
+                        ["docker", "logs", c.name],
+                        stdout=log,
+                        stderr=log,
+                    )
+                    p.communicate()
+                    print(f"container({c.name}) logs dumped")
+                    if p.returncode != 0:
+                        q = subprocess.Popen(
+                            ["docker", "logs", c.name],
+                            stdout=subprocess.PIPE,
+                            stderr=subprocess.PIPE,
+                        )
+                        stdout, stderr = q.communicate()
+                        print(f"{c.name} container log dump failed with stdout: ", stdout)
+                        print(f"{c.name} container logs dump failed with stderr: ", stderr)
+
+            p = subprocess.Popen(
+                [
+                    "buildkite-agent",
+                    "artifact",
+                    "upload",
+                    "{dir}/**/*".format(dir=logs_dir),
+                ],
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+            )
+            stdout, stderr = p.communicate()
+            print("Buildkite artifact added with stdout: ", stdout)
+            print("Buildkite artifact added with stderr: ", stderr)
         return
 
     try:

integration_tests/test_suites/backcompat-test-suite/tox.ini

@@ -5,7 +5,7 @@ skipsdist = True
 [testenv]
 setenv =
   VIRTUALENV_PIP=21.3.1
-passenv = CI_* COVERALLS_REPO_TOKEN BUILDKITE BACKCOMPAT_TESTS_DAGIT_HOST EARLIEST_TESTED_RELEASE
+passenv = CI_* COVERALLS_REPO_TOKEN BUILDKITE* BACKCOMPAT_TESTS_DAGIT_HOST EARLIEST_TESTED_RELEASE
 
 deps =
   -e ../../../python_modules/dagster[mypy,test]