diff --git a/tests/conftest.py b/tests/conftest.py index 58195ea4e..86858c678 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -224,6 +224,28 @@ def application_docker_images(docker_client: DockerClient) -> Mapping[str, Image "python": { "": {}, "libpython": dict(dockerfile="libpython.Dockerfile"), + "2.7-glibc-python": dict(dockerfile="matrix.Dockerfile", buildargs={"PYTHON_IMAGE_TAG": "2.7-slim"}), + "2.7-musl-python": dict(dockerfile="matrix.Dockerfile", buildargs={"PYTHON_IMAGE_TAG": "2.7-alpine"}), + "3.5-glibc-python": dict(dockerfile="matrix.Dockerfile", buildargs={"PYTHON_IMAGE_TAG": "3.5-slim"}), + "3.5-musl-python": dict(dockerfile="matrix.Dockerfile", buildargs={"PYTHON_IMAGE_TAG": "3.5-alpine"}), + "3.6-glibc-python": dict(dockerfile="matrix.Dockerfile", buildargs={"PYTHON_IMAGE_TAG": "3.6-slim"}), + "3.6-musl-python": dict(dockerfile="matrix.Dockerfile", buildargs={"PYTHON_IMAGE_TAG": "3.6-alpine"}), + "3.7-glibc-python": dict(dockerfile="matrix.Dockerfile", buildargs={"PYTHON_IMAGE_TAG": "3.7-slim"}), + "3.7-musl-python": dict(dockerfile="matrix.Dockerfile", buildargs={"PYTHON_IMAGE_TAG": "3.7-alpine"}), + "3.8-glibc-python": dict(dockerfile="matrix.Dockerfile", buildargs={"PYTHON_IMAGE_TAG": "3.8-slim"}), + "3.8-musl-python": dict(dockerfile="matrix.Dockerfile", buildargs={"PYTHON_IMAGE_TAG": "3.8-alpine"}), + "3.9-glibc-python": dict(dockerfile="matrix.Dockerfile", buildargs={"PYTHON_IMAGE_TAG": "3.9-slim"}), + "3.9-musl-python": dict(dockerfile="matrix.Dockerfile", buildargs={"PYTHON_IMAGE_TAG": "3.9-alpine"}), + "3.10-glibc-python": dict(dockerfile="matrix.Dockerfile", buildargs={"PYTHON_IMAGE_TAG": "3.10-slim"}), + "3.10-musl-python": dict(dockerfile="matrix.Dockerfile", buildargs={"PYTHON_IMAGE_TAG": "3.10-alpine"}), + "2.7-glibc-uwsgi": dict( + dockerfile="uwsgi.Dockerfile", buildargs={"PYTHON_IMAGE_TAG": "2.7"} + ), # not slim - need gcc + "2.7-musl-uwsgi": dict(dockerfile="uwsgi.Dockerfile", buildargs={"PYTHON_IMAGE_TAG": "2.7-alpine"}), + "3.7-glibc-uwsgi": dict( + dockerfile="uwsgi.Dockerfile", buildargs={"PYTHON_IMAGE_TAG": "3.7"} + ), # not slim - need gcc + "3.7-musl-uwsgi": dict(dockerfile="uwsgi.Dockerfile", buildargs={"PYTHON_IMAGE_TAG": "3.7-alpine"}), }, "ruby": {"": {}}, } diff --git a/tests/containers/python/lister.py b/tests/containers/python/lister.py index b81c45eb0..0a0f2c7a4 100755 --- a/tests/containers/python/lister.py +++ b/tests/containers/python/lister.py @@ -6,22 +6,27 @@ import os from threading import Thread -import yaml - class Lister(object): @classmethod - def lister(cls) -> None: + def lister(cls): + # type: () -> None os.listdir("/") # have some kernel stacks & Python stacks from a class method class Burner(object): - def burner(self) -> None: + def burner(self): + # type: () -> None while True: # have some Python stacks from an instance method pass -def parser() -> None: +def parser(): + # type: () -> None + try: + import yaml + except ImportError: + return # not required in this test while True: # Have some package stacks. # Notice the name of the package name (PyYAML) is different from the name of the module (yaml) diff --git a/tests/containers/python/matrix.Dockerfile b/tests/containers/python/matrix.Dockerfile new file mode 100644 index 000000000..e38fc2e36 --- /dev/null +++ b/tests/containers/python/matrix.Dockerfile @@ -0,0 +1,6 @@ +ARG PYTHON_IMAGE_TAG +FROM python:${PYTHON_IMAGE_TAG} + +WORKDIR /app +ADD lister.py /app +CMD ["python", "lister.py"] diff --git a/tests/containers/python/uwsgi.Dockerfile b/tests/containers/python/uwsgi.Dockerfile new file mode 100644 index 000000000..9520db920 --- /dev/null +++ b/tests/containers/python/uwsgi.Dockerfile @@ -0,0 +1,12 @@ +ARG PYTHON_IMAGE_TAG +FROM python:${PYTHON_IMAGE_TAG} + +WORKDIR /app + +# to build uwsgi +RUN if grep -q Alpine /etc/os-release; then apk add gcc libc-dev linux-headers; fi + +RUN pip install uwsgi + +ADD lister.py /app +CMD ["uwsgi", "--py", "lister.py"] diff --git a/tests/test_python.py b/tests/test_python.py index 88e3d24c3..e0d92f3eb 100644 --- a/tests/test_python.py +++ b/tests/test_python.py @@ -2,15 +2,17 @@ # Copyright (c) Granulate. All rights reserved. # Licensed under the AGPL3 License. See LICENSE.md in the project root for license information. # +import os from pathlib import Path from threading import Event +import psutil import pytest -from docker.models.containers import Container +from granulate_utils.linux.process import is_musl from gprofiler.profilers.python import PythonProfiler from tests.conftest import AssertInCollapsed -from tests.utils import snapshot_pid_collapsed +from tests.utils import assert_function_in_collapsed, snapshot_pid_collapsed, snapshot_pid_profile @pytest.fixture @@ -22,7 +24,7 @@ def runtime() -> str: @pytest.mark.parametrize("application_image_tag", ["libpython"]) def test_python_select_by_libpython( tmp_path: Path, - application_docker_container: Container, + application_pid: int, assert_collapsed: AssertInCollapsed, ) -> None: """ @@ -32,6 +34,80 @@ def test_python_select_by_libpython( This test runs a Python named "shmython". """ with PythonProfiler(1000, 1, Event(), str(tmp_path), False, "pyspy", True, None) as profiler: - process_collapsed = snapshot_pid_collapsed(profiler, application_docker_container.attrs["State"]["Pid"]) + process_collapsed = snapshot_pid_collapsed(profiler, application_pid) assert_collapsed(process_collapsed) assert all(stack.startswith("shmython") for stack in process_collapsed.keys()) + + +@pytest.mark.parametrize("in_container", [True]) +@pytest.mark.parametrize( + "application_image_tag", + [ + "2.7-glibc-python", + "2.7-musl-python", + "3.5-glibc-python", + "3.5-musl-python", + "3.6-glibc-python", + "3.6-musl-python", + "3.7-glibc-python", + "3.7-musl-python", + "3.8-glibc-python", + "3.8-musl-python", + "3.9-glibc-python", + "3.9-musl-python", + "3.10-glibc-python", + "3.10-musl-python", + "2.7-glibc-uwsgi", + "2.7-musl-uwsgi", + "3.7-glibc-uwsgi", + "3.7-musl-uwsgi", + ], +) +@pytest.mark.parametrize("profiler_type", ["py-spy", "pyperf"]) +def test_python_matrix( + tmp_path: Path, + application_pid: int, + assert_collapsed: AssertInCollapsed, + profiler_type: str, + application_image_tag: str, +) -> None: + python_version, libc, app = application_image_tag.split("-") + + if python_version == "3.5" and profiler_type == "pyperf": + pytest.skip("PyPerf doesn't support Python 3.5!") + + if python_version == "2.7" and profiler_type == "pyperf" and app == "uwsgi": + pytest.xfail("This combination fails, see https://github.com/Granulate/gprofiler/issues/485") + + with PythonProfiler(1000, 2, Event(), str(tmp_path), False, profiler_type, True, None) as profiler: + profile = snapshot_pid_profile(profiler, application_pid) + + collapsed = profile.stacks + + assert_collapsed(collapsed) + # searching for "python_version.", because ours is without the patchlevel. + assert_function_in_collapsed(f"standard-library=={python_version}.", collapsed) + + assert libc in ("musl", "glibc") + assert (libc == "musl") == is_musl(psutil.Process(application_pid)) + + if profiler_type == "pyperf": + # we expect to see kernel code + assert_function_in_collapsed("do_syscall_64_[k]", collapsed) + # and native user code + assert_function_in_collapsed( + "PyEval_EvalFrameEx_[pn]" if python_version == "2.7" else "_PyEval_EvalFrameDefault_[pn]", collapsed + ) + # ensure class name exists for instance methods + assert_function_in_collapsed("lister.Burner.burner", collapsed) + # ensure class name exists for class methods + assert_function_in_collapsed("lister.Lister.lister", collapsed) + + assert profile.app_metadata is not None + assert os.path.basename(profile.app_metadata["execfn"]) == app + # searching for "python_version.", because ours is without the patchlevel. + assert profile.app_metadata["python_version"].startswith(f"Python {python_version}.") + if python_version == "2.7" and app == "python": + assert profile.app_metadata["sys_maxunicode"] == "1114111" + else: + assert profile.app_metadata["sys_maxunicode"] is None