Tests: Reduce test flakiness (#109)

* Tests: Do not allow docker pull by default, but 
   pre-load default image per session.

* Improve profile detection algorithm.

* Tests: Mark the test_instance_pull test as slow.

* Use getsentry/responses for request mocking.

Instead of requests-mock due to flakiness.

* Tests: Expand start_stop_reset test.

aiidalab_launch/profile.py

@@ -29,6 +29,9 @@ def _default_port() -> int:  # explicit function required to enable test patchin
     return DEFAULT_PORT
 
 
+_DEFAULT_IMAGE = "aiidalab/aiidalab-docker-stack:latest"
+
+
 def _get_configured_host_port(container: Container) -> int | None:
     try:
         host_config = container.attrs["HostConfig"]
@@ -51,7 +54,7 @@ class Profile:
     port: int | None = field(default_factory=_default_port)
     default_apps: list[str] = field(default_factory=lambda: ["aiidalab-widgets-base"])
     system_user: str = "aiida"
-    image: str = "aiidalab/aiidalab-docker-stack:latest"
+    image: str = _DEFAULT_IMAGE
     home_mount: str | None = None
 
     def __post_init__(self):
@@ -96,13 +99,20 @@ def from_container(cls, container: Container) -> Profile:
             )
 
         system_user = get_docker_env(container, "SYSTEM_USER")
+
+        image_tag = (
+            _DEFAULT_IMAGE
+            if _DEFAULT_IMAGE in container.image.tags
+            else container.image.tags[0]
+        )
+
         return Profile(
             name=profile_name,
             port=_get_configured_host_port(container),
             default_apps=_get_aiidalab_default_apps(container),
             home_mount=str(
                 docker_mount_for(container, PurePosixPath("/", "home", system_user))
             ),
-            image=container.image.tags[0],
+            image=image_tag,
             system_user=system_user,
         )

setup.cfg

@@ -49,7 +49,7 @@ tests =
     pytest==6.2.5
     pytest-asyncio==0.17.2
     pytest-cov==3.0.0
-    requests-mock==1.9.3
+    responses==0.18.0
 
 [flake8]
 ignore =

tests/conftest.py

@@ -20,7 +20,7 @@
 import docker
 import pytest
 import requests
-from requests_mock import ANY
+import responses
 
 import aiidalab_launch
 from aiidalab_launch.application_state import ApplicationState
@@ -57,6 +57,14 @@ def docker_client():
         pytest.skip("docker not available")
 
 
+@pytest.fixture(scope="session", autouse=True)
+def _pull_docker_image(docker_client):
+    try:
+        docker_client.images.pull(aiidalab_launch.profile._DEFAULT_IMAGE)
+    except docker.errors.APIError:
+        pytest.skip("unable to pull docker image")
+
+
 # Avoid interfering with used ports on the host system.
 @pytest.fixture(scope="session", autouse=True)
 def _default_port(monkeypatch_session):
@@ -147,29 +155,49 @@ async def started_instance(instance):
 
 
 @pytest.fixture(autouse=True)
-def _enable_docker_requests(requests_mock):
+def _mocked_responses():
+    "Setup mocker for all HTTP requests."
+    with responses.RequestsMock(assert_all_requests_are_fired=False) as rsps:
+        yield rsps
+
+
+@pytest.fixture(autouse=True)
+def _enable_docker_requests(_mocked_responses):
+    "Pass-through all docker requests."
     docker_uris = re.compile(r"http\+docker:\/\/")
-    requests_mock.register_uri(ANY, docker_uris, real_http=True)
+    _mocked_responses.add_passthru(docker_uris)
+
+
+@pytest.fixture
+def _pypi_response():
+    "A minimal, but valid PyPI response for this package."
+    return dict(
+        url="https://pypi.python.org/pypi/aiidalab-launch/json",
+        json={"releases": {"2022.1010": [{"yanked": False}]}},
+    )
 
 
 # Do not request package information from PyPI
 @pytest.fixture(autouse=True)
-def mock_pypi_request(monkeypatch, requests_mock):
+def mock_pypi_request(monkeypatch, _mocked_responses, _pypi_response):
+    "Mock the PyPI request."
+    # Need to monkeypatch to prevent caching to interfere with the test.
     monkeypatch.setattr(aiidalab_launch.util, "SESSION", requests.Session())
-    requests_mock.register_uri(
-        "GET",
-        "https://pypi.python.org/pypi/aiidalab-launch/json",
-        json={"releases": {"2022.1010": [{"yanked": False}]}},
-    )
+    # Setup the mocked response for PyPI.
+    _mocked_responses.upsert(responses.GET, **_pypi_response)
 
 
 @pytest.fixture
-def mock_pypi_request_timeout(requests_mock):
-    requests_mock.register_uri(
-        "GET",
-        "https://pypi.python.org/pypi/aiidalab-launch/json",
-        exc=requests.exceptions.Timeout,
+def mock_pypi_request_timeout(_mocked_responses, _pypi_response):
+    "Simulate a timeout while trying to reach the PyPI server."
+    # Setup the timeout response.
+    timeout_response = dict(
+        url=_pypi_response["url"], body=requests.exceptions.Timeout()
     )
+    _mocked_responses.upsert(responses.GET, **timeout_response)
+    yield
+    # Restore the valid mocked response for PyPI.
+    _mocked_responses.upsert(responses.GET, **_pypi_response)
 
 
 @pytest.fixture(scope="session")
@@ -185,6 +213,20 @@ def invalid_image_id(docker_client):
         pytest.xfail("Unable to generate invalid Docker image id.")
 
 
+@pytest.fixture(autouse=True)
+def _disable_docker_pull(monkeypatch):
+    def no_pull(self, *args, **kwargs):
+        pytest.skip("Test tried to pull docker image.")
+
+    monkeypatch.setattr(docker.api.image.ImageApiMixin, "pull", no_pull)
+    return monkeypatch
+
+
+@pytest.fixture()
+def enable_docker_pull(_disable_docker_pull):
+    _disable_docker_pull.undo()
+
+
 def pytest_addoption(parser):
     parser.addoption(
         "--slow",

tests/test_cli.py

@@ -188,17 +188,28 @@ def assert_status_down():
             assert "down" in result.output
             assert "http" not in result.output
 
-        # Start instance.
+        # Start instance (non-blocking).
         runner: CliRunner = CliRunner()
-        result: Result = runner.invoke(cli.cli, ["start", "--no-browser", "--wait=300"])
+        result: Result = runner.invoke(
+            cli.cli, ["start", "--no-browser", "--no-pull", "--wait=0"]
+        )
+        assert result.exit_code == 0
+
+        # Start instance again (blocking, should be no-op).
+        runner: CliRunner = CliRunner()
+        result: Result = runner.invoke(
+            cli.cli, ["start", "--no-browser", "--no-pull", "--wait=300"]
+        )
         assert result.exit_code == 0
 
         assert_status_up()
         assert get_volume(instance.profile.home_mount)
         assert get_volume(instance.profile.conda_volume_name())
 
         # Start instance again – should be noop.
-        result: Result = runner.invoke(cli.cli, ["start", "--no-browser", "--wait=300"])
+        result: Result = runner.invoke(
+            cli.cli, ["start", "--no-browser", "--no-pull", "--wait=300"]
+        )
         assert "Container was already running" in result.output.strip()
         assert result.exit_code == 0
         assert_status_up()

tests/test_instance.py

@@ -19,7 +19,8 @@ async def test_instance_init(instance):
     assert await instance.status() is instance.AiidaLabInstanceStatus.DOWN
 
 
-def test_instance_pull(instance):
+@pytest.mark.slow
+def test_instance_pull(instance, enable_docker_pull):
     assert (
         "hello-world:latest"
         in replace(instance, profile=replace(instance.profile, image="hello-world"))