Merge pull request #1578 from mathbunnyru/asalikhov/unify_docker_usage

Unify docker usage in tests

.github/workflows/docker-amd64.yml

@@ -76,3 +76,13 @@ jobs:
 
       - name: Test Docker Images
         run: make -C main test-all
+
+      - name: Clone Wiki
+        uses: actions/checkout@v2
+        with:
+          repository: ${{github.repository}}.wiki
+          path: wiki
+
+      - name: Run Post-Build Hooks
+        id: hook-all
+        run: make -C main hook-all

all-spark-notebook/test/test_spark_notebooks.py

@@ -22,22 +22,24 @@ def test_nbconvert(container: TrackedContainer, test_file: str) -> None:
     host_data_dir = THIS_DIR / "data"
     cont_data_dir = "/home/jovyan/data"
     output_dir = "/tmp"
-    timeout_ms = 600
+    conversion_timeout_ms = 600
     LOGGER.info(f"Test that {test_file} notebook can be executed ...")
     command = (
         "jupyter nbconvert --to markdown "
-        + f"--ExecutePreprocessor.timeout={timeout_ms} "
+        + f"--ExecutePreprocessor.timeout={conversion_timeout_ms} "
         + f"--output-dir {output_dir} "
         + f"--execute {cont_data_dir}/{test_file}.ipynb"
     )
-    c = container.run(
+    logs = container.run_and_wait(
+        timeout=60,
+        no_warnings=False,
         volumes={str(host_data_dir): {"bind": cont_data_dir, "mode": "ro"}},
         tty=True,
         command=["start.sh", "bash", "-c", command],
     )
-    rv = c.wait(timeout=timeout_ms / 10 + 10)
-    logs = c.logs(stdout=True).decode("utf-8")
-    LOGGER.debug(logs)
-    assert rv == 0 or rv["StatusCode"] == 0, f"Command {command} failed"
+    warnings = TrackedContainer.get_warnings(logs)
+    # Some Spark warnings
+    assert len(warnings) == 5
+
     expected_file = f"{output_dir}/{test_file}.md"
     assert expected_file in logs, f"Expected file {expected_file} not generated"

base-notebook/test/test_container_options.py

@@ -1,5 +1,6 @@
 # Copyright (c) Jupyter Development Team.
 # Distributed under the terms of the Modified BSD License.
+import pathlib
 import time
 import logging
 
@@ -14,15 +15,15 @@
 def test_cli_args(container: TrackedContainer, http_client: requests.Session) -> None:
     """Container should respect notebook server command line args
     (e.g., disabling token security)"""
-    c = container.run(command=["start-notebook.sh", "--NotebookApp.token=''"])
+    running_container = container.run_detached(
+        command=["start-notebook.sh", "--NotebookApp.token=''"]
+    )
     resp = http_client.get("http://localhost:8888")
     resp.raise_for_status()
-    logs = c.logs(stdout=True).decode("utf-8")
+    logs = running_container.logs().decode("utf-8")
     LOGGER.debug(logs)
     assert "ERROR" not in logs
-    warnings = [
-        warning for warning in logs.split("\n") if warning.startswith("WARNING")
-    ]
+    warnings = TrackedContainer.get_warnings(logs)
     assert not warnings
     assert "login_submit" not in resp.text
 
@@ -34,7 +35,7 @@ def test_unsigned_ssl(
     """Container should generate a self-signed SSL certificate
     and notebook server should use it to enable HTTPS.
     """
-    c = container.run(environment=["GEN_CERT=yes"])
+    running_container = container.run_detached(environment=["GEN_CERT=yes"])
     # NOTE: The requests.Session backing the http_client fixture does not retry
     # properly while the server is booting up. An SSL handshake error seems to
     # abort the retry logic. Forcing a long sleep for the moment until I have
@@ -43,52 +44,41 @@ def test_unsigned_ssl(
     resp = http_client.get("https://localhost:8888", verify=False)
     resp.raise_for_status()
     assert "login_submit" in resp.text
-    logs = c.logs(stdout=True).decode("utf-8")
+    logs = running_container.logs().decode("utf-8")
     assert "ERROR" not in logs
-    warnings = [
-        warning for warning in logs.split("\n") if warning.startswith("WARNING")
-    ]
+    warnings = TrackedContainer.get_warnings(logs)
     assert not warnings
 
 
 def test_uid_change(container: TrackedContainer) -> None:
     """Container should change the UID of the default user."""
-    c = container.run(
+    logs = container.run_and_wait(
+        timeout=120,  # usermod is slow so give it some time
         tty=True,
         user="root",
         environment=["NB_UID=1010"],
         command=["start.sh", "bash", "-c", "id && touch /opt/conda/test-file"],
     )
-    # usermod is slow so give it some time
-    rv = c.wait(timeout=120)
-    logs = c.logs(stdout=True).decode("utf-8")
-    assert "ERROR" not in logs
-    assert "WARNING" not in logs
-    assert rv == 0 or rv["StatusCode"] == 0
-    assert "uid=1010(jovyan)" in c.logs(stdout=True).decode("utf-8")
+    assert "uid=1010(jovyan)" in logs
 
 
 def test_gid_change(container: TrackedContainer) -> None:
     """Container should change the GID of the default user."""
-    c = container.run(
+    logs = container.run_and_wait(
+        timeout=10,
         tty=True,
         user="root",
         environment=["NB_GID=110"],
         command=["start.sh", "id"],
     )
-    rv = c.wait(timeout=10)
-    assert rv == 0 or rv["StatusCode"] == 0
-    logs = c.logs(stdout=True).decode("utf-8")
-    assert "ERROR" not in logs
-    assert "WARNING" not in logs
     assert "gid=110(jovyan)" in logs
     assert "groups=110(jovyan),100(users)" in logs
 
 
 def test_nb_user_change(container: TrackedContainer) -> None:
     """Container should change the user name (`NB_USER`) of the default user."""
     nb_user = "nayvoj"
-    running_container = container.run(
+    running_container = container.run_detached(
         tty=True,
         user="root",
         environment=[f"NB_USER={nb_user}", "CHOWN_HOME=yes"],
@@ -99,7 +89,7 @@ def test_nb_user_change(container: TrackedContainer) -> None:
     # container sleeps forever.
     time.sleep(10)
     LOGGER.info(f"Checking if the user is changed to {nb_user} by the start script ...")
-    output = running_container.logs(stdout=True).decode("utf-8")
+    output = running_container.logs().decode("utf-8")
     assert "ERROR" not in output
     assert "WARNING" not in output
     assert (
@@ -137,7 +127,8 @@ def test_nb_user_change(container: TrackedContainer) -> None:
 def test_chown_extra(container: TrackedContainer) -> None:
     """Container should change the UID/GID of a comma separated
     CHOWN_EXTRA list of folders."""
-    c = container.run(
+    logs = container.run_and_wait(
+        timeout=120,  # chown is slow so give it some time
         tty=True,
         user="root",
         environment=[
@@ -153,20 +144,15 @@ def test_chown_extra(container: TrackedContainer) -> None:
             "stat -c '%n:%u:%g' /home/jovyan/.bashrc /opt/conda/bin/jupyter",
         ],
     )
-    # chown is slow so give it some time
-    rv = c.wait(timeout=120)
-    assert rv == 0 or rv["StatusCode"] == 0
-    logs = c.logs(stdout=True).decode("utf-8")
-    assert "ERROR" not in logs
-    assert "WARNING" not in logs
     assert "/home/jovyan/.bashrc:1010:101" in logs
     assert "/opt/conda/bin/jupyter:1010:101" in logs
 
 
 def test_chown_home(container: TrackedContainer) -> None:
     """Container should change the NB_USER home directory owner and
     group to the current value of NB_UID and NB_GID."""
-    c = container.run(
+    logs = container.run_and_wait(
+        timeout=120,  # chown is slow so give it some time
         tty=True,
         user="root",
         environment=[
@@ -178,58 +164,41 @@ def test_chown_home(container: TrackedContainer) -> None:
         ],
         command=["start.sh", "bash", "-c", "stat -c '%n:%u:%g' /home/kitten/.bashrc"],
     )
-    rv = c.wait(timeout=120)
-    assert rv == 0 or rv["StatusCode"] == 0
-    logs = c.logs(stdout=True).decode("utf-8")
-    assert "ERROR" not in logs
-    assert "WARNING" not in logs
     assert "/home/kitten/.bashrc:1010:101" in logs
 
 
 def test_sudo(container: TrackedContainer) -> None:
     """Container should grant passwordless sudo to the default user."""
-    c = container.run(
+    logs = container.run_and_wait(
+        timeout=10,
         tty=True,
         user="root",
         environment=["GRANT_SUDO=yes"],
         command=["start.sh", "sudo", "id"],
     )
-    rv = c.wait(timeout=10)
-    assert rv == 0 or rv["StatusCode"] == 0
-    logs = c.logs(stdout=True).decode("utf-8")
-    assert "ERROR" not in logs
-    assert "WARNING" not in logs
     assert "uid=0(root)" in logs
 
 
 def test_sudo_path(container: TrackedContainer) -> None:
     """Container should include /opt/conda/bin in the sudo secure_path."""
-    c = container.run(
+    logs = container.run_and_wait(
+        timeout=10,
         tty=True,
         user="root",
         environment=["GRANT_SUDO=yes"],
         command=["start.sh", "sudo", "which", "jupyter"],
     )
-    rv = c.wait(timeout=10)
-    assert rv == 0 or rv["StatusCode"] == 0
-    logs = c.logs(stdout=True).decode("utf-8")
-    assert "ERROR" not in logs
-    assert "WARNING" not in logs
     assert logs.rstrip().endswith("/opt/conda/bin/jupyter")
 
 
 def test_sudo_path_without_grant(container: TrackedContainer) -> None:
     """Container should include /opt/conda/bin in the sudo secure_path."""
-    c = container.run(
+    logs = container.run_and_wait(
+        timeout=10,
         tty=True,
         user="root",
         command=["start.sh", "which", "jupyter"],
     )
-    rv = c.wait(timeout=10)
-    assert rv == 0 or rv["StatusCode"] == 0
-    logs = c.logs(stdout=True).decode("utf-8")
-    assert "ERROR" not in logs
-    assert "WARNING" not in logs
     assert logs.rstrip().endswith("/opt/conda/bin/jupyter")
 
 
@@ -238,18 +207,14 @@ def test_group_add(container: TrackedContainer) -> None:
     group. It won't be possible to modify /etc/passwd since gid is nonzero, so
     additionally verify that setting gid=0 is suggested in a warning.
     """
-    c = container.run(
+    logs = container.run_and_wait(
+        timeout=5,
+        no_warnings=False,
         user="1010:1010",
         group_add=["users"],  # Ensures write access to /home/jovyan
         command=["start.sh", "id"],
     )
-    rv = c.wait(timeout=5)
-    assert rv == 0 or rv["StatusCode"] == 0
-    logs = c.logs(stdout=True).decode("utf-8")
-    assert "ERROR" not in logs
-    warnings = [
-        warning for warning in logs.split("\n") if warning.startswith("WARNING")
-    ]
+    warnings = TrackedContainer.get_warnings(logs)
     assert len(warnings) == 1
     assert "Try setting gid=0" in warnings[0]
     assert "uid=1010 gid=1010 groups=1010,100(users)" in logs
@@ -261,43 +226,37 @@ def test_set_uid(container: TrackedContainer) -> None:
     Additionally verify that "--group-add=users" is suggested in a warning to restore
     write access.
     """
-    c = container.run(
+    logs = container.run_and_wait(
+        timeout=5,
+        no_warnings=False,
         user="1010",
         command=["start.sh", "id"],
     )
-    rv = c.wait(timeout=5)
-    assert rv == 0 or rv["StatusCode"] == 0
-    logs = c.logs(stdout=True).decode("utf-8")
-    assert "ERROR" not in logs
     assert "uid=1010(jovyan) gid=0(root)" in logs
-    warnings = [
-        warning for warning in logs.split("\n") if warning.startswith("WARNING")
-    ]
+    warnings = TrackedContainer.get_warnings(logs)
     assert len(warnings) == 1
     assert "--group-add=users" in warnings[0]
 
 
 def test_set_uid_and_nb_user(container: TrackedContainer) -> None:
     """Container should run with the specified uid and NB_USER."""
-    c = container.run(
+    logs = container.run_and_wait(
+        timeout=5,
+        no_warnings=False,
         user="1010",
         environment=["NB_USER=kitten"],
         group_add=["users"],  # Ensures write access to /home/jovyan
         command=["start.sh", "id"],
     )
-    rv = c.wait(timeout=5)
-    assert rv == 0 or rv["StatusCode"] == 0
-    logs = c.logs(stdout=True).decode("utf-8")
-    assert "ERROR" not in logs
     assert "uid=1010(kitten) gid=0(root)" in logs
-    warnings = [
-        warning for warning in logs.split("\n") if warning.startswith("WARNING")
-    ]
+    warnings = TrackedContainer.get_warnings(logs)
     assert len(warnings) == 1
     assert "user is kitten but home is /home/jovyan" in warnings[0]
 
 
-def test_container_not_delete_bind_mount(container: TrackedContainer, tmp_path) -> None:
+def test_container_not_delete_bind_mount(
+    container: TrackedContainer, tmp_path: pathlib.Path
+) -> None:
     """Container should not delete host system files when using the (docker)
     -v bind mount flag and mapping to /home/jovyan.
     """
@@ -306,7 +265,8 @@ def test_container_not_delete_bind_mount(container: TrackedContainer, tmp_path)
     p = d / "foo.txt"
     p.write_text("some-content")
 
-    c = container.run(
+    container.run_and_wait(
+        timeout=5,
         tty=True,
         user="root",
         working_dir="/home/",
@@ -317,11 +277,6 @@ def test_container_not_delete_bind_mount(container: TrackedContainer, tmp_path)
         volumes={d: {"bind": "/home/jovyan/data", "mode": "rw"}},
         command=["start.sh", "ls"],
     )
-    rv = c.wait(timeout=5)
-    logs = c.logs(stdout=True).decode("utf-8")
-    assert "ERROR" not in logs
-    assert "WARNING" not in logs
-    assert rv == 0 or rv["StatusCode"] == 0
     assert p.read_text() == "some-content"
     assert len(list(tmp_path.iterdir())) == 1
 
@@ -333,7 +288,8 @@ def test_jupyter_env_vars_to_unset_as_root(
     """Environment variables names listed in JUPYTER_ENV_VARS_TO_UNSET
     should be unset in the final environment."""
     root_args = {"user": "root"} if enable_root else {}
-    c = container.run(
+    logs = container.run_and_wait(
+        timeout=10,
         tty=True,
         environment=[
             "JUPYTER_ENV_VARS_TO_UNSET=SECRET_ANIMAL,UNUSED_ENV,SECRET_FRUIT",
@@ -349,9 +305,4 @@ def test_jupyter_env_vars_to_unset_as_root(
         ],
         **root_args,
     )
-    rv = c.wait(timeout=10)
-    assert rv == 0 or rv["StatusCode"] == 0
-    logs = c.logs(stdout=True).decode("utf-8")
-    assert "ERROR" not in logs
-    assert "WARNING" not in logs
     assert "I like bananas and stuff, and love to keep secrets!" in logs

base-notebook/test/test_package_managers.py

@@ -26,15 +26,8 @@ def test_package_manager(
     LOGGER.info(
         f"Test that the package manager {package_manager} is working properly ..."
     )
-    c = container.run(
+    container.run_and_wait(
+        timeout=5,
         tty=True,
         command=["start.sh", "bash", "-c", f"{package_manager} {version_arg}"],
     )
-    rv = c.wait(timeout=5)
-    logs = c.logs(stdout=True).decode("utf-8")
-    LOGGER.debug(logs)
-    assert "ERROR" not in logs
-    assert "WARNING" not in logs
-    assert (
-        rv == 0 or rv["StatusCode"] == 0
-    ), f"Package manager {package_manager} not working"