Let examples to have their own e2e tests

examples/09-testing/README.md

@@ -0,0 +1,12 @@
+# Kopf example for testing the operator
+
+Kopf provides some basic tools to test the Kopf-based operators.
+With these tools, the testing frameworks (pytest in this case)
+can run the operator-under-test in the background, while the test
+performs the resource manipulation.
+
+To run the tests:
+
+```bash
+pytest
+```

examples/09-testing/example.py

@@ -0,0 +1,6 @@
+import kopf
+
+
+@kopf.on.create('zalando.org', 'v1', 'kopfexamples')
+def create_fn(logger, **kwargs):
+    logger.info("Something was logged here.")

examples/09-testing/test_example_09.py

@@ -0,0 +1,43 @@
+import os.path
+import subprocess
+import time
+
+import pytest
+
+import kopf.testing
+
+crd_yaml = os.path.relpath(os.path.join(os.path.dirname(__file__), '..', 'crd.yaml'))
+obj_yaml = os.path.relpath(os.path.join(os.path.dirname(__file__), '..', 'obj.yaml'))
+example_py = os.path.relpath(os.path.join(os.path.dirname(__file__), 'example.py'))
+
+
+@pytest.fixture(autouse=True)
+def crd_exists():
+    subprocess.run(f"kubectl apply -f {crd_yaml}", shell=True, check=True)
+
+
+@pytest.fixture(autouse=True)
+def obj_absent():
+    subprocess.run(f"kubectl delete -f {obj_yaml}", shell=True, check=False)
+
+
+def test_resource_lifecycle(mocker):
+
+    # To prevent lengthy threads in the loop executor when the process exits.
+    mocker.patch('kopf.clients.watching.DEFAULT_STREAM_TIMEOUT', 10)
+
+    # Run an operator and simulate some activity with the operated resource.
+    with kopf.testing.KopfRunner(['run', '--verbose', '--standalone', example_py]) as runner:
+        subprocess.run(f"kubectl create -f {obj_yaml}", shell=True, check=True)
+        time.sleep(5)  # give it some time to react
+        subprocess.run(f"kubectl delete -f {obj_yaml}", shell=True, check=True)
+        time.sleep(1)  # give it some time to react
+
+    # Ensure that the operator did not die on start, or during the operation.
+    assert runner.exception is None
+    assert runner.exit_code == 0
+
+    # There are usually more than these messages, but we only check for the certain ones.
+    assert '[default/kopf-example-1] Creation event:' in runner.stdout
+    assert '[default/kopf-example-1] Something was logged here.' in runner.stdout
+    assert '[default/kopf-example-1] Deletion event:' in runner.stdout

tests/conftest.py

@@ -1,4 +1,5 @@
 import asyncio
+import os
 import re
 import time
 
@@ -9,12 +10,40 @@
 from kopf.reactor.registries import Resource
 
 
-# Make all tests in this directory and below asyncio-compatible by default.
+def pytest_configure(config):
+    config.addinivalue_line('markers', "e2e: end-to-end tests with real operators.")
+
+
+# This logic is not applied if pytest is started explicitly on ./examples/.
+# In that case, regular pytest behaviour applies -- this is intended.
 def pytest_collection_modifyitems(items):
+
+    # Make all tests in this directory and below asyncio-compatible by default.
     for item in items:
         if asyncio.iscoroutinefunction(item.function):
             item.add_marker('asyncio')
 
+    # Put all e2e tests to the end, as they are assumed to be slow.
+    def _is_e2e(item):
+        path = item.location[0]
+        return path.startswith('tests/e2e/') or path.startswith('examples/')
+    etc = [item for item in items if not _is_e2e(item)]
+    e2e = [item for item in items if _is_e2e(item)]
+    items[:] = etc + e2e
+
+    # Mark all e2e tests, no matter how they were detected. Just for filtering.
+    mark_e2e = pytest.mark.e2e
+    for item in e2e:
+        item.add_marker(mark_e2e)
+
+    # Minikube tests are heavy and require a cluster. Skip them by default,
+    # so that the contributors can run pytest without initial tweaks.
+    mark_skip = pytest.mark.skip(reason="E2E tests are not enabled. "
+                                        "Set E2E env var to enable.")
+    if not os.environ.get('E2E'):
+        for item in e2e:
+            item.add_marker(mark_skip)
+
 
 # Substitute the regular mock with the async-aware mock in the `mocker` fixture.
 @pytest.fixture(scope='session', autouse=True)

tests/e2e/conftest.py

@@ -10,6 +10,7 @@
 root_dir = os.path.relpath(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))
 examples = sorted(glob.glob(os.path.join(root_dir, 'examples/*/')))
 assert examples  # if empty, it is just the detection failed
+examples = [path for path in examples if not glob.glob((os.path.join(path, 'test*.py')))]
 
 
 @pytest.fixture(params=examples)
@@ -41,18 +42,3 @@ def no_crd():
 @pytest.fixture()
 def no_peering():
     subprocess.run("kubectl delete customresourcedefinition kopfpeerings.zalando.org", shell=True, check=True)
-
-
-@pytest.fixture(autouse=True)
-def _skip_if_not_explicitly_enabled():
-    # Minikube tests are heavy and require a cluster. Skip them by default,
-    # so that the contributors can run pytest without initial tweaks.
-    if not os.environ.get('E2E'):
-        pytest.skip('e2e tests are not explicitly enabled; set E2E env var to enable.')
-
-
-def pytest_collection_modifyitems(config, items):
-    # Put the e2e tests to the end always, since they are a bit lengthy.
-    etc = [item for item in items if '/e2e/' not in item.nodeid]
-    e2e = [item for item in items if '/e2e/' in item.nodeid]
-    items[:] = etc + e2e