Fix test utils and provide test cases for karton.core.test

docs/unit_tests.rst

@@ -12,9 +12,11 @@ What's more important however, is that it runs without any Redis or MinIO intera
 
 .. code-block:: python
 
-    from math_karton import MathKarton
+    from .math_karton import MathKarton
+    from karton.core import Task
     from karton.core.test import KartonTestCase
 
+
     class MathKartonTestCase(KartonTestCase):
         """Test a karton that accepts an array of integers in "numbers" payload and
         returns their sum in "result".
@@ -33,14 +35,14 @@ What's more important however, is that it runs without any Redis or MinIO intera
             results = self.run_task(task)
 
             # prepare a expected output task and check if it matches the one produced
-            expected_task = Task(
+            expected_task = Task({
+                "origin": "karton.math",
                 "type": "math-result"
             }, payload={
                 "result": 10,
             })
 
-            self.assertTasksEqual(results, expected_task)
-
+            self.assertTasksEqual(results, [expected_task])
 
 Testing resources
 -----------------
@@ -52,22 +54,22 @@ That was pretty simple, but what about testing karton systems that accept and sp
 
 .. code-block:: python
 
-    from reverse import ReverserKarton
+    from .reverser_karton import ReverserKarton
+    from karton.core import Task, Resource
     from karton.core.test import KartonTestCase
-    from karton.core import Resource
+
 
     class ReverserKartonTestCase(KartonTestCase):
-        """Test a karton that expects a KartonResource in "file" key and spawns a new
+        """
+        Test a karton that expects a KartonResource in "file" key and spawns a new
         task containing that file reversed.
         """
 
         karton_class = ReverserKarton
 
         def test_reverse(self) -> None:
-            # load data from testcase files
-            with open("testdata/file.txt", "rb") as f:
-                input_data = f.read()
-            
+            # prepare input data
+            input_data = b"foobarbaz"
             # create fake, mini-independent resources
             input_sample = Resource("sample.txt", input_data)
             output_sample = Resource("sample.txt", input_data[::-1])
@@ -83,10 +85,11 @@ That was pretty simple, but what about testing karton systems that accept and sp
             results = self.run_task(task)
 
             # prepare a expected output task and check if it matches the one produced
-            expected_task = Task(
+            expected_task = Task({
+                "origin": "karton.reverser",
                 "type": "reverse-result"
             }, payload={
                 "file": output_sample,
             })
 
-            self.assertTasksEqual(results, expected_task)
+            self.assertTasksEqual(results, [expected_task])

karton/core/test.py

@@ -6,12 +6,12 @@
 import logging
 import unittest
 from collections import defaultdict
-from typing import Any, BinaryIO, Dict, List, Optional, Union, cast
+from typing import Any, BinaryIO, Dict, List, Union, cast
 from unittest import mock
 
-from .backend import KartonMetrics
-from .karton import Config
-from .resource import Resource, ResourceBase
+from .backend import KartonBackend, KartonMetrics
+from .config import Config
+from .resource import LocalResource, RemoteResource, ResourceBase
 from .task import Task, TaskState
 from .utils import get_function_arg_num
 
@@ -22,11 +22,13 @@
 
 
 class ConfigMock(Config):
-    def __init__(self) -> None:
+    def __init__(self):
         self.config = configparser.ConfigParser()
+        self.config.add_section("minio")
+        self.config.add_section("redis")
 
 
-class KartonBackendMock:
+class BackendMock:
     def __init__(self) -> None:
         self.produced_tasks: List[Task] = []
         # A custom MinIO system mock
@@ -36,12 +38,10 @@ def __init__(self) -> None:
     def default_bucket_name(self) -> str:
         return "karton.test"
 
-    def register_task(self, task: Task) -> None:
+    def register_task(self, task: Task, pipe=None) -> None:
         log.debug("Registering a new task in Redis: %s", task.serialize())
 
-    def set_task_status(
-        self, task: Task, status: TaskState, consumer: Optional[str] = None
-    ) -> None:
+    def set_task_status(self, task: Task, status: TaskState, pipe=None) -> None:
         log.debug("Setting task %s status to %s", task.uid, status)
 
     def produce_unrouted_task(self, task: Task) -> None:
@@ -128,15 +128,19 @@ def test_karton_service(self):
 
     def setUp(self) -> None:
         kwargs: Dict[Any, Any] = self.kwargs or {}
+        if self.config is None:
+            self.config = ConfigMock()
+        self.backend = BackendMock()
         self.karton = self.karton_class(  # type: ignore
-            config=ConfigMock(), backend=KartonBackendMock(), **kwargs
+            config=self.config, backend=self.backend, **kwargs
         )
 
     def get_resource_sha256(self, resource: ResourceBase) -> str:
-        """Calculate SHA256 hash for a given resource
+        """
+        Calculate SHA256 hash for a given resource
 
         :param resource: Resource to be hashed
-        :return: Hexencoded SHA256 digest
+        :return: Hex-encoded SHA256 digest
         """
         h = hashlib.sha256()
         if resource._path is not None:
@@ -172,7 +176,8 @@ def assertResourceEqual(
     def assertPayloadBagEqual(
         self, payload: Dict[str, Any], expected: Dict[str, Any], payload_bag_name: str
     ) -> None:
-        """Assert that two payload bags are equal
+        """
+        Assert that two payload bags are equal
 
         :param payload: Result payload bag
         :param expected: Expected payload bag
@@ -196,7 +201,9 @@ def assertPayloadBagEqual(
                 self.assertResourceEqual(value, other_value, path)
 
     def assertTaskEqual(self, task: Task, expected: Task) -> None:
-        """Assert that two tasks objects are equal
+        """
+        Assert that two tasks objects are equal
+
         :param task: Result task
         :param expected: Expected task
         """
@@ -209,21 +216,53 @@ def assertTaskEqual(self, task: Task, expected: Task) -> None:
     def assertTasksEqual(self, tasks: List[Task], expected: List[Task]) -> None:
         """
         Assert that two task lists are equal
+
         :param tasks: Result tasks list
         :param expected: Expected tasks list
         """
         self.assertEqual(len(tasks), len(expected), "Incorrect number of tasks sent")
         for task, other in zip(tasks, expected):
             self.assertTaskEqual(task, other)
 
+    def _process_task(self, incoming_task: Task):
+        """
+        Converts task from outgoing to incoming including transformation
+        of LocalResources to RemoteResources
+        """
+        task = incoming_task.fork_task()
+        task.status = TaskState.STARTED
+        task.headers.update({"receiver": self.karton.identity})
+        for payload_bag, key, resource in task.walk_payload_bags():
+            if not isinstance(resource, ResourceBase):
+                continue
+            if not isinstance(resource, LocalResource):
+                raise ValueError("Test task must contain only LocalResource objects")
+            backend = cast(KartonBackend, self.backend)
+            resource.bucket = backend.default_bucket_name
+            resource.upload(backend)
+            remote_resource = RemoteResource(
+                name=resource.name,
+                bucket=resource.bucket,
+                metadata=resource.metadata,
+                uid=resource.uid,
+                size=resource.size,
+                backend=backend,
+                sha256=resource.sha256,
+                _flags=resource._flags,
+            )
+            payload_bag[key] = remote_resource
+        return task
+
     def run_task(self, task: Task) -> List[Task]:
         """
         Spawns task into tested Karton subsystem instance
+
         :param task: Task to be spawned
         :return: Result tasks sent by Karton Service
         """
         self.karton.backend.produced_tasks = []
-        self.karton.current_task = task
+        outgoing_task = self._process_task(task)
+        self.karton.current_task = outgoing_task
 
         # `consumer.process` might accept the incoming task as an argument or not
         if get_function_arg_num(self.karton.process) == 0:
@@ -235,4 +274,4 @@ def run_task(self, task: Task) -> List[Task]:
 
 
 # Backward compatibility
-TestResource = Resource
+TestResource = LocalResource

tests/math_test_case/math_karton.py

@@ -0,0 +1,23 @@
+from karton.core import Karton, Task
+
+
+class MathKarton(Karton):
+    identity = "karton.math"
+    filters = [
+        {
+            "type": "math-task"
+        }
+    ]
+
+    def process(self, task: Task):
+        numbers = task.get_payload("numbers")
+        sum_of_numbers = sum(numbers)
+        result = Task(
+            headers={
+                "type": "math-result"
+            },
+            payload={
+                "result": sum_of_numbers
+            }
+        )
+        self.send_task(result)

tests/math_test_case/test_math_karton.py

@@ -0,0 +1,31 @@
+from .math_karton import MathKarton
+from karton.core import Task
+from karton.core.test import KartonTestCase
+
+
+class MathKartonTestCase(KartonTestCase):
+    """Test a karton that accepts an array of integers in "numbers" payload and
+    returns their sum in "result".
+    """
+    karton_class = MathKarton
+
+    def test_addition(self) -> None:
+        # prepare a fake test task that matches the production format
+        task = Task({
+            "type": "math-task",
+        }, payload={
+            "numbers": [1, 2, 3, 4],
+        })
+
+        # dry-run the fake task on the wrapped karton system
+        results = self.run_task(task)
+
+        # prepare a expected output task and check if it matches the one produced
+        expected_task = Task({
+            "origin": "karton.math",
+            "type": "math-result"
+        }, payload={
+            "result": 10,
+        })
+
+        self.assertTasksEqual(results, [expected_task])

tests/reverser_test_case/reverser_karton.py

@@ -0,0 +1,23 @@
+from karton.core import Karton, Task, Resource
+
+
+class ReverserKarton(Karton):
+    identity = "karton.reverser"
+    filters = [
+        {
+            "type": "reverse-task"
+        }
+    ]
+
+    def process(self, task: Task):
+        input_file = task.get_resource("file")
+        reversed_content = input_file.content[::-1]
+        result = Task(
+            headers={
+                "type": "reverse-result"
+            },
+            payload={
+                "file": Resource("sample.txt", content=reversed_content)
+            }
+        )
+        self.send_task(result)

tests/reverser_test_case/test_reverser_karton.py

@@ -0,0 +1,39 @@
+from .reverser_karton import ReverserKarton
+from karton.core import Task, Resource
+from karton.core.test import KartonTestCase
+
+
+class ReverserKartonTestCase(KartonTestCase):
+    """
+    Test a karton that expects a KartonResource in "file" key and spawns a new
+    task containing that file reversed.
+    """
+
+    karton_class = ReverserKarton
+
+    def test_reverse(self) -> None:
+        # prepare input data
+        input_data = b"foobarbaz"
+        # create fake, mini-independent resources
+        input_sample = Resource("sample.txt", input_data)
+        output_sample = Resource("sample.txt", input_data[::-1])
+
+        # prepare a fake test task that matches the production format
+        task = Task({
+            "type": "reverse-task",
+        }, payload={
+            "file": input_sample
+        })
+
+        # dry-run the fake task on the wrapped karton system
+        results = self.run_task(task)
+
+        # prepare a expected output task and check if it matches the one produced
+        expected_task = Task({
+            "origin": "karton.reverser",
+            "type": "reverse-result"
+        }, payload={
+            "file": output_sample,
+        })
+
+        self.assertTasksEqual(results, [expected_task])