Add worker and card parsing task

backend/dev_settings.py

@@ -1,3 +1,7 @@
 MONGO_HOST = "mongo"
 MONGO_PORT = 27017
 MONGO_DATABASE = "core"
+REDIS_HOST = "redis"
+REDIS_PORT = 6379
+REDIS_DB = 0
+TASK_QUEUE_NAME = "tasks"

backend/tasks/parse.py

@@ -0,0 +1,14 @@
+import mistune
+
+from backend.storage.card import CardDAO
+from backend.tasks.task import task
+
+
+@task
+def parse_card_markup(card_dao: CardDAO, card_id: str):
+    card = card_dao.get_by_id(card_id)
+    card.html = _parse_markdown(card.markdown)
+    card_dao.update(card)
+
+
+_parse_markdown = mistune.Markdown(escape=True, hard_wrap=False)

backend/tasks/task.py

@@ -0,0 +1,19 @@
+import functools
+from typing import Callable
+
+from backend.wiring import Wiring
+
+
+def task(func: Callable):
+    varnames = func.__code__.co_varnames
+
+    @functools.wraps(func)
+    def result(*args, **kwargs):
+        wiring: Wiring = kwargs.pop("wiring")
+        wired_objects_by_name = wiring.__dict__
+        for arg_name in varnames:
+            if arg_name in wired_objects_by_name:
+                kwargs[arg_name] = wired_objects_by_name[arg_name]
+        return func(*args, **kwargs)
+
+    return result

backend/wiring.py

@@ -2,6 +2,8 @@
 
 from pymongo import MongoClient
 from pymongo.database import Database
+import redis
+import rq
 
 import backend.dev_settings
 from backend.storage.card import CardDAO
@@ -22,3 +24,11 @@ def __init__(self, env=None):
             port=self.settings.MONGO_PORT)
         self.mongo_database: Database = self.mongo_client[self.settings.MONGO_DATABASE]
         self.card_dao: CardDAO = MongoCardDAO(self.mongo_database)
+
+        self.redis: redis.Redis = redis.StrictRedis(
+            host=self.settings.REDIS_HOST,
+            port=self.settings.REDIS_PORT,
+            db=self.settings.REDIS_DB)
+        self.task_queue: rq.Queue = rq.Queue(
+            name=self.settings.TASK_QUEUE_NAME,
+            connection=self.redis)

requirements.txt

@@ -2,4 +2,7 @@ flask
 flask-cors
 gevent
 gunicorn
+mistune
 pymongo
+redis
+rq

tools/add_test_content.py

@@ -1,11 +1,18 @@
-from backend.storage.card import Card
+from backend.storage.card import Card, CardNotFound
+from backend.tasks.parse import parse_card_markup
 from backend.wiring import Wiring
 
 wiring = Wiring()
 
-wiring.card_dao.create(Card(
-    slug="helloworld",
-    name="Hello, world!",
-    markdown="""
+try:
+    card = wiring.card_dao.get_by_slug("helloworld")
+except CardNotFound:
+    card = wiring.card_dao.create(Card(
+        slug="helloworld",
+        name="Hello, world!",
+        markdown="""
 This is a hello-world page.
-"""))
+    """))
+
+wiring.task_queue.enqueue_call(
+    parse_card_markup, kwargs={"card_id": card.id})

worker/__main__.py

@@ -0,0 +1,39 @@
+import argparse
+import uuid
+
+import rq
+from rq.job import Job
+
+import backend.wiring
+
+parser = argparse.ArgumentParser(description="Run worker.")
+parser.add_argument(
+    "--burst",
+    action="store_const",
+    const=True,
+    default=False,
+    help="enable burst mode")
+args = parser.parse_args()
+
+wiring = backend.wiring.Wiring()
+
+
+class JobWithWiring(Job):
+
+    @property
+    def kwargs(self):
+        result = dict(super().kwargs)
+        result["wiring"] = backend.wiring.Wiring()
+        return result
+
+    @kwargs.setter
+    def kwargs(self, value):
+        super().kwargs = value
+
+
+with rq.Connection(wiring.redis):
+    w = rq.Worker(
+        queues=[wiring.settings.TASK_QUEUE_NAME],
+        name=uuid.uuid4().hex,
+        job_class=JobWithWiring)
+    w.work(burst=args.burst)