backend: make crawlconfigs mutable! (#656)

backend/btrixcloud/crawl_job.py

@@ -17,6 +17,7 @@
 
 from .db import init_db
 from .crawls import Crawl, CrawlFile, CrawlCompleteIn, dt_now
+from .crawlconfigs import CrawlConfig
 
 
 # Seconds before allowing another shutdown attempt
@@ -46,6 +47,8 @@ def __init__(self):
         self.cid = uuid.UUID(os.environ["CRAWL_CONFIG_ID"])
         self.userid = uuid.UUID(os.environ["USER_ID"])
 
+        self.rev = int(os.environ["REV"])
+
         self.is_manual = os.environ.get("RUN_MANUAL") == "1"
         self.tags = os.environ.get("TAGS", "").split(",")
 
@@ -79,15 +82,23 @@ def __init__(self):
     async def async_init(self, template, params):
         """async init for k8s job"""
         crawl = await self._get_crawl()
+        crawlconfig = None
 
-        self.scale = await self.load_initial_scale(crawl)
+        try:
+            result = await self.crawl_configs.find_one({"_id": self.cid})
+            crawlconfig = CrawlConfig.from_dict(result)
+            self.scale = self._get_crawl_scale(crawl) or crawlconfig.scale
+
+        # pylint: disable=broad-except
+        except Exception as exc:
+            print(exc)
 
         # if doesn't exist, create, using scale from config
         if not crawl:
             params["scale"] = self.scale
             await self.init_job_objects(template, params)
 
-        await self.init_crawl()
+        await self.init_crawl(crawlconfig)
         prev_start_time = None
 
         retry = 3
@@ -249,10 +260,10 @@ async def update_crawl(self, **kwargs):
         """update crawl state, and optionally mark as finished"""
         await self.crawls.find_one_and_update({"_id": self.job_id}, {"$set": kwargs})
 
-    async def init_crawl(self):
+    async def init_crawl(self, crawlconfig):
         """create crawl, doesn't exist, mark as starting"""
         try:
-            crawl = self._make_crawl("starting", self.scale)
+            crawl = self._make_crawl("starting", self.scale, crawlconfig)
             await self.crawls.insert_one(crawl.to_dict())
         except pymongo.errors.DuplicateKeyError:
             await self.update_crawl(state="starting", scale=self.scale)
@@ -321,27 +332,17 @@ async def cancel(self):
 
         return {"success": True}
 
-    # pylint: disable=unused-argument
-    async def load_initial_scale(self, crawl=None):
-        """load scale from config or crawl object if not set"""
-        if self.scale:
-            return self.scale
-
-        try:
-            result = await self.crawl_configs.find_one(
-                {"_id": self.cid}, {"scale": True}
-            )
-            return result["scale"]
-        # pylint: disable=broad-except
-        except Exception as exc:
-            print(exc)
-            return 1
-
-    def _make_crawl(self, state, scale):
+    def _make_crawl(self, state, scale, crawlconfig):
         """Create crawl object for partial or fully complete crawl"""
         return Crawl(
             id=self.job_id,
             state=state,
+            config=crawlconfig.config,
+            jobType=crawlconfig.jobType,
+            profileid=crawlconfig.profileid,
+            cid_rev=crawlconfig.rev,
+            schedule=crawlconfig.schedule,
+            crawlTimeout=crawlconfig.crawlTimeout,
             userid=self.userid,
             oid=self.oid,
             cid=self.cid,
@@ -383,9 +384,9 @@ async def cancel():
         async def healthz():
             return {}
 
-        @app.post("/change_config/{cid}")
-        async def change_config(cid: str):
-            return await self._change_crawl_config(cid)
+        @app.post("/rollover")
+        async def restart():
+            return await self._rollover_restart()
 
     @abstractmethod
     async def init_job_objects(self, template, params):
@@ -399,6 +400,10 @@ async def delete_job_objects(self, job_id):
     async def _get_crawl(self):
         """get runnable object representing this crawl"""
 
+    @abstractmethod
+    def _get_crawl_scale(self, crawl):
+        """get scale from crawl, if any"""
+
     @abstractmethod
     async def _do_scale(self, new_scale):
         """set number of replicas"""
@@ -408,7 +413,7 @@ async def _send_shutdown_signal(self, signame):
         """gracefully shutdown crawl"""
 
     @abstractmethod
-    async def _change_crawl_config(self, cid):
+    async def _rollover_restart(self):
         """change crawl config for this crawl"""
 
     @property