Make task chunks more evenly sized.

lsst · Feb 22, 2023 · 89f8ddc · 89f8ddc
1 parent 14ca26c
commit 89f8ddc
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 2 deletions.
diff --git a/doc/changes/DM-38101.misc.rst b/doc/changes/DM-38101.misc.rst
@@ -0,0 +1 @@
+Modifying PanDA task chunking to be evenly divided.
diff --git a/python/lsst/ctrl/bps/panda/utils.py b/python/lsst/ctrl/bps/panda/utils.py
@@ -483,9 +483,14 @@ def add_idds_work(config, generic_workflow, idds_workflow):
         job_count = 0  # Number of jobs in idds task used for task chunking
         task_chunk = 1  # Task chunk number within job label used for unique name
         work = None
-        for gwjob in generic_workflow.get_jobs_by_label(job_label):
+        jobs_by_label = generic_workflow.get_jobs_by_label(job_label)
+        num_chunks = -(-len(jobs_by_label) // max_jobs_per_task)  # ceil
+        max_jobs_per_task_this_label = -(-len(jobs_by_label) // num_chunks)
+        _LOG.debug("For job_label = %s, num jobs = %s, num_chunks = %s, max_jobs = %s",
+                   job_label, len(jobs_by_label), num_chunks, max_jobs_per_task_this_label)
+        for gwjob in jobs_by_label:
             job_count += 1
-            if job_count >= max_jobs_per_task:
+            if job_count > max_jobs_per_task_this_label:
                 job_count = 1
                 task_chunk += 1