Skip to content
Permalink
Browse files
perf: improve job selection performance in case of potential ambiguit…
…y that is resolved by comprehensive ruleorder statements. (#1147)

* perf: improve job selection performance in case of potential ambiguity that is resolved by comprehensive ruleorder statements.

* fmt

* improve order checking optimization after correct remarks from @dkuzminov.
  • Loading branch information
johanneskoester committed Sep 6, 2021
1 parent 7110f9d commit 921f4f715e3814fc2e22a4f6527ff62e066cc5da
Showing 1 changed file with 17 additions and 3 deletions.
@@ -735,7 +735,18 @@ def update(
exceptions = list()
cycles = list()

for job in jobs:
# check if all potential producers are strictly ordered
jobs = sorted(jobs, reverse=True)
discarded_jobs = set()

def is_strictly_higher_ordered(pivot_job):
return all(
pivot_job > job
for job in jobs
if job is not pivot_job and job not in discarded_jobs
)

for i, job in enumerate(jobs):
logger.dag_debug(dict(status="candidate", job=job))
if file in job.input:
cycles.append(job)
@@ -753,16 +764,19 @@ def update(
progress=progress,
create_inventory=create_inventory,
)
# TODO this might fail if a rule discarded here is needed
# elsewhere
producers.append(job)
if is_strictly_higher_ordered(job):
# All other jobs are either discarded or strictly less given
# any defined ruleorder.
break
except (
MissingInputException,
CyclicGraphException,
PeriodicWildcardError,
WorkflowError,
) as ex:
exceptions.append(ex)
discarded_jobs.add(job)
except RecursionError as e:
raise WorkflowError(
e,

0 comments on commit 921f4f7

Please sign in to comment.