From 8c95a5bc7ed0f2cbb2ce126ffb128c5eaa12c32f Mon Sep 17 00:00:00 2001 From: Stu Hood Date: Tue, 28 Jul 2020 10:23:25 -0700 Subject: [PATCH] Make independent owners requests per file to improve memoization (#10491) ### Problem #10441 caused a performance regression from about 10s to run `./pants dependencies --transitive ::` with inference enabled, to about 22s. ### Solution Make independent owners requests per file we'd like to find owners, which allows the lookups for each file to be memoized independently. ### Result `./pants dependencies --transitive ::` takes 15s. Although we are doing more work than before (before #10441, conftest discovery would only happen at `test` time), this is not ideal, and we should do further optimization before launch. But there are a few variables that will impact this soon that make it not the best time to optimize: 1) an intrinisic `PathGlobs->Paths` operation, 2) possibly enabling inference by default, allowing all of the strategies to use the module_mapper. [ci skip-rust-tests] --- .../python/dependency_inference/rules.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/python/pants/backend/python/dependency_inference/rules.py b/src/python/pants/backend/python/dependency_inference/rules.py index 5069a9e1067..085ac182997 100644 --- a/src/python/pants/backend/python/dependency_inference/rules.py +++ b/src/python/pants/backend/python/dependency_inference/rules.py @@ -1,6 +1,7 @@ # Copyright 2020 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). +import itertools from pathlib import PurePath from pants.backend.python.dependency_inference import module_mapper @@ -123,11 +124,11 @@ async def infer_python_init_dependencies( ) # And add dependencies on their owners. - return InferredDependencies( - await Get( - Owners, OwnersRequest(extra_init_files.snapshot.files, OwnersNotFoundBehavior.error) - ) + owners = await MultiGet( + Get(Owners, OwnersRequest((f,), OwnersNotFoundBehavior.error)) + for f in extra_init_files.snapshot.files ) + return InferredDependencies(itertools.chain.from_iterable(owners)) class InferConftestDependencies(InferDependenciesRequest): @@ -149,11 +150,11 @@ async def infer_python_conftest_dependencies( ) # And add dependencies on their owners. - return InferredDependencies( - await Get( - Owners, OwnersRequest(extra_conftest_files.snapshot.files, OwnersNotFoundBehavior.error) - ) + owners = await MultiGet( + Get(Owners, OwnersRequest((f,), OwnersNotFoundBehavior.error)) + for f in extra_conftest_files.snapshot.files ) + return InferredDependencies(itertools.chain.from_iterable(owners)) def rules():