Skip to content

Commit 17276d4

Browse files
coeuvrecopybara-github
authored andcommitted
Prefetch inputs before acquiring the resource and sending Executing event.
prefetchInputs could wait for network I/Os to download all the inptus for this local action which might take a long time. It is possiable that local actions acqure resource and then wait for the network I/Os which is a waste. The prefetch is also moved before sending Executing event so that the UI will display the action is being prepared instead of running. i.e. ``` [Prepa] Linking xxx ``` instead of ``` Linking xxx ``` when prefetching inputs for the action. PiperOrigin-RevId: 481874944 Change-Id: I5cd9d379d3ddc68ddf444d4c0ebce5f1ae3ef905
1 parent 0eeac8b commit 17276d4

File tree

3 files changed

+22
-19
lines changed

3 files changed

+22
-19
lines changed

src/main/java/com/google/devtools/build/lib/exec/local/LocalSpawnRunner.java

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -141,6 +141,9 @@ public SpawnResult exec(Spawn spawn, SpawnExecutionContext context)
141141
spawn.getEnvironment(),
142142
context.getFileOutErr(),
143143
xattrProvider);
144+
if (Spawns.shouldPrefetchInputsForLocalExecution(spawn)) {
145+
context.prefetchInputsAndWait();
146+
}
144147
spawnMetrics.addSetupTime(setupTimeStopwatch.elapsed());
145148

146149
try (SilentCloseable c =
@@ -351,12 +354,6 @@ private SpawnResult start()
351354
.build();
352355
}
353356

354-
if (Spawns.shouldPrefetchInputsForLocalExecution(spawn)) {
355-
stepLog(INFO, "prefetching inputs for local execution");
356-
setState(State.PREFETCHING_LOCAL_INPUTS);
357-
context.prefetchInputsAndWait();
358-
}
359-
360357
spawnMetrics.setInputFiles(spawn.getInputFiles().memoizedFlattenAndGetSize());
361358
Stopwatch setupTimeStopwatch = Stopwatch.createStarted();
362359
for (ActionInput input : spawn.getInputFiles().toList()) {

src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnCache.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -157,10 +157,10 @@ public CacheHandle lookup(Spawn spawn, SpawnExecutionContext context)
157157
}
158158
}
159159

160-
context.report(SPAWN_EXECUTING_EVENT);
161-
162160
context.prefetchInputsAndWait();
163161

162+
context.report(SPAWN_EXECUTING_EVENT);
163+
164164
if (shouldUploadLocalResults) {
165165
return new CacheHandle() {
166166
@Override

src/main/java/com/google/devtools/build/lib/sandbox/AbstractSandboxSpawnRunner.java

Lines changed: 17 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -87,14 +87,23 @@ public final SpawnResult exec(Spawn spawn, SpawnExecutionContext context)
8787
throws ExecException, InterruptedException {
8888
ActionExecutionMetadata owner = spawn.getResourceOwner();
8989
context.report(SpawnSchedulingEvent.create(getName()));
90-
try (ResourceHandle ignored =
91-
resourceManager.acquireResources(
92-
owner,
93-
spawn.getLocalResources(),
94-
context.speculating() ? ResourcePriority.DYNAMIC_STANDALONE : ResourcePriority.LOCAL)) {
95-
context.report(SpawnExecutingEvent.create(getName()));
96-
SandboxedSpawn sandbox = prepareSpawn(spawn, context);
97-
return runSpawn(spawn, sandbox, context);
90+
91+
try {
92+
try (SilentCloseable c = Profiler.instance().profile("context.prefetchInputs")) {
93+
context.prefetchInputsAndWait();
94+
}
95+
96+
try (ResourceHandle ignored =
97+
resourceManager.acquireResources(
98+
owner,
99+
spawn.getLocalResources(),
100+
context.speculating()
101+
? ResourcePriority.DYNAMIC_STANDALONE
102+
: ResourcePriority.LOCAL)) {
103+
context.report(SpawnExecutingEvent.create(getName()));
104+
SandboxedSpawn sandbox = prepareSpawn(spawn, context);
105+
return runSpawn(spawn, sandbox, context);
106+
}
98107
} catch (IOException e) {
99108
FailureDetail failureDetail =
100109
createFailureDetail(
@@ -129,9 +138,6 @@ private SpawnResult runSpawn(
129138
sandbox.createFileSystem();
130139
}
131140
FileOutErr outErr = context.getFileOutErr();
132-
try (SilentCloseable c = Profiler.instance().profile("context.prefetchInputs")) {
133-
context.prefetchInputsAndWait();
134-
}
135141

136142
SpawnResult result;
137143
try (SilentCloseable c = Profiler.instance().profile("subprocess.run")) {

0 commit comments

Comments
 (0)