From e7a0a71f50b69df5d38a8a85fefd36d211e12e8d Mon Sep 17 00:00:00 2001 From: larsrc Date: Tue, 9 Mar 2021 07:15:10 -0800 Subject: [PATCH] More properly destroy workers on interrupt. Interrupts did not wake the workers out of waiting for a response. When running under dynamic execution, that's all we can do until cancellation is implemented. For workers without dynamic execution, the only interrupt is from the build itself getting interrupted, and in that case we don't want to wait - it could take minutes. Instead we then use a busy-wait to notice interrupts and destroy the worker if interrupted. This will be useful even when cancellation is implemented, since not all workers may be able to implement a sensible cancellation. This also renames mustBeSandboxed on WorkerKey to isSpeculative to separate cause and effect. RELNOTES: None. PiperOrigin-RevId: 361802760 --- .../build/lib/worker/JsonWorkerProtocol.java | 11 +++ .../build/lib/worker/ProtoWorkerProtocol.java | 9 ++- .../build/lib/worker/SingleplexWorker.java | 22 +++++- .../build/lib/worker/WorkerFactory.java | 38 ++++------ .../devtools/build/lib/worker/WorkerKey.java | 33 ++++----- .../build/lib/worker/WorkerModule.java | 27 +------ .../build/lib/worker/WorkerMultiplexer.java | 2 + .../devtools/build/lib/worker/WorkerPool.java | 24 ++----- .../build/lib/worker/WorkerSpawnRunner.java | 1 - .../build/lib/worker/WorkerKeyTest.java | 71 +++++++++++-------- .../build/lib/worker/WorkerPoolTest.java | 46 ------------ 11 files changed, 119 insertions(+), 165 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/worker/JsonWorkerProtocol.java b/src/main/java/com/google/devtools/build/lib/worker/JsonWorkerProtocol.java index 3990a9bdc7518a..8c594515eedd26 100644 --- a/src/main/java/com/google/devtools/build/lib/worker/JsonWorkerProtocol.java +++ b/src/main/java/com/google/devtools/build/lib/worker/JsonWorkerProtocol.java @@ -54,6 +54,17 @@ public void putRequest(WorkRequest request) throws IOException { @Override public WorkResponse getResponse() throws IOException { + boolean interrupted = Thread.interrupted(); + try { + return parseResponse(); + } finally { + if (interrupted) { + Thread.currentThread().interrupt(); + } + } + } + + private WorkResponse parseResponse() throws IOException { Integer exitCode = null; String output = null; Integer requestId = null; diff --git a/src/main/java/com/google/devtools/build/lib/worker/ProtoWorkerProtocol.java b/src/main/java/com/google/devtools/build/lib/worker/ProtoWorkerProtocol.java index 6eed2ca8463ada..199f24f7cf66ca 100644 --- a/src/main/java/com/google/devtools/build/lib/worker/ProtoWorkerProtocol.java +++ b/src/main/java/com/google/devtools/build/lib/worker/ProtoWorkerProtocol.java @@ -41,7 +41,14 @@ public void putRequest(WorkRequest request) throws IOException { @Override public WorkResponse getResponse() throws IOException { - return WorkResponse.parseDelimitedFrom(workersStdout); + boolean interrupted = Thread.interrupted(); + try { + return WorkResponse.parseDelimitedFrom(workersStdout); + } finally { + if (interrupted) { + Thread.currentThread().interrupt(); + } + } } @Override diff --git a/src/main/java/com/google/devtools/build/lib/worker/SingleplexWorker.java b/src/main/java/com/google/devtools/build/lib/worker/SingleplexWorker.java index f3a1e31a5bee78..dbe0c494db3f11 100644 --- a/src/main/java/com/google/devtools/build/lib/worker/SingleplexWorker.java +++ b/src/main/java/com/google/devtools/build/lib/worker/SingleplexWorker.java @@ -118,8 +118,28 @@ void putRequest(WorkRequest request) throws IOException { } @Override - WorkResponse getResponse(int requestId) throws IOException { + WorkResponse getResponse(int requestId) throws IOException, InterruptedException { recordingInputStream.startRecording(4096); + // Ironically, we don't allow interrupts during dynamic execution, since we can't cancel + // the worker short of destroying it. + if (!workerKey.isSpeculative()) { + while (recordingInputStream.available() == 0) { + try { + Thread.sleep(10); + } catch (InterruptedException e) { + // This should only happen when not in dynamic execution, so we can safely kill the + // worker. + destroy(); + throw e; + } + if (!process.isAlive()) { + throw new IOException( + String.format( + "Worker process for %s died while waiting for response", + workerKey.getMnemonic())); + } + } + } return workerProtocol.getResponse(); } diff --git a/src/main/java/com/google/devtools/build/lib/worker/WorkerFactory.java b/src/main/java/com/google/devtools/build/lib/worker/WorkerFactory.java index 5088d50071c219..f05b8fe31c6296 100644 --- a/src/main/java/com/google/devtools/build/lib/worker/WorkerFactory.java +++ b/src/main/java/com/google/devtools/build/lib/worker/WorkerFactory.java @@ -60,7 +60,7 @@ public Worker create(WorkerKey key) { workerBaseDir.getRelative(workTypeName + "-" + workerId + "-" + key.getMnemonic() + ".log"); Worker worker; - boolean sandboxed = workerOptions.workerSandboxing || key.mustBeSandboxed(); + boolean sandboxed = workerOptions.workerSandboxing || key.isSpeculative(); if (sandboxed) { Path workDir = getSandboxedWorkerPath(key, workerId); worker = new SandboxedWorker(key, workerId, workDir, logFile); @@ -124,30 +124,18 @@ public boolean validateObject(WorkerKey key, PooledObject p) { Worker worker = p.getObject(); Optional exitValue = worker.getExitValue(); if (exitValue.isPresent()) { - if (workerOptions.workerVerbose) { - if (worker.diedUnexpectedly()) { - String msg = - String.format( - "%s %s (id %d) has unexpectedly died with exit code %d.", - key.getMnemonic(), - key.getWorkerTypeName(), - worker.getWorkerId(), - exitValue.get()); - ErrorMessage errorMessage = - ErrorMessage.builder() - .message(msg) - .logFile(worker.getLogFile()) - .logSizeLimit(4096) - .build(); - reporter.handle(Event.warn(errorMessage.toString())); - } else { - // Can't rule this out entirely, but it's not an unexpected death. - String msg = - String.format( - "%s %s (id %d) was destroyed, but is still in the worker pool.", - key.getMnemonic(), key.getWorkerTypeName(), worker.getWorkerId()); - reporter.handle(Event.info(msg)); - } + if (workerOptions.workerVerbose && worker.diedUnexpectedly()) { + String msg = + String.format( + "%s %s (id %d) has unexpectedly died with exit code %d.", + key.getMnemonic(), key.getWorkerTypeName(), worker.getWorkerId(), exitValue.get()); + ErrorMessage errorMessage = + ErrorMessage.builder() + .message(msg) + .logFile(worker.getLogFile()) + .logSizeLimit(4096) + .build(); + reporter.handle(Event.warn(errorMessage.toString())); } return false; } diff --git a/src/main/java/com/google/devtools/build/lib/worker/WorkerKey.java b/src/main/java/com/google/devtools/build/lib/worker/WorkerKey.java index c4741d6ed85f4a..d04ecaba4f2f0a 100644 --- a/src/main/java/com/google/devtools/build/lib/worker/WorkerKey.java +++ b/src/main/java/com/google/devtools/build/lib/worker/WorkerKey.java @@ -32,9 +32,13 @@ * break various things as well as render the workers less useful. */ final class WorkerKey { + /** Build options. */ private final ImmutableList args; + /** Environment variables. */ private final ImmutableMap env; + /** Execution root of Bazel process. */ private final Path execRoot; + /** Mnemonic of the worker. */ private final String mnemonic; /** @@ -43,8 +47,10 @@ final class WorkerKey { * methods. */ private final HashCode workerFilesCombinedHash; + /** Worker files with the corresponding hash code. */ private final SortedMap workerFilesWithHashes; - private final boolean mustBeSandboxed; + /** Set it to true if this job is running speculatively and thus likely to be interrupted. */ + private final boolean isSpeculative; /** A WorkerProxy will be instantiated if true, instantiate a regular Worker if false. */ private final boolean proxied; /** @@ -52,7 +58,7 @@ final class WorkerKey { * (ImmutableMap and ImmutableList do not cache their hashcodes. */ private final int hash; - + /** The format of the worker protocol sent to and read from the worker. */ private final WorkerProtocolFormat protocolFormat; WorkerKey( @@ -62,26 +68,17 @@ final class WorkerKey { String mnemonic, HashCode workerFilesCombinedHash, SortedMap workerFilesWithHashes, - boolean mustBeSandboxed, + boolean isSpeculative, boolean proxied, WorkerProtocolFormat protocolFormat) { - /** Build options. */ this.args = Preconditions.checkNotNull(args); - /** Environment variables. */ this.env = Preconditions.checkNotNull(env); - /** Execution root of Bazel process. */ this.execRoot = Preconditions.checkNotNull(execRoot); - /** Mnemonic of the worker. */ this.mnemonic = Preconditions.checkNotNull(mnemonic); - /** One combined hash code for all files. */ this.workerFilesCombinedHash = Preconditions.checkNotNull(workerFilesCombinedHash); - /** Worker files with the corresponding hash code. */ this.workerFilesWithHashes = Preconditions.checkNotNull(workerFilesWithHashes); - /** Set it to true if this job should be run in sandbox. */ - this.mustBeSandboxed = mustBeSandboxed; - /** Set it to true if this job should be run with WorkerProxy. */ + this.isSpeculative = isSpeculative; this.proxied = proxied; - /** The format of the worker protocol sent to and read from the worker. */ this.protocolFormat = protocolFormat; hash = calculateHashCode(); @@ -117,9 +114,9 @@ public SortedMap getWorkerFilesWithHashes() { return workerFilesWithHashes; } - /** Getter function for variable mustBeSandboxed. */ - public boolean mustBeSandboxed() { - return mustBeSandboxed; + /** Returns true if workers are run speculatively. */ + public boolean isSpeculative() { + return isSpeculative; } /** Getter function for variable proxied. */ @@ -128,7 +125,7 @@ public boolean getProxied() { } public boolean isMultiplex() { - return getProxied() && !mustBeSandboxed; + return getProxied() && !isSpeculative; } /** Returns the format of the worker protocol. */ @@ -147,7 +144,7 @@ public static String makeWorkerTypeName(boolean proxied, boolean mustBeSandboxed /** Returns a user-friendly name for this worker type. */ public String getWorkerTypeName() { - return makeWorkerTypeName(proxied, mustBeSandboxed); + return makeWorkerTypeName(proxied, isSpeculative); } @Override diff --git a/src/main/java/com/google/devtools/build/lib/worker/WorkerModule.java b/src/main/java/com/google/devtools/build/lib/worker/WorkerModule.java index 43a3c058b6fdce..a34c9bd0fb8445 100644 --- a/src/main/java/com/google/devtools/build/lib/worker/WorkerModule.java +++ b/src/main/java/com/google/devtools/build/lib/worker/WorkerModule.java @@ -20,7 +20,6 @@ import com.google.common.collect.ImmutableMap; import com.google.common.eventbus.Subscribe; import com.google.devtools.build.lib.buildtool.buildevent.BuildCompleteEvent; -import com.google.devtools.build.lib.buildtool.buildevent.BuildInterruptedEvent; import com.google.devtools.build.lib.buildtool.buildevent.BuildStartingEvent; import com.google.devtools.build.lib.events.Event; import com.google.devtools.build.lib.exec.ExecutionOptions; @@ -72,7 +71,8 @@ public void cleanStarting(CleanStartingEvent event) { this.options = event.getOptionsProvider().getOptions(WorkerOptions.class); workerFactory.setReporter(env.getReporter()); workerFactory.setOptions(options); - shutdownPool("Clean command is running, shutting down worker pool..."); + shutdownPool( + "Clean command is running, shutting down worker pool...", /* alwaysLog= */ false); } } @@ -179,31 +179,10 @@ public void registerSpawnStrategies( @Subscribe public void buildComplete(BuildCompleteEvent event) { if (options != null && options.workerQuitAfterBuild) { - shutdownPool("Build completed, shutting down worker pool..."); - } - } - - /** - * Stops any workers that are still executing. - * - *

This currently kills off some amount of workers, losing the warmed-up state. - * TODO(b/119701157): Cancel running workers instead (requires some way to reach each worker). - */ - @Subscribe - public void buildInterrupted(BuildInterruptedEvent event) { - if (workerPool != null) { - if ((options != null && options.workerVerbose)) { - env.getReporter().handle(Event.info("Build interrupted, stopping active workers...")); - } - workerPool.stopWork(); + shutdownPool("Build completed, shutting down worker pool...", /* alwaysLog= */ false); } } - /** Shuts down the worker pool and sets {#code workerPool} to null. */ - private void shutdownPool(String reason) { - shutdownPool(reason, /* alwaysLog= */ false); - } - /** Shuts down the worker pool and sets {#code workerPool} to null. */ private void shutdownPool(String reason, boolean alwaysLog) { Preconditions.checkArgument(!reason.isEmpty()); diff --git a/src/main/java/com/google/devtools/build/lib/worker/WorkerMultiplexer.java b/src/main/java/com/google/devtools/build/lib/worker/WorkerMultiplexer.java index 407a0b18c865e5..ab537e101becba 100644 --- a/src/main/java/com/google/devtools/build/lib/worker/WorkerMultiplexer.java +++ b/src/main/java/com/google/devtools/build/lib/worker/WorkerMultiplexer.java @@ -314,6 +314,8 @@ private boolean sendRequest() { * *

This is only called on the readResponses subthread and so cannot be interrupted by dynamic * execution cancellation, but only by a call to {@link #destroyProcess()}. + * + * @return True if the worker is still in a consistent state. */ private boolean readResponse() { WorkResponse parsedResponse; diff --git a/src/main/java/com/google/devtools/build/lib/worker/WorkerPool.java b/src/main/java/com/google/devtools/build/lib/worker/WorkerPool.java index abfa98c7ddf170..ba630a52578da9 100644 --- a/src/main/java/com/google/devtools/build/lib/worker/WorkerPool.java +++ b/src/main/java/com/google/devtools/build/lib/worker/WorkerPool.java @@ -189,28 +189,12 @@ private void waitForHighPriorityWorkersToFinish() throws InterruptedException { } } + /** + * Closes all the worker pools, destroying the workers in the process. This waits for any + * currently-ongoing work to finish. + */ public void close() { workerPools.values().forEach(GenericKeyedObjectPool::close); multiplexPools.values().forEach(GenericKeyedObjectPool::close); } - - /** Stops any ongoing work in the worker pools. This may entail killing the worker processes. */ - public void stopWork() { - workerPools - .values() - .forEach( - pool -> { - if (pool.getNumActive() > 0) { - pool.clear(); - } - }); - multiplexPools - .values() - .forEach( - pool -> { - if (pool.getNumActive() > 0) { - pool.clear(); - } - }); - } } diff --git a/src/main/java/com/google/devtools/build/lib/worker/WorkerSpawnRunner.java b/src/main/java/com/google/devtools/build/lib/worker/WorkerSpawnRunner.java index d7f1cc99308726..86798ec46f86c8 100644 --- a/src/main/java/com/google/devtools/build/lib/worker/WorkerSpawnRunner.java +++ b/src/main/java/com/google/devtools/build/lib/worker/WorkerSpawnRunner.java @@ -384,7 +384,6 @@ WorkResponse execInWorker( Worker worker = null; WorkResponse response; WorkRequest request; - ActionExecutionMetadata owner = spawn.getResourceOwner(); try { Stopwatch setupInputsStopwatch = Stopwatch.createStarted(); diff --git a/src/test/java/com/google/devtools/build/lib/worker/WorkerKeyTest.java b/src/test/java/com/google/devtools/build/lib/worker/WorkerKeyTest.java index 0a078b63a6fe16..e8d5aa6a0be816 100644 --- a/src/test/java/com/google/devtools/build/lib/worker/WorkerKeyTest.java +++ b/src/test/java/com/google/devtools/build/lib/worker/WorkerKeyTest.java @@ -23,7 +23,6 @@ import com.google.devtools.build.lib.actions.ExecutionRequirements.WorkerProtocolFormat; import com.google.devtools.build.lib.vfs.DigestHashFunction; import com.google.devtools.build.lib.vfs.FileSystem; -import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem; import org.junit.Test; import org.junit.runner.RunWith; @@ -34,40 +33,53 @@ public class WorkerKeyTest { final FileSystem fs = new InMemoryFileSystem(DigestHashFunction.SHA256); - Path workerBaseDir = fs.getPath("/outputbase/bazel-workers"); - WorkerKey workerKey = - new WorkerKey( - /* args= */ ImmutableList.of("arg1", "arg2", "arg3"), - /* env= */ ImmutableMap.of("env1", "foo", "env2", "bar"), - /* execRoot= */ fs.getPath("/outputbase/execroot/workspace"), - /* mnemonic= */ "dummy", - /* workerFilesCombinedHash= */ HashCode.fromInt(0), - /* workerFilesWithHashes= */ ImmutableSortedMap.of(), - /* mustBeSandboxed= */ true, - /* proxied= */ true, - WorkerProtocolFormat.PROTO); + private WorkerKey makeWorkerKey(boolean multiplex, boolean dynamic) { + return new WorkerKey( + /* args= */ ImmutableList.of("arg1", "arg2", "arg3"), + /* env= */ ImmutableMap.of("env1", "foo", "env2", "bar"), + /* execRoot= */ fs.getPath("/outputbase/execroot/workspace"), + /* mnemonic= */ "dummy", + /* workerFilesCombinedHash= */ HashCode.fromInt(0), + /* workerFilesWithHashes= */ ImmutableSortedMap.of(), + /* isSpeculative= */ dynamic, + /* proxied= */ multiplex, + WorkerProtocolFormat.PROTO); + } @Test public void testWorkerKeyGetter() { - assertThat(workerKey.mustBeSandboxed()).isTrue(); - assertThat(workerKey.getProxied()).isTrue(); - assertThat(workerKey.isMultiplex()).isFalse(); - assertThat(workerKey.getWorkerTypeName()).isEqualTo("worker"); - assertThat(WorkerKey.makeWorkerTypeName(/* proxied=*/ false, /* mustBeSandboxed=*/ false)) - .isEqualTo("worker"); - assertThat(WorkerKey.makeWorkerTypeName(/* proxied=*/ false, /* mustBeSandboxed=*/ true)) - .isEqualTo("worker"); - assertThat(WorkerKey.makeWorkerTypeName(/* proxied=*/ true, /* mustBeSandboxed=*/ false)) - .isEqualTo("multiplex-worker"); - assertThat(WorkerKey.makeWorkerTypeName(/* proxied=*/ true, /* mustBeSandboxed=*/ true)) - .isEqualTo("worker"); + WorkerKey keyNomultiNodynamic = makeWorkerKey(false, false); + assertThat(keyNomultiNodynamic.isSpeculative()).isFalse(); + assertThat(keyNomultiNodynamic.getProxied()).isFalse(); + assertThat(keyNomultiNodynamic.isMultiplex()).isFalse(); + assertThat(keyNomultiNodynamic.getWorkerTypeName()).isEqualTo("worker"); + + WorkerKey keyMultiNoDynamic = makeWorkerKey(true, false); + assertThat(keyMultiNoDynamic.isSpeculative()).isFalse(); + assertThat(keyMultiNoDynamic.getProxied()).isTrue(); + assertThat(keyMultiNoDynamic.isMultiplex()).isTrue(); + assertThat(keyMultiNoDynamic.getWorkerTypeName()).isEqualTo("multiplex-worker"); + + WorkerKey keyNoMultiDynamic = makeWorkerKey(false, true); + assertThat(keyNoMultiDynamic.isSpeculative()).isTrue(); + assertThat(keyNoMultiDynamic.getProxied()).isFalse(); + assertThat(keyNoMultiDynamic.isMultiplex()).isFalse(); + assertThat(keyNoMultiDynamic.getWorkerTypeName()).isEqualTo("worker"); + + WorkerKey keyMultiDynamic = makeWorkerKey(true, true); + assertThat(keyMultiDynamic.isSpeculative()).isTrue(); + assertThat(keyMultiDynamic.getProxied()).isTrue(); + assertThat(keyMultiDynamic.isMultiplex()).isFalse(); + assertThat(keyMultiDynamic.getWorkerTypeName()).isEqualTo("worker"); + // Hash code contains args, env, execRoot, proxied, and mnemonic. - assertThat(workerKey.hashCode()).isEqualTo(1605714200); - assertThat(workerKey.getProtocolFormat()).isEqualTo(WorkerProtocolFormat.PROTO); + assertThat(keyMultiDynamic.hashCode()).isEqualTo(1605714200); + assertThat(keyMultiDynamic.getProtocolFormat()).isEqualTo(WorkerProtocolFormat.PROTO); } @Test public void testWorkerKeyEquality() { + WorkerKey workerKey = makeWorkerKey(true, true); WorkerKey workerKeyWithSameFields = new WorkerKey( workerKey.getArgs(), @@ -76,7 +88,7 @@ public void testWorkerKeyEquality() { workerKey.getMnemonic(), workerKey.getWorkerFilesCombinedHash(), workerKey.getWorkerFilesWithHashes(), - workerKey.mustBeSandboxed(), + workerKey.isSpeculative(), workerKey.getProxied(), workerKey.getProtocolFormat()); assertThat(workerKey).isEqualTo(workerKeyWithSameFields); @@ -84,6 +96,7 @@ public void testWorkerKeyEquality() { @Test public void testWorkerKeyInequality_protocol() { + WorkerKey workerKey = makeWorkerKey(true, true); WorkerKey workerKeyWithDifferentProtocol = new WorkerKey( workerKey.getArgs(), @@ -92,7 +105,7 @@ public void testWorkerKeyInequality_protocol() { workerKey.getMnemonic(), workerKey.getWorkerFilesCombinedHash(), workerKey.getWorkerFilesWithHashes(), - workerKey.mustBeSandboxed(), + workerKey.isSpeculative(), workerKey.getProxied(), WorkerProtocolFormat.JSON); assertThat(workerKey).isNotEqualTo(workerKeyWithDifferentProtocol); diff --git a/src/test/java/com/google/devtools/build/lib/worker/WorkerPoolTest.java b/src/test/java/com/google/devtools/build/lib/worker/WorkerPoolTest.java index d1f426d8638169..036ca7b11be835 100644 --- a/src/test/java/com/google/devtools/build/lib/worker/WorkerPoolTest.java +++ b/src/test/java/com/google/devtools/build/lib/worker/WorkerPoolTest.java @@ -238,50 +238,4 @@ public void testBorrow_twoHiPrioBlocks() throws Exception { verify(factoryMock, times(2)).makeObject(workerKey1); verify(factoryMock, times(1)).makeObject(workerKey2); } - - @Test - public void testStopWork_activePoolsStopped() throws Exception { - WorkerPool pool = - new WorkerPool( - factoryMock, - // Have to declare the mnemonics, or they all fall into the default SimpleWorkerPool. - ImmutableMap.of("mnem1", 2, "mnem2", 2), - ImmutableMap.of("mnem2", 2, "mnem3", 2), - Lists.newArrayList()); - WorkerKey singleKey1 = createWorkerKey(fileSystem, "mnem1", false); - // These workers get borrowed, then both get destroyed in stopWork because they share mnemonic - WorkerKey singleKey1a = createWorkerKey(fileSystem, "mnem1", false, "anArg"); - pool.borrowObject(singleKey1); - Worker worker1a = pool.borrowObject(singleKey1a); - pool.returnObject(singleKey1a, worker1a); - WorkerKey singleKey2 = createWorkerKey(fileSystem, "mnem2", false); - // This worker gets borrowed, then returned, doesn't get destroyed in stopWork - Worker worker1 = pool.borrowObject(singleKey2); - pool.returnObject(singleKey2, worker1); - WorkerKey multiKey1 = createWorkerKey(fileSystem, "mnem2", true); - // This worker gets borrowed, then destroyed in stopWork, but separately from the singleplex - // worker2 even though they share a mnemonic. - pool.borrowObject(multiKey1); - WorkerKey multiKey2 = createWorkerKey(fileSystem, "mnem3", true); - // This worker gets borrowed, then returned, doesn't get destroyed during stopWork. - Worker worker2 = pool.borrowObject(multiKey2); - pool.returnObject(multiKey2, worker2); - verify(factoryMock, times(1)).makeObject(singleKey1); - verify(factoryMock, times(1)).makeObject(singleKey1a); - verify(factoryMock, times(1)).makeObject(singleKey2); - verify(factoryMock, times(1)).makeObject(multiKey1); - verify(factoryMock, times(1)).makeObject(multiKey2); - pool.stopWork(); - pool.borrowObject(singleKey1); - pool.borrowObject(singleKey1a); - pool.borrowObject(singleKey2); - pool.borrowObject(multiKey1); - pool.borrowObject(multiKey2); - // After stopWork, we had to create new workers for the keys that got their pools destroyed. - verify(factoryMock, times(2)).makeObject(singleKey1); - verify(factoryMock, times(2)).makeObject(singleKey1a); - verify(factoryMock, times(1)).makeObject(singleKey2); - verify(factoryMock, times(2)).makeObject(multiKey1); - verify(factoryMock, times(1)).makeObject(multiKey2); - } }