Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 2 additions & 31 deletions pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -369,37 +369,8 @@
</site>
</distributionManagement>

<repositories>
<repository>
<id>smx.m2</id>
<name>Apache ServiceMix M2</name>
<url>https://svn.apache.org/repos/asf/servicemix/m2-repo/</url>
</repository>
<!-- Apache snapshots -->
<repository>
<id>apache-snapshots</id>
<name>Apache Snapshots Repository</name>
<url>https://repository.apache.org/content/groups/snapshots-group</url>
<releases>
<enabled>false</enabled>
</releases>
<snapshots>
<enabled>true</enabled>
</snapshots>
</repository>
<!-- OPS4J SNAPSHOT repository -->
<repository>
<id>ops4j.sonatype.snapshots.deploy</id>
<name>OPS4J snapshot repository</name>
<url>https://oss.sonatype.org/content/repositories/ops4j-snapshots/</url>
<releases>
<enabled>false</enabled>
</releases>
<snapshots>
<enabled>true</enabled>
</snapshots>
</repository>
</repositories>
<!-- UNOMI-971: no project repositories — Central + Apache snapshots come from the parent POM.
Extra repos (ServiceMix SVN, OPS4J Sonatype snapshots) stalled CI on dead hosts. -->

<modules>
<module>bom</module>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,13 @@ public class TaskExecutionManager {
private final Map<String, Set<String>> executingTasksByType;
private final Map<String, LockRenewalHandle> activeLockRenewals = new ConcurrentHashMap<>();
private final AtomicBoolean running = new AtomicBoolean(false);
/**
* Set at the start of {@link #shutdown()} before canceling in-flight work. Failures that race
* with shutdown must not schedule retries: {@link ScheduledExecutorService#isShutdown()} only
* flips after {@code scheduler.shutdown()}, which runs after {@code future.cancel(true)} and
* can race with {@link #handleTaskError}.
*/
private final AtomicBoolean shuttingDown = new AtomicBoolean(false);
private ScheduledFuture<?> taskCheckerFuture;
private SchedulerServiceImpl schedulerService;
private TaskExecutorRegistry executorRegistry;
Expand Down Expand Up @@ -776,8 +783,10 @@ private void handleTaskError(ScheduledTask task, String error, long startTime) {
metricsManager.updateMetric(TaskMetricsManager.METRIC_TASKS_EXECUTION_TIME, executionTime);

if (scheduleRetry) {
// Only schedule retry if scheduler is not shutting down
if (!scheduler.isShutdown() && !scheduler.isTerminated()) {
// Only schedule retry if this manager is not shutting down. Check shuttingDown before
// scheduler.isShutdown(): canceling in-flight futures can invoke handleTaskError before
// scheduler.shutdown() runs, which would otherwise queue a retry that fires after teardown.
if (!shuttingDown.get() && !scheduler.isShutdown() && !scheduler.isTerminated()) {
try {
Runnable retryTask = () -> {
TaskExecutor executor = executorRegistry.getExecutor(task.getTaskType());
Expand Down Expand Up @@ -844,6 +853,8 @@ public void cancelTask(String taskId) {
* Shuts down the execution manager
*/
public void shutdown() {
// Mark before canceling futures so in-flight failures cannot schedule retries.
shuttingDown.set(true);
stopTaskChecker();

// Stop all lock heartbeats so held locks age out and peers can recover the work
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -324,8 +324,22 @@ public void testHandleTaskErrorSkipsRetryScheduleDuringShutdown() throws Excepti

executionManager.executeTask(task, executor);
assertTrue(inFlight.await(5, TimeUnit.SECONDS));
// Ignore the stubbing / any pre-shutdown registry lookups; we only care about retries after shutdown.
clearInvocations(executorRegistry);
// Release the in-flight task shortly after shutdown starts so awaitTermination can finish
// without waiting the full timeout (shutdown cancels futures then awaits the pool).
Thread releaser = new Thread(() -> {
try {
Thread.sleep(50);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
allowFail.countDown();
}, "shut-retry-releaser");
releaser.setDaemon(true);
releaser.start();
executionManager.shutdown();
allowFail.countDown();
releaser.join(2000);
Thread.sleep(200);
assertEquals(ScheduledTask.TaskStatus.SCHEDULED, task.getStatus());
assertEquals(1, task.getFailureCount());
Expand Down
Loading