Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

ML: Fix error race condition on stop _all datafeeds and close _all jobs #38113

Merged
merged 3 commits into from Feb 1, 2019
Merged

ML: Fix error race condition on stop _all datafeeds and close _all jobs #38113

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
d86fb82
ML: Ignore when task is not found for _all
benwtrent Jan 31, 2019
2a9c194
Addressing PR comments
benwtrent Feb 1, 2019
23946c7
Update TransportStopDatafeedAction.java
benwtrent Feb 1, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
20 changes: 15 additions & 5 deletions ...ck/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportCloseJobAction.java
View file
Open in desktop
Expand Up @@ -6,6 +6,7 @@
package org.elasticsearch.xpack.ml.action;

import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.ResourceNotFoundException;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.ActionListenerResponseHandler;
import org.elasticsearch.action.FailedNodeException;
Expand All @@ -16,6 +17,7 @@
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.node.DiscoveryNodes;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.util.concurrent.AbstractRunnable;
import org.elasticsearch.common.util.concurrent.AtomicArray;
Expand Down Expand Up @@ -272,7 +274,12 @@ protected void taskOperation(CloseJobAction.Request request, TransportOpenJobAct
threadPool.executor(MachineLearning.UTILITY_THREAD_POOL_NAME).execute(new AbstractRunnable() {
@Override
public void onFailure(Exception e) {
listener.onFailure(e);
if (e instanceof ResourceNotFoundException && Strings.isAllOrWildcard(new String[]{request.getJobId()})) {
jobTask.closeJob("close job (api)");
listener.onResponse(new CloseJobAction.Response(true));
} else {
listener.onFailure(e);
}
}

@Override
Expand Down Expand Up @@ -331,10 +338,13 @@ public void onResponse(PersistentTasksCustomMetaData.PersistentTask<?> task) {

@Override
public void onFailure(Exception e) {
final int slot = counter.incrementAndGet();
failures.set(slot - 1, e);
if (slot == numberOfJobs) {
sendResponseOrFailure(request.getJobId(), listener, failures);
if ((e instanceof ResourceNotFoundException &&
Strings.isAllOrWildcard(new String[]{request.getJobId()})) == false) {
final int slot = counter.incrementAndGet();
benwtrent marked this conversation as resolved.
Show resolved Hide resolved
failures.set(slot - 1, e);
if (slot == numberOfJobs) {
benwtrent marked this conversation as resolved.
Show resolved Hide resolved
sendResponseOrFailure(request.getJobId(), listener, failures);
}
}
}

Expand Down
21 changes: 16 additions & 5 deletions ...lugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStopDatafeedAction.java
View file
Open in desktop
Expand Up @@ -16,6 +16,7 @@
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.node.DiscoveryNodes;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.util.concurrent.AbstractRunnable;
import org.elasticsearch.common.util.concurrent.AtomicArray;
Expand Down Expand Up @@ -186,10 +187,14 @@ public void onResponse(PersistentTasksCustomMetaData.PersistentTask<?> persisten

@Override
public void onFailure(Exception e) {
final int slot = counter.incrementAndGet();
failures.set(slot - 1, e);
if (slot == startedDatafeeds.size()) {
sendResponseOrFailure(request.getDatafeedId(), listener, failures);
if ((e instanceof ResourceNotFoundException &&
Strings.isAllOrWildcard(new String[]{request.getDatafeedId()})) == false) {

final int slot = counter.incrementAndGet();
failures.set(slot - 1, e);
benwtrent marked this conversation as resolved.
Show resolved Hide resolved
if (slot == startedDatafeeds.size()) {
sendResponseOrFailure(request.getDatafeedId(), listener, failures);
}
}
}
});
Expand All @@ -215,7 +220,13 @@ protected void taskOperation(StopDatafeedAction.Request request, TransportStartD
threadPool.executor(MachineLearning.UTILITY_THREAD_POOL_NAME).execute(new AbstractRunnable() {
@Override
public void onFailure(Exception e) {
listener.onFailure(e);
if ((e instanceof ResourceNotFoundException &&
Strings.isAllOrWildcard(new String[]{request.getDatafeedId()}))) {
datafeedTask.stop("stop_datafeed (api)", request.getStopTimeout());
listener.onResponse(new StopDatafeedAction.Response(true));
} else {
listener.onFailure(e);
}
}

@Override
Expand Down