Skip to content

Commit 3e96b6d

Browse files
davemoore-davemoore-
committed
Catch any ElasticsearchException, fail the job, and include the logged query in the response. Allow IndexNotFoundException and skip those indices in future hops.
1 parent 32baed6 commit 3e96b6d

File tree

1 file changed

+91
-37
lines changed
  • src/main/java/io/zentity/resolution

1 file changed

+91
-37
lines changed

‎src/main/java/io/zentity/resolution/Job.java‎

Lines changed: 91 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
package io.zentity.resolution;
22

3+
import com.fasterxml.jackson.core.JsonProcessingException;
34
import com.fasterxml.jackson.databind.JsonNode;
45
import com.fasterxml.jackson.databind.node.ArrayNode;
56
import com.fasterxml.jackson.databind.node.ObjectNode;
@@ -14,16 +15,20 @@
1415
import io.zentity.resolution.input.Term;
1516
import io.zentity.resolution.input.value.StringValue;
1617
import io.zentity.resolution.input.value.Value;
18+
import org.elasticsearch.ElasticsearchException;
1719
import org.elasticsearch.action.search.SearchAction;
1820
import org.elasticsearch.action.search.SearchRequestBuilder;
1921
import org.elasticsearch.action.search.SearchResponse;
2022
import org.elasticsearch.client.node.NodeClient;
23+
import org.elasticsearch.common.Strings;
2124
import org.elasticsearch.common.settings.Settings;
2225
import org.elasticsearch.common.xcontent.DeprecationHandler;
2326
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
27+
import org.elasticsearch.common.xcontent.ToXContent;
2428
import org.elasticsearch.common.xcontent.XContentFactory;
2529
import org.elasticsearch.common.xcontent.XContentParser;
2630
import org.elasticsearch.common.xcontent.XContentType;
31+
import org.elasticsearch.index.IndexNotFoundException;
2732
import org.elasticsearch.search.SearchModule;
2833
import org.elasticsearch.search.builder.SearchSourceBuilder;
2934

@@ -41,6 +46,7 @@
4146
import java.util.concurrent.atomic.AtomicInteger;
4247
import java.util.regex.Pattern;
4348

49+
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
4450
import static io.zentity.common.Patterns.COLON;
4551

4652
public class Job {
@@ -72,15 +78,55 @@ public class Job {
7278
private Map<String, Attribute> attributes = new TreeMap<>();
7379
private NodeClient client;
7480
private Map<String, Set<String>> docIds = new TreeMap<>();
81+
private String error = null;
7582
private List<String> hits = new ArrayList<>();
7683
private int hop = 0;
84+
private Set<String> missingIndices = new TreeSet<>();
7785
private List<String> queries = new ArrayList<>();
7886
private boolean ran = false;
7987

8088
public Job(NodeClient client) {
8189
this.client = client;
8290
}
8391

92+
public static String serializeElasticsearchException(ElasticsearchException e) throws IOException {
93+
String cause = Strings.toString(e.toXContent(jsonBuilder().startObject(), ToXContent.EMPTY_PARAMS).endObject());
94+
return "{\"error\":{\"root_cause\":[" + cause + "],\"type\":\"" + ElasticsearchException.getExceptionName(e) + "\",\"reason\":\"" + e.getMessage() + "\"},\"status\":" + e.status().getStatus() + "}";
95+
}
96+
97+
public static String serializeLoggedQuery(Input input, int _hop, int _query, String indexName, String request, String response, List<String> resolvers, TreeMap<Integer, TreeMap<String, TreeMap>> resolversFilterTreeGrouped, List<String> termResolvers, TreeMap<String, TreeMap> termResolversFilterTree) throws JsonProcessingException {
98+
List<String> filtersLoggedList = new ArrayList<>();
99+
if (!resolvers.isEmpty() && !resolversFilterTreeGrouped.isEmpty()) {
100+
List<String> attributesResolversSummary = new ArrayList<>();
101+
for (String resolverName : resolvers) {
102+
List<String> resolversAttributes = new ArrayList<>();
103+
for (String attributeName : input.model().resolvers().get(resolverName).attributes())
104+
resolversAttributes.add("\"" + attributeName + "\"");
105+
attributesResolversSummary.add("\"" + resolverName + "\":{\"attributes\":[" + String.join(",", resolversAttributes) + "]}");
106+
}
107+
String attributesResolversFilterTreeLogged = Json.ORDERED_MAPPER.writeValueAsString(resolversFilterTreeGrouped);
108+
filtersLoggedList.add("\"attributes\":{\"tree\":" + attributesResolversFilterTreeLogged + ",\"resolvers\":{" + String.join(",", attributesResolversSummary) + "}}");
109+
} else {
110+
filtersLoggedList.add("\"attributes\":null");
111+
}
112+
if (!termResolvers.isEmpty() && !termResolversFilterTree.isEmpty()) {
113+
List<String> termsResolversSummary = new ArrayList<>();
114+
for (String resolverName : termResolvers) {
115+
List<String> resolverAttributes = new ArrayList<>();
116+
for (String attributeName : input.model().resolvers().get(resolverName).attributes())
117+
resolverAttributes.add("\"" + attributeName + "\"");
118+
termsResolversSummary.add("\"" + resolverName + "\":{\"attributes\":[" + String.join(",", resolverAttributes) + "]}");
119+
}
120+
String termResolversFilterTreeLogged = Json.ORDERED_MAPPER.writeValueAsString(termResolversFilterTree);
121+
filtersLoggedList.add("\"terms\":{\"tree\":{\"0\":" + termResolversFilterTreeLogged + "},\"resolvers\":{" + String.join(",", termsResolversSummary) + "}}");
122+
} else {
123+
filtersLoggedList.add("\"terms\":null");
124+
}
125+
String filtersLogged = String.join(",", filtersLoggedList);
126+
String searchLogged = "{\"request\":" + request + ",\"response\":" + response + "}";
127+
return "{\"_hop\":" + _hop + ",\"_query\":" + _query + ",\"_index\":\"" + indexName + "\",\"filters\":{" + filtersLogged + "},\"search\":" + searchLogged + "}";
128+
}
129+
84130
public static String makeScriptFieldsClause(Input input, String indexName) throws ValidationException {
85131
List<String> scriptFieldClauses = new ArrayList<>();
86132

@@ -468,8 +514,10 @@ public static TreeMap<Integer, List<String>> groupResolversByWeight(Model model,
468514
private void resetState() {
469515
this.attributes = new TreeMap<>(this.input().attributes());
470516
this.docIds = new TreeMap<>();
517+
this.error = null;
471518
this.hits = new ArrayList<>();
472519
this.hop = 0;
520+
this.missingIndices = new TreeSet<>();
473521
this.queries = new ArrayList<>();
474522
this.ran = false;
475523
}
@@ -589,6 +637,10 @@ private void traverse() throws IOException, ValidationException {
589637
// Construct a query for each index that maps to a resolver.
590638
for (String indexName : this.input.model().indices().keySet()) {
591639

640+
// Skip this index if a prior hop determined the index to be missing.
641+
if (this.missingIndices.contains(indexName))
642+
continue;
643+
592644
// Track _ids for this index.
593645
if (!this.docIds.containsKey(indexName))
594646
this.docIds.put(indexName, new TreeSet<>());
@@ -933,55 +985,55 @@ else if (!resolversClause.isEmpty())
933985
query = "{" + String.join(",", topLevelClauses) + "}";
934986

935987
// Submit query to Elasticsearch.
936-
SearchResponse response = this.search(indexName, query);
988+
SearchResponse response = null;
989+
ElasticsearchException responseError = null;
990+
boolean fatalError = false;
991+
try {
992+
response = this.search(indexName, query);
993+
} catch (IndexNotFoundException e) {
994+
// Don't fail the job if an index was missing.
995+
this.missingIndices.add(e.getIndex().getName());
996+
responseError = e;
997+
} catch (ElasticsearchException e) {
998+
// Fail the job for any other error.
999+
fatalError = true;
1000+
responseError = e;
1001+
}
9371002

9381003
// Read response from Elasticsearch.
939-
String responseBody = response.toString();
940-
JsonNode responseData = Json.ORDERED_MAPPER.readTree(responseBody);
1004+
JsonNode responseData = null;
1005+
if (response != null)
1006+
responseData = Json.ORDERED_MAPPER.readTree(response.toString());
9411007

9421008
// Log queries.
9431009
if (this.includeQueries || this.profile) {
944-
JsonNode responseDataCopy = responseData.deepCopy();
945-
ObjectNode responseDataCopyObj = (ObjectNode) responseDataCopy;
946-
if (responseDataCopyObj.has("hits")) {
947-
ObjectNode responseDataCopyObjHits = (ObjectNode) responseDataCopyObj.get("hits");
948-
if (responseDataCopyObjHits.has("hits"))
949-
responseDataCopyObjHits.remove("hits");
950-
}
951-
List<String> filtersLoggedList = new ArrayList<>();
952-
if (!resolvers.isEmpty() && !resolversFilterTreeGrouped.isEmpty()) {
953-
List<String> attributesResolversSummary = new ArrayList<>();
954-
for (String resolverName : resolvers) {
955-
List<String> resolversAttributes = new ArrayList<>();
956-
for (String attributeName : input.model().resolvers().get(resolverName).attributes())
957-
resolversAttributes.add("\"" + attributeName + "\"");
958-
attributesResolversSummary.add("\"" + resolverName + "\":{\"attributes\":[" + String.join(",", resolversAttributes) + "]}");
1010+
String responseString;
1011+
if (responseData != null) {
1012+
JsonNode responseDataCopy = responseData.deepCopy();
1013+
ObjectNode responseDataCopyObj = (ObjectNode) responseDataCopy;
1014+
if (responseDataCopyObj.has("hits")) {
1015+
ObjectNode responseDataCopyObjHits = (ObjectNode) responseDataCopyObj.get("hits");
1016+
if (responseDataCopyObjHits.has("hits"))
1017+
responseDataCopyObjHits.remove("hits");
9591018
}
960-
String attributesResolversFilterTreeLogged = Json.ORDERED_MAPPER.writeValueAsString(resolversFilterTreeGrouped);
961-
filtersLoggedList.add("\"attributes\":{\"tree\":" + attributesResolversFilterTreeLogged + ",\"resolvers\":{" + String.join(",", attributesResolversSummary) + "}}");
1019+
responseString = responseDataCopyObj.toString();
9621020
} else {
963-
filtersLoggedList.add("\"attributes\":null");
1021+
responseString = serializeElasticsearchException(responseError);
9641022
}
965-
if (!termResolvers.isEmpty() && !termResolversFilterTree.isEmpty()) {
966-
List<String> termsResolversSummary = new ArrayList<>();
967-
for (String resolverName : termResolvers) {
968-
List<String> resolverAttributes = new ArrayList<>();
969-
for (String attributeName : input.model().resolvers().get(resolverName).attributes())
970-
resolverAttributes.add("\"" + attributeName + "\"");
971-
termsResolversSummary.add("\"" + resolverName + "\":{\"attributes\":[" + String.join(",", resolverAttributes) + "]}");
972-
}
973-
String termResolversFilterTreeLogged = Json.ORDERED_MAPPER.writeValueAsString(termResolversFilterTree);
974-
filtersLoggedList.add("\"terms\":{\"tree\":{\"0\":" + termResolversFilterTreeLogged + "},\"resolvers\":{" + String.join(",", termsResolversSummary) + "}}");
975-
} else {
976-
filtersLoggedList.add("\"terms\":null");
977-
}
978-
String filtersLogged = String.join(",", filtersLoggedList);
979-
String searchLogged = "{\"request\":" + query + ",\"response\":" + responseDataCopyObj + "}";
980-
String logged = "{\"_hop\":" + this.hop + ",\"_query\":" + _query + ",\"_index\":\"" + indexName + "\",\"filters\":{" + filtersLogged + "},\"search\":" + searchLogged + "}";
1023+
String logged = serializeLoggedQuery(this.input, this.hop, _query, indexName, query, responseString, resolvers, resolversFilterTreeGrouped, termResolvers, termResolversFilterTree);
9811024
this.queries.add(logged);
9821025
}
9831026

1027+
// Stop traversing if there was an error not due to a missing index.
1028+
// Include the logged query in the response.
1029+
if (fatalError) {
1030+
this.error = serializeLoggedQuery(this.input, this.hop, _query, indexName, query, serializeElasticsearchException(responseError), resolvers, resolversFilterTreeGrouped, termResolvers, termResolversFilterTree);
1031+
return;
1032+
}
1033+
9841034
// Read the hits
1035+
if (responseData == null)
1036+
continue;
9851037
if (!responseData.has("hits"))
9861038
continue;
9871039
if (!responseData.get("hits").has("hits"))
@@ -1183,6 +1235,8 @@ public String run() throws IOException, ValidationException {
11831235
// Format response
11841236
List<String> responseParts = new ArrayList<>();
11851237
responseParts.add("\"took\":" + Long.toString(took));
1238+
if (this.error != null)
1239+
responseParts.add("\"error\":" + this.error);
11861240
if (this.includeHits)
11871241
responseParts.add("\"hits\":{\"total\":" + this.hits.size() + ",\"hits\":[" + String.join(",", this.hits) + "]}");
11881242
if (this.includeQueries || this.profile)

0 commit comments

Comments
 (0)