diff --git a/server/src/main/java/org/elasticsearch/TransportVersion.java b/server/src/main/java/org/elasticsearch/TransportVersion.java index fc029d9e722c9..3def6b7aac6d3 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersion.java +++ b/server/src/main/java/org/elasticsearch/TransportVersion.java @@ -132,12 +132,13 @@ private static TransportVersion registerTransportVersion(int id, String uniqueId public static final TransportVersion V_8_500_006 = registerTransportVersion(8_500_006, "7BB5621A-80AC-425F-BA88-75543C442F23"); public static final TransportVersion V_8_500_007 = registerTransportVersion(8_500_007, "77261d43-4149-40af-89c5-7e71e0454fce"); public static final TransportVersion V_8_500_008 = registerTransportVersion(8_500_008, "8884ab9d-94cd-4bac-aff8-01f2c394f47c"); + public static final TransportVersion V_8_500_009 = registerTransportVersion(8_500_009, "35091358-fd41-4106-a6e2-d2a1315494c1"); /** * Reference to the most recent transport version. * This should be the transport version with the highest id. */ - public static final TransportVersion CURRENT = findCurrent(V_8_500_008); + public static final TransportVersion CURRENT = findCurrent(V_8_500_009); /** * Reference to the earliest compatible transport version to this version of the codebase. diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java b/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java index 915585240d32e..4900f0a9ae26f 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java @@ -516,9 +516,17 @@ public Clusters(int total, int successful, int skipped, int remoteClusters, bool * We are not tracking number of remote clusters in this search. */ public Clusters(int total, int successful, int skipped) { - assert total >= 0 && successful >= 0 && skipped >= 0 + this(total, successful, skipped, true); + } + + /** + * @param finalState if true, then do an assert that total = successful + skipped. This is true + * only when the cluster is in its final state, not an initial or intermediate state. + */ + Clusters(int total, int successful, int skipped, boolean finalState) { + assert total >= 0 && successful >= 0 && skipped >= 0 && successful <= total : "total: " + total + " successful: " + successful + " skipped: " + skipped; - assert successful <= total && skipped == total - successful + assert finalState == false || skipped == total - successful : "total: " + total + " successful: " + successful + " skipped: " + skipped; this.total = total; this.successful = successful; @@ -527,8 +535,9 @@ public Clusters(int total, int successful, int skipped) { this.ccsMinimizeRoundtrips = false; } - private Clusters(StreamInput in) throws IOException { - this(in.readVInt(), in.readVInt(), in.readVInt()); + public Clusters(StreamInput in) throws IOException { + // when coming across the wire, we don't have context to know if this Cluster is in a final state, so set finalState=false + this(in.readVInt(), in.readVInt(), in.readVInt(), false); } @Override diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/MutableSearchResponse.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/MutableSearchResponse.java index ce7a620aa6d58..5a3c07ca521fb 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/MutableSearchResponse.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/MutableSearchResponse.java @@ -241,6 +241,11 @@ synchronized AsyncSearchResponse toAsyncSearchResponse(AsyncSearchTask task, lon * @return response representing the status of async search */ synchronized AsyncStatusResponse toStatusResponse(String asyncExecutionId, long startTime, long expirationTime) { + SearchResponse.Clusters clustersInStatus = null; + if (clusters != null && clusters.getTotal() > 0) { + // include clusters in the status if present and not Clusters.EMPTY (the case for local searches only) + clustersInStatus = clusters; + } if (finalResponse != null) { return new AsyncStatusResponse( asyncExecutionId, @@ -252,7 +257,8 @@ synchronized AsyncStatusResponse toStatusResponse(String asyncExecutionId, long finalResponse.getSuccessfulShards(), finalResponse.getSkippedShards(), finalResponse.getShardFailures() != null ? finalResponse.getShardFailures().length : 0, - finalResponse.status() + finalResponse.status(), + clustersInStatus ); } if (failure != null) { @@ -266,7 +272,8 @@ synchronized AsyncStatusResponse toStatusResponse(String asyncExecutionId, long successfulShards, skippedShards, queryFailures == null ? 0 : queryFailures.nonNullLength(), - ExceptionsHelper.status(ExceptionsHelper.unwrapCause(failure)) + ExceptionsHelper.status(ExceptionsHelper.unwrapCause(failure)), + clustersInStatus ); } return new AsyncStatusResponse( @@ -279,7 +286,8 @@ synchronized AsyncStatusResponse toStatusResponse(String asyncExecutionId, long successfulShards, skippedShards, queryFailures == null ? 0 : queryFailures.nonNullLength(), - null // for a still running search, completion status is null + null, // for a still running search, completion status is null + clustersInStatus ); } diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncStatusResponseTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncStatusResponseTests.java index 12231681f6470..b7561a166154b 100644 --- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncStatusResponseTests.java +++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncStatusResponseTests.java @@ -7,14 +7,18 @@ package org.elasticsearch.xpack.search; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.ShardSearchFailure; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.search.internal.InternalSearchResponse; import org.elasticsearch.test.AbstractWireSerializingTestCase; import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.core.search.action.AsyncSearchResponse; import org.elasticsearch.xpack.core.search.action.AsyncStatusResponse; import java.io.IOException; @@ -36,6 +40,12 @@ protected AsyncStatusResponse createTestInstance() { int skippedShards = randomIntBetween(0, 5); int failedShards = totalShards - successfulShards - skippedShards; RestStatus completionStatus = isRunning ? null : randomBoolean() ? RestStatus.OK : RestStatus.SERVICE_UNAVAILABLE; + SearchResponse.Clusters clusters = switch (randomIntBetween(0, 3)) { + case 1 -> SearchResponse.Clusters.EMPTY; + case 2 -> new SearchResponse.Clusters(1, 1, 0); + case 3 -> new SearchResponse.Clusters(4, 1, 0, 3, true); + default -> null; // case 0 + }; return new AsyncStatusResponse( id, isRunning, @@ -46,7 +56,8 @@ protected AsyncStatusResponse createTestInstance() { successfulShards, skippedShards, failedShards, - completionStatus + completionStatus, + clusters ); } @@ -61,6 +72,12 @@ protected AsyncStatusResponse mutateInstance(AsyncStatusResponse instance) { boolean isRunning = instance.isRunning() == false; boolean isPartial = isRunning ? randomBoolean() : false; RestStatus completionStatus = isRunning ? null : randomBoolean() ? RestStatus.OK : RestStatus.SERVICE_UNAVAILABLE; + SearchResponse.Clusters clusters = switch (randomIntBetween(0, 3)) { + case 1 -> SearchResponse.Clusters.EMPTY; + case 2 -> new SearchResponse.Clusters(1, 1, 0); + case 3 -> new SearchResponse.Clusters(4, 1, 0, 3, true); + default -> null; // case 0 + }; return new AsyncStatusResponse( instance.getId(), isRunning, @@ -71,43 +88,246 @@ protected AsyncStatusResponse mutateInstance(AsyncStatusResponse instance) { instance.getSuccessfulShards(), instance.getSkippedShards(), instance.getFailedShards(), - completionStatus + completionStatus, + clusters ); } public void testToXContent() throws IOException { AsyncStatusResponse response = createTestInstance(); try (XContentBuilder builder = XContentBuilder.builder(XContentType.JSON.xContent())) { - Object[] args = new Object[] { - response.getId(), - response.isRunning(), - response.isPartial(), - response.getStartTime(), - response.getExpirationTime(), - response.getTotalShards(), - response.getSuccessfulShards(), - response.getSkippedShards(), - response.getFailedShards(), - response.getCompletionStatus() == null ? "" : Strings.format(""" - ,"completion_status" : %s""", response.getCompletionStatus().getStatus()) }; - String expectedJson = Strings.format(""" - { - "id" : "%s", - "is_running" : %s, - "is_partial" : %s, - "start_time_in_millis" : %s, - "expiration_time_in_millis" : %s, - "_shards" : { - "total" : %s, - "successful" : %s, - "skipped" : %s, - "failed" : %s - } - %s - } - """, args); + String expectedJson; + SearchResponse.Clusters clusters = response.getClusters(); + if (clusters == null || clusters.getTotal() == 0) { + Object[] args = new Object[] { + response.getId(), + response.isRunning(), + response.isPartial(), + response.getStartTime(), + response.getExpirationTime(), + response.getTotalShards(), + response.getSuccessfulShards(), + response.getSkippedShards(), + response.getFailedShards(), + response.getCompletionStatus() == null ? "" : Strings.format(""" + ,"completion_status" : %s""", response.getCompletionStatus().getStatus()) }; + + expectedJson = Strings.format(""" + { + "id" : "%s", + "is_running" : %s, + "is_partial" : %s, + "start_time_in_millis" : %s, + "expiration_time_in_millis" : %s, + "_shards" : { + "total" : %s, + "successful" : %s, + "skipped" : %s, + "failed" : %s + } + %s + } + """, args); + } else { + Object[] args = new Object[] { + response.getId(), + response.isRunning(), + response.isPartial(), + response.getStartTime(), + response.getExpirationTime(), + response.getTotalShards(), + response.getSuccessfulShards(), + response.getSkippedShards(), + response.getFailedShards(), + clusters.getTotal(), + clusters.getSuccessful(), + clusters.getSkipped(), + response.getCompletionStatus() == null ? "" : Strings.format(""" + ,"completion_status" : %s""", response.getCompletionStatus().getStatus()) }; + + expectedJson = Strings.format(""" + { + "id" : "%s", + "is_running" : %s, + "is_partial" : %s, + "start_time_in_millis" : %s, + "expiration_time_in_millis" : %s, + "_shards" : { + "total" : %s, + "successful" : %s, + "skipped" : %s, + "failed" : %s + }, + "_clusters": { + "total": %s, + "successful": %s, + "skipped": %s + } + %s + } + """, args); + } response.toXContent(builder, ToXContent.EMPTY_PARAMS); assertEquals(XContentHelper.stripWhitespace(expectedJson), Strings.toString(builder)); } } + + public void testGetStatusFromStoredSearchRandomizedInputs() { + String searchId = randomSearchId(); + AsyncSearchResponse asyncSearchResponse = AsyncSearchResponseTests.randomAsyncSearchResponse( + searchId, + AsyncSearchResponseTests.randomSearchResponse() + ); + + if (asyncSearchResponse.getSearchResponse() == null + && asyncSearchResponse.getFailure() == null + && asyncSearchResponse.isRunning() == false) { + // if no longer running, the search should have recorded either a failure or a search response + // if not an Exception should be thrown + expectThrows( + IllegalStateException.class, + () -> AsyncStatusResponse.getStatusFromStoredSearch(asyncSearchResponse, 100, searchId) + ); + } else { + AsyncStatusResponse statusFromStoredSearch = AsyncStatusResponse.getStatusFromStoredSearch(asyncSearchResponse, 100, searchId); + assertNotNull(statusFromStoredSearch); + if (statusFromStoredSearch.isRunning()) { + assertNull( + "completion_status should only be present if search is no longer running", + statusFromStoredSearch.getCompletionStatus() + ); + } else { + assertNotNull( + "completion_status should be present if search is no longer running", + statusFromStoredSearch.getCompletionStatus() + ); + } + } + } + + public void testGetStatusFromStoredSearchFailureScenario() { + String searchId = randomSearchId(); + Exception error = new IllegalArgumentException("dummy"); + AsyncSearchResponse asyncSearchResponse = new AsyncSearchResponse(searchId, null, error, true, false, 100, 200); + AsyncStatusResponse statusFromStoredSearch = AsyncStatusResponse.getStatusFromStoredSearch(asyncSearchResponse, 100, searchId); + assertNotNull(statusFromStoredSearch); + assertEquals(statusFromStoredSearch.getCompletionStatus(), RestStatus.BAD_REQUEST); + assertTrue(statusFromStoredSearch.isPartial()); + assertNull(statusFromStoredSearch.getClusters()); + assertEquals(0, statusFromStoredSearch.getTotalShards()); + assertEquals(0, statusFromStoredSearch.getSuccessfulShards()); + assertEquals(0, statusFromStoredSearch.getSkippedShards()); + } + + public void testGetStatusFromStoredSearchFailedShardsScenario() { + String searchId = randomSearchId(); + + long tookInMillis = randomNonNegativeLong(); + int totalShards = randomIntBetween(1, Integer.MAX_VALUE); + int successfulShards = randomIntBetween(0, totalShards); + int skippedShards = randomIntBetween(0, successfulShards); + InternalSearchResponse internalSearchResponse = InternalSearchResponse.EMPTY_WITH_TOTAL_HITS; + SearchResponse.Clusters clusters = new SearchResponse.Clusters(100, 99, 1, 99, false); + SearchResponse searchResponse = new SearchResponse( + internalSearchResponse, + null, + totalShards, + successfulShards, + skippedShards, + tookInMillis, + new ShardSearchFailure[] { new ShardSearchFailure(new RuntimeException("foo")) }, + clusters + ); + + AsyncSearchResponse asyncSearchResponse = new AsyncSearchResponse(searchId, searchResponse, null, false, false, 100, 200); + AsyncStatusResponse statusFromStoredSearch = AsyncStatusResponse.getStatusFromStoredSearch(asyncSearchResponse, 100, searchId); + assertNotNull(statusFromStoredSearch); + assertEquals(1, statusFromStoredSearch.getFailedShards()); + assertEquals(statusFromStoredSearch.getCompletionStatus(), RestStatus.OK); + } + + public void testGetStatusFromStoredSearchWithEmptyClustersSuccessfullyCompleted() { + String searchId = randomSearchId(); + + long tookInMillis = randomNonNegativeLong(); + int totalShards = randomIntBetween(1, Integer.MAX_VALUE); + int successfulShards = randomIntBetween(0, totalShards); + int skippedShards = randomIntBetween(0, successfulShards); + InternalSearchResponse internalSearchResponse = InternalSearchResponse.EMPTY_WITH_TOTAL_HITS; + SearchResponse searchResponse = new SearchResponse( + internalSearchResponse, + null, + totalShards, + successfulShards, + skippedShards, + tookInMillis, + ShardSearchFailure.EMPTY_ARRAY, + SearchResponse.Clusters.EMPTY + ); + + AsyncSearchResponse asyncSearchResponse = new AsyncSearchResponse(searchId, searchResponse, null, false, false, 100, 200); + AsyncStatusResponse statusFromStoredSearch = AsyncStatusResponse.getStatusFromStoredSearch(asyncSearchResponse, 100, searchId); + assertNotNull(statusFromStoredSearch); + assertEquals(statusFromStoredSearch.getCompletionStatus(), RestStatus.OK); + assertNull(statusFromStoredSearch.getClusters()); + } + + public void testGetStatusFromStoredSearchWithNonEmptyClustersSuccessfullyCompleted() { + String searchId = randomSearchId(); + + long tookInMillis = randomNonNegativeLong(); + int totalShards = randomIntBetween(1, Integer.MAX_VALUE); + int successfulShards = randomIntBetween(0, totalShards); + int skippedShards = randomIntBetween(0, successfulShards); + InternalSearchResponse internalSearchResponse = InternalSearchResponse.EMPTY_WITH_TOTAL_HITS; + SearchResponse.Clusters clusters = new SearchResponse.Clusters(100, 99, 1, 99, false); + SearchResponse searchResponse = new SearchResponse( + internalSearchResponse, + null, + totalShards, + successfulShards, + skippedShards, + tookInMillis, + ShardSearchFailure.EMPTY_ARRAY, + clusters + ); + + AsyncSearchResponse asyncSearchResponse = new AsyncSearchResponse(searchId, searchResponse, null, false, false, 100, 200); + AsyncStatusResponse statusFromStoredSearch = AsyncStatusResponse.getStatusFromStoredSearch(asyncSearchResponse, 100, searchId); + assertNotNull(statusFromStoredSearch); + assertEquals(0, statusFromStoredSearch.getFailedShards()); + assertEquals(statusFromStoredSearch.getCompletionStatus(), RestStatus.OK); + assertEquals(100, statusFromStoredSearch.getClusters().getTotal()); + } + + public void testGetStatusFromStoredSearchWithNonEmptyClustersStillRunning() { + String searchId = randomSearchId(); + + long tookInMillis = randomNonNegativeLong(); + int totalShards = randomIntBetween(1, Integer.MAX_VALUE); + int successfulShards = randomIntBetween(0, totalShards); + int skippedShards = randomIntBetween(0, successfulShards); + InternalSearchResponse internalSearchResponse = InternalSearchResponse.EMPTY_WITH_TOTAL_HITS; + SearchResponse.Clusters clusters = new SearchResponse.Clusters(100, 2, 3, 99, true); + SearchResponse searchResponse = new SearchResponse( + internalSearchResponse, + null, + totalShards, + successfulShards, + skippedShards, + tookInMillis, + ShardSearchFailure.EMPTY_ARRAY, + clusters + ); + + boolean isRunning = true; + AsyncSearchResponse asyncSearchResponse = new AsyncSearchResponse(searchId, searchResponse, null, false, isRunning, 100, 200); + AsyncStatusResponse statusFromStoredSearch = AsyncStatusResponse.getStatusFromStoredSearch(asyncSearchResponse, 100, searchId); + assertNotNull(statusFromStoredSearch); + assertEquals(0, statusFromStoredSearch.getFailedShards()); + assertNull("completion_status should not be present if still running", statusFromStoredSearch.getCompletionStatus()); + assertEquals(100, statusFromStoredSearch.getClusters().getTotal()); + assertEquals(2, statusFromStoredSearch.getClusters().getSuccessful()); + assertEquals(3, statusFromStoredSearch.getClusters().getSkipped()); + } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncStatusResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncStatusResponse.java index a0388b048654c..b3eaaff917ede 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncStatusResponse.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncStatusResponse.java @@ -7,11 +7,13 @@ package org.elasticsearch.xpack.core.search.action; import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.TransportVersion; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.StatusToXContentObject; +import org.elasticsearch.core.Nullable; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.rest.action.RestActions; import org.elasticsearch.xcontent.XContentBuilder; @@ -36,6 +38,10 @@ public class AsyncStatusResponse extends ActionResponse implements SearchStatusR private final int failedShards; private final RestStatus completionStatus; + // Non-null for cross cluster searches + @Nullable + private final SearchResponse.Clusters clusters; + public AsyncStatusResponse( String id, boolean isRunning, @@ -46,7 +52,8 @@ public AsyncStatusResponse( int successfulShards, int skippedShards, int failedShards, - RestStatus completionStatus + RestStatus completionStatus, + SearchResponse.Clusters clusters ) { this.id = id; this.isRunning = isRunning; @@ -58,6 +65,7 @@ public AsyncStatusResponse( this.skippedShards = skippedShards; this.failedShards = failedShards; this.completionStatus = completionStatus; + this.clusters = clusters; } /** @@ -76,6 +84,7 @@ public static AsyncStatusResponse getStatusFromStoredSearch( int successfulShards = 0; int skippedShards = 0; int failedShards = 0; + SearchResponse.Clusters clusters = null; RestStatus completionStatus = null; SearchResponse searchResponse = asyncSearchResponse.getSearchResponse(); if (searchResponse != null) { @@ -83,15 +92,18 @@ public static AsyncStatusResponse getStatusFromStoredSearch( successfulShards = searchResponse.getSuccessfulShards(); skippedShards = searchResponse.getSkippedShards(); failedShards = searchResponse.getFailedShards(); + if (searchResponse.getClusters() != null && searchResponse.getClusters() != SearchResponse.Clusters.EMPTY) { + clusters = searchResponse.getClusters(); + } } if (asyncSearchResponse.isRunning() == false) { - if (searchResponse != null) { + Exception failure = asyncSearchResponse.getFailure(); + if (failure != null) { + completionStatus = ExceptionsHelper.status(ExceptionsHelper.unwrapCause(failure)); + } else if (searchResponse != null) { completionStatus = searchResponse.status(); } else { - Exception failure = asyncSearchResponse.getFailure(); - if (failure != null) { - completionStatus = ExceptionsHelper.status(ExceptionsHelper.unwrapCause(failure)); - } + throw new IllegalStateException("Unable to retrieve async_search status. No SearchResponse or Exception could be found."); } } return new AsyncStatusResponse( @@ -104,7 +116,8 @@ public static AsyncStatusResponse getStatusFromStoredSearch( successfulShards, skippedShards, failedShards, - completionStatus + completionStatus, + clusters ); } @@ -119,6 +132,11 @@ public AsyncStatusResponse(StreamInput in) throws IOException { this.skippedShards = in.readVInt(); this.failedShards = in.readVInt(); this.completionStatus = (this.isRunning == false) ? RestStatus.readFrom(in) : null; + if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_500_009)) { + this.clusters = in.readOptionalWriteable(SearchResponse.Clusters::new); + } else { + this.clusters = null; + } } @Override @@ -135,6 +153,10 @@ public void writeTo(StreamOutput out) throws IOException { if (isRunning == false) { RestStatus.writeTo(out, completionStatus); } + if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_500_009)) { + // optional since only CCS uses is; it is null for local-only searches + out.writeOptionalWriteable(clusters); + } } @Override @@ -151,6 +173,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.timeField("start_time_in_millis", "start_time", startTimeMillis); builder.timeField("expiration_time_in_millis", "expiration_time", expirationTimeMillis); RestActions.buildBroadcastShardsHeader(builder, params, totalShards, successfulShards, skippedShards, failedShards, null); + if (clusters != null) { + builder = clusters.toXContent(builder, null); + } if (isRunning == false) { // completion status information is only available for a completed search builder.field("completion_status", completionStatus.getStatus()); } @@ -172,7 +197,8 @@ public boolean equals(Object obj) { && successfulShards == other.successfulShards && skippedShards == other.skippedShards && failedShards == other.failedShards - && Objects.equals(completionStatus, other.completionStatus); + && Objects.equals(completionStatus, other.completionStatus) + && Objects.equals(clusters, other.clusters); } @Override @@ -187,7 +213,8 @@ public int hashCode() { successfulShards, skippedShards, failedShards, - completionStatus + completionStatus, + clusters ); } @@ -265,4 +292,12 @@ public int getFailedShards() { public RestStatus getCompletionStatus() { return completionStatus; } + + /** + * @return For CCS, clusters object that has information about the clustes being searched, such as total count + * successful count and skipped. Will be null for local-only searches. + */ + public SearchResponse.Clusters getClusters() { + return clusters; + } }