Merge remote-tracking branch 'origin/main' into only_refresh_unpromot…

…ables_if_not_fast_refresh

build-tools-internal/version.properties

@@ -1,5 +1,5 @@
 elasticsearch     = 8.9.0
-lucene            = 9.7.0-snapshot-24df30cca69
+lucene            = 9.7.0-snapshot-41cd1f7a88c
 
 bundled_jdk_vendor = openjdk
 bundled_jdk = 20.0.1+9@b4887098932d415489976708ad6d1a4b

docs/changelog/96262.yaml

@@ -0,0 +1,5 @@
+pr: 96262
+summary: Fix `retry_on_conflict` parameter in update API to not retry indefinitely
+area: CRUD
+type: bug
+issues: []

docs/changelog/96550.yaml

@@ -0,0 +1,5 @@
+pr: 96550
+summary: "[Profiling] Allow to upgrade managed ILM policy"
+area: Application
+type: enhancement
+issues: []

docs/changelog/96613.yaml

@@ -0,0 +1,6 @@
+pr: 96613
+summary: Add `script` information to the cluster info endpoint
+area: Stats
+type: enhancement
+issues:
+ - 95394

docs/changelog/96741.yaml

@@ -0,0 +1,5 @@
+pr: 96741
+summary: Upgrade to new lucene snapshot 9.7.0-snapshot-a8602d6ef88
+area: Search
+type: upgrade
+issues: []

docs/reference/cluster/cluster-info.asciidoc

@@ -44,6 +44,9 @@ Ingest information.
 
 `thread_pool`::
 Statistics about each thread pool, including current size, queue size and rejected tasks.
+
+`script`::
+Contains script statistics of the cluster.
 --
 
 [role="child_attributes"]
@@ -282,6 +285,65 @@ Number of tasks completed by the thread pool executor.
 =======
 ======
 
+[[cluster-info-api-response-body-script]]
+`script`::
+(object)
+Contains script statistics of the cluster.
++
+.Properties of `script`
+[%collapsible%open]
+======
+`compilations`::
+(integer)
+Total number of inline script compilations performed by the cluster.
+
+`compilations_history`::
+(object)
+Contains the recent history of script compilations.
+
+.Properties of `compilations_history`
+[%collapsible%open]
+=======
+`5m`::
+(long)
+The number of script compilations in the last five minutes.
+`15m`::
+(long)
+The number of script compilations in the last fifteen minutes.
+`24h`::
+(long)
+The number of script compilations in the last twenty-four hours.
+=======
+
+`cache_evictions`::
+(integer)
+Total number of times the script cache has evicted old data.
+
+
+`cache_evictions_history`::
+(object)
+Contains the recent history of script cache evictions.
+
+.Properties of `cache_evictions`
+[%collapsible%open]
+=======
+`5m`::
+(long)
+The number of script cache evictions in the last five minutes.
+`15m`::
+(long)
+The number of script cache evictions in the last fifteen minutes.
+`24h`::
+(long)
+The number of script cache evictions in the last twenty-four hours.
+=======
+
+`compilation_limit_triggered`::
+(integer)
+Total number of times the <<script-compilation-circuit-breaker,script
+compilation>> circuit breaker has limited inline script compilations.
+======
+
 [[cluster-info-api-example]]
 ==== {api-examples-title}
 
@@ -299,6 +361,9 @@ GET /_info/ingest
 # returns the thread_pool info of the cluster
 GET /_info/thread_pool
 
+# returns the script info of the cluster
+GET /_info/script
+
 # returns the http and ingest info of the cluster
 GET /_info/http,ingest
 ----

docs/reference/search/search-your-data/knn-search.asciidoc

@@ -334,8 +334,6 @@ calculated on the combined set of `knn` and `query` matches.
 [[semantic-search]]
 ==== Perform semantic search
 
-experimental[]
-
 kNN search enables you to perform semantic search by using a previously deployed
 {ml-docs}/ml-nlp-search-compare.html#ml-nlp-text-embedding[text embedding model].
 Instead of literal matching on search terms, semantic search retrieves results

docs/reference/transform/apis/schedule-now-transform.asciidoc

@@ -21,13 +21,14 @@ Instantly runs a {transform} to process data.
 * Requires the `manage_transform` cluster privilege. This privilege is included
 in the `transform_admin` built-in role.
 
-[schedule-now-transform-desc]]
+[[schedule-now-transform-desc]]
 == {api-description-title}
 
-When you run this API, the {transform} processes the new data instantly,
-without waiting for the configured `frequency` interval.
-Subsequently, the transform will be processed again at
-`now + frequency` unless the API is called again in the meantime.
+When you run this API, processing for the next checkpoint is started immediately 
+without waiting for the configured `frequency` interval. The API returns 
+immediately, data processing happens in the background. Subsequently, the 
+{transform} will be processed again at `now + frequency` unless the API is 
+called again in the meantime.
 
 [[schedule-now-transform-path-parms]]
 == {api-path-parms-title}

modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CommonAnalysisFactoryTests.java

@@ -137,6 +137,7 @@ protected Map<String, Class<?>> getTokenFilters() {
         filters.put("germanstem", GermanStemTokenFilterFactory.class);
         // this filter is not exposed and should only be used internally
         filters.put("fixedshingle", Void.class);
+        filters.put("word2vecsynonym", Void.class); // not exposed
         return filters;
     }
 

rest-api-spec/src/main/resources/rest-api-spec/api/cluster.info.json

@@ -23,7 +23,8 @@
                               "_all",
                               "http",
                               "ingest",
-                              "thread_pool"
+                              "thread_pool",
+                              "script"
                           ],
                           "description":"Limit the information returned to the specified target."
                       }

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.info/10_info_all.yml

@@ -15,6 +15,7 @@ setup:
   - is_true: http
   - is_true: ingest
   - is_true: thread_pool
+  - is_true: script
 
 ---
 "Cluster Info fails when mixing _all with other targets":

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.info/40_info_script.yml

@@ -0,0 +1,16 @@
+---
+"Cluster HTTP Info":
+  - skip:
+      version: " - 8.8.99"
+      reason: "/_info/script only available from v8.9"
+
+  - do:
+      cluster.info:
+        target: [ script ]
+
+  - is_true: cluster_name
+  - is_true: script
+
+  - gte: { script.compilations: 0 }
+  - gte: { script.cache_evictions: 0 }
+  - gte: { script.compilation_limit_triggered: 0 }

server/src/main/java/org/elasticsearch/action/bulk/BulkPrimaryExecutionContext.java

@@ -58,7 +58,7 @@ enum ItemProcessingState {
     private ItemProcessingState currentItemState;
     private DocWriteRequest<?> requestToExecute;
     private BulkItemResponse executionResult;
-    private int retryCounter;
+    private int updateRetryCounter;
 
     BulkPrimaryExecutionContext(BulkShardRequest request, IndexShard primary) {
         this.request = request;
@@ -84,7 +84,7 @@ private void advance() {
             : "moving to next but current item wasn't completed (state: " + currentItemState + ")";
         currentItemState = ItemProcessingState.INITIAL;
         currentIndex = findNextNonAborted(currentIndex + 1);
-        retryCounter = 0;
+        updateRetryCounter = 0;
         requestToExecute = null;
         executionResult = null;
         assert assertInvariants(ItemProcessingState.INITIAL);
@@ -105,9 +105,9 @@ public BulkItemResponse getExecutionResult() {
         return executionResult;
     }
 
-    /** returns the number of times the current operation has been retried */
-    public int getRetryCounter() {
-        return retryCounter;
+    /** returns the number of times the current update operation has been retried */
+    public int getUpdateRetryCounter() {
+        return updateRetryCounter;
     }
 
     /** returns true if the request needs to wait for a mapping update to arrive from the master */
@@ -178,8 +178,19 @@ public void markAsRequiringMappingUpdate() {
         assert assertInvariants(ItemProcessingState.WAIT_FOR_MAPPING_UPDATE);
     }
 
+    public void resetForUpdateRetry() {
+        assert assertInvariants(ItemProcessingState.EXECUTED);
+        updateRetryCounter++;
+        resetForExecutionRetry();
+    }
+
+    public void resetForMappingUpdateRetry() {
+        assert assertInvariants(ItemProcessingState.WAIT_FOR_MAPPING_UPDATE);
+        resetForExecutionRetry();
+    }
+
     /** resets the current item state, prepare for a new execution */
-    public void resetForExecutionForRetry() {
+    private void resetForExecutionRetry() {
         assert assertInvariants(ItemProcessingState.WAIT_FOR_MAPPING_UPDATE, ItemProcessingState.EXECUTED);
         currentItemState = ItemProcessingState.INITIAL;
         requestToExecute = null;
@@ -292,7 +303,7 @@ private boolean assertInvariants(ItemProcessingState... expectedCurrentState) {
         assert Arrays.asList(expectedCurrentState).contains(currentItemState)
             : "expected current state [" + currentItemState + "] to be one of " + Arrays.toString(expectedCurrentState);
         assert currentIndex >= 0 : currentIndex;
-        assert retryCounter >= 0 : retryCounter;
+        assert updateRetryCounter >= 0 : updateRetryCounter;
         switch (currentItemState) {
             case INITIAL:
                 assert requestToExecute == null : requestToExecute;

server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java

@@ -385,7 +385,7 @@ public void onResponse(Void v) {
                         @Override
                         public void onResponse(Void v) {
                             assert context.requiresWaitingForMappingUpdate();
-                            context.resetForExecutionForRetry();
+                            context.resetForMappingUpdateRetry();
                         }
 
                         @Override
@@ -425,8 +425,8 @@ private static void onComplete(Engine.Result r, BulkPrimaryExecutionContext cont
         if (isUpdate
             && isFailed
             && isConflictException(executionResult.getFailure().getCause())
-            && context.getRetryCounter() < ((UpdateRequest) docWriteRequest).retryOnConflict()) {
-            context.resetForExecutionForRetry();
+            && context.getUpdateRetryCounter() < ((UpdateRequest) docWriteRequest).retryOnConflict()) {
+            context.resetForUpdateRetry();
             return;
         }
         final BulkItemResponse response;

server/src/main/java/org/elasticsearch/action/get/TransportGetAction.java

@@ -20,9 +20,9 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.routing.OperationRouting;
 import org.elasticsearch.cluster.routing.PlainShardIterator;
 import org.elasticsearch.cluster.routing.ShardIterator;
-import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -96,7 +96,10 @@ protected ShardIterator shards(ClusterState state, InternalRequest request) {
         if (iterator == null) {
             return null;
         }
-        return new PlainShardIterator(iterator.shardId(), iterator.getShardRoutings().stream().filter(ShardRouting::isSearchable).toList());
+        return new PlainShardIterator(
+            iterator.shardId(),
+            iterator.getShardRoutings().stream().filter(shardRouting -> OperationRouting.canSearchShard(shardRouting, state)).toList()
+        );
     }
 
     @Override
@@ -110,11 +113,13 @@ protected void asyncShardOperation(GetRequest request, ShardId shardId, ActionLi
         IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex());
         IndexShard indexShard = indexService.getShard(shardId.id());
         if (indexShard.routingEntry().isPromotableToPrimary() == false) {
+            assert indexShard.indexSettings().isFastRefresh() == false
+                : "a search shard should not receive a TransportGetAction for an index with fast refresh";
             handleGetOnUnpromotableShard(request, indexShard, listener);
             return;
         }
-        assert DiscoveryNode.isStateless(clusterService.getSettings()) == false
-            : "A TransportGetAction should always be handled by a search shard in Stateless";
+        assert DiscoveryNode.isStateless(clusterService.getSettings()) == false || indexShard.indexSettings().isFastRefresh()
+            : "in Stateless a promotable to primary shard can receive a TransportGetAction only if an index has the fast refresh setting";
         if (request.realtime()) { // we are not tied to a refresh cycle here anyway
             asyncGet(request, shardId, listener);
         } else {

server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java

@@ -14,6 +14,7 @@
 import org.apache.logging.log4j.core.config.Configurator;
 import org.apache.lucene.util.Constants;
 import org.apache.lucene.util.StringHelper;
+import org.apache.lucene.util.VectorUtil;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.support.SubscribableListener;
@@ -186,7 +187,9 @@ private static void initPhase2(Bootstrap bootstrap) throws IOException {
             ReferenceDocs.class,
             // The following classes use MethodHandles.lookup during initialization, load them now (before SM) to be sure they succeed
             AbstractRefCounted.class,
-            SubscribableListener.class
+            SubscribableListener.class,
+            // We eagerly initialize to work around log4j permissions & JDK-8309727
+            VectorUtil.class
         );
 
         // install SM after natives, shutdown hooks, etc.

server/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java

@@ -30,6 +30,8 @@
 import java.util.Set;
 import java.util.stream.Collectors;
 
+import static org.elasticsearch.index.IndexSettings.INDEX_FAST_REFRESH_SETTING;
+
 public class OperationRouting {
 
     public static final Setting<Boolean> USE_ADAPTIVE_REPLICA_SELECTION_SETTING = Setting.boolSetting(
@@ -122,8 +124,11 @@ public GroupShardsIterator<ShardIterator> searchShards(
                 nodeCounts
             );
             if (iterator != null) {
-                var searchableShards = iterator.getShardRoutings().stream().filter(ShardRouting::isSearchable).toList();
-                set.add(new PlainShardIterator(iterator.shardId(), searchableShards));
+                var shardsThatCanHandleSearches = iterator.getShardRoutings()
+                    .stream()
+                    .filter(shardRouting -> canSearchShard(shardRouting, clusterState))
+                    .toList();
+                set.add(new PlainShardIterator(iterator.shardId(), shardsThatCanHandleSearches));
             }
         }
         return GroupShardsIterator.sortAndCreate(new ArrayList<>(set));
@@ -262,4 +267,12 @@ public ShardId shardId(ClusterState clusterState, String index, String id, @Null
         IndexMetadata indexMetadata = indexMetadata(clusterState, index);
         return new ShardId(indexMetadata.getIndex(), IndexRouting.fromIndexMetadata(indexMetadata).getShard(id, routing));
     }
+
+    public static boolean canSearchShard(ShardRouting shardRouting, ClusterState clusterState) {
+        if (INDEX_FAST_REFRESH_SETTING.get(clusterState.metadata().index(shardRouting.index()).getSettings())) {
+            return shardRouting.isPromotableToPrimary();
+        } else {
+            return shardRouting.isSearchable();
+        }
+    }
 }

server/src/main/java/org/elasticsearch/index/IndexSettings.java

@@ -638,6 +638,7 @@ public Iterator<Setting<?>> settings() {
     private volatile Translog.Durability durability;
     private volatile TimeValue syncInterval;
     private volatile TimeValue refreshInterval;
+    private final boolean fastRefresh;
     private volatile ByteSizeValue flushThresholdSize;
     private volatile TimeValue flushThresholdAge;
     private volatile ByteSizeValue generationThresholdSize;
@@ -787,7 +788,8 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti
         defaultFields = scopedSettings.get(DEFAULT_FIELD_SETTING);
         syncInterval = INDEX_TRANSLOG_SYNC_INTERVAL_SETTING.get(settings);
         refreshInterval = scopedSettings.get(INDEX_REFRESH_INTERVAL_SETTING);
-        if (scopedSettings.get(INDEX_FAST_REFRESH_SETTING) && DiscoveryNode.isStateless(nodeSettings) == false) {
+        fastRefresh = scopedSettings.get(INDEX_FAST_REFRESH_SETTING);
+        if (fastRefresh && DiscoveryNode.isStateless(nodeSettings) == false) {
             throw new IllegalArgumentException(INDEX_FAST_REFRESH_SETTING.getKey() + " is allowed only in stateless");
         }
         flushThresholdSize = scopedSettings.get(INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING);
@@ -1119,6 +1121,13 @@ public TimeValue getRefreshInterval() {
         return refreshInterval;
     }
 
+    /**
+     * Only intended for stateless.
+     */
+    public boolean isFastRefresh() {
+        return fastRefresh;
+    }
+
     /**
      * Returns the transaction log threshold size when to forcefully flush the index and clear the transaction log.
      */