-
Notifications
You must be signed in to change notification settings - Fork 24.3k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Browse files
Browse the repository at this point in the history
…0284) Data frame analytics jobs that work with very large datasets may produce bulk requests that are over the memory limit for indexing. This commit adds a helper class that bundles index requests in bulk requests that steer away from the memory limit. We then use this class both from the results joiner and the inference runner ensuring data frame analytics jobs do not generate bulk requests that are too large. Note the limit was implemented in #58885. Backport of #60219
- Loading branch information
1 parent
6ae66b1
commit 0ccdcf9
Showing
9 changed files
with
224 additions
and
55 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
66 changes: 66 additions & 0 deletions
66
.../ml/src/main/java/org/elasticsearch/xpack/ml/utils/persistence/LimitAwareBulkIndexer.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,66 @@ | ||
| /* | ||
| * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one | ||
| * or more contributor license agreements. Licensed under the Elastic License; | ||
| * you may not use this file except in compliance with the Elastic License. | ||
| */ | ||
|
|
||
| package org.elasticsearch.xpack.ml.utils.persistence; | ||
|
|
||
| import org.apache.logging.log4j.LogManager; | ||
| import org.apache.logging.log4j.Logger; | ||
| import org.elasticsearch.action.bulk.BulkRequest; | ||
| import org.elasticsearch.action.index.IndexRequest; | ||
| import org.elasticsearch.common.settings.Settings; | ||
| import org.elasticsearch.index.IndexingPressure; | ||
|
|
||
| import java.util.Objects; | ||
| import java.util.function.Consumer; | ||
|
|
||
| /** | ||
| * A helper class that gathers index requests in bulk requests | ||
| * that do exceed a 1000 operations or half the available memory | ||
| * limit for indexing. | ||
| */ | ||
| public class LimitAwareBulkIndexer implements AutoCloseable { | ||
|
|
||
| private static final Logger LOGGER = LogManager.getLogger(LimitAwareBulkIndexer.class); | ||
|
|
||
| private static final int BATCH_SIZE = 1000; | ||
|
|
||
| private final long bytesLimit; | ||
| private final Consumer<BulkRequest> executor; | ||
| private BulkRequest currentBulkRequest = new BulkRequest(); | ||
| private long currentRamBytes; | ||
|
|
||
| public LimitAwareBulkIndexer(Settings settings, Consumer<BulkRequest> executor) { | ||
| this((long) Math.ceil(0.5 * IndexingPressure.MAX_INDEXING_BYTES.get(settings).getBytes()), executor); | ||
| } | ||
|
|
||
| LimitAwareBulkIndexer(long bytesLimit, Consumer<BulkRequest> executor) { | ||
| this.bytesLimit = bytesLimit; | ||
| this.executor = Objects.requireNonNull(executor); | ||
| } | ||
|
|
||
| public void addAndExecuteIfNeeded(IndexRequest indexRequest) { | ||
| if (currentRamBytes + indexRequest.ramBytesUsed() > bytesLimit || currentBulkRequest.numberOfActions() == BATCH_SIZE) { | ||
| execute(); | ||
| } | ||
| currentBulkRequest.add(indexRequest); | ||
| currentRamBytes += indexRequest.ramBytesUsed(); | ||
| } | ||
|
|
||
| private void execute() { | ||
| if (currentBulkRequest.numberOfActions() > 0) { | ||
| LOGGER.debug("Executing bulk request; current bytes [{}]; bytes limit [{}]; number of actions [{}]", | ||
| currentRamBytes, bytesLimit, currentBulkRequest.numberOfActions()); | ||
| executor.accept(currentBulkRequest); | ||
| currentBulkRequest = new BulkRequest(); | ||
| currentRamBytes = 0; | ||
| } | ||
| } | ||
|
|
||
| @Override | ||
| public void close() { | ||
| execute(); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.