Skip to content
This repository has been archived by the owner on Aug 2, 2022. It is now read-only.

Commit

Permalink
Adjust resizable queue lower bounds to align with the default capacit…
Browse files Browse the repository at this point in the history 
…y(200/1000) suggested by ES (#483)
  • Loading branch information
@rguo-aws
rguo-aws committed Oct 22, 2020
1 parent f0155b5 commit 202a8a6
Show file tree
Hide file tree
Showing 6 changed files with 77 additions and 28 deletions.
41 changes: 34 additions & 7 deletions pa_config/rca_idle_master.conf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -109,22 +109,49 @@
// Default Values are added in the config specific file as well
// Modify them as well whenever changing the default values (DeciderConfig.java)
"decider-config-settings": {
// Decreasing order of priority for the type of workload we can expect on the cluster.
// Priority order in the list goes from most expected to the lease expected workload type.
"workload-type": {
"priority-order": ["ingest", "search"]
"prefer-ingest": true,
"prefer-search": true
},
// Decreasing order of priority for the type of cache which is expected to be consumed more.
// Priority order in the list goes from most used to the lease used cache type.
"cache-type": {
"priority-order": ["fielddata-cache", "shard-request-cache", "query-cache", "bitset-filter-cache"]
},
// cache decider - Needs to be updated as per the performance test results
"cache-bounds": {
"field-data-cache-upper-bound" : 0.4,
"shard-request-cache-upper-bound" : 0.05
"old-gen-decision-policy-config": {
"queue-bucket-size" : 20,
"old-gen-threshold-level-one" : 0.6,
"old-gen-threshold-level-two" : 0.75,
"old-gen-threshold-level-three" : 0.9
}
},
// Action Configurations
"action-config-settings": {
// Cache Max Size bounds are expressed as %age of JVM heap size
"cache-settings": {
"total-step-count" : 20,
"fielddata": {
"upper-bound": 0.4,
"lower-bound": 0.1
},
"shard-request": {
"upper-bound": 0.05,
"lower-bound": 0.01
}
},
// Queue Capacity bounds are expressed as absolute queue size
"queue-settings": {
"total-step-count" : 20,
"search": {
"upper-bound": 3000,
"lower-bound": 1000
},
"write": {
"upper-bound": 1000,
"lower-bound": 200
}
}
},
"bucketization": {
"old-gen": {
"UNDER_UTILIZED": 10.0,
Expand Down
2 changes: 1 addition & 1 deletion pa_config/rca_master.conf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -148,7 +148,7 @@
},
"write": {
"upper-bound": 1000,
"lower-bound": 50
"lower-bound": 200
}
}
},
Expand Down
8 changes: 4 additions & 4 deletions ...ro/elasticsearch/performanceanalyzer/decisionmaker/actions/configs/QueueActionConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,11 +36,11 @@
* "total-step-count": 20,
* "search": {
* "upper-bound": 3000,
* "lower-bound": 500
* "lower-bound": 1000
* },
* "write": {
* "upper-bound": 1000,
* "lower-bound": 50
* "lower-bound": 200
* }
* }
* }
Expand All @@ -58,9 +58,9 @@ public class QueueActionConfig {
private static final String TOTAL_STEP_COUNT_CONFIG_NAME = "total-step-count";
public static final int DEFAULT_TOTAL_STEP_COUNT = 20;
public static final int DEFAULT_SEARCH_QUEUE_UPPER_BOUND = 3000;
public static final int DEFAULT_SEARCH_QUEUE_LOWER_BOUND = 500;
public static final int DEFAULT_SEARCH_QUEUE_LOWER_BOUND = 1000;
public static final int DEFAULT_WRITE_QUEUE_UPPER_BOUND = 1000;
public static final int DEFAULT_WRITE_QUEUE_LOWER_BOUND = 50;
public static final int DEFAULT_WRITE_QUEUE_LOWER_BOUND = 200;

public QueueActionConfig(RcaConf conf) {
Map<String, Object> actionConfig = conf.getActionConfigSettings();
Expand Down
38 changes: 24 additions & 14 deletions ...asticsearch/performanceanalyzer/decisionmaker/deciders/jvm/LevelTwoActionBuilderTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.store.collector.NodeConfigCache;
import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.store.rca.cluster.NodeKey;
import java.util.List;
import java.util.Random;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
Expand Down Expand Up @@ -87,8 +88,10 @@ public void init() throws Exception {
public void testDownSizeAllResources() {
final double fielddataCacheSizeInPercent = 0.3;
final double shardRequestCacheSizeInPercent = 0.04;
final int writeQueueSize = 800;
final int searchQueueSize = 2000;
//bucket index = 2
final int writeQueueSize = generateQueueSize(ResourceEnum.WRITE_THREADPOOL, 6);
//bucket index = 2
final int searchQueueSize = generateQueueSize(ResourceEnum.SEARCH_THREADPOOL, 6);
dummyCache.put(node, ResourceUtil.FIELD_DATA_CACHE_MAX_SIZE,
(long) (heapMaxSizeInBytes * fielddataCacheSizeInPercent));
dummyCache.put(node, ResourceUtil.SHARD_REQUEST_CACHE_MAX_SIZE,
Expand Down Expand Up @@ -203,16 +206,26 @@ private void updateWorkLoadType(boolean preferIngest) throws Exception {
rcaConf.readConfigFromString(configStr);
}

//generate a random queue size within the given bucket index
private int generateQueueSize(ResourceEnum queueType, int index) {
Random rand = new Random();
int queueStepSize = rcaConf.getQueueActionConfig().getStepSize(queueType);
int lowerBound = rcaConf.getQueueActionConfig().getThresholdConfig(queueType).lowerBound();
return lowerBound + index * queueStepSize + rand.nextInt(queueStepSize);
}

@Test
public void testSameBucketsAndPreferIngest() throws Exception {
updateWorkLoadType(true);
final double fielddataCacheSizeInPercent = CacheActionConfig.DEFAULT_FIELDDATA_CACHE_LOWER_BOUND;
final double shardRequestCacheSizeInPercent = CacheActionConfig.DEFAULT_SHARD_REQUEST_CACHE_LOWER_BOUND;
// bucket size for search queue = 500 / write queue = 190
int writeQueueStepSize = rcaConf.getQueueActionConfig().getStepSize(ResourceEnum.WRITE_THREADPOOL);
int searchQueueStepSize = rcaConf.getQueueActionConfig().getStepSize(ResourceEnum.SEARCH_THREADPOOL);
// bucket size for search queue = 100 / write queue = 40
//bucket index = 2
final int writeQueueSize = 155;
final int writeQueueSize = generateQueueSize(ResourceEnum.WRITE_THREADPOOL, 2);
//bucket index = 2
final int searchQueueSize = 770;
final int searchQueueSize = generateQueueSize(ResourceEnum.SEARCH_THREADPOOL, 2);
dummyCache.put(node, ResourceUtil.FIELD_DATA_CACHE_MAX_SIZE,
(long) (heapMaxSizeInBytes * fielddataCacheSizeInPercent));
dummyCache.put(node, ResourceUtil.SHARD_REQUEST_CACHE_MAX_SIZE,
Expand Down Expand Up @@ -244,11 +257,10 @@ public void testSameBucketsAndPreferSearch() throws Exception {
updateWorkLoadType(false);
final double fielddataCacheSizeInPercent = CacheActionConfig.DEFAULT_FIELDDATA_CACHE_LOWER_BOUND;
final double shardRequestCacheSizeInPercent = CacheActionConfig.DEFAULT_SHARD_REQUEST_CACHE_LOWER_BOUND;
// bucket size for search queue = 500 / write queue = 190
//bucket index = 0
final int writeQueueSize = QueueActionConfig.DEFAULT_WRITE_QUEUE_LOWER_BOUND + 5;
final int writeQueueSize = generateQueueSize(ResourceEnum.WRITE_THREADPOOL, 0);
//bucket index = 0
final int searchQueueSize = QueueActionConfig.DEFAULT_SEARCH_QUEUE_LOWER_BOUND + 20;
final int searchQueueSize = generateQueueSize(ResourceEnum.SEARCH_THREADPOOL, 0);
dummyCache.put(node, ResourceUtil.FIELD_DATA_CACHE_MAX_SIZE,
(long) (heapMaxSizeInBytes * fielddataCacheSizeInPercent));
dummyCache.put(node, ResourceUtil.SHARD_REQUEST_CACHE_MAX_SIZE,
Expand Down Expand Up @@ -278,11 +290,10 @@ public void testSearchQueueHasLargerBucketIndex() throws Exception {
updateWorkLoadType(false);
final double fielddataCacheSizeInPercent = CacheActionConfig.DEFAULT_FIELDDATA_CACHE_LOWER_BOUND;
final double shardRequestCacheSizeInPercent = CacheActionConfig.DEFAULT_SHARD_REQUEST_CACHE_LOWER_BOUND;
// bucket size for search queue = 500 / write queue = 190
//bucket index = 1
final int writeQueueSize = 280;
final int writeQueueSize = generateQueueSize(ResourceEnum.WRITE_THREADPOOL, 1);
//bucket index = 5
final int searchQueueSize = 2100;
final int searchQueueSize = generateQueueSize(ResourceEnum.SEARCH_THREADPOOL, 5);
dummyCache.put(node, ResourceUtil.FIELD_DATA_CACHE_MAX_SIZE,
(long) (heapMaxSizeInBytes * fielddataCacheSizeInPercent));
dummyCache.put(node, ResourceUtil.SHARD_REQUEST_CACHE_MAX_SIZE,
Expand Down Expand Up @@ -314,11 +325,10 @@ public void testWriteQueueHasLargerBucketIndex() throws Exception {
updateWorkLoadType(true);
final double fielddataCacheSizeInPercent = CacheActionConfig.DEFAULT_FIELDDATA_CACHE_LOWER_BOUND;
final double shardRequestCacheSizeInPercent = CacheActionConfig.DEFAULT_SHARD_REQUEST_CACHE_LOWER_BOUND;
// bucket size for search queue = 500 / write queue = 190
//bucket index = 4
final int writeQueueSize = 690;
final int writeQueueSize = generateQueueSize(ResourceEnum.WRITE_THREADPOOL, 4);
//bucket index = 2
final int searchQueueSize = 900;
final int searchQueueSize = generateQueueSize(ResourceEnum.SEARCH_THREADPOOL, 2);
dummyCache.put(node, ResourceUtil.FIELD_DATA_CACHE_MAX_SIZE,
(long) (heapMaxSizeInBytes * fielddataCacheSizeInPercent));
dummyCache.put(node, ResourceUtil.SHARD_REQUEST_CACHE_MAX_SIZE,
Expand Down
8 changes: 7 additions & 1 deletion ...zer/rca/integTests/tests/queue_tuning/validator/QDeciderNoActionOnUnhealthyValidator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,17 +3,22 @@
import com.amazon.opendistro.elasticsearch.performanceanalyzer.AppContext;
import com.amazon.opendistro.elasticsearch.performanceanalyzer.decisionmaker.actions.ModifyQueueCapacityAction;
import com.amazon.opendistro.elasticsearch.performanceanalyzer.grpc.ResourceEnum;
import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.NodeRole;
import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.RcaControllerHelper;
import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.core.RcaConf;
import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.integTests.framework.api.IValidator;
import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.persistence.actions.PersistedAction;
import org.junit.Assert;

public class QDeciderNoActionOnUnhealthyValidator implements IValidator {
AppContext appContext;
RcaConf rcaConf;
long startTime;

public QDeciderNoActionOnUnhealthyValidator() {
appContext = new AppContext();
startTime = System.currentTimeMillis();
rcaConf = RcaControllerHelper.pickRcaConfForRole(NodeRole.ELECTED_MASTER);
}

/**
Expand Down Expand Up @@ -59,7 +64,8 @@ private boolean checkPersistedAction(final PersistedAction persistedAction) {
Assert.assertTrue(persistedAction.isActionable());
Assert.assertFalse(persistedAction.isMuted());
Assert.assertEquals(ResourceEnum.WRITE_THREADPOOL, modifyQueueCapacityAction.getThreadPool());
Assert.assertEquals(547, modifyQueueCapacityAction.getDesiredCapacity());
int writeQueueStepSize = rcaConf.getQueueActionConfig().getStepSize(ResourceEnum.WRITE_THREADPOOL);
Assert.assertEquals(500 + writeQueueStepSize, modifyQueueCapacityAction.getDesiredCapacity());
Assert.assertEquals(500, modifyQueueCapacityAction.getCurrentCapacity());
return true;
}
Expand Down
8 changes: 7 additions & 1 deletion ...erformanceanalyzer/rca/integTests/tests/queue_tuning/validator/QueueDeciderValidator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,17 +18,22 @@
import com.amazon.opendistro.elasticsearch.performanceanalyzer.AppContext;
import com.amazon.opendistro.elasticsearch.performanceanalyzer.decisionmaker.actions.ModifyQueueCapacityAction;
import com.amazon.opendistro.elasticsearch.performanceanalyzer.grpc.ResourceEnum;
import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.NodeRole;
import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.RcaControllerHelper;
import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.core.RcaConf;
import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.integTests.framework.api.IValidator;
import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.persistence.actions.PersistedAction;
import org.junit.Assert;

public class QueueDeciderValidator implements IValidator {
AppContext appContext;
RcaConf rcaConf;
long startTime;

public QueueDeciderValidator() {
appContext = new AppContext();
startTime = System.currentTimeMillis();
rcaConf = RcaControllerHelper.pickRcaConfForRole(NodeRole.ELECTED_MASTER);
}

/**
Expand Down Expand Up @@ -74,7 +79,8 @@ private boolean checkPersistedAction(final PersistedAction persistedAction) {
Assert.assertTrue(persistedAction.isActionable());
Assert.assertFalse(persistedAction.isMuted());
Assert.assertEquals(ResourceEnum.WRITE_THREADPOOL, modifyQueueCapacityAction.getThreadPool());
Assert.assertEquals(547, modifyQueueCapacityAction.getDesiredCapacity());
int writeQueueStepSize = rcaConf.getQueueActionConfig().getStepSize(ResourceEnum.WRITE_THREADPOOL);
Assert.assertEquals(500 + writeQueueStepSize, modifyQueueCapacityAction.getDesiredCapacity());
Assert.assertEquals(500, modifyQueueCapacityAction.getCurrentCapacity());
return true;
}
Expand Down

0 comments on commit 202a8a6

Please sign in to comment.