Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Set min translated per partition size to 1 instead of 2 #12824

Merged
merged 1 commit into from Apr 11, 2018
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
Expand Up @@ -53,7 +53,7 @@
public class EvictionChecker {

protected static final double ONE_HUNDRED_PERCENT = 100D;
private static final int MIN_SANE_PER_PARTITION_SIZE = 2;
private static final int MIN_TRANSLATED_PARTITION_SIZE = 1;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Minor: can it be just MIN_PARTITION_SIZE? I didn't like the "sane" attribute as it implicates that setting it too low is "insane" and I'm not sure what "translated" is referring to.


protected final ILogger logger;
protected final MapServiceContext mapServiceContext;
Expand Down Expand Up @@ -107,15 +107,15 @@ public boolean checkEvictable(RecordStore recordStore) {
}

protected boolean checkPerNodeEviction(RecordStore recordStore) {
double recordStoreSize = translatePerNodeSizeToRecordStoreSize(recordStore);
return recordStore.size() > recordStoreSize;
double maxAllowedStoreSize = translatePerNodeSizeToPartitionSize(recordStore);
return recordStore.size() > maxAllowedStoreSize;
}

/**
* Calculates and returns the expected maximum size of an evicted record-store
* when {@link com.hazelcast.config.MaxSizeConfig.MaxSizePolicy#PER_NODE PER_NODE} max-size-policy is used.
*/
public double translatePerNodeSizeToRecordStoreSize(RecordStore recordStore) {
public double translatePerNodeSizeToPartitionSize(RecordStore recordStore) {
MapConfig mapConfig = recordStore.getMapContainer().getMapConfig();
MaxSizeConfig maxSizeConfig = mapConfig.getMaxSizeConfig();
NodeEngine nodeEngine = mapServiceContext.getNodeEngine();
Expand All @@ -124,19 +124,19 @@ public double translatePerNodeSizeToRecordStoreSize(RecordStore recordStore) {
int memberCount = nodeEngine.getClusterService().getSize(DATA_MEMBER_SELECTOR);
int partitionCount = nodeEngine.getPartitionService().getPartitionCount();

double perNodeMaxRecordStoreSize = (1D * configuredMaxSize * memberCount / partitionCount);
if (perNodeMaxRecordStoreSize < 1) {
perNodeMaxRecordStoreSize = MIN_SANE_PER_PARTITION_SIZE;
double translatedPartitionSize = (1D * configuredMaxSize * memberCount / partitionCount);
if (translatedPartitionSize < 1) {
translatedPartitionSize = MIN_TRANSLATED_PARTITION_SIZE;
if (misconfiguredPerNodeMaxSizeWarningLogged.compareAndSet(false, true)) {
int minMaxSize = (int) Math.ceil((1D * partitionCount / memberCount));
int newSize = MIN_SANE_PER_PARTITION_SIZE * partitionCount / memberCount;
int newSize = MIN_TRANSLATED_PARTITION_SIZE * partitionCount / memberCount;
logger.warning(format("The max size configuration for map \"%s\" does not allow any data in the map. "
+ "Given the current cluster size of %d members with %d partitions, max size should be at "
+ "least %d. Map size is forced set to %d for backward compatibility", mapConfig.getName(),
memberCount, partitionCount, minMaxSize, newSize));
}
}
return perNodeMaxRecordStoreSize;
return translatedPartitionSize;
}

protected boolean checkPerPartitionEviction(String mapName, MaxSizeConfig maxSizeConfig, int partitionId) {
Expand Down
Expand Up @@ -65,7 +65,6 @@
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;


@RunWith(HazelcastParallelClassRunner.class)
@Category({QuickTest.class, ParallelTest.class})
public class EvictionMaxSizePolicyTest extends HazelcastTestSupport {
Expand All @@ -90,7 +89,7 @@ public void testPerNodePolicy_afterGracefulShutdown() {
int perNodeMaxSize = 1000;

// eviction takes place if a partitions size exceeds this number
// see EvictionChecker#translatePerNodeSizeToRecordStoreSize
// see EvictionChecker#translatePerNodeSizeToPartitionSize
double maxPartitionSize = 1D * nodeCount * perNodeMaxSize / PARTITION_COUNT;

String mapName = "testPerNodePolicy_afterGracefulShutdown";
Expand All @@ -113,15 +112,36 @@ public void testPerNodePolicy_afterGracefulShutdown() {
int mapSize = node.getMap(mapName).size();
String message = format("map size is %d and it should be smaller "
+ "than maxPartitionSize * PARTITION_COUNT which is %.0f",
mapSize, maxPartitionSize * PARTITION_COUNT);

System.err.println(message);
mapSize, maxPartitionSize * PARTITION_COUNT);

assertTrue(message, mapSize <= maxPartitionSize * PARTITION_COUNT);
}
}
}

/**
* Eviction starts if a partitions' size exceeds this number:
*
* double maxPartitionSize = 1D * nodeCount * perNodeMaxSize / PARTITION_COUNT;
*
* when calculated `maxPartitionSize` is under 1, we should forcibly set it
* to 1, otherwise all puts will immediately be removed by eviction.
*/
@Test
public void testPerNodePolicy_does_not_cause_unexpected_eviction_when_translated_partition_size_under_one() {
String mapName = randomMapName();
int maxSize = PARTITION_COUNT / 2;
Config config = createConfig(PER_NODE, maxSize, mapName);
HazelcastInstance node = createHazelcastInstance(config);
IMap map = node.getMap(mapName);

for (int i = 0; i < PARTITION_COUNT; i++) {
String keyForPartition = generateKeyForPartition(node, i);
map.put(keyForPartition, i);
assertEquals(i, map.get(keyForPartition));
}
}

@Test
public void testOwnerAndBackupEntryCountsAreEqualAfterEviction_whenPerNodeMaxSizePolicyIsUsed() throws Exception {
String mapName = randomMapName();
Expand Down