Skip to content
Permalink
Browse files
IGNITE-16392 PageMemory data regions configuration; porting of PageMe…
…moryNoStoreImpl and all other necessary components. (#591)
  • Loading branch information
ibessonov committed Jan 28, 2022
1 parent c99f7a8 commit 3b7af82338e9687d008302822f0a2ff6a3eb0a61
Showing 35 changed files with 3,449 additions and 67 deletions.
@@ -22,6 +22,9 @@
import org.apache.ignite.configuration.annotation.ConfigurationType;
import org.apache.ignite.configuration.schemas.runner.ClusterConfiguration;
import org.apache.ignite.configuration.schemas.store.DataStorageConfiguration;
import org.apache.ignite.configuration.schemas.store.PageMemoryDataRegionConfigurationSchema;
import org.apache.ignite.configuration.schemas.store.RocksDbDataRegionConfigurationSchema;
import org.apache.ignite.configuration.schemas.store.UnsafeMemoryAllocatorConfigurationSchema;
import org.apache.ignite.configuration.schemas.table.HashIndexConfigurationSchema;
import org.apache.ignite.configuration.schemas.table.PartialIndexConfigurationSchema;
import org.apache.ignite.configuration.schemas.table.SortedIndexConfigurationSchema;
@@ -54,7 +57,10 @@ public Collection<Class<?>> polymorphicSchemaExtensions() {
return List.of(
HashIndexConfigurationSchema.class,
SortedIndexConfigurationSchema.class,
PartialIndexConfigurationSchema.class
PartialIndexConfigurationSchema.class,
RocksDbDataRegionConfigurationSchema.class,
PageMemoryDataRegionConfigurationSchema.class,
UnsafeMemoryAllocatorConfigurationSchema.class
);
}
}
@@ -17,48 +17,24 @@

package org.apache.ignite.configuration.schemas.store;

import org.apache.ignite.configuration.annotation.Config;
import org.apache.ignite.configuration.annotation.Value;
import static org.apache.ignite.configuration.schemas.store.RocksDbDataRegionConfigurationSchema.ROCKSDB_DATA_REGION_TYPE;

import org.apache.ignite.configuration.annotation.InjectedName;
import org.apache.ignite.configuration.annotation.PolymorphicConfig;
import org.apache.ignite.configuration.annotation.PolymorphicId;
import org.apache.ignite.configuration.validation.Immutable;
import org.apache.ignite.configuration.validation.Min;
import org.apache.ignite.configuration.validation.OneOf;

/**
* Configuration schema for data region. Currently it represents configuration for rocksdb storage engine only.
* Configuration schema for data region.
*/
@Config
@PolymorphicConfig
public class DataRegionConfigurationSchema {
/** Type of the RocksDB data region. */
public static final String ROCKSDB_DATA_REGION_TYPE = "rocksdb";

/** Cache type for the RocksDB LRU cache. */
public static final String ROCKSDB_LRU_CACHE = "lru";

/** Cache type for the RocksDB LRU cache. */
public static final String ROCKSDB_CLOCK_CACHE = "clock";

/** Type for the future polymorphic configuration schemas. */
@Immutable
@OneOf(ROCKSDB_DATA_REGION_TYPE)
@Value(hasDefault = true)
@PolymorphicId(hasDefault = true)
public String type = ROCKSDB_DATA_REGION_TYPE;

/** Size of the rocksdb offheap cache. */
@Value(hasDefault = true)
public long size = 256 * 1024 * 1024;

/** Size of rocksdb write buffer. */
@Value(hasDefault = true)
@Min(1)
public long writeBufferSize = 64 * 1024 * 1024;

/** Cache type - only {@code LRU} is supported at the moment. {@code Clock} implementation has known bugs. */
@OneOf({ROCKSDB_LRU_CACHE})
@Value(hasDefault = true)
public String cache = ROCKSDB_LRU_CACHE;

/** The cache is sharded to 2^numShardBits shards, by hash of the key. */
@Min(-1)
@Value(hasDefault = true)
public int numShardBits = -1;
/** Name of the data region. */
@InjectedName
public String name;
}
@@ -20,6 +20,7 @@
import org.apache.ignite.configuration.annotation.ConfigValue;
import org.apache.ignite.configuration.annotation.ConfigurationRoot;
import org.apache.ignite.configuration.annotation.ConfigurationType;
import org.apache.ignite.configuration.annotation.Name;
import org.apache.ignite.configuration.annotation.NamedConfigValue;
import org.apache.ignite.configuration.validation.ExceptKeys;

@@ -33,6 +34,7 @@ public class DataStorageConfigurationSchema {

/** Default data region. */
@ConfigValue
@Name(DEFAULT_DATA_REGION_NAME)
public DataRegionConfigurationSchema defaultRegion;

/** Other data regions. */
@@ -0,0 +1,32 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.ignite.configuration.schemas.store;

import static org.apache.ignite.configuration.schemas.store.UnsafeMemoryAllocatorConfigurationSchema.UNSAFE_MEMORY_ALLOCATOR_TYPE;

import org.apache.ignite.configuration.annotation.PolymorphicConfig;
import org.apache.ignite.configuration.annotation.PolymorphicId;

/**
* Configuration schema for memory allocation strategies.
*/
@PolymorphicConfig
public class MemoryAllocatorConfigurationSchema {
@PolymorphicId(hasDefault = true)
public String type = UNSAFE_MEMORY_ALLOCATOR_TYPE;
}
@@ -0,0 +1,95 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.ignite.configuration.schemas.store;

import static org.apache.ignite.configuration.schemas.store.PageMemoryDataRegionConfigurationSchema.PAGE_MEMORY_DATA_REGION_TYPE;

import org.apache.ignite.configuration.annotation.ConfigValue;
import org.apache.ignite.configuration.annotation.PolymorphicConfigInstance;
import org.apache.ignite.configuration.annotation.Value;
import org.apache.ignite.configuration.validation.Immutable;
import org.apache.ignite.configuration.validation.OneOf;

/**
* Data region configuration for Page Memory storage engine.
*/
@PolymorphicConfigInstance(PAGE_MEMORY_DATA_REGION_TYPE)
public class PageMemoryDataRegionConfigurationSchema extends DataRegionConfigurationSchema {
/** Type of the Page Memory data region. */
public static final String PAGE_MEMORY_DATA_REGION_TYPE = "pagemem";

/** Default initial size. */
public static final long DFLT_DATA_REGION_INITIAL_SIZE = 256 * 1024 * 1024;

/** Default max size. */
public static final long DFLT_DATA_REGION_MAX_SIZE = 256 * 1024 * 1024;

/** Eviction is disabled. */
public static final String DISABLED_EVICTION_MODE = "DISABLED";

/** Random-LRU algorithm. */
public static final String RANDOM_LRU_EVICTION_MODE = "RANDOM_LRU";

/** Random-2-LRU algorithm: scan-resistant version of Random-LRU. */
public static final String RANDOM_2_LRU_EVICTION_MODE = "RANDOM_2_LRU";

/** Random-LRU algorithm. */
public static final String RANDOM_LRU_REPLACEMENT_MODE = "RANDOM_LRU";

/** Segmented-LRU algorithm. */
public static final String SEGMENTED_LRU_REPLACEMENT_MODE = "SEGMENTED_LRU";

/** CLOCK algorithm. */
public static final String CLOCK_REPLACEMENT_MODE = "CLOCK";

@Immutable
@Value(hasDefault = true)
public int pageSize = 16 * 1024;

@Value(hasDefault = true)
public boolean persistent = false;

@Value(hasDefault = true)
public long initSize = DFLT_DATA_REGION_INITIAL_SIZE;

@Value(hasDefault = true)
public long maxSize = DFLT_DATA_REGION_MAX_SIZE;

@ConfigValue
public MemoryAllocatorConfigurationSchema memoryAllocator;

@OneOf({DISABLED_EVICTION_MODE, RANDOM_LRU_EVICTION_MODE, RANDOM_2_LRU_EVICTION_MODE})
@Value(hasDefault = true)
public String evictionMode = DISABLED_EVICTION_MODE;

@OneOf({RANDOM_LRU_REPLACEMENT_MODE, SEGMENTED_LRU_REPLACEMENT_MODE, CLOCK_REPLACEMENT_MODE})
@Value(hasDefault = true)
public String replacementMode = CLOCK_REPLACEMENT_MODE;

@Value(hasDefault = true)
public double evictionThreshold = 0.9;

@Value(hasDefault = true)
public int emptyPagesPoolSize = 100;

@Value(hasDefault = true)
public long checkpointPageBufSize = 0;

@Value(hasDefault = true)
public boolean lazyMemoryAllocation = true;
}
@@ -0,0 +1,59 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.ignite.configuration.schemas.store;

import static org.apache.ignite.configuration.schemas.store.RocksDbDataRegionConfigurationSchema.ROCKSDB_DATA_REGION_TYPE;

import org.apache.ignite.configuration.annotation.PolymorphicConfigInstance;
import org.apache.ignite.configuration.annotation.Value;
import org.apache.ignite.configuration.validation.Min;
import org.apache.ignite.configuration.validation.OneOf;

/**
* Data region configuration for rocksdb storage engine.
*/
@PolymorphicConfigInstance(ROCKSDB_DATA_REGION_TYPE)
public class RocksDbDataRegionConfigurationSchema extends DataRegionConfigurationSchema {
/** Type of the RocksDB data region. */
public static final String ROCKSDB_DATA_REGION_TYPE = "rocksdb";

/** Cache type for the RocksDB LRU cache. */
public static final String ROCKSDB_LRU_CACHE = "lru";

/** Cache type for the RocksDB LRU cache. */
public static final String ROCKSDB_CLOCK_CACHE = "clock";

/** Size of the rocksdb offheap cache. */
@Value(hasDefault = true)
public long size = 256 * 1024 * 1024;

/** Size of rocksdb write buffer. */
@Value(hasDefault = true)
@Min(1)
public long writeBufferSize = 64 * 1024 * 1024;

/** Cache type - only {@code LRU} is supported at the moment. {@code Clock} implementation has known bugs. */
@OneOf(ROCKSDB_LRU_CACHE)
@Value(hasDefault = true)
public String cache = ROCKSDB_LRU_CACHE;

/** The cache is sharded to 2^numShardBits shards, by hash of the key. */
@Min(-1)
@Value(hasDefault = true)
public int numShardBits = -1;
}
@@ -0,0 +1,30 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.ignite.configuration.schemas.store;

import static org.apache.ignite.configuration.schemas.store.UnsafeMemoryAllocatorConfigurationSchema.UNSAFE_MEMORY_ALLOCATOR_TYPE;

import org.apache.ignite.configuration.annotation.PolymorphicConfigInstance;

/**
* Memory allocator that allocates data in offheap using {@link sun.misc.Unsafe}.
*/
@PolymorphicConfigInstance(UNSAFE_MEMORY_ALLOCATOR_TYPE)
public class UnsafeMemoryAllocatorConfigurationSchema extends MemoryAllocatorConfigurationSchema {
public static final String UNSAFE_MEMORY_ALLOCATOR_TYPE = "unsafe";
}
@@ -0,0 +1,32 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.ignite.internal.configuration;

import org.apache.ignite.configuration.ConfigurationTree;

/**
* Utility methods, related to testing configurations.
*/
public class ConfigurationTestUtils {
/**
* Casts dynamic configuration to its specific type. Can be applied to polymorphic configuration instances.
*/
public static <C extends ConfigurationTree> C fixConfiguration(C cfg) {
return (C) ((DynamicConfiguration) cfg).specificConfigTree();
}
}
@@ -61,6 +61,12 @@ public abstract class GridUnsafe {
/** Unsafe. */
private static final Unsafe UNSAFE = unsafe();

/** Page size. */
private static final int PAGE_SIZE = UNSAFE.pageSize();

/** Empty page. */
private static final byte[] EMPTY_PAGE = new byte[PAGE_SIZE];

/** Unaligned flag. */
private static final boolean UNALIGNED = unaligned();

@@ -1257,6 +1263,24 @@ public static void setMemory(long addr, long len, byte val) {
UNSAFE.setMemory(addr, len, val);
}

/**
* Fills memory with zeroes.
*
* @param addr Address.
* @param len Length.
*/
public static void zeroMemory(long addr, long len) {
long off = 0;

for (; off + PAGE_SIZE <= len; off += PAGE_SIZE) {
GridUnsafe.copyMemory(EMPTY_PAGE, GridUnsafe.BYTE_ARR_OFF, null, addr + off, PAGE_SIZE);
}

if (len != off) {
GridUnsafe.copyMemory(EMPTY_PAGE, GridUnsafe.BYTE_ARR_OFF, null, addr + off, len - off);
}
}

/**
* Copy memory between offheap locations.
*
@@ -1499,7 +1523,7 @@ public static void putObjectVolatile(Object obj, long off, Object val) {
* @return Page size.
*/
public static int pageSize() {
return UNSAFE.pageSize();
return PAGE_SIZE;
}

/**

0 comments on commit 3b7af82

Please sign in to comment.