diff --git a/core/src/main/java/com/orientechnologies/orient/core/db/tool/ODatabaseImport.java b/core/src/main/java/com/orientechnologies/orient/core/db/tool/ODatabaseImport.java index 14fb4dc690c..bf8911453ba 100755 --- a/core/src/main/java/com/orientechnologies/orient/core/db/tool/ODatabaseImport.java +++ b/core/src/main/java/com/orientechnologies/orient/core/db/tool/ODatabaseImport.java @@ -31,20 +31,29 @@ import com.orientechnologies.orient.core.db.document.ODocumentFieldWalker; import com.orientechnologies.orient.core.db.record.OClassTrigger; import com.orientechnologies.orient.core.db.record.OIdentifiable; -import com.orientechnologies.orient.core.db.tool.importer.*; +import com.orientechnologies.orient.core.db.tool.importer.OConverterData; +import com.orientechnologies.orient.core.db.tool.importer.OLinksRewriter; import com.orientechnologies.orient.core.exception.OConfigurationException; import com.orientechnologies.orient.core.exception.ODatabaseException; import com.orientechnologies.orient.core.exception.OSchemaException; import com.orientechnologies.orient.core.exception.OSerializationException; import com.orientechnologies.orient.core.id.ORID; import com.orientechnologies.orient.core.id.ORecordId; -import com.orientechnologies.orient.core.index.*; -import com.orientechnologies.orient.core.storage.index.hashindex.local.OHashIndexFactory; -import com.orientechnologies.orient.core.storage.index.hashindex.local.OMurmurHash3HashFunction; +import com.orientechnologies.orient.core.index.OIndex; +import com.orientechnologies.orient.core.index.OIndexDefinition; +import com.orientechnologies.orient.core.index.OIndexFactory; +import com.orientechnologies.orient.core.index.OIndexManager; +import com.orientechnologies.orient.core.index.OIndexes; +import com.orientechnologies.orient.core.index.ORuntimeKeyIndexDefinition; +import com.orientechnologies.orient.core.index.OSimpleKeyIndexDefinition; import com.orientechnologies.orient.core.intent.OIntentMassiveInsert; import com.orientechnologies.orient.core.metadata.OMetadataDefault; import com.orientechnologies.orient.core.metadata.function.OFunction; -import com.orientechnologies.orient.core.metadata.schema.*; +import com.orientechnologies.orient.core.metadata.schema.OClass; +import com.orientechnologies.orient.core.metadata.schema.OClassImpl; +import com.orientechnologies.orient.core.metadata.schema.OPropertyImpl; +import com.orientechnologies.orient.core.metadata.schema.OSchema; +import com.orientechnologies.orient.core.metadata.schema.OType; import com.orientechnologies.orient.core.metadata.security.OIdentity; import com.orientechnologies.orient.core.metadata.security.ORole; import com.orientechnologies.orient.core.metadata.security.OSecurityShared; @@ -59,12 +68,29 @@ import com.orientechnologies.orient.core.serialization.serializer.record.string.ORecordSerializerJSON; import com.orientechnologies.orient.core.storage.OPhysicalPosition; import com.orientechnologies.orient.core.storage.OStorage; +import com.orientechnologies.orient.core.storage.index.hashindex.local.OHashIndexFactory; +import com.orientechnologies.orient.core.storage.index.hashindex.local.OMurmurHash3HashFunction; -import java.io.*; +import java.io.BufferedInputStream; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.PrintWriter; +import java.io.StringWriter; import java.lang.reflect.InvocationTargetException; import java.text.ParseException; -import java.util.*; +import java.util.AbstractList; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; import java.util.Map.Entry; +import java.util.Set; import java.util.zip.GZIPInputStream; /** @@ -118,8 +144,7 @@ public void onMessage(String iText) { inStream = bf; } - OMurmurHash3HashFunction keyHashFunction = new OMurmurHash3HashFunction(); - keyHashFunction.setValueSerializer(OLinkSerializer.INSTANCE); + OMurmurHash3HashFunction keyHashFunction = new OMurmurHash3HashFunction(OLinkSerializer.INSTANCE); jsonReader = new OJSONReader(new InputStreamReader(inStream)); database.declareIntent(new OIntentMassiveInsert()); diff --git a/core/src/main/java/com/orientechnologies/orient/core/sharding/auto/OAutoShardingIndexEngine.java b/core/src/main/java/com/orientechnologies/orient/core/sharding/auto/OAutoShardingIndexEngine.java index bee2c3a1e4e..87dec056e8f 100755 --- a/core/src/main/java/com/orientechnologies/orient/core/sharding/auto/OAutoShardingIndexEngine.java +++ b/core/src/main/java/com/orientechnologies/orient/core/sharding/auto/OAutoShardingIndexEngine.java @@ -36,10 +36,12 @@ import com.orientechnologies.orient.core.metadata.schema.OType; import com.orientechnologies.orient.core.record.impl.ODocument; import com.orientechnologies.orient.core.storage.impl.local.OAbstractPaginatedStorage; +import com.orientechnologies.orient.core.storage.index.hashindex.local.OHashFunction; import com.orientechnologies.orient.core.storage.index.hashindex.local.OHashIndexBucket; import com.orientechnologies.orient.core.storage.index.hashindex.local.OHashTable; import com.orientechnologies.orient.core.storage.index.hashindex.local.OLocalHashTable; import com.orientechnologies.orient.core.storage.index.hashindex.local.OMurmurHash3HashFunction; +import com.orientechnologies.orient.core.storage.index.hashindex.local.OSHA256HashFunction; import java.util.ArrayList; import java.util.List; @@ -61,7 +63,6 @@ public final class OAutoShardingIndexEngine implements OIndexEngine { private final OAbstractPaginatedStorage storage; private final boolean durableInNonTx; - private final OMurmurHash3HashFunction hashFunction; private List> partitions; private OAutoShardingStrategy strategy; private int version; @@ -73,7 +74,6 @@ public OAutoShardingIndexEngine(final String iName, final Boolean iDurableInNonT final int iVersion) { this.name = iName; this.storage = iStorage; - this.hashFunction = new OMurmurHash3HashFunction(); if (iDurableInNonTxMode == null) durableInNonTx = iStorage.getConfiguration().getContextConfiguration() @@ -99,8 +99,14 @@ public void create(final OBinarySerializer valueSerializer, final boolean isAuto final Map engineProperties, final ODocument metadata, OEncryption encryption) { this.strategy = new OAutoShardingMurmurStrategy(keySerializer); - this.hashFunction.setValueSerializer(keySerializer); - this.hashFunction.setEncryption(encryption); + + final OHashFunction hashFunction; + + if (encryption != null) { + hashFunction = new OSHA256HashFunction<>(keySerializer); + } else { + hashFunction = new OMurmurHash3HashFunction<>(keySerializer); + } this.partitionSize = clustersToIndex.size(); if (metadata != null && metadata.containsField("partitions")) @@ -111,7 +117,7 @@ public void create(final OBinarySerializer valueSerializer, final boolean isAuto init(); for (OHashTable p : partitions) { - p.create(keySerializer, valueSerializer, keyTypes, encryption, nullPointerSupport); + p.create(keySerializer, valueSerializer, keyTypes, encryption, hashFunction, nullPointerSupport); } } @@ -132,12 +138,18 @@ public void load(final String indexName, final OBinarySerializer valueSerializer init(); int i = 0; + + final OHashFunction hashFunction; + + if (encryption != null) { + hashFunction = new OSHA256HashFunction<>(keySerializer); + } else { + hashFunction = new OMurmurHash3HashFunction<>(keySerializer); + } + for (OHashTable p : partitions) - p.load(indexName + "_" + (i++), keyTypes, nullPointerSupport, encryption); + p.load(indexName + "_" + (i++), keyTypes, nullPointerSupport, encryption, hashFunction); } - - hashFunction.setValueSerializer(keySerializer); - hashFunction.setEncryption(encryption); } @Override @@ -174,7 +186,7 @@ private void init() { for (int i = 0; i < partitionSize; ++i) { partitions.add( new OLocalHashTable(name + "_" + i, SUBINDEX_METADATA_FILE_EXTENSION, SUBINDEX_TREE_FILE_EXTENSION, - SUBINDEX_BUCKET_FILE_EXTENSION, SUBINDEX_NULL_BUCKET_FILE_EXTENSION, hashFunction, storage)); + SUBINDEX_BUCKET_FILE_EXTENSION, SUBINDEX_NULL_BUCKET_FILE_EXTENSION, storage)); } } diff --git a/core/src/main/java/com/orientechnologies/orient/core/sharding/auto/OAutoShardingMurmurStrategy.java b/core/src/main/java/com/orientechnologies/orient/core/sharding/auto/OAutoShardingMurmurStrategy.java index 9be94606a2d..831379bcfef 100755 --- a/core/src/main/java/com/orientechnologies/orient/core/sharding/auto/OAutoShardingMurmurStrategy.java +++ b/core/src/main/java/com/orientechnologies/orient/core/sharding/auto/OAutoShardingMurmurStrategy.java @@ -27,14 +27,14 @@ /** * Auto-sharding strategy implementation that uses Murmur hashing. * - * @since 3.0 * @author Luca Garulli (l.garulli--(at)--orientdb.com) + * @since 3.0 */ public final class OAutoShardingMurmurStrategy implements OAutoShardingStrategy { - private OMurmurHash3HashFunction hashFunction = new OMurmurHash3HashFunction(); + private OMurmurHash3HashFunction hashFunction; public OAutoShardingMurmurStrategy(final OBinarySerializer keySerializer) { - hashFunction.setValueSerializer(keySerializer); + hashFunction = new OMurmurHash3HashFunction(keySerializer); } public int getPartitionsId(final Object iKey, final int partitionSize) { diff --git a/core/src/main/java/com/orientechnologies/orient/core/storage/index/engine/OHashTableIndexEngine.java b/core/src/main/java/com/orientechnologies/orient/core/storage/index/engine/OHashTableIndexEngine.java index 0b79a5804b3..10082d3852f 100755 --- a/core/src/main/java/com/orientechnologies/orient/core/storage/index/engine/OHashTableIndexEngine.java +++ b/core/src/main/java/com/orientechnologies/orient/core/storage/index/engine/OHashTableIndexEngine.java @@ -35,11 +35,13 @@ import com.orientechnologies.orient.core.metadata.schema.OType; import com.orientechnologies.orient.core.record.impl.ODocument; import com.orientechnologies.orient.core.storage.impl.local.OAbstractPaginatedStorage; +import com.orientechnologies.orient.core.storage.index.hashindex.local.OHashFunction; import com.orientechnologies.orient.core.storage.index.hashindex.local.OHashIndexBucket; import com.orientechnologies.orient.core.storage.index.hashindex.local.OHashTable; import com.orientechnologies.orient.core.storage.index.hashindex.local.OLocalHashTable; import com.orientechnologies.orient.core.storage.index.hashindex.local.OLocalHashTable20; import com.orientechnologies.orient.core.storage.index.hashindex.local.OMurmurHash3HashFunction; +import com.orientechnologies.orient.core.storage.index.hashindex.local.OSHA256HashFunction; import java.util.Collections; import java.util.Iterator; @@ -59,16 +61,14 @@ public final class OHashTableIndexEngine implements OIndexEngine { public static final String BUCKET_FILE_EXTENSION = ".hib"; public static final String NULL_BUCKET_FILE_EXTENSION = ".hnb"; - private final OHashTable hashTable; - private final OMurmurHash3HashFunction hashFunction; - private final AtomicLong bonsayFileId = new AtomicLong(0); + private final OHashTable hashTable; + private final AtomicLong bonsayFileId = new AtomicLong(0); private int version; private final String name; public OHashTableIndexEngine(String name, Boolean durableInNonTxMode, OAbstractPaginatedStorage storage, int version) { - hashFunction = new OMurmurHash3HashFunction(); boolean durableInNonTx; if (durableInNonTxMode == null) @@ -78,12 +78,13 @@ public OHashTableIndexEngine(String name, Boolean durableInNonTxMode, OAbstractP durableInNonTx = durableInNonTxMode; this.version = version; - if (version < 2) + if (version < 2) { hashTable = new OLocalHashTable20(name, METADATA_FILE_EXTENSION, TREE_FILE_EXTENSION, BUCKET_FILE_EXTENSION, - NULL_BUCKET_FILE_EXTENSION, hashFunction, durableInNonTx, storage); - else + NULL_BUCKET_FILE_EXTENSION, durableInNonTx, storage); + } else { hashTable = new OLocalHashTable(name, METADATA_FILE_EXTENSION, TREE_FILE_EXTENSION, BUCKET_FILE_EXTENSION, - NULL_BUCKET_FILE_EXTENSION, hashFunction, storage); + NULL_BUCKET_FILE_EXTENSION, storage); + } this.name = name; } @@ -101,10 +102,15 @@ public String getName() { public void create(OBinarySerializer valueSerializer, boolean isAutomatic, OType[] keyTypes, boolean nullPointerSupport, OBinarySerializer keySerializer, int keySize, Set clustersToIndex, Map engineProperties, ODocument metadata, OEncryption encryption) { - hashFunction.setValueSerializer(keySerializer); - hashFunction.setEncryption(encryption); + final OHashFunction hashFunction; + + if (encryption != null) { + hashFunction = new OSHA256HashFunction<>(keySerializer); + } else { + hashFunction = new OMurmurHash3HashFunction<>(keySerializer); + } - hashTable.create(keySerializer, valueSerializer, keyTypes, encryption, nullPointerSupport); + hashTable.create(keySerializer, valueSerializer, keyTypes, encryption, hashFunction, nullPointerSupport); } @Override @@ -129,9 +135,15 @@ public void delete() { @Override public void load(String indexName, OBinarySerializer valueSerializer, boolean isAutomatic, OBinarySerializer keySerializer, OType[] keyTypes, boolean nullPointerSupport, int keySize, Map engineProperties, OEncryption encryption) { - hashTable.load(indexName, keyTypes, nullPointerSupport, encryption); - hashFunction.setValueSerializer(hashTable.getKeySerializer()); - hashFunction.setEncryption(encryption); + + final OHashFunction hashFunction; + + if (encryption != null) { + hashFunction = new OSHA256HashFunction<>(keySerializer); + } else { + hashFunction = new OMurmurHash3HashFunction<>(keySerializer); + } + hashTable.load(indexName, keyTypes, nullPointerSupport, encryption, hashFunction); } @Override diff --git a/core/src/main/java/com/orientechnologies/orient/core/storage/index/hashindex/local/MessageDigestHolder.java b/core/src/main/java/com/orientechnologies/orient/core/storage/index/hashindex/local/MessageDigestHolder.java new file mode 100755 index 00000000000..37bd70938cb --- /dev/null +++ b/core/src/main/java/com/orientechnologies/orient/core/storage/index/hashindex/local/MessageDigestHolder.java @@ -0,0 +1,24 @@ +package com.orientechnologies.orient.core.storage.index.hashindex.local; + +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; + +public class MessageDigestHolder { + private static final MessageDigestHolder INSTANCE = new MessageDigestHolder(); + + private final ThreadLocal messageDigest = ThreadLocal.withInitial(() -> { + try { + return MessageDigest.getInstance("SHA-256"); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException("SHA-256 algorithm is not implemented", e); + } + }); + + public static MessageDigestHolder instance() { + return INSTANCE; + } + + public MessageDigest get() { + return messageDigest.get(); + } +} diff --git a/core/src/main/java/com/orientechnologies/orient/core/storage/index/hashindex/local/OHashTable.java b/core/src/main/java/com/orientechnologies/orient/core/storage/index/hashindex/local/OHashTable.java index 6466d406d2d..3b8645eb856 100755 --- a/core/src/main/java/com/orientechnologies/orient/core/storage/index/hashindex/local/OHashTable.java +++ b/core/src/main/java/com/orientechnologies/orient/core/storage/index/hashindex/local/OHashTable.java @@ -33,7 +33,7 @@ */ public interface OHashTable { void create(OBinarySerializer keySerializer, OBinarySerializer valueSerializer, OType[] keyTypes, OEncryption encryption, - boolean nullKeyIsSupported); + OHashFunction keyHashFunction, boolean nullKeyIsSupported); OBinarySerializer getKeySerializer(); @@ -68,7 +68,7 @@ void create(OBinarySerializer keySerializer, OBinarySerializer valueSerial OHashIndexBucket.Entry[] higherEntries(K key, int limit); - void load(String name, OType[] keyTypes, boolean nullKeyIsSupported, OEncryption encryption); + void load(String name, OType[] keyTypes, boolean nullKeyIsSupported, OEncryption encryption, OHashFunction keyHashFunction); void deleteWithoutLoad(String name); diff --git a/core/src/main/java/com/orientechnologies/orient/core/storage/index/hashindex/local/OLocalHashTable.java b/core/src/main/java/com/orientechnologies/orient/core/storage/index/hashindex/local/OLocalHashTable.java index 06d061a0d96..da3e4af229d 100755 --- a/core/src/main/java/com/orientechnologies/orient/core/storage/index/hashindex/local/OLocalHashTable.java +++ b/core/src/main/java/com/orientechnologies/orient/core/storage/index/hashindex/local/OLocalHashTable.java @@ -89,13 +89,13 @@ public class OLocalHashTable extends ODurableComponent implements OHashTab private static final int LEVEL_MASK = Integer.MAX_VALUE >>> (31 - MAX_LEVEL_DEPTH); - private final OHashFunction keyHashFunction; + private OHashFunction keyHashFunction; private OBinarySerializer keySerializer; private OBinarySerializer valueSerializer; private OType[] keyTypes; - private final OHashTable.KeyHashCodeComparator comparator; + private OHashTable.KeyHashCodeComparator comparator; private boolean nullKeyIsSupported; private long nullBucketFileId = -1; @@ -111,22 +111,18 @@ public class OLocalHashTable extends ODurableComponent implements OHashTab private OEncryption encryption; public OLocalHashTable(String name, String metadataConfigurationFileExtension, String treeStateFileExtension, - String bucketFileExtension, String nullBucketFileExtension, OHashFunction keyHashFunction, - OAbstractPaginatedStorage abstractPaginatedStorage) { + String bucketFileExtension, String nullBucketFileExtension, OAbstractPaginatedStorage abstractPaginatedStorage) { super(abstractPaginatedStorage, name, bucketFileExtension, name + bucketFileExtension); this.metadataConfigurationFileExtension = metadataConfigurationFileExtension; this.treeStateFileExtension = treeStateFileExtension; - this.keyHashFunction = keyHashFunction; this.nullBucketFileExtension = nullBucketFileExtension; - - this.comparator = new OHashTable.KeyHashCodeComparator<>(this.keyHashFunction); } @SuppressFBWarnings("DLS_DEAD_LOCAL_STORE") @Override public void create(OBinarySerializer keySerializer, OBinarySerializer valueSerializer, OType[] keyTypes, - OEncryption encryption, boolean nullKeyIsSupported) { + OEncryption encryption, OHashFunction keyHashFunction, boolean nullKeyIsSupported) { startOperation(); try { final OAtomicOperation atomicOperation; @@ -139,6 +135,8 @@ public void create(OBinarySerializer keySerializer, OBinarySerializer valu acquireExclusiveLock(); try { try { + this.keyHashFunction = keyHashFunction; + this.comparator = new OHashTable.KeyHashCodeComparator<>(this.keyHashFunction); this.encryption = encryption; if (keyTypes != null) @@ -619,11 +617,14 @@ public OHashIndexBucket.Entry[] higherEntries(K key, int limit) { } @Override - public void load(String name, OType[] keyTypes, boolean nullKeyIsSupported, OEncryption encryption) { + public void load(String name, OType[] keyTypes, boolean nullKeyIsSupported, OEncryption encryption, + OHashFunction keyHashFunction) { startOperation(); try { acquireExclusiveLock(); try { + this.keyHashFunction = keyHashFunction; + this.comparator = new OHashTable.KeyHashCodeComparator<>(this.keyHashFunction); if (keyTypes != null) this.keyTypes = Arrays.copyOf(keyTypes, keyTypes.length); else diff --git a/core/src/main/java/com/orientechnologies/orient/core/storage/index/hashindex/local/OLocalHashTable20.java b/core/src/main/java/com/orientechnologies/orient/core/storage/index/hashindex/local/OLocalHashTable20.java index 7e329a372c4..8ded6b36ccb 100755 --- a/core/src/main/java/com/orientechnologies/orient/core/storage/index/hashindex/local/OLocalHashTable20.java +++ b/core/src/main/java/com/orientechnologies/orient/core/storage/index/hashindex/local/OLocalHashTable20.java @@ -46,12 +46,10 @@ * to classic algorithm because of its big memory consumption in case of non-uniform data distribution instead it is implemented * according too "Multilevel Extendible Hashing Sven Helmer, Thomas Neumann, Guido Moerkotte April 17, 2002". Which has much less * memory consumption in case of nonuniform data distribution. - *

* Index itself uses so called "muiltilevel schema" when first level contains 256 buckets, when bucket is split it is put at the * end of other file which represents second level. So if data which are put has distribution close to uniform (this index was * designed to be use as rid index for DHT storage) buckets split will be preformed in append only manner to speed up index write * speed. - *

* So hash index bucket itself has following structure: *

    *
  1. Bucket depth - 1 byte.
    2. @@ -60,30 +58,23 @@ *
  2. Offsets of entities stored in this bucket relatively to it's beginning. It is array of int values of undefined size.
    3. *
  3. Entities itself
    4. *
- *

* So if 1-st and 2-nd fields are clear. We should discuss the last ones. - *

- *

* Entities in bucket are sorted by key's hash code so each entity has following storage format in bucket: key's hash code (8 * bytes), key, value. Because entities are stored in sorted order it means that every time when we insert new entity old ones * should be moved. - *

* There are 2 reasons why it is bad: *

    *
  1. It will generate write ahead log of enormous size.
    2. *
  2. The more amount of memory is affected in operation the less speed we will have. In worst case 60 kb of memory should be * moved.
    3. *
- *

* To avoid disadvantages listed above entries ara appended to the end of bucket, but their offsets are stored at the beginning of * bucket. Offsets are stored in sorted order (ordered by hash code of entity's key) so we need to move only small amount of memory * to store entities in sorted order. - *

* About indexes of parents of current bucket. When item is removed from bucket we check space which is needed to store all entities * of this bucket, it's buddy bucket (bucket which was also created from parent bucket during split) and if space of single bucket * is enough to save all entities from both buckets we remove these buckets and put all content in parent bucket. That is why we * need indexes of parents of current bucket. - *

* Also hash index has special file of one page long which contains information about state of each level of buckets in index. This * information is stored as array index of which equals to file level. All array item has following structure: *

    @@ -111,7 +102,7 @@ public class OLocalHashTable20 extends ODurableComponent implements OHashT public static final int LEVEL_MASK = Integer.MAX_VALUE >>> (31 - MAX_LEVEL_DEPTH); - private final OHashFunction keyHashFunction; + private OHashFunction keyHashFunction; private OBinarySerializer keySerializer; private OBinarySerializer valueSerializer; @@ -119,9 +110,9 @@ public class OLocalHashTable20 extends ODurableComponent implements OHashT private final KeyHashCodeComparator comparator; - private boolean nullKeyIsSupported; - private long nullBucketFileId = -1; - private final String nullBucketFileExtension; + private boolean nullKeyIsSupported; + private long nullBucketFileId = -1; + private final String nullBucketFileExtension; private long fileStateId; @@ -132,13 +123,12 @@ public class OLocalHashTable20 extends ODurableComponent implements OHashT private final boolean durableInNonTxMode; public OLocalHashTable20(String name, String metadataConfigurationFileExtension, String treeStateFileExtension, - String bucketFileExtension, String nullBucketFileExtension, OHashFunction keyHashFunction, boolean durableInNonTxMode, + String bucketFileExtension, String nullBucketFileExtension, boolean durableInNonTxMode, OAbstractPaginatedStorage abstractPaginatedStorage) { super(abstractPaginatedStorage, name, bucketFileExtension, name + bucketFileExtension); this.metadataConfigurationFileExtension = metadataConfigurationFileExtension; this.treeStateFileExtension = treeStateFileExtension; - this.keyHashFunction = keyHashFunction; this.nullBucketFileExtension = nullBucketFileExtension; this.durableInNonTxMode = durableInNonTxMode; @@ -147,7 +137,7 @@ public OLocalHashTable20(String name, String metadataConfigurationFileExtension, @Override public void create(OBinarySerializer keySerializer, OBinarySerializer valueSerializer, OType[] keyTypes, - OEncryption encryption, boolean nullKeyIsSupported) { + OEncryption encryption, OHashFunction keyHashFunction, boolean nullKeyIsSupported) { final OAtomicOperation atomicOperation; try { atomicOperation = startAtomicOperation(false); @@ -159,6 +149,7 @@ public void create(OBinarySerializer keySerializer, OBinarySerializer valu try { try { + this.keyHashFunction = keyHashFunction; if (keyTypes != null) this.keyTypes = Arrays.copyOf(keyTypes, keyTypes.length); else @@ -621,9 +612,11 @@ public OHashIndexBucket.Entry[] higherEntries(K key, int limit) { } @Override - public void load(String name, OType[] keyTypes, boolean nullKeyIsSupported, OEncryption encryption) { + public void load(String name, OType[] keyTypes, boolean nullKeyIsSupported, OEncryption encryption, + OHashFunction keyHashFunction) { acquireExclusiveLock(); try { + this.keyHashFunction = keyHashFunction; if (keyTypes != null) this.keyTypes = Arrays.copyOf(keyTypes, keyTypes.length); else diff --git a/core/src/main/java/com/orientechnologies/orient/core/storage/index/hashindex/local/OMurmurHash3HashFunction.java b/core/src/main/java/com/orientechnologies/orient/core/storage/index/hashindex/local/OMurmurHash3HashFunction.java index 56c2fffaa74..f2f9bfbb04c 100755 --- a/core/src/main/java/com/orientechnologies/orient/core/storage/index/hashindex/local/OMurmurHash3HashFunction.java +++ b/core/src/main/java/com/orientechnologies/orient/core/storage/index/hashindex/local/OMurmurHash3HashFunction.java @@ -21,7 +21,6 @@ import com.orientechnologies.common.hash.OMurmurHash3; import com.orientechnologies.common.serialization.types.OBinarySerializer; -import com.orientechnologies.orient.core.encryption.OEncryption; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; /** @@ -32,18 +31,9 @@ public class OMurmurHash3HashFunction implements OHashFunction { private static final int SEED = 362498820; - private OBinarySerializer valueSerializer; - private OEncryption encryption; + private final OBinarySerializer valueSerializer; - public OBinarySerializer getValueSerializer() { - return valueSerializer; - } - - public void setEncryption(OEncryption encryption) { - this.encryption = encryption; - } - - public void setValueSerializer(OBinarySerializer valueSerializer) { + public OMurmurHash3HashFunction(OBinarySerializer valueSerializer) { this.valueSerializer = valueSerializer; } @@ -52,11 +42,6 @@ public long hashCode(final V value) { final byte[] serializedValue = new byte[valueSerializer.getObjectSize(value)]; valueSerializer.serializeNativeObject(value, serializedValue, 0); - if (encryption == null) { - return OMurmurHash3.murmurHash3_x64_64(serializedValue, SEED); - } else { - final byte[] encrypted = encryption.encrypt(serializedValue); - return OMurmurHash3.murmurHash3_x64_64(encrypted, SEED); - } + return OMurmurHash3.murmurHash3_x64_64(serializedValue, SEED); } } diff --git a/core/src/main/java/com/orientechnologies/orient/core/storage/index/hashindex/local/OSHA256HashFunction.java b/core/src/main/java/com/orientechnologies/orient/core/storage/index/hashindex/local/OSHA256HashFunction.java new file mode 100755 index 00000000000..01273a68b83 --- /dev/null +++ b/core/src/main/java/com/orientechnologies/orient/core/storage/index/hashindex/local/OSHA256HashFunction.java @@ -0,0 +1,21 @@ +package com.orientechnologies.orient.core.storage.index.hashindex.local; + +import com.orientechnologies.common.serialization.types.OBinarySerializer; +import com.orientechnologies.common.serialization.types.OLongSerializer; + +public class OSHA256HashFunction implements OHashFunction { + private final OBinarySerializer valueSerializer; + + public OSHA256HashFunction(OBinarySerializer valueSerializer) { + this.valueSerializer = valueSerializer; + } + + @Override + public long hashCode(V value) { + final byte[] serializedValue = new byte[valueSerializer.getObjectSize(value)]; + valueSerializer.serializeNativeObject(value, serializedValue, 0); + + final byte[] digest = MessageDigestHolder.instance().get().digest(serializedValue); + return OLongSerializer.INSTANCE.deserializeNative(digest, 0); + } +} diff --git a/core/src/test/java/com/orientechnologies/orient/core/storage/index/hashindex/local/LocalHashTableIterationTestIT.java b/core/src/test/java/com/orientechnologies/orient/core/storage/index/hashindex/local/LocalHashTableIterationTestIT.java index 75b8c5fd2aa..398eec989df 100755 --- a/core/src/test/java/com/orientechnologies/orient/core/storage/index/hashindex/local/LocalHashTableIterationTestIT.java +++ b/core/src/test/java/com/orientechnologies/orient/core/storage/index/hashindex/local/LocalHashTableIterationTestIT.java @@ -51,12 +51,11 @@ public long hashCode(Integer value) { }; localHashTable = new OLocalHashTable("localHashTableIterationTest", ".imc", ".tsc", ".obf", ".nbh", - hashFunction, (OAbstractPaginatedStorage) databaseDocumentTx.getStorage()); + (OAbstractPaginatedStorage) databaseDocumentTx.getStorage()); localHashTable .create(OIntegerSerializer.INSTANCE, OBinarySerializerFactory.getInstance().getObjectSerializer(OType.STRING), null, - null, - true); + null, hashFunction, true); } @After diff --git a/core/src/test/java/com/orientechnologies/orient/core/storage/index/hashindex/local/OHashTableDirectoryTest.java b/core/src/test/java/com/orientechnologies/orient/core/storage/index/hashindex/local/OHashTableDirectoryTest.java index 083c1028494..95f99c1a764 100755 --- a/core/src/test/java/com/orientechnologies/orient/core/storage/index/hashindex/local/OHashTableDirectoryTest.java +++ b/core/src/test/java/com/orientechnologies/orient/core/storage/index/hashindex/local/OHashTableDirectoryTest.java @@ -1,10 +1,14 @@ package com.orientechnologies.orient.core.storage.index.hashindex.local; -import com.orientechnologies.common.serialization.types.OIntegerSerializer; import com.orientechnologies.orient.core.db.document.ODatabaseDocumentTx; import com.orientechnologies.orient.core.storage.impl.local.OAbstractPaginatedStorage; import org.assertj.core.api.Assertions; -import org.junit.*; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; import java.io.IOException; @@ -31,9 +35,6 @@ public static void beforeClass() throws IOException { databaseDocumentTx.create(); - OMurmurHash3HashFunction murmurHash3HashFunction = new OMurmurHash3HashFunction(); - murmurHash3HashFunction.setValueSerializer(OIntegerSerializer.INSTANCE); - directory = new OHashTableDirectory(".tsc", "hashTableDirectoryTest", "hashTableDirectoryTest", (OAbstractPaginatedStorage) databaseDocumentTx.getStorage()); directory.create(); diff --git a/core/src/test/java/com/orientechnologies/orient/core/storage/index/hashindex/local/OLocalHashTableBase.java b/core/src/test/java/com/orientechnologies/orient/core/storage/index/hashindex/local/OLocalHashTableBase.java index 2459c063cb5..37339962c82 100755 --- a/core/src/test/java/com/orientechnologies/orient/core/storage/index/hashindex/local/OLocalHashTableBase.java +++ b/core/src/test/java/com/orientechnologies/orient/core/storage/index/hashindex/local/OLocalHashTableBase.java @@ -1,6 +1,5 @@ package com.orientechnologies.orient.core.storage.index.hashindex.local; -import com.orientechnologies.orient.core.db.document.ODatabaseDocumentTx; import org.junit.Assert; import org.junit.Test; @@ -13,9 +12,8 @@ * Created by frank on 24/04/2016. */ public abstract class OLocalHashTableBase { - protected static final int KEYS_COUNT = 500000; - protected ODatabaseDocumentTx databaseDocumentTx; - protected OLocalHashTable localHashTable; + protected static final int KEYS_COUNT = 500000; + protected OLocalHashTable localHashTable; @Test public void testKeyPut() throws IOException { diff --git a/core/src/test/java/com/orientechnologies/orient/core/storage/index/hashindex/local/OLocalHashTableEncryptionTestIT.java b/core/src/test/java/com/orientechnologies/orient/core/storage/index/hashindex/local/OLocalHashTableEncryptionTestIT.java new file mode 100755 index 00000000000..3af673eb7c5 --- /dev/null +++ b/core/src/test/java/com/orientechnologies/orient/core/storage/index/hashindex/local/OLocalHashTableEncryptionTestIT.java @@ -0,0 +1,54 @@ +package com.orientechnologies.orient.core.storage.index.hashindex.local; + +import com.orientechnologies.common.io.OFileUtils; +import com.orientechnologies.common.serialization.types.OIntegerSerializer; +import com.orientechnologies.orient.core.db.ODatabaseInternal; +import com.orientechnologies.orient.core.db.ODatabaseSession; +import com.orientechnologies.orient.core.db.ODatabaseType; +import com.orientechnologies.orient.core.db.OrientDB; +import com.orientechnologies.orient.core.db.OrientDBConfig; +import com.orientechnologies.orient.core.encryption.OEncryption; +import com.orientechnologies.orient.core.encryption.OEncryptionFactory; +import com.orientechnologies.orient.core.metadata.schema.OType; +import com.orientechnologies.orient.core.serialization.serializer.binary.OBinarySerializerFactory; +import com.orientechnologies.orient.core.storage.impl.local.OAbstractPaginatedStorage; +import org.junit.After; +import org.junit.Before; + +import java.io.File; + +public class OLocalHashTableEncryptionTestIT extends OLocalHashTableBase { + private OrientDB orientDB; + + private static final String DB_NAME = "localHashTableEncryptionTest"; + + @Before + public void before() { + String buildDirectory = System.getProperty("buildDirectory", "."); + final File dbDirectory = new File(buildDirectory, DB_NAME); + + OFileUtils.deleteRecursively(dbDirectory); + orientDB = new OrientDB("plocal:" + buildDirectory, OrientDBConfig.defaultConfig()); + orientDB.create(DB_NAME, ODatabaseType.PLOCAL); + + ODatabaseSession databaseDocumentTx = orientDB.open(DB_NAME, "admin", "admin"); + + final OEncryption encryption = OEncryptionFactory.INSTANCE.getEncryption("aes/gcm", "T1JJRU5UREJfSVNfQ09PTA=="); + + OSHA256HashFunction SHA256HashFunction = new OSHA256HashFunction<>(OIntegerSerializer.INSTANCE); + + localHashTable = new OLocalHashTable<>("localHashTableEncryptionTest", ".imc", ".tsc", ".obf", ".nbh", + (OAbstractPaginatedStorage) ((ODatabaseInternal) databaseDocumentTx).getStorage()); + + localHashTable + .create(OIntegerSerializer.INSTANCE, OBinarySerializerFactory.getInstance().getObjectSerializer(OType.STRING), null, + encryption, SHA256HashFunction, true); + + } + + @After + public void after() { + orientDB.drop(DB_NAME); + orientDB.close(); + } +} diff --git a/core/src/test/java/com/orientechnologies/orient/core/storage/index/hashindex/local/OLocalHashTableTestIT.java b/core/src/test/java/com/orientechnologies/orient/core/storage/index/hashindex/local/OLocalHashTableTestIT.java index 52096dbda85..34dd4de1100 100755 --- a/core/src/test/java/com/orientechnologies/orient/core/storage/index/hashindex/local/OLocalHashTableTestIT.java +++ b/core/src/test/java/com/orientechnologies/orient/core/storage/index/hashindex/local/OLocalHashTableTestIT.java @@ -1,54 +1,55 @@ package com.orientechnologies.orient.core.storage.index.hashindex.local; +import com.orientechnologies.common.io.OFileUtils; import com.orientechnologies.common.serialization.types.OIntegerSerializer; -import com.orientechnologies.orient.core.db.document.ODatabaseDocumentTx; +import com.orientechnologies.orient.core.db.ODatabaseInternal; +import com.orientechnologies.orient.core.db.ODatabaseSession; +import com.orientechnologies.orient.core.db.ODatabaseType; +import com.orientechnologies.orient.core.db.OrientDB; +import com.orientechnologies.orient.core.db.OrientDBConfig; import com.orientechnologies.orient.core.metadata.schema.OType; import com.orientechnologies.orient.core.serialization.serializer.binary.OBinarySerializerFactory; import com.orientechnologies.orient.core.storage.impl.local.OAbstractPaginatedStorage; import org.junit.After; +import org.junit.Before; import java.io.File; -import java.io.IOException; /** * @author Andrey Lomakin (a.lomakin-at-orientdb.com) * @since 19.02.13 */ public class OLocalHashTableTestIT extends OLocalHashTableBase { + private OrientDB orientDB; - public OLocalHashTableTestIT() throws Exception { + private static final String DB_NAME = "localHashTableTest"; - String buildDirectory = System.getProperty("buildDirectory"); - if (buildDirectory == null) - buildDirectory = "."; + @Before + public void before() { + String buildDirectory = System.getProperty("buildDirectory", "."); + final File dbDirectory = new File(buildDirectory, DB_NAME); - final File dbDirectory = new File(buildDirectory, "localHashTableTest"); - System.out.println(this.getClass().getSimpleName() + " test is initializing using DB directory = " + dbDirectory); + OFileUtils.deleteRecursively(dbDirectory); + orientDB = new OrientDB("plocal:" + buildDirectory, OrientDBConfig.defaultConfig()); - databaseDocumentTx = new ODatabaseDocumentTx("plocal:" + dbDirectory.getCanonicalPath()); - if (databaseDocumentTx.exists()) { - databaseDocumentTx.open("admin", "admin"); - databaseDocumentTx.drop(); - } + orientDB.create(DB_NAME, ODatabaseType.PLOCAL); + final ODatabaseSession databaseDocumentTx = orientDB.open(DB_NAME, "admin", "admin"); - databaseDocumentTx.create(); + OMurmurHash3HashFunction murmurHash3HashFunction = new OMurmurHash3HashFunction(OIntegerSerializer.INSTANCE); - OMurmurHash3HashFunction murmurHash3HashFunction = new OMurmurHash3HashFunction(); - murmurHash3HashFunction.setValueSerializer(OIntegerSerializer.INSTANCE); - - localHashTable = new OLocalHashTable("localHashTableTest", ".imc", ".tsc", ".obf", ".nbh", - murmurHash3HashFunction, (OAbstractPaginatedStorage) databaseDocumentTx.getStorage()); + localHashTable = new OLocalHashTable<>("localHashTableTest", ".imc", ".tsc", ".obf", ".nbh", + (OAbstractPaginatedStorage) ((ODatabaseInternal) databaseDocumentTx).getStorage()); localHashTable - .create(OIntegerSerializer.INSTANCE, OBinarySerializerFactory.getInstance().getObjectSerializer(OType.STRING), null, - null, - true); + .create(OIntegerSerializer.INSTANCE, OBinarySerializerFactory.getInstance().getObjectSerializer(OType.STRING), null, null, + murmurHash3HashFunction, true); } @After - public void afterMethod() throws IOException { - localHashTable.clear(); + public void after() { + orientDB.drop(DB_NAME); + orientDB.close(); } } diff --git a/core/src/test/java/com/orientechnologies/orient/core/storage/index/hashindex/local/OLocalHashTableWALTestIT.java b/core/src/test/java/com/orientechnologies/orient/core/storage/index/hashindex/local/OLocalHashTableWALTestIT.java index 80cb5ca04d7..1e6a792dc67 100755 --- a/core/src/test/java/com/orientechnologies/orient/core/storage/index/hashindex/local/OLocalHashTableWALTestIT.java +++ b/core/src/test/java/com/orientechnologies/orient/core/storage/index/hashindex/local/OLocalHashTableWALTestIT.java @@ -1,8 +1,12 @@ package com.orientechnologies.orient.core.storage.index.hashindex.local; +import com.orientechnologies.common.io.OFileUtils; import com.orientechnologies.common.serialization.types.OIntegerSerializer; -import com.orientechnologies.orient.core.config.OGlobalConfiguration; -import com.orientechnologies.orient.core.db.document.ODatabaseDocumentTx; +import com.orientechnologies.orient.core.db.ODatabaseInternal; +import com.orientechnologies.orient.core.db.ODatabaseSession; +import com.orientechnologies.orient.core.db.ODatabaseType; +import com.orientechnologies.orient.core.db.OrientDB; +import com.orientechnologies.orient.core.db.OrientDBConfig; import com.orientechnologies.orient.core.metadata.schema.OType; import com.orientechnologies.orient.core.serialization.serializer.binary.OBinarySerializerFactory; import com.orientechnologies.orient.core.storage.cache.OCacheEntry; @@ -44,57 +48,47 @@ * @since 5/19/14 */ public class OLocalHashTableWALTestIT extends OLocalHashTableBase { - static { - OGlobalConfiguration.FILE_LOCK.setValue(false); - } - - private String buildDirectory; + private static final String ACTUAL_DB_NAME = OLocalHashTableWALTestIT.class.getSimpleName() + "Actual"; + private static final String EXPECTED_DB_NAME = OLocalHashTableWALTestIT.class.getSimpleName() + "Expected"; private String actualStorageDir; private String expectedStorageDir; + private ODatabaseSession databaseDocumentTx; + private OWOWCache actualWriteCache; private OWOWCache expectedWriteCache; - private ODatabaseDocumentTx expectedDatabaseDocumentTx; + private ODatabaseSession expectedDatabaseDocumentTx; + + private OrientDB orientDB; @Before - public void beforeMethod() throws IOException { - buildDirectory = System.getProperty("buildDirectory", "."); + public void before() { + String buildDirectory = System.getProperty("buildDirectory", "."); buildDirectory += "/" + this.getClass().getSimpleName(); final File buildDir = new File(buildDirectory); - if (buildDir.exists()) - buildDir.delete(); + OFileUtils.deleteRecursively(buildDir); - buildDir.mkdir(); + orientDB = new OrientDB("plocal:" + buildDirectory, OrientDBConfig.defaultConfig()); - final String actualStorageName = this.getClass().getSimpleName() + "Actual"; - databaseDocumentTx = new ODatabaseDocumentTx("plocal:" + buildDirectory + File.separator + actualStorageName); - if (databaseDocumentTx.exists()) { - databaseDocumentTx.open("admin", "admin"); - databaseDocumentTx.drop(); - } + orientDB.create(ACTUAL_DB_NAME, ODatabaseType.PLOCAL); + databaseDocumentTx = orientDB.open(ACTUAL_DB_NAME, "admin", "admin"); - databaseDocumentTx.create(); + orientDB.create(EXPECTED_DB_NAME, ODatabaseType.PLOCAL); + expectedDatabaseDocumentTx = orientDB.open(EXPECTED_DB_NAME, "admin", "admin"); - final String expectedStorageName = this.getClass().getSimpleName() + "Expected"; - expectedDatabaseDocumentTx = new ODatabaseDocumentTx("plocal:" + buildDirectory + File.separator + expectedStorageName); - if (expectedDatabaseDocumentTx.exists()) { - expectedDatabaseDocumentTx.open("admin", "admin"); - expectedDatabaseDocumentTx.drop(); - } - - expectedDatabaseDocumentTx.create(); + actualStorageDir = ((OLocalPaginatedStorage) ((ODatabaseInternal) databaseDocumentTx).getStorage()).getStoragePath().toString(); + expectedStorageDir = ((OLocalPaginatedStorage) ((ODatabaseInternal) expectedDatabaseDocumentTx).getStorage()).getStoragePath() + .toString(); - actualStorageDir = ((OLocalPaginatedStorage) databaseDocumentTx.getStorage()).getStoragePath().toString(); - expectedStorageDir = ((OLocalPaginatedStorage) expectedDatabaseDocumentTx.getStorage()).getStoragePath().toString(); + actualWriteCache = (OWOWCache) ((OLocalPaginatedStorage) ((ODatabaseInternal) databaseDocumentTx).getStorage()).getWriteCache(); + expectedWriteCache = (OWOWCache) ((OLocalPaginatedStorage) ((ODatabaseInternal) expectedDatabaseDocumentTx).getStorage()) + .getWriteCache(); - actualWriteCache = (OWOWCache) ((OLocalPaginatedStorage) databaseDocumentTx.getStorage()).getWriteCache(); - expectedWriteCache = (OWOWCache) ((OLocalPaginatedStorage) expectedDatabaseDocumentTx.getStorage()).getWriteCache(); - - OLocalPaginatedStorage actualStorage = (OLocalPaginatedStorage) databaseDocumentTx.getStorage(); + OLocalPaginatedStorage actualStorage = (OLocalPaginatedStorage) ((ODatabaseInternal) databaseDocumentTx).getStorage(); ODiskWriteAheadLog diskWriteAheadLog = (ODiskWriteAheadLog) actualStorage.getWALInstance(); actualStorage.synch(); @@ -104,32 +98,20 @@ public void beforeMethod() throws IOException { } @After - public void after() throws IOException { - if (databaseDocumentTx.isClosed()) - databaseDocumentTx.open("admin", "admin"); - - databaseDocumentTx.activateOnCurrentThread(); - databaseDocumentTx.drop(); - - if (expectedDatabaseDocumentTx.isClosed()) - expectedDatabaseDocumentTx.open("admin", "admin"); - - expectedDatabaseDocumentTx.activateOnCurrentThread(); - expectedDatabaseDocumentTx.drop(); - - Assert.assertTrue(new File(buildDirectory).delete()); + public void after() { + orientDB.drop(ACTUAL_DB_NAME); + orientDB.drop(EXPECTED_DB_NAME); + orientDB.close(); } - private void createActualHashTable() throws IOException { - OMurmurHash3HashFunction murmurHash3HashFunction = new OMurmurHash3HashFunction(); - murmurHash3HashFunction.setValueSerializer(OIntegerSerializer.INSTANCE); + private void createActualHashTable() { + OMurmurHash3HashFunction murmurHash3HashFunction = new OMurmurHash3HashFunction<>(OIntegerSerializer.INSTANCE); - localHashTable = new OLocalHashTable("actualLocalHashTable", ".imc", ".tsc", ".obf", ".nbh", - murmurHash3HashFunction, (OAbstractPaginatedStorage) databaseDocumentTx.getStorage()); + localHashTable = new OLocalHashTable<>("actualLocalHashTable", ".imc", ".tsc", ".obf", ".nbh", + (OAbstractPaginatedStorage) ((ODatabaseInternal) databaseDocumentTx).getStorage()); localHashTable - .create(OIntegerSerializer.INSTANCE, OBinarySerializerFactory.getInstance().getObjectSerializer(OType.STRING), null, - null, - true); + .create(OIntegerSerializer.INSTANCE, OBinarySerializerFactory.getInstance().getObjectSerializer(OType.STRING), null, null, + murmurHash3HashFunction, true); } @Override @@ -137,7 +119,8 @@ public void testKeyPut() throws IOException { super.testKeyPut(); Assert.assertNull( - ((OAbstractPaginatedStorage) databaseDocumentTx.getStorage()).getAtomicOperationsManager().getCurrentOperation()); + ((OAbstractPaginatedStorage) ((ODatabaseInternal) databaseDocumentTx).getStorage()).getAtomicOperationsManager() + .getCurrentOperation()); assertFileRestoreFromWAL(); } @@ -147,7 +130,8 @@ public void testKeyPutRandomUniform() throws IOException { super.testKeyPutRandomUniform(); Assert.assertNull( - ((OAbstractPaginatedStorage) databaseDocumentTx.getStorage()).getAtomicOperationsManager().getCurrentOperation()); + ((OAbstractPaginatedStorage) ((ODatabaseInternal) databaseDocumentTx).getStorage()).getAtomicOperationsManager() + .getCurrentOperation()); assertFileRestoreFromWAL(); } @@ -157,7 +141,8 @@ public void testKeyPutRandomGaussian() throws IOException { super.testKeyPutRandomGaussian(); Assert.assertNull( - ((OAbstractPaginatedStorage) databaseDocumentTx.getStorage()).getAtomicOperationsManager().getCurrentOperation()); + ((OAbstractPaginatedStorage) ((ODatabaseInternal) databaseDocumentTx).getStorage()).getAtomicOperationsManager() + .getCurrentOperation()); assertFileRestoreFromWAL(); } @@ -167,7 +152,8 @@ public void testKeyDelete() throws IOException { super.testKeyDelete(); Assert.assertNull( - ((OAbstractPaginatedStorage) databaseDocumentTx.getStorage()).getAtomicOperationsManager().getCurrentOperation()); + ((OAbstractPaginatedStorage) ((ODatabaseInternal) databaseDocumentTx).getStorage()).getAtomicOperationsManager() + .getCurrentOperation()); assertFileRestoreFromWAL(); } @@ -177,7 +163,8 @@ public void testKeyDeleteRandomUniform() throws IOException { super.testKeyDeleteRandomUniform(); Assert.assertNull( - ((OAbstractPaginatedStorage) databaseDocumentTx.getStorage()).getAtomicOperationsManager().getCurrentOperation()); + ((OAbstractPaginatedStorage) ((ODatabaseInternal) databaseDocumentTx).getStorage()).getAtomicOperationsManager() + .getCurrentOperation()); assertFileRestoreFromWAL(); } @@ -187,7 +174,8 @@ public void testKeyDeleteRandomGaussian() throws IOException { super.testKeyDeleteRandomGaussian(); Assert.assertNull( - ((OAbstractPaginatedStorage) databaseDocumentTx.getStorage()).getAtomicOperationsManager().getCurrentOperation()); + ((OAbstractPaginatedStorage) ((ODatabaseInternal) databaseDocumentTx).getStorage()).getAtomicOperationsManager() + .getCurrentOperation()); assertFileRestoreFromWAL(); } @@ -197,7 +185,8 @@ public void testKeyAddDelete() throws IOException { super.testKeyAddDelete(); Assert.assertNull( - ((OAbstractPaginatedStorage) databaseDocumentTx.getStorage()).getAtomicOperationsManager().getCurrentOperation()); + ((OAbstractPaginatedStorage) ((ODatabaseInternal) databaseDocumentTx).getStorage()).getAtomicOperationsManager() + .getCurrentOperation()); assertFileRestoreFromWAL(); } @@ -207,7 +196,8 @@ public void testKeyPutRemoveNullKey() throws IOException { super.testKeyPutRemoveNullKey(); Assert.assertNull( - ((OAbstractPaginatedStorage) databaseDocumentTx.getStorage()).getAtomicOperationsManager().getCurrentOperation()); + ((OAbstractPaginatedStorage) ((ODatabaseInternal) databaseDocumentTx).getStorage()).getAtomicOperationsManager() + .getCurrentOperation()); assertFileRestoreFromWAL(); } @@ -257,12 +247,12 @@ private void assertFileRestoreFromWAL() throws IOException { } private void restoreDataFromWAL() throws IOException { - OWriteAheadLog log = ((OAbstractPaginatedStorage) databaseDocumentTx.getStorage()).getWALInstance(); + OWriteAheadLog log = ((OAbstractPaginatedStorage) ((ODatabaseInternal) databaseDocumentTx).getStorage()).getWALInstance(); OLogSequenceNumber lsn = log.begin(); - List atomicUnit = new ArrayList(); - List batch = new ArrayList(); + List atomicUnit = new ArrayList<>(); + List batch = new ArrayList<>(); boolean atomicChangeIsProcessed = false; while (lsn != null) { @@ -271,7 +261,7 @@ private void restoreDataFromWAL() throws IOException { if (batch.size() >= 1000) { atomicChangeIsProcessed = restoreDataFromBatch(atomicChangeIsProcessed, atomicUnit, batch); - batch = new ArrayList(); + batch = new ArrayList<>(); } lsn = log.next(lsn); @@ -279,20 +269,22 @@ private void restoreDataFromWAL() throws IOException { if (batch.size() > 0) { restoreDataFromBatch(atomicChangeIsProcessed, atomicUnit, batch); - batch = null; } Assert.assertTrue(atomicUnit.isEmpty()); - OWriteCache writeCache = ((OAbstractPaginatedStorage) expectedDatabaseDocumentTx.getStorage()).getWriteCache(); + OWriteCache writeCache = ((OAbstractPaginatedStorage) ((ODatabaseInternal) expectedDatabaseDocumentTx).getStorage()) + .getWriteCache(); writeCache.flush(); } private boolean restoreDataFromBatch(boolean atomicChangeIsProcessed, List atomicUnit, List records) throws IOException { - final OReadCache expectedReadCache = ((OAbstractPaginatedStorage) expectedDatabaseDocumentTx.getStorage()).getReadCache(); - final OWriteCache expectedWriteCache = ((OAbstractPaginatedStorage) expectedDatabaseDocumentTx.getStorage()).getWriteCache(); + final OReadCache expectedReadCache = ((OAbstractPaginatedStorage) ((ODatabaseInternal) expectedDatabaseDocumentTx).getStorage()) + .getReadCache(); + final OWriteCache expectedWriteCache = ((OAbstractPaginatedStorage) ((ODatabaseInternal) expectedDatabaseDocumentTx) + .getStorage()).getWriteCache(); for (OWALRecord walRecord : records) { if (walRecord instanceof OOperationUnitBodyRecord) @@ -361,30 +353,30 @@ private void assertFileContentIsTheSame(String expectedIMCFile, String actualIMC } private void assertCompareFilesAreTheSame(File expectedFile, File actualFile) throws IOException { - RandomAccessFile fileOne = new RandomAccessFile(expectedFile, "r"); - RandomAccessFile fileTwo = new RandomAccessFile(actualFile, "r"); + try (RandomAccessFile fileOne = new RandomAccessFile(expectedFile, "r")) { + try (RandomAccessFile fileTwo = new RandomAccessFile(actualFile, "r")) { - Assert.assertEquals(fileOne.length(), fileTwo.length()); + Assert.assertEquals(fileOne.length(), fileTwo.length()); - byte[] expectedContent = new byte[OClusterPage.PAGE_SIZE]; - byte[] actualContent = new byte[OClusterPage.PAGE_SIZE]; + byte[] expectedContent = new byte[OClusterPage.PAGE_SIZE]; + byte[] actualContent = new byte[OClusterPage.PAGE_SIZE]; - fileOne.seek(OFileClassic.HEADER_SIZE); - fileTwo.seek(OFileClassic.HEADER_SIZE); + fileOne.seek(OFileClassic.HEADER_SIZE); + fileTwo.seek(OFileClassic.HEADER_SIZE); - int bytesRead = fileOne.read(expectedContent); - while (bytesRead >= 0) { - fileTwo.readFully(actualContent, 0, bytesRead); + int bytesRead = fileOne.read(expectedContent); + while (bytesRead >= 0) { + fileTwo.readFully(actualContent, 0, bytesRead); - Assert.assertArrayEquals(Arrays.copyOfRange(expectedContent, ODurablePage.NEXT_FREE_POSITION, ODurablePage.MAX_PAGE_SIZE_BYTES), - Arrays.copyOfRange(actualContent, ODurablePage.NEXT_FREE_POSITION, ODurablePage.MAX_PAGE_SIZE_BYTES)); + Assert.assertArrayEquals( + Arrays.copyOfRange(expectedContent, ODurablePage.NEXT_FREE_POSITION, ODurablePage.MAX_PAGE_SIZE_BYTES), + Arrays.copyOfRange(actualContent, ODurablePage.NEXT_FREE_POSITION, ODurablePage.MAX_PAGE_SIZE_BYTES)); - expectedContent = new byte[OClusterPage.PAGE_SIZE]; - actualContent = new byte[OClusterPage.PAGE_SIZE]; - bytesRead = fileOne.read(expectedContent); + expectedContent = new byte[OClusterPage.PAGE_SIZE]; + actualContent = new byte[OClusterPage.PAGE_SIZE]; + bytesRead = fileOne.read(expectedContent); + } + } } - - fileOne.close(); - fileTwo.close(); } } diff --git a/core/src/test/java/com/orientechnologies/orient/core/storage/index/sbtree/local/SBTreeTestEncryptionTestIT.java b/core/src/test/java/com/orientechnologies/orient/core/storage/index/sbtree/local/SBTreeTestEncryptionTestIT.java new file mode 100755 index 00000000000..860300c87c8 --- /dev/null +++ b/core/src/test/java/com/orientechnologies/orient/core/storage/index/sbtree/local/SBTreeTestEncryptionTestIT.java @@ -0,0 +1,36 @@ +package com.orientechnologies.orient.core.storage.index.sbtree.local; + +import com.orientechnologies.common.io.OFileUtils; +import com.orientechnologies.common.serialization.types.OIntegerSerializer; +import com.orientechnologies.orient.core.db.ODatabaseInternal; +import com.orientechnologies.orient.core.db.ODatabaseType; +import com.orientechnologies.orient.core.db.OrientDB; +import com.orientechnologies.orient.core.db.OrientDBConfig; +import com.orientechnologies.orient.core.encryption.OEncryption; +import com.orientechnologies.orient.core.encryption.OEncryptionFactory; +import com.orientechnologies.orient.core.serialization.serializer.binary.impl.OLinkSerializer; +import com.orientechnologies.orient.core.storage.impl.local.OAbstractPaginatedStorage; + +import java.io.File; + +public class SBTreeTestEncryptionTestIT extends SBTreeTestIT { + @Override + public void before() { + buildDirectory = System.getProperty("buildDirectory", "."); + + dbName = "localSBTreeEncryptedTest"; + final File dbDirectory = new File(buildDirectory, dbName); + OFileUtils.deleteRecursively(dbDirectory); + + orientDB = new OrientDB("plocal:" + buildDirectory, OrientDBConfig.defaultConfig()); + + orientDB.create(dbName, ODatabaseType.PLOCAL); + databaseDocumentTx = orientDB.open(dbName, "admin", "admin"); + + sbTree = new OSBTree<>("sbTreeEncrypted", ".sbt", ".nbt", + (OAbstractPaginatedStorage) ((ODatabaseInternal) databaseDocumentTx).getStorage()); + + final OEncryption encryption = OEncryptionFactory.INSTANCE.getEncryption("aes/gcm", "T1JJRU5UREJfSVNfQ09PTA=="); + sbTree.create(OIntegerSerializer.INSTANCE, OLinkSerializer.INSTANCE, null, 1, false, encryption); + } +} diff --git a/core/src/test/java/com/orientechnologies/orient/core/storage/index/sbtree/local/SBTreeTestIT.java b/core/src/test/java/com/orientechnologies/orient/core/storage/index/sbtree/local/SBTreeTestIT.java index 20f94dea270..9a446b28b8e 100755 --- a/core/src/test/java/com/orientechnologies/orient/core/storage/index/sbtree/local/SBTreeTestIT.java +++ b/core/src/test/java/com/orientechnologies/orient/core/storage/index/sbtree/local/SBTreeTestIT.java @@ -1,7 +1,12 @@ package com.orientechnologies.orient.core.storage.index.sbtree.local; +import com.orientechnologies.common.io.OFileUtils; import com.orientechnologies.common.serialization.types.OIntegerSerializer; -import com.orientechnologies.orient.core.db.document.ODatabaseDocumentTx; +import com.orientechnologies.orient.core.db.ODatabaseInternal; +import com.orientechnologies.orient.core.db.ODatabaseSession; +import com.orientechnologies.orient.core.db.ODatabaseType; +import com.orientechnologies.orient.core.db.OrientDB; +import com.orientechnologies.orient.core.db.OrientDBConfig; import com.orientechnologies.orient.core.db.record.OIdentifiable; import com.orientechnologies.orient.core.id.ORID; import com.orientechnologies.orient.core.id.ORecordId; @@ -9,6 +14,7 @@ import com.orientechnologies.orient.core.storage.impl.local.OAbstractPaginatedStorage; import org.junit.After; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; import java.io.File; @@ -28,40 +34,35 @@ */ public class SBTreeTestIT { private static final int KEYS_COUNT = 500000; - protected OSBTree sbTree; - protected ODatabaseDocumentTx databaseDocumentTx; - private String buildDirectory; + OSBTree sbTree; + protected ODatabaseSession databaseDocumentTx; + protected String buildDirectory; + protected OrientDB orientDB; - public SBTreeTestIT() { - buildDirectory = System.getProperty("buildDirectory"); - if (buildDirectory == null) - buildDirectory = "."; + String dbName; - final File dbDirectory = new File(buildDirectory, "localSBTreeTest"); - System.out.println(this.getClass().getSimpleName() + " test is initializing using DB directory = " + dbDirectory); + @Before + public void before() { + buildDirectory = System.getProperty("buildDirectory", ".") + File.separator + SBTreeTestIT.class.getSimpleName(); - databaseDocumentTx = new ODatabaseDocumentTx("plocal:" + buildDirectory + "/localSBTreeTest"); - if (databaseDocumentTx.exists()) { - databaseDocumentTx.open("admin", "admin"); - databaseDocumentTx.drop(); - } + dbName = "localSBTreeTest"; + final File dbDirectory = new File(buildDirectory, dbName); + OFileUtils.deleteRecursively(dbDirectory); + + orientDB = new OrientDB("plocal:" + buildDirectory, OrientDBConfig.defaultConfig()); + orientDB.create(dbName, ODatabaseType.PLOCAL); - databaseDocumentTx.create(); + databaseDocumentTx = orientDB.open(dbName, "admin", "admin"); - sbTree = new OSBTree("sbTree", ".sbt", ".nbt", - (OAbstractPaginatedStorage) databaseDocumentTx.getStorage()); + sbTree = new OSBTree<>("sbTree", ".sbt", ".nbt", + (OAbstractPaginatedStorage) ((ODatabaseInternal) databaseDocumentTx).getStorage()); sbTree.create(OIntegerSerializer.INSTANCE, OLinkSerializer.INSTANCE, null, 1, false, null); } @After public void afterMethod() throws Exception { - sbTree.clear(); - } - - public void afterClass() throws Exception { - sbTree.clear(); - sbTree.delete(); - databaseDocumentTx.drop(); + orientDB.drop(dbName); + orientDB.close(); } @Test @@ -86,7 +87,7 @@ public void testKeyPut() throws Exception { @Test public void testKeyPutRandomUniform() throws Exception { - final NavigableSet keys = new TreeSet(); + final NavigableSet keys = new TreeSet<>(); final Random random = new Random(); while (keys.size() < KEYS_COUNT) { @@ -108,7 +109,7 @@ public void testKeyPutRandomUniform() throws Exception { @Test public void testKeyPutRandomGaussian() throws Exception { - NavigableSet keys = new TreeSet(); + NavigableSet keys = new TreeSet<>(); long seed = System.currentTimeMillis(); System.out.println("testKeyPutRandomGaussian seed : " + seed); @@ -137,7 +138,7 @@ public void testKeyPutRandomGaussian() throws Exception { @Test public void testKeyDeleteRandomUniform() throws Exception { - NavigableSet keys = new TreeSet(); + NavigableSet keys = new TreeSet<>(); for (int i = 0; i < KEYS_COUNT; i++) { sbTree.put(i, new ORecordId(i % 32000, i)); keys.add(i); @@ -166,7 +167,7 @@ public void testKeyDeleteRandomUniform() throws Exception { @Test public void testKeyDeleteRandomGaussian() throws Exception { - NavigableSet keys = new TreeSet(); + NavigableSet keys = new TreeSet<>(); long seed = System.currentTimeMillis(); @@ -219,6 +220,7 @@ public void testKeyDelete() throws Exception { } Assert.assertEquals((int) sbTree.firstKey(), 1); + //noinspection ConstantConditions Assert.assertEquals((int) sbTree.lastKey(), (KEYS_COUNT - 1) % 3 == 0 ? KEYS_COUNT - 2 : KEYS_COUNT - 1); for (int i = 0; i < KEYS_COUNT; i++) { @@ -263,7 +265,7 @@ public void testKeyAddDelete() throws Exception { @Test public void testIterateEntriesMajor() { - NavigableMap keyValues = new TreeMap(); + NavigableMap keyValues = new TreeMap<>(); Random random = new Random(); while (keyValues.size() < KEYS_COUNT) { @@ -285,7 +287,7 @@ public void testIterateEntriesMajor() { @Test public void testIterateEntriesMinor() { - NavigableMap keyValues = new TreeMap(); + NavigableMap keyValues = new TreeMap<>(); Random random = new Random(); while (keyValues.size() < KEYS_COUNT) { @@ -307,7 +309,7 @@ public void testIterateEntriesMinor() { @Test public void testIterateEntriesBetween() { - NavigableMap keyValues = new TreeMap(); + NavigableMap keyValues = new TreeMap<>(); Random random = new Random(); while (keyValues.size() < KEYS_COUNT) { @@ -394,7 +396,7 @@ public void testAddKeyValuesAndRemoveFirstMiddleAndLastPages() throws Exception Assert.assertEquals((int) sbTree.firstKey(), 1730); Assert.assertEquals((int) sbTree.lastKey(), 8599); - Set identifiables = new HashSet(); + Set identifiables = new HashSet<>(); OSBTree.OSBTreeCursor cursor = sbTree.iterateEntriesMinor(7200, true, true); cursorToSet(identifiables, cursor); @@ -489,8 +491,8 @@ public void testAddKeyValuesAndRemoveFirstMiddleAndLastPages() throws Exception @Test public void testNullKeysInSBTree() { - final OSBTree nullSBTree = new OSBTree("nullSBTree", ".sbt", ".nbt", - (OAbstractPaginatedStorage) databaseDocumentTx.getStorage()); + final OSBTree nullSBTree = new OSBTree<>("nullSBTree", ".sbt", ".nbt", + (OAbstractPaginatedStorage) ((ODatabaseInternal) databaseDocumentTx).getStorage()); nullSBTree.create(OIntegerSerializer.INSTANCE, OLinkSerializer.INSTANCE, null, 1, true, null); try { diff --git a/core/src/test/java/com/orientechnologies/orient/core/storage/index/sbtree/local/SBTreeWALTestIT.java b/core/src/test/java/com/orientechnologies/orient/core/storage/index/sbtree/local/SBTreeWALTestIT.java index 9e87a453206..d2db0d9b809 100755 --- a/core/src/test/java/com/orientechnologies/orient/core/storage/index/sbtree/local/SBTreeWALTestIT.java +++ b/core/src/test/java/com/orientechnologies/orient/core/storage/index/sbtree/local/SBTreeWALTestIT.java @@ -1,16 +1,18 @@ package com.orientechnologies.orient.core.storage.index.sbtree.local; +import com.orientechnologies.common.io.OFileUtils; import com.orientechnologies.common.serialization.types.OIntegerSerializer; import com.orientechnologies.orient.core.config.OGlobalConfiguration; -import com.orientechnologies.orient.core.db.document.ODatabaseDocumentTx; +import com.orientechnologies.orient.core.db.ODatabaseInternal; +import com.orientechnologies.orient.core.db.ODatabaseSession; +import com.orientechnologies.orient.core.db.ODatabaseType; +import com.orientechnologies.orient.core.db.OrientDB; +import com.orientechnologies.orient.core.db.OrientDBConfig; import com.orientechnologies.orient.core.serialization.serializer.binary.impl.OLinkSerializer; -import com.orientechnologies.orient.core.storage.OStorage; import com.orientechnologies.orient.core.storage.cache.OCacheEntry; import com.orientechnologies.orient.core.storage.cache.OReadCache; import com.orientechnologies.orient.core.storage.cache.OWriteCache; -import com.orientechnologies.orient.core.storage.cache.local.OWOWCache; import com.orientechnologies.orient.core.storage.fs.OFileClassic; -import com.orientechnologies.orient.core.storage.impl.local.OAbstractPaginatedStorage; import com.orientechnologies.orient.core.storage.impl.local.paginated.OClusterPage; import com.orientechnologies.orient.core.storage.impl.local.paginated.OLocalPaginatedStorage; import com.orientechnologies.orient.core.storage.impl.local.paginated.base.ODurablePage; @@ -26,7 +28,6 @@ import com.orientechnologies.orient.core.storage.impl.local.paginated.wal.OUpdatePageRecord; import com.orientechnologies.orient.core.storage.impl.local.paginated.wal.OWALPage; import com.orientechnologies.orient.core.storage.impl.local.paginated.wal.OWALRecord; -import com.orientechnologies.orient.core.storage.impl.local.paginated.wal.OWriteAheadLog; import org.assertj.core.api.Assertions; import org.junit.After; import org.junit.Assert; @@ -50,28 +51,30 @@ public class SBTreeWALTestIT extends SBTreeTestIT { OGlobalConfiguration.FILE_LOCK.setValue(false); } - private String buildDirectory; + private OLocalPaginatedStorage actualStorage; + private OWriteCache actualWriteCache; - private ODatabaseDocumentTx expectedDatabaseDocumentTx; + private ODatabaseSession expectedDatabaseDocumentTx; + private OLocalPaginatedStorage expectedStorage; + private OReadCache expectedReadCache; + private OWriteCache expectedWriteCache; - private String actualStorageDir; private String expectedStorageDir; + private String actualStorageDir; - private OLocalPaginatedStorage actualStorage; - private OWriteCache actualWriteCache; - - private OReadCache expectedReadCache; - private OWriteCache expectedWriteCache; - private OWriteAheadLog expectedWAL; + private static final String DIR_NAME = SBTreeWALTestIT.class.getSimpleName(); + private static final String ACTUAL_DB_NAME = "sbtreeWithWALTestActual"; + private static final String EXPECTED_DB_NAME = "sbtreeWithWALTestExpected"; @Before - public void before() throws IOException { - buildDirectory = System.getProperty("buildDirectory", "."); - buildDirectory += "/sbtreeWithWALTest"; + public void before() { + String buildDirectory = System.getProperty("buildDirectory", "."); + buildDirectory += "/" + DIR_NAME; final File buildDir = new File(buildDirectory); - if (!buildDir.exists()) - buildDir.mkdirs(); + OFileUtils.deleteRecursively(buildDir); + + orientDB = new OrientDB("plocal:" + buildDir, OrientDBConfig.defaultConfig()); createExpectedSBTree(); createActualSBTree(); @@ -80,72 +83,37 @@ public void before() throws IOException { @After @Override public void afterMethod() throws Exception { - databaseDocumentTx.open("admin", "admin"); - databaseDocumentTx.drop(); - - expectedDatabaseDocumentTx.open("admin", "admin"); - expectedDatabaseDocumentTx.drop(); - - final File actualStorage = new File(actualStorageDir); - if (actualStorage.exists()) - Assert.assertTrue(actualStorage.delete()); - - final File expectedStorage = new File(expectedStorageDir); - if (expectedStorage.exists()) - Assert.assertTrue(expectedStorage.delete()); - - final File buildDir = new File(buildDirectory); - if (buildDir.exists()) - Assert.assertTrue(buildDir.delete()); + orientDB.drop(ACTUAL_DB_NAME); + orientDB.drop(EXPECTED_DB_NAME); + orientDB.close(); } - private void createActualSBTree() throws IOException { - actualStorageDir = buildDirectory + "/sbtreeWithWALTestActual"; - - File actualStorageDirFile = new File(actualStorageDir); - if (!actualStorageDirFile.exists()) - actualStorageDirFile.mkdirs(); + private void createActualSBTree() { + orientDB.create(ACTUAL_DB_NAME, ODatabaseType.PLOCAL); - databaseDocumentTx = new ODatabaseDocumentTx("plocal:" + actualStorageDir); - if (databaseDocumentTx.exists()) { - databaseDocumentTx.open("admin", "admin"); - databaseDocumentTx.drop(); - } else { - databaseDocumentTx.create(); - } - - actualStorage = (OLocalPaginatedStorage) databaseDocumentTx.getStorage(); + databaseDocumentTx = orientDB.open(ACTUAL_DB_NAME, "admin", "admin"); + actualStorage = (OLocalPaginatedStorage) ((ODatabaseInternal) databaseDocumentTx).getStorage(); + actualStorageDir = actualStorage.getStoragePath().toString(); ODiskWriteAheadLog writeAheadLog = (ODiskWriteAheadLog) actualStorage.getWALInstance(); actualStorage.synch(); writeAheadLog.addCutTillLimit(writeAheadLog.getFlushedLsn()); - OReadCache actualReadCache = ((OAbstractPaginatedStorage) databaseDocumentTx.getStorage()).getReadCache(); - actualWriteCache = ((OAbstractPaginatedStorage) databaseDocumentTx.getStorage()).getWriteCache(); + actualWriteCache = actualStorage.getWriteCache(); sbTree = new OSBTree<>("actualSBTree", ".sbt", ".nbt", actualStorage); sbTree.create(OIntegerSerializer.INSTANCE, OLinkSerializer.INSTANCE, null, 1, false, null); } private void createExpectedSBTree() { - expectedStorageDir = buildDirectory + "/sbtreeWithWALTestExpected"; - - File expectedStorageDirFile = new File(expectedStorageDir); - if (!expectedStorageDirFile.exists()) - expectedStorageDirFile.mkdirs(); - - expectedDatabaseDocumentTx = new ODatabaseDocumentTx("plocal:" + expectedStorageDir); - if (expectedDatabaseDocumentTx.exists()) { - expectedDatabaseDocumentTx.open("admin", "admin"); - expectedDatabaseDocumentTx.drop(); - } else { - expectedDatabaseDocumentTx.create(); - } + orientDB.create(EXPECTED_DB_NAME, ODatabaseType.PLOCAL); - OLocalPaginatedStorage expectedStorage = (OLocalPaginatedStorage) expectedDatabaseDocumentTx.getStorage(); + expectedDatabaseDocumentTx = orientDB.open(EXPECTED_DB_NAME, "admin", "admin"); + expectedStorage = (OLocalPaginatedStorage) ((ODatabaseInternal) expectedDatabaseDocumentTx).getStorage(); expectedReadCache = expectedStorage.getReadCache(); expectedWriteCache = expectedStorage.getWriteCache(); - expectedWAL = expectedStorage.getWALInstance(); + + expectedStorageDir = expectedStorage.getStoragePath().toString(); } @Override @@ -258,22 +226,20 @@ public void testIterateEntriesBetween() { private void assertFileRestoreFromWAL() throws IOException { long sbTreeFileId = actualWriteCache.fileIdByName(sbTree.getName() + ".sbt"); - String nativeSBTreeFileName = ((OWOWCache) actualWriteCache).nativeFileNameById(sbTreeFileId); + String nativeSBTreeFileName = actualWriteCache.nativeFileNameById(sbTreeFileId); - OStorage storage = databaseDocumentTx.getStorage(); databaseDocumentTx.activateOnCurrentThread(); databaseDocumentTx.close(); - storage.close(true, false); + actualStorage.close(true, false); restoreDataFromWAL(); long expectedSBTreeFileId = expectedWriteCache.fileIdByName("expectedSBTree.sbt"); - String expectedSBTreeNativeFileName = ((OWOWCache) expectedWriteCache).nativeFileNameById(expectedSBTreeFileId); + String expectedSBTreeNativeFileName = expectedWriteCache.nativeFileNameById(expectedSBTreeFileId); expectedDatabaseDocumentTx.activateOnCurrentThread(); expectedDatabaseDocumentTx.close(); - storage = expectedDatabaseDocumentTx.getStorage(); - storage.close(true, false); + expectedStorage.close(true, false); assertFileContentIsTheSame(expectedSBTreeNativeFileName, nativeSBTreeFileName); } @@ -283,7 +249,7 @@ private void restoreDataFromWAL() throws IOException { 16 * OWALPage.PAGE_SIZE, 120); OLogSequenceNumber lsn = log.begin(); - List atomicUnit = new ArrayList(); + List atomicUnit = new ArrayList<>(); boolean atomicChangeIsProcessed = false; while (lsn != null) { @@ -353,29 +319,30 @@ private void restoreDataFromWAL() throws IOException { private void assertFileContentIsTheSame(String expectedBTreeFileName, String actualBTreeFileName) throws IOException { File expectedFile = new File(expectedStorageDir, expectedBTreeFileName); - RandomAccessFile fileOne = new RandomAccessFile(expectedFile, "r"); - RandomAccessFile fileTwo = new RandomAccessFile(new File(actualStorageDir, actualBTreeFileName), "r"); + try (RandomAccessFile fileOne = new RandomAccessFile(expectedFile, "r")) { + try (RandomAccessFile fileTwo = new RandomAccessFile(new File(actualStorageDir, actualBTreeFileName), "r")) { - Assert.assertEquals(fileOne.length(), fileTwo.length()); + Assert.assertEquals(fileOne.length(), fileTwo.length()); - byte[] expectedContent = new byte[OClusterPage.PAGE_SIZE]; - byte[] actualContent = new byte[OClusterPage.PAGE_SIZE]; + byte[] expectedContent = new byte[OClusterPage.PAGE_SIZE]; + byte[] actualContent = new byte[OClusterPage.PAGE_SIZE]; - fileOne.seek(OFileClassic.HEADER_SIZE); - fileTwo.seek(OFileClassic.HEADER_SIZE); + fileOne.seek(OFileClassic.HEADER_SIZE); + fileTwo.seek(OFileClassic.HEADER_SIZE); - int bytesRead = fileOne.read(expectedContent); - while (bytesRead >= 0) { - fileTwo.readFully(actualContent, 0, bytesRead); + int bytesRead = fileOne.read(expectedContent); + while (bytesRead >= 0) { + fileTwo.readFully(actualContent, 0, bytesRead); - Assertions.assertThat(Arrays.copyOfRange(expectedContent, ODurablePage.NEXT_FREE_POSITION, ODurablePage.MAX_PAGE_SIZE_BYTES)) - .isEqualTo(Arrays.copyOfRange(actualContent, ODurablePage.NEXT_FREE_POSITION, ODurablePage.MAX_PAGE_SIZE_BYTES)); - expectedContent = new byte[OClusterPage.PAGE_SIZE]; - actualContent = new byte[OClusterPage.PAGE_SIZE]; - bytesRead = fileOne.read(expectedContent); - } + Assertions + .assertThat(Arrays.copyOfRange(expectedContent, ODurablePage.NEXT_FREE_POSITION, ODurablePage.MAX_PAGE_SIZE_BYTES)) + .isEqualTo(Arrays.copyOfRange(actualContent, ODurablePage.NEXT_FREE_POSITION, ODurablePage.MAX_PAGE_SIZE_BYTES)); + expectedContent = new byte[OClusterPage.PAGE_SIZE]; + actualContent = new byte[OClusterPage.PAGE_SIZE]; + bytesRead = fileOne.read(expectedContent); + } - fileOne.close(); - fileTwo.close(); + } + } } } diff --git a/tests/src/test/java/com/orientechnologies/orient/test/database/auto/AutoShardingTest.java b/tests/src/test/java/com/orientechnologies/orient/test/database/auto/AutoShardingTest.java index c301cede0d4..bfb2c6ed71d 100755 --- a/tests/src/test/java/com/orientechnologies/orient/test/database/auto/AutoShardingTest.java +++ b/tests/src/test/java/com/orientechnologies/orient/test/database/auto/AutoShardingTest.java @@ -15,33 +15,32 @@ */ package com.orientechnologies.orient.test.database.auto; -import org.testng.Assert; -import org.testng.annotations.BeforeMethod; -import org.testng.annotations.Optional; -import org.testng.annotations.Parameters; -import org.testng.annotations.Test; - import com.orientechnologies.common.listener.OProgressListener; import com.orientechnologies.common.serialization.types.OIntegerSerializer; import com.orientechnologies.orient.core.index.OIndex; import com.orientechnologies.orient.core.index.OIndexKeyCursor; -import com.orientechnologies.orient.core.storage.index.hashindex.local.OMurmurHash3HashFunction; import com.orientechnologies.orient.core.metadata.schema.OClass; import com.orientechnologies.orient.core.metadata.schema.OType; import com.orientechnologies.orient.core.record.impl.ODocument; import com.orientechnologies.orient.core.sharding.auto.OAutoShardingClusterSelectionStrategy; import com.orientechnologies.orient.core.sql.OCommandSQL; +import com.orientechnologies.orient.core.storage.index.hashindex.local.OMurmurHash3HashFunction; +import org.testng.Assert; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Optional; +import org.testng.annotations.Parameters; +import org.testng.annotations.Test; /** * Tests Auto-Sharding indexes (Since v2.2.0). */ @Test public class AutoShardingTest extends DocumentDBBaseTest { - private static final int ITERATIONS = 500; - private OClass cls; - private OIndex idx; - private final OMurmurHash3HashFunction hashFunction = new OMurmurHash3HashFunction(); - private int[] clusterIds; + private static final int ITERATIONS = 500; + private OClass cls; + private OIndex idx; + private final OMurmurHash3HashFunction hashFunction = new OMurmurHash3HashFunction(new OIntegerSerializer()); + private int[] clusterIds; @Parameters(value = "url") public AutoShardingTest(@Optional String url) { @@ -52,8 +51,6 @@ public AutoShardingTest(@Optional String url) { public void beforeMethod() throws Exception { super.beforeMethod(); - hashFunction.setValueSerializer(new OIntegerSerializer()); - if (database.getMetadata().getSchema().existsClass("AutoShardingTest")) database.getMetadata().getSchema().dropClass("AutoShardingTest");