Tests for encrypted indexes are added.

Few bugs in hash index encryption were fixed.

core/src/main/java/com/orientechnologies/orient/core/db/tool/ODatabaseImport.java

@@ -31,20 +31,29 @@
 import com.orientechnologies.orient.core.db.document.ODocumentFieldWalker;
 import com.orientechnologies.orient.core.db.record.OClassTrigger;
 import com.orientechnologies.orient.core.db.record.OIdentifiable;
-import com.orientechnologies.orient.core.db.tool.importer.*;
+import com.orientechnologies.orient.core.db.tool.importer.OConverterData;
+import com.orientechnologies.orient.core.db.tool.importer.OLinksRewriter;
 import com.orientechnologies.orient.core.exception.OConfigurationException;
 import com.orientechnologies.orient.core.exception.ODatabaseException;
 import com.orientechnologies.orient.core.exception.OSchemaException;
 import com.orientechnologies.orient.core.exception.OSerializationException;
 import com.orientechnologies.orient.core.id.ORID;
 import com.orientechnologies.orient.core.id.ORecordId;
-import com.orientechnologies.orient.core.index.*;
-import com.orientechnologies.orient.core.storage.index.hashindex.local.OHashIndexFactory;
-import com.orientechnologies.orient.core.storage.index.hashindex.local.OMurmurHash3HashFunction;
+import com.orientechnologies.orient.core.index.OIndex;
+import com.orientechnologies.orient.core.index.OIndexDefinition;
+import com.orientechnologies.orient.core.index.OIndexFactory;
+import com.orientechnologies.orient.core.index.OIndexManager;
+import com.orientechnologies.orient.core.index.OIndexes;
+import com.orientechnologies.orient.core.index.ORuntimeKeyIndexDefinition;
+import com.orientechnologies.orient.core.index.OSimpleKeyIndexDefinition;
 import com.orientechnologies.orient.core.intent.OIntentMassiveInsert;
 import com.orientechnologies.orient.core.metadata.OMetadataDefault;
 import com.orientechnologies.orient.core.metadata.function.OFunction;
-import com.orientechnologies.orient.core.metadata.schema.*;
+import com.orientechnologies.orient.core.metadata.schema.OClass;
+import com.orientechnologies.orient.core.metadata.schema.OClassImpl;
+import com.orientechnologies.orient.core.metadata.schema.OPropertyImpl;
+import com.orientechnologies.orient.core.metadata.schema.OSchema;
+import com.orientechnologies.orient.core.metadata.schema.OType;
 import com.orientechnologies.orient.core.metadata.security.OIdentity;
 import com.orientechnologies.orient.core.metadata.security.ORole;
 import com.orientechnologies.orient.core.metadata.security.OSecurityShared;
@@ -59,12 +68,29 @@
 import com.orientechnologies.orient.core.serialization.serializer.record.string.ORecordSerializerJSON;
 import com.orientechnologies.orient.core.storage.OPhysicalPosition;
 import com.orientechnologies.orient.core.storage.OStorage;
+import com.orientechnologies.orient.core.storage.index.hashindex.local.OHashIndexFactory;
+import com.orientechnologies.orient.core.storage.index.hashindex.local.OMurmurHash3HashFunction;
 
-import java.io.*;
+import java.io.BufferedInputStream;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.PrintWriter;
+import java.io.StringWriter;
 import java.lang.reflect.InvocationTargetException;
 import java.text.ParseException;
-import java.util.*;
+import java.util.AbstractList;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
 import java.util.Map.Entry;
+import java.util.Set;
 import java.util.zip.GZIPInputStream;
 
 /**
@@ -118,8 +144,7 @@ public void onMessage(String iText) {
       inStream = bf;
     }
 
-    OMurmurHash3HashFunction<OIdentifiable> keyHashFunction = new OMurmurHash3HashFunction<OIdentifiable>();
-    keyHashFunction.setValueSerializer(OLinkSerializer.INSTANCE);
+    OMurmurHash3HashFunction<OIdentifiable> keyHashFunction = new OMurmurHash3HashFunction<OIdentifiable>(OLinkSerializer.INSTANCE);
 
     jsonReader = new OJSONReader(new InputStreamReader(inStream));
     database.declareIntent(new OIntentMassiveInsert());

core/src/main/java/com/orientechnologies/orient/core/sharding/auto/OAutoShardingIndexEngine.java

@@ -36,10 +36,12 @@
 import com.orientechnologies.orient.core.metadata.schema.OType;
 import com.orientechnologies.orient.core.record.impl.ODocument;
 import com.orientechnologies.orient.core.storage.impl.local.OAbstractPaginatedStorage;
+import com.orientechnologies.orient.core.storage.index.hashindex.local.OHashFunction;
 import com.orientechnologies.orient.core.storage.index.hashindex.local.OHashIndexBucket;
 import com.orientechnologies.orient.core.storage.index.hashindex.local.OHashTable;
 import com.orientechnologies.orient.core.storage.index.hashindex.local.OLocalHashTable;
 import com.orientechnologies.orient.core.storage.index.hashindex.local.OMurmurHash3HashFunction;
+import com.orientechnologies.orient.core.storage.index.hashindex.local.OSHA256HashFunction;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -61,7 +63,6 @@ public final class OAutoShardingIndexEngine implements OIndexEngine {
 
   private final OAbstractPaginatedStorage        storage;
   private final boolean                          durableInNonTx;
-  private final OMurmurHash3HashFunction<Object> hashFunction;
   private       List<OHashTable<Object, Object>> partitions;
   private       OAutoShardingStrategy            strategy;
   private       int                              version;
@@ -73,7 +74,6 @@ public OAutoShardingIndexEngine(final String iName, final Boolean iDurableInNonT
       final int iVersion) {
     this.name = iName;
     this.storage = iStorage;
-    this.hashFunction = new OMurmurHash3HashFunction<Object>();
 
     if (iDurableInNonTxMode == null)
       durableInNonTx = iStorage.getConfiguration().getContextConfiguration()
@@ -99,8 +99,14 @@ public void create(final OBinarySerializer valueSerializer, final boolean isAuto
       final Map<String, String> engineProperties, final ODocument metadata, OEncryption encryption) {
 
     this.strategy = new OAutoShardingMurmurStrategy(keySerializer);
-    this.hashFunction.setValueSerializer(keySerializer);
-    this.hashFunction.setEncryption(encryption);
+
+    final OHashFunction<Object> hashFunction;
+
+    if (encryption != null) {
+      hashFunction = new OSHA256HashFunction<>(keySerializer);
+    } else {
+      hashFunction = new OMurmurHash3HashFunction<>(keySerializer);
+    }
 
     this.partitionSize = clustersToIndex.size();
     if (metadata != null && metadata.containsField("partitions"))
@@ -111,7 +117,7 @@ public void create(final OBinarySerializer valueSerializer, final boolean isAuto
     init();
 
     for (OHashTable<Object, Object> p : partitions) {
-      p.create(keySerializer, valueSerializer, keyTypes, encryption, nullPointerSupport);
+      p.create(keySerializer, valueSerializer, keyTypes, encryption, hashFunction, nullPointerSupport);
     }
   }
 
@@ -132,12 +138,18 @@ public void load(final String indexName, final OBinarySerializer valueSerializer
       init();
 
       int i = 0;
+
+      final OHashFunction<Object> hashFunction;
+
+      if (encryption != null) {
+        hashFunction = new OSHA256HashFunction<>(keySerializer);
+      } else {
+        hashFunction = new OMurmurHash3HashFunction<>(keySerializer);
+      }
+
       for (OHashTable<Object, Object> p : partitions)
-        p.load(indexName + "_" + (i++), keyTypes, nullPointerSupport, encryption);
+        p.load(indexName + "_" + (i++), keyTypes, nullPointerSupport, encryption, hashFunction);
     }
-
-    hashFunction.setValueSerializer(keySerializer);
-    hashFunction.setEncryption(encryption);
   }
 
   @Override
@@ -174,7 +186,7 @@ private void init() {
     for (int i = 0; i < partitionSize; ++i) {
       partitions.add(
           new OLocalHashTable<Object, Object>(name + "_" + i, SUBINDEX_METADATA_FILE_EXTENSION, SUBINDEX_TREE_FILE_EXTENSION,
-              SUBINDEX_BUCKET_FILE_EXTENSION, SUBINDEX_NULL_BUCKET_FILE_EXTENSION, hashFunction, storage));
+              SUBINDEX_BUCKET_FILE_EXTENSION, SUBINDEX_NULL_BUCKET_FILE_EXTENSION, storage));
     }
   }
 

core/src/main/java/com/orientechnologies/orient/core/sharding/auto/OAutoShardingMurmurStrategy.java

@@ -27,14 +27,14 @@
 /**
  * Auto-sharding strategy implementation that uses Murmur hashing.
  *
- * @since 3.0
  * @author Luca Garulli (l.garulli--(at)--orientdb.com)
+ * @since 3.0
  */
 public final class OAutoShardingMurmurStrategy implements OAutoShardingStrategy {
-  private OMurmurHash3HashFunction hashFunction = new OMurmurHash3HashFunction<Object>();
+  private OMurmurHash3HashFunction hashFunction;
 
   public OAutoShardingMurmurStrategy(final OBinarySerializer keySerializer) {
-    hashFunction.setValueSerializer(keySerializer);
+    hashFunction = new OMurmurHash3HashFunction<Object>(keySerializer);
   }
 
   public int getPartitionsId(final Object iKey, final int partitionSize) {

core/src/main/java/com/orientechnologies/orient/core/storage/index/engine/OHashTableIndexEngine.java

@@ -35,11 +35,13 @@
 import com.orientechnologies.orient.core.metadata.schema.OType;
 import com.orientechnologies.orient.core.record.impl.ODocument;
 import com.orientechnologies.orient.core.storage.impl.local.OAbstractPaginatedStorage;
+import com.orientechnologies.orient.core.storage.index.hashindex.local.OHashFunction;
 import com.orientechnologies.orient.core.storage.index.hashindex.local.OHashIndexBucket;
 import com.orientechnologies.orient.core.storage.index.hashindex.local.OHashTable;
 import com.orientechnologies.orient.core.storage.index.hashindex.local.OLocalHashTable;
 import com.orientechnologies.orient.core.storage.index.hashindex.local.OLocalHashTable20;
 import com.orientechnologies.orient.core.storage.index.hashindex.local.OMurmurHash3HashFunction;
+import com.orientechnologies.orient.core.storage.index.hashindex.local.OSHA256HashFunction;
 
 import java.util.Collections;
 import java.util.Iterator;
@@ -59,16 +61,14 @@ public final class OHashTableIndexEngine implements OIndexEngine {
   public static final String BUCKET_FILE_EXTENSION      = ".hib";
   public static final String NULL_BUCKET_FILE_EXTENSION = ".hnb";
 
-  private final OHashTable<Object, Object>       hashTable;
-  private final OMurmurHash3HashFunction<Object> hashFunction;
-  private final AtomicLong bonsayFileId = new AtomicLong(0);
+  private final OHashTable<Object, Object> hashTable;
+  private final AtomicLong                 bonsayFileId = new AtomicLong(0);
 
   private int version;
 
   private final String name;
 
   public OHashTableIndexEngine(String name, Boolean durableInNonTxMode, OAbstractPaginatedStorage storage, int version) {
-    hashFunction = new OMurmurHash3HashFunction<Object>();
 
     boolean durableInNonTx;
     if (durableInNonTxMode == null)
@@ -78,12 +78,13 @@ public OHashTableIndexEngine(String name, Boolean durableInNonTxMode, OAbstractP
       durableInNonTx = durableInNonTxMode;
 
     this.version = version;
-    if (version < 2)
+    if (version < 2) {
       hashTable = new OLocalHashTable20<Object, Object>(name, METADATA_FILE_EXTENSION, TREE_FILE_EXTENSION, BUCKET_FILE_EXTENSION,
-          NULL_BUCKET_FILE_EXTENSION, hashFunction, durableInNonTx, storage);
-    else
+          NULL_BUCKET_FILE_EXTENSION, durableInNonTx, storage);
+    } else {
       hashTable = new OLocalHashTable<Object, Object>(name, METADATA_FILE_EXTENSION, TREE_FILE_EXTENSION, BUCKET_FILE_EXTENSION,
-          NULL_BUCKET_FILE_EXTENSION, hashFunction, storage);
+          NULL_BUCKET_FILE_EXTENSION, storage);
+    }
 
     this.name = name;
   }
@@ -101,10 +102,15 @@ public String getName() {
   public void create(OBinarySerializer valueSerializer, boolean isAutomatic, OType[] keyTypes, boolean nullPointerSupport,
       OBinarySerializer keySerializer, int keySize, Set<String> clustersToIndex, Map<String, String> engineProperties,
       ODocument metadata, OEncryption encryption) {
-    hashFunction.setValueSerializer(keySerializer);
-    hashFunction.setEncryption(encryption);
+    final OHashFunction<Object> hashFunction;
+
+    if (encryption != null) {
+      hashFunction = new OSHA256HashFunction<>(keySerializer);
+    } else {
+      hashFunction = new OMurmurHash3HashFunction<>(keySerializer);
+    }
 
-    hashTable.create(keySerializer, valueSerializer, keyTypes, encryption, nullPointerSupport);
+    hashTable.create(keySerializer, valueSerializer, keyTypes, encryption, hashFunction, nullPointerSupport);
   }
 
   @Override
@@ -129,9 +135,15 @@ public void delete() {
   @Override
   public void load(String indexName, OBinarySerializer valueSerializer, boolean isAutomatic, OBinarySerializer keySerializer,
       OType[] keyTypes, boolean nullPointerSupport, int keySize, Map<String, String> engineProperties, OEncryption encryption) {
-    hashTable.load(indexName, keyTypes, nullPointerSupport, encryption);
-    hashFunction.setValueSerializer(hashTable.getKeySerializer());
-    hashFunction.setEncryption(encryption);
+
+    final OHashFunction<Object> hashFunction;
+
+    if (encryption != null) {
+      hashFunction = new OSHA256HashFunction<>(keySerializer);
+    } else {
+      hashFunction = new OMurmurHash3HashFunction<>(keySerializer);
+    }
+    hashTable.load(indexName, keyTypes, nullPointerSupport, encryption, hashFunction);
   }
 
   @Override

core/src/main/java/com/orientechnologies/orient/core/storage/index/hashindex/local/MessageDigestHolder.java

@@ -0,0 +1,24 @@
+package com.orientechnologies.orient.core.storage.index.hashindex.local;
+
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+
+public class MessageDigestHolder {
+  private static final MessageDigestHolder INSTANCE = new MessageDigestHolder();
+
+  private final ThreadLocal<MessageDigest> messageDigest = ThreadLocal.withInitial(() -> {
+    try {
+      return MessageDigest.getInstance("SHA-256");
+    } catch (NoSuchAlgorithmException e) {
+      throw new IllegalStateException("SHA-256 algorithm is not implemented", e);
+    }
+  });
+
+  public static MessageDigestHolder instance() {
+    return INSTANCE;
+  }
+
+  public MessageDigest get() {
+    return messageDigest.get();
+  }
+}

core/src/main/java/com/orientechnologies/orient/core/storage/index/hashindex/local/OHashTable.java

@@ -33,7 +33,7 @@
  */
 public interface OHashTable<K, V> {
   void create(OBinarySerializer<K> keySerializer, OBinarySerializer<V> valueSerializer, OType[] keyTypes, OEncryption encryption,
-      boolean nullKeyIsSupported);
+      OHashFunction<K> keyHashFunction, boolean nullKeyIsSupported);
 
   OBinarySerializer<K> getKeySerializer();
 
@@ -68,7 +68,7 @@ void create(OBinarySerializer<K> keySerializer, OBinarySerializer<V> valueSerial
 
   OHashIndexBucket.Entry<K, V>[] higherEntries(K key, int limit);
 
-  void load(String name, OType[] keyTypes, boolean nullKeyIsSupported, OEncryption encryption);
+  void load(String name, OType[] keyTypes, boolean nullKeyIsSupported, OEncryption encryption, OHashFunction<K> keyHashFunction);
 
   void deleteWithoutLoad(String name);
 

core/src/main/java/com/orientechnologies/orient/core/storage/index/hashindex/local/OLocalHashTable.java

@@ -89,13 +89,13 @@ public class OLocalHashTable<K, V> extends ODurableComponent implements OHashTab
 
   private static final int LEVEL_MASK = Integer.MAX_VALUE >>> (31 - MAX_LEVEL_DEPTH);
 
-  private final OHashFunction<K> keyHashFunction;
+  private OHashFunction<K> keyHashFunction;
 
   private OBinarySerializer<K> keySerializer;
   private OBinarySerializer<V> valueSerializer;
   private OType[]              keyTypes;
 
-  private final OHashTable.KeyHashCodeComparator<K> comparator;
+  private OHashTable.KeyHashCodeComparator<K> comparator;
 
   private       boolean nullKeyIsSupported;
   private       long    nullBucketFileId = -1;
@@ -111,22 +111,18 @@ public class OLocalHashTable<K, V> extends ODurableComponent implements OHashTab
   private OEncryption encryption;
 
   public OLocalHashTable(String name, String metadataConfigurationFileExtension, String treeStateFileExtension,
-      String bucketFileExtension, String nullBucketFileExtension, OHashFunction<K> keyHashFunction,
-      OAbstractPaginatedStorage abstractPaginatedStorage) {
+      String bucketFileExtension, String nullBucketFileExtension, OAbstractPaginatedStorage abstractPaginatedStorage) {
     super(abstractPaginatedStorage, name, bucketFileExtension, name + bucketFileExtension);
 
     this.metadataConfigurationFileExtension = metadataConfigurationFileExtension;
     this.treeStateFileExtension = treeStateFileExtension;
-    this.keyHashFunction = keyHashFunction;
     this.nullBucketFileExtension = nullBucketFileExtension;
-
-    this.comparator = new OHashTable.KeyHashCodeComparator<>(this.keyHashFunction);
   }
 
   @SuppressFBWarnings("DLS_DEAD_LOCAL_STORE")
   @Override
   public void create(OBinarySerializer<K> keySerializer, OBinarySerializer<V> valueSerializer, OType[] keyTypes,
-      OEncryption encryption, boolean nullKeyIsSupported) {
+      OEncryption encryption, OHashFunction<K> keyHashFunction, boolean nullKeyIsSupported) {
     startOperation();
     try {
       final OAtomicOperation atomicOperation;
@@ -139,6 +135,8 @@ public void create(OBinarySerializer<K> keySerializer, OBinarySerializer<V> valu
       acquireExclusiveLock();
       try {
         try {
+          this.keyHashFunction = keyHashFunction;
+          this.comparator = new OHashTable.KeyHashCodeComparator<>(this.keyHashFunction);
           this.encryption = encryption;
 
           if (keyTypes != null)
@@ -619,11 +617,14 @@ public OHashIndexBucket.Entry<K, V>[] higherEntries(K key, int limit) {
   }
 
   @Override
-  public void load(String name, OType[] keyTypes, boolean nullKeyIsSupported, OEncryption encryption) {
+  public void load(String name, OType[] keyTypes, boolean nullKeyIsSupported, OEncryption encryption,
+      OHashFunction<K> keyHashFunction) {
     startOperation();
     try {
       acquireExclusiveLock();
       try {
+        this.keyHashFunction = keyHashFunction;
+        this.comparator = new OHashTable.KeyHashCodeComparator<>(this.keyHashFunction);
         if (keyTypes != null)
           this.keyTypes = Arrays.copyOf(keyTypes, keyTypes.length);
         else