Autosharding: completed creation of index by taking params by class c…

…onfifuration
orientechnologies · Jan 2, 2016 · fff3c36 · fff3c36
1 parent b78f63c
commit fff3c36
Show file tree

Hide file tree

Showing 17 changed files with 81 additions and 57 deletions.
diff --git a/core/src/main/java/com/orientechnologies/orient/core/index/OIndexAbstract.java b/core/src/main/java/com/orientechnologies/orient/core/index/OIndexAbstract.java
@@ -229,7 +229,7 @@ public OIndexInternal<?> create(final OIndexDefinition indexDefinition, final St
       final Boolean durableInNonTxMode = isDurableInNonTxMode();
 
       indexId = storage.addIndexEngine(name, algorithm, type, indexDefinition, valueSerializer, isAutomatic(), durableInNonTxMode,
-          version, getEngineProperties(), metadata);
+          version, getEngineProperties(), clustersToIndex, metadata);
       assert indexId >= 0;
 
       onIndexEngineChange(indexId);
@@ -424,7 +424,7 @@ public long rebuild(final OProgressListener iProgressListener) {
       removeValuesContainer();
 
       indexId = storage.addIndexEngine(name, algorithm, type, indexDefinition, determineValueSerializer(), isAutomatic(),
-          isDurableInNonTxMode(), version, getEngineProperties(), metadata);
+          isDurableInNonTxMode(), version, getEngineProperties(), clustersToIndex, metadata);
 
       onIndexEngineChange(indexId);
 

diff --git a/core/src/main/java/com/orientechnologies/orient/core/index/OIndexEngine.java b/core/src/main/java/com/orientechnologies/orient/core/index/OIndexEngine.java
@@ -26,6 +26,7 @@
 import com.orientechnologies.orient.core.record.impl.ODocument;
 
 import java.util.Collection;
+import java.util.Set;
 
 /**
  * @author Andrey Lomakin
@@ -37,7 +38,7 @@ public interface OIndexEngine {
   void flush();
 
   void create(OBinarySerializer valueSerializer, boolean isAutomatic, OType[] keyTypes, boolean nullPointerSupport,
-                 OBinarySerializer keySerializer, int keySize, ODocument metadata);
+                 OBinarySerializer keySerializer, int keySize, Set<String> clustersToIndex, ODocument metadata);
 
   void delete();
 

diff --git a/core/src/main/java/com/orientechnologies/orient/core/index/OIndexManagerAbstract.java b/core/src/main/java/com/orientechnologies/orient/core/index/OIndexManagerAbstract.java
@@ -43,7 +43,7 @@
 import com.orientechnologies.orient.core.metadata.schema.OType;
 import com.orientechnologies.orient.core.record.ORecord;
 import com.orientechnologies.orient.core.record.impl.ODocument;
-import com.orientechnologies.orient.core.sharding.OAutoShardingIndexFactory;
+import com.orientechnologies.orient.core.sharding.auto.OAutoShardingIndexFactory;
 import com.orientechnologies.orient.core.storage.OStorage;
 import com.orientechnologies.orient.core.storage.OStorageProxy;
 import com.orientechnologies.orient.core.type.ODocumentWrapper;

diff --git a/core/src/main/java/com/orientechnologies/orient/core/index/engine/OHashTableIndexEngine.java b/core/src/main/java/com/orientechnologies/orient/core/index/engine/OHashTableIndexEngine.java
@@ -43,6 +43,7 @@
 import java.util.Collections;
 import java.util.Iterator;
 import java.util.Map;
+import java.util.Set;
 
 /**
  * @author Andrey Lomakin
@@ -94,7 +95,7 @@ public String getName() {
 
   @Override
   public void create(OBinarySerializer valueSerializer, boolean isAutomatic, OType[] keyTypes, boolean nullPointerSupport,
-                        OBinarySerializer keySerializer, int keySize, ODocument metadata) {
+                        OBinarySerializer keySerializer, int keySize, Set<String> clustersToIndex, ODocument metadata) {
     hashFunction.setValueSerializer(keySerializer);
 
     hashTable.create(keySerializer, valueSerializer, keyTypes, nullPointerSupport);

diff --git a/core/src/main/java/com/orientechnologies/orient/core/index/engine/ORemoteIndexEngine.java b/core/src/main/java/com/orientechnologies/orient/core/index/engine/ORemoteIndexEngine.java
@@ -30,6 +30,7 @@
 import com.orientechnologies.orient.core.record.impl.ODocument;
 
 import java.util.Map;
+import java.util.Set;
 
 /**
  * @author Andrey Lomakin
@@ -57,7 +58,7 @@ public void flush() {
 
   @Override
   public void create(OBinarySerializer valueSerializer, boolean isAutomatic, OType[] keyTypes, boolean nullPointerSupport,
-                        OBinarySerializer keySerializer, int keySize, ODocument metadata) {
+                        OBinarySerializer keySerializer, int keySize, Set<String> clustersToIndex, ODocument metadata) {
   }
 
   @Override

diff --git a/core/src/main/java/com/orientechnologies/orient/core/index/engine/OSBTreeIndexEngine.java b/core/src/main/java/com/orientechnologies/orient/core/index/engine/OSBTreeIndexEngine.java
@@ -37,6 +37,7 @@
 
 import java.util.Iterator;
 import java.util.Map;
+import java.util.Set;
 
 /**
  * @author Andrey Lomakin
@@ -90,7 +91,7 @@ public void flush() {
 
   @Override
   public void create(OBinarySerializer valueSerializer, boolean isAutomatic, OType[] keyTypes, boolean nullPointerSupport,
-                        OBinarySerializer keySerializer, int keySize, ODocument metadata) {
+                        OBinarySerializer keySerializer, int keySize, Set<String> clustersToIndex, ODocument metadata) {
     acquireExclusiveLock();
     try {
       sbTree.create(keySerializer, valueSerializer, keyTypes, keySize, nullPointerSupport);

diff --git a/core/src/main/java/com/orientechnologies/orient/core/metadata/schema/OClassImpl.java b/core/src/main/java/com/orientechnologies/orient/core/metadata/schema/OClassImpl.java
@@ -52,7 +52,6 @@
 import com.orientechnologies.orient.core.serialization.OBinaryProtocol;
 import com.orientechnologies.orient.core.serialization.serializer.record.ORecordSerializerFactory;
 import com.orientechnologies.orient.core.serialization.serializer.record.string.ORecordSerializerSchemaAware2CSV;
-import com.orientechnologies.orient.core.sharding.OAutoShardingClusterSelectionStrategy;
 import com.orientechnologies.orient.core.sql.OCommandSQL;
 import com.orientechnologies.orient.core.sql.query.OSQLAsynchQuery;
 import com.orientechnologies.orient.core.storage.OAutoshardedStorage;

diff --git a/...AutoShardingClusterSelectionStrategy.java → ...AutoShardingClusterSelectionStrategy.java b/...AutoShardingClusterSelectionStrategy.java → ...AutoShardingClusterSelectionStrategy.java
@@ -13,7 +13,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package com.orientechnologies.orient.core.sharding;
+package com.orientechnologies.orient.core.sharding.auto;
 
 import com.orientechnologies.orient.core.db.ODatabaseRecordThreadLocal;
 import com.orientechnologies.orient.core.exception.OConfigurationException;

diff --git a/...re/sharding/OAutoShardingIndexEngine.java → ...arding/auto/OAutoShardingIndexEngine.java b/...re/sharding/OAutoShardingIndexEngine.java → ...arding/auto/OAutoShardingIndexEngine.java
@@ -17,7 +17,7 @@
  *  * For more information: http://www.orientechnologies.com
  *
  */
-package com.orientechnologies.orient.core.sharding;
+package com.orientechnologies.orient.core.sharding.auto;
 
 import com.orientechnologies.common.io.OIOUtils;
 import com.orientechnologies.common.serialization.types.OBinarySerializer;
@@ -35,12 +35,15 @@
 import com.orientechnologies.orient.core.index.hashindex.local.OMurmurHash3HashFunction;
 import com.orientechnologies.orient.core.metadata.schema.OType;
 import com.orientechnologies.orient.core.record.impl.ODocument;
+import com.orientechnologies.orient.core.storage.OStorage;
 import com.orientechnologies.orient.core.storage.impl.local.OAbstractPaginatedStorage;
+import com.orientechnologies.orient.core.storage.impl.local.paginated.OLocalPaginatedStorage;
 
 import java.io.File;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Set;
 
 /**
  * Index engine implementation that relies on multiple hash indexes partitioned by key.
@@ -49,20 +52,20 @@
  */
 public final class OAutoShardingIndexEngine implements OIndexEngine {
   public static final int    VERSION                             = 1;
-  public static final String METADATA_FILE_EXTENSION             = ".smet";
-  public static final String SUBINDEX_METADATA_FILE_EXTENSION    = ".shim";
-  public static final String SUBINDEX_TREE_FILE_EXTENSION        = ".shit";
-  public static final String SUBINDEX_BUCKET_FILE_EXTENSION      = ".shib";
-  public static final String SUBINDEX_NULL_BUCKET_FILE_EXTENSION = ".shnb";
+  public static final String METADATA_FILE_EXTENSION             = ".asmm";
+  public static final String SUBINDEX_METADATA_FILE_EXTENSION    = ".asm";
+  public static final String SUBINDEX_TREE_FILE_EXTENSION        = ".ast";
+  public static final String SUBINDEX_BUCKET_FILE_EXTENSION      = ".asb";
+  public static final String SUBINDEX_NULL_BUCKET_FILE_EXTENSION = ".asn";
 
   private final OAbstractPaginatedStorage        storage;
   private final boolean                          durableInNonTx;
   private final OMurmurHash3HashFunction<Object> hashFunction;
   private List<OHashTable<Object, Object>>       partitions;
-  private       OAutoShardingStrategy strategy;
-  private       int                   version;
-  private final String                name;
-  private       int                   partitionSize;
+  private OAutoShardingStrategy                  strategy;
+  private int                                    version;
+  private final String                           name;
+  private int                                    partitionSize;
 
   public OAutoShardingIndexEngine(final String iName, final Boolean iDurableInNonTxMode, final OAbstractPaginatedStorage iStorage,
       final int iVersion) {
@@ -87,24 +90,34 @@ public OAutoShardingStrategy getStrategy() {
     return strategy;
   }
 
-
   @Override
-  public void create(OBinarySerializer valueSerializer, boolean isAutomatic, OType[] keyTypes, boolean nullPointerSupport,
-      OBinarySerializer keySerializer, int keySize, ODocument metadata) {
+  public void create(final OBinarySerializer valueSerializer, final boolean isAutomatic, final OType[] keyTypes,
+      final boolean nullPointerSupport, final OBinarySerializer keySerializer, final int keySize, final Set<String> clustersToIndex,
+      ODocument metadata) {
+
     this.strategy = new OAutoShardingMurmurStrategy(keySerializer);
     this.hashFunction.setValueSerializer(keySerializer);
-
-    // FILE METADATA
-    final File fileMetadata = new File(name + METADATA_FILE_EXTENSION);
-    if (fileMetadata.exists())
-      fileMetadata.delete();
-    try {
-      OIOUtils.writeFile(fileMetadata, metadata.toJSON());
-    } catch (IOException e1) {
-      throw new OConfigurationException("Cannot create sharded index metadata file '" + fileMetadata + "'");
+    this.partitionSize = clustersToIndex.size();
+
+    final OStorage storage = getDatabase().getStorage().getUnderlying();
+    if (storage instanceof OLocalPaginatedStorage) {
+      // WRITE INDEX METADATA INFORMATION
+      final String path = ((OLocalPaginatedStorage) storage).getStoragePath();
+
+      final File fileMetadata = new File(path + "/" + name + METADATA_FILE_EXTENSION);
+      if (fileMetadata.exists())
+        fileMetadata.delete();
+      try {
+        if (metadata == null)
+          metadata = new ODocument();
+        metadata.field("partitions", partitionSize);
+        OIOUtils.writeFile(fileMetadata, metadata.toJSON());
+      } catch (IOException e1) {
+        throw new OConfigurationException("Cannot create sharded index metadata file '" + fileMetadata + "'");
+      }
     }
 
-    init(metadata);
+    init();
     for (OHashTable<Object, Object> p : partitions)
       p.create(keySerializer, valueSerializer, keyTypes, nullPointerSupport);
   }
@@ -115,17 +128,24 @@ public void load(final String indexName, final OBinarySerializer valueSerializer
 
     this.strategy = new OAutoShardingMurmurStrategy(keySerializer);
 
-    // FILE METADATA
-    final File fileMetadata = new File(name + METADATA_FILE_EXTENSION);
-    if (!fileMetadata.exists())
-      throw new OConfigurationException("Cannot find sharded index metadata file '" + fileMetadata + "'");
-    try {
-      final ODocument metadata = new ODocument();
-      metadata.fromJSON(OIOUtils.readFileAsString(fileMetadata));
-      init(metadata);
-
-    } catch (IOException e1) {
-      throw new OConfigurationException("Cannot load sharded index metadata file '" + fileMetadata + "'");
+    final OStorage storage = getDatabase().getStorage().getUnderlying();
+    if (storage instanceof OLocalPaginatedStorage) {
+      // LOAD INDEX METADATA INFORMATION
+      final String path = ((OLocalPaginatedStorage) storage).getStoragePath();
+
+      final File fileMetadata = new File(path + "/" + name + METADATA_FILE_EXTENSION);
+
+      if (!fileMetadata.exists())
+        throw new OConfigurationException("Cannot find sharded index metadata file '" + fileMetadata + "'");
+      try {
+        final ODocument metadata = new ODocument();
+        metadata.fromJSON(OIOUtils.readFileAsString(fileMetadata));
+        partitionSize = metadata.field("partitions");
+        init();
+
+      } catch (IOException e1) {
+        throw new OConfigurationException("Cannot load sharded index metadata file '" + fileMetadata + "'");
+      }
     }
 
     int i = 0;
@@ -158,11 +178,9 @@ public void init(final String indexName, final String indexType, final OIndexDef
       final boolean isAutomatic, final ODocument metadata) {
   }
 
-  private void init(final ODocument metadata) {
-    if (metadata == null || !metadata.containsField("partitions"))
-      throw new IllegalArgumentException("Index metadata does not contain 'partitions' field");
-
-    partitionSize = metadata.field("partitions");
+  private void init() {
+    if (partitions != null)
+      return;
 
     partitions = new ArrayList<OHashTable<Object, Object>>();
     for (int i = 0; i < partitionSize; ++i) {

diff --git a/...e/sharding/OAutoShardingIndexFactory.java → ...rding/auto/OAutoShardingIndexFactory.java b/...e/sharding/OAutoShardingIndexFactory.java → ...rding/auto/OAutoShardingIndexFactory.java
@@ -13,7 +13,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package com.orientechnologies.orient.core.sharding;
+package com.orientechnologies.orient.core.sharding.auto;
 
 import com.orientechnologies.orient.core.db.ODatabaseDocumentInternal;
 import com.orientechnologies.orient.core.exception.OConfigurationException;

diff --git a/...sharding/OAutoShardingMurmurStrategy.java → ...ing/auto/OAutoShardingMurmurStrategy.java b/...sharding/OAutoShardingMurmurStrategy.java → ...ing/auto/OAutoShardingMurmurStrategy.java
@@ -17,7 +17,7 @@
  *  * For more information: http://www.orientechnologies.com
  *
  */
-package com.orientechnologies.orient.core.sharding;
+package com.orientechnologies.orient.core.sharding.auto;
 
 import com.orientechnologies.common.serialization.types.OBinarySerializer;
 import com.orientechnologies.orient.core.index.hashindex.local.OMurmurHash3HashFunction;

diff --git a/.../core/sharding/OAutoShardingStrategy.java → .../sharding/auto/OAutoShardingStrategy.java b/.../core/sharding/OAutoShardingStrategy.java → .../sharding/auto/OAutoShardingStrategy.java
@@ -17,7 +17,7 @@
  *  * For more information: http://www.orientechnologies.com
  *
  */
-package com.orientechnologies.orient.core.sharding;
+package com.orientechnologies.orient.core.sharding.auto;
 
 /**
  * Auto-sharding strategy interface.

diff --git a/.../java/com/orientechnologies/orient/core/storage/impl/local/OAbstractPaginatedStorage.java b/.../java/com/orientechnologies/orient/core/storage/impl/local/OAbstractPaginatedStorage.java
@@ -1410,7 +1410,8 @@ public int loadExternalIndexEngine(String engineName, String algorithm, String i
 
   public int addIndexEngine(String engineName, final String algorithm, final String indexType,
       final OIndexDefinition indexDefinition, final OBinarySerializer valueSerializer, final boolean isAutomatic,
-      final Boolean durableInNonTxMode, final int version, final Map<String, String> engineProperties, final ODocument metadata) {
+      final Boolean durableInNonTxMode, final int version, final Map<String, String> engineProperties,
+      final Set<String> clustersToIndex, final ODocument metadata) {
     checkOpeness();
 
     stateLock.acquireWriteLock();
@@ -1444,7 +1445,7 @@ public int addIndexEngine(String engineName, final String algorithm, final Strin
 
       final OIndexEngine engine = OIndexes.createIndexEngine(originalName, algorithm, indexType, durableInNonTxMode, this, version,
           engineProperties);
-      engine.create(valueSerializer, isAutomatic, keyTypes, nullValuesSupport, keySerializer, keySize, metadata);
+      engine.create(valueSerializer, isAutomatic, keyTypes, nullValuesSupport, keySerializer, keySize, clustersToIndex, metadata);
 
       indexEngineNameMap.put(engineName, engine);
 

diff --git a/...rc/main/resources/META-INF/services/com.orientechnologies.orient.core.index.OIndexFactory b/...rc/main/resources/META-INF/services/com.orientechnologies.orient.core.index.OIndexFactory
@@ -40,4 +40,4 @@
 
 com.orientechnologies.orient.core.index.ODefaultIndexFactory
 com.orientechnologies.orient.core.index.hashindex.local.OHashIndexFactory
-com.orientechnologies.orient.core.sharding.OAutoShardingIndexFactory
+com.orientechnologies.orient.core.sharding.auto.OAutoShardingIndexFactory
diff --git a/lucene/src/main/java/com/orientechnologies/lucene/engine/OLuceneFullTextExpIndexEngine.java b/lucene/src/main/java/com/orientechnologies/lucene/engine/OLuceneFullTextExpIndexEngine.java
@@ -196,7 +196,7 @@ public void flush() {
 
   @Override
   public void create(OBinarySerializer valueSerializer, boolean isAutomatic, OType[] keyTypes, boolean nullPointerSupport,
-      OBinarySerializer keySerializer, int keySize, ODocument metadata) {
+      OBinarySerializer keySerializer, int keySize, Set<String> clustersToIndex, ODocument metadata) {
     luceneStorage.create(valueSerializer, isAutomatic, keyTypes, nullPointerSupport, keySerializer, keySize);
   }
 

diff --git a/lucene/src/main/java/com/orientechnologies/lucene/engine/OLuceneIndexEngineAbstract.java b/lucene/src/main/java/com/orientechnologies/lucene/engine/OLuceneIndexEngineAbstract.java
@@ -21,6 +21,7 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.TimerTask;
 import java.util.concurrent.atomic.AtomicBoolean;
 
@@ -104,7 +105,7 @@ public OLuceneIndexEngineAbstract(String indexName) {
 
   @Override
   public void create(OBinarySerializer valueSerializer, boolean isAutomatic, OType[] keyTypes, boolean nullPointerSupport,
-      OBinarySerializer keySerializer, int keySize, ODocument metadata) {
+      OBinarySerializer keySerializer, int keySize, Set<String> clustersToIndex, ODocument metadata) {
     // initIndex(indexName, null, isAutomatic, metadata);
   }
 

diff --git a/lucene/src/main/java/com/orientechnologies/lucene/engine/OLuceneIndexEngineDelegate.java b/lucene/src/main/java/com/orientechnologies/lucene/engine/OLuceneIndexEngineDelegate.java
@@ -19,6 +19,7 @@
 package com.orientechnologies.lucene.engine;
 
 import java.io.IOException;
+import java.util.Set;
 
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.document.Document;
@@ -68,7 +69,7 @@ public void flush() {
 
   @Override
   public void create(OBinarySerializer valueSerializer, boolean isAutomatic, OType[] keyTypes, boolean nullPointerSupport,
-      OBinarySerializer keySerializer, int keySize, ODocument metadata) {
+      OBinarySerializer keySerializer, int keySize, Set<String> clustersToIndex, ODocument metadata) {
   }
 
   @Override