first support Tx for embedded indexing

pom.xml

@@ -318,11 +318,11 @@
             <scope>test</scope>
         </dependency>
 
-
         <dependency>
             <groupId>com.vividsolutions</groupId>
             <artifactId>jts</artifactId>
             <version>1.13</version>
+            <optional>true</optional>
         </dependency>
         <dependency>
             <groupId>com.google.code.gson</groupId>

src/main/java/com/orientechnologies/lucene/builder/DocBuilder.java

@@ -23,10 +23,13 @@
 import com.orientechnologies.orient.core.record.impl.ODocument;
 import org.apache.lucene.document.Document;
 
+import java.util.Map;
+
 /**
  * Created by Enrico Risa on 02/09/15.
  */
 public interface DocBuilder {
 
-  public Document build(OIndexDefinition definition,Object key,OIdentifiable value,ODocument metadata);
+  public Document build(OIndexDefinition definition, Object key, OIdentifiable value, Map<String,Boolean> fieldsToStore,
+      ODocument metadata);
 }

src/main/java/com/orientechnologies/lucene/builder/ODocBuilder.java

@@ -29,13 +29,15 @@
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Map;
 
 /**
  * Created by Enrico Risa on 02/09/15.
  */
 public class ODocBuilder implements DocBuilder {
   @Override
-  public Document build(OIndexDefinition definition, Object key, OIdentifiable value, ODocument metadata) {
+  public Document build(OIndexDefinition definition, Object key, OIdentifiable value, Map<String, Boolean> fieldsToStore,
+      ODocument metadata) {
     Document doc = new Document();
 
     if (value != null) {
@@ -48,6 +50,10 @@ public Document build(OIndexDefinition definition, Object key, OIdentifiable val
     for (String f : definition.getFields()) {
       Object val = formattedKey.get(i);
       i++;
+      if (isToStore(f, fieldsToStore).equals(Field.Store.YES)) {
+        doc.add(OLuceneIndexType.createField(f + OLuceneIndexEngineAbstract.STORED, val, Field.Store.YES,
+            Field.Index.NOT_ANALYZED_NO_NORMS));
+      }
       doc.add(OLuceneIndexType.createField(f, val, Field.Store.NO, Field.Index.ANALYZED));
     }
     return doc;
@@ -66,10 +72,14 @@ private List<Object> formatKeys(OIndexDefinition definition, Object key) {
       keys.add(key);
     }
 
-    //a sort of pa
+    // a sort of pa
     for (int i = keys.size(); i < definition.getFields().size(); i++) {
       keys.add("");
     }
     return keys;
   }
+
+  protected Field.Store isToStore(String f, Map<String, Boolean> collectionFields) {
+    return collectionFields.get(f) ? Field.Store.YES : Field.Store.NO;
+  }
 }

src/main/java/com/orientechnologies/lucene/collections/OLuceneTxResultSet.java

@@ -21,7 +21,7 @@
 import com.orientechnologies.common.log.OLogManager;
 import com.orientechnologies.lucene.engine.OLuceneIndexEngineAbstract;
 import com.orientechnologies.lucene.query.QueryContext;
-import com.orientechnologies.lucene.tx.OLuceneTxChanges;
+import com.orientechnologies.lucene.tx.OLuceneTxChangesAbstract;
 import com.orientechnologies.orient.core.db.record.OIdentifiable;
 import com.orientechnologies.orient.core.id.OContextualRecordId;
 import org.apache.lucene.document.Document;
@@ -126,23 +126,23 @@ private OContextualRecordId toRecordId(Document doc, ScoreDoc score) {
     }
 
     private boolean isToSkip(OContextualRecordId res, Document doc) {
-      return isDeleted(res) || isUpdatedDiskMatch(res, doc);
+      return isDeleted(res, doc) || isUpdatedDiskMatch(res, doc);
     }
 
     private boolean isUpdatedDiskMatch(OIdentifiable value, Document doc) {
       return isUpdated(value) && !isTempMatch(doc);
     }
 
     private boolean isTempMatch(Document doc) {
-      return doc.get(OLuceneTxChanges.TMP) != null;
+      return doc.get(OLuceneTxChangesAbstract.TMP) != null;
     }
 
     private boolean isUpdated(OIdentifiable value) {
       return queryContext.changes().isUpdated(null, null, value);
     }
 
-    private boolean isDeleted(OIdentifiable value) {
-      return queryContext.changes().isDeleted(null, null, value);
+    private boolean isDeleted(OIdentifiable value, Document doc) {
+      return queryContext.changes().isDeleted(doc, null, value);
     }
 
     protected ScoreDoc fetchNext() {

src/main/java/com/orientechnologies/lucene/engine/OLuceneFullTextIndexEngine.java

@@ -274,7 +274,7 @@ public int getVersion() {
 
   @Override
   public Document buildDocument(Object key, OIdentifiable value) {
-    return builder.build(index, key, value, metadata);
+    return builder.build(index, key, value,collectionFields, metadata);
   }
 
   @Override

src/main/java/com/orientechnologies/lucene/engine/OLuceneIndexEngineAbstract.java

@@ -23,6 +23,8 @@
 import com.orientechnologies.lucene.OLuceneMapEntryIterator;
 import com.orientechnologies.lucene.query.QueryContext;
 import com.orientechnologies.lucene.tx.OLuceneTxChanges;
+import com.orientechnologies.lucene.tx.OLuceneTxChangesMultiRid;
+import com.orientechnologies.lucene.tx.OLuceneTxChangesSingleRid;
 import com.orientechnologies.lucene.utils.OLuceneIndexUtils;
 import com.orientechnologies.orient.core.OOrientListener;
 import com.orientechnologies.orient.core.Orient;
@@ -61,7 +63,6 @@
 import java.util.*;
 import java.util.concurrent.atomic.AtomicBoolean;
 
-//TODO: (frank) this is more and engine rather than a manager, maybe rename
 public abstract class OLuceneIndexEngineAbstract<V> extends OSharedResourceAdaptiveExternal implements OLuceneIndexEngine,
     OOrientListener {
 
@@ -295,35 +296,35 @@ public void setRebuilding(boolean rebuilding) {
   protected Analyzer getAnalyzer(final ODocument metadata) {
     if (metadata != null && metadata.field("analyzer") != null) {
       final String analyzerFQN = metadata.field("analyzer");
-        try {
-
-          final Class classAnalyzer = Class.forName(analyzerFQN);
-          final Constructor constructor = classAnalyzer.getConstructor();
+      try {
 
-          return (Analyzer) constructor.newInstance();
-        } catch (ClassNotFoundException e) {
-          throw new OIndexException("Analyzer: " + analyzerFQN + " not found", e);
-        } catch (NoSuchMethodException e) {
-          Class classAnalyzer = null;
-          try {
-            classAnalyzer = Class.forName(analyzerFQN);
-            return  (Analyzer) classAnalyzer.newInstance();
+        final Class classAnalyzer = Class.forName(analyzerFQN);
+        final Constructor constructor = classAnalyzer.getConstructor();
 
-          } catch (Throwable e1) {
-            throw new OIndexException("Couldn't instantiate analyzer:  public constructor  not found", e1);
-          }
+        return (Analyzer) constructor.newInstance();
+      } catch (ClassNotFoundException e) {
+        throw new OIndexException("Analyzer: " + analyzerFQN + " not found", e);
+      } catch (NoSuchMethodException e) {
+        Class classAnalyzer = null;
+        try {
+          classAnalyzer = Class.forName(analyzerFQN);
+          return (Analyzer) classAnalyzer.newInstance();
 
-        } catch (Exception e) {
-          OLogManager.instance().error(this, "Error on getting analyzer for Lucene index", e);
+        } catch (Throwable e1) {
+          throw new OIndexException("Couldn't instantiate analyzer:  public constructor  not found", e1);
         }
+
+      } catch (Exception e) {
+        OLogManager.instance().error(this, "Error on getting analyzer for Lucene index", e);
+      }
     }
     return new StandardAnalyzer();
   }
 
   public void initIndex(String indexName, String indexType, OIndexDefinition indexDefinition, boolean isAutomatic,
       ODocument metadata) {
 
-    //FIXME how many timers are around?
+    // FIXME how many timers are around?
     Orient.instance().registerListener(this);
     commitTask = new TimerTask() {
       @Override
@@ -434,7 +435,6 @@ private void reOpen(final ODocument metadata) throws IOException {
     mgrWriter = new TrackingIndexWriter(indexWriter);
     searcherManager = new SearcherManager(indexWriter, true, null);
 
-
     if (nrt != null) {
       nrt.close();
     }
@@ -535,6 +535,10 @@ public void onStorageUnregistered(OStorage storage) {
 
   @Override
   public OLuceneTxChanges buildTxChanges() throws IOException {
-    return new OLuceneTxChanges(this, createIndexWriter(new RAMDirectory()));
+    if (isCollectionDelete()) {
+      return new OLuceneTxChangesMultiRid(this, createIndexWriter(new RAMDirectory()));
+    } else {
+      return new OLuceneTxChangesSingleRid(this, createIndexWriter(new RAMDirectory()));
+    }
   }
 }

src/main/java/com/orientechnologies/lucene/tx/OLuceneTxChanges.java

@@ -18,85 +18,30 @@
 
 package com.orientechnologies.lucene.tx;
 
-import com.orientechnologies.common.log.OLogManager;
-import com.orientechnologies.lucene.OLuceneIndexType;
-import com.orientechnologies.lucene.engine.OLuceneIndexEngine;
 import com.orientechnologies.orient.core.db.record.OIdentifiable;
 import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.search.IndexSearcher;
 
 import java.io.IOException;
-import java.util.HashSet;
 import java.util.Set;
 
 /**
  * Created by Enrico Risa on 15/09/15.
  */
-public class OLuceneTxChanges {
+public interface OLuceneTxChanges {
 
-  public static final String       TMP         = "_tmp_rid";
+  public void put(Object key, OIdentifiable value, Document doc) throws IOException;
 
-  private final IndexWriter        writer;
-  private final OLuceneIndexEngine engine;
-  private final Set<String>        deleted     = new HashSet<String>();
-  private final Set<String>        updated     = new HashSet<String>();
-  private final Set<Document>      deletedDocs = new HashSet<Document>();
+  public void remove(Object key, OIdentifiable value) throws IOException;
 
-  public OLuceneTxChanges(OLuceneIndexEngine engine, IndexWriter writer) {
-    this.writer = writer;
-    this.engine = engine;
-  }
+  public IndexSearcher searcher() throws IOException;
 
-  public void put(Object key, OIdentifiable value, Document doc) throws IOException {
-    if (deleted.remove(value.getIdentity().toString())) {
-      doc.add(OLuceneIndexType.createField(TMP, value.getIdentity().toString(), Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS));
-      updated.add(value.getIdentity().toString());
-    }
-    writer.addDocument(doc);
-  }
+  public long numDocs();
 
-  public void remove(Object key, OIdentifiable value) throws IOException {
+  public Set<Document> getDeletedDocs();
 
-    if (value.getIdentity().isTemporary()) {
-      writer.deleteDocuments(engine.deleteQuery(key, value));
-    } else {
-      deleted.add(value.getIdentity().toString());
-      deletedDocs.add(engine.buildDocument(key, value));
-    }
-  }
+  public boolean isDeleted(Document document, Object key, OIdentifiable value);
 
-  public IndexSearcher searcher() {
-    // TODO optimize
-    try {
-      return new IndexSearcher(DirectoryReader.open(writer, true));
-    } catch (IOException e) {
-      OLogManager.instance().error(this, "Error during searcher instantiation", e);
-    }
+  public boolean isUpdated(Document document, Object key, OIdentifiable value);
 
-    return null;
-  }
-
-  public long numDocs() {
-    return searcher().getIndexReader().numDocs() - deleted.size() - updated.size();
-  }
-
-  public Set<Document> getDeletedDocs() {
-    return deletedDocs;
-  }
-
-  // TODO is ok for full text on string but with [] ?
-  public boolean isDeleted(Document document, Object key, OIdentifiable value) {
-    return deleted.contains(value.getIdentity().toString());
-  }
-
-  public boolean isUpdated(Document document, Object key, OIdentifiable value) {
-    return updated.contains(value.getIdentity().toString());
-  }
-
-  public boolean isChanged(Document document, Object key, OIdentifiable value) {
-    return isDeleted(document, key, value) || isUpdated(document, key, value);
-  }
 }

src/main/java/com/orientechnologies/lucene/tx/OLuceneTxChangesAbstract.java

@@ -0,0 +1,54 @@
+/*
+ *
+ *  * Copyright 2014 Orient Technologies.
+ *  *
+ *  * Licensed under the Apache License, Version 2.0 (the "License");
+ *  * you may not use this file except in compliance with the License.
+ *  * You may obtain a copy of the License at
+ *  *
+ *  *      http://www.apache.org/licenses/LICENSE-2.0
+ *  *
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS,
+ *  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  * See the License for the specific language governing permissions and
+ *  * limitations under the License.
+ *
+ */
+
+package com.orientechnologies.lucene.tx;
+
+import com.orientechnologies.common.log.OLogManager;
+import com.orientechnologies.lucene.engine.OLuceneIndexEngine;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.search.IndexSearcher;
+
+import java.io.IOException;
+
+/**
+ * Created by Enrico Risa on 28/09/15.
+ */
+public abstract class OLuceneTxChangesAbstract implements OLuceneTxChanges {
+
+  public static final String         TMP = "_tmp_rid";
+
+  protected final IndexWriter        writer;
+  protected final OLuceneIndexEngine engine;
+
+  public OLuceneTxChangesAbstract(OLuceneIndexEngine engine, IndexWriter writer) {
+    this.writer = writer;
+    this.engine = engine;
+  }
+
+  public IndexSearcher searcher() {
+    // TODO optimize
+    try {
+      return new IndexSearcher(DirectoryReader.open(writer, true));
+    } catch (IOException e) {
+      OLogManager.instance().error(this, "Error during searcher instantiation", e);
+    }
+
+    return null;
+  }
+}