Skip to content

Commit

Permalink
Index Engine Refactor and Support For Legacy Spatial Index
Browse files Browse the repository at this point in the history
  • Loading branch information
wolf4ood committed Sep 26, 2015
1 parent 399a216 commit 3bce24b
Show file tree
Hide file tree
Showing 32 changed files with 956 additions and 304 deletions.
7 changes: 7 additions & 0 deletions pom.xml
Expand Up @@ -268,6 +268,7 @@
<scope>compile</scope>
</dependency>


<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-core</artifactId>
Expand Down Expand Up @@ -323,6 +324,12 @@
<artifactId>jts</artifactId>
<version>1.13</version>
</dependency>
<dependency>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
<version>2.2.4</version>
<scope>test</scope>
</dependency>

</dependencies>
</project>
10 changes: 5 additions & 5 deletions src/main/java/com/orientechnologies/lucene/OLuceneIndexType.java
Expand Up @@ -16,7 +16,7 @@

package com.orientechnologies.lucene;

import com.orientechnologies.lucene.manager.OLuceneIndexManagerAbstract;
import com.orientechnologies.lucene.engine.OLuceneIndexEngineAbstract;
import com.orientechnologies.orient.core.db.record.OIdentifiable;
import com.orientechnologies.orient.core.index.OCompositeKey;
import com.orientechnologies.orient.core.index.OIndexDefinition;
Expand Down Expand Up @@ -68,7 +68,7 @@ public static Query createExactQuery(OIndexDefinition index, Object key) {
queryBuilder.add(new TermQuery(new Term(idx, key.toString())), BooleanClause.Occur.SHOULD);
}
} else {
queryBuilder.add(new TermQuery(new Term(OLuceneIndexManagerAbstract.KEY, key.toString())), BooleanClause.Occur.SHOULD);
queryBuilder.add(new TermQuery(new Term(OLuceneIndexEngineAbstract.KEY, key.toString())), BooleanClause.Occur.SHOULD);
}
query = queryBuilder.build();
} else if (key instanceof OCompositeKey) {
Expand All @@ -87,13 +87,13 @@ public static Query createExactQuery(OIndexDefinition index, Object key) {
}

public static Query createQueryId(OIdentifiable value) {
return new TermQuery(new Term(OLuceneIndexManagerAbstract.RID, value.getIdentity().toString()));
return new TermQuery(new Term(OLuceneIndexEngineAbstract.RID, value.getIdentity().toString()));
}

public static Query createDeleteQuery(OIdentifiable value, List<String> fields, Object key) {

final BooleanQuery.Builder queryBuilder = new BooleanQuery.Builder();
queryBuilder.add(new TermQuery(new Term(OLuceneIndexManagerAbstract.RID, value.getIdentity().toString())),
queryBuilder.add(new TermQuery(new Term(OLuceneIndexEngineAbstract.RID, value.getIdentity().toString())),
BooleanClause.Occur.MUST);

Map<String, String> values = new HashMap<String, String>();
Expand All @@ -104,7 +104,7 @@ public static Query createDeleteQuery(OIdentifiable value, List<String> fields,
values.put(fields.iterator().next(), key.toString());
}
for (String s : values.keySet()) {
queryBuilder.add(new TermQuery(new Term(s + OLuceneIndexManagerAbstract.STORED, values.get(s))), BooleanClause.Occur.MUST);
queryBuilder.add(new TermQuery(new Term(s + OLuceneIndexEngineAbstract.STORED, values.get(s))), BooleanClause.Occur.MUST);
}
return queryBuilder.build();
}
Expand Down
Expand Up @@ -17,7 +17,7 @@
package com.orientechnologies.lucene;

import com.orientechnologies.common.log.OLogManager;
import com.orientechnologies.lucene.manager.OLuceneIndexManagerAbstract;
import com.orientechnologies.lucene.engine.OLuceneIndexEngineAbstract;
import com.orientechnologies.orient.core.id.ORecordId;
import com.orientechnologies.orient.core.index.OIndexDefinition;
import org.apache.lucene.document.Document;
Expand Down Expand Up @@ -56,10 +56,10 @@ public Map.Entry<K, V> next() {
val += doc.get(field);
}
} else {
val = doc.get(OLuceneIndexManagerAbstract.KEY);
val = doc.get(OLuceneIndexEngineAbstract.KEY);
}
final String finalVal = val;
final ORecordId id = new ORecordId(doc.get(OLuceneIndexManagerAbstract.RID));
final ORecordId id = new ORecordId(doc.get(OLuceneIndexEngineAbstract.RID));
currentIdx++;
return new Map.Entry<K, V>() {
@Override
Expand Down
Expand Up @@ -19,7 +19,7 @@
package com.orientechnologies.lucene.builder;

import com.orientechnologies.lucene.OLuceneIndexType;
import com.orientechnologies.lucene.manager.OLuceneIndexManagerAbstract;
import com.orientechnologies.lucene.engine.OLuceneIndexEngineAbstract;
import com.orientechnologies.orient.core.db.record.OIdentifiable;
import com.orientechnologies.orient.core.index.OCompositeKey;
import com.orientechnologies.orient.core.index.OIndexDefinition;
Expand All @@ -39,7 +39,7 @@ public Document build(OIndexDefinition definition, Object key, OIdentifiable val
Document doc = new Document();

if (value != null) {
doc.add(OLuceneIndexType.createField(OLuceneIndexManagerAbstract.RID, value.getIdentity().toString(), Field.Store.YES,
doc.add(OLuceneIndexType.createField(OLuceneIndexEngineAbstract.RID, value.getIdentity().toString(), Field.Store.YES,
Field.Index.NOT_ANALYZED_NO_NORMS));
}
List<Object> formattedKey = formatKeys(definition, key);
Expand Down
Expand Up @@ -19,12 +19,11 @@
package com.orientechnologies.lucene.collections;

import com.orientechnologies.common.log.OLogManager;
import com.orientechnologies.lucene.manager.OLuceneIndexManagerAbstract;
import com.orientechnologies.lucene.engine.OLuceneIndexEngineAbstract;
import com.orientechnologies.lucene.query.QueryContext;
import com.orientechnologies.orient.core.db.record.OIdentifiable;
import com.orientechnologies.orient.core.id.OContextualRecordId;
import org.apache.lucene.document.Document;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TopDocs;

Expand All @@ -36,7 +35,7 @@
*/
public class LuceneResultSet extends OLuceneAbstractResultSet {

public LuceneResultSet(OLuceneIndexManagerAbstract manager, QueryContext queryContext) {
public LuceneResultSet(OLuceneIndexEngineAbstract manager, QueryContext queryContext) {
super(manager, queryContext);
}

Expand Down Expand Up @@ -83,7 +82,7 @@ public OIdentifiable next() {
OContextualRecordId res = null;
try {
ret = queryContext.getSearcher().doc(score.doc);
String rId = ret.get(OLuceneIndexManagerAbstract.RID);
String rId = ret.get(OLuceneIndexEngineAbstract.RID);
res = new OContextualRecordId(rId);
manager.onRecordAddedToResultSet(queryContext, res, ret, score);
} catch (IOException e) {
Expand Down
Expand Up @@ -18,7 +18,7 @@

package com.orientechnologies.lucene.collections;

import com.orientechnologies.lucene.manager.OLuceneIndexManagerAbstract;
import com.orientechnologies.lucene.engine.OLuceneIndexEngineAbstract;
import com.orientechnologies.lucene.query.QueryContext;

/**
Expand All @@ -31,7 +31,7 @@ public class LuceneResultSetFactory {
protected LuceneResultSetFactory() {
}

public OLuceneAbstractResultSet create(OLuceneIndexManagerAbstract manager, QueryContext queryContext) {
public OLuceneAbstractResultSet create(OLuceneIndexEngineAbstract manager, QueryContext queryContext) {

if (queryContext.isInTx()) {
return new OLuceneTxResultSet(manager, queryContext);
Expand Down
Expand Up @@ -19,7 +19,7 @@
package com.orientechnologies.lucene.collections;

import com.orientechnologies.common.log.OLogManager;
import com.orientechnologies.lucene.manager.OLuceneIndexManagerAbstract;
import com.orientechnologies.lucene.engine.OLuceneIndexEngineAbstract;
import com.orientechnologies.lucene.query.QueryContext;
import com.orientechnologies.orient.core.db.record.OIdentifiable;
import org.apache.lucene.search.Query;
Expand All @@ -36,11 +36,11 @@ public abstract class OLuceneAbstractResultSet implements Set<OIdentifiable> {

protected TopDocs topDocs;
protected Query query;
protected OLuceneIndexManagerAbstract manager;
protected OLuceneIndexEngineAbstract manager;
protected QueryContext queryContext;
protected static Integer PAGE_SIZE = 50;
protected static Integer PAGE_SIZE = 10000;

public OLuceneAbstractResultSet(OLuceneIndexManagerAbstract manager, QueryContext queryContext) {
public OLuceneAbstractResultSet(OLuceneIndexEngineAbstract manager, QueryContext queryContext) {
this.manager = manager;
this.queryContext = queryContext;
this.query = enhanceQuery(queryContext.query);
Expand Down
Expand Up @@ -19,7 +19,7 @@
package com.orientechnologies.lucene.collections;

import com.orientechnologies.common.log.OLogManager;
import com.orientechnologies.lucene.manager.OLuceneIndexManagerAbstract;
import com.orientechnologies.lucene.engine.OLuceneIndexEngineAbstract;
import com.orientechnologies.lucene.query.QueryContext;
import com.orientechnologies.lucene.tx.OLuceneTxChanges;
import com.orientechnologies.orient.core.db.record.OIdentifiable;
Expand All @@ -41,7 +41,7 @@ public class OLuceneTxResultSet extends OLuceneAbstractResultSet {

protected int deletedMatchCount = 0;

public OLuceneTxResultSet(OLuceneIndexManagerAbstract manager, QueryContext queryContext) {
public OLuceneTxResultSet(OLuceneIndexEngineAbstract manager, QueryContext queryContext) {
super(manager, queryContext);

deletedMatchCount = calculateDeletedMatch();
Expand Down Expand Up @@ -119,7 +119,7 @@ private Document toDocument(ScoreDoc score) {
}

private OContextualRecordId toRecordId(Document doc, ScoreDoc score) {
String rId = doc.get(OLuceneIndexManagerAbstract.RID);
String rId = doc.get(OLuceneIndexEngineAbstract.RID);
OContextualRecordId res = new OContextualRecordId(rId);
manager.onRecordAddedToResultSet(queryContext, res, doc, score);
return res;
Expand Down
Expand Up @@ -16,7 +16,7 @@
*
*/

package com.orientechnologies.lucene.manager;
package com.orientechnologies.lucene.engine;

import com.orientechnologies.common.log.OLogManager;
import com.orientechnologies.lucene.query.QueryContext;
Expand Down Expand Up @@ -57,10 +57,10 @@ public class OLuceneFacetManager {
protected FacetsConfig config = new FacetsConfig();
protected String facetField;
// protected String facetDim;
private OLuceneIndexManagerAbstract owner;
private OLuceneIndexEngineAbstract owner;
private ODocument metadata;

public OLuceneFacetManager(OLuceneIndexManagerAbstract owner, ODocument metadata) throws IOException {
public OLuceneFacetManager(OLuceneIndexEngineAbstract owner, ODocument metadata) throws IOException {
this.owner = owner;

this.metadata = metadata;
Expand Down
Expand Up @@ -14,7 +14,7 @@
* limitations under the License.
*/

package com.orientechnologies.lucene.manager;
package com.orientechnologies.lucene.engine;

import com.orientechnologies.common.log.OLogManager;
import com.orientechnologies.lucene.OLuceneIndexType;
Expand Down Expand Up @@ -44,13 +44,13 @@
import java.util.*;

//TODO: (frank) this is more and engine rather than a manager, maybe rename
public class OLuceneFullTextIndexManager extends OLuceneIndexManagerAbstract {
public class OLuceneFullTextIndexEngine extends OLuceneIndexEngineAbstract {

protected OLuceneFacetManager facetManager;
private DocBuilder builder;
private OQueryBuilder queryBuilder;

public OLuceneFullTextIndexManager(String idxName, DocBuilder builder, OQueryBuilder queryBuilder) {
public OLuceneFullTextIndexEngine(String idxName, DocBuilder builder, OQueryBuilder queryBuilder) {
super(idxName);
this.builder = builder;
this.queryBuilder = queryBuilder;
Expand Down
@@ -0,0 +1,110 @@
/*
*
* * Copyright 2014 Orient Technologies.
* *
* * Licensed under the Apache License, Version 2.0 (the "License");
* * you may not use this file except in compliance with the License.
* * You may obtain a copy of the License at
* *
* * http://www.apache.org/licenses/LICENSE-2.0
* *
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS,
* * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* * See the License for the specific language governing permissions and
* * limitations under the License.
*
*/

package com.orientechnologies.lucene.engine;

import com.orientechnologies.common.log.OLogManager;
import com.orientechnologies.lucene.collections.LuceneResultSet;
import com.orientechnologies.lucene.query.QueryContext;
import com.orientechnologies.lucene.query.SpatialQueryContext;
import com.orientechnologies.lucene.tx.OLuceneTxChanges;
import com.orientechnologies.orient.core.db.record.OIdentifiable;
import com.orientechnologies.orient.core.id.OContextualRecordId;
import com.orientechnologies.orient.core.index.OIndexDefinition;
import com.orientechnologies.orient.core.record.impl.ODocument;
import com.orientechnologies.orient.spatial.shape.OShapeBuilder;
import com.spatial4j.core.distance.DistanceUtils;
import com.spatial4j.core.shape.Point;
import org.apache.lucene.document.Document;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.spatial.SpatialStrategy;
import org.apache.lucene.spatial.bbox.BBoxStrategy;

import java.util.Collection;
import java.util.HashMap;
import java.util.Map;

public class OLuceneGeoSpatialIndexEngine extends OLuceneSpatialIndexEngineAbstract {

public OLuceneGeoSpatialIndexEngine(String name, OShapeBuilder factory) {
super(name, factory);
}

@Override
protected SpatialStrategy createSpatialStrategy(OIndexDefinition indexDefinition, ODocument metadata) {
return new BBoxStrategy(ctx, "location");
}

@Override
public Object get(Object key) {
return getInTx(key, null);
}

@Override
public Object getInTx(Object key, OLuceneTxChanges changes) {
try {
if (key instanceof Map) {
return newGeoSearch((Map<String, Object>) key);

} else {
// TODO HANDLE EXCEPTION
}
} catch (Exception e) {
OLogManager.instance().error(this, "Error on getting entry against Lucene index", e);
}

return null;
}

private Object newGeoSearch(Map<String, Object> key) throws Exception {
return new LuceneResultSet(this, queryStrategy.build(key));

}

@Override
public void onRecordAddedToResultSet(QueryContext queryContext, OContextualRecordId recordId, Document doc, ScoreDoc score) {

SpatialQueryContext spatialContext = (SpatialQueryContext) queryContext;
if (spatialContext.spatialArgs != null) {
Point docPoint = (Point) ctx.readShape(doc.get(strategy.getFieldName()));
double docDistDEG = ctx.getDistCalc().distance(spatialContext.spatialArgs.getShape().getCenter(), docPoint);
final double docDistInKM = DistanceUtils.degrees2Dist(docDistDEG, DistanceUtils.EARTH_EQUATORIAL_RADIUS_KM);
recordId.setContext(new HashMap<String, Object>() {
{
put("distance", docDistInKM);
}
});
}
}

@Override
public void put(Object key, Object value) {

if (key instanceof OIdentifiable) {

ODocument location = ((OIdentifiable) key).getRecord();
Collection<OIdentifiable> container = (Collection<OIdentifiable>) value;
for (OIdentifiable oIdentifiable : container) {
addDocument(newGeoDocument(oIdentifiable, factory.fromDoc(location)));
}
} else {

}
}

}
Expand Up @@ -14,14 +14,13 @@
* limitations under the License.
*/

package com.orientechnologies.lucene.manager;
package com.orientechnologies.lucene.engine;

import com.orientechnologies.common.concur.resource.OSharedResourceAdaptiveExternal;
import com.orientechnologies.common.log.OLogManager;
import com.orientechnologies.common.serialization.types.OBinarySerializer;
import com.orientechnologies.lucene.OLuceneIndexType;
import com.orientechnologies.lucene.OLuceneMapEntryIterator;
import com.orientechnologies.lucene.engine.OLuceneIndexEngine;
import com.orientechnologies.lucene.query.QueryContext;
import com.orientechnologies.lucene.tx.OLuceneTxChanges;
import com.orientechnologies.lucene.utils.OLuceneIndexUtils;
Expand Down Expand Up @@ -63,7 +62,7 @@
import java.util.concurrent.atomic.AtomicBoolean;

//TODO: (frank) this is more and engine rather than a manager, maybe rename
public abstract class OLuceneIndexManagerAbstract<V> extends OSharedResourceAdaptiveExternal implements OLuceneIndexEngine,
public abstract class OLuceneIndexEngineAbstract<V> extends OSharedResourceAdaptiveExternal implements OLuceneIndexEngine,
OOrientListener {

public static final String RID = "RID";
Expand All @@ -88,7 +87,7 @@ public abstract class OLuceneIndexManagerAbstract<V> extends OSharedResourceAdap
private boolean rebuilding;
private long reopenToken;

public OLuceneIndexManagerAbstract(String indexName) {
public OLuceneIndexEngineAbstract(String indexName) {
super(OGlobalConfiguration.ENVIRONMENT_CONCURRENT.getValueAsBoolean(), OGlobalConfiguration.MVRBTREE_TIMEOUT
.getValueAsInteger(), true);
this.indexName = indexName;
Expand Down

0 comments on commit 3bce24b

Please sign in to comment.