Skip to content

Commit

Permalink
index analyzers configurability
Browse files Browse the repository at this point in the history 
#5190
  • Loading branch information
@robfrank
robfrank committed Nov 30, 2015
1 parent 58ba8dc commit 513d9e0
Show file tree
Hide file tree
Showing 4 changed files with 90 additions and 22 deletions.
10 changes: 5 additions & 5 deletions lucene/src/main/java/com/orientechnologies/lucene/engine/OLuceneFullTextIndexEngine.java
View file
Expand Up @@ -60,8 +60,8 @@ public OLuceneFullTextIndexEngine(String idxName, DocBuilder builder, OQueryBuil
@Override @Override
public IndexWriter createIndexWriter(Directory directory) throws IOException { public IndexWriter createIndexWriter(Directory directory) throws IOException {


Analyzer analyzer = getAnalyzer(metadata); configureAnalyzers(metadata);
IndexWriterConfig iwc = new IndexWriterConfig(analyzer); IndexWriterConfig iwc = new IndexWriterConfig(indexAnalyzer());
iwc.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND); iwc.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND);


facetManager = new OLuceneFacetManager(this, metadata); facetManager = new OLuceneFacetManager(this, metadata);
Expand All @@ -73,8 +73,8 @@ public IndexWriter createIndexWriter(Directory directory) throws IOException {


@Override @Override
public IndexWriter openIndexWriter(Directory directory) throws IOException { public IndexWriter openIndexWriter(Directory directory) throws IOException {
Analyzer analyzer = getAnalyzer(metadata); configureAnalyzers(metadata);
IndexWriterConfig iwc = new IndexWriterConfig(analyzer); IndexWriterConfig iwc = new IndexWriterConfig(indexAnalyzer());
iwc.setOpenMode(IndexWriterConfig.OpenMode.APPEND); iwc.setOpenMode(IndexWriterConfig.OpenMode.APPEND);


// TODO: use writer config params to tune writer behaviour // TODO: use writer config params to tune writer behaviour
Expand Down Expand Up @@ -108,7 +108,7 @@ public Object get(Object key) {
public Object getInTx(Object key, OLuceneTxChanges changes) { public Object getInTx(Object key, OLuceneTxChanges changes) {
Query q = null; Query q = null;
try { try {
q = queryBuilder.query(index, key, mgrWriter.getIndexWriter().getAnalyzer()); q = queryBuilder.query(index, key, queryAnalyzer());
OCommandContext context = null; OCommandContext context = null;
if (key instanceof OFullTextCompositeKey) { if (key instanceof OFullTextCompositeKey) {
context = ((OFullTextCompositeKey) key).getContext(); context = ((OFullTextCompositeKey) key).getContext();
Expand Down
67 changes: 58 additions & 9 deletions lucene/src/main/java/com/orientechnologies/lucene/engine/OLuceneIndexEngineAbstract.java
View file
Expand Up @@ -53,7 +53,12 @@
import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.TrackingIndexWriter; import org.apache.lucene.index.TrackingIndexWriter;
import org.apache.lucene.search.*; import org.apache.lucene.search.ControlledRealTimeReopenThread;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.SearcherManager;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.Directory; import org.apache.lucene.store.Directory;
import org.apache.lucene.store.NIOFSDirectory; import org.apache.lucene.store.NIOFSDirectory;
import org.apache.lucene.store.RAMDirectory; import org.apache.lucene.store.RAMDirectory;
Expand All @@ -62,7 +67,12 @@
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.lang.reflect.Constructor; import java.lang.reflect.Constructor;
import java.util.*; import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.TimerTask;
import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicBoolean;


public abstract class OLuceneIndexEngineAbstract<V> extends OSharedResourceAdaptiveExternal public abstract class OLuceneIndexEngineAbstract<V> extends OSharedResourceAdaptiveExternal
Expand Down Expand Up @@ -304,18 +314,59 @@ public void setRebuilding(boolean rebuilding) {
this.rebuilding = rebuilding; this.rebuilding = rebuilding;
} }


protected Analyzer getAnalyzer(final ODocument metadata) { protected void configureAnalyzers(final ODocument metadata) {
String analyzerFQN = metadata.field("analyzer"); String defaultAnalyzerFQN = metadata.field("analyzer");
if (analyzerFQN == null)
analyzerFQN = StandardAnalyzer.class.getName(); //default analyzer for all fields
if (defaultAnalyzerFQN != null) {
for (String field : index.getFields()) {
indexAnalyzer.add(field, buildAnalyzer(defaultAnalyzerFQN));
queryAnalyzer.add(field, buildAnalyzer(defaultAnalyzerFQN));
}
}
String indexAnalyzerFQN = metadata.field("index_analyzer");

//default analyzer for indexing
if (indexAnalyzerFQN != null) {
for (String field : index.getFields()) {
indexAnalyzer.add(field, buildAnalyzer(indexAnalyzerFQN));
}
}

String queryAnalyzerFQN = metadata.field("query_analyzer");

//default analyzer for query
if (queryAnalyzerFQN != null) {
for (String field : index.getFields()) {
queryAnalyzer.add(field, buildAnalyzer(queryAnalyzerFQN));
}
}


//specialized for each field
for (String meta : metadata.fieldNames()) {
if (meta.contains("index") || meta.contains("query")) {
String fieldName = meta.substring(0, meta.indexOf("_"));
if (meta.contains("index")) {
indexAnalyzer.add(fieldName, buildAnalyzer(metadata.<String>field(meta)));
} else if (meta.contains("query")) {
queryAnalyzer.add(fieldName, buildAnalyzer(metadata.<String>field(meta)));
}
}
}

}

private Analyzer buildAnalyzer(String analyzerFQN) {

OLogManager.instance().debug(this, "looking for analyzer:: " + analyzerFQN);
try { try {


final Class classAnalyzer = Class.forName(analyzerFQN); final Class classAnalyzer = Class.forName(analyzerFQN);
final Constructor constructor = classAnalyzer.getConstructor(); final Constructor constructor = classAnalyzer.getConstructor();


return (Analyzer) constructor.newInstance(); return (Analyzer) constructor.newInstance();
} catch (ClassNotFoundException e) { } catch (ClassNotFoundException e) {

throw OException.wrapException(new OIndexException("Analyzer: " + analyzerFQN + " not found"), e); throw OException.wrapException(new OIndexException("Analyzer: " + analyzerFQN + " not found"), e);
} catch (NoSuchMethodException e) { } catch (NoSuchMethodException e) {
Class classAnalyzer = null; Class classAnalyzer = null;
Expand All @@ -324,7 +375,6 @@ protected Analyzer getAnalyzer(final ODocument metadata) {
return (Analyzer) classAnalyzer.newInstance(); return (Analyzer) classAnalyzer.newInstance();


} catch (Throwable e1) { } catch (Throwable e1) {

throw OException.wrapException(new OIndexException("Couldn't instantiate analyzer: public constructor not found"), e); throw OException.wrapException(new OIndexException("Couldn't instantiate analyzer: public constructor not found"), e);
} }


Expand Down Expand Up @@ -498,7 +548,6 @@ public static void sendLookupTime(String indexName, OCommandContext context, fin
} }
} }



private String getIndexPath(OLocalPaginatedStorage storageLocalAbstract) { private String getIndexPath(OLocalPaginatedStorage storageLocalAbstract) {
return getIndexPath(storageLocalAbstract, indexName); return getIndexPath(storageLocalAbstract, indexName);
} }
Expand Down
8 changes: 4 additions & 4 deletions .../src/main/java/com/orientechnologies/lucene/engine/OLuceneSpatialIndexEngineAbstract.java
View file
Expand Up @@ -76,16 +76,16 @@ public void initIndex(String indexName, String indexType, OIndexDefinition index


@Override @Override
public IndexWriter openIndexWriter(Directory directory) throws IOException { public IndexWriter openIndexWriter(Directory directory) throws IOException {
Analyzer analyzer = getAnalyzer(metadata); configureAnalyzers(metadata);
IndexWriterConfig iwc = new IndexWriterConfig(analyzer); IndexWriterConfig iwc = new IndexWriterConfig(indexAnalyzer());
iwc.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND); iwc.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND);
return new IndexWriter(directory, iwc); return new IndexWriter(directory, iwc);
} }


@Override @Override
public IndexWriter createIndexWriter(Directory directory) throws IOException { public IndexWriter createIndexWriter(Directory directory) throws IOException {
Analyzer analyzer = getAnalyzer(metadata); configureAnalyzers(metadata);
IndexWriterConfig iwc = new IndexWriterConfig(analyzer); IndexWriterConfig iwc = new IndexWriterConfig(indexAnalyzer());
iwc.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND); iwc.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND);
return new IndexWriter(directory, iwc); return new IndexWriter(directory, iwc);
} }
Expand Down
27 changes: 23 additions & 4 deletions lucene/src/test/java/com/orientechnologies/lucene/test/LuceneMultiFieldTest.java
View file
Expand Up @@ -25,13 +25,18 @@
import com.orientechnologies.orient.core.record.impl.ODocument; import com.orientechnologies.orient.core.record.impl.ODocument;
import com.orientechnologies.orient.core.sql.OCommandSQL; import com.orientechnologies.orient.core.sql.OCommandSQL;
import com.orientechnologies.orient.core.sql.query.OSQLSynchQuery; import com.orientechnologies.orient.core.sql.query.OSQLSynchQuery;
import org.apache.lucene.analysis.en.EnglishAnalyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.assertj.core.api.Assertions;
import org.testng.Assert;
import org.testng.annotations.AfterClass; import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test; import org.testng.annotations.Test;


import java.io.InputStream; import java.io.InputStream;
import java.util.List; import java.util.List;


import static org.assertj.core.api.Assertions.*;
import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThat;


/** /**
Expand Down Expand Up @@ -59,9 +64,21 @@ public void init() {
song.createProperty("title", OType.STRING); song.createProperty("title", OType.STRING);
song.createProperty("author", OType.STRING); song.createProperty("author", OType.STRING);


databaseDocumentTx.command(new OCommandSQL("create index Song.title_author on Song (title,author) FULLTEXT ENGINE LUCENE")) // databaseDocumentTx.command(new OCommandSQL("create index Song.title_author on Song (title,author) FULLTEXT ENGINE LUCENE"))
.execute(); // .execute();


databaseDocumentTx.command(new OCommandSQL(
"create index Song.title_author on Song (title,author) FULLTEXT ENGINE LUCENE METADATA {"
+ "\"title_index_analyzer\":\"" + EnglishAnalyzer.class.getName() + "\" , "
+ "\"title_query_analyzer\":\"" + EnglishAnalyzer.class.getName() + "\" , "
+ "\"author_index_analyzer\":\""
+ StandardAnalyzer.class.getName() +"\"}")).execute();


final ODocument index = databaseDocumentTx.getMetadata().getIndexManager().getIndex("Song.title_author").getMetadata();

assertThat(index.field("author_index_analyzer")).isEqualTo(StandardAnalyzer.class.getName());
assertThat(index.field("title_index_analyzer")).isEqualTo(EnglishAnalyzer.class.getName());
InputStream stream = ClassLoader.getSystemResourceAsStream("testLuceneIndex.sql"); InputStream stream = ClassLoader.getSystemResourceAsStream("testLuceneIndex.sql");


databaseDocumentTx.command(new OCommandScript("sql", getScriptFromStream(stream))).execute(); databaseDocumentTx.command(new OCommandScript("sql", getScriptFromStream(stream))).execute();
Expand All @@ -77,6 +94,8 @@ public void testSelectSingleDocumentWithAndOperator() {


List<ODocument> docs = databaseDocumentTx.query( List<ODocument> docs = databaseDocumentTx.query(
new OSQLSynchQuery<ODocument>("select * from Song where [title,author] LUCENE \"(title:mountain AND author:Fabbio)\"")); new OSQLSynchQuery<ODocument>("select * from Song where [title,author] LUCENE \"(title:mountain AND author:Fabbio)\""));
//List<ODocument> docs = databaseDocumentTx.query(
// new OSQLSynchQuery<ODocument>("select * from Song where [title,author] LUCENE \"(title:mountains)\""));


assertThat(docs).hasSize(1); assertThat(docs).hasSize(1);


Expand All @@ -88,7 +107,7 @@ public void testSelectMultipleDocumentsWithOrOperator() {
List<ODocument> docs = databaseDocumentTx.query( List<ODocument> docs = databaseDocumentTx.query(
new OSQLSynchQuery<ODocument>("select * from Song where [title,author] LUCENE \"(title:mountain OR author:Fabbio)\"")); new OSQLSynchQuery<ODocument>("select * from Song where [title,author] LUCENE \"(title:mountain OR author:Fabbio)\""));


assertThat(docs).hasSize(90); assertThat(docs).hasSize(91);


} }


Expand All @@ -98,7 +117,7 @@ public void testSelectOnTitleAndAuthorWithMatchOnTitle() {
List<ODocument> docs = databaseDocumentTx List<ODocument> docs = databaseDocumentTx
.query(new OSQLSynchQuery<ODocument>("select * from Song where [title,author] LUCENE \"mountain\"")); .query(new OSQLSynchQuery<ODocument>("select * from Song where [title,author] LUCENE \"mountain\""));


assertThat(docs).hasSize(4); assertThat(docs).hasSize(5);


} }


Expand Down

0 comments on commit 513d9e0

Please sign in to comment.