Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP

Loading…

Lucene 3.0.3 et FileAttachment -> Blob #14

Merged
2 commits merged into from

2 participants

@sgodbillon
Collaborator

:)

@jfp
Owner

Thank's for sharing !

This issue was closed.
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Commits on Feb 23, 2011
  1. @sgodbillon

    lucene 3.0.3

    sgodbillon authored
Commits on Feb 24, 2011
  1. @sgodbillon
This page is out of date. Refresh to see the latest.
View
BIN  lib/lucene-analyzers-2.3.1.jar
Binary file not shown
View
BIN  lib/lucene-analyzers-3.0.3.jar
Binary file not shown
View
BIN  lib/lucene-core-2.3.1.jar
Binary file not shown
View
BIN  lib/lucene-core-3.0.3.jar
Binary file not shown
View
45 src/play/modules/search/Query.java
@@ -6,8 +6,10 @@
import org.apache.lucene.queryParser.ParseException;
import org.apache.lucene.queryParser.QueryParser;
import org.apache.lucene.search.BooleanQuery;
-import org.apache.lucene.search.Hits;
+import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Sort;
+import org.apache.lucene.search.SortField;
+import org.apache.lucene.search.TopDocs;
import play.Play;
import play.db.jpa.JPA;
@@ -36,12 +38,15 @@
private boolean reverse = false;
- private Hits hits = null;
+ private IndexSearcher indexSearcher;
+
+ private TopDocs topDocs = null;
protected Query(String query, Class<JPABase> clazz, Store store) {
this.query = query;
this.clazz = clazz;
this.store = store;
+ indexSearcher = store.getIndexSearcher(clazz.getName());
}
public Query page(int offset, int pageSize) {
@@ -74,10 +79,14 @@ private Sort getSort() {
if (reverse) {
if (order.length != 1)
throw new SearchException("reverse can be used while sorting only one field with oderBy");
- else
- sort.setSort(order[0], reverse);
- } else
- sort.setSort(order);
+ sort.setSort(new SortField(order[0], SortField.SCORE, true));
+ } else {
+ SortField[] fields = new SortField[order.length];
+ for (int i = 0; i < fields.length; i++) {
+ fields[i] = new SortField(order[i], SortField.SCORE);
+ }
+ sort.setSort(fields);
+ }
}
return sort;
}
@@ -117,9 +126,9 @@ private Sort getSort() {
public long count() throws SearchException {
try {
- org.apache.lucene.search.Query luceneQuery = new QueryParser("_docID", Search.getAnalyser()).parse(query);
- hits = store.getIndexSearcher(clazz.getName()).search(luceneQuery, getSort());
- return hits.length();
+ org.apache.lucene.search.Query luceneQuery = new QueryParser(Search.getLuceneVersion(), "_docID", Search.getAnalyser()).parse(query);
+ topDocs = store.getIndexSearcher(clazz.getName()).search(luceneQuery, null, Integer.MAX_VALUE, getSort());
+ return topDocs.totalHits;
} catch (ParseException e) {
throw new SearchException(e);
} catch (Exception e) {
@@ -136,18 +145,18 @@ public long count() throws SearchException {
*/
public List<QueryResult> executeQuery(boolean fetch) throws SearchException {
try {
- if (hits == null) {
+ if (topDocs == null) {
org.apache.lucene.search.Query luceneQuery =
- new QueryParser("_docID", Search.getAnalyser()).parse(query);
+ new QueryParser(Search.getLuceneVersion(), "_docID", Search.getAnalyser()).parse(query);
BooleanQuery.setMaxClauseCount(Integer.parseInt(Play.configuration.getProperty(
"play.search.maxClauseCount", "1024")));
- hits = store.getIndexSearcher(clazz.getName()).search(luceneQuery, getSort());
+ topDocs = indexSearcher.search(luceneQuery, null, Integer.MAX_VALUE, getSort());
}
List<QueryResult> results = new ArrayList<QueryResult>();
- if (hits == null)
+ if (topDocs == null)
return results;
- int l = hits.length();
+ int l = topDocs.totalHits;
if (offset > l) {
return results;
}
@@ -155,8 +164,8 @@ public long count() throws SearchException {
if (pageSize > 0) {
for (int i = offset; i < (offset + pageSize > l ? l : offset + pageSize); i++) {
QueryResult qresult = new QueryResult();
- qresult.score = hits.score(i);
- qresult.id = hits.doc(i).get("_docID");
+ qresult.score = topDocs.scoreDocs[i].score;
+ qresult.id = indexSearcher.doc(topDocs.scoreDocs[i].doc).get("_docID");
if (fetch) {
Object objectId = ConvertionUtils.getIdValueFromIndex(clazz, qresult.id);
qresult.object = (JPABase)JPA.em().find(clazz, objectId);
@@ -168,8 +177,8 @@ public long count() throws SearchException {
} else {
for (int i = 0; i < l; i++) {
QueryResult qresult = new QueryResult();
- qresult.score = hits.score(i);
- qresult.id = hits.doc(i).get("_docID");
+ qresult.score = topDocs.scoreDocs[i].score;
+ qresult.id = indexSearcher.doc(topDocs.scoreDocs[i].doc).get("_docID");
if (fetch) {
Object objectId = ConvertionUtils.getIdValueFromIndex(clazz, qresult.id);
qresult.object = (JPABase)JPA.em().find(clazz, objectId);
View
11 src/play/modules/search/Search.java
@@ -1,6 +1,7 @@
package play.modules.search;
import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.util.Version;
import play.Logger;
import play.Play;
@@ -22,6 +23,7 @@
*/
public class Search {
private static String ANALYSER_CLASS;
+ private static Version LUCENE_VERSION;
private static Store store;
public static void init() {
@@ -31,6 +33,7 @@ public static void init() {
Logger.error(e, "Error while shutting down search module");
}
ANALYSER_CLASS = Play.configuration.getProperty("play.search.analyser", "org.apache.lucene.analysis.standard.StandardAnalyzer");
+ LUCENE_VERSION = Version.valueOf("LUCENE_" + Play.configuration.getProperty("play.search.lucene.version", "30"));
String storeClassName = Play.configuration.getProperty("play.search.store","play.modules.search.store.FilesystemStore");
try {
store = (Store) Class.forName(storeClassName).newInstance();
@@ -42,12 +45,16 @@ public static void init() {
public static Analyzer getAnalyser() {
try {
- Class clazz = Class.forName(ANALYSER_CLASS);
- return (Analyzer) clazz.newInstance();
+ Class<Analyzer> clazz = (Class<Analyzer>) Class.forName(ANALYSER_CLASS);
+ return clazz.getConstructor(Version.class).newInstance(getLuceneVersion());
} catch (Exception e) {
throw new RuntimeException(e);
}
}
+
+ public static Version getLuceneVersion() {
+ return LUCENE_VERSION;
+ }
public static Store getCurrentStore () {
return store;
View
10 src/play/modules/search/SearchPlugin.java
@@ -4,9 +4,10 @@
import play.PlayPlugin;
import play.Play.Mode;
import play.exceptions.UnexpectedException;
+import play.modules.search.store.FileExtractor;
import play.mvc.Router;
/**
- * Integrated to Play's lifcycle, SearchPlugin
+ * Integrated to Play's lifecycle, SearchPlugin
* will trap JPA events and drive the Search
* service.
* @author jfp
@@ -15,11 +16,12 @@
public class SearchPlugin extends PlayPlugin {
@Override
- public void onApplicationStart() {
+ public void onApplicationStart() {
Search.init();
- }
+ FileExtractor.init();
+ }
- @Override
+ @Override
public void onApplicationStop() {
try {
Search.shutdown();
View
22 src/play/modules/search/store/ConvertionUtils.java
@@ -10,7 +10,7 @@
import play.Logger;
import play.data.binding.Binder;
-import play.db.jpa.FileAttachment;
+import play.db.jpa.Blob;
import play.db.jpa.JPABase;
import play.db.jpa.Model;
import play.exceptions.UnexpectedException;
@@ -37,7 +37,7 @@ public static Document toDocument(Object object) throws Exception {
return null;
JPABase jpaBase = (JPABase) object;
Document document = new Document();
- document.add(new Field("_docID", getIdValueFor(jpaBase) + "", Field.Store.YES, Field.Index.UN_TOKENIZED));
+ document.add(new Field("_docID", getIdValueFor(jpaBase) + "", Field.Store.YES, Field.Index.NOT_ANALYZED));
StringBuffer allValue = new StringBuffer();
for (java.lang.reflect.Field field : object.getClass().getFields()) {
play.modules.search.Field index = field.getAnnotation(play.modules.search.Field.class);
@@ -67,14 +67,14 @@ public static Document toDocument(Object object) throws Exception {
continue;
document.add(new Field(name, value, index.stored() ? Field.Store.YES : Field.Store.NO,
- index.tokenize() ? Field.Index.TOKENIZED : Field.Index.UN_TOKENIZED));
+ index.tokenize() ? Field.Index.ANALYZED : Field.Index.NOT_ANALYZED));
if (index.tokenize() && index.sortable()) {
document.add(new Field(name + "_untokenized", value, index.stored() ? Field.Store.YES : Field.Store.NO,
- Field.Index.UN_TOKENIZED));
+ Field.Index.NOT_ANALYZED));
}
allValue.append(value).append(' ');
}
- document.add(new Field("allfield", allValue.toString(), Field.Store.NO, Field.Index.TOKENIZED));
+ document.add(new Field("allfield", allValue.toString(), Field.Store.NO, Field.Index.ANALYZED));
return document;
}
@@ -82,8 +82,8 @@ public static String valueOf(Object object, java.lang.reflect.Field field) throw
if (field.getType().equals(String.class)) {
return (String ) field.get(object);
}
- if (field.getType().equals(FileAttachment.class) && field.get(object) != null) {
- return FileExtractor.getText((FileAttachment ) field.get(object));
+ if (field.getType().equals(Blob.class) && field.get(object) != null) {
+ return FileExtractor.getText((Blob) field.get(object));
}
Object o = field.get(object);
@@ -103,9 +103,9 @@ public static String valueOf(Object object, java.lang.reflect.Field field) throw
* @param indexValue String value of the id, taken from index
* @return Object id expected to build query
*/
- public static Object getIdValueFromIndex(Class clazz, String indexValue) {
+ public static Object getIdValueFromIndex(Class<?> clazz, String indexValue) {
java.lang.reflect.Field field = getIdField(clazz);
- Class parameter = field.getType();
+ Class<?> parameter = field.getType();
try {
return Binder.directBind(indexValue, parameter);
} catch (Exception e) {
@@ -119,7 +119,7 @@ public static Object getIdValueFromIndex(Class clazz, String indexValue) {
* @param clazz JPABase target class
* @return corresponding field
*/
- public static java.lang.reflect.Field getIdField(Class clazz) {
+ public static java.lang.reflect.Field getIdField(Class<?> clazz) {
for (java.lang.reflect.Field field : clazz.getFields()) {
if (field.getAnnotation(Id.class) != null) {
return field;
@@ -152,7 +152,7 @@ public static Object getIdValueFor(JPABase jpaBase) {
return val;
}
- public static boolean isForcedUntokenized(Class clazz, String fieldName) {
+ public static boolean isForcedUntokenized(Class<?> clazz, String fieldName) {
try {
java.lang.reflect.Field field = clazz.getField(fieldName);
play.modules.search.Field index = field.getAnnotation(play.modules.search.Field.class);
View
20 src/play/modules/search/store/FileExtractor.java
@@ -6,7 +6,7 @@
import play.Logger;
import play.Play;
import play.classloading.ApplicationClasses.ApplicationClass;
-import play.db.jpa.FileAttachment;
+import play.db.jpa.Blob;
import play.modules.search.store.extractors.TextExtractor;
import play.modules.search.store.mime.ExtensionGuesser;
import play.modules.search.store.mime.MimeGuesser;
@@ -20,28 +20,34 @@
public static List<TextExtractor> extractors = new ArrayList<TextExtractor>();
public static MimeGuesser mimeGuesser = new ExtensionGuesser();
- static {
+
+ public static void init() {
+ Logger.debug("init FileExtractor");
List<ApplicationClass> classes = Play.classes.getAssignableClasses(TextExtractor.class);
+ List<TextExtractor> extractors = new ArrayList<TextExtractor>();
for (ApplicationClass applicationClass : classes) {
try {
+ Logger.trace("adding %s as a TextExtractor", applicationClass.name);
extractors.add((TextExtractor) applicationClass.javaClass.newInstance());
} catch (Exception e) {
Logger.warn(e,"Could not instanciate text extractor %s",applicationClass.javaClass.getName());
}
}
+ FileExtractor.extractors = extractors;
}
- public static String getText (FileAttachment file) {
+ public static String getText (Blob blob) {
// Guess mime
- String mime = mimeGuesser.guess (file);
+ String mime = mimeGuesser.guess (blob);
// Invoke the handlers
+ String fileName = blob.getFile().getName();
for (TextExtractor extractor : extractors) {
if (extractor.handles(mime)) {
- Logger.debug ("Using %s extractor to handle file %s, mime=%s",extractor.getClass().getName(),file.filename,mime);
- return extractor.extract(file);
+ Logger.debug ("Using %s extractor to handle blob %s, mime=%s", extractor.getClass().getName(), fileName, blob.type());
+ return extractor.extract(blob);
}
}
- Logger.warn("No handlers able to index %s mime type, file was %s",mime,file.filename);
+ Logger.warn("No handlers able to index %s mime type, file was %s", mime, fileName);
return null;
}
}
View
16 src/play/modules/search/store/FilesystemStore.java
@@ -13,6 +13,7 @@
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.Term;
+import org.apache.lucene.index.IndexWriter.MaxFieldLength;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.store.FSDirectory;
@@ -46,7 +47,7 @@ public void unIndex(Object object) {
String index = object.getClass().getName();
getIndexWriter(index).deleteDocuments(new Term("_docID", ConvertionUtils.getIdValueFor(jpaBase) + ""));
if (sync) {
- getIndexWriter(index).flush();
+ getIndexWriter(index).commit();
dirtyReader(index);
}
} catch (Exception e) {
@@ -67,11 +68,11 @@ public void index(Object object, String index) {
getIndexWriter(index).deleteDocuments(new Term("_docID", ConvertionUtils.getIdValueFor(jpaABase) + ""));
getIndexWriter(index).addDocument(document);
if (sync) {
- getIndexWriter(index).flush();
+ getIndexWriter(index).commit();
dirtyReader(index);
} else {
if (getIndexWriter(index).ramSizeInBytes() > 1024 * 1024 * 48) {
- getIndexWriter(index).flush();
+ getIndexWriter(index).commit();
dirtyReader(index);
}
}
@@ -87,7 +88,7 @@ public IndexSearcher getIndexSearcher(String name) {
File root = new File(DATA_PATH, name);
if (!root.exists())
getIndexWriter(name);
- IndexSearcher reader = new IndexSearcher(FSDirectory.getDirectory(root));
+ IndexSearcher reader = new IndexSearcher(FSDirectory.open(root));
indexSearchers.put(name, reader);
}
}
@@ -125,7 +126,7 @@ private IndexWriter getIndexWriter(String name) {
root.mkdirs();
if (new File(root, "write.lock").exists())
new File(root, "write.lock").delete();
- IndexWriter writer = new IndexWriter(FSDirectory.getDirectory(root), true, Search.getAnalyser());
+ IndexWriter writer = new IndexWriter(FSDirectory.open(root), Search.getAnalyser(), MaxFieldLength.UNLIMITED);
indexWriters.put(name, writer);
}
}
@@ -189,7 +190,8 @@ public void stop() throws Exception {
public void optimize(String name) {
try {
- getIndexWriter(name).optimize();
+ getIndexWriter(name).optimize(true);
+ getIndexWriter(name).commit();
dirtyReader(name);
} catch (Exception e) {
throw new UnexpectedException(e);
@@ -212,7 +214,7 @@ public void rebuild(String name) {
indexWriter.addDocument(document);
}
- getIndexWriter(index).flush();
+ getIndexWriter(index).commit();
dirtyReader(index);
getIndexSearcher(name).close();
indexSearchers.remove(name);
View
4 src/play/modules/search/store/extractors/TextExtractor.java
@@ -1,8 +1,8 @@
package play.modules.search.store.extractors;
-import play.db.jpa.FileAttachment;
+import play.db.jpa.Blob;
public interface TextExtractor {
public boolean handles (String mime);
- public String extract (FileAttachment file);
+ public String extract (Blob blob);
}
View
14 src/play/modules/search/store/mime/ExtensionGuesser.java
@@ -3,7 +3,7 @@
import java.util.HashMap;
import java.util.Map;
-import play.db.jpa.FileAttachment;
+import play.db.jpa.Blob;
public class ExtensionGuesser implements MimeGuesser {
public static Map<String, String> extensions = new HashMap<String, String> ();
@@ -11,12 +11,10 @@
extensions.put("pdf", "application/pdf");
//FIXME: complete
}
- public String guess (FileAttachment file) {
- if (file == null || file.filename == null) {
- return null;
- }
- String extension = file.filename.substring(file.filename.lastIndexOf('.')+1);
- return extensions.get(extension);
- }
+ public String guess (Blob blob) {
+ if(blob != null)
+ return blob.type();
+ return null;
+ }
}
View
4 src/play/modules/search/store/mime/MimeGuesser.java
@@ -1,7 +1,7 @@
package play.modules.search.store.mime;
-import play.db.jpa.FileAttachment;
+import play.db.jpa.Blob;
public interface MimeGuesser {
- public String guess (FileAttachment file);
+ public String guess (Blob blob);
}
Something went wrong with that request. Please try again.