Permalink
Browse files

Merge pull request #45 from wolfgangmm/develop

[feature] Merge new range index branch into develop
  • Loading branch information...
shabanovd committed Sep 18, 2013
2 parents b3f59fd + 14318d4 commit 8be8f34bf4488523bb60392fb2c4a6708a925ee9
Showing with 7,849 additions and 1,048 deletions.
  1. +1 −0 build/scripts/build-impl.xml
  2. +8 −1 conf.xml.tmpl
  3. +2 −1 eXist.iml
  4. +2 −0 eXist.ipr
  5. +2 −0 extensions/build.properties
  6. +4 −0 extensions/indexes/build.properties
  7. BIN extensions/indexes/lucene/lib/lucene-analyzers-3.6.1.jar
  8. BIN extensions/indexes/lucene/lib/lucene-analyzers-common-4.4.0.jar
  9. BIN extensions/indexes/lucene/lib/lucene-core-3.6.1.jar
  10. BIN extensions/indexes/lucene/lib/lucene-core-4.4.0.jar
  11. BIN extensions/indexes/lucene/lib/lucene-facet-4.4.0.jar
  12. BIN extensions/indexes/lucene/lib/lucene-queries-3.6.1.jar
  13. BIN extensions/indexes/lucene/lib/lucene-queries-4.4.0.jar
  14. BIN extensions/indexes/lucene/lib/lucene-queryparser-4.4.0.jar
  15. +67 −0 extensions/indexes/lucene/src/org/exist/indexing/lucene/BinaryTokenStream.java
  16. +60 −20 extensions/indexes/lucene/src/org/exist/indexing/lucene/LuceneIndex.java
  17. +1 −1 extensions/indexes/lucene/src/org/exist/indexing/lucene/LuceneIndexConfig.java
  18. +369 −554 extensions/indexes/lucene/src/org/exist/indexing/lucene/LuceneIndexWorker.java
  19. +3 −2 extensions/indexes/lucene/src/org/exist/indexing/lucene/LuceneMatchListener.java
  20. +76 −16 extensions/indexes/lucene/src/org/exist/indexing/lucene/LuceneUtil.java
  21. +0 −1 extensions/indexes/lucene/src/org/exist/indexing/lucene/MarkableTokenFilter.java
  22. +1 −0 extensions/indexes/lucene/src/org/exist/indexing/lucene/PlainTextHighlighter.java
  23. +165 −0 extensions/indexes/lucene/src/org/exist/indexing/lucene/QueryDocuments.java
  24. +254 −0 extensions/indexes/lucene/src/org/exist/indexing/lucene/QueryFacetCollector.java
  25. +222 −0 extensions/indexes/lucene/src/org/exist/indexing/lucene/QueryNodes.java
  26. +32 −0 extensions/indexes/lucene/src/org/exist/indexing/lucene/SearchCallback.java
  27. +57 −57 extensions/indexes/lucene/src/org/exist/indexing/lucene/XMLToQuery.java
  28. +19 −12 extensions/indexes/lucene/src/org/exist/indexing/lucene/analyzers/NoDiacriticsStandardAnalyzer.java
  29. +1 −1 extensions/indexes/lucene/src/org/exist/xquery/modules/lucene/GetField.java
  30. +2 −3 extensions/indexes/lucene/src/org/exist/xquery/modules/lucene/Query.java
  31. +5 −18 extensions/indexes/lucene/src/org/exist/xquery/modules/lucene/QueryField.java
  32. +4 −4 extensions/indexes/lucene/test/src/org/exist/indexing/lucene/LuceneIndexTest.java
  33. +6 −4 extensions/indexes/ngram/src/org/exist/indexing/ngram/NGramIndexWorker.java
  34. +11 −0 extensions/indexes/range/build.properties
  35. +47 −0 extensions/indexes/range/extension.xml
  36. +114 −0 extensions/indexes/range/src/org/exist/indexing/range/ComplexRangeIndexConfigElement.java
  37. +75 −0 extensions/indexes/range/src/org/exist/indexing/range/ComplexTextCollector.java
  38. +76 −0 extensions/indexes/range/src/org/exist/indexing/range/NodesFilter.java
  39. +90 −0 extensions/indexes/range/src/org/exist/indexing/range/RangeIndex.java
  40. +185 −0 extensions/indexes/range/src/org/exist/indexing/range/RangeIndexConfig.java
  41. +199 −0 extensions/indexes/range/src/org/exist/indexing/range/RangeIndexConfigElement.java
  42. +115 −0 extensions/indexes/range/src/org/exist/indexing/range/RangeIndexConfigField.java
  43. +63 −0 extensions/indexes/range/src/org/exist/indexing/range/RangeIndexDoc.java
  44. +971 −0 extensions/indexes/range/src/org/exist/indexing/range/RangeIndexWorker.java
  45. +61 −0 extensions/indexes/range/src/org/exist/indexing/range/SimpleTextCollector.java
  46. +71 −0 extensions/indexes/range/src/org/exist/indexing/range/TextCollector.java
  47. +335 −0 extensions/indexes/range/src/org/exist/xquery/modules/range/FieldLookup.java
  48. +62 −0 extensions/indexes/range/src/org/exist/xquery/modules/range/IndexKeys.java
  49. +336 −0 extensions/indexes/range/src/org/exist/xquery/modules/range/Lookup.java
  50. +61 −0 extensions/indexes/range/src/org/exist/xquery/modules/range/Optimize.java
  51. +95 −0 extensions/indexes/range/src/org/exist/xquery/modules/range/RangeIndexModule.java
  52. +308 −0 extensions/indexes/range/src/org/exist/xquery/modules/range/RangeQueryRewriter.java
  53. +177 −0 extensions/indexes/range/test/src/xquery/fields.xql
  54. +405 −0 extensions/indexes/range/test/src/xquery/optimizer.xql
  55. +392 −0 extensions/indexes/range/test/src/xquery/range.xql
  56. +12 −0 extensions/indexes/range/test/src/xquery/range/RunTests.java
  57. +8 −0 extensions/indexes/range/test/src/xquery/suite-fields.xql
  58. +8 −0 extensions/indexes/range/test/src/xquery/suite-optimizer.xql
  59. +8 −0 extensions/indexes/range/test/src/xquery/suite.xql
  60. +6 −0 extensions/indexes/sort/src/org/exist/indexing/sort/SortIndexWorker.java
  61. +0 −13 extensions/metadata/build.properties
  62. +26 −3 extensions/metadata/interface/src/main/java/org/exist/storage/md/MDStorageManager.java
  63. +4 −2 extensions/metadata/interface/src/main/java/org/exist/storage/md/xquery/Reindex.java
  64. +1 −1 extensions/metadata/interface/src/main/java/org/exist/storage/md/xquery/Search.java
  65. +1,525 −0 extensions/metadata/interface/src/test/java/org/exist/storage/md/LuceneSearchTest.java
  66. +19 −19 extensions/metadata/interface/src/test/java/org/exist/storage/md/SearchTest.java
  67. +3 −3 extensions/metadata/sleepycat/extension.xml
  68. +231 −226 extensions/metadata/sleepycat/src/main/java/org/exist/indexing/lucene/PlugToLucene.java
  69. +35 −15 extensions/metadata/sleepycat/src/main/java/org/exist/storage/md/MetaDataImpl.java
  70. +3 −0 src/org/exist/Database.java
  71. +18 −3 src/org/exist/collections/CollectionConfigurationManager.java
  72. +6 −0 src/org/exist/fulltext/FTIndexWorker.java
  73. +1 −1 src/org/exist/indexing/AbstractIndex.java
  74. +0 −4 src/org/exist/indexing/AbstractStreamListener.java
  75. +22 −0 src/org/exist/indexing/IndexController.java
  76. +10 −1 src/org/exist/indexing/IndexWorker.java
  77. +0 −9 src/org/exist/indexing/StreamListener.java
  78. +6 −0 src/org/exist/storage/BrokerPool.java
  79. +36 −0 src/org/exist/storage/MetaStorage.java
  80. +39 −0 src/org/exist/storage/MetaStreamListener.java
  81. +1 −1 src/org/exist/storage/NativeBroker.java
  82. +19 −3 src/org/exist/storage/NodePath.java
  83. 0 {extensions/metadata/interface/src/main/java → src}/org/exist/storage/md/Meta.java
  84. +5 −6 {extensions/metadata/interface/src/main/java → src}/org/exist/storage/md/MetaData.java
  85. 0 {extensions/metadata/interface/src/main/java → src}/org/exist/storage/md/Metas.java
  86. +6 −0 src/org/exist/storage/statistics/IndexStatisticsWorker.java
  87. +2 −2 src/org/exist/storage/structural/NativeStructuralIndex.java
  88. +5 −0 src/org/exist/storage/structural/NativeStructuralIndexWorker.java
  89. +26 −0 src/org/exist/util/XMLString.java
  90. +1 −1 src/org/exist/xquery/Constants.java
  91. +3 −0 src/org/exist/xquery/GeneralComparison.java
  92. +17 −1 src/org/exist/xquery/Optimizer.java
  93. +54 −0 src/org/exist/xquery/QueryRewriter.java
  94. +16 −0 test/src/org/exist/storage/NodePathTest.java
  95. +10 −0 test/src/org/exist/util/XMLStringTest.java
  96. +1 −1 test/src/org/exist/xmldb/concurrent/action/XQueryAction.java
  97. +44 −38 test/src/xquery/TestRunner.java
@@ -636,6 +636,7 @@
<delete dir="${src.webapp}/WEB-INF/data/fs"/>
<delete dir="${src.webapp}/WEB-INF/data/fs.journal"/>
<delete dir="${src.webapp}/WEB-INF/data/lucene"/>
+ <delete dir="${src.webapp}/WEB-INF/data/range"/>
<delete dir="${src.webapp}/WEB-INF/data/sanity"/>
<delete>
<fileset dir="${src.webapp}/WEB-INF/data" includes="*.dbx,*.log,*.lck,spatial_index.*,counters,restxq.registry"
View
@@ -415,6 +415,12 @@
-->
<module id="sort-index" class="org.exist.indexing.sort.SortIndex"/>
+ <!--
+ New range index based on Apache Lucene. Replaces the old range index which is
+ hard-wired into eXist core.
+ -->
+ <module id="range-index" class="org.exist.indexing.range.RangeIndex"/>
+
<!--
The full text index is always required and should
not be disabled. We still have some dependencies on
@@ -723,7 +729,8 @@
<module uri="http://exist-db.org/xquery/lucene" class="org.exist.xquery.modules.lucene.LuceneModule" />
<module uri="http://exist-db.org/xquery/ngram" class="org.exist.xquery.modules.ngram.NGramModule" />
<module uri="http://exist-db.org/xquery/sort" class="org.exist.xquery.modules.sort.SortModule" />
-
+ <module uri="http://exist-db.org/xquery/range" class="org.exist.xquery.modules.range.RangeIndexModule" />
+
<!--
<module uri="http://exist-db.org/xquery/spatial" class="org.exist.xquery.modules.spatial.SpatialModule" />
-->
View
@@ -19,6 +19,7 @@
<sourceFolder url="file://$MODULE_DIR$/tools/izpack/src" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/tools/requestlog/src" isTestSource="true" />
<sourceFolder url="file://$MODULE_DIR$/tools/wrapper/src" isTestSource="false" />
+ <sourceFolder url="file://$MODULE_DIR$/extensions/indexes/range/src" isTestSource="false" />
<excludeFolder url="file://$MODULE_DIR$/extensions/modules/src/org/exist/xquery/modules/cqlparser" />
<excludeFolder url="file://$MODULE_DIR$/extensions/modules/src/org/exist/xquery/modules/jfreechart" />
<excludeFolder url="file://$MODULE_DIR$/extensions/modules/src/org/exist/xquery/modules/memcached" />
@@ -31,7 +32,7 @@
<excludeFolder url="file://$MODULE_DIR$/webapp/WEB-INF/data" />
<excludeFolder url="file://$MODULE_DIR$/webapp/WEB-INF/logs" />
</content>
- <orderEntry type="jdk" jdkName="1.6" jdkType="JavaSDK" />
+ <orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
<orderEntry type="library" exported="" name="tools" level="project" />
<orderEntry type="library" exported="" name="core" level="project" />
View
@@ -366,12 +366,14 @@
<root url="file://$PROJECT_DIR$/extensions/indexes/lucene/lib" />
<root url="file://$PROJECT_DIR$/lib/extensions" />
<root url="file://$PROJECT_DIR$/extensions/xqdoc/lib" />
+ <root url="file://$PROJECT_DIR$/extensions/webdav/lib" />
</CLASSES>
<JAVADOC />
<SOURCES />
<jarDirectory url="file://$PROJECT_DIR$/extensions/fluent/lib" recursive="false" />
<jarDirectory url="file://$PROJECT_DIR$/extensions/indexes/lucene/lib" recursive="false" />
<jarDirectory url="file://$PROJECT_DIR$/extensions/indexes/spatial/lib" recursive="false" />
+ <jarDirectory url="file://$PROJECT_DIR$/extensions/webdav/lib" recursive="false" />
<jarDirectory url="file://$PROJECT_DIR$/extensions/xqdoc/lib" recursive="false" />
<jarDirectory url="file://$PROJECT_DIR$/lib/extensions" recursive="false" />
</library>
@@ -43,6 +43,8 @@ include.feature.fluent = true
# Metadata Storage
include.feature.metadata = false
+# sleepycat backend
+include.feature.metadata.sleepycat = false
# Extension for editing stored docs
# directly from browser in desktop application
@@ -17,3 +17,7 @@ include.index.spatial = false
# Sort index
include.index.sort = true
+
+# New Lucene-based range index
+# Requires include.index.lucene
+include.index.range = true
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,67 @@
+package org.exist.indexing.lucene;
+
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute;
+import org.apache.lucene.util.AttributeImpl;
+import org.apache.lucene.util.BytesRef;
+
+/**
+ * A binary tokenstream that lets you index a single
+ * binary token (BytesRef value).
+ *
+ */
+public final class BinaryTokenStream extends TokenStream {
+ private final ByteTermAttribute bytesAtt = addAttribute(ByteTermAttribute.class);
+ private boolean available = true;
+
+ public BinaryTokenStream(BytesRef bytes) {
+ bytesAtt.setBytesRef(bytes);
+ }
+
+ @Override
+ public boolean incrementToken() {
+ if (available) {
+ available = false;
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ public void reset() {
+ available = true;
+ }
+
+ public interface ByteTermAttribute extends TermToBytesRefAttribute {
+ public void setBytesRef(BytesRef bytes);
+ }
+
+ public static class ByteTermAttributeImpl extends AttributeImpl implements ByteTermAttribute,TermToBytesRefAttribute {
+ private BytesRef bytes;
+
+ @Override
+ public int fillBytesRef() {
+ return bytes.hashCode();
+ }
+
+ @Override
+ public BytesRef getBytesRef() {
+ return bytes;
+ }
+
+ @Override
+ public void setBytesRef(BytesRef bytes) {
+ this.bytes = bytes;
+ }
+
+ @Override
+ public void clear() {}
+
+ @Override
+ public void copyTo(AttributeImpl target) {
+ ByteTermAttributeImpl other = (ByteTermAttributeImpl) target;
+ other.bytes = bytes;
+ }
+ }
+}
+
@@ -3,8 +3,11 @@
import org.apache.log4j.Logger;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
-import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.facet.taxonomy.TaxonomyReader;
+import org.apache.lucene.facet.taxonomy.TaxonomyWriter;
+import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyReader;
+import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter;
+import org.apache.lucene.index.*;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
@@ -25,13 +28,11 @@
import java.io.InputStream;
import java.io.OutputStream;
-import org.apache.lucene.index.CorruptIndexException;
-import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.util.Version;
public class LuceneIndex extends AbstractIndex implements RawBackupSupport {
- public final static Version LUCENE_VERSION_IN_USE = Version.LUCENE_34;
+ public final static Version LUCENE_VERSION_IN_USE = Version.LUCENE_43;
private static final Logger LOG = Logger.getLogger(LuceneIndexWorker.class);
@@ -42,7 +43,7 @@
protected Directory directory;
protected Analyzer defaultAnalyzer;
- protected double bufferSize = IndexWriter.DEFAULT_RAM_BUFFER_SIZE_MB;
+ protected double bufferSize = IndexWriterConfig.DEFAULT_RAM_BUFFER_SIZE_MB;
protected IndexWriter cachedWriter = null;
protected int writerUseCount = 0;
@@ -52,13 +53,26 @@
protected int writingReaderUseCount = 0;
protected IndexSearcher cachedSearcher = null;
protected int searcherUseCount = 0;
-
+
protected boolean singleWriter = false;
+ //Taxonomy staff
+ protected Directory taxonomyDirectory;
+
+ protected TaxonomyWriter cachedTaxonomyWriter = null;
+ protected int taxonomyWriterUseCount = 0;
+
+ protected TaxonomyReader cachedTaxonomyReader = null;
+ protected int taxonomyReaderUseCount = 0;
+
public LuceneIndex() {
//Nothing special to do
}
+ public String getDirName() {
+ return DIR_NAME;
+ }
+
@Override
public void configure(BrokerPool pool, String dataDir, Element config) throws DatabaseConfigurationException {
super.configure(pool, dataDir, config);
@@ -90,7 +104,7 @@ public void configure(BrokerPool pool, String dataDir, Element config) throws Da
@Override
public void open() throws DatabaseConfigurationException {
- File dir = new File(getDataDir(), DIR_NAME);
+ File dir = new File(getDataDir(), getDirName());
if (LOG.isDebugEnabled())
LOG.debug("Opening Lucene index directory: " + dir.getAbsolutePath());
if (dir.exists()) {
@@ -102,7 +116,10 @@ public void open() throws DatabaseConfigurationException {
IndexWriter writer = null;
try {
directory = FSDirectory.open(dir);
+ taxonomyDirectory = FSDirectory.open(new File(dir, "taxonomy"));
+
writer = getWriter();
+
} catch (IOException e) {
throw new DatabaseConfigurationException("Exception while reading lucene index directory: " +
e.getMessage(), e);
@@ -116,9 +133,14 @@ public synchronized void close() throws DBException {
try {
if (cachedWriter != null) {
commit();
+
+ cachedTaxonomyWriter.close();
cachedWriter.close();
+
+ cachedTaxonomyWriter = null;
cachedWriter = null;
}
+ taxonomyDirectory.close();
directory.close();
} catch (IOException e) {
throw new DBException("Caught exception while closing lucene indexes: " + e.getMessage());
@@ -161,11 +183,11 @@ protected Analyzer getDefaultAnalyzer() {
protected boolean needsCommit = false;
- protected IndexWriter getWriter() throws IOException {
+ public IndexWriter getWriter() throws IOException {
return getWriter(false);
}
- protected synchronized IndexWriter getWriter(boolean exclusive) throws IOException {
+ public synchronized IndexWriter getWriter(boolean exclusive) throws IOException {
while (writingReaderUseCount > 0) {
try {
wait();
@@ -194,13 +216,14 @@ protected synchronized IndexWriter getWriter(boolean exclusive) throws IOExcepti
now we have to commit ourselves, this is done manually in releaseWriter()
*/
cachedWriter = new IndexWriter(directory, idxWriterConfig);
+ cachedTaxonomyWriter = new DirectoryTaxonomyWriter(taxonomyDirectory);
writerUseCount = 1;
}
notifyAll();
return cachedWriter;
}
- protected synchronized void releaseWriter(IndexWriter writer) {
+ public synchronized void releaseWriter(IndexWriter writer) {
if (writer == null)
return;
if (writer != cachedWriter)
@@ -223,6 +246,7 @@ protected void commit() {
}
if (cachedWriter != null) {
+ cachedTaxonomyWriter.commit();
cachedWriter.commit();
}
needsCommit = false;
@@ -233,18 +257,19 @@ protected void commit() {
}
}
- protected synchronized IndexReader getReader() throws IOException {
+ public synchronized IndexReader getReader() throws IOException {
commit();
if (cachedReader != null) {
readerUseCount++;
} else {
- cachedReader = IndexReader.open(directory);
+ cachedReader = DirectoryReader.open(directory);
+ cachedTaxonomyReader = new DirectoryTaxonomyReader(taxonomyDirectory);
readerUseCount = 1;
}
return cachedReader;
}
- protected synchronized void releaseReader(IndexReader reader) {
+ public synchronized void releaseReader(IndexReader reader) {
if (reader == null)
return;
if (reader != cachedReader)
@@ -273,7 +298,7 @@ protected synchronized IndexReader getWritingReader() throws IOException {
if (cachedWritingReader != null) {
writingReaderUseCount++;
} else {
- cachedWritingReader = IndexReader.open(directory, false);
+ cachedWritingReader = DirectoryReader.open(directory);
writingReaderUseCount = 1;
}
notifyAll();
@@ -315,17 +340,22 @@ private void reopenReaders() {
if (cachedReader == null)
return;
try {
+ cachedTaxonomyReader.close();
cachedReader.close();
+
+ cachedTaxonomyReader = null;
cachedReader = null;
- if (cachedSearcher != null)
- cachedSearcher.close();
+
+ //XXX: understand - is it right to comment close out? ... Closed by "cachedReader.close();"?
+// if (cachedSearcher != null)
+// cachedSearcher.close();
cachedSearcher = null;
} catch (IOException e) {
LOG.warn("Exception while refreshing lucene index: " + e.getMessage(), e);
}
}
- protected synchronized IndexSearcher getSearcher() throws IOException {
+ public synchronized IndexSearcher getSearcher() throws IOException {
commit();
if (cachedSearcher != null) {
searcherUseCount++;
@@ -337,19 +367,29 @@ protected synchronized IndexSearcher getSearcher() throws IOException {
return cachedSearcher;
}
- protected synchronized void releaseSearcher(IndexSearcher searcher) {
+ public synchronized void releaseSearcher(IndexSearcher searcher) {
if (searcher == null)
return;
if (searcher != cachedSearcher)
throw new IllegalStateException("IndexSearcher was not obtained from getWritingReader().");
searcherUseCount--;
notifyAll();
}
+
+ //DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir);
+
+ public synchronized TaxonomyWriter getTaxonomyWriter() throws IOException {
+ return cachedTaxonomyWriter;
+ }
+
+ public synchronized TaxonomyReader getTaxonomyReader() throws IOException {
+ return cachedTaxonomyReader;
+ }
@Override
public void backupToArchive(RawDataBackup backup) throws IOException {
for (String name : directory.listAll()) {
- String path = DIR_NAME + "/" + name;
+ String path = getDirName() + "/" + name;
OutputStream os = backup.newEntry(path);
InputStream is = new FileInputStream(new File(getDataDir(), path));
byte[] buf = new byte[4096];
@@ -145,7 +145,7 @@ public boolean isInlineNode(QName qname) {
return specialNodes != null && specialNodes.get(qname) == N_INLINE;
}
- protected static QName parseQName(Element config, Map<String, String> namespaces) throws DatabaseConfigurationException {
+ public static QName parseQName(Element config, Map<String, String> namespaces) throws DatabaseConfigurationException {
String name = config.getAttribute(QNAME_ATTR);
if (name == null || name.length() == 0)
throw new DatabaseConfigurationException("Lucene index configuration error: element " + config.getNodeName() +
Oops, something went wrong.

0 comments on commit 8be8f34

Please sign in to comment.