Fulltext addon fuzzy query support

neo4j · Sep 12, 2017 · 63e46a3 · 63e46a3
1 parent dbd5189
commit 63e46a3
Show file tree

Hide file tree

Showing 9 changed files with 152 additions and 57 deletions.
diff --git a/...rise/fulltext-addon/src/main/java/org/neo4j/kernel/api/impl/fulltext/FulltextFactory.java b/...rise/fulltext-addon/src/main/java/org/neo4j/kernel/api/impl/fulltext/FulltextFactory.java
@@ -46,7 +46,7 @@ public LuceneFulltext createFulltextHelper( String identifier, FULLTEXT_HELPER_T
     {
         LuceneIndexStorageBuilder storageBuilder = LuceneIndexStorageBuilder.create();
         storageBuilder.withFileSystem( fileSystem ).withIndexIdentifier( identifier ).withDirectoryFactory(
-                directoryFactory( false, this.fileSystem ) ).withIndexRootFolder( Paths.get( this.storeDir.getAbsolutePath(), "fulltextHelper" ).toFile() );
+                directoryFactory( false, this.fileSystem ) ).withIndexRootFolder( Paths.get( this.storeDir.getAbsolutePath(), "fulltext" ).toFile() );
         return new LuceneFulltext( storageBuilder.build(), partitionFactory, properties, analyzer, identifier, type );
     }
 

diff --git a/...ise/fulltext-addon/src/main/java/org/neo4j/kernel/api/impl/fulltext/FulltextProvider.java b/...ise/fulltext-addon/src/main/java/org/neo4j/kernel/api/impl/fulltext/FulltextProvider.java
@@ -127,7 +127,7 @@ public Set<WritableFulltext> relationshipIndices()
         return relationshipIndices.values().stream().map( WritableFulltext::new ).collect( Collectors.toSet() );
     }
 
-    public FulltextReader getReader( String identifier, FulltextFactory.FULLTEXT_HELPER_TYPE type ) throws IOException
+    public ReadOnlyFulltext getReader( String identifier, FulltextFactory.FULLTEXT_HELPER_TYPE type ) throws IOException
     {
         if ( type == FulltextFactory.FULLTEXT_HELPER_TYPE.NODES )
         {

diff --git a/...prise/fulltext-addon/src/main/java/org/neo4j/kernel/api/impl/fulltext/LuceneFulltext.java b/...prise/fulltext-addon/src/main/java/org/neo4j/kernel/api/impl/fulltext/LuceneFulltext.java
@@ -97,7 +97,7 @@ public PartitionedIndexWriter getIndexWriter( WritableFulltext writableIndex ) t
         return new PartitionedIndexWriter( writableIndex );
     }
 
-    public FulltextReader getIndexReader() throws IOException
+    public ReadOnlyFulltext getIndexReader() throws IOException
     {
         ensureOpen();
         List<AbstractIndexPartition> partitions = getPartitions();

diff --git a/...ext-addon/src/main/java/org/neo4j/kernel/api/impl/fulltext/PartitionedFulltextReader.java b/...ext-addon/src/main/java/org/neo4j/kernel/api/impl/fulltext/PartitionedFulltextReader.java
@@ -38,32 +38,44 @@
  *
  * @see SimpleFulltextReader
  */
-class PartitionedFulltextReader implements FulltextReader
+class PartitionedFulltextReader implements ReadOnlyFulltext
 {
 
-    private final List<FulltextReader> indexReaders;
+    private final List<ReadOnlyFulltext> indexReaders;
 
     PartitionedFulltextReader( List<PartitionSearcher> partitionSearchers, String[] properties, Analyzer analyzer )
     {
-        this( partitionSearchers.stream().map( partitionSearcher -> new SimpleFulltextReader( partitionSearcher, properties, analyzer ) )
+        this( partitionSearchers.stream().map( partitionSearcher -> new SimpleFulltextReader( partitionSearcher, properties, analyzer ) ).collect(
-                .collect( Collectors.toList() ) );
+                Collectors.toList() ) );
     }
 
-    private PartitionedFulltextReader( List<FulltextReader> readers )
+    private PartitionedFulltextReader( List<ReadOnlyFulltext> readers )
     {
         this.indexReaders = readers;
     }
 
+    @Override
     public PrimitiveLongIterator query( String... query )
     {
         return partitionedOperation( reader -> innerQuery( reader, query ) );
     }
 
-    private PrimitiveLongIterator innerQuery( FulltextReader reader, String... query )
+    @Override
+    public PrimitiveLongIterator fuzzyQuery( String... query )
+    {
+        return partitionedOperation( reader -> innerFuzzyQuery( reader, query ) );
+    }
+
+    private PrimitiveLongIterator innerQuery( ReadOnlyFulltext reader, String... query )
     {
 
         return reader.query( query );
+    }
+
+    private PrimitiveLongIterator innerFuzzyQuery( ReadOnlyFulltext reader, String... query )
+    {
 
+        return reader.fuzzyQuery( query );
     }
 
     public void close()
@@ -78,10 +90,8 @@ public void close()
         }
     }
 
-    private PrimitiveLongIterator partitionedOperation(
+    private PrimitiveLongIterator partitionedOperation( Function<ReadOnlyFulltext,PrimitiveLongIterator> readerFunction )
-            Function<FulltextReader,PrimitiveLongIterator> readerFunction )
     {
-        return PrimitiveLongCollections
+        return PrimitiveLongCollections.concat( indexReaders.parallelStream().map( readerFunction::apply ).collect( Collectors.toList() ) );
-                .concat( indexReaders.parallelStream().map( readerFunction::apply ).collect( Collectors.toList() ) );
     }
 }
diff --git a/...nel/api/impl/fulltext/FulltextReader.java → ...l/api/impl/fulltext/ReadOnlyFulltext.java b/...nel/api/impl/fulltext/FulltextReader.java → ...l/api/impl/fulltext/ReadOnlyFulltext.java
@@ -21,7 +21,8 @@
 
 import org.neo4j.collection.primitive.PrimitiveLongIterator;
 
-public interface FulltextReader extends AutoCloseable
+public interface ReadOnlyFulltext extends AutoCloseable
 {
     PrimitiveLongIterator query( String... query );
+    PrimitiveLongIterator fuzzyQuery( String... query );
 }
diff --git a/...fulltext-addon/src/main/java/org/neo4j/kernel/api/impl/fulltext/SimpleFulltextReader.java b/...fulltext-addon/src/main/java/org/neo4j/kernel/api/impl/fulltext/SimpleFulltextReader.java
@@ -21,6 +21,7 @@
 
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.queryparser.classic.MultiFieldQueryParser;
+import org.apache.lucene.queryparser.classic.ParseException;
 import org.apache.lucene.queryparser.classic.QueryParser;
 import org.apache.lucene.search.BooleanClause;
 import org.apache.lucene.search.BooleanQuery;
@@ -42,7 +43,7 @@
  *
  * @see PartitionedFulltextReader
  */
-class SimpleFulltextReader implements FulltextReader
+class SimpleFulltextReader implements ReadOnlyFulltext
 {
     private final PartitionSearcher partitionSearcher;
     private final Analyzer analyzer;
@@ -55,12 +56,28 @@ class SimpleFulltextReader implements FulltextReader
         this.analyzer = analyzer;
     }
 
-    public PrimitiveLongIterator query( String... query )
+    @Override
+    public PrimitiveLongIterator query( String... tokens )
     {
-        BooleanQuery.Builder builder = new BooleanQuery.Builder();
+        MultiFieldQueryParser multiFieldQueryParser = new MultiFieldQueryParser( properties, analyzer );
-        QueryParser multiFieldQueryParser = new MultiFieldQueryParser( properties, analyzer );
         multiFieldQueryParser.setDefaultOperator( QueryParser.Operator.OR );
-        for ( String s : query )
+        Query query;
+        try
+        {
+            query = multiFieldQueryParser.parse( String.join( " ", tokens ) );
+        }
+        catch ( ParseException e )
+        {
+            query = parseFallbackBooleanQuery( multiFieldQueryParser, tokens );
+        }
+        return query( query );
+    }
+
+    private Query parseFallbackBooleanQuery( MultiFieldQueryParser multiFieldQueryParser, String[] tokens )
+    {
+        Query query;
+        BooleanQuery.Builder builder = new BooleanQuery.Builder();
+        for ( String s : tokens )
         {
             for ( String property : properties )
             {
@@ -71,9 +88,28 @@ public PrimitiveLongIterator query( String... query )
                 }
             }
         }
-        return query( builder.build() );
+        query = builder.build();
+        return query;
+    }
+
+    @Override
+    public PrimitiveLongIterator fuzzyQuery( String... tokens )
+    {
+        MultiFieldQueryParser multiFieldQueryParser = new MultiFieldQueryParser( properties, analyzer );
+        multiFieldQueryParser.setDefaultOperator( QueryParser.Operator.OR );
+        Query query;
+        try
+        {
+            query = multiFieldQueryParser.parse(  String.join( "~ ", tokens ) + "~" );
+        }
+        catch ( ParseException e )
+        {
+            query = parseFallbackBooleanQuery( multiFieldQueryParser, tokens );
+        }
+        return query( query );
     }
 
+    @Override
     public void close()
     {
         try
@@ -86,7 +122,7 @@ public void close()
         }
     }
 
-    protected PrimitiveLongIterator query( Query query )
+    private PrimitiveLongIterator query( Query query )
     {
         try
         {

diff --git a/...n/src/main/java/org/neo4j/kernel/api/impl/fulltext/integrations/bloom/BloomProcedure.java b/...n/src/main/java/org/neo4j/kernel/api/impl/fulltext/integrations/bloom/BloomProcedure.java
@@ -19,13 +19,14 @@
  */
 package org.neo4j.kernel.api.impl.fulltext.integrations.bloom;
 
+import java.util.ArrayList;
 import java.util.Arrays;
 
 import org.neo4j.collection.RawIterator;
 import org.neo4j.collection.primitive.PrimitiveLongIterator;
 import org.neo4j.kernel.api.exceptions.ProcedureException;
 import org.neo4j.kernel.api.exceptions.Status;
-import org.neo4j.kernel.api.impl.fulltext.FulltextReader;
+import org.neo4j.kernel.api.impl.fulltext.ReadOnlyFulltext;
 import org.neo4j.kernel.api.impl.fulltext.LuceneFulltext;
 import org.neo4j.kernel.api.proc.CallableProcedure;
 import org.neo4j.kernel.api.proc.Context;
@@ -36,9 +37,9 @@
 
 public class BloomProcedure extends CallableProcedure.BasicProcedure
 {
+    public static final String OUTPUT_NAME = "entityid";
     private static final String PROCEDURE_NAME = "bloomFulltext";
     private static final String[] PROCEDURE_NAMESPACE = {"db", "fulltext"};
-    public static final String OUTPUT_NAME = "entityid";
     private final LuceneFulltext luceneFulltext;
     private String type;
 
@@ -54,10 +55,10 @@ public class BloomProcedure extends CallableProcedure.BasicProcedure
     @Override
     public RawIterator<Object[],ProcedureException> apply( Context ctx, Object[] input ) throws ProcedureException
     {
-        String[] query = Arrays.stream( input ).map( Object::toString ).toArray( String[]::new );
+        String[] query = ((ArrayList<String>) input[0]).toArray( new String[0] );
-        try ( FulltextReader indexReader = luceneFulltext.getIndexReader() )
+        try ( ReadOnlyFulltext indexReader = luceneFulltext.getIndexReader() )
         {
-            PrimitiveLongIterator primitiveLongIterator = indexReader.query( query );
+            PrimitiveLongIterator primitiveLongIterator = indexReader.fuzzyQuery( query );
             return new RawIterator<Object[],ProcedureException>()
             {
                 @Override

diff --git a/...fulltext-addon/src/test/java/org/neo4j/kernel/api/impl/fulltext/FulltextAnalyzerTest.java b/...fulltext-addon/src/test/java/org/neo4j/kernel/api/impl/fulltext/FulltextAnalyzerTest.java
@@ -72,7 +72,7 @@ public void shouldBeAbleToSpecifyEnglishAnalyzer() throws Exception
                 tx.success();
             }
 
-            try ( FulltextReader reader = provider.getReader( "bloomNodes", FULLTEXT_HELPER_TYPE.NODES ) )
+            try ( ReadOnlyFulltext reader = provider.getReader( "bloomNodes", FULLTEXT_HELPER_TYPE.NODES ) )
             {
 
                 assertFalse( reader.query( "and" ).hasNext() );
@@ -108,7 +108,7 @@ public void shouldBeAbleToSpecifySwedishAnalyzer() throws Exception
                 tx.success();
             }
 
-            try ( FulltextReader reader = provider.getReader( "bloomNodes", FULLTEXT_HELPER_TYPE.NODES ) )
+            try ( ReadOnlyFulltext reader = provider.getReader( "bloomNodes", FULLTEXT_HELPER_TYPE.NODES ) )
             {
                 assertEquals( firstID, reader.query( "and" ).next() );
                 assertEquals( firstID, reader.query( "in" ).next() );