V3.0.1 (#139)

* bump version to v3.0.1 * a little test to make sure things work with materialized views * make sure all ubuntu-based Dockerfiles for the build system first update themselves (and --fix-missing). Boggles my mind why this used to work and now doesn't. * fix terrible bug where ANDed terms + phrases against the same field would be rewritten as ORs! * code cleanup * doc update * organize query_parser classes into subpackages and cleanup method/member visibility along the way. * get everything compiling across all distros and pg versions * fixing issue #129: support for Debian Jessie as build artifacts and in top-level Dockerfile * resolving issue #133: support the 'citext' contrib datatype and treat it just as we do the 'text' datatype. * docs update * fixing issue #132 * implementing issue #134: ZDB now supports querying multi-fields that might be configured via zdb_define_mapping(). * remove unused file * fixing merge issues * fixup tests list * doc fixes
zombodb · Aug 7, 2016 · bfca10a · bfca10a
1 parent 61783e0
commit bfca10a
Show file tree

Hide file tree

Showing 70 changed files with 880 additions and 417 deletions.
diff --git a/.gitignore b/.gitignore
@@ -11,7 +11,7 @@
 target/
 .DS_store
 foo.*
-postgres/src/main/sql/zombodb--3.0.0.sql
+postgres/src/main/sql/zombodb--3.0.1.sql
 postgres/src/test/sql/setup.sql
 postgres/src/test/sql/so_comments.sql
 postgres/src/test/sql/TUTORIAL.sql

diff --git a/LICENSE.md b/LICENSE.md
@@ -1,4 +1,4 @@
-Portions Copyright 2013-2015 Technology Concepts & Design, Inc.
+Portions Copyright 2013-2015 Technology Concepts & Design, Inc.  
 Portions Copyright 2015-2016 ZomboDB, LLC
 
 Licensed under the Apache License, Version 2.0 (the "License");

diff --git a/docker/elasticsearch/Dockerfile b/docker/elasticsearch/Dockerfile
@@ -9,7 +9,7 @@
 FROM java:7
 
 ENV ES_PKG_NAME elasticsearch-1.7.5
-ENV ZOMBODB_VER 3.0.0
+ENV ZOMBODB_VER 3.0.1
 
 # Install Elasticsearch.
 RUN \

diff --git a/docker/postgres/Dockerfile b/docker/postgres/Dockerfile
@@ -4,7 +4,7 @@
 
 # Pull base image.
 FROM postgres:9.5
-ENV ZOMBODB_VER 3.0.0
+ENV ZOMBODB_VER 3.0.1
 
 # Fetch wget
 RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates wget && rm -rf /var/lib/apt/lists/*
@@ -16,11 +16,11 @@ RUN \
   cd / && \
   mkdir zombodb && \
   cd zombodb && \
-  wget https://github.com/zombodb/zombodb/releases/download/v${ZOMBODB_VER}/zombodb_trusty_pg95-${ZOMBODB_VER}_amd64.deb
+  wget https://github.com/zombodb/zombodb/releases/download/v${ZOMBODB_VER}/zombodb_jessie_pg95-${ZOMBODB_VER}_amd64.deb
 
 RUN \
   cd /zombodb && \
-  dpkg -i zombodb_trusty_pg95-${ZOMBODB_VER}_amd64.deb
+  dpkg -i zombodb_jessie_pg95-${ZOMBODB_VER}_amd64.deb
 
 COPY zombodb-entrypoint.sh /
 

diff --git a/elasticsearch/pom.xml b/elasticsearch/pom.xml
@@ -7,7 +7,7 @@
     <parent>
         <groupId>com.tcdi.elasticsearch</groupId>
         <artifactId>zombodb-parent</artifactId>
-        <version>3.0.0</version>
+        <version>3.0.1</version>
     </parent>
 
     <artifactId>zombodb-plugin</artifactId>

diff --git a/elasticsearch/src/main/java/com/tcdi/zombodb/highlight/AnalyzedField.java b/elasticsearch/src/main/java/com/tcdi/zombodb/highlight/AnalyzedField.java
@@ -17,6 +17,8 @@
 package com.tcdi.zombodb.highlight;
 
 import com.tcdi.zombodb.query_parser.*;
+import com.tcdi.zombodb.query_parser.metadata.IndexMetadataManager;
+import com.tcdi.zombodb.query_parser.utils.Utils;
 import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequest;
 import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequestBuilder;
 import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse;
@@ -205,16 +207,16 @@ public String getDescription() {
                 }
             } else {
                 if (value.contains("*") || value.contains("?") || value.contains("*")) {
-                    toKeep = Utils.convertToProximityForHighlighting(phrase);
+                    toKeep = Utils.convertToProximityForHighlighting(metadataManager, phrase);
                 } else {
                     try {
                         if (client == null)
                             return;
                         AnalyzeResponse response = analyzePhrase(value);
-                        toKeep = Utils.convertToProximityForHighlighting(phrase.getFieldname(), response.getTokens());
+                        toKeep = Utils.convertToProximityForHighlighting(metadataManager, phrase.getFieldname(), response.getTokens());
                     } catch (Exception e2) {
                         try {
-                            toKeep = Utils.convertToProximityForHighlighting(phrase);
+                            toKeep = Utils.convertToProximityForHighlighting(metadataManager, phrase);
                         } catch (Exception e) {
                             throw new RuntimeException(e);
                         }
@@ -367,6 +369,7 @@ public List<Token> reduce() {
     }
 
     private final Client client;
+    private final IndexMetadataManager metadataManager;
     private final String indexName;
     private final Object primaryKey;
     private final String fieldName;
@@ -383,8 +386,9 @@ public List<Token> get(Object key) {
     };
     private Proxy proxy = null;
 
-    public AnalyzedField(Client client, String indexName, Object primaryKey, String fieldName, int arrayIndex, AnalyzeResponse analysis) {
+    public AnalyzedField(Client client, IndexMetadataManager metadataManager, String indexName, Object primaryKey, String fieldName, int arrayIndex, AnalyzeResponse analysis) {
         this.client = client;
+        this.metadataManager = metadataManager;
         this.indexName = indexName;
         this.primaryKey = primaryKey;
         this.fieldName = fieldName;
@@ -525,7 +529,7 @@ private List<Token> match(ASTPhrase phrase) {
         if (phrase.getOperator() == QueryParserNode.Operator.REGEX)
             return matchRegex(phrase);
 
-        QueryParserNode node = Utils.convertToProximityForHighlighting(phrase);
+        QueryParserNode node = Utils.convertToProximityForHighlighting(metadataManager, phrase);
         if (node instanceof ASTProximity) {
             ASTProximity prox = (ASTProximity) node;
             List<ProximityGroup> scratch = new ArrayList<>();

diff --git a/elasticsearch/src/main/java/com/tcdi/zombodb/highlight/DocumentHighlighter.java b/elasticsearch/src/main/java/com/tcdi/zombodb/highlight/DocumentHighlighter.java
@@ -17,6 +17,8 @@
 package com.tcdi.zombodb.highlight;
 
 import com.tcdi.zombodb.query_parser.*;
+import com.tcdi.zombodb.query_parser.metadata.IndexMetadataManager;
+import com.tcdi.zombodb.query_parser.utils.Utils;
 import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequestBuilder;
 import org.elasticsearch.client.Client;
 
@@ -27,6 +29,7 @@
 public class DocumentHighlighter {
 
     private final Client client;
+    private final IndexMetadataManager metadataManager;
     private final ASTQueryTree query;
     private final Map<String, List<AnalyzedField>> analyzedFields = new HashMap<String, List<AnalyzedField>>() {
         @Override
@@ -44,7 +47,8 @@ public DocumentHighlighter(Client client, String indexName, String primaryKeyFie
         QueryParser parser = new QueryParser(new StringReader(newQuery.toString().toLowerCase()));
 
         this.client = client;
-        this.query = parser.parse(true);
+        this.metadataManager = new IndexMetadataManager(client, indexName);
+        this.query = parser.parse(metadataManager, true);
 
         analyzeFields(parser, indexName, primaryKeyFieldname, documentData);
     }
@@ -154,7 +158,7 @@ private void analyzeData(String indexName, String primaryKeyFieldname, Set<Strin
                             rb.setAnalyzer("phrase");
 
                             try {
-                                fields.add(new AnalyzedField(client, indexName, baseDocumentData.get(primaryKeyFieldname), baseFn == null ? fieldName : baseFn + "." + fieldName, idx++, client.admin().indices().analyze(rb.request()).get()));
+                                fields.add(new AnalyzedField(client, metadataManager, indexName, baseDocumentData.get(primaryKeyFieldname), baseFn == null ? fieldName : baseFn + "." + fieldName, idx++, client.admin().indices().analyze(rb.request()).get()));
                             } catch (InterruptedException | ExecutionException e) {
                                 // ignore
                             }
@@ -165,7 +169,7 @@ private void analyzeData(String indexName, String primaryKeyFieldname, Set<Strin
                     rb.setAnalyzer("phrase");
 
                     try {
-                        fields.add(new AnalyzedField(client, indexName, baseDocumentData.get(primaryKeyFieldname), baseFn == null ? fieldName : baseFn + "." + fieldName, idx, client.admin().indices().analyze(rb.request()).get()));
+                        fields.add(new AnalyzedField(client, metadataManager, indexName, baseDocumentData.get(primaryKeyFieldname), baseFn == null ? fieldName : baseFn + "." + fieldName, idx, client.admin().indices().analyze(rb.request()).get()));
                     } catch (InterruptedException | ExecutionException e) {
                         // ignore
                     }

diff --git a/elasticsearch/src/main/java/com/tcdi/zombodb/postgres/PostgresAggregationAction.java b/elasticsearch/src/main/java/com/tcdi/zombodb/postgres/PostgresAggregationAction.java
@@ -16,7 +16,7 @@
  */
 package com.tcdi.zombodb.postgres;
 
-import com.tcdi.zombodb.query_parser.QueryRewriter;
+import com.tcdi.zombodb.query_parser.rewriters.QueryRewriter;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;

diff --git a/elasticsearch/src/main/java/com/tcdi/zombodb/postgres/PostgresMappingAction.java b/elasticsearch/src/main/java/com/tcdi/zombodb/postgres/PostgresMappingAction.java
@@ -16,7 +16,7 @@
  */
 package com.tcdi.zombodb.postgres;
 
-import com.tcdi.zombodb.query_parser.QueryRewriter;
+import com.tcdi.zombodb.query_parser.rewriters.QueryRewriter;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;

diff --git a/elasticsearch/src/main/java/com/tcdi/zombodb/postgres/PostgresTIDResponseAction.java b/elasticsearch/src/main/java/com/tcdi/zombodb/postgres/PostgresTIDResponseAction.java
@@ -16,7 +16,8 @@
  */
 package com.tcdi.zombodb.postgres;
 
-import com.tcdi.zombodb.query_parser.QueryRewriter;
+import com.tcdi.zombodb.query_parser.rewriters.QueryRewriter;
+import com.tcdi.zombodb.query_parser.utils.Utils;
 import org.elasticsearch.action.ActionFuture;
 import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;

diff --git a/elasticsearch/src/main/java/com/tcdi/zombodb/postgres/ZombodbMultiSearchAction.java b/elasticsearch/src/main/java/com/tcdi/zombodb/postgres/ZombodbMultiSearchAction.java
@@ -16,7 +16,7 @@
 package com.tcdi.zombodb.postgres;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
-import com.tcdi.zombodb.query_parser.QueryRewriter;
+import com.tcdi.zombodb.query_parser.rewriters.QueryRewriter;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.search.MultiSearchRequestBuilder;
 import org.elasticsearch.action.search.MultiSearchResponse;

diff --git a/elasticsearch/src/main/java/com/tcdi/zombodb/postgres/ZombodbQueryAction.java b/elasticsearch/src/main/java/com/tcdi/zombodb/postgres/ZombodbQueryAction.java
@@ -15,7 +15,7 @@
  */
 package com.tcdi.zombodb.postgres;
 
-import com.tcdi.zombodb.query_parser.QueryRewriter;
+import com.tcdi.zombodb.query_parser.rewriters.QueryRewriter;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.common.inject.Inject;

diff --git a/elasticsearch/src/main/java/com/tcdi/zombodb/query_parser/QueryParserNode.java b/elasticsearch/src/main/java/com/tcdi/zombodb/query_parser/QueryParserNode.java
@@ -16,6 +16,10 @@
  */
 package com.tcdi.zombodb.query_parser;
 
+import com.tcdi.zombodb.query_parser.metadata.IndexMetadata;
+import com.tcdi.zombodb.query_parser.metadata.IndexMetadataManager;
+import com.tcdi.zombodb.query_parser.utils.Utils;
+
 import java.util.*;
 
 public class QueryParserNode extends SimpleNode implements Iterable<QueryParserNode>, Cloneable {
@@ -85,14 +89,26 @@ public float getBoost() {
         return boost;
     }
 
+    public void setBoost(float boost) {
+        this.boost = boost;
+    }
+
     public int getDistance() {
         return distance;
     }
 
+    public void setDistance(int distance) {
+        this.distance = distance;
+    }
+
     public boolean isOrdered() {
         return ordered;
     }
 
+    public void setOrdered(boolean ordered) {
+        this.ordered = ordered;
+    }
+
     public Object getValue() {
         Object value = jjtGetValue();
         return value instanceof String ? Utils.unescape((String) value) : value;
@@ -125,7 +141,16 @@ public int getFuzzyness() {
         return fuzzyness;
     }
 
-    public boolean isNested() {
+    public void setFuzzyness(int fuzzyness) {
+        this.fuzzyness = fuzzyness;
+    }
+
+    public boolean isNested(IndexMetadataManager metadataManager) {
+        if (fieldname != null && fieldname.contains(".")) {
+            IndexMetadata md = metadataManager.getMetadataForField(fieldname);
+            if (md != null && md.isMultiField(fieldname))
+                return false;
+        }
         return fieldname != null && fieldname.contains(".");
     }
 
@@ -202,20 +227,20 @@ public void forceFieldname(String fieldname) {
             child.forceFieldname(fieldname);
     }
 
-    protected void adoptChildren(QueryParserNode node) {
+    public void adoptChildren(QueryParserNode node) {
         for (QueryParserNode child : node) {
             jjtAddChild(child, jjtGetNumChildren());
             child.jjtSetParent(this);
         }
     }
 
-    protected void removeNode(int idx) {
+    public void removeNode(int idx) {
         if (children == null)
             return;
         children.remove(idx);
     }
 
-    protected void removeNode(QueryParserNode node) {
+    public void removeNode(QueryParserNode node) {
         if (children == null)
             return;
 
@@ -257,10 +282,7 @@ public int size() {
         };
     }
 
-    public Collection<QueryParserNode> getChildren() {
-        List<QueryParserNode> children = new ArrayList<>();
-        for (QueryParserNode node : this)
-            children.add(node);
+    public Map<Number, Node> getChildren() {
         return children;
     }
 

diff --git a/...m/tcdi/zombodb/query_parser/Dijkstra.java → ...mbodb/query_parser/metadata/Dijkstra.java b/...m/tcdi/zombodb/query_parser/Dijkstra.java → ...mbodb/query_parser/metadata/Dijkstra.java
@@ -13,7 +13,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package com.tcdi.zombodb.query_parser;
+package com.tcdi.zombodb.query_parser.metadata;
 
 import java.util.*;
 

diff --git a/...mbodb/query_parser/FieldAndIndexPair.java → ...ry_parser/metadata/FieldAndIndexPair.java b/...mbodb/query_parser/FieldAndIndexPair.java → ...ry_parser/metadata/FieldAndIndexPair.java
@@ -13,12 +13,14 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package com.tcdi.zombodb.query_parser;
+package com.tcdi.zombodb.query_parser.metadata;
 
-class FieldAndIndexPair {
-    ASTIndexLink link;
-    String fieldname;
-    IndexMetadata md;
+import com.tcdi.zombodb.query_parser.ASTIndexLink;
+
+public class FieldAndIndexPair {
+    public final ASTIndexLink link;
+    public final String fieldname;
+    public final IndexMetadata md;
 
     FieldAndIndexPair(ASTIndexLink link, String fieldname, IndexMetadata md) {
         this.link = link;