v3.2.0 (#229)

Change how ZomboDB tracks and resolves visibility information to drastically improve performance over all previous versions.

.gitignore

@@ -11,7 +11,7 @@
 target/
 .DS_store
 foo.*
-postgres/src/main/sql/zombodb--3.1.15.sql
+postgres/src/main/sql/zombodb--3.2.0.sql
 postgres/src/test/sql/setup.sql
 postgres/src/test/sql/so_comments.sql
 postgres/src/test/sql/TUTORIAL.sql

docker/elasticsearch/Dockerfile

@@ -9,7 +9,7 @@
 FROM java:7
 
 ENV ES_PKG_NAME elasticsearch-1.7.5
-ENV ZOMBODB_VER 3.1.15
+ENV ZOMBODB_VER 3.2.0
 
 # Install Elasticsearch.
 RUN \

docker/postgres/Dockerfile

@@ -4,7 +4,7 @@
 
 # Pull base image.
 FROM postgres:9.5
-ENV ZOMBODB_VER 3.1.15
+ENV ZOMBODB_VER 3.2.0
 
 # Fetch wget
 RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates wget && rm -rf /var/lib/apt/lists/*

elasticsearch/pom.xml

@@ -7,7 +7,7 @@
     <parent>
         <groupId>com.tcdi.elasticsearch</groupId>
         <artifactId>zombodb-parent</artifactId>
-        <version>3.1.15</version>
+        <version>3.2.0</version>
     </parent>
 
     <artifactId>zombodb-plugin</artifactId>

elasticsearch/src/main/java/com/tcdi/zombodb/ZombodbPlugin.java

@@ -46,7 +46,10 @@ public void onModule(RestModule module) {
         module.addRestAction(RestTermlistAction.class);
         module.addRestAction(ZombodbBulkAction.class);
         module.addRestAction(ZombodbCommitXIDAction.class);
+        module.addRestAction(ZombodbDeleteTuplesAction.class);
+        module.addRestAction(ZombodbGetXidVacuumCandidatesAction.class);
         module.addRestAction(ZombodbVacuumSupportAction.class);
+        module.addRestAction(ZombodbVacuumCleanupAction.class);
     }
 
     public void onModule(ActionModule module) {

elasticsearch/src/main/java/com/tcdi/zombodb/postgres/PostgresAggregationAction.java

@@ -54,7 +54,7 @@ protected void handleRequest(RestRequest request, RestChannel channel, Client cl
             SearchRequestBuilder builder = new SearchRequestBuilder(client);
             String input = request.content().toUtf8();
             final QueryRewriter rewriter = QueryRewriter.Factory.create(client, request.param("index"), request.param("preference"), input, true, true, true);
-            QueryBuilder qb = rewriter.rewriteQuery(true);
+            QueryBuilder qb = rewriter.rewriteQuery();
             AbstractAggregationBuilder ab = rewriter.rewriteAggregations();
             SuggestBuilder.SuggestionBuilder tsb = rewriter.rewriteSuggestions();
 

elasticsearch/src/main/java/com/tcdi/zombodb/postgres/PostgresCountAction.java

@@ -16,7 +16,6 @@
  */
 package com.tcdi.zombodb.postgres;
 
-import org.elasticsearch.action.admin.indices.refresh.RefreshRequestBuilder;
 import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.search.SearchType;
@@ -47,7 +46,7 @@ protected void handleRequest(RestRequest request, RestChannel channel, Client cl
             BytesRestResponse response;
             QueryAndIndexPair query;
 
-            query = PostgresTIDResponseAction.buildJsonQueryFromRequestContent(client, request, !isSelectivityQuery, true, true, true);
+            query = PostgresTIDResponseAction.buildJsonQueryFromRequestContent(client, request, !isSelectivityQuery, true, true);
 
             if (query.hasLimit() && isSelectivityQuery) {
                 count = query.getLimit().getLimit();
@@ -58,7 +57,7 @@ protected void handleRequest(RestRequest request, RestChannel channel, Client cl
                 builder.setSize(0);
                 builder.setSearchType(SearchType.COUNT);
                 builder.setPreference(request.param("preference"));
-                builder.setQueryCache(false);
+                builder.setQueryCache(true);
                 builder.setFetchSource(false);
                 builder.setTrackScores(false);
                 builder.setNoFields();

elasticsearch/src/main/java/com/tcdi/zombodb/postgres/PostgresMappingAction.java

@@ -42,7 +42,7 @@ protected void handleRequest(RestRequest request, RestChannel channel, Client cl
         BytesRestResponse response;
 
         QueryRewriter rewriter = QueryRewriter.Factory.create(client, request.param("index"), request.param("preference"), request.content().toUtf8(), true, false, false);
-        rewriter.rewriteQuery(false);
+        rewriter.rewriteQuery();
         Map<String, ?> properties = rewriter.describedNestedObject(request.param("fieldname"));
 
         response = new BytesRestResponse(RestStatus.OK, "application/json", JsonXContent.contentBuilder().map(properties).bytes());

elasticsearch/src/main/java/com/tcdi/zombodb/postgres/PostgresTIDResponseAction.java

@@ -107,7 +107,7 @@ protected void handleRequest(RestRequest request, RestChannel channel, Client cl
 
         try {
             parseStart = System.nanoTime();
-            query = buildJsonQueryFromRequestContent(client, request, true, false, false, false);
+            query = buildJsonQueryFromRequestContent(client, request, true, false, false);
             parseEnd = System.nanoTime();
 
             SearchRequestBuilder builder = new SearchRequestBuilder(client);
@@ -153,7 +153,7 @@ protected void handleRequest(RestRequest request, RestChannel channel, Client cl
         }
     }
 
-    public static QueryAndIndexPair buildJsonQueryFromRequestContent(Client client, RestRequest request, boolean doFullFieldDataLookups, boolean canDoSingleIndex, boolean needVisibilityOnTopLevel, boolean all) {
+    public static QueryAndIndexPair buildJsonQueryFromRequestContent(Client client, RestRequest request, boolean doFullFieldDataLookups, boolean canDoSingleIndex, boolean needVisibilityOnTopLevel) {
         String queryString = request.content().toUtf8();
         String indexName = request.param("index");
 
@@ -163,7 +163,7 @@ public static QueryAndIndexPair buildJsonQueryFromRequestContent(Client client,
 
             if (queryString != null && queryString.trim().length() > 0) {
                 QueryRewriter qr = QueryRewriter.Factory.create(client, indexName, request.param("preference"), queryString, doFullFieldDataLookups, canDoSingleIndex, needVisibilityOnTopLevel);
-                query = qr.rewriteQuery(all);
+                query = qr.rewriteQuery();
                 indexName = qr.getSearchIndexName();
                 limit = qr.getLimit();
             } else {

elasticsearch/src/main/java/com/tcdi/zombodb/postgres/RoutingHelper.java

@@ -0,0 +1,54 @@
+/*
+ * Copyright 2017 ZomboDB, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.tcdi.zombodb.postgres;
+
+import org.elasticsearch.client.Client;
+import org.elasticsearch.cluster.ClusterService;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.routing.operation.OperationRouting;
+
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+public class RoutingHelper {
+    private static Map<String, String[]> ROUTING_TABLE = new ConcurrentHashMap<>();
+
+    public static String[] getRoutingTable(Client client, ClusterService clusterService, String index, int shards) {
+        String key = index+"."+shards;
+
+        String[] routingTable = ROUTING_TABLE.get(key);
+
+        if (routingTable != null)
+            return routingTable;
+
+        ClusterState clusterState = clusterService.state();
+        OperationRouting operationRouting = clusterService.operationRouting();
+
+        routingTable = new String[shards];
+        for (int i=0; i<shards; i++) {
+            String routing = String.valueOf(i);
+
+            int cnt=0;
+            while ( (operationRouting.indexShards(clusterState, index, "aborted", routing, null).shardId()).id() != i)
+                routing = String.valueOf(i + ++cnt);
+            routingTable[i] = routing;
+        }
+
+        ROUTING_TABLE.put(key, routingTable);
+        return routingTable;
+
+    }
+}