Merge pull request #11279 from jpountz/fix/simplify_compression

Internal: tighten up our compression framework.

src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java

@@ -43,7 +43,7 @@
 import org.elasticsearch.cluster.routing.*;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.collect.Tuple;
-import org.elasticsearch.common.compress.CompressedString;
+import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.lease.Releasable;
 import org.elasticsearch.common.lease.Releasables;
 import org.elasticsearch.common.settings.Settings;
@@ -1080,7 +1080,7 @@ protected final WriteResult<IndexResponse> executeIndexRequestOnPrimary(BulkShar
                 // to find a _meta document
                 // So we have no choice but to index first and send mappings afterwards
                 MapperService mapperService = indexShard.indexService().mapperService();
-                mapperService.merge(request.type(), new CompressedString(update.toBytes()), true);
+                mapperService.merge(request.type(), new CompressedXContent(update.toBytes()), true);
                 created = operation.execute(indexShard);
                 mappingUpdatedAction.updateMappingOnMasterAsynchronously(indexName, request.type(), update);
             } else {

src/main/java/org/elasticsearch/client/transport/TransportClient.java

@@ -20,8 +20,14 @@
 package org.elasticsearch.client.transport;
 
 import com.google.common.collect.ImmutableList;
+
 import org.elasticsearch.Version;
-import org.elasticsearch.action.*;
+import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.ActionModule;
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionRequestBuilder;
+import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.cache.recycler.PageCacheRecycler;
 import org.elasticsearch.client.support.AbstractClient;
 import org.elasticsearch.client.support.Headers;
@@ -30,7 +36,6 @@
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.component.LifecycleComponent;
-import org.elasticsearch.common.compress.CompressorFactory;
 import org.elasticsearch.common.inject.Injector;
 import org.elasticsearch.common.inject.ModulesBuilder;
 import org.elasticsearch.common.network.NetworkModule;
@@ -122,8 +127,6 @@ public TransportClient build() {
 
             Version version = Version.CURRENT;
 
-            CompressorFactory.configure(this.settings);
-
             final ThreadPool threadPool = new ThreadPool(settings);
 
             boolean success = false;

src/main/java/org/elasticsearch/cluster/ClusterState.java

@@ -37,7 +37,7 @@
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
-import org.elasticsearch.common.compress.CompressedString;
+import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -422,7 +422,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
                 builder.endObject();
 
                 builder.startObject("mappings");
-                for (ObjectObjectCursor<String, CompressedString> cursor1 : templateMetaData.mappings()) {
+                for (ObjectObjectCursor<String, CompressedXContent> cursor1 : templateMetaData.mappings()) {
                     byte[] mappingSource = cursor1.value.uncompressed();
                     XContentParser parser = XContentFactory.xContent(mappingSource).createParser(mappingSource);
                     Map<String, Object> mapping = parser.map();

src/main/java/org/elasticsearch/cluster/metadata/AliasMetaData.java

@@ -23,7 +23,7 @@
 import org.elasticsearch.ElasticsearchGenerationException;
 import org.elasticsearch.cluster.AbstractDiffable;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.compress.CompressedString;
+import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.ToXContent;
@@ -45,15 +45,15 @@ public class AliasMetaData extends AbstractDiffable<AliasMetaData> {
 
     private final String alias;
 
-    private final CompressedString filter;
+    private final CompressedXContent filter;
 
     private final String indexRouting;
 
     private final String searchRouting;
 
     private final Set<String> searchRoutingValues;
 
-    private AliasMetaData(String alias, CompressedString filter, String indexRouting, String searchRouting) {
+    private AliasMetaData(String alias, CompressedXContent filter, String indexRouting, String searchRouting) {
         this.alias = alias;
         this.filter = filter;
         this.indexRouting = indexRouting;
@@ -77,11 +77,11 @@ public String getAlias() {
         return alias();
     }
 
-    public CompressedString filter() {
+    public CompressedXContent filter() {
         return filter;
     }
 
-    public CompressedString getFilter() {
+    public CompressedXContent getFilter() {
         return filter();
     }
 
@@ -176,9 +176,9 @@ public void writeTo(StreamOutput out) throws IOException {
     @Override
     public AliasMetaData readFrom(StreamInput in) throws IOException {
         String alias = in.readString();
-        CompressedString filter = null;
+        CompressedXContent filter = null;
         if (in.readBoolean()) {
-            filter = CompressedString.readCompressedString(in);
+            filter = CompressedXContent.readCompressedString(in);
         }
         String indexRouting = null;
         if (in.readBoolean()) {
@@ -195,7 +195,7 @@ public static class Builder {
 
         private final String alias;
 
-        private CompressedString filter;
+        private CompressedXContent filter;
 
         private String indexRouting;
 
@@ -217,7 +217,7 @@ public String alias() {
             return alias;
         }
 
-        public Builder filter(CompressedString filter) {
+        public Builder filter(CompressedXContent filter) {
             this.filter = filter;
             return this;
         }
@@ -244,7 +244,7 @@ public Builder filter(Map<String, Object> filter) {
             }
             try {
                 XContentBuilder builder = XContentFactory.jsonBuilder().map(filter);
-                this.filter = new CompressedString(builder.bytes());
+                this.filter = new CompressedXContent(builder.bytes());
                 return this;
             } catch (IOException e) {
                 throw new ElasticsearchGenerationException("Failed to build json for alias request", e);
@@ -324,7 +324,7 @@ public static AliasMetaData fromXContent(XContentParser parser) throws IOExcepti
                     }
                 } else if (token == XContentParser.Token.VALUE_EMBEDDED_OBJECT) {
                     if ("filter".equals(currentFieldName)) {
-                        builder.filter(new CompressedString(parser.binaryValue()));
+                        builder.filter(new CompressedXContent(parser.binaryValue()));
                     }
                 } else if (token == XContentParser.Token.VALUE_STRING) {
                     if ("routing".equals(currentFieldName)) {

src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java

@@ -35,7 +35,7 @@
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.collect.MapBuilder;
-import org.elasticsearch.common.compress.CompressedString;
+import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Settings;
@@ -874,7 +874,7 @@ public static IndexMetaData fromXContent(XContentParser parser) throws IOExcepti
                     if ("mappings".equals(currentFieldName)) {
                         while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
                             if (token == XContentParser.Token.VALUE_EMBEDDED_OBJECT) {
-                                builder.putMapping(new MappingMetaData(new CompressedString(parser.binaryValue())));
+                                builder.putMapping(new MappingMetaData(new CompressedXContent(parser.binaryValue())));
                             } else {
                                 Map<String, Object> mapping = parser.mapOrdered();
                                 if (mapping.size() == 1) {

src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java

@@ -24,7 +24,7 @@
 import org.elasticsearch.cluster.AbstractDiffable;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.collect.MapBuilder;
-import org.elasticsearch.common.compress.CompressedString;
+import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Settings;
@@ -54,13 +54,13 @@ public class IndexTemplateMetaData extends AbstractDiffable<IndexTemplateMetaDat
     private final Settings settings;
 
     // the mapping source should always include the type as top level
-    private final ImmutableOpenMap<String, CompressedString> mappings;
+    private final ImmutableOpenMap<String, CompressedXContent> mappings;
 
     private final ImmutableOpenMap<String, AliasMetaData> aliases;
 
     private final ImmutableOpenMap<String, IndexMetaData.Custom> customs;
 
-    public IndexTemplateMetaData(String name, int order, String template, Settings settings, ImmutableOpenMap<String, CompressedString> mappings,
+    public IndexTemplateMetaData(String name, int order, String template, Settings settings, ImmutableOpenMap<String, CompressedXContent> mappings,
                                  ImmutableOpenMap<String, AliasMetaData> aliases, ImmutableOpenMap<String, IndexMetaData.Custom> customs) {
         this.name = name;
         this.order = order;
@@ -103,11 +103,11 @@ public Settings getSettings() {
         return settings();
     }
 
-    public ImmutableOpenMap<String, CompressedString> mappings() {
+    public ImmutableOpenMap<String, CompressedXContent> mappings() {
         return this.mappings;
     }
 
-    public ImmutableOpenMap<String, CompressedString> getMappings() {
+    public ImmutableOpenMap<String, CompressedXContent> getMappings() {
         return this.mappings;
     }
 
@@ -170,7 +170,7 @@ public IndexTemplateMetaData readFrom(StreamInput in) throws IOException {
         builder.settings(Settings.readSettingsFromStream(in));
         int mappingsSize = in.readVInt();
         for (int i = 0; i < mappingsSize; i++) {
-            builder.putMapping(in.readString(), CompressedString.readCompressedString(in));
+            builder.putMapping(in.readString(), CompressedXContent.readCompressedString(in));
         }
         int aliasesSize = in.readVInt();
         for (int i = 0; i < aliasesSize; i++) {
@@ -193,7 +193,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeString(template);
         Settings.writeSettingsToStream(settings, out);
         out.writeVInt(mappings.size());
-        for (ObjectObjectCursor<String, CompressedString> cursor : mappings) {
+        for (ObjectObjectCursor<String, CompressedXContent> cursor : mappings) {
             out.writeString(cursor.key);
             cursor.value.writeTo(out);
         }
@@ -223,7 +223,7 @@ public static class Builder {
 
         private Settings settings = Settings.Builder.EMPTY_SETTINGS;
 
-        private final ImmutableOpenMap.Builder<String, CompressedString> mappings;
+        private final ImmutableOpenMap.Builder<String, CompressedXContent> mappings;
 
         private final ImmutableOpenMap.Builder<String, AliasMetaData> aliases;
 
@@ -276,13 +276,13 @@ public Builder removeMapping(String mappingType) {
             return this;
         }
 
-        public Builder putMapping(String mappingType, CompressedString mappingSource) throws IOException {
+        public Builder putMapping(String mappingType, CompressedXContent mappingSource) throws IOException {
             mappings.put(mappingType, mappingSource);
             return this;
         }
 
         public Builder putMapping(String mappingType, String mappingSource) throws IOException {
-            mappings.put(mappingType, new CompressedString(mappingSource));
+            mappings.put(mappingType, new CompressedXContent(mappingSource));
             return this;
         }
 
@@ -327,7 +327,7 @@ public static void toXContent(IndexTemplateMetaData indexTemplateMetaData, XCont
 
             if (params.paramAsBoolean("reduce_mappings", false)) {
                 builder.startObject("mappings");
-                for (ObjectObjectCursor<String, CompressedString> cursor : indexTemplateMetaData.mappings()) {
+                for (ObjectObjectCursor<String, CompressedXContent> cursor : indexTemplateMetaData.mappings()) {
                     byte[] mappingSource = cursor.value.uncompressed();
                     XContentParser parser = XContentFactory.xContent(mappingSource).createParser(mappingSource);
                     Map<String, Object> mapping = parser.map();
@@ -341,7 +341,7 @@ public static void toXContent(IndexTemplateMetaData indexTemplateMetaData, XCont
                 builder.endObject();
             } else {
                 builder.startArray("mappings");
-                for (ObjectObjectCursor<String, CompressedString> cursor : indexTemplateMetaData.mappings()) {
+                for (ObjectObjectCursor<String, CompressedXContent> cursor : indexTemplateMetaData.mappings()) {
                     byte[] data = cursor.value.uncompressed();
                     XContentParser parser = XContentFactory.xContent(data).createParser(data);
                     Map<String, Object> mapping = parser.mapOrderedAndClose();

src/main/java/org/elasticsearch/cluster/metadata/MappingMetaData.java

@@ -23,7 +23,7 @@
 import org.elasticsearch.cluster.AbstractDiffable;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.compress.CompressedString;
+import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.joda.FormatDateTimeFormatter;
@@ -276,7 +276,7 @@ public int hashCode() {
 
     private final String type;
 
-    private final CompressedString source;
+    private final CompressedXContent source;
 
     private Id id;
     private Routing routing;
@@ -294,9 +294,9 @@ public MappingMetaData(DocumentMapper docMapper) {
         this.hasParentField = docMapper.parentFieldMapper().active();
     }
 
-    public MappingMetaData(CompressedString mapping) throws IOException {
+    public MappingMetaData(CompressedXContent mapping) throws IOException {
         this.source = mapping;
-        Map<String, Object> mappingMap = XContentHelper.createParser(mapping.compressed(), 0, mapping.compressed().length).mapOrderedAndClose();
+        Map<String, Object> mappingMap = XContentHelper.createParser(mapping.compressedReference()).mapOrderedAndClose();
         if (mappingMap.size() != 1) {
             throw new IllegalStateException("Can't derive type from mapping, no root type: " + mapping.string());
         }
@@ -311,7 +311,7 @@ public MappingMetaData(Map<String, Object> mapping) throws IOException {
     public MappingMetaData(String type, Map<String, Object> mapping) throws IOException {
         this.type = type;
         XContentBuilder mappingBuilder = XContentFactory.jsonBuilder().map(mapping);
-        this.source = new CompressedString(mappingBuilder.bytes());
+        this.source = new CompressedXContent(mappingBuilder.bytes());
         Map<String, Object> withoutType = mapping;
         if (mapping.size() == 1 && mapping.containsKey(type)) {
             withoutType = (Map<String, Object>) mapping.get(type);
@@ -322,7 +322,7 @@ public MappingMetaData(String type, Map<String, Object> mapping) throws IOExcept
     private MappingMetaData() {
         this.type = "";
         try {
-            this.source = new CompressedString("");
+            this.source = new CompressedXContent("{}");
         } catch (IOException ex) {
             throw new IllegalStateException("Cannot create MappingMetaData prototype", ex);
         }
@@ -393,7 +393,7 @@ private void initMappers(Map<String, Object> withoutType) {
         }
     }
 
-    public MappingMetaData(String type, CompressedString source, Id id, Routing routing, Timestamp timestamp, boolean hasParentField) {
+    public MappingMetaData(String type, CompressedXContent source, Id id, Routing routing, Timestamp timestamp, boolean hasParentField) {
         this.type = type;
         this.source = source;
         this.id = id;
@@ -418,7 +418,7 @@ public String type() {
         return this.type;
     }
 
-    public CompressedString source() {
+    public CompressedXContent source() {
         return this.source;
     }
 
@@ -430,7 +430,7 @@ public boolean hasParentField() {
      * Converts the serialized compressed form of the mappings into a parsed map.
      */
     public Map<String, Object> sourceAsMap() throws IOException {
-        Map<String, Object> mapping = XContentHelper.convertToMap(source.compressed(), 0, source.compressed().length, true).v2();
+        Map<String, Object> mapping = XContentHelper.convertToMap(source.compressedReference(), true).v2();
         if (mapping.size() == 1 && mapping.containsKey(type())) {
             // the type name is the root value, reduce it
             mapping = (Map<String, Object>) mapping.get(type());
@@ -599,7 +599,7 @@ public int hashCode() {
 
     public MappingMetaData readFrom(StreamInput in) throws IOException {
         String type = in.readString();
-        CompressedString source = CompressedString.readCompressedString(in);
+        CompressedXContent source = CompressedXContent.readCompressedString(in);
         // id
         Id id = new Id(in.readBoolean() ? in.readString() : null);
         // routing