diff --git a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/ProtobufMessageHandler.java b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/ProtobufMessageHandler.java index 2e01ff24d477..962e5dfae860 100644 --- a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/ProtobufMessageHandler.java +++ b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/ProtobufMessageHandler.java @@ -18,21 +18,60 @@ package org.apache.hadoop.hbase.rest; import java.io.IOException; +import java.io.OutputStream; import org.apache.yetus.audience.InterfaceAudience; +import org.apache.hbase.thirdparty.com.google.protobuf.CodedOutputStream; +import org.apache.hbase.thirdparty.com.google.protobuf.Message; + /** * Common interface for models capable of supporting protobuf marshalling and unmarshalling. Hooks * up to the ProtobufMessageBodyConsumer and ProtobufMessageBodyProducer adapters. */ @InterfaceAudience.Private public interface ProtobufMessageHandler { - /** Returns the protobuf represention of the model */ - byte[] createProtobufOutput(); + + // The Jetty 9.4 HttpOutput default commit size is 32K/4 = 8K. We use that size to avoid + // double buffering (and copying) in HttpOutput. If we ever increase the HttpOutput commit size, + // we need to adjust this accordingly. We should also revisit this when Jetty is upgraded. + int BUFFER_SIZE = 8 * 1024; + + /** Writes the protobuf represention of the model to os */ + default void writeProtobufOutput(OutputStream os) throws IOException { + // Creating an explicit CodedOutputStream for the following reasons : + // 1. This avoids the cost of pre-computing the message size + // 2. This lets us set the buffer size explicitly + CodedOutputStream cos = CodedOutputStream.newInstance(os, BUFFER_SIZE); + messageFromObject().writeTo(cos); + cos.flush(); + } + + /** + * Returns the protobuf represention of the model in a byte array Use + * {@link org.apache.hadoop.hbase.rest.ProtobufMessageHandler#writeProtobufOutput(OutputStream)} + * for better performance + * @return the protobuf encoded object in a byte array + */ + default byte[] createProtobufOutput() { + return messageFromObject().toByteArray(); + } + + /** + * Convert to model to a protobuf Message object + * @return the protobuf Message object + */ + Message messageFromObject(); /** * Initialize the model from a protobuf representation. * @param message the raw bytes of the protobuf message * @return reference to self for convenience */ + // TODO implement proper stream handling for unmarshalling. + // Using byte array here lets us use ProtobufUtil.mergeFrom in the implementations to + // avoid the CodedOutputStream size limitation, but is slow + // and memory intensive. We could use the ProtobufUtil.mergeFrom() variant that takes + // an inputStream and sets the size limit to maxInt. + // This would help both on the client side, and when processing large Puts on the server. ProtobufMessageHandler getObjectFromMessage(byte[] message) throws IOException; } diff --git a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/CellModel.java b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/CellModel.java index 4284727e4380..3d8806b7dc00 100644 --- a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/CellModel.java +++ b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/CellModel.java @@ -36,6 +36,7 @@ import org.apache.hadoop.hbase.rest.ProtobufMessageHandler; import org.apache.yetus.audience.InterfaceAudience; +import org.apache.hbase.thirdparty.com.google.protobuf.Message; import org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations; import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; @@ -202,7 +203,7 @@ public int getValueLength() { } @Override - public byte[] createProtobufOutput() { + public Message messageFromObject() { Cell.Builder builder = Cell.newBuilder(); builder.setColumn(UnsafeByteOperations.unsafeWrap(getColumn())); if (valueLength == MAGIC_LENGTH) { @@ -213,7 +214,7 @@ public byte[] createProtobufOutput() { if (hasUserTimestamp()) { builder.setTimestamp(getTimestamp()); } - return builder.build().toByteArray(); + return builder.build(); } @Override diff --git a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/CellSetModel.java b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/CellSetModel.java index 8908ec7e6c88..8486be2762fe 100644 --- a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/CellSetModel.java +++ b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/CellSetModel.java @@ -31,6 +31,7 @@ import org.apache.hadoop.hbase.rest.ProtobufMessageHandler; import org.apache.yetus.audience.InterfaceAudience; +import org.apache.hbase.thirdparty.com.google.protobuf.Message; import org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations; import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; @@ -108,7 +109,7 @@ public List getRows() { } @Override - public byte[] createProtobufOutput() { + public Message messageFromObject() { CellSet.Builder builder = CellSet.newBuilder(); for (RowModel row : getRows()) { CellSet.Row.Builder rowBuilder = CellSet.Row.newBuilder(); @@ -134,7 +135,7 @@ public byte[] createProtobufOutput() { } builder.addRows(rowBuilder); } - return builder.build().toByteArray(); + return builder.build(); } @Override diff --git a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/NamespacesInstanceModel.java b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/NamespacesInstanceModel.java index 64b46f2956c8..78f647203851 100644 --- a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/NamespacesInstanceModel.java +++ b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/NamespacesInstanceModel.java @@ -31,6 +31,8 @@ import org.apache.hadoop.hbase.rest.ProtobufMessageHandler; import org.apache.yetus.audience.InterfaceAudience; +import org.apache.hbase.thirdparty.com.google.protobuf.Message; + import org.apache.hadoop.hbase.shaded.rest.protobuf.generated.NamespacePropertiesMessage.NamespaceProperties; /** @@ -140,7 +142,7 @@ public String toString() { } @Override - public byte[] createProtobufOutput() { + public Message messageFromObject() { NamespaceProperties.Builder builder = NamespaceProperties.newBuilder(); if (properties != null) { for (Map.Entry entry : properties.entrySet()) { @@ -151,7 +153,7 @@ public byte[] createProtobufOutput() { builder.addProps(property); } } - return builder.build().toByteArray(); + return builder.build(); } @Override diff --git a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/NamespacesModel.java b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/NamespacesModel.java index e866c7a935d1..90e4f6560a51 100644 --- a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/NamespacesModel.java +++ b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/NamespacesModel.java @@ -31,6 +31,8 @@ import org.apache.hadoop.hbase.rest.ProtobufMessageHandler; import org.apache.yetus.audience.InterfaceAudience; +import org.apache.hbase.thirdparty.com.google.protobuf.Message; + import org.apache.hadoop.hbase.shaded.rest.protobuf.generated.NamespacesMessage.Namespaces; /** @@ -95,10 +97,10 @@ public String toString() { } @Override - public byte[] createProtobufOutput() { + public Message messageFromObject() { Namespaces.Builder builder = Namespaces.newBuilder(); builder.addAllNamespace(namespaces); - return builder.build().toByteArray(); + return builder.build(); } @Override diff --git a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/RowModel.java b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/RowModel.java index 8b660ac362fc..e200dfbc1f35 100644 --- a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/RowModel.java +++ b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/RowModel.java @@ -36,6 +36,8 @@ import org.apache.hadoop.hbase.util.Bytes; import org.apache.yetus.audience.InterfaceAudience; +import org.apache.hbase.thirdparty.com.google.protobuf.Message; + /** * Representation of a row. A row is a related set of cells, grouped by common row key. RowModels do * not appear in results by themselves. They are always encapsulated within CellSetModels. @@ -179,7 +181,7 @@ public List getCells() { } @Override - public byte[] createProtobufOutput() { + public Message messageFromObject() { // there is no standalone row protobuf message throw new UnsupportedOperationException("no protobuf equivalent to RowModel"); } diff --git a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/ScannerModel.java b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/ScannerModel.java index 3655a3798041..4c241753e5e8 100644 --- a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/ScannerModel.java +++ b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/ScannerModel.java @@ -71,6 +71,7 @@ import org.apache.hbase.thirdparty.com.fasterxml.jackson.jaxrs.json.JacksonJaxbJsonProvider; import org.apache.hbase.thirdparty.com.google.protobuf.ByteString; +import org.apache.hbase.thirdparty.com.google.protobuf.Message; import org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations; import org.apache.hbase.thirdparty.javax.ws.rs.core.MediaType; @@ -809,7 +810,7 @@ public void setFilter(String filter) { } @Override - public byte[] createProtobufOutput() { + public Message messageFromObject() { Scanner.Builder builder = Scanner.newBuilder(); if (!Bytes.equals(startRow, HConstants.EMPTY_START_ROW)) { builder.setStartRow(UnsafeByteOperations.unsafeWrap(startRow)); @@ -842,7 +843,7 @@ public byte[] createProtobufOutput() { builder.addLabels(label); } builder.setCacheBlocks(cacheBlocks); - return builder.build().toByteArray(); + return builder.build(); } @Override diff --git a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/StorageClusterStatusModel.java b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/StorageClusterStatusModel.java index e0102811142a..c9370cad901b 100644 --- a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/StorageClusterStatusModel.java +++ b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/StorageClusterStatusModel.java @@ -30,6 +30,7 @@ import org.apache.hadoop.hbase.util.Bytes; import org.apache.yetus.audience.InterfaceAudience; +import org.apache.hbase.thirdparty.com.google.protobuf.Message; import org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations; import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; @@ -672,7 +673,7 @@ public String toString() { } @Override - public byte[] createProtobufOutput() { + public Message messageFromObject() { StorageClusterStatus.Builder builder = StorageClusterStatus.newBuilder(); builder.setRegions(regions); builder.setRequests(requests); @@ -708,7 +709,7 @@ public byte[] createProtobufOutput() { for (String node : deadNodes) { builder.addDeadNodes(node); } - return builder.build().toByteArray(); + return builder.build(); } @Override diff --git a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/TableInfoModel.java b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/TableInfoModel.java index 74d0732ec918..43b131fcb701 100644 --- a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/TableInfoModel.java +++ b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/TableInfoModel.java @@ -27,6 +27,7 @@ import org.apache.hadoop.hbase.rest.ProtobufMessageHandler; import org.apache.yetus.audience.InterfaceAudience; +import org.apache.hbase.thirdparty.com.google.protobuf.Message; import org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations; import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; @@ -123,7 +124,7 @@ public String toString() { } @Override - public byte[] createProtobufOutput() { + public Message messageFromObject() { TableInfo.Builder builder = TableInfo.newBuilder(); builder.setName(name); for (TableRegionModel aRegion : regions) { @@ -135,7 +136,7 @@ public byte[] createProtobufOutput() { regionBuilder.setLocation(aRegion.getLocation()); builder.addRegions(regionBuilder); } - return builder.build().toByteArray(); + return builder.build(); } @Override diff --git a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/TableListModel.java b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/TableListModel.java index 76854acdf6ae..63b2e809279c 100644 --- a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/TableListModel.java +++ b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/TableListModel.java @@ -26,6 +26,8 @@ import org.apache.hadoop.hbase.rest.ProtobufMessageHandler; import org.apache.yetus.audience.InterfaceAudience; +import org.apache.hbase.thirdparty.com.google.protobuf.Message; + import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.shaded.rest.protobuf.generated.TableListMessage.TableList; @@ -90,12 +92,12 @@ public String toString() { } @Override - public byte[] createProtobufOutput() { + public Message messageFromObject() { TableList.Builder builder = TableList.newBuilder(); for (TableModel aTable : tables) { builder.addName(aTable.getName()); } - return builder.build().toByteArray(); + return builder.build(); } @Override diff --git a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/TableSchemaModel.java b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/TableSchemaModel.java index 06abe355859a..f2a8c4c7060d 100644 --- a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/TableSchemaModel.java +++ b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/TableSchemaModel.java @@ -42,6 +42,8 @@ import org.apache.hadoop.hbase.util.Bytes; import org.apache.yetus.audience.InterfaceAudience; +import org.apache.hbase.thirdparty.com.google.protobuf.Message; + import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.shaded.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema; import org.apache.hadoop.hbase.shaded.rest.protobuf.generated.TableSchemaMessage.TableSchema; @@ -248,7 +250,7 @@ public void __setReadOnly(boolean value) { } @Override - public byte[] createProtobufOutput() { + public Message messageFromObject() { TableSchema.Builder builder = TableSchema.newBuilder(); builder.setName(name); for (Map.Entry e : attrs.entrySet()) { @@ -281,7 +283,7 @@ public byte[] createProtobufOutput() { if (attrs.containsKey(READONLY)) { builder.setReadOnly(Boolean.parseBoolean(attrs.get(READONLY).toString())); } - return builder.build().toByteArray(); + return builder.build(); } @Override diff --git a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/VersionModel.java b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/VersionModel.java index e5d79af5e55c..65eca57ac5a3 100644 --- a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/VersionModel.java +++ b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/VersionModel.java @@ -26,6 +26,7 @@ import org.apache.hadoop.hbase.rest.RESTServlet; import org.apache.yetus.audience.InterfaceAudience; +import org.apache.hbase.thirdparty.com.google.protobuf.Message; import org.apache.hbase.thirdparty.org.glassfish.jersey.servlet.ServletContainer; import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; @@ -162,14 +163,14 @@ public String toString() { } @Override - public byte[] createProtobufOutput() { + public Message messageFromObject() { Version.Builder builder = Version.newBuilder(); builder.setRestVersion(restVersion); builder.setJvmVersion(jvmVersion); builder.setOsVersion(osVersion); builder.setServerVersion(serverVersion); builder.setJerseyVersion(jerseyVersion); - return builder.build().toByteArray(); + return builder.build(); } @Override diff --git a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/provider/producer/ProtobufMessageBodyProducer.java b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/provider/producer/ProtobufMessageBodyProducer.java index 1d95e6f343e7..4a7806e652b1 100644 --- a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/provider/producer/ProtobufMessageBodyProducer.java +++ b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/provider/producer/ProtobufMessageBodyProducer.java @@ -59,6 +59,6 @@ public long getSize(ProtobufMessageHandler m, Class type, Type genericType, public void writeTo(ProtobufMessageHandler m, Class type, Type genericType, Annotation[] annotations, MediaType mediaType, MultivaluedMap httpHeaders, OutputStream entityStream) throws IOException, WebApplicationException { - entityStream.write(m.createProtobufOutput()); + m.writeProtobufOutput(entityStream); } }