From 90a52a82829d674df8d96cd6157f05848a82b967 Mon Sep 17 00:00:00 2001 From: Haozhun Jin Date: Fri, 20 Apr 2018 12:53:10 -0700 Subject: [PATCH] Simplify Block serialization/deserialization Previously, there are 3 sections in a serialized Block: * Name - written by BlockEncodingSerde * e.g. "ARRAY" indicating it's an ArrayBlock * Metadata - written by BlockEncodingFactory * e.g. "INT_ARRAY" indicating the inner Block type of an ArrayBlock * e.g. 4 indicating the entry size of FixedWidthBlock * Data - written by BlockEncoding This 3-section scheme makes things more complicated: * Not recursive - For an ArrayBlock with IntArrayBlock inside, the encoding of IntArrayBlock isn't the same as a stand-alone IntArrayBlock. Specifically, part of it is put in the metadata section, and part of it in the data section. This requires each invididual encoding to handle nested blocks, which introduces both code duplication and complexity. * It is harder to learn and understand the implementations with two interfaces (BlockEncodingFactory and BlockEncoding), which don't have a very clear separation of responsibility, for each Block. This commit solves both problems: * Only a single class (BlockEncoding) is now needed. * A block inside another block now serializes the exact same way as a standalone one. The outer block no longer need to handle any of the details of the inner one. It also resolves the TODO item that replacementBlockForWrite was not respected for nested Blocks. --- .../presto/block/BlockEncodingManager.java | 112 ++++++++++-------- .../facebook/presto/block/BlockSerdeUtil.java | 21 +--- .../presto/operator/GroupByIdBlock.java | 5 +- .../facebook/presto/server/PluginManager.java | 8 +- .../presto/server/ServerMainModule.java | 4 +- .../presto/block/AbstractTestBlock.java | 18 ++- .../presto/block/ColumnarTestUtils.java | 18 ++- .../java/com/facebook/presto/spi/Plugin.java | 5 +- .../presto/spi/block/AbstractArrayBlock.java | 4 +- .../spi/block/AbstractFixedWidthBlock.java | 4 +- .../presto/spi/block/AbstractMapBlock.java | 4 +- .../presto/spi/block/AbstractRowBlock.java | 8 +- .../spi/block/AbstractSingleArrayBlock.java | 2 +- .../spi/block/AbstractVariableWidthBlock.java | 4 +- .../presto/spi/block/ArrayBlockEncoding.java | 40 +------ .../com/facebook/presto/spi/block/Block.java | 2 +- .../presto/spi/block/BlockEncoding.java | 4 +- .../spi/block/BlockEncodingFactory.java | 35 ------ .../presto/spi/block/BlockEncodingSerde.java | 4 +- .../presto/spi/block/ByteArrayBlock.java | 4 +- .../spi/block/ByteArrayBlockBuilder.java | 4 +- .../spi/block/ByteArrayBlockEncoding.java | 27 +---- .../presto/spi/block/DictionaryBlock.java | 4 +- .../spi/block/DictionaryBlockEncoding.java | 46 +------ .../spi/block/FixedWidthBlockEncoding.java | 48 +------- .../presto/spi/block/IntArrayBlock.java | 4 +- .../spi/block/IntArrayBlockBuilder.java | 4 +- .../spi/block/IntArrayBlockEncoding.java | 27 +---- .../facebook/presto/spi/block/LazyBlock.java | 4 +- .../presto/spi/block/LazyBlockEncoding.java | 10 +- .../presto/spi/block/LongArrayBlock.java | 4 +- .../spi/block/LongArrayBlockBuilder.java | 4 +- .../spi/block/LongArrayBlockEncoding.java | 27 +---- .../presto/spi/block/MapBlockEncoding.java | 77 +++--------- .../presto/spi/block/RowBlockEncoding.java | 64 ++-------- .../spi/block/RunLengthBlockEncoding.java | 47 +------- .../spi/block/RunLengthEncodedBlock.java | 4 +- .../presto/spi/block/ShortArrayBlock.java | 4 +- .../spi/block/ShortArrayBlockBuilder.java | 4 +- .../spi/block/ShortArrayBlockEncoding.java | 27 +---- .../presto/spi/block/SingleMapBlock.java | 6 +- .../spi/block/SingleMapBlockEncoding.java | 75 +++--------- .../spi/block/SingleMapBlockWriter.java | 2 +- .../presto/spi/block/SingleRowBlock.java | 13 +- .../spi/block/SingleRowBlockEncoding.java | 56 ++------- .../spi/block/SingleRowBlockWriter.java | 2 +- .../spi/block/VariableWidthBlockEncoding.java | 27 +---- .../block/TestDictionaryBlockEncoding.java | 8 +- .../block/TestVariableWidthBlockEncoding.java | 8 +- .../spi/block/TestingBlockEncodingSerde.java | 96 +++++++-------- .../spi/block/TestingBlockJsonSerde.java | 10 +- 51 files changed, 314 insertions(+), 735 deletions(-) delete mode 100644 presto-spi/src/main/java/com/facebook/presto/spi/block/BlockEncodingFactory.java diff --git a/presto-main/src/main/java/com/facebook/presto/block/BlockEncodingManager.java b/presto-main/src/main/java/com/facebook/presto/block/BlockEncodingManager.java index 83708f05a178..5144e68abe0f 100644 --- a/presto-main/src/main/java/com/facebook/presto/block/BlockEncodingManager.java +++ b/presto-main/src/main/java/com/facebook/presto/block/BlockEncodingManager.java @@ -13,22 +13,23 @@ */ package com.facebook.presto.block; -import com.facebook.presto.spi.block.ArrayBlockEncoding.ArrayBlockEncodingFactory; +import com.facebook.presto.spi.block.ArrayBlockEncoding; +import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.block.BlockEncoding; -import com.facebook.presto.spi.block.BlockEncodingFactory; import com.facebook.presto.spi.block.BlockEncodingSerde; -import com.facebook.presto.spi.block.ByteArrayBlockEncoding.ByteArrayBlockEncodingFactory; -import com.facebook.presto.spi.block.DictionaryBlockEncoding.DictionaryBlockEncodingFactory; -import com.facebook.presto.spi.block.FixedWidthBlockEncoding.FixedWidthBlockEncodingFactory; -import com.facebook.presto.spi.block.IntArrayBlockEncoding.IntArrayBlockEncodingFactory; -import com.facebook.presto.spi.block.LongArrayBlockEncoding.LongArrayBlockEncodingFactory; -import com.facebook.presto.spi.block.MapBlockEncoding.MapBlockEncodingFactory; -import com.facebook.presto.spi.block.RowBlockEncoding.RowBlockEncodingFactory; -import com.facebook.presto.spi.block.RunLengthBlockEncoding.RunLengthBlockEncodingFactory; -import com.facebook.presto.spi.block.ShortArrayBlockEncoding.ShortArrayBlockEncodingFactory; -import com.facebook.presto.spi.block.SingleMapBlockEncoding.SingleMapBlockEncodingFactory; -import com.facebook.presto.spi.block.SingleRowBlockEncoding.SingleRowBlockEncodingFactory; -import com.facebook.presto.spi.block.VariableWidthBlockEncoding.VariableWidthBlockEncodingFactory; +import com.facebook.presto.spi.block.ByteArrayBlockEncoding; +import com.facebook.presto.spi.block.DictionaryBlockEncoding; +import com.facebook.presto.spi.block.FixedWidthBlockEncoding; +import com.facebook.presto.spi.block.IntArrayBlockEncoding; +import com.facebook.presto.spi.block.LazyBlockEncoding; +import com.facebook.presto.spi.block.LongArrayBlockEncoding; +import com.facebook.presto.spi.block.MapBlockEncoding; +import com.facebook.presto.spi.block.RowBlockEncoding; +import com.facebook.presto.spi.block.RunLengthBlockEncoding; +import com.facebook.presto.spi.block.ShortArrayBlockEncoding; +import com.facebook.presto.spi.block.SingleMapBlockEncoding; +import com.facebook.presto.spi.block.SingleRowBlockEncoding; +import com.facebook.presto.spi.block.VariableWidthBlockEncoding; import com.facebook.presto.spi.type.TypeManager; import com.google.common.collect.ImmutableSet; import io.airlift.slice.SliceInput; @@ -36,6 +37,7 @@ import javax.inject.Inject; +import java.util.Optional; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -47,73 +49,85 @@ public final class BlockEncodingManager implements BlockEncodingSerde { - private final ConcurrentMap> blockEncodings = new ConcurrentHashMap<>(); + private final ConcurrentMap blockEncodings = new ConcurrentHashMap<>(); - public BlockEncodingManager(TypeManager typeManager, BlockEncodingFactory... blockEncodingFactories) + public BlockEncodingManager(TypeManager typeManager, BlockEncoding... blockEncodings) { - this(typeManager, ImmutableSet.copyOf(blockEncodingFactories)); + this(typeManager, ImmutableSet.copyOf(blockEncodings)); } @Inject - public BlockEncodingManager(TypeManager typeManager, Set> blockEncodingFactories) + public BlockEncodingManager(TypeManager typeManager, Set blockEncodings) { // This function should be called from Guice and tests only - // always add the built-in BlockEncodingFactories - addBlockEncodingFactory(new VariableWidthBlockEncodingFactory()); - addBlockEncodingFactory(new FixedWidthBlockEncodingFactory()); - addBlockEncodingFactory(new ByteArrayBlockEncodingFactory()); - addBlockEncodingFactory(new ShortArrayBlockEncodingFactory()); - addBlockEncodingFactory(new IntArrayBlockEncodingFactory()); - addBlockEncodingFactory(new LongArrayBlockEncodingFactory()); - addBlockEncodingFactory(new DictionaryBlockEncodingFactory()); - addBlockEncodingFactory(new ArrayBlockEncodingFactory()); - addBlockEncodingFactory(new MapBlockEncodingFactory(typeManager)); - addBlockEncodingFactory(new SingleMapBlockEncodingFactory(typeManager)); - addBlockEncodingFactory(new RowBlockEncodingFactory()); - addBlockEncodingFactory(new SingleRowBlockEncodingFactory()); - addBlockEncodingFactory(new RunLengthBlockEncodingFactory()); - - for (BlockEncodingFactory factory : requireNonNull(blockEncodingFactories, "blockEncodingFactories is null")) { - addBlockEncodingFactory(factory); + // always add the built-in BlockEncodings + addBlockEncoding(new VariableWidthBlockEncoding()); + addBlockEncoding(new FixedWidthBlockEncoding()); + addBlockEncoding(new ByteArrayBlockEncoding()); + addBlockEncoding(new ShortArrayBlockEncoding()); + addBlockEncoding(new IntArrayBlockEncoding()); + addBlockEncoding(new LongArrayBlockEncoding()); + addBlockEncoding(new DictionaryBlockEncoding()); + addBlockEncoding(new ArrayBlockEncoding()); + addBlockEncoding(new MapBlockEncoding(typeManager)); + addBlockEncoding(new SingleMapBlockEncoding(typeManager)); + addBlockEncoding(new RowBlockEncoding()); + addBlockEncoding(new SingleRowBlockEncoding()); + addBlockEncoding(new RunLengthBlockEncoding()); + addBlockEncoding(new LazyBlockEncoding()); + + for (BlockEncoding blockEncoding : requireNonNull(blockEncodings, "blockEncodings is null")) { + addBlockEncoding(blockEncoding); } } - public void addBlockEncodingFactory(BlockEncodingFactory blockEncoding) + public void addBlockEncoding(BlockEncoding blockEncoding) { requireNonNull(blockEncoding, "blockEncoding is null"); - BlockEncodingFactory existingEntry = blockEncodings.putIfAbsent(blockEncoding.getName(), blockEncoding); + BlockEncoding existingEntry = blockEncodings.putIfAbsent(blockEncoding.getName(), blockEncoding); checkArgument(existingEntry == null, "Encoding %s is already registered", blockEncoding.getName()); } @Override - public BlockEncoding readBlockEncoding(SliceInput input) + public Block readBlock(SliceInput input) { // read the encoding name String encodingName = readLengthPrefixedString(input); // look up the encoding factory - BlockEncodingFactory blockEncoding = blockEncodings.get(encodingName); + BlockEncoding blockEncoding = blockEncodings.get(encodingName); checkArgument(blockEncoding != null, "Unknown block encoding %s", encodingName); // load read the encoding factory from the output stream - return blockEncoding.readEncoding(this, input); + return blockEncoding.readBlock(this, input); } @Override - public void writeBlockEncoding(SliceOutput output, BlockEncoding encoding) + public void writeBlock(SliceOutput output, Block block) { - // get the encoding name - String encodingName = encoding.getName(); + while (true) { + // get the encoding name + String encodingName = block.getEncodingName(); - // look up the encoding factory - BlockEncodingFactory blockEncoding = blockEncodings.get(encodingName); + // look up the BlockEncoding + BlockEncoding blockEncoding = blockEncodings.get(encodingName); + + // see if a replacement block should be written instead + Optional replacementBlock = blockEncoding.replacementBlockForWrite(block); + if (replacementBlock.isPresent()) { + block = replacementBlock.get(); + continue; + } - // write the name to the output - writeLengthPrefixedString(output, encodingName); + // write the name to the output + writeLengthPrefixedString(output, encodingName); - // write the encoding to the output - blockEncoding.writeEncoding(this, output, encoding); + // write the block to the output + blockEncoding.writeBlock(this, output, block); + + break; + } } private static String readLengthPrefixedString(SliceInput input) diff --git a/presto-main/src/main/java/com/facebook/presto/block/BlockSerdeUtil.java b/presto-main/src/main/java/com/facebook/presto/block/BlockSerdeUtil.java index c3bf7cb304d0..05240fc640ee 100644 --- a/presto-main/src/main/java/com/facebook/presto/block/BlockSerdeUtil.java +++ b/presto-main/src/main/java/com/facebook/presto/block/BlockSerdeUtil.java @@ -14,14 +14,12 @@ package com.facebook.presto.block; import com.facebook.presto.spi.block.Block; -import com.facebook.presto.spi.block.BlockEncoding; import com.facebook.presto.spi.block.BlockEncodingSerde; import io.airlift.slice.Slice; import io.airlift.slice.SliceInput; import io.airlift.slice.SliceOutput; import java.lang.invoke.MethodHandle; -import java.util.Optional; import static com.facebook.presto.util.Reflection.methodHandle; @@ -40,26 +38,11 @@ public static Block readBlock(BlockEncodingSerde blockEncodingSerde, Slice slice public static Block readBlock(BlockEncodingSerde blockEncodingSerde, SliceInput input) { - BlockEncoding blockEncoding = blockEncodingSerde.readBlockEncoding(input); - return blockEncoding.readBlock(input); + return blockEncodingSerde.readBlock(input); } public static void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput output, Block block) { - while (true) { - // TODO: respect replacementBlockForWrite for Blocks nested in other Block - // A proposed simplified design for block encoding decoding will address this to-do item. - - BlockEncoding encoding = block.getEncoding(); - Optional replaceBlock = encoding.replacementBlockForWrite(block); - if (!replaceBlock.isPresent()) { - break; - } - block = replaceBlock.get(); - } - - BlockEncoding encoding = block.getEncoding(); - blockEncodingSerde.writeBlockEncoding(output, encoding); - encoding.writeBlock(output, block); + blockEncodingSerde.writeBlock(output, block); } } diff --git a/presto-main/src/main/java/com/facebook/presto/operator/GroupByIdBlock.java b/presto-main/src/main/java/com/facebook/presto/operator/GroupByIdBlock.java index 37351033874a..c8993466b33c 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/GroupByIdBlock.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/GroupByIdBlock.java @@ -15,7 +15,6 @@ import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.block.BlockBuilder; -import com.facebook.presto.spi.block.BlockEncoding; import io.airlift.slice.Slice; import org.openjdk.jol.info.ClassLayout; @@ -190,9 +189,9 @@ public void retainedBytesForEachPart(BiConsumer consumer) } @Override - public BlockEncoding getEncoding() + public String getEncodingName() { - return block.getEncoding(); + throw new UnsupportedOperationException("GroupByIdBlock does not support serialization"); } @Override diff --git a/presto-main/src/main/java/com/facebook/presto/server/PluginManager.java b/presto-main/src/main/java/com/facebook/presto/server/PluginManager.java index f867a1bf0857..f2a7948bf40d 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/PluginManager.java +++ b/presto-main/src/main/java/com/facebook/presto/server/PluginManager.java @@ -21,7 +21,7 @@ import com.facebook.presto.security.AccessControlManager; import com.facebook.presto.server.security.PasswordAuthenticatorManager; import com.facebook.presto.spi.Plugin; -import com.facebook.presto.spi.block.BlockEncodingFactory; +import com.facebook.presto.spi.block.BlockEncoding; import com.facebook.presto.spi.classloader.ThreadContextClassLoader; import com.facebook.presto.spi.connector.ConnectorFactory; import com.facebook.presto.spi.eventlistener.EventListenerFactory; @@ -175,9 +175,9 @@ private void loadPlugin(URLClassLoader pluginClassLoader) public void installPlugin(Plugin plugin) { - for (BlockEncodingFactory blockEncodingFactory : plugin.getBlockEncodingFactories(blockEncodingManager)) { - log.info("Registering block encoding %s", blockEncodingFactory.getName()); - blockEncodingManager.addBlockEncodingFactory(blockEncodingFactory); + for (BlockEncoding blockEncoding : plugin.getBlockEncodings()) { + log.info("Registering block encoding %s", blockEncoding.getName()); + blockEncodingManager.addBlockEncoding(blockEncoding); } for (Type type : plugin.getTypes()) { diff --git a/presto-main/src/main/java/com/facebook/presto/server/ServerMainModule.java b/presto-main/src/main/java/com/facebook/presto/server/ServerMainModule.java index ada40e7a3a73..91024aab371a 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/ServerMainModule.java +++ b/presto-main/src/main/java/com/facebook/presto/server/ServerMainModule.java @@ -104,7 +104,7 @@ import com.facebook.presto.spi.PageIndexerFactory; import com.facebook.presto.spi.PageSorter; import com.facebook.presto.spi.block.Block; -import com.facebook.presto.spi.block.BlockEncodingFactory; +import com.facebook.presto.spi.block.BlockEncoding; import com.facebook.presto.spi.block.BlockEncodingSerde; import com.facebook.presto.spi.type.Type; import com.facebook.presto.spi.type.TypeManager; @@ -472,7 +472,7 @@ protected void setup(Binder binder) // block encodings binder.bind(BlockEncodingManager.class).in(Scopes.SINGLETON); binder.bind(BlockEncodingSerde.class).to(BlockEncodingManager.class).in(Scopes.SINGLETON); - newSetBinder(binder, new TypeLiteral>() {}); + newSetBinder(binder, BlockEncoding.class); jsonBinder(binder).addSerializerBinding(Block.class).to(BlockJsonSerde.Serializer.class); jsonBinder(binder).addDeserializerBinding(Block.class).to(BlockJsonSerde.Deserializer.class); diff --git a/presto-main/src/test/java/com/facebook/presto/block/AbstractTestBlock.java b/presto-main/src/test/java/com/facebook/presto/block/AbstractTestBlock.java index 94b5a5fc5457..8599d5765fd8 100644 --- a/presto-main/src/test/java/com/facebook/presto/block/AbstractTestBlock.java +++ b/presto-main/src/test/java/com/facebook/presto/block/AbstractTestBlock.java @@ -13,11 +13,15 @@ */ package com.facebook.presto.block; +import com.facebook.presto.metadata.FunctionRegistry; import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.block.BlockBuilder; import com.facebook.presto.spi.block.BlockBuilderStatus; -import com.facebook.presto.spi.block.BlockEncoding; +import com.facebook.presto.spi.block.BlockEncodingSerde; import com.facebook.presto.spi.block.DictionaryId; +import com.facebook.presto.spi.type.TypeManager; +import com.facebook.presto.sql.analyzer.FeaturesConfig; +import com.facebook.presto.type.TypeRegistry; import com.google.common.collect.ImmutableList; import io.airlift.slice.DynamicSliceOutput; import io.airlift.slice.Slice; @@ -51,6 +55,13 @@ @Test public abstract class AbstractTestBlock { + private static final TypeManager TYPE_MANAGER = new TypeRegistry(); + private static final BlockEncodingSerde BLOCK_ENCODING_SERDE = new BlockEncodingManager(TYPE_MANAGER); + static { + // associate TYPE_MANAGER with a function registry + new FunctionRegistry(TYPE_MANAGER, new BlockEncodingManager(TYPE_MANAGER), new FeaturesConfig()); + } + protected void assertBlock(Block block, Supplier newBlockBuilder, T[] expectedValues) { assertBlockPositions(block, newBlockBuilder, expectedValues); @@ -363,9 +374,8 @@ protected boolean isSliceAccessSupported() private static Block copyBlockViaBlockSerde(Block block) { DynamicSliceOutput sliceOutput = new DynamicSliceOutput(1024); - BlockEncoding blockEncoding = block.getEncoding(); - blockEncoding.writeBlock(sliceOutput, block); - return blockEncoding.readBlock(sliceOutput.slice().getInput()); + BLOCK_ENCODING_SERDE.writeBlock(sliceOutput, block); + return BLOCK_ENCODING_SERDE.readBlock(sliceOutput.slice().getInput()); } private static Block copyBlockViaWritePositionTo(Block block, Supplier newBlockBuilder) diff --git a/presto-main/src/test/java/com/facebook/presto/block/ColumnarTestUtils.java b/presto-main/src/test/java/com/facebook/presto/block/ColumnarTestUtils.java index a5bbb30ab9ab..db9ffc80a784 100644 --- a/presto-main/src/test/java/com/facebook/presto/block/ColumnarTestUtils.java +++ b/presto-main/src/test/java/com/facebook/presto/block/ColumnarTestUtils.java @@ -13,10 +13,14 @@ */ package com.facebook.presto.block; +import com.facebook.presto.metadata.FunctionRegistry; import com.facebook.presto.spi.block.Block; -import com.facebook.presto.spi.block.BlockEncoding; +import com.facebook.presto.spi.block.BlockEncodingSerde; import com.facebook.presto.spi.block.DictionaryBlock; import com.facebook.presto.spi.block.RunLengthEncodedBlock; +import com.facebook.presto.spi.type.TypeManager; +import com.facebook.presto.sql.analyzer.FeaturesConfig; +import com.facebook.presto.type.TypeRegistry; import io.airlift.slice.DynamicSliceOutput; import io.airlift.slice.Slice; @@ -29,6 +33,13 @@ final class ColumnarTestUtils { + private static final TypeManager TYPE_MANAGER = new TypeRegistry(); + private static final BlockEncodingSerde BLOCK_ENCODING_SERDE = new BlockEncodingManager(TYPE_MANAGER); + static { + // associate TYPE_MANAGER with a function registry + new FunctionRegistry(TYPE_MANAGER, new BlockEncodingManager(TYPE_MANAGER), new FeaturesConfig()); + } + private ColumnarTestUtils() {} public static void assertBlock(Block block, T[] expectedValues) @@ -109,9 +120,8 @@ public static T[] alternatingNullValues(T[] objects) private static Block copyBlock(Block block) { DynamicSliceOutput sliceOutput = new DynamicSliceOutput(1024); - BlockEncoding blockEncoding = block.getEncoding(); - blockEncoding.writeBlock(sliceOutput, block); - return blockEncoding.readBlock(sliceOutput.slice().getInput()); + BLOCK_ENCODING_SERDE.writeBlock(sliceOutput, block); + return BLOCK_ENCODING_SERDE.readBlock(sliceOutput.slice().getInput()); } public static DictionaryBlock createTestDictionaryBlock(Block block) diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/Plugin.java b/presto-spi/src/main/java/com/facebook/presto/spi/Plugin.java index fd30f176abb4..c87c27736113 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/Plugin.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/Plugin.java @@ -13,8 +13,7 @@ */ package com.facebook.presto.spi; -import com.facebook.presto.spi.block.BlockEncodingFactory; -import com.facebook.presto.spi.block.BlockEncodingSerde; +import com.facebook.presto.spi.block.BlockEncoding; import com.facebook.presto.spi.connector.ConnectorFactory; import com.facebook.presto.spi.eventlistener.EventListenerFactory; import com.facebook.presto.spi.resourceGroups.ResourceGroupConfigurationManagerFactory; @@ -36,7 +35,7 @@ default Iterable getConnectorFactories() return emptyList(); } - default Iterable> getBlockEncodingFactories(BlockEncodingSerde serde) + default Iterable getBlockEncodings() { return emptyList(); } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/AbstractArrayBlock.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/AbstractArrayBlock.java index 4929557c7d4c..6fedb364f352 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/AbstractArrayBlock.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/AbstractArrayBlock.java @@ -36,9 +36,9 @@ int getOffset(int position) } @Override - public BlockEncoding getEncoding() + public String getEncodingName() { - return new ArrayBlockEncoding(getValues().getEncoding()); + return ArrayBlockEncoding.NAME; } @Override diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/AbstractFixedWidthBlock.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/AbstractFixedWidthBlock.java index a98dc68808f7..1997533a761e 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/AbstractFixedWidthBlock.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/AbstractFixedWidthBlock.java @@ -145,9 +145,9 @@ public void writePositionTo(int position, BlockBuilder blockBuilder) } @Override - public BlockEncoding getEncoding() + public String getEncodingName() { - return new FixedWidthBlockEncoding(fixedSize); + return FixedWidthBlockEncoding.NAME; } @Override diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/AbstractMapBlock.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/AbstractMapBlock.java index 1247fd6dd465..3c16b32ec3b9 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/AbstractMapBlock.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/AbstractMapBlock.java @@ -71,9 +71,9 @@ int getOffset(int position) } @Override - public BlockEncoding getEncoding() + public String getEncodingName() { - return new MapBlockEncoding(keyType, keyBlockNativeEquals, keyNativeHashCode, getKeys().getEncoding(), getValues().getEncoding()); + return MapBlockEncoding.NAME; } @Override diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/AbstractRowBlock.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/AbstractRowBlock.java index 76414b0c38db..51b98723736f 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/AbstractRowBlock.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/AbstractRowBlock.java @@ -48,13 +48,9 @@ protected AbstractRowBlock(int numFields) } @Override - public RowBlockEncoding getEncoding() + public String getEncodingName() { - BlockEncoding[] fieldBlockEncodings = new BlockEncoding[numFields]; - for (int i = 0; i < numFields; i++) { - fieldBlockEncodings[i] = getFieldBlocks()[i].getEncoding(); - } - return new RowBlockEncoding(fieldBlockEncodings); + return RowBlockEncoding.NAME; } @Override diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/AbstractSingleArrayBlock.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/AbstractSingleArrayBlock.java index 69351eb2f99c..1de03a0d3eb0 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/AbstractSingleArrayBlock.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/AbstractSingleArrayBlock.java @@ -147,7 +147,7 @@ public boolean isNull(int position) } @Override - public BlockEncoding getEncoding() + public String getEncodingName() { // SingleArrayBlockEncoding does not exist throw new UnsupportedOperationException(); diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/AbstractVariableWidthBlock.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/AbstractVariableWidthBlock.java index f864d1217680..9728348e8b71 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/AbstractVariableWidthBlock.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/AbstractVariableWidthBlock.java @@ -30,9 +30,9 @@ public abstract class AbstractVariableWidthBlock protected abstract boolean isEntryNull(int position); @Override - public BlockEncoding getEncoding() + public String getEncodingName() { - return new VariableWidthBlockEncoding(); + return VariableWidthBlockEncoding.NAME; } @Override diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/ArrayBlockEncoding.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/ArrayBlockEncoding.java index 2ed8e736c242..097f94f884cf 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/ArrayBlockEncoding.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/ArrayBlockEncoding.java @@ -22,14 +22,7 @@ public class ArrayBlockEncoding implements BlockEncoding { - private static final String NAME = "ARRAY"; - - private final BlockEncoding valueBlockEncoding; - - public ArrayBlockEncoding(BlockEncoding valueBlockEncoding) - { - this.valueBlockEncoding = valueBlockEncoding; - } + public static final String NAME = "ARRAY"; @Override public String getName() @@ -38,7 +31,7 @@ public String getName() } @Override - public void writeBlock(SliceOutput sliceOutput, Block block) + public void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput sliceOutput, Block block) { AbstractArrayBlock arrayBlock = (AbstractArrayBlock) block; @@ -50,7 +43,7 @@ public void writeBlock(SliceOutput sliceOutput, Block block) int valuesStartOffset = offsets[offsetBase]; int valuesEndOffset = offsets[offsetBase + positionCount]; Block values = arrayBlock.getValues().getRegion(valuesStartOffset, valuesEndOffset - valuesStartOffset); - valueBlockEncoding.writeBlock(sliceOutput, values); + blockEncodingSerde.writeBlock(sliceOutput, values); sliceOutput.appendInt(positionCount); for (int position = 0; position < positionCount + 1; position++) { @@ -60,9 +53,9 @@ public void writeBlock(SliceOutput sliceOutput, Block block) } @Override - public Block readBlock(SliceInput sliceInput) + public Block readBlock(BlockEncodingSerde blockEncodingSerde, SliceInput sliceInput) { - Block values = valueBlockEncoding.readBlock(sliceInput); + Block values = blockEncodingSerde.readBlock(sliceInput); int positionCount = sliceInput.readInt(); int[] offsets = new int[positionCount + 1]; @@ -70,27 +63,4 @@ public Block readBlock(SliceInput sliceInput) boolean[] valueIsNull = EncoderUtil.decodeNullBits(sliceInput, positionCount); return createArrayBlockInternal(0, positionCount, valueIsNull, offsets, values); } - - public static class ArrayBlockEncodingFactory - implements BlockEncodingFactory - { - @Override - public String getName() - { - return NAME; - } - - @Override - public ArrayBlockEncoding readEncoding(BlockEncodingSerde serde, SliceInput input) - { - BlockEncoding valueBlockEncoding = serde.readBlockEncoding(input); - return new ArrayBlockEncoding(valueBlockEncoding); - } - - @Override - public void writeEncoding(BlockEncodingSerde serde, SliceOutput output, ArrayBlockEncoding blockEncoding) - { - serde.writeBlockEncoding(output, blockEncoding.valueBlockEncoding); - } - } } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/Block.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/Block.java index 494ae0ebb66f..71b39da9fb4e 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/Block.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/Block.java @@ -192,7 +192,7 @@ default int compareTo(int leftPosition, int leftOffset, int leftLength, Block ri /** * Get the encoding for this block. */ - BlockEncoding getEncoding(); + String getEncodingName(); /** * Create a new block from the current block by keeping the same elements diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/BlockEncoding.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/BlockEncoding.java index bb0441e62e39..d3c4b5c3f337 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/BlockEncoding.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/BlockEncoding.java @@ -29,12 +29,12 @@ public interface BlockEncoding * Read a block from the specified input. The returned * block should begin at the specified position. */ - Block readBlock(SliceInput input); + Block readBlock(BlockEncodingSerde blockEncodingSerde, SliceInput input); /** * Write the specified block to the specified output */ - void writeBlock(SliceOutput sliceOutput, Block block); + void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput sliceOutput, Block block); /** * This method allows the implementor to specify a replacement object that will be serialized instead of the original one. diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/BlockEncodingFactory.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/BlockEncodingFactory.java deleted file mode 100644 index b3b6699ef418..000000000000 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/BlockEncodingFactory.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.facebook.presto.spi.block; - -import io.airlift.slice.SliceInput; -import io.airlift.slice.SliceOutput; - -public interface BlockEncodingFactory -{ - /** - * Gets the unique name of this encoding. - */ - String getName(); - - /** - * Reads the encoding from the specified input. - */ - T readEncoding(BlockEncodingSerde serde, SliceInput input); - - /** - * Writes this encoding to the output stream. - */ - void writeEncoding(BlockEncodingSerde serde, SliceOutput output, T blockEncoding); -} diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/BlockEncodingSerde.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/BlockEncodingSerde.java index 0c8442a225dd..bcf001d52efc 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/BlockEncodingSerde.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/BlockEncodingSerde.java @@ -21,10 +21,10 @@ public interface BlockEncodingSerde /** * Read a block encoding from the input. */ - BlockEncoding readBlockEncoding(SliceInput input); + Block readBlock(SliceInput input); /** * Write a blockEncoding to the output. */ - void writeBlockEncoding(SliceOutput output, BlockEncoding encoding); + void writeBlock(SliceOutput output, Block block); } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/ByteArrayBlock.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/ByteArrayBlock.java index ad5362d815dd..ff8b2f9fba8f 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/ByteArrayBlock.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/ByteArrayBlock.java @@ -172,9 +172,9 @@ public Block copyRegion(int positionOffset, int length) } @Override - public BlockEncoding getEncoding() + public String getEncodingName() { - return new ByteArrayBlockEncoding(); + return ByteArrayBlockEncoding.NAME; } @Override diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/ByteArrayBlockBuilder.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/ByteArrayBlockBuilder.java index c13e43814b43..e04bb042efe4 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/ByteArrayBlockBuilder.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/ByteArrayBlockBuilder.java @@ -227,9 +227,9 @@ public Block copyRegion(int positionOffset, int length) } @Override - public BlockEncoding getEncoding() + public String getEncodingName() { - return new ByteArrayBlockEncoding(); + return ByteArrayBlockEncoding.NAME; } @Override diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/ByteArrayBlockEncoding.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/ByteArrayBlockEncoding.java index 27ad2e625bb6..8612e88e8803 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/ByteArrayBlockEncoding.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/ByteArrayBlockEncoding.java @@ -22,7 +22,7 @@ public class ByteArrayBlockEncoding implements BlockEncoding { - private static final String NAME = "BYTE_ARRAY"; + public static final String NAME = "BYTE_ARRAY"; @Override public String getName() @@ -31,7 +31,7 @@ public String getName() } @Override - public void writeBlock(SliceOutput sliceOutput, Block block) + public void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput sliceOutput, Block block) { int positionCount = block.getPositionCount(); sliceOutput.appendInt(positionCount); @@ -46,7 +46,7 @@ public void writeBlock(SliceOutput sliceOutput, Block block) } @Override - public Block readBlock(SliceInput sliceInput) + public Block readBlock(BlockEncodingSerde blockEncodingSerde, SliceInput sliceInput) { int positionCount = sliceInput.readInt(); @@ -61,25 +61,4 @@ public Block readBlock(SliceInput sliceInput) return new ByteArrayBlock(positionCount, valueIsNull, values); } - - public static class ByteArrayBlockEncodingFactory - implements BlockEncodingFactory - { - @Override - public String getName() - { - return NAME; - } - - @Override - public ByteArrayBlockEncoding readEncoding(BlockEncodingSerde serde, SliceInput input) - { - return new ByteArrayBlockEncoding(); - } - - @Override - public void writeEncoding(BlockEncodingSerde serde, SliceOutput output, ByteArrayBlockEncoding blockEncoding) - { - } - } } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/DictionaryBlock.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/DictionaryBlock.java index 43d876cec4fc..edd8a5ba3e44 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/DictionaryBlock.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/DictionaryBlock.java @@ -258,9 +258,9 @@ public void retainedBytesForEachPart(BiConsumer consumer) } @Override - public BlockEncoding getEncoding() + public String getEncodingName() { - return new DictionaryBlockEncoding(dictionary.getEncoding()); + return DictionaryBlockEncoding.NAME; } @Override diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/DictionaryBlockEncoding.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/DictionaryBlockEncoding.java index d4e0133e33bc..8c0b2a52c755 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/DictionaryBlockEncoding.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/DictionaryBlockEncoding.java @@ -17,18 +17,10 @@ import io.airlift.slice.SliceOutput; import io.airlift.slice.Slices; -import static java.util.Objects.requireNonNull; - public class DictionaryBlockEncoding implements BlockEncoding { - private static final String NAME = "DICTIONARY"; - private final BlockEncoding dictionaryEncoding; - - public DictionaryBlockEncoding(BlockEncoding dictionaryEncoding) - { - this.dictionaryEncoding = requireNonNull(dictionaryEncoding, "dictionaryEncoding is null"); - } + public static final String NAME = "DICTIONARY"; @Override public String getName() @@ -37,7 +29,7 @@ public String getName() } @Override - public void writeBlock(SliceOutput sliceOutput, Block block) + public void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput sliceOutput, Block block) { // The down casts here are safe because it is the block itself the provides this encoding implementation. DictionaryBlock dictionaryBlock = (DictionaryBlock) block; @@ -50,7 +42,7 @@ public void writeBlock(SliceOutput sliceOutput, Block block) // dictionary Block dictionary = dictionaryBlock.getDictionary(); - dictionaryEncoding.writeBlock(sliceOutput, dictionary); + blockEncodingSerde.writeBlock(sliceOutput, dictionary); // ids sliceOutput.writeBytes(dictionaryBlock.getIds()); @@ -62,13 +54,13 @@ public void writeBlock(SliceOutput sliceOutput, Block block) } @Override - public Block readBlock(SliceInput sliceInput) + public Block readBlock(BlockEncodingSerde blockEncodingSerde, SliceInput sliceInput) { // positionCount int positionCount = sliceInput.readInt(); // dictionary - Block dictionaryBlock = dictionaryEncoding.readBlock(sliceInput); + Block dictionaryBlock = blockEncodingSerde.readBlock(sliceInput); // ids int[] ids = new int[positionCount]; @@ -84,32 +76,4 @@ public Block readBlock(SliceInput sliceInput) // TODO: fix DictionaryBlock so that dictionaryIsCompacted can be set to true when the underlying block over-retains memory. return new DictionaryBlock(positionCount, dictionaryBlock, ids, false, new DictionaryId(mostSignificantBits, leastSignificantBits, sequenceId)); } - - public BlockEncoding getDictionaryEncoding() - { - return dictionaryEncoding; - } - - public static class DictionaryBlockEncodingFactory - implements BlockEncodingFactory - { - @Override - public String getName() - { - return NAME; - } - - @Override - public DictionaryBlockEncoding readEncoding(BlockEncodingSerde serde, SliceInput input) - { - BlockEncoding dictionaryEncoding = serde.readBlockEncoding(input); - return new DictionaryBlockEncoding(dictionaryEncoding); - } - - @Override - public void writeEncoding(BlockEncodingSerde serde, SliceOutput output, DictionaryBlockEncoding blockEncoding) - { - serde.writeBlockEncoding(output, blockEncoding.getDictionaryEncoding()); - } - } } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/FixedWidthBlockEncoding.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/FixedWidthBlockEncoding.java index bc229346ad06..fd33787f6007 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/FixedWidthBlockEncoding.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/FixedWidthBlockEncoding.java @@ -24,16 +24,7 @@ public class FixedWidthBlockEncoding implements BlockEncoding { - private static final String NAME = "FIXED_WIDTH"; - private final int fixedSize; - - public FixedWidthBlockEncoding(int fixedSize) - { - if (fixedSize < 0) { - throw new IllegalArgumentException("fixedSize is negative"); - } - this.fixedSize = fixedSize; - } + public static final String NAME = "FIXED_WIDTH"; @Override public String getName() @@ -41,18 +32,13 @@ public String getName() return NAME; } - public int getFixedSize() - { - return fixedSize; - } - @Override - public void writeBlock(SliceOutput sliceOutput, Block block) + public void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput sliceOutput, Block block) { AbstractFixedWidthBlock fixedWidthBlock = (AbstractFixedWidthBlock) block; - int positionCount = fixedWidthBlock.getPositionCount(); - sliceOutput.appendInt(positionCount); + sliceOutput.appendInt(fixedWidthBlock.fixedSize); + sliceOutput.appendInt(fixedWidthBlock.getPositionCount()); // write null bits 8 at a time encodeNullsAsBits(sliceOutput, fixedWidthBlock); @@ -64,8 +50,9 @@ public void writeBlock(SliceOutput sliceOutput, Block block) } @Override - public Block readBlock(SliceInput sliceInput) + public Block readBlock(BlockEncodingSerde blockEncodingSerde, SliceInput sliceInput) { + int fixedSize = sliceInput.readInt(); int positionCount = sliceInput.readInt(); boolean[] valueIsNull = decodeNullBits(sliceInput, positionCount); @@ -75,27 +62,4 @@ public Block readBlock(SliceInput sliceInput) return new FixedWidthBlock(fixedSize, positionCount, slice, Slices.wrappedBooleanArray(valueIsNull)); } - - public static class FixedWidthBlockEncodingFactory - implements BlockEncodingFactory - { - @Override - public String getName() - { - return NAME; - } - - @Override - public FixedWidthBlockEncoding readEncoding(BlockEncodingSerde serde, SliceInput input) - { - int entrySize = input.readInt(); - return new FixedWidthBlockEncoding(entrySize); - } - - @Override - public void writeEncoding(BlockEncodingSerde serde, SliceOutput output, FixedWidthBlockEncoding blockEncoding) - { - output.writeInt(blockEncoding.getFixedSize()); - } - } } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/IntArrayBlock.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/IntArrayBlock.java index a7d5f6e3ec98..b16a1dcb6464 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/IntArrayBlock.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/IntArrayBlock.java @@ -172,9 +172,9 @@ public Block copyRegion(int positionOffset, int length) } @Override - public BlockEncoding getEncoding() + public String getEncodingName() { - return new IntArrayBlockEncoding(); + return IntArrayBlockEncoding.NAME; } @Override diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/IntArrayBlockBuilder.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/IntArrayBlockBuilder.java index 44f25c1741f7..59e2bb9b79db 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/IntArrayBlockBuilder.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/IntArrayBlockBuilder.java @@ -228,9 +228,9 @@ public Block copyRegion(int positionOffset, int length) } @Override - public BlockEncoding getEncoding() + public String getEncodingName() { - return new IntArrayBlockEncoding(); + return IntArrayBlockEncoding.NAME; } @Override diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/IntArrayBlockEncoding.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/IntArrayBlockEncoding.java index 6d065a24d2dd..fb320493931b 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/IntArrayBlockEncoding.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/IntArrayBlockEncoding.java @@ -22,7 +22,7 @@ public class IntArrayBlockEncoding implements BlockEncoding { - private static final String NAME = "INT_ARRAY"; + public static final String NAME = "INT_ARRAY"; @Override public String getName() @@ -31,7 +31,7 @@ public String getName() } @Override - public void writeBlock(SliceOutput sliceOutput, Block block) + public void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput sliceOutput, Block block) { int positionCount = block.getPositionCount(); sliceOutput.appendInt(positionCount); @@ -46,7 +46,7 @@ public void writeBlock(SliceOutput sliceOutput, Block block) } @Override - public Block readBlock(SliceInput sliceInput) + public Block readBlock(BlockEncodingSerde blockEncodingSerde, SliceInput sliceInput) { int positionCount = sliceInput.readInt(); @@ -61,25 +61,4 @@ public Block readBlock(SliceInput sliceInput) return new IntArrayBlock(positionCount, valueIsNull, values); } - - public static class IntArrayBlockEncodingFactory - implements BlockEncodingFactory - { - @Override - public String getName() - { - return NAME; - } - - @Override - public IntArrayBlockEncoding readEncoding(BlockEncodingSerde serde, SliceInput input) - { - return new IntArrayBlockEncoding(); - } - - @Override - public void writeEncoding(BlockEncodingSerde serde, SliceOutput output, IntArrayBlockEncoding blockEncoding) - { - } - } } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/LazyBlock.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/LazyBlock.java index cb02145031f2..99a78e4d7c25 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/LazyBlock.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/LazyBlock.java @@ -193,10 +193,10 @@ public void retainedBytesForEachPart(BiConsumer consumer) } @Override - public BlockEncoding getEncoding() + public String getEncodingName() { assureLoaded(); - return new LazyBlockEncoding(); + return LazyBlockEncoding.NAME; } @Override diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/LazyBlockEncoding.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/LazyBlockEncoding.java index 958653c8316a..be2922ce9b5d 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/LazyBlockEncoding.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/LazyBlockEncoding.java @@ -21,7 +21,7 @@ public class LazyBlockEncoding implements BlockEncoding { - private static final String NAME = "LAZY"; + public static final String NAME = "LAZY"; public LazyBlockEncoding() {} @@ -32,16 +32,16 @@ public String getName() } @Override - public void writeBlock(SliceOutput sliceOutput, Block block) + public Block readBlock(BlockEncodingSerde blockEncodingSerde, SliceInput input) { - // We implemented replacementBlockForWrite, so we will never need to write a lazy block + // We write the actual underlying block, so we will never need to read a lazy block throw new UnsupportedOperationException(); } @Override - public Block readBlock(SliceInput sliceInput) + public void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput sliceOutput, Block block) { - // We write the actual underlying block, so we will never need to read a lazy block + // We implemented replacementBlockForWrite, so we will never need to write a lazy block throw new UnsupportedOperationException(); } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/LongArrayBlock.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/LongArrayBlock.java index f6c422634706..1dfac3e1a1c7 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/LongArrayBlock.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/LongArrayBlock.java @@ -219,9 +219,9 @@ public Block copyRegion(int positionOffset, int length) } @Override - public BlockEncoding getEncoding() + public String getEncodingName() { - return new LongArrayBlockEncoding(); + return LongArrayBlockEncoding.NAME; } @Override diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/LongArrayBlockBuilder.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/LongArrayBlockBuilder.java index b8d1d90e7af3..bad02a3b9850 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/LongArrayBlockBuilder.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/LongArrayBlockBuilder.java @@ -275,9 +275,9 @@ public Block copyRegion(int positionOffset, int length) } @Override - public BlockEncoding getEncoding() + public String getEncodingName() { - return new LongArrayBlockEncoding(); + return LongArrayBlockEncoding.NAME; } @Override diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/LongArrayBlockEncoding.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/LongArrayBlockEncoding.java index 772c424d7bca..35eeb7453f8b 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/LongArrayBlockEncoding.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/LongArrayBlockEncoding.java @@ -22,7 +22,7 @@ public class LongArrayBlockEncoding implements BlockEncoding { - private static final String NAME = "LONG_ARRAY"; + public static final String NAME = "LONG_ARRAY"; @Override public String getName() @@ -31,7 +31,7 @@ public String getName() } @Override - public void writeBlock(SliceOutput sliceOutput, Block block) + public void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput sliceOutput, Block block) { int positionCount = block.getPositionCount(); sliceOutput.appendInt(positionCount); @@ -46,7 +46,7 @@ public void writeBlock(SliceOutput sliceOutput, Block block) } @Override - public Block readBlock(SliceInput sliceInput) + public Block readBlock(BlockEncodingSerde blockEncodingSerde, SliceInput sliceInput) { int positionCount = sliceInput.readInt(); @@ -61,25 +61,4 @@ public Block readBlock(SliceInput sliceInput) return new LongArrayBlock(positionCount, valueIsNull, values); } - - public static class LongArrayBlockEncodingFactory - implements BlockEncodingFactory - { - @Override - public String getName() - { - return NAME; - } - - @Override - public LongArrayBlockEncoding readEncoding(BlockEncodingSerde serde, SliceInput input) - { - return new LongArrayBlockEncoding(); - } - - @Override - public void writeEncoding(BlockEncodingSerde serde, SliceOutput output, LongArrayBlockEncoding blockEncoding) - { - } - } } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/MapBlockEncoding.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/MapBlockEncoding.java index f83c226aa9a9..34968c2b6845 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/MapBlockEncoding.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/MapBlockEncoding.java @@ -31,28 +31,17 @@ import static java.lang.String.format; import static java.util.Arrays.asList; import static java.util.Collections.singletonList; -import static java.util.Objects.requireNonNull; public class MapBlockEncoding implements BlockEncoding { - private static final String NAME = "MAP"; + public static final String NAME = "MAP"; - private final Type keyType; - private final MethodHandle keyNativeHashCode; - private final MethodHandle keyBlockNativeEquals; - private final BlockEncoding keyBlockEncoding; - private final BlockEncoding valueBlockEncoding; + private final TypeManager typeManager; - public MapBlockEncoding(Type keyType, MethodHandle keyBlockNativeEquals, MethodHandle keyNativeHashCode, BlockEncoding keyBlockEncoding, BlockEncoding valueBlockEncoding) + public MapBlockEncoding(TypeManager typeManager) { - this.keyType = requireNonNull(keyType, "keyType is null"); - // keyNativeHashCode can only be null due to map block kill switch. deprecated.new-map-block - this.keyNativeHashCode = keyNativeHashCode; - // keyBlockNativeEquals can only be null due to map block kill switch. deprecated.new-map-block - this.keyBlockNativeEquals = keyBlockNativeEquals; - this.keyBlockEncoding = requireNonNull(keyBlockEncoding, "keyBlockEncoding is null"); - this.valueBlockEncoding = requireNonNull(valueBlockEncoding, "valueBlockEncoding is null"); + this.typeManager = typeManager; } @Override @@ -62,7 +51,7 @@ public String getName() } @Override - public void writeBlock(SliceOutput sliceOutput, Block block) + public void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput sliceOutput, Block block) { AbstractMapBlock mapBlock = (AbstractMapBlock) block; @@ -74,8 +63,11 @@ public void writeBlock(SliceOutput sliceOutput, Block block) int entriesStartOffset = offsets[offsetBase]; int entriesEndOffset = offsets[offsetBase + positionCount]; - keyBlockEncoding.writeBlock(sliceOutput, mapBlock.getKeys().getRegion(entriesStartOffset, entriesEndOffset - entriesStartOffset)); - valueBlockEncoding.writeBlock(sliceOutput, mapBlock.getValues().getRegion(entriesStartOffset, entriesEndOffset - entriesStartOffset)); + + TypeSerde.writeType(sliceOutput, mapBlock.keyType); + + blockEncodingSerde.writeBlock(sliceOutput, mapBlock.getKeys().getRegion(entriesStartOffset, entriesEndOffset - entriesStartOffset)); + blockEncodingSerde.writeBlock(sliceOutput, mapBlock.getValues().getRegion(entriesStartOffset, entriesEndOffset - entriesStartOffset)); sliceOutput.appendInt((entriesEndOffset - entriesStartOffset) * HASH_MULTIPLIER); sliceOutput.writeBytes(wrappedIntArray(hashTable, entriesStartOffset * HASH_MULTIPLIER, (entriesEndOffset - entriesStartOffset) * HASH_MULTIPLIER)); @@ -88,10 +80,15 @@ public void writeBlock(SliceOutput sliceOutput, Block block) } @Override - public Block readBlock(SliceInput sliceInput) + public Block readBlock(BlockEncodingSerde blockEncodingSerde, SliceInput sliceInput) { - Block keyBlock = keyBlockEncoding.readBlock(sliceInput); - Block valueBlock = valueBlockEncoding.readBlock(sliceInput); + Type keyType = TypeSerde.readType(typeManager, sliceInput); + MethodHandle keyNativeEquals = typeManager.resolveOperator(OperatorType.EQUAL, asList(keyType, keyType)); + MethodHandle keyBlockNativeEquals = compose(keyNativeEquals, nativeValueGetter(keyType)); + MethodHandle keyNativeHashCode = typeManager.resolveOperator(OperatorType.HASH_CODE, singletonList(keyType)); + + Block keyBlock = blockEncodingSerde.readBlock(sliceInput); + Block valueBlock = blockEncodingSerde.readBlock(sliceInput); int[] hashTable = new int[sliceInput.readInt()]; sliceInput.readBytes(wrappedIntArray(hashTable)); @@ -107,42 +104,4 @@ public Block readBlock(SliceInput sliceInput) boolean[] mapIsNull = EncoderUtil.decodeNullBits(sliceInput, positionCount); return createMapBlockInternal(0, positionCount, mapIsNull, offsets, keyBlock, valueBlock, hashTable, keyType, keyBlockNativeEquals, keyNativeHashCode); } - - public static class MapBlockEncodingFactory - implements BlockEncodingFactory - { - private final TypeManager typeManager; - - public MapBlockEncodingFactory(TypeManager typeManager) - { - this.typeManager = requireNonNull(typeManager, "typeManager is null"); - } - - @Override - public String getName() - { - return NAME; - } - - @Override - public MapBlockEncoding readEncoding(BlockEncodingSerde serde, SliceInput input) - { - Type keyType = TypeSerde.readType(typeManager, input); - MethodHandle keyNativeEquals = typeManager.resolveOperator(OperatorType.EQUAL, asList(keyType, keyType)); - MethodHandle keyBlockNativeEquals = compose(keyNativeEquals, nativeValueGetter(keyType)); - MethodHandle keyNativeHashCode = typeManager.resolveOperator(OperatorType.HASH_CODE, singletonList(keyType)); - - BlockEncoding keyBlockEncoding = serde.readBlockEncoding(input); - BlockEncoding valueBlockEncoding = serde.readBlockEncoding(input); - return new MapBlockEncoding(keyType, keyBlockNativeEquals, keyNativeHashCode, keyBlockEncoding, valueBlockEncoding); - } - - @Override - public void writeEncoding(BlockEncodingSerde serde, SliceOutput output, MapBlockEncoding blockEncoding) - { - TypeSerde.writeType(output, blockEncoding.keyType); - serde.writeBlockEncoding(output, blockEncoding.keyBlockEncoding); - serde.writeBlockEncoding(output, blockEncoding.valueBlockEncoding); - } - } } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/RowBlockEncoding.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/RowBlockEncoding.java index 1923b49b54bd..fb3c3401f0ea 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/RowBlockEncoding.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/RowBlockEncoding.java @@ -19,19 +19,11 @@ import static com.facebook.presto.spi.block.RowBlock.createRowBlockInternal; import static io.airlift.slice.Slices.wrappedIntArray; -import static java.util.Objects.requireNonNull; public class RowBlockEncoding implements BlockEncoding { - private static final String NAME = "ROW"; - - private final BlockEncoding[] fieldBlockEncodings; - - public RowBlockEncoding(BlockEncoding[] fieldBlockEncodings) - { - this.fieldBlockEncodings = requireNonNull(fieldBlockEncodings, "fieldBlockEncodings is null"); - } + public static final String NAME = "ROW"; @Override public String getName() @@ -40,24 +32,21 @@ public String getName() } @Override - public void writeBlock(SliceOutput sliceOutput, Block block) + public void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput sliceOutput, Block block) { AbstractRowBlock rowBlock = (AbstractRowBlock) block; - - if (rowBlock.numFields != fieldBlockEncodings.length) { - throw new IllegalArgumentException( - "argument block differs in length (" + rowBlock.numFields + ") with this encoding (" + fieldBlockEncodings.length + ")"); - } + int numFields = rowBlock.numFields; int positionCount = rowBlock.getPositionCount(); int offsetBase = rowBlock.getOffsetBase(); int[] fieldBlockOffsets = rowBlock.getFieldBlockOffsets(); - int startFieldBlockOffset = fieldBlockOffsets[offsetBase]; int endFieldBlockOffset = fieldBlockOffsets[offsetBase + positionCount]; - for (int i = 0; i < fieldBlockEncodings.length; i++) { - fieldBlockEncodings[i].writeBlock(sliceOutput, rowBlock.getFieldBlocks()[i].getRegion(startFieldBlockOffset, endFieldBlockOffset - startFieldBlockOffset)); + + sliceOutput.appendInt(numFields); + for (int i = 0; i < numFields; i++) { + blockEncodingSerde.writeBlock(sliceOutput, rowBlock.getFieldBlocks()[i].getRegion(startFieldBlockOffset, endFieldBlockOffset - startFieldBlockOffset)); } sliceOutput.appendInt(positionCount); @@ -68,11 +57,12 @@ public void writeBlock(SliceOutput sliceOutput, Block block) } @Override - public Block readBlock(SliceInput sliceInput) + public Block readBlock(BlockEncodingSerde blockEncodingSerde, SliceInput sliceInput) { - Block[] fieldBlocks = new Block[fieldBlockEncodings.length]; - for (int i = 0; i < fieldBlockEncodings.length; i++) { - fieldBlocks[i] = fieldBlockEncodings[i].readBlock(sliceInput); + int numFields = sliceInput.readInt(); + Block[] fieldBlocks = new Block[numFields]; + for (int i = 0; i < numFields; i++) { + fieldBlocks[i] = blockEncodingSerde.readBlock(sliceInput); } int positionCount = sliceInput.readInt(); @@ -81,34 +71,4 @@ public Block readBlock(SliceInput sliceInput) boolean[] rowIsNull = EncoderUtil.decodeNullBits(sliceInput, positionCount); return createRowBlockInternal(0, positionCount, rowIsNull, fieldBlockOffsets, fieldBlocks); } - - public static class RowBlockEncodingFactory - implements BlockEncodingFactory - { - @Override - public String getName() - { - return NAME; - } - - @Override - public RowBlockEncoding readEncoding(BlockEncodingSerde serde, SliceInput input) - { - int numFields = input.readInt(); - BlockEncoding[] fieldBlockEncodings = new BlockEncoding[numFields]; - for (int i = 0; i < numFields; i++) { - fieldBlockEncodings[i] = serde.readBlockEncoding(input); - } - return new RowBlockEncoding(fieldBlockEncodings); - } - - @Override - public void writeEncoding(BlockEncodingSerde serde, SliceOutput output, RowBlockEncoding blockEncoding) - { - output.appendInt(blockEncoding.fieldBlockEncodings.length); - for (BlockEncoding fieldBlockEncoding : blockEncoding.fieldBlockEncodings) { - serde.writeBlockEncoding(output, fieldBlockEncoding); - } - } - } } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/RunLengthBlockEncoding.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/RunLengthBlockEncoding.java index 4ec843c992d8..32b87bb22bcf 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/RunLengthBlockEncoding.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/RunLengthBlockEncoding.java @@ -16,19 +16,10 @@ import io.airlift.slice.SliceInput; import io.airlift.slice.SliceOutput; -import static java.util.Objects.requireNonNull; - public class RunLengthBlockEncoding implements BlockEncoding { - private static final String NAME = "RLE"; - - private final BlockEncoding valueBlockEncoding; - - public RunLengthBlockEncoding(BlockEncoding valueBlockEncoding) - { - this.valueBlockEncoding = requireNonNull(valueBlockEncoding, "valueBlockEncoding is null"); - } + public static final String NAME = "RLE"; @Override public String getName() @@ -36,13 +27,8 @@ public String getName() return NAME; } - public BlockEncoding getValueBlockEncoding() - { - return valueBlockEncoding; - } - @Override - public void writeBlock(SliceOutput sliceOutput, Block block) + public void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput sliceOutput, Block block) { RunLengthEncodedBlock rleBlock = (RunLengthEncodedBlock) block; @@ -50,41 +36,18 @@ public void writeBlock(SliceOutput sliceOutput, Block block) sliceOutput.writeInt(rleBlock.getPositionCount()); // write the value - getValueBlockEncoding().writeBlock(sliceOutput, rleBlock.getValue()); + blockEncodingSerde.writeBlock(sliceOutput, rleBlock.getValue()); } @Override - public RunLengthEncodedBlock readBlock(SliceInput sliceInput) + public RunLengthEncodedBlock readBlock(BlockEncodingSerde blockEncodingSerde, SliceInput sliceInput) { // read the run length int positionCount = sliceInput.readInt(); // read the value - Block value = getValueBlockEncoding().readBlock(sliceInput); + Block value = blockEncodingSerde.readBlock(sliceInput); return new RunLengthEncodedBlock(value, positionCount); } - - public static class RunLengthBlockEncodingFactory - implements BlockEncodingFactory - { - @Override - public String getName() - { - return NAME; - } - - @Override - public RunLengthBlockEncoding readEncoding(BlockEncodingSerde serde, SliceInput input) - { - BlockEncoding valueBlockEncoding = serde.readBlockEncoding(input); - return new RunLengthBlockEncoding(valueBlockEncoding); - } - - @Override - public void writeEncoding(BlockEncodingSerde serde, SliceOutput output, RunLengthBlockEncoding blockEncoding) - { - serde.writeBlockEncoding(output, blockEncoding.getValueBlockEncoding()); - } - } } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/RunLengthEncodedBlock.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/RunLengthEncodedBlock.java index c971b7439274..6f053e238b09 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/RunLengthEncodedBlock.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/RunLengthEncodedBlock.java @@ -91,9 +91,9 @@ public void retainedBytesForEachPart(BiConsumer consumer) } @Override - public RunLengthBlockEncoding getEncoding() + public String getEncodingName() { - return new RunLengthBlockEncoding(value.getEncoding()); + return RunLengthBlockEncoding.NAME; } @Override diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/ShortArrayBlock.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/ShortArrayBlock.java index c297ece19100..79abdf672755 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/ShortArrayBlock.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/ShortArrayBlock.java @@ -172,9 +172,9 @@ public Block copyRegion(int positionOffset, int length) } @Override - public BlockEncoding getEncoding() + public String getEncodingName() { - return new ShortArrayBlockEncoding(); + return ShortArrayBlockEncoding.NAME; } @Override diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/ShortArrayBlockBuilder.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/ShortArrayBlockBuilder.java index 8608a9d4e8b3..91ba02b78e1f 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/ShortArrayBlockBuilder.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/ShortArrayBlockBuilder.java @@ -228,9 +228,9 @@ public Block copyRegion(int positionOffset, int length) } @Override - public BlockEncoding getEncoding() + public String getEncodingName() { - return new ShortArrayBlockEncoding(); + return ShortArrayBlockEncoding.NAME; } @Override diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/ShortArrayBlockEncoding.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/ShortArrayBlockEncoding.java index 35426307a881..671cc87cb9ff 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/ShortArrayBlockEncoding.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/ShortArrayBlockEncoding.java @@ -22,7 +22,7 @@ public class ShortArrayBlockEncoding implements BlockEncoding { - private static final String NAME = "SHORT_ARRAY"; + public static final String NAME = "SHORT_ARRAY"; @Override public String getName() @@ -31,7 +31,7 @@ public String getName() } @Override - public void writeBlock(SliceOutput sliceOutput, Block block) + public void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput sliceOutput, Block block) { int positionCount = block.getPositionCount(); sliceOutput.appendInt(positionCount); @@ -46,7 +46,7 @@ public void writeBlock(SliceOutput sliceOutput, Block block) } @Override - public Block readBlock(SliceInput sliceInput) + public Block readBlock(BlockEncodingSerde blockEncodingSerde, SliceInput sliceInput) { int positionCount = sliceInput.readInt(); @@ -61,25 +61,4 @@ public Block readBlock(SliceInput sliceInput) return new ShortArrayBlock(positionCount, valueIsNull, values); } - - public static class ShortArrayBlockEncodingFactory - implements BlockEncodingFactory - { - @Override - public String getName() - { - return NAME; - } - - @Override - public ShortArrayBlockEncoding readEncoding(BlockEncodingSerde serde, SliceInput input) - { - return new ShortArrayBlockEncoding(); - } - - @Override - public void writeEncoding(BlockEncodingSerde serde, SliceOutput output, ShortArrayBlockEncoding blockEncoding) - { - } - } } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/SingleMapBlock.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/SingleMapBlock.java index fad5e33f4cef..afdb560d9eac 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/SingleMapBlock.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/SingleMapBlock.java @@ -38,7 +38,7 @@ public class SingleMapBlock private final Block keyBlock; private final Block valueBlock; private final int[] hashTable; - private final Type keyType; + final Type keyType; private final MethodHandle keyNativeHashCode; private final MethodHandle keyBlockNativeEquals; @@ -84,9 +84,9 @@ public void retainedBytesForEachPart(BiConsumer consumer) } @Override - public BlockEncoding getEncoding() + public String getEncodingName() { - return new SingleMapBlockEncoding(keyType, keyNativeHashCode, keyBlockNativeEquals, keyBlock.getEncoding(), valueBlock.getEncoding()); + return SingleMapBlockEncoding.NAME; } @Override diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/SingleMapBlockEncoding.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/SingleMapBlockEncoding.java index cbd736a5f52a..338a9380d017 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/SingleMapBlockEncoding.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/SingleMapBlockEncoding.java @@ -35,23 +35,13 @@ public class SingleMapBlockEncoding implements BlockEncoding { - private static final String NAME = "MAP_ELEMENT"; + public static final String NAME = "MAP_ELEMENT"; - private final Type keyType; - private final MethodHandle keyNativeHashCode; - private final MethodHandle keyBlockNativeEquals; - private final BlockEncoding keyBlockEncoding; - private final BlockEncoding valueBlockEncoding; + private final TypeManager typeManager; - public SingleMapBlockEncoding(Type keyType, MethodHandle keyNativeHashCode, MethodHandle keyBlockNativeEquals, BlockEncoding keyBlockEncoding, BlockEncoding valueBlockEncoding) + public SingleMapBlockEncoding(TypeManager typeManager) { - this.keyType = requireNonNull(keyType, "keyType is null"); - // keyNativeHashCode can only be null due to map block kill switch. deprecated.new-map-block - this.keyNativeHashCode = keyNativeHashCode; - // keyBlockNativeEquals can only be null due to map block kill switch. deprecated.new-map-block - this.keyBlockNativeEquals = keyBlockNativeEquals; - this.keyBlockEncoding = requireNonNull(keyBlockEncoding, "keyBlockEncoding is null"); - this.valueBlockEncoding = requireNonNull(valueBlockEncoding, "valueBlockEncoding is null"); + this.typeManager = requireNonNull(typeManager, "typeManager is null"); } @Override @@ -61,23 +51,30 @@ public String getName() } @Override - public void writeBlock(SliceOutput sliceOutput, Block block) + public void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput sliceOutput, Block block) { SingleMapBlock singleMapBlock = (SingleMapBlock) block; + TypeSerde.writeType(sliceOutput, singleMapBlock.keyType); + int offset = singleMapBlock.getOffset(); int positionCount = singleMapBlock.getPositionCount(); - keyBlockEncoding.writeBlock(sliceOutput, singleMapBlock.getKeyBlock().getRegion(offset / 2, positionCount / 2)); - valueBlockEncoding.writeBlock(sliceOutput, singleMapBlock.getValueBlock().getRegion(offset / 2, positionCount / 2)); + blockEncodingSerde.writeBlock(sliceOutput, singleMapBlock.getKeyBlock().getRegion(offset / 2, positionCount / 2)); + blockEncodingSerde.writeBlock(sliceOutput, singleMapBlock.getValueBlock().getRegion(offset / 2, positionCount / 2)); int[] hashTable = singleMapBlock.getHashTable(); sliceOutput.appendInt(positionCount / 2 * HASH_MULTIPLIER); sliceOutput.writeBytes(wrappedIntArray(hashTable, offset / 2 * HASH_MULTIPLIER, positionCount / 2 * HASH_MULTIPLIER)); } @Override - public Block readBlock(SliceInput sliceInput) + public Block readBlock(BlockEncodingSerde blockEncodingSerde, SliceInput sliceInput) { - Block keyBlock = keyBlockEncoding.readBlock(sliceInput); - Block valueBlock = valueBlockEncoding.readBlock(sliceInput); + Type keyType = TypeSerde.readType(typeManager, sliceInput); + MethodHandle keyNativeEquals = typeManager.resolveOperator(OperatorType.EQUAL, asList(keyType, keyType)); + MethodHandle keyBlockNativeEquals = compose(keyNativeEquals, nativeValueGetter(keyType)); + MethodHandle keyNativeHashCode = typeManager.resolveOperator(OperatorType.HASH_CODE, singletonList(keyType)); + + Block keyBlock = blockEncodingSerde.readBlock(sliceInput); + Block valueBlock = blockEncodingSerde.readBlock(sliceInput); int[] hashTable = new int[sliceInput.readInt()]; sliceInput.readBytes(wrappedIntArray(hashTable)); @@ -89,42 +86,4 @@ public Block readBlock(SliceInput sliceInput) return new SingleMapBlock(0, keyBlock.getPositionCount() * 2, keyBlock, valueBlock, hashTable, keyType, keyNativeHashCode, keyBlockNativeEquals); } - - public static class SingleMapBlockEncodingFactory - implements BlockEncodingFactory - { - private final TypeManager typeManager; - - public SingleMapBlockEncodingFactory(TypeManager typeManager) - { - this.typeManager = requireNonNull(typeManager, "typeManager is null"); - } - - @Override - public String getName() - { - return NAME; - } - - @Override - public SingleMapBlockEncoding readEncoding(BlockEncodingSerde serde, SliceInput input) - { - Type keyType = TypeSerde.readType(typeManager, input); - MethodHandle keyNativeHashCode = typeManager.resolveOperator(OperatorType.HASH_CODE, singletonList(keyType)); - MethodHandle keyNativeEquals = typeManager.resolveOperator(OperatorType.EQUAL, asList(keyType, keyType)); - MethodHandle keyBlockNativeEquals = compose(keyNativeEquals, nativeValueGetter(keyType)); - - BlockEncoding keyBlockEncoding = serde.readBlockEncoding(input); - BlockEncoding valueBlockEncoding = serde.readBlockEncoding(input); - return new SingleMapBlockEncoding(keyType, keyNativeHashCode, keyBlockNativeEquals, keyBlockEncoding, valueBlockEncoding); - } - - @Override - public void writeEncoding(BlockEncodingSerde serde, SliceOutput output, SingleMapBlockEncoding blockEncoding) - { - TypeSerde.writeType(output, blockEncoding.keyType); - serde.writeBlockEncoding(output, blockEncoding.keyBlockEncoding); - serde.writeBlockEncoding(output, blockEncoding.valueBlockEncoding); - } - } } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/SingleMapBlockWriter.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/SingleMapBlockWriter.java index dc5d866d344d..f0b7adafcce5 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/SingleMapBlockWriter.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/SingleMapBlockWriter.java @@ -218,7 +218,7 @@ public int getPositionCount() } @Override - public BlockEncoding getEncoding() + public String getEncodingName() { throw new UnsupportedOperationException(); } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/SingleRowBlock.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/SingleRowBlock.java index 91b8d8596fcf..19af31ed8aab 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/SingleRowBlock.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/SingleRowBlock.java @@ -33,6 +33,11 @@ public class SingleRowBlock this.fieldBlocks = fieldBlocks; } + int getNumFields() + { + return fieldBlocks.length; + } + @Override protected Block getFieldBlock(int fieldIndex) { @@ -75,13 +80,9 @@ public void retainedBytesForEachPart(BiConsumer consumer) } @Override - public BlockEncoding getEncoding() + public String getEncodingName() { - BlockEncoding[] fieldBlockEncodings = new BlockEncoding[fieldBlocks.length]; - for (int i = 0; i < fieldBlocks.length; i++) { - fieldBlockEncodings[i] = fieldBlocks[i].getEncoding(); - } - return new SingleRowBlockEncoding(fieldBlockEncodings); + return SingleRowBlockEncoding.NAME; } public int getRowIndex() diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/SingleRowBlockEncoding.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/SingleRowBlockEncoding.java index 26bd13d19fe0..214c80a1a390 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/SingleRowBlockEncoding.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/SingleRowBlockEncoding.java @@ -17,19 +17,10 @@ import io.airlift.slice.SliceInput; import io.airlift.slice.SliceOutput; -import static java.util.Objects.requireNonNull; - public class SingleRowBlockEncoding implements BlockEncoding { - private static final String NAME = "ROW_ELEMENT"; - - private final BlockEncoding[] fieldBlockEncodings; - - public SingleRowBlockEncoding(BlockEncoding[] fieldBlockEncodings) - { - this.fieldBlockEncodings = requireNonNull(fieldBlockEncodings, "fieldBlockEncodings is null"); - } + public static final String NAME = "ROW_ELEMENT"; @Override public String getName() @@ -38,52 +29,25 @@ public String getName() } @Override - public void writeBlock(SliceOutput sliceOutput, Block block) + public void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput sliceOutput, Block block) { SingleRowBlock singleRowBlock = (SingleRowBlock) block; + int numFields = singleRowBlock.getNumFields(); int rowIndex = singleRowBlock.getRowIndex(); - for (int i = 0; i < fieldBlockEncodings.length; i++) { - fieldBlockEncodings[i].writeBlock(sliceOutput, singleRowBlock.getFieldBlock(i).getRegion(rowIndex, 1)); + sliceOutput.appendInt(numFields); + for (int i = 0; i < numFields; i++) { + blockEncodingSerde.writeBlock(sliceOutput, singleRowBlock.getFieldBlock(i).getRegion(rowIndex, 1)); } } @Override - public Block readBlock(SliceInput sliceInput) + public Block readBlock(BlockEncodingSerde blockEncodingSerde, SliceInput sliceInput) { - Block[] fieldBlocks = new Block[fieldBlockEncodings.length]; + int numFields = sliceInput.readInt(); + Block[] fieldBlocks = new Block[numFields]; for (int i = 0; i < fieldBlocks.length; i++) { - fieldBlocks[i] = fieldBlockEncodings[i].readBlock(sliceInput); + fieldBlocks[i] = blockEncodingSerde.readBlock(sliceInput); } return new SingleRowBlock(0, fieldBlocks); } - - public static class SingleRowBlockEncodingFactory - implements BlockEncodingFactory - { - @Override - public String getName() - { - return NAME; - } - - @Override - public SingleRowBlockEncoding readEncoding(BlockEncodingSerde serde, SliceInput input) - { - int numFields = input.readInt(); - BlockEncoding[] fieldBlockEncodings = new BlockEncoding[numFields]; - for (int i = 0; i < numFields; i++) { - fieldBlockEncodings[i] = serde.readBlockEncoding(input); - } - return new SingleRowBlockEncoding(fieldBlockEncodings); - } - - @Override - public void writeEncoding(BlockEncodingSerde serde, SliceOutput output, SingleRowBlockEncoding blockEncoding) - { - output.appendInt(blockEncoding.fieldBlockEncodings.length); - for (BlockEncoding fieldBlockEncoding : blockEncoding.fieldBlockEncodings) { - serde.writeBlockEncoding(output, fieldBlockEncoding); - } - } - } } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/SingleRowBlockWriter.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/SingleRowBlockWriter.java index bfcfaffc3968..60cb2d9df924 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/SingleRowBlockWriter.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/SingleRowBlockWriter.java @@ -196,7 +196,7 @@ public int getPositionCount() } @Override - public BlockEncoding getEncoding() + public String getEncodingName() { throw new UnsupportedOperationException(); } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/VariableWidthBlockEncoding.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/VariableWidthBlockEncoding.java index f01f5a734da9..dc81afe327b1 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/VariableWidthBlockEncoding.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/VariableWidthBlockEncoding.java @@ -25,7 +25,7 @@ public class VariableWidthBlockEncoding implements BlockEncoding { - private static final String NAME = "VARIABLE_WIDTH"; + public static final String NAME = "VARIABLE_WIDTH"; @Override public String getName() @@ -34,7 +34,7 @@ public String getName() } @Override - public void writeBlock(SliceOutput sliceOutput, Block block) + public void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput sliceOutput, Block block) { // The down casts here are safe because it is the block itself the provides this encoding implementation. AbstractVariableWidthBlock variableWidthBlock = (AbstractVariableWidthBlock) block; @@ -58,7 +58,7 @@ public void writeBlock(SliceOutput sliceOutput, Block block) } @Override - public Block readBlock(SliceInput sliceInput) + public Block readBlock(BlockEncodingSerde blockEncodingSerde, SliceInput sliceInput) { int positionCount = sliceInput.readInt(); @@ -72,25 +72,4 @@ public Block readBlock(SliceInput sliceInput) return new VariableWidthBlock(positionCount, slice, offsets, valueIsNull); } - - public static class VariableWidthBlockEncodingFactory - implements BlockEncodingFactory - { - @Override - public String getName() - { - return NAME; - } - - @Override - public VariableWidthBlockEncoding readEncoding(BlockEncodingSerde serde, SliceInput input) - { - return new VariableWidthBlockEncoding(); - } - - @Override - public void writeEncoding(BlockEncodingSerde serde, SliceOutput output, VariableWidthBlockEncoding blockEncoding) - { - } - } } diff --git a/presto-spi/src/test/java/com/facebook/presto/spi/block/TestDictionaryBlockEncoding.java b/presto-spi/src/test/java/com/facebook/presto/spi/block/TestDictionaryBlockEncoding.java index 1619cb77a93a..34fa1c6cca19 100644 --- a/presto-spi/src/test/java/com/facebook/presto/spi/block/TestDictionaryBlockEncoding.java +++ b/presto-spi/src/test/java/com/facebook/presto/spi/block/TestDictionaryBlockEncoding.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.spi.block; +import com.facebook.presto.spi.type.TestingTypeManager; import com.facebook.presto.spi.type.Type; import io.airlift.slice.DynamicSliceOutput; import org.testng.annotations.Test; @@ -24,6 +25,8 @@ public class TestDictionaryBlockEncoding { + private final BlockEncodingSerde blockEncodingSerde = new TestingBlockEncodingSerde(new TestingTypeManager()); + @Test public void testRoundTrip() { @@ -43,12 +46,11 @@ public void testRoundTrip() ids[i] = i % 4; } - BlockEncoding blockEncoding = new DictionaryBlockEncoding(new VariableWidthBlockEncoding()); DictionaryBlock dictionaryBlock = new DictionaryBlock(dictionary, ids); DynamicSliceOutput sliceOutput = new DynamicSliceOutput(1024); - blockEncoding.writeBlock(sliceOutput, dictionaryBlock); - Block actualBlock = blockEncoding.readBlock(sliceOutput.slice().getInput()); + blockEncodingSerde.writeBlock(sliceOutput, dictionaryBlock); + Block actualBlock = blockEncodingSerde.readBlock(sliceOutput.slice().getInput()); assertTrue(actualBlock instanceof DictionaryBlock); DictionaryBlock actualDictionaryBlock = (DictionaryBlock) actualBlock; diff --git a/presto-spi/src/test/java/com/facebook/presto/spi/block/TestVariableWidthBlockEncoding.java b/presto-spi/src/test/java/com/facebook/presto/spi/block/TestVariableWidthBlockEncoding.java index 68380ab0ca55..6efc0f2a8da9 100644 --- a/presto-spi/src/test/java/com/facebook/presto/spi/block/TestVariableWidthBlockEncoding.java +++ b/presto-spi/src/test/java/com/facebook/presto/spi/block/TestVariableWidthBlockEncoding.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.spi.block; +import com.facebook.presto.spi.type.TestingTypeManager; import com.facebook.presto.spi.type.Type; import io.airlift.slice.DynamicSliceOutput; import org.testng.annotations.Test; @@ -23,6 +24,8 @@ public class TestVariableWidthBlockEncoding { + private final BlockEncodingSerde blockEncodingSerde = new TestingBlockEncodingSerde(new TestingTypeManager()); + @Test public void testRoundTrip() { @@ -34,9 +37,8 @@ public void testRoundTrip() Block expectedBlock = expectedBlockBuilder.build(); DynamicSliceOutput sliceOutput = new DynamicSliceOutput(1024); - BlockEncoding blockEncoding = new VariableWidthBlockEncoding(); - blockEncoding.writeBlock(sliceOutput, expectedBlock); - Block actualBlock = blockEncoding.readBlock(sliceOutput.slice().getInput()); + blockEncodingSerde.writeBlock(sliceOutput, expectedBlock); + Block actualBlock = blockEncodingSerde.readBlock(sliceOutput.slice().getInput()); assertBlockEquals(VARCHAR, actualBlock, expectedBlock); } diff --git a/presto-spi/src/test/java/com/facebook/presto/spi/block/TestingBlockEncodingSerde.java b/presto-spi/src/test/java/com/facebook/presto/spi/block/TestingBlockEncodingSerde.java index 4b6342c4e8a8..bd46375c6b6c 100644 --- a/presto-spi/src/test/java/com/facebook/presto/spi/block/TestingBlockEncodingSerde.java +++ b/presto-spi/src/test/java/com/facebook/presto/spi/block/TestingBlockEncodingSerde.java @@ -13,24 +13,12 @@ */ package com.facebook.presto.spi.block; -import com.facebook.presto.spi.block.ArrayBlockEncoding.ArrayBlockEncodingFactory; -import com.facebook.presto.spi.block.ByteArrayBlockEncoding.ByteArrayBlockEncodingFactory; -import com.facebook.presto.spi.block.DictionaryBlockEncoding.DictionaryBlockEncodingFactory; -import com.facebook.presto.spi.block.FixedWidthBlockEncoding.FixedWidthBlockEncodingFactory; -import com.facebook.presto.spi.block.IntArrayBlockEncoding.IntArrayBlockEncodingFactory; -import com.facebook.presto.spi.block.LongArrayBlockEncoding.LongArrayBlockEncodingFactory; -import com.facebook.presto.spi.block.MapBlockEncoding.MapBlockEncodingFactory; -import com.facebook.presto.spi.block.RowBlockEncoding.RowBlockEncodingFactory; -import com.facebook.presto.spi.block.RunLengthBlockEncoding.RunLengthBlockEncodingFactory; -import com.facebook.presto.spi.block.ShortArrayBlockEncoding.ShortArrayBlockEncodingFactory; -import com.facebook.presto.spi.block.SingleMapBlockEncoding.SingleMapBlockEncodingFactory; -import com.facebook.presto.spi.block.SingleRowBlockEncoding.SingleRowBlockEncodingFactory; -import com.facebook.presto.spi.block.VariableWidthBlockEncoding.VariableWidthBlockEncodingFactory; import com.facebook.presto.spi.type.TypeManager; import com.google.common.collect.ImmutableSet; import io.airlift.slice.SliceInput; import io.airlift.slice.SliceOutput; +import java.util.Optional; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -44,74 +32,86 @@ public final class TestingBlockEncodingSerde implements BlockEncodingSerde { - private final ConcurrentMap> blockEncodings = new ConcurrentHashMap<>(); + private final ConcurrentMap blockEncodings = new ConcurrentHashMap<>(); - public TestingBlockEncodingSerde(TypeManager typeManager, BlockEncodingFactory... blockEncodingFactories) + public TestingBlockEncodingSerde(TypeManager typeManager, BlockEncoding... blockEncodings) { - this(typeManager, ImmutableSet.copyOf(blockEncodingFactories)); + this(typeManager, ImmutableSet.copyOf(blockEncodings)); } - public TestingBlockEncodingSerde(TypeManager typeManager, Set> blockEncodingFactories) + public TestingBlockEncodingSerde(TypeManager typeManager, Set blockEncodings) { // This function should be called from Guice and tests only requireNonNull(typeManager, "typeManager is null"); - // always add the built-in BlockEncodingFactories - addBlockEncodingFactory(new VariableWidthBlockEncodingFactory()); - addBlockEncodingFactory(new FixedWidthBlockEncodingFactory()); - addBlockEncodingFactory(new ByteArrayBlockEncodingFactory()); - addBlockEncodingFactory(new ShortArrayBlockEncodingFactory()); - addBlockEncodingFactory(new IntArrayBlockEncodingFactory()); - addBlockEncodingFactory(new LongArrayBlockEncodingFactory()); - addBlockEncodingFactory(new DictionaryBlockEncodingFactory()); - addBlockEncodingFactory(new ArrayBlockEncodingFactory()); - addBlockEncodingFactory(new MapBlockEncodingFactory(typeManager)); - addBlockEncodingFactory(new SingleMapBlockEncodingFactory(typeManager)); - addBlockEncodingFactory(new RowBlockEncodingFactory()); - addBlockEncodingFactory(new SingleRowBlockEncodingFactory()); - addBlockEncodingFactory(new RunLengthBlockEncodingFactory()); - - for (BlockEncodingFactory factory : requireNonNull(blockEncodingFactories, "blockEncodingFactories is null")) { - addBlockEncodingFactory(factory); + // always add the built-in BlockEncodings + addBlockEncoding(new VariableWidthBlockEncoding()); + addBlockEncoding(new FixedWidthBlockEncoding()); + addBlockEncoding(new ByteArrayBlockEncoding()); + addBlockEncoding(new ShortArrayBlockEncoding()); + addBlockEncoding(new IntArrayBlockEncoding()); + addBlockEncoding(new LongArrayBlockEncoding()); + addBlockEncoding(new DictionaryBlockEncoding()); + addBlockEncoding(new ArrayBlockEncoding()); + addBlockEncoding(new MapBlockEncoding(typeManager)); + addBlockEncoding(new SingleMapBlockEncoding(typeManager)); + addBlockEncoding(new RowBlockEncoding()); + addBlockEncoding(new SingleRowBlockEncoding()); + addBlockEncoding(new RunLengthBlockEncoding()); + addBlockEncoding(new LazyBlockEncoding()); + + for (BlockEncoding blockEncoding : requireNonNull(blockEncodings, "blockEncodings is null")) { + addBlockEncoding(blockEncoding); } } - public void addBlockEncodingFactory(BlockEncodingFactory blockEncoding) + public void addBlockEncoding(BlockEncoding blockEncoding) { requireNonNull(blockEncoding, "blockEncoding is null"); - BlockEncodingFactory existingEntry = blockEncodings.putIfAbsent(blockEncoding.getName(), blockEncoding); + BlockEncoding existingEntry = blockEncodings.putIfAbsent(blockEncoding.getName(), blockEncoding); checkArgument(existingEntry == null, "Encoding %s is already registered", blockEncoding.getName()); } @Override - public BlockEncoding readBlockEncoding(SliceInput input) + public Block readBlock(SliceInput input) { // read the encoding name String encodingName = readLengthPrefixedString(input); // look up the encoding factory - BlockEncodingFactory blockEncoding = blockEncodings.get(encodingName); + BlockEncoding blockEncoding = blockEncodings.get(encodingName); checkArgument(blockEncoding != null, "Unknown block encoding %s", encodingName); // load read the encoding factory from the output stream - return blockEncoding.readEncoding(this, input); + return blockEncoding.readBlock(this, input); } @Override - public void writeBlockEncoding(SliceOutput output, BlockEncoding encoding) + public void writeBlock(SliceOutput output, Block block) { - // get the encoding name - String encodingName = encoding.getName(); + while (true) { + // get the encoding name + String encodingName = block.getEncodingName(); - // look up the encoding factory - BlockEncodingFactory blockEncoding = blockEncodings.get(encodingName); + // look up the encoding factory + BlockEncoding blockEncoding = blockEncodings.get(encodingName); + + // see if a replacement block should be written instead + Optional replacementBlock = blockEncoding.replacementBlockForWrite(block); + if (replacementBlock.isPresent()) { + block = replacementBlock.get(); + continue; + } - // write the name to the output - writeLengthPrefixedString(output, encodingName); + // write the name to the output + writeLengthPrefixedString(output, encodingName); - // write the encoding to the output - blockEncoding.writeEncoding(this, output, encoding); + // write the block to the output + blockEncoding.writeBlock(this, output, block); + + break; + } } private static String readLengthPrefixedString(SliceInput input) diff --git a/presto-spi/src/test/java/com/facebook/presto/spi/block/TestingBlockJsonSerde.java b/presto-spi/src/test/java/com/facebook/presto/spi/block/TestingBlockJsonSerde.java index 63fb14ba0f0f..77875b075088 100644 --- a/presto-spi/src/test/java/com/facebook/presto/spi/block/TestingBlockJsonSerde.java +++ b/presto-spi/src/test/java/com/facebook/presto/spi/block/TestingBlockJsonSerde.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.spi.block; +import com.facebook.presto.spi.type.TestingTypeManager; import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.databind.DeserializationContext; @@ -31,6 +32,8 @@ public final class TestingBlockJsonSerde { + private final BlockEncodingSerde blockEncodingSerde = new TestingBlockEncodingSerde(new TestingTypeManager()); + private TestingBlockJsonSerde() {} public static class Serializer @@ -48,9 +51,7 @@ public void serialize(Block block, JsonGenerator jsonGenerator, SerializerProvid throws IOException { SliceOutput output = new DynamicSliceOutput(64); - BlockEncoding encoding = block.getEncoding(); - blockEncodingSerde.writeBlockEncoding(output, encoding); - encoding.writeBlock(output, block); + blockEncodingSerde.writeBlock(output, block); String encoded = Base64.getEncoder().encodeToString(output.slice().getBytes()); jsonGenerator.writeString(encoded); } @@ -72,8 +73,7 @@ public Block deserialize(JsonParser jsonParser, DeserializationContext deseriali { byte[] decoded = Base64.getDecoder().decode(jsonParser.readValueAs(String.class)); BasicSliceInput input = Slices.wrappedBuffer(decoded).getInput(); - BlockEncoding blockEncoding = blockEncodingSerde.readBlockEncoding(input); - return blockEncoding.readBlock(input); + return blockEncodingSerde.readBlock(input); } } }