diff --git a/presto-main/src/main/java/com/facebook/presto/block/BlockEncodingManager.java b/presto-main/src/main/java/com/facebook/presto/block/BlockEncodingManager.java index 83708f05a178..5144e68abe0f 100644 --- a/presto-main/src/main/java/com/facebook/presto/block/BlockEncodingManager.java +++ b/presto-main/src/main/java/com/facebook/presto/block/BlockEncodingManager.java @@ -13,22 +13,23 @@ */ package com.facebook.presto.block; -import com.facebook.presto.spi.block.ArrayBlockEncoding.ArrayBlockEncodingFactory; +import com.facebook.presto.spi.block.ArrayBlockEncoding; +import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.block.BlockEncoding; -import com.facebook.presto.spi.block.BlockEncodingFactory; import com.facebook.presto.spi.block.BlockEncodingSerde; -import com.facebook.presto.spi.block.ByteArrayBlockEncoding.ByteArrayBlockEncodingFactory; -import com.facebook.presto.spi.block.DictionaryBlockEncoding.DictionaryBlockEncodingFactory; -import com.facebook.presto.spi.block.FixedWidthBlockEncoding.FixedWidthBlockEncodingFactory; -import com.facebook.presto.spi.block.IntArrayBlockEncoding.IntArrayBlockEncodingFactory; -import com.facebook.presto.spi.block.LongArrayBlockEncoding.LongArrayBlockEncodingFactory; -import com.facebook.presto.spi.block.MapBlockEncoding.MapBlockEncodingFactory; -import com.facebook.presto.spi.block.RowBlockEncoding.RowBlockEncodingFactory; -import com.facebook.presto.spi.block.RunLengthBlockEncoding.RunLengthBlockEncodingFactory; -import com.facebook.presto.spi.block.ShortArrayBlockEncoding.ShortArrayBlockEncodingFactory; -import com.facebook.presto.spi.block.SingleMapBlockEncoding.SingleMapBlockEncodingFactory; -import com.facebook.presto.spi.block.SingleRowBlockEncoding.SingleRowBlockEncodingFactory; -import com.facebook.presto.spi.block.VariableWidthBlockEncoding.VariableWidthBlockEncodingFactory; +import com.facebook.presto.spi.block.ByteArrayBlockEncoding; +import com.facebook.presto.spi.block.DictionaryBlockEncoding; +import com.facebook.presto.spi.block.FixedWidthBlockEncoding; +import com.facebook.presto.spi.block.IntArrayBlockEncoding; +import com.facebook.presto.spi.block.LazyBlockEncoding; +import com.facebook.presto.spi.block.LongArrayBlockEncoding; +import com.facebook.presto.spi.block.MapBlockEncoding; +import com.facebook.presto.spi.block.RowBlockEncoding; +import com.facebook.presto.spi.block.RunLengthBlockEncoding; +import com.facebook.presto.spi.block.ShortArrayBlockEncoding; +import com.facebook.presto.spi.block.SingleMapBlockEncoding; +import com.facebook.presto.spi.block.SingleRowBlockEncoding; +import com.facebook.presto.spi.block.VariableWidthBlockEncoding; import com.facebook.presto.spi.type.TypeManager; import com.google.common.collect.ImmutableSet; import io.airlift.slice.SliceInput; @@ -36,6 +37,7 @@ import javax.inject.Inject; +import java.util.Optional; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -47,73 +49,85 @@ public final class BlockEncodingManager implements BlockEncodingSerde { - private final ConcurrentMap> blockEncodings = new ConcurrentHashMap<>(); + private final ConcurrentMap blockEncodings = new ConcurrentHashMap<>(); - public BlockEncodingManager(TypeManager typeManager, BlockEncodingFactory... blockEncodingFactories) + public BlockEncodingManager(TypeManager typeManager, BlockEncoding... blockEncodings) { - this(typeManager, ImmutableSet.copyOf(blockEncodingFactories)); + this(typeManager, ImmutableSet.copyOf(blockEncodings)); } @Inject - public BlockEncodingManager(TypeManager typeManager, Set> blockEncodingFactories) + public BlockEncodingManager(TypeManager typeManager, Set blockEncodings) { // This function should be called from Guice and tests only - // always add the built-in BlockEncodingFactories - addBlockEncodingFactory(new VariableWidthBlockEncodingFactory()); - addBlockEncodingFactory(new FixedWidthBlockEncodingFactory()); - addBlockEncodingFactory(new ByteArrayBlockEncodingFactory()); - addBlockEncodingFactory(new ShortArrayBlockEncodingFactory()); - addBlockEncodingFactory(new IntArrayBlockEncodingFactory()); - addBlockEncodingFactory(new LongArrayBlockEncodingFactory()); - addBlockEncodingFactory(new DictionaryBlockEncodingFactory()); - addBlockEncodingFactory(new ArrayBlockEncodingFactory()); - addBlockEncodingFactory(new MapBlockEncodingFactory(typeManager)); - addBlockEncodingFactory(new SingleMapBlockEncodingFactory(typeManager)); - addBlockEncodingFactory(new RowBlockEncodingFactory()); - addBlockEncodingFactory(new SingleRowBlockEncodingFactory()); - addBlockEncodingFactory(new RunLengthBlockEncodingFactory()); - - for (BlockEncodingFactory factory : requireNonNull(blockEncodingFactories, "blockEncodingFactories is null")) { - addBlockEncodingFactory(factory); + // always add the built-in BlockEncodings + addBlockEncoding(new VariableWidthBlockEncoding()); + addBlockEncoding(new FixedWidthBlockEncoding()); + addBlockEncoding(new ByteArrayBlockEncoding()); + addBlockEncoding(new ShortArrayBlockEncoding()); + addBlockEncoding(new IntArrayBlockEncoding()); + addBlockEncoding(new LongArrayBlockEncoding()); + addBlockEncoding(new DictionaryBlockEncoding()); + addBlockEncoding(new ArrayBlockEncoding()); + addBlockEncoding(new MapBlockEncoding(typeManager)); + addBlockEncoding(new SingleMapBlockEncoding(typeManager)); + addBlockEncoding(new RowBlockEncoding()); + addBlockEncoding(new SingleRowBlockEncoding()); + addBlockEncoding(new RunLengthBlockEncoding()); + addBlockEncoding(new LazyBlockEncoding()); + + for (BlockEncoding blockEncoding : requireNonNull(blockEncodings, "blockEncodings is null")) { + addBlockEncoding(blockEncoding); } } - public void addBlockEncodingFactory(BlockEncodingFactory blockEncoding) + public void addBlockEncoding(BlockEncoding blockEncoding) { requireNonNull(blockEncoding, "blockEncoding is null"); - BlockEncodingFactory existingEntry = blockEncodings.putIfAbsent(blockEncoding.getName(), blockEncoding); + BlockEncoding existingEntry = blockEncodings.putIfAbsent(blockEncoding.getName(), blockEncoding); checkArgument(existingEntry == null, "Encoding %s is already registered", blockEncoding.getName()); } @Override - public BlockEncoding readBlockEncoding(SliceInput input) + public Block readBlock(SliceInput input) { // read the encoding name String encodingName = readLengthPrefixedString(input); // look up the encoding factory - BlockEncodingFactory blockEncoding = blockEncodings.get(encodingName); + BlockEncoding blockEncoding = blockEncodings.get(encodingName); checkArgument(blockEncoding != null, "Unknown block encoding %s", encodingName); // load read the encoding factory from the output stream - return blockEncoding.readEncoding(this, input); + return blockEncoding.readBlock(this, input); } @Override - public void writeBlockEncoding(SliceOutput output, BlockEncoding encoding) + public void writeBlock(SliceOutput output, Block block) { - // get the encoding name - String encodingName = encoding.getName(); + while (true) { + // get the encoding name + String encodingName = block.getEncodingName(); - // look up the encoding factory - BlockEncodingFactory blockEncoding = blockEncodings.get(encodingName); + // look up the BlockEncoding + BlockEncoding blockEncoding = blockEncodings.get(encodingName); + + // see if a replacement block should be written instead + Optional replacementBlock = blockEncoding.replacementBlockForWrite(block); + if (replacementBlock.isPresent()) { + block = replacementBlock.get(); + continue; + } - // write the name to the output - writeLengthPrefixedString(output, encodingName); + // write the name to the output + writeLengthPrefixedString(output, encodingName); - // write the encoding to the output - blockEncoding.writeEncoding(this, output, encoding); + // write the block to the output + blockEncoding.writeBlock(this, output, block); + + break; + } } private static String readLengthPrefixedString(SliceInput input) diff --git a/presto-main/src/main/java/com/facebook/presto/block/BlockSerdeUtil.java b/presto-main/src/main/java/com/facebook/presto/block/BlockSerdeUtil.java index c3bf7cb304d0..05240fc640ee 100644 --- a/presto-main/src/main/java/com/facebook/presto/block/BlockSerdeUtil.java +++ b/presto-main/src/main/java/com/facebook/presto/block/BlockSerdeUtil.java @@ -14,14 +14,12 @@ package com.facebook.presto.block; import com.facebook.presto.spi.block.Block; -import com.facebook.presto.spi.block.BlockEncoding; import com.facebook.presto.spi.block.BlockEncodingSerde; import io.airlift.slice.Slice; import io.airlift.slice.SliceInput; import io.airlift.slice.SliceOutput; import java.lang.invoke.MethodHandle; -import java.util.Optional; import static com.facebook.presto.util.Reflection.methodHandle; @@ -40,26 +38,11 @@ public static Block readBlock(BlockEncodingSerde blockEncodingSerde, Slice slice public static Block readBlock(BlockEncodingSerde blockEncodingSerde, SliceInput input) { - BlockEncoding blockEncoding = blockEncodingSerde.readBlockEncoding(input); - return blockEncoding.readBlock(input); + return blockEncodingSerde.readBlock(input); } public static void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput output, Block block) { - while (true) { - // TODO: respect replacementBlockForWrite for Blocks nested in other Block - // A proposed simplified design for block encoding decoding will address this to-do item. - - BlockEncoding encoding = block.getEncoding(); - Optional replaceBlock = encoding.replacementBlockForWrite(block); - if (!replaceBlock.isPresent()) { - break; - } - block = replaceBlock.get(); - } - - BlockEncoding encoding = block.getEncoding(); - blockEncodingSerde.writeBlockEncoding(output, encoding); - encoding.writeBlock(output, block); + blockEncodingSerde.writeBlock(output, block); } } diff --git a/presto-main/src/main/java/com/facebook/presto/operator/GroupByIdBlock.java b/presto-main/src/main/java/com/facebook/presto/operator/GroupByIdBlock.java index 37351033874a..c8993466b33c 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/GroupByIdBlock.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/GroupByIdBlock.java @@ -15,7 +15,6 @@ import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.block.BlockBuilder; -import com.facebook.presto.spi.block.BlockEncoding; import io.airlift.slice.Slice; import org.openjdk.jol.info.ClassLayout; @@ -190,9 +189,9 @@ public void retainedBytesForEachPart(BiConsumer consumer) } @Override - public BlockEncoding getEncoding() + public String getEncodingName() { - return block.getEncoding(); + throw new UnsupportedOperationException("GroupByIdBlock does not support serialization"); } @Override diff --git a/presto-main/src/main/java/com/facebook/presto/server/PluginManager.java b/presto-main/src/main/java/com/facebook/presto/server/PluginManager.java index f867a1bf0857..f2a7948bf40d 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/PluginManager.java +++ b/presto-main/src/main/java/com/facebook/presto/server/PluginManager.java @@ -21,7 +21,7 @@ import com.facebook.presto.security.AccessControlManager; import com.facebook.presto.server.security.PasswordAuthenticatorManager; import com.facebook.presto.spi.Plugin; -import com.facebook.presto.spi.block.BlockEncodingFactory; +import com.facebook.presto.spi.block.BlockEncoding; import com.facebook.presto.spi.classloader.ThreadContextClassLoader; import com.facebook.presto.spi.connector.ConnectorFactory; import com.facebook.presto.spi.eventlistener.EventListenerFactory; @@ -175,9 +175,9 @@ private void loadPlugin(URLClassLoader pluginClassLoader) public void installPlugin(Plugin plugin) { - for (BlockEncodingFactory blockEncodingFactory : plugin.getBlockEncodingFactories(blockEncodingManager)) { - log.info("Registering block encoding %s", blockEncodingFactory.getName()); - blockEncodingManager.addBlockEncodingFactory(blockEncodingFactory); + for (BlockEncoding blockEncoding : plugin.getBlockEncodings()) { + log.info("Registering block encoding %s", blockEncoding.getName()); + blockEncodingManager.addBlockEncoding(blockEncoding); } for (Type type : plugin.getTypes()) { diff --git a/presto-main/src/main/java/com/facebook/presto/server/ServerMainModule.java b/presto-main/src/main/java/com/facebook/presto/server/ServerMainModule.java index ada40e7a3a73..91024aab371a 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/ServerMainModule.java +++ b/presto-main/src/main/java/com/facebook/presto/server/ServerMainModule.java @@ -104,7 +104,7 @@ import com.facebook.presto.spi.PageIndexerFactory; import com.facebook.presto.spi.PageSorter; import com.facebook.presto.spi.block.Block; -import com.facebook.presto.spi.block.BlockEncodingFactory; +import com.facebook.presto.spi.block.BlockEncoding; import com.facebook.presto.spi.block.BlockEncodingSerde; import com.facebook.presto.spi.type.Type; import com.facebook.presto.spi.type.TypeManager; @@ -472,7 +472,7 @@ protected void setup(Binder binder) // block encodings binder.bind(BlockEncodingManager.class).in(Scopes.SINGLETON); binder.bind(BlockEncodingSerde.class).to(BlockEncodingManager.class).in(Scopes.SINGLETON); - newSetBinder(binder, new TypeLiteral>() {}); + newSetBinder(binder, BlockEncoding.class); jsonBinder(binder).addSerializerBinding(Block.class).to(BlockJsonSerde.Serializer.class); jsonBinder(binder).addDeserializerBinding(Block.class).to(BlockJsonSerde.Deserializer.class); diff --git a/presto-main/src/test/java/com/facebook/presto/block/AbstractTestBlock.java b/presto-main/src/test/java/com/facebook/presto/block/AbstractTestBlock.java index 94b5a5fc5457..8599d5765fd8 100644 --- a/presto-main/src/test/java/com/facebook/presto/block/AbstractTestBlock.java +++ b/presto-main/src/test/java/com/facebook/presto/block/AbstractTestBlock.java @@ -13,11 +13,15 @@ */ package com.facebook.presto.block; +import com.facebook.presto.metadata.FunctionRegistry; import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.block.BlockBuilder; import com.facebook.presto.spi.block.BlockBuilderStatus; -import com.facebook.presto.spi.block.BlockEncoding; +import com.facebook.presto.spi.block.BlockEncodingSerde; import com.facebook.presto.spi.block.DictionaryId; +import com.facebook.presto.spi.type.TypeManager; +import com.facebook.presto.sql.analyzer.FeaturesConfig; +import com.facebook.presto.type.TypeRegistry; import com.google.common.collect.ImmutableList; import io.airlift.slice.DynamicSliceOutput; import io.airlift.slice.Slice; @@ -51,6 +55,13 @@ @Test public abstract class AbstractTestBlock { + private static final TypeManager TYPE_MANAGER = new TypeRegistry(); + private static final BlockEncodingSerde BLOCK_ENCODING_SERDE = new BlockEncodingManager(TYPE_MANAGER); + static { + // associate TYPE_MANAGER with a function registry + new FunctionRegistry(TYPE_MANAGER, new BlockEncodingManager(TYPE_MANAGER), new FeaturesConfig()); + } + protected void assertBlock(Block block, Supplier newBlockBuilder, T[] expectedValues) { assertBlockPositions(block, newBlockBuilder, expectedValues); @@ -363,9 +374,8 @@ protected boolean isSliceAccessSupported() private static Block copyBlockViaBlockSerde(Block block) { DynamicSliceOutput sliceOutput = new DynamicSliceOutput(1024); - BlockEncoding blockEncoding = block.getEncoding(); - blockEncoding.writeBlock(sliceOutput, block); - return blockEncoding.readBlock(sliceOutput.slice().getInput()); + BLOCK_ENCODING_SERDE.writeBlock(sliceOutput, block); + return BLOCK_ENCODING_SERDE.readBlock(sliceOutput.slice().getInput()); } private static Block copyBlockViaWritePositionTo(Block block, Supplier newBlockBuilder) diff --git a/presto-main/src/test/java/com/facebook/presto/block/ColumnarTestUtils.java b/presto-main/src/test/java/com/facebook/presto/block/ColumnarTestUtils.java index a5bbb30ab9ab..db9ffc80a784 100644 --- a/presto-main/src/test/java/com/facebook/presto/block/ColumnarTestUtils.java +++ b/presto-main/src/test/java/com/facebook/presto/block/ColumnarTestUtils.java @@ -13,10 +13,14 @@ */ package com.facebook.presto.block; +import com.facebook.presto.metadata.FunctionRegistry; import com.facebook.presto.spi.block.Block; -import com.facebook.presto.spi.block.BlockEncoding; +import com.facebook.presto.spi.block.BlockEncodingSerde; import com.facebook.presto.spi.block.DictionaryBlock; import com.facebook.presto.spi.block.RunLengthEncodedBlock; +import com.facebook.presto.spi.type.TypeManager; +import com.facebook.presto.sql.analyzer.FeaturesConfig; +import com.facebook.presto.type.TypeRegistry; import io.airlift.slice.DynamicSliceOutput; import io.airlift.slice.Slice; @@ -29,6 +33,13 @@ final class ColumnarTestUtils { + private static final TypeManager TYPE_MANAGER = new TypeRegistry(); + private static final BlockEncodingSerde BLOCK_ENCODING_SERDE = new BlockEncodingManager(TYPE_MANAGER); + static { + // associate TYPE_MANAGER with a function registry + new FunctionRegistry(TYPE_MANAGER, new BlockEncodingManager(TYPE_MANAGER), new FeaturesConfig()); + } + private ColumnarTestUtils() {} public static void assertBlock(Block block, T[] expectedValues) @@ -109,9 +120,8 @@ public static T[] alternatingNullValues(T[] objects) private static Block copyBlock(Block block) { DynamicSliceOutput sliceOutput = new DynamicSliceOutput(1024); - BlockEncoding blockEncoding = block.getEncoding(); - blockEncoding.writeBlock(sliceOutput, block); - return blockEncoding.readBlock(sliceOutput.slice().getInput()); + BLOCK_ENCODING_SERDE.writeBlock(sliceOutput, block); + return BLOCK_ENCODING_SERDE.readBlock(sliceOutput.slice().getInput()); } public static DictionaryBlock createTestDictionaryBlock(Block block) diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/Plugin.java b/presto-spi/src/main/java/com/facebook/presto/spi/Plugin.java index fd30f176abb4..c87c27736113 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/Plugin.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/Plugin.java @@ -13,8 +13,7 @@ */ package com.facebook.presto.spi; -import com.facebook.presto.spi.block.BlockEncodingFactory; -import com.facebook.presto.spi.block.BlockEncodingSerde; +import com.facebook.presto.spi.block.BlockEncoding; import com.facebook.presto.spi.connector.ConnectorFactory; import com.facebook.presto.spi.eventlistener.EventListenerFactory; import com.facebook.presto.spi.resourceGroups.ResourceGroupConfigurationManagerFactory; @@ -36,7 +35,7 @@ default Iterable getConnectorFactories() return emptyList(); } - default Iterable> getBlockEncodingFactories(BlockEncodingSerde serde) + default Iterable getBlockEncodings() { return emptyList(); } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/AbstractArrayBlock.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/AbstractArrayBlock.java index 4929557c7d4c..6fedb364f352 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/AbstractArrayBlock.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/AbstractArrayBlock.java @@ -36,9 +36,9 @@ int getOffset(int position) } @Override - public BlockEncoding getEncoding() + public String getEncodingName() { - return new ArrayBlockEncoding(getValues().getEncoding()); + return ArrayBlockEncoding.NAME; } @Override diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/AbstractFixedWidthBlock.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/AbstractFixedWidthBlock.java index a98dc68808f7..1997533a761e 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/AbstractFixedWidthBlock.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/AbstractFixedWidthBlock.java @@ -145,9 +145,9 @@ public void writePositionTo(int position, BlockBuilder blockBuilder) } @Override - public BlockEncoding getEncoding() + public String getEncodingName() { - return new FixedWidthBlockEncoding(fixedSize); + return FixedWidthBlockEncoding.NAME; } @Override diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/AbstractMapBlock.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/AbstractMapBlock.java index 1247fd6dd465..3c16b32ec3b9 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/AbstractMapBlock.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/AbstractMapBlock.java @@ -71,9 +71,9 @@ int getOffset(int position) } @Override - public BlockEncoding getEncoding() + public String getEncodingName() { - return new MapBlockEncoding(keyType, keyBlockNativeEquals, keyNativeHashCode, getKeys().getEncoding(), getValues().getEncoding()); + return MapBlockEncoding.NAME; } @Override diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/AbstractRowBlock.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/AbstractRowBlock.java index 76414b0c38db..51b98723736f 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/AbstractRowBlock.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/AbstractRowBlock.java @@ -48,13 +48,9 @@ protected AbstractRowBlock(int numFields) } @Override - public RowBlockEncoding getEncoding() + public String getEncodingName() { - BlockEncoding[] fieldBlockEncodings = new BlockEncoding[numFields]; - for (int i = 0; i < numFields; i++) { - fieldBlockEncodings[i] = getFieldBlocks()[i].getEncoding(); - } - return new RowBlockEncoding(fieldBlockEncodings); + return RowBlockEncoding.NAME; } @Override diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/AbstractSingleArrayBlock.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/AbstractSingleArrayBlock.java index 69351eb2f99c..1de03a0d3eb0 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/AbstractSingleArrayBlock.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/AbstractSingleArrayBlock.java @@ -147,7 +147,7 @@ public boolean isNull(int position) } @Override - public BlockEncoding getEncoding() + public String getEncodingName() { // SingleArrayBlockEncoding does not exist throw new UnsupportedOperationException(); diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/AbstractVariableWidthBlock.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/AbstractVariableWidthBlock.java index f864d1217680..9728348e8b71 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/AbstractVariableWidthBlock.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/AbstractVariableWidthBlock.java @@ -30,9 +30,9 @@ public abstract class AbstractVariableWidthBlock protected abstract boolean isEntryNull(int position); @Override - public BlockEncoding getEncoding() + public String getEncodingName() { - return new VariableWidthBlockEncoding(); + return VariableWidthBlockEncoding.NAME; } @Override diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/ArrayBlockEncoding.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/ArrayBlockEncoding.java index 2ed8e736c242..097f94f884cf 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/ArrayBlockEncoding.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/ArrayBlockEncoding.java @@ -22,14 +22,7 @@ public class ArrayBlockEncoding implements BlockEncoding { - private static final String NAME = "ARRAY"; - - private final BlockEncoding valueBlockEncoding; - - public ArrayBlockEncoding(BlockEncoding valueBlockEncoding) - { - this.valueBlockEncoding = valueBlockEncoding; - } + public static final String NAME = "ARRAY"; @Override public String getName() @@ -38,7 +31,7 @@ public String getName() } @Override - public void writeBlock(SliceOutput sliceOutput, Block block) + public void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput sliceOutput, Block block) { AbstractArrayBlock arrayBlock = (AbstractArrayBlock) block; @@ -50,7 +43,7 @@ public void writeBlock(SliceOutput sliceOutput, Block block) int valuesStartOffset = offsets[offsetBase]; int valuesEndOffset = offsets[offsetBase + positionCount]; Block values = arrayBlock.getValues().getRegion(valuesStartOffset, valuesEndOffset - valuesStartOffset); - valueBlockEncoding.writeBlock(sliceOutput, values); + blockEncodingSerde.writeBlock(sliceOutput, values); sliceOutput.appendInt(positionCount); for (int position = 0; position < positionCount + 1; position++) { @@ -60,9 +53,9 @@ public void writeBlock(SliceOutput sliceOutput, Block block) } @Override - public Block readBlock(SliceInput sliceInput) + public Block readBlock(BlockEncodingSerde blockEncodingSerde, SliceInput sliceInput) { - Block values = valueBlockEncoding.readBlock(sliceInput); + Block values = blockEncodingSerde.readBlock(sliceInput); int positionCount = sliceInput.readInt(); int[] offsets = new int[positionCount + 1]; @@ -70,27 +63,4 @@ public Block readBlock(SliceInput sliceInput) boolean[] valueIsNull = EncoderUtil.decodeNullBits(sliceInput, positionCount); return createArrayBlockInternal(0, positionCount, valueIsNull, offsets, values); } - - public static class ArrayBlockEncodingFactory - implements BlockEncodingFactory - { - @Override - public String getName() - { - return NAME; - } - - @Override - public ArrayBlockEncoding readEncoding(BlockEncodingSerde serde, SliceInput input) - { - BlockEncoding valueBlockEncoding = serde.readBlockEncoding(input); - return new ArrayBlockEncoding(valueBlockEncoding); - } - - @Override - public void writeEncoding(BlockEncodingSerde serde, SliceOutput output, ArrayBlockEncoding blockEncoding) - { - serde.writeBlockEncoding(output, blockEncoding.valueBlockEncoding); - } - } } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/Block.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/Block.java index 494ae0ebb66f..71b39da9fb4e 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/Block.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/Block.java @@ -192,7 +192,7 @@ default int compareTo(int leftPosition, int leftOffset, int leftLength, Block ri /** * Get the encoding for this block. */ - BlockEncoding getEncoding(); + String getEncodingName(); /** * Create a new block from the current block by keeping the same elements diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/BlockEncoding.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/BlockEncoding.java index bb0441e62e39..d3c4b5c3f337 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/BlockEncoding.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/BlockEncoding.java @@ -29,12 +29,12 @@ public interface BlockEncoding * Read a block from the specified input. The returned * block should begin at the specified position. */ - Block readBlock(SliceInput input); + Block readBlock(BlockEncodingSerde blockEncodingSerde, SliceInput input); /** * Write the specified block to the specified output */ - void writeBlock(SliceOutput sliceOutput, Block block); + void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput sliceOutput, Block block); /** * This method allows the implementor to specify a replacement object that will be serialized instead of the original one. diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/BlockEncodingFactory.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/BlockEncodingFactory.java deleted file mode 100644 index b3b6699ef418..000000000000 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/BlockEncodingFactory.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.facebook.presto.spi.block; - -import io.airlift.slice.SliceInput; -import io.airlift.slice.SliceOutput; - -public interface BlockEncodingFactory -{ - /** - * Gets the unique name of this encoding. - */ - String getName(); - - /** - * Reads the encoding from the specified input. - */ - T readEncoding(BlockEncodingSerde serde, SliceInput input); - - /** - * Writes this encoding to the output stream. - */ - void writeEncoding(BlockEncodingSerde serde, SliceOutput output, T blockEncoding); -} diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/BlockEncodingSerde.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/BlockEncodingSerde.java index 0c8442a225dd..bcf001d52efc 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/BlockEncodingSerde.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/BlockEncodingSerde.java @@ -21,10 +21,10 @@ public interface BlockEncodingSerde /** * Read a block encoding from the input. */ - BlockEncoding readBlockEncoding(SliceInput input); + Block readBlock(SliceInput input); /** * Write a blockEncoding to the output. */ - void writeBlockEncoding(SliceOutput output, BlockEncoding encoding); + void writeBlock(SliceOutput output, Block block); } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/ByteArrayBlock.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/ByteArrayBlock.java index ad5362d815dd..ff8b2f9fba8f 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/ByteArrayBlock.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/ByteArrayBlock.java @@ -172,9 +172,9 @@ public Block copyRegion(int positionOffset, int length) } @Override - public BlockEncoding getEncoding() + public String getEncodingName() { - return new ByteArrayBlockEncoding(); + return ByteArrayBlockEncoding.NAME; } @Override diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/ByteArrayBlockBuilder.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/ByteArrayBlockBuilder.java index c13e43814b43..e04bb042efe4 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/ByteArrayBlockBuilder.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/ByteArrayBlockBuilder.java @@ -227,9 +227,9 @@ public Block copyRegion(int positionOffset, int length) } @Override - public BlockEncoding getEncoding() + public String getEncodingName() { - return new ByteArrayBlockEncoding(); + return ByteArrayBlockEncoding.NAME; } @Override diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/ByteArrayBlockEncoding.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/ByteArrayBlockEncoding.java index 27ad2e625bb6..8612e88e8803 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/ByteArrayBlockEncoding.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/ByteArrayBlockEncoding.java @@ -22,7 +22,7 @@ public class ByteArrayBlockEncoding implements BlockEncoding { - private static final String NAME = "BYTE_ARRAY"; + public static final String NAME = "BYTE_ARRAY"; @Override public String getName() @@ -31,7 +31,7 @@ public String getName() } @Override - public void writeBlock(SliceOutput sliceOutput, Block block) + public void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput sliceOutput, Block block) { int positionCount = block.getPositionCount(); sliceOutput.appendInt(positionCount); @@ -46,7 +46,7 @@ public void writeBlock(SliceOutput sliceOutput, Block block) } @Override - public Block readBlock(SliceInput sliceInput) + public Block readBlock(BlockEncodingSerde blockEncodingSerde, SliceInput sliceInput) { int positionCount = sliceInput.readInt(); @@ -61,25 +61,4 @@ public Block readBlock(SliceInput sliceInput) return new ByteArrayBlock(positionCount, valueIsNull, values); } - - public static class ByteArrayBlockEncodingFactory - implements BlockEncodingFactory - { - @Override - public String getName() - { - return NAME; - } - - @Override - public ByteArrayBlockEncoding readEncoding(BlockEncodingSerde serde, SliceInput input) - { - return new ByteArrayBlockEncoding(); - } - - @Override - public void writeEncoding(BlockEncodingSerde serde, SliceOutput output, ByteArrayBlockEncoding blockEncoding) - { - } - } } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/DictionaryBlock.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/DictionaryBlock.java index 43d876cec4fc..edd8a5ba3e44 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/DictionaryBlock.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/DictionaryBlock.java @@ -258,9 +258,9 @@ public void retainedBytesForEachPart(BiConsumer consumer) } @Override - public BlockEncoding getEncoding() + public String getEncodingName() { - return new DictionaryBlockEncoding(dictionary.getEncoding()); + return DictionaryBlockEncoding.NAME; } @Override diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/DictionaryBlockEncoding.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/DictionaryBlockEncoding.java index d4e0133e33bc..8c0b2a52c755 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/DictionaryBlockEncoding.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/DictionaryBlockEncoding.java @@ -17,18 +17,10 @@ import io.airlift.slice.SliceOutput; import io.airlift.slice.Slices; -import static java.util.Objects.requireNonNull; - public class DictionaryBlockEncoding implements BlockEncoding { - private static final String NAME = "DICTIONARY"; - private final BlockEncoding dictionaryEncoding; - - public DictionaryBlockEncoding(BlockEncoding dictionaryEncoding) - { - this.dictionaryEncoding = requireNonNull(dictionaryEncoding, "dictionaryEncoding is null"); - } + public static final String NAME = "DICTIONARY"; @Override public String getName() @@ -37,7 +29,7 @@ public String getName() } @Override - public void writeBlock(SliceOutput sliceOutput, Block block) + public void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput sliceOutput, Block block) { // The down casts here are safe because it is the block itself the provides this encoding implementation. DictionaryBlock dictionaryBlock = (DictionaryBlock) block; @@ -50,7 +42,7 @@ public void writeBlock(SliceOutput sliceOutput, Block block) // dictionary Block dictionary = dictionaryBlock.getDictionary(); - dictionaryEncoding.writeBlock(sliceOutput, dictionary); + blockEncodingSerde.writeBlock(sliceOutput, dictionary); // ids sliceOutput.writeBytes(dictionaryBlock.getIds()); @@ -62,13 +54,13 @@ public void writeBlock(SliceOutput sliceOutput, Block block) } @Override - public Block readBlock(SliceInput sliceInput) + public Block readBlock(BlockEncodingSerde blockEncodingSerde, SliceInput sliceInput) { // positionCount int positionCount = sliceInput.readInt(); // dictionary - Block dictionaryBlock = dictionaryEncoding.readBlock(sliceInput); + Block dictionaryBlock = blockEncodingSerde.readBlock(sliceInput); // ids int[] ids = new int[positionCount]; @@ -84,32 +76,4 @@ public Block readBlock(SliceInput sliceInput) // TODO: fix DictionaryBlock so that dictionaryIsCompacted can be set to true when the underlying block over-retains memory. return new DictionaryBlock(positionCount, dictionaryBlock, ids, false, new DictionaryId(mostSignificantBits, leastSignificantBits, sequenceId)); } - - public BlockEncoding getDictionaryEncoding() - { - return dictionaryEncoding; - } - - public static class DictionaryBlockEncodingFactory - implements BlockEncodingFactory - { - @Override - public String getName() - { - return NAME; - } - - @Override - public DictionaryBlockEncoding readEncoding(BlockEncodingSerde serde, SliceInput input) - { - BlockEncoding dictionaryEncoding = serde.readBlockEncoding(input); - return new DictionaryBlockEncoding(dictionaryEncoding); - } - - @Override - public void writeEncoding(BlockEncodingSerde serde, SliceOutput output, DictionaryBlockEncoding blockEncoding) - { - serde.writeBlockEncoding(output, blockEncoding.getDictionaryEncoding()); - } - } } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/FixedWidthBlockEncoding.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/FixedWidthBlockEncoding.java index bc229346ad06..fd33787f6007 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/FixedWidthBlockEncoding.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/FixedWidthBlockEncoding.java @@ -24,16 +24,7 @@ public class FixedWidthBlockEncoding implements BlockEncoding { - private static final String NAME = "FIXED_WIDTH"; - private final int fixedSize; - - public FixedWidthBlockEncoding(int fixedSize) - { - if (fixedSize < 0) { - throw new IllegalArgumentException("fixedSize is negative"); - } - this.fixedSize = fixedSize; - } + public static final String NAME = "FIXED_WIDTH"; @Override public String getName() @@ -41,18 +32,13 @@ public String getName() return NAME; } - public int getFixedSize() - { - return fixedSize; - } - @Override - public void writeBlock(SliceOutput sliceOutput, Block block) + public void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput sliceOutput, Block block) { AbstractFixedWidthBlock fixedWidthBlock = (AbstractFixedWidthBlock) block; - int positionCount = fixedWidthBlock.getPositionCount(); - sliceOutput.appendInt(positionCount); + sliceOutput.appendInt(fixedWidthBlock.fixedSize); + sliceOutput.appendInt(fixedWidthBlock.getPositionCount()); // write null bits 8 at a time encodeNullsAsBits(sliceOutput, fixedWidthBlock); @@ -64,8 +50,9 @@ public void writeBlock(SliceOutput sliceOutput, Block block) } @Override - public Block readBlock(SliceInput sliceInput) + public Block readBlock(BlockEncodingSerde blockEncodingSerde, SliceInput sliceInput) { + int fixedSize = sliceInput.readInt(); int positionCount = sliceInput.readInt(); boolean[] valueIsNull = decodeNullBits(sliceInput, positionCount); @@ -75,27 +62,4 @@ public Block readBlock(SliceInput sliceInput) return new FixedWidthBlock(fixedSize, positionCount, slice, Slices.wrappedBooleanArray(valueIsNull)); } - - public static class FixedWidthBlockEncodingFactory - implements BlockEncodingFactory - { - @Override - public String getName() - { - return NAME; - } - - @Override - public FixedWidthBlockEncoding readEncoding(BlockEncodingSerde serde, SliceInput input) - { - int entrySize = input.readInt(); - return new FixedWidthBlockEncoding(entrySize); - } - - @Override - public void writeEncoding(BlockEncodingSerde serde, SliceOutput output, FixedWidthBlockEncoding blockEncoding) - { - output.writeInt(blockEncoding.getFixedSize()); - } - } } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/IntArrayBlock.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/IntArrayBlock.java index a7d5f6e3ec98..b16a1dcb6464 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/IntArrayBlock.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/IntArrayBlock.java @@ -172,9 +172,9 @@ public Block copyRegion(int positionOffset, int length) } @Override - public BlockEncoding getEncoding() + public String getEncodingName() { - return new IntArrayBlockEncoding(); + return IntArrayBlockEncoding.NAME; } @Override diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/IntArrayBlockBuilder.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/IntArrayBlockBuilder.java index 44f25c1741f7..59e2bb9b79db 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/IntArrayBlockBuilder.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/IntArrayBlockBuilder.java @@ -228,9 +228,9 @@ public Block copyRegion(int positionOffset, int length) } @Override - public BlockEncoding getEncoding() + public String getEncodingName() { - return new IntArrayBlockEncoding(); + return IntArrayBlockEncoding.NAME; } @Override diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/IntArrayBlockEncoding.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/IntArrayBlockEncoding.java index 6d065a24d2dd..fb320493931b 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/IntArrayBlockEncoding.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/IntArrayBlockEncoding.java @@ -22,7 +22,7 @@ public class IntArrayBlockEncoding implements BlockEncoding { - private static final String NAME = "INT_ARRAY"; + public static final String NAME = "INT_ARRAY"; @Override public String getName() @@ -31,7 +31,7 @@ public String getName() } @Override - public void writeBlock(SliceOutput sliceOutput, Block block) + public void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput sliceOutput, Block block) { int positionCount = block.getPositionCount(); sliceOutput.appendInt(positionCount); @@ -46,7 +46,7 @@ public void writeBlock(SliceOutput sliceOutput, Block block) } @Override - public Block readBlock(SliceInput sliceInput) + public Block readBlock(BlockEncodingSerde blockEncodingSerde, SliceInput sliceInput) { int positionCount = sliceInput.readInt(); @@ -61,25 +61,4 @@ public Block readBlock(SliceInput sliceInput) return new IntArrayBlock(positionCount, valueIsNull, values); } - - public static class IntArrayBlockEncodingFactory - implements BlockEncodingFactory - { - @Override - public String getName() - { - return NAME; - } - - @Override - public IntArrayBlockEncoding readEncoding(BlockEncodingSerde serde, SliceInput input) - { - return new IntArrayBlockEncoding(); - } - - @Override - public void writeEncoding(BlockEncodingSerde serde, SliceOutput output, IntArrayBlockEncoding blockEncoding) - { - } - } } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/LazyBlock.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/LazyBlock.java index cb02145031f2..99a78e4d7c25 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/LazyBlock.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/LazyBlock.java @@ -193,10 +193,10 @@ public void retainedBytesForEachPart(BiConsumer consumer) } @Override - public BlockEncoding getEncoding() + public String getEncodingName() { assureLoaded(); - return new LazyBlockEncoding(); + return LazyBlockEncoding.NAME; } @Override diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/LazyBlockEncoding.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/LazyBlockEncoding.java index 958653c8316a..be2922ce9b5d 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/LazyBlockEncoding.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/LazyBlockEncoding.java @@ -21,7 +21,7 @@ public class LazyBlockEncoding implements BlockEncoding { - private static final String NAME = "LAZY"; + public static final String NAME = "LAZY"; public LazyBlockEncoding() {} @@ -32,16 +32,16 @@ public String getName() } @Override - public void writeBlock(SliceOutput sliceOutput, Block block) + public Block readBlock(BlockEncodingSerde blockEncodingSerde, SliceInput input) { - // We implemented replacementBlockForWrite, so we will never need to write a lazy block + // We write the actual underlying block, so we will never need to read a lazy block throw new UnsupportedOperationException(); } @Override - public Block readBlock(SliceInput sliceInput) + public void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput sliceOutput, Block block) { - // We write the actual underlying block, so we will never need to read a lazy block + // We implemented replacementBlockForWrite, so we will never need to write a lazy block throw new UnsupportedOperationException(); } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/LongArrayBlock.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/LongArrayBlock.java index f6c422634706..1dfac3e1a1c7 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/LongArrayBlock.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/LongArrayBlock.java @@ -219,9 +219,9 @@ public Block copyRegion(int positionOffset, int length) } @Override - public BlockEncoding getEncoding() + public String getEncodingName() { - return new LongArrayBlockEncoding(); + return LongArrayBlockEncoding.NAME; } @Override diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/LongArrayBlockBuilder.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/LongArrayBlockBuilder.java index b8d1d90e7af3..bad02a3b9850 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/LongArrayBlockBuilder.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/LongArrayBlockBuilder.java @@ -275,9 +275,9 @@ public Block copyRegion(int positionOffset, int length) } @Override - public BlockEncoding getEncoding() + public String getEncodingName() { - return new LongArrayBlockEncoding(); + return LongArrayBlockEncoding.NAME; } @Override diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/LongArrayBlockEncoding.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/LongArrayBlockEncoding.java index 772c424d7bca..35eeb7453f8b 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/LongArrayBlockEncoding.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/LongArrayBlockEncoding.java @@ -22,7 +22,7 @@ public class LongArrayBlockEncoding implements BlockEncoding { - private static final String NAME = "LONG_ARRAY"; + public static final String NAME = "LONG_ARRAY"; @Override public String getName() @@ -31,7 +31,7 @@ public String getName() } @Override - public void writeBlock(SliceOutput sliceOutput, Block block) + public void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput sliceOutput, Block block) { int positionCount = block.getPositionCount(); sliceOutput.appendInt(positionCount); @@ -46,7 +46,7 @@ public void writeBlock(SliceOutput sliceOutput, Block block) } @Override - public Block readBlock(SliceInput sliceInput) + public Block readBlock(BlockEncodingSerde blockEncodingSerde, SliceInput sliceInput) { int positionCount = sliceInput.readInt(); @@ -61,25 +61,4 @@ public Block readBlock(SliceInput sliceInput) return new LongArrayBlock(positionCount, valueIsNull, values); } - - public static class LongArrayBlockEncodingFactory - implements BlockEncodingFactory - { - @Override - public String getName() - { - return NAME; - } - - @Override - public LongArrayBlockEncoding readEncoding(BlockEncodingSerde serde, SliceInput input) - { - return new LongArrayBlockEncoding(); - } - - @Override - public void writeEncoding(BlockEncodingSerde serde, SliceOutput output, LongArrayBlockEncoding blockEncoding) - { - } - } } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/MapBlockEncoding.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/MapBlockEncoding.java index f83c226aa9a9..34968c2b6845 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/MapBlockEncoding.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/MapBlockEncoding.java @@ -31,28 +31,17 @@ import static java.lang.String.format; import static java.util.Arrays.asList; import static java.util.Collections.singletonList; -import static java.util.Objects.requireNonNull; public class MapBlockEncoding implements BlockEncoding { - private static final String NAME = "MAP"; + public static final String NAME = "MAP"; - private final Type keyType; - private final MethodHandle keyNativeHashCode; - private final MethodHandle keyBlockNativeEquals; - private final BlockEncoding keyBlockEncoding; - private final BlockEncoding valueBlockEncoding; + private final TypeManager typeManager; - public MapBlockEncoding(Type keyType, MethodHandle keyBlockNativeEquals, MethodHandle keyNativeHashCode, BlockEncoding keyBlockEncoding, BlockEncoding valueBlockEncoding) + public MapBlockEncoding(TypeManager typeManager) { - this.keyType = requireNonNull(keyType, "keyType is null"); - // keyNativeHashCode can only be null due to map block kill switch. deprecated.new-map-block - this.keyNativeHashCode = keyNativeHashCode; - // keyBlockNativeEquals can only be null due to map block kill switch. deprecated.new-map-block - this.keyBlockNativeEquals = keyBlockNativeEquals; - this.keyBlockEncoding = requireNonNull(keyBlockEncoding, "keyBlockEncoding is null"); - this.valueBlockEncoding = requireNonNull(valueBlockEncoding, "valueBlockEncoding is null"); + this.typeManager = typeManager; } @Override @@ -62,7 +51,7 @@ public String getName() } @Override - public void writeBlock(SliceOutput sliceOutput, Block block) + public void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput sliceOutput, Block block) { AbstractMapBlock mapBlock = (AbstractMapBlock) block; @@ -74,8 +63,11 @@ public void writeBlock(SliceOutput sliceOutput, Block block) int entriesStartOffset = offsets[offsetBase]; int entriesEndOffset = offsets[offsetBase + positionCount]; - keyBlockEncoding.writeBlock(sliceOutput, mapBlock.getKeys().getRegion(entriesStartOffset, entriesEndOffset - entriesStartOffset)); - valueBlockEncoding.writeBlock(sliceOutput, mapBlock.getValues().getRegion(entriesStartOffset, entriesEndOffset - entriesStartOffset)); + + TypeSerde.writeType(sliceOutput, mapBlock.keyType); + + blockEncodingSerde.writeBlock(sliceOutput, mapBlock.getKeys().getRegion(entriesStartOffset, entriesEndOffset - entriesStartOffset)); + blockEncodingSerde.writeBlock(sliceOutput, mapBlock.getValues().getRegion(entriesStartOffset, entriesEndOffset - entriesStartOffset)); sliceOutput.appendInt((entriesEndOffset - entriesStartOffset) * HASH_MULTIPLIER); sliceOutput.writeBytes(wrappedIntArray(hashTable, entriesStartOffset * HASH_MULTIPLIER, (entriesEndOffset - entriesStartOffset) * HASH_MULTIPLIER)); @@ -88,10 +80,15 @@ public void writeBlock(SliceOutput sliceOutput, Block block) } @Override - public Block readBlock(SliceInput sliceInput) + public Block readBlock(BlockEncodingSerde blockEncodingSerde, SliceInput sliceInput) { - Block keyBlock = keyBlockEncoding.readBlock(sliceInput); - Block valueBlock = valueBlockEncoding.readBlock(sliceInput); + Type keyType = TypeSerde.readType(typeManager, sliceInput); + MethodHandle keyNativeEquals = typeManager.resolveOperator(OperatorType.EQUAL, asList(keyType, keyType)); + MethodHandle keyBlockNativeEquals = compose(keyNativeEquals, nativeValueGetter(keyType)); + MethodHandle keyNativeHashCode = typeManager.resolveOperator(OperatorType.HASH_CODE, singletonList(keyType)); + + Block keyBlock = blockEncodingSerde.readBlock(sliceInput); + Block valueBlock = blockEncodingSerde.readBlock(sliceInput); int[] hashTable = new int[sliceInput.readInt()]; sliceInput.readBytes(wrappedIntArray(hashTable)); @@ -107,42 +104,4 @@ public Block readBlock(SliceInput sliceInput) boolean[] mapIsNull = EncoderUtil.decodeNullBits(sliceInput, positionCount); return createMapBlockInternal(0, positionCount, mapIsNull, offsets, keyBlock, valueBlock, hashTable, keyType, keyBlockNativeEquals, keyNativeHashCode); } - - public static class MapBlockEncodingFactory - implements BlockEncodingFactory - { - private final TypeManager typeManager; - - public MapBlockEncodingFactory(TypeManager typeManager) - { - this.typeManager = requireNonNull(typeManager, "typeManager is null"); - } - - @Override - public String getName() - { - return NAME; - } - - @Override - public MapBlockEncoding readEncoding(BlockEncodingSerde serde, SliceInput input) - { - Type keyType = TypeSerde.readType(typeManager, input); - MethodHandle keyNativeEquals = typeManager.resolveOperator(OperatorType.EQUAL, asList(keyType, keyType)); - MethodHandle keyBlockNativeEquals = compose(keyNativeEquals, nativeValueGetter(keyType)); - MethodHandle keyNativeHashCode = typeManager.resolveOperator(OperatorType.HASH_CODE, singletonList(keyType)); - - BlockEncoding keyBlockEncoding = serde.readBlockEncoding(input); - BlockEncoding valueBlockEncoding = serde.readBlockEncoding(input); - return new MapBlockEncoding(keyType, keyBlockNativeEquals, keyNativeHashCode, keyBlockEncoding, valueBlockEncoding); - } - - @Override - public void writeEncoding(BlockEncodingSerde serde, SliceOutput output, MapBlockEncoding blockEncoding) - { - TypeSerde.writeType(output, blockEncoding.keyType); - serde.writeBlockEncoding(output, blockEncoding.keyBlockEncoding); - serde.writeBlockEncoding(output, blockEncoding.valueBlockEncoding); - } - } } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/RowBlockEncoding.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/RowBlockEncoding.java index 1923b49b54bd..fb3c3401f0ea 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/RowBlockEncoding.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/RowBlockEncoding.java @@ -19,19 +19,11 @@ import static com.facebook.presto.spi.block.RowBlock.createRowBlockInternal; import static io.airlift.slice.Slices.wrappedIntArray; -import static java.util.Objects.requireNonNull; public class RowBlockEncoding implements BlockEncoding { - private static final String NAME = "ROW"; - - private final BlockEncoding[] fieldBlockEncodings; - - public RowBlockEncoding(BlockEncoding[] fieldBlockEncodings) - { - this.fieldBlockEncodings = requireNonNull(fieldBlockEncodings, "fieldBlockEncodings is null"); - } + public static final String NAME = "ROW"; @Override public String getName() @@ -40,24 +32,21 @@ public String getName() } @Override - public void writeBlock(SliceOutput sliceOutput, Block block) + public void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput sliceOutput, Block block) { AbstractRowBlock rowBlock = (AbstractRowBlock) block; - - if (rowBlock.numFields != fieldBlockEncodings.length) { - throw new IllegalArgumentException( - "argument block differs in length (" + rowBlock.numFields + ") with this encoding (" + fieldBlockEncodings.length + ")"); - } + int numFields = rowBlock.numFields; int positionCount = rowBlock.getPositionCount(); int offsetBase = rowBlock.getOffsetBase(); int[] fieldBlockOffsets = rowBlock.getFieldBlockOffsets(); - int startFieldBlockOffset = fieldBlockOffsets[offsetBase]; int endFieldBlockOffset = fieldBlockOffsets[offsetBase + positionCount]; - for (int i = 0; i < fieldBlockEncodings.length; i++) { - fieldBlockEncodings[i].writeBlock(sliceOutput, rowBlock.getFieldBlocks()[i].getRegion(startFieldBlockOffset, endFieldBlockOffset - startFieldBlockOffset)); + + sliceOutput.appendInt(numFields); + for (int i = 0; i < numFields; i++) { + blockEncodingSerde.writeBlock(sliceOutput, rowBlock.getFieldBlocks()[i].getRegion(startFieldBlockOffset, endFieldBlockOffset - startFieldBlockOffset)); } sliceOutput.appendInt(positionCount); @@ -68,11 +57,12 @@ public void writeBlock(SliceOutput sliceOutput, Block block) } @Override - public Block readBlock(SliceInput sliceInput) + public Block readBlock(BlockEncodingSerde blockEncodingSerde, SliceInput sliceInput) { - Block[] fieldBlocks = new Block[fieldBlockEncodings.length]; - for (int i = 0; i < fieldBlockEncodings.length; i++) { - fieldBlocks[i] = fieldBlockEncodings[i].readBlock(sliceInput); + int numFields = sliceInput.readInt(); + Block[] fieldBlocks = new Block[numFields]; + for (int i = 0; i < numFields; i++) { + fieldBlocks[i] = blockEncodingSerde.readBlock(sliceInput); } int positionCount = sliceInput.readInt(); @@ -81,34 +71,4 @@ public Block readBlock(SliceInput sliceInput) boolean[] rowIsNull = EncoderUtil.decodeNullBits(sliceInput, positionCount); return createRowBlockInternal(0, positionCount, rowIsNull, fieldBlockOffsets, fieldBlocks); } - - public static class RowBlockEncodingFactory - implements BlockEncodingFactory - { - @Override - public String getName() - { - return NAME; - } - - @Override - public RowBlockEncoding readEncoding(BlockEncodingSerde serde, SliceInput input) - { - int numFields = input.readInt(); - BlockEncoding[] fieldBlockEncodings = new BlockEncoding[numFields]; - for (int i = 0; i < numFields; i++) { - fieldBlockEncodings[i] = serde.readBlockEncoding(input); - } - return new RowBlockEncoding(fieldBlockEncodings); - } - - @Override - public void writeEncoding(BlockEncodingSerde serde, SliceOutput output, RowBlockEncoding blockEncoding) - { - output.appendInt(blockEncoding.fieldBlockEncodings.length); - for (BlockEncoding fieldBlockEncoding : blockEncoding.fieldBlockEncodings) { - serde.writeBlockEncoding(output, fieldBlockEncoding); - } - } - } } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/RunLengthBlockEncoding.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/RunLengthBlockEncoding.java index 4ec843c992d8..32b87bb22bcf 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/RunLengthBlockEncoding.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/RunLengthBlockEncoding.java @@ -16,19 +16,10 @@ import io.airlift.slice.SliceInput; import io.airlift.slice.SliceOutput; -import static java.util.Objects.requireNonNull; - public class RunLengthBlockEncoding implements BlockEncoding { - private static final String NAME = "RLE"; - - private final BlockEncoding valueBlockEncoding; - - public RunLengthBlockEncoding(BlockEncoding valueBlockEncoding) - { - this.valueBlockEncoding = requireNonNull(valueBlockEncoding, "valueBlockEncoding is null"); - } + public static final String NAME = "RLE"; @Override public String getName() @@ -36,13 +27,8 @@ public String getName() return NAME; } - public BlockEncoding getValueBlockEncoding() - { - return valueBlockEncoding; - } - @Override - public void writeBlock(SliceOutput sliceOutput, Block block) + public void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput sliceOutput, Block block) { RunLengthEncodedBlock rleBlock = (RunLengthEncodedBlock) block; @@ -50,41 +36,18 @@ public void writeBlock(SliceOutput sliceOutput, Block block) sliceOutput.writeInt(rleBlock.getPositionCount()); // write the value - getValueBlockEncoding().writeBlock(sliceOutput, rleBlock.getValue()); + blockEncodingSerde.writeBlock(sliceOutput, rleBlock.getValue()); } @Override - public RunLengthEncodedBlock readBlock(SliceInput sliceInput) + public RunLengthEncodedBlock readBlock(BlockEncodingSerde blockEncodingSerde, SliceInput sliceInput) { // read the run length int positionCount = sliceInput.readInt(); // read the value - Block value = getValueBlockEncoding().readBlock(sliceInput); + Block value = blockEncodingSerde.readBlock(sliceInput); return new RunLengthEncodedBlock(value, positionCount); } - - public static class RunLengthBlockEncodingFactory - implements BlockEncodingFactory - { - @Override - public String getName() - { - return NAME; - } - - @Override - public RunLengthBlockEncoding readEncoding(BlockEncodingSerde serde, SliceInput input) - { - BlockEncoding valueBlockEncoding = serde.readBlockEncoding(input); - return new RunLengthBlockEncoding(valueBlockEncoding); - } - - @Override - public void writeEncoding(BlockEncodingSerde serde, SliceOutput output, RunLengthBlockEncoding blockEncoding) - { - serde.writeBlockEncoding(output, blockEncoding.getValueBlockEncoding()); - } - } } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/RunLengthEncodedBlock.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/RunLengthEncodedBlock.java index c971b7439274..6f053e238b09 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/RunLengthEncodedBlock.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/RunLengthEncodedBlock.java @@ -91,9 +91,9 @@ public void retainedBytesForEachPart(BiConsumer consumer) } @Override - public RunLengthBlockEncoding getEncoding() + public String getEncodingName() { - return new RunLengthBlockEncoding(value.getEncoding()); + return RunLengthBlockEncoding.NAME; } @Override diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/ShortArrayBlock.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/ShortArrayBlock.java index c297ece19100..79abdf672755 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/ShortArrayBlock.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/ShortArrayBlock.java @@ -172,9 +172,9 @@ public Block copyRegion(int positionOffset, int length) } @Override - public BlockEncoding getEncoding() + public String getEncodingName() { - return new ShortArrayBlockEncoding(); + return ShortArrayBlockEncoding.NAME; } @Override diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/ShortArrayBlockBuilder.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/ShortArrayBlockBuilder.java index 8608a9d4e8b3..91ba02b78e1f 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/ShortArrayBlockBuilder.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/ShortArrayBlockBuilder.java @@ -228,9 +228,9 @@ public Block copyRegion(int positionOffset, int length) } @Override - public BlockEncoding getEncoding() + public String getEncodingName() { - return new ShortArrayBlockEncoding(); + return ShortArrayBlockEncoding.NAME; } @Override diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/ShortArrayBlockEncoding.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/ShortArrayBlockEncoding.java index 35426307a881..671cc87cb9ff 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/ShortArrayBlockEncoding.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/ShortArrayBlockEncoding.java @@ -22,7 +22,7 @@ public class ShortArrayBlockEncoding implements BlockEncoding { - private static final String NAME = "SHORT_ARRAY"; + public static final String NAME = "SHORT_ARRAY"; @Override public String getName() @@ -31,7 +31,7 @@ public String getName() } @Override - public void writeBlock(SliceOutput sliceOutput, Block block) + public void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput sliceOutput, Block block) { int positionCount = block.getPositionCount(); sliceOutput.appendInt(positionCount); @@ -46,7 +46,7 @@ public void writeBlock(SliceOutput sliceOutput, Block block) } @Override - public Block readBlock(SliceInput sliceInput) + public Block readBlock(BlockEncodingSerde blockEncodingSerde, SliceInput sliceInput) { int positionCount = sliceInput.readInt(); @@ -61,25 +61,4 @@ public Block readBlock(SliceInput sliceInput) return new ShortArrayBlock(positionCount, valueIsNull, values); } - - public static class ShortArrayBlockEncodingFactory - implements BlockEncodingFactory - { - @Override - public String getName() - { - return NAME; - } - - @Override - public ShortArrayBlockEncoding readEncoding(BlockEncodingSerde serde, SliceInput input) - { - return new ShortArrayBlockEncoding(); - } - - @Override - public void writeEncoding(BlockEncodingSerde serde, SliceOutput output, ShortArrayBlockEncoding blockEncoding) - { - } - } } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/SingleMapBlock.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/SingleMapBlock.java index fad5e33f4cef..afdb560d9eac 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/SingleMapBlock.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/SingleMapBlock.java @@ -38,7 +38,7 @@ public class SingleMapBlock private final Block keyBlock; private final Block valueBlock; private final int[] hashTable; - private final Type keyType; + final Type keyType; private final MethodHandle keyNativeHashCode; private final MethodHandle keyBlockNativeEquals; @@ -84,9 +84,9 @@ public void retainedBytesForEachPart(BiConsumer consumer) } @Override - public BlockEncoding getEncoding() + public String getEncodingName() { - return new SingleMapBlockEncoding(keyType, keyNativeHashCode, keyBlockNativeEquals, keyBlock.getEncoding(), valueBlock.getEncoding()); + return SingleMapBlockEncoding.NAME; } @Override diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/SingleMapBlockEncoding.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/SingleMapBlockEncoding.java index cbd736a5f52a..338a9380d017 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/SingleMapBlockEncoding.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/SingleMapBlockEncoding.java @@ -35,23 +35,13 @@ public class SingleMapBlockEncoding implements BlockEncoding { - private static final String NAME = "MAP_ELEMENT"; + public static final String NAME = "MAP_ELEMENT"; - private final Type keyType; - private final MethodHandle keyNativeHashCode; - private final MethodHandle keyBlockNativeEquals; - private final BlockEncoding keyBlockEncoding; - private final BlockEncoding valueBlockEncoding; + private final TypeManager typeManager; - public SingleMapBlockEncoding(Type keyType, MethodHandle keyNativeHashCode, MethodHandle keyBlockNativeEquals, BlockEncoding keyBlockEncoding, BlockEncoding valueBlockEncoding) + public SingleMapBlockEncoding(TypeManager typeManager) { - this.keyType = requireNonNull(keyType, "keyType is null"); - // keyNativeHashCode can only be null due to map block kill switch. deprecated.new-map-block - this.keyNativeHashCode = keyNativeHashCode; - // keyBlockNativeEquals can only be null due to map block kill switch. deprecated.new-map-block - this.keyBlockNativeEquals = keyBlockNativeEquals; - this.keyBlockEncoding = requireNonNull(keyBlockEncoding, "keyBlockEncoding is null"); - this.valueBlockEncoding = requireNonNull(valueBlockEncoding, "valueBlockEncoding is null"); + this.typeManager = requireNonNull(typeManager, "typeManager is null"); } @Override @@ -61,23 +51,30 @@ public String getName() } @Override - public void writeBlock(SliceOutput sliceOutput, Block block) + public void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput sliceOutput, Block block) { SingleMapBlock singleMapBlock = (SingleMapBlock) block; + TypeSerde.writeType(sliceOutput, singleMapBlock.keyType); + int offset = singleMapBlock.getOffset(); int positionCount = singleMapBlock.getPositionCount(); - keyBlockEncoding.writeBlock(sliceOutput, singleMapBlock.getKeyBlock().getRegion(offset / 2, positionCount / 2)); - valueBlockEncoding.writeBlock(sliceOutput, singleMapBlock.getValueBlock().getRegion(offset / 2, positionCount / 2)); + blockEncodingSerde.writeBlock(sliceOutput, singleMapBlock.getKeyBlock().getRegion(offset / 2, positionCount / 2)); + blockEncodingSerde.writeBlock(sliceOutput, singleMapBlock.getValueBlock().getRegion(offset / 2, positionCount / 2)); int[] hashTable = singleMapBlock.getHashTable(); sliceOutput.appendInt(positionCount / 2 * HASH_MULTIPLIER); sliceOutput.writeBytes(wrappedIntArray(hashTable, offset / 2 * HASH_MULTIPLIER, positionCount / 2 * HASH_MULTIPLIER)); } @Override - public Block readBlock(SliceInput sliceInput) + public Block readBlock(BlockEncodingSerde blockEncodingSerde, SliceInput sliceInput) { - Block keyBlock = keyBlockEncoding.readBlock(sliceInput); - Block valueBlock = valueBlockEncoding.readBlock(sliceInput); + Type keyType = TypeSerde.readType(typeManager, sliceInput); + MethodHandle keyNativeEquals = typeManager.resolveOperator(OperatorType.EQUAL, asList(keyType, keyType)); + MethodHandle keyBlockNativeEquals = compose(keyNativeEquals, nativeValueGetter(keyType)); + MethodHandle keyNativeHashCode = typeManager.resolveOperator(OperatorType.HASH_CODE, singletonList(keyType)); + + Block keyBlock = blockEncodingSerde.readBlock(sliceInput); + Block valueBlock = blockEncodingSerde.readBlock(sliceInput); int[] hashTable = new int[sliceInput.readInt()]; sliceInput.readBytes(wrappedIntArray(hashTable)); @@ -89,42 +86,4 @@ public Block readBlock(SliceInput sliceInput) return new SingleMapBlock(0, keyBlock.getPositionCount() * 2, keyBlock, valueBlock, hashTable, keyType, keyNativeHashCode, keyBlockNativeEquals); } - - public static class SingleMapBlockEncodingFactory - implements BlockEncodingFactory - { - private final TypeManager typeManager; - - public SingleMapBlockEncodingFactory(TypeManager typeManager) - { - this.typeManager = requireNonNull(typeManager, "typeManager is null"); - } - - @Override - public String getName() - { - return NAME; - } - - @Override - public SingleMapBlockEncoding readEncoding(BlockEncodingSerde serde, SliceInput input) - { - Type keyType = TypeSerde.readType(typeManager, input); - MethodHandle keyNativeHashCode = typeManager.resolveOperator(OperatorType.HASH_CODE, singletonList(keyType)); - MethodHandle keyNativeEquals = typeManager.resolveOperator(OperatorType.EQUAL, asList(keyType, keyType)); - MethodHandle keyBlockNativeEquals = compose(keyNativeEquals, nativeValueGetter(keyType)); - - BlockEncoding keyBlockEncoding = serde.readBlockEncoding(input); - BlockEncoding valueBlockEncoding = serde.readBlockEncoding(input); - return new SingleMapBlockEncoding(keyType, keyNativeHashCode, keyBlockNativeEquals, keyBlockEncoding, valueBlockEncoding); - } - - @Override - public void writeEncoding(BlockEncodingSerde serde, SliceOutput output, SingleMapBlockEncoding blockEncoding) - { - TypeSerde.writeType(output, blockEncoding.keyType); - serde.writeBlockEncoding(output, blockEncoding.keyBlockEncoding); - serde.writeBlockEncoding(output, blockEncoding.valueBlockEncoding); - } - } } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/SingleMapBlockWriter.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/SingleMapBlockWriter.java index dc5d866d344d..f0b7adafcce5 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/SingleMapBlockWriter.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/SingleMapBlockWriter.java @@ -218,7 +218,7 @@ public int getPositionCount() } @Override - public BlockEncoding getEncoding() + public String getEncodingName() { throw new UnsupportedOperationException(); } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/SingleRowBlock.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/SingleRowBlock.java index 91b8d8596fcf..19af31ed8aab 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/SingleRowBlock.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/SingleRowBlock.java @@ -33,6 +33,11 @@ public class SingleRowBlock this.fieldBlocks = fieldBlocks; } + int getNumFields() + { + return fieldBlocks.length; + } + @Override protected Block getFieldBlock(int fieldIndex) { @@ -75,13 +80,9 @@ public void retainedBytesForEachPart(BiConsumer consumer) } @Override - public BlockEncoding getEncoding() + public String getEncodingName() { - BlockEncoding[] fieldBlockEncodings = new BlockEncoding[fieldBlocks.length]; - for (int i = 0; i < fieldBlocks.length; i++) { - fieldBlockEncodings[i] = fieldBlocks[i].getEncoding(); - } - return new SingleRowBlockEncoding(fieldBlockEncodings); + return SingleRowBlockEncoding.NAME; } public int getRowIndex() diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/SingleRowBlockEncoding.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/SingleRowBlockEncoding.java index 26bd13d19fe0..214c80a1a390 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/SingleRowBlockEncoding.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/SingleRowBlockEncoding.java @@ -17,19 +17,10 @@ import io.airlift.slice.SliceInput; import io.airlift.slice.SliceOutput; -import static java.util.Objects.requireNonNull; - public class SingleRowBlockEncoding implements BlockEncoding { - private static final String NAME = "ROW_ELEMENT"; - - private final BlockEncoding[] fieldBlockEncodings; - - public SingleRowBlockEncoding(BlockEncoding[] fieldBlockEncodings) - { - this.fieldBlockEncodings = requireNonNull(fieldBlockEncodings, "fieldBlockEncodings is null"); - } + public static final String NAME = "ROW_ELEMENT"; @Override public String getName() @@ -38,52 +29,25 @@ public String getName() } @Override - public void writeBlock(SliceOutput sliceOutput, Block block) + public void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput sliceOutput, Block block) { SingleRowBlock singleRowBlock = (SingleRowBlock) block; + int numFields = singleRowBlock.getNumFields(); int rowIndex = singleRowBlock.getRowIndex(); - for (int i = 0; i < fieldBlockEncodings.length; i++) { - fieldBlockEncodings[i].writeBlock(sliceOutput, singleRowBlock.getFieldBlock(i).getRegion(rowIndex, 1)); + sliceOutput.appendInt(numFields); + for (int i = 0; i < numFields; i++) { + blockEncodingSerde.writeBlock(sliceOutput, singleRowBlock.getFieldBlock(i).getRegion(rowIndex, 1)); } } @Override - public Block readBlock(SliceInput sliceInput) + public Block readBlock(BlockEncodingSerde blockEncodingSerde, SliceInput sliceInput) { - Block[] fieldBlocks = new Block[fieldBlockEncodings.length]; + int numFields = sliceInput.readInt(); + Block[] fieldBlocks = new Block[numFields]; for (int i = 0; i < fieldBlocks.length; i++) { - fieldBlocks[i] = fieldBlockEncodings[i].readBlock(sliceInput); + fieldBlocks[i] = blockEncodingSerde.readBlock(sliceInput); } return new SingleRowBlock(0, fieldBlocks); } - - public static class SingleRowBlockEncodingFactory - implements BlockEncodingFactory - { - @Override - public String getName() - { - return NAME; - } - - @Override - public SingleRowBlockEncoding readEncoding(BlockEncodingSerde serde, SliceInput input) - { - int numFields = input.readInt(); - BlockEncoding[] fieldBlockEncodings = new BlockEncoding[numFields]; - for (int i = 0; i < numFields; i++) { - fieldBlockEncodings[i] = serde.readBlockEncoding(input); - } - return new SingleRowBlockEncoding(fieldBlockEncodings); - } - - @Override - public void writeEncoding(BlockEncodingSerde serde, SliceOutput output, SingleRowBlockEncoding blockEncoding) - { - output.appendInt(blockEncoding.fieldBlockEncodings.length); - for (BlockEncoding fieldBlockEncoding : blockEncoding.fieldBlockEncodings) { - serde.writeBlockEncoding(output, fieldBlockEncoding); - } - } - } } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/SingleRowBlockWriter.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/SingleRowBlockWriter.java index bfcfaffc3968..60cb2d9df924 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/SingleRowBlockWriter.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/SingleRowBlockWriter.java @@ -196,7 +196,7 @@ public int getPositionCount() } @Override - public BlockEncoding getEncoding() + public String getEncodingName() { throw new UnsupportedOperationException(); } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/VariableWidthBlockEncoding.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/VariableWidthBlockEncoding.java index f01f5a734da9..dc81afe327b1 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/VariableWidthBlockEncoding.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/VariableWidthBlockEncoding.java @@ -25,7 +25,7 @@ public class VariableWidthBlockEncoding implements BlockEncoding { - private static final String NAME = "VARIABLE_WIDTH"; + public static final String NAME = "VARIABLE_WIDTH"; @Override public String getName() @@ -34,7 +34,7 @@ public String getName() } @Override - public void writeBlock(SliceOutput sliceOutput, Block block) + public void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput sliceOutput, Block block) { // The down casts here are safe because it is the block itself the provides this encoding implementation. AbstractVariableWidthBlock variableWidthBlock = (AbstractVariableWidthBlock) block; @@ -58,7 +58,7 @@ public void writeBlock(SliceOutput sliceOutput, Block block) } @Override - public Block readBlock(SliceInput sliceInput) + public Block readBlock(BlockEncodingSerde blockEncodingSerde, SliceInput sliceInput) { int positionCount = sliceInput.readInt(); @@ -72,25 +72,4 @@ public Block readBlock(SliceInput sliceInput) return new VariableWidthBlock(positionCount, slice, offsets, valueIsNull); } - - public static class VariableWidthBlockEncodingFactory - implements BlockEncodingFactory - { - @Override - public String getName() - { - return NAME; - } - - @Override - public VariableWidthBlockEncoding readEncoding(BlockEncodingSerde serde, SliceInput input) - { - return new VariableWidthBlockEncoding(); - } - - @Override - public void writeEncoding(BlockEncodingSerde serde, SliceOutput output, VariableWidthBlockEncoding blockEncoding) - { - } - } } diff --git a/presto-spi/src/test/java/com/facebook/presto/spi/block/TestDictionaryBlockEncoding.java b/presto-spi/src/test/java/com/facebook/presto/spi/block/TestDictionaryBlockEncoding.java index 1619cb77a93a..34fa1c6cca19 100644 --- a/presto-spi/src/test/java/com/facebook/presto/spi/block/TestDictionaryBlockEncoding.java +++ b/presto-spi/src/test/java/com/facebook/presto/spi/block/TestDictionaryBlockEncoding.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.spi.block; +import com.facebook.presto.spi.type.TestingTypeManager; import com.facebook.presto.spi.type.Type; import io.airlift.slice.DynamicSliceOutput; import org.testng.annotations.Test; @@ -24,6 +25,8 @@ public class TestDictionaryBlockEncoding { + private final BlockEncodingSerde blockEncodingSerde = new TestingBlockEncodingSerde(new TestingTypeManager()); + @Test public void testRoundTrip() { @@ -43,12 +46,11 @@ public void testRoundTrip() ids[i] = i % 4; } - BlockEncoding blockEncoding = new DictionaryBlockEncoding(new VariableWidthBlockEncoding()); DictionaryBlock dictionaryBlock = new DictionaryBlock(dictionary, ids); DynamicSliceOutput sliceOutput = new DynamicSliceOutput(1024); - blockEncoding.writeBlock(sliceOutput, dictionaryBlock); - Block actualBlock = blockEncoding.readBlock(sliceOutput.slice().getInput()); + blockEncodingSerde.writeBlock(sliceOutput, dictionaryBlock); + Block actualBlock = blockEncodingSerde.readBlock(sliceOutput.slice().getInput()); assertTrue(actualBlock instanceof DictionaryBlock); DictionaryBlock actualDictionaryBlock = (DictionaryBlock) actualBlock; diff --git a/presto-spi/src/test/java/com/facebook/presto/spi/block/TestVariableWidthBlockEncoding.java b/presto-spi/src/test/java/com/facebook/presto/spi/block/TestVariableWidthBlockEncoding.java index 68380ab0ca55..6efc0f2a8da9 100644 --- a/presto-spi/src/test/java/com/facebook/presto/spi/block/TestVariableWidthBlockEncoding.java +++ b/presto-spi/src/test/java/com/facebook/presto/spi/block/TestVariableWidthBlockEncoding.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.spi.block; +import com.facebook.presto.spi.type.TestingTypeManager; import com.facebook.presto.spi.type.Type; import io.airlift.slice.DynamicSliceOutput; import org.testng.annotations.Test; @@ -23,6 +24,8 @@ public class TestVariableWidthBlockEncoding { + private final BlockEncodingSerde blockEncodingSerde = new TestingBlockEncodingSerde(new TestingTypeManager()); + @Test public void testRoundTrip() { @@ -34,9 +37,8 @@ public void testRoundTrip() Block expectedBlock = expectedBlockBuilder.build(); DynamicSliceOutput sliceOutput = new DynamicSliceOutput(1024); - BlockEncoding blockEncoding = new VariableWidthBlockEncoding(); - blockEncoding.writeBlock(sliceOutput, expectedBlock); - Block actualBlock = blockEncoding.readBlock(sliceOutput.slice().getInput()); + blockEncodingSerde.writeBlock(sliceOutput, expectedBlock); + Block actualBlock = blockEncodingSerde.readBlock(sliceOutput.slice().getInput()); assertBlockEquals(VARCHAR, actualBlock, expectedBlock); } diff --git a/presto-spi/src/test/java/com/facebook/presto/spi/block/TestingBlockEncodingSerde.java b/presto-spi/src/test/java/com/facebook/presto/spi/block/TestingBlockEncodingSerde.java index 4b6342c4e8a8..bd46375c6b6c 100644 --- a/presto-spi/src/test/java/com/facebook/presto/spi/block/TestingBlockEncodingSerde.java +++ b/presto-spi/src/test/java/com/facebook/presto/spi/block/TestingBlockEncodingSerde.java @@ -13,24 +13,12 @@ */ package com.facebook.presto.spi.block; -import com.facebook.presto.spi.block.ArrayBlockEncoding.ArrayBlockEncodingFactory; -import com.facebook.presto.spi.block.ByteArrayBlockEncoding.ByteArrayBlockEncodingFactory; -import com.facebook.presto.spi.block.DictionaryBlockEncoding.DictionaryBlockEncodingFactory; -import com.facebook.presto.spi.block.FixedWidthBlockEncoding.FixedWidthBlockEncodingFactory; -import com.facebook.presto.spi.block.IntArrayBlockEncoding.IntArrayBlockEncodingFactory; -import com.facebook.presto.spi.block.LongArrayBlockEncoding.LongArrayBlockEncodingFactory; -import com.facebook.presto.spi.block.MapBlockEncoding.MapBlockEncodingFactory; -import com.facebook.presto.spi.block.RowBlockEncoding.RowBlockEncodingFactory; -import com.facebook.presto.spi.block.RunLengthBlockEncoding.RunLengthBlockEncodingFactory; -import com.facebook.presto.spi.block.ShortArrayBlockEncoding.ShortArrayBlockEncodingFactory; -import com.facebook.presto.spi.block.SingleMapBlockEncoding.SingleMapBlockEncodingFactory; -import com.facebook.presto.spi.block.SingleRowBlockEncoding.SingleRowBlockEncodingFactory; -import com.facebook.presto.spi.block.VariableWidthBlockEncoding.VariableWidthBlockEncodingFactory; import com.facebook.presto.spi.type.TypeManager; import com.google.common.collect.ImmutableSet; import io.airlift.slice.SliceInput; import io.airlift.slice.SliceOutput; +import java.util.Optional; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -44,74 +32,86 @@ public final class TestingBlockEncodingSerde implements BlockEncodingSerde { - private final ConcurrentMap> blockEncodings = new ConcurrentHashMap<>(); + private final ConcurrentMap blockEncodings = new ConcurrentHashMap<>(); - public TestingBlockEncodingSerde(TypeManager typeManager, BlockEncodingFactory... blockEncodingFactories) + public TestingBlockEncodingSerde(TypeManager typeManager, BlockEncoding... blockEncodings) { - this(typeManager, ImmutableSet.copyOf(blockEncodingFactories)); + this(typeManager, ImmutableSet.copyOf(blockEncodings)); } - public TestingBlockEncodingSerde(TypeManager typeManager, Set> blockEncodingFactories) + public TestingBlockEncodingSerde(TypeManager typeManager, Set blockEncodings) { // This function should be called from Guice and tests only requireNonNull(typeManager, "typeManager is null"); - // always add the built-in BlockEncodingFactories - addBlockEncodingFactory(new VariableWidthBlockEncodingFactory()); - addBlockEncodingFactory(new FixedWidthBlockEncodingFactory()); - addBlockEncodingFactory(new ByteArrayBlockEncodingFactory()); - addBlockEncodingFactory(new ShortArrayBlockEncodingFactory()); - addBlockEncodingFactory(new IntArrayBlockEncodingFactory()); - addBlockEncodingFactory(new LongArrayBlockEncodingFactory()); - addBlockEncodingFactory(new DictionaryBlockEncodingFactory()); - addBlockEncodingFactory(new ArrayBlockEncodingFactory()); - addBlockEncodingFactory(new MapBlockEncodingFactory(typeManager)); - addBlockEncodingFactory(new SingleMapBlockEncodingFactory(typeManager)); - addBlockEncodingFactory(new RowBlockEncodingFactory()); - addBlockEncodingFactory(new SingleRowBlockEncodingFactory()); - addBlockEncodingFactory(new RunLengthBlockEncodingFactory()); - - for (BlockEncodingFactory factory : requireNonNull(blockEncodingFactories, "blockEncodingFactories is null")) { - addBlockEncodingFactory(factory); + // always add the built-in BlockEncodings + addBlockEncoding(new VariableWidthBlockEncoding()); + addBlockEncoding(new FixedWidthBlockEncoding()); + addBlockEncoding(new ByteArrayBlockEncoding()); + addBlockEncoding(new ShortArrayBlockEncoding()); + addBlockEncoding(new IntArrayBlockEncoding()); + addBlockEncoding(new LongArrayBlockEncoding()); + addBlockEncoding(new DictionaryBlockEncoding()); + addBlockEncoding(new ArrayBlockEncoding()); + addBlockEncoding(new MapBlockEncoding(typeManager)); + addBlockEncoding(new SingleMapBlockEncoding(typeManager)); + addBlockEncoding(new RowBlockEncoding()); + addBlockEncoding(new SingleRowBlockEncoding()); + addBlockEncoding(new RunLengthBlockEncoding()); + addBlockEncoding(new LazyBlockEncoding()); + + for (BlockEncoding blockEncoding : requireNonNull(blockEncodings, "blockEncodings is null")) { + addBlockEncoding(blockEncoding); } } - public void addBlockEncodingFactory(BlockEncodingFactory blockEncoding) + public void addBlockEncoding(BlockEncoding blockEncoding) { requireNonNull(blockEncoding, "blockEncoding is null"); - BlockEncodingFactory existingEntry = blockEncodings.putIfAbsent(blockEncoding.getName(), blockEncoding); + BlockEncoding existingEntry = blockEncodings.putIfAbsent(blockEncoding.getName(), blockEncoding); checkArgument(existingEntry == null, "Encoding %s is already registered", blockEncoding.getName()); } @Override - public BlockEncoding readBlockEncoding(SliceInput input) + public Block readBlock(SliceInput input) { // read the encoding name String encodingName = readLengthPrefixedString(input); // look up the encoding factory - BlockEncodingFactory blockEncoding = blockEncodings.get(encodingName); + BlockEncoding blockEncoding = blockEncodings.get(encodingName); checkArgument(blockEncoding != null, "Unknown block encoding %s", encodingName); // load read the encoding factory from the output stream - return blockEncoding.readEncoding(this, input); + return blockEncoding.readBlock(this, input); } @Override - public void writeBlockEncoding(SliceOutput output, BlockEncoding encoding) + public void writeBlock(SliceOutput output, Block block) { - // get the encoding name - String encodingName = encoding.getName(); + while (true) { + // get the encoding name + String encodingName = block.getEncodingName(); - // look up the encoding factory - BlockEncodingFactory blockEncoding = blockEncodings.get(encodingName); + // look up the encoding factory + BlockEncoding blockEncoding = blockEncodings.get(encodingName); + + // see if a replacement block should be written instead + Optional replacementBlock = blockEncoding.replacementBlockForWrite(block); + if (replacementBlock.isPresent()) { + block = replacementBlock.get(); + continue; + } - // write the name to the output - writeLengthPrefixedString(output, encodingName); + // write the name to the output + writeLengthPrefixedString(output, encodingName); - // write the encoding to the output - blockEncoding.writeEncoding(this, output, encoding); + // write the block to the output + blockEncoding.writeBlock(this, output, block); + + break; + } } private static String readLengthPrefixedString(SliceInput input) diff --git a/presto-spi/src/test/java/com/facebook/presto/spi/block/TestingBlockJsonSerde.java b/presto-spi/src/test/java/com/facebook/presto/spi/block/TestingBlockJsonSerde.java index 63fb14ba0f0f..77875b075088 100644 --- a/presto-spi/src/test/java/com/facebook/presto/spi/block/TestingBlockJsonSerde.java +++ b/presto-spi/src/test/java/com/facebook/presto/spi/block/TestingBlockJsonSerde.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.spi.block; +import com.facebook.presto.spi.type.TestingTypeManager; import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.databind.DeserializationContext; @@ -31,6 +32,8 @@ public final class TestingBlockJsonSerde { + private final BlockEncodingSerde blockEncodingSerde = new TestingBlockEncodingSerde(new TestingTypeManager()); + private TestingBlockJsonSerde() {} public static class Serializer @@ -48,9 +51,7 @@ public void serialize(Block block, JsonGenerator jsonGenerator, SerializerProvid throws IOException { SliceOutput output = new DynamicSliceOutput(64); - BlockEncoding encoding = block.getEncoding(); - blockEncodingSerde.writeBlockEncoding(output, encoding); - encoding.writeBlock(output, block); + blockEncodingSerde.writeBlock(output, block); String encoded = Base64.getEncoder().encodeToString(output.slice().getBytes()); jsonGenerator.writeString(encoded); } @@ -72,8 +73,7 @@ public Block deserialize(JsonParser jsonParser, DeserializationContext deseriali { byte[] decoded = Base64.getDecoder().decode(jsonParser.readValueAs(String.class)); BasicSliceInput input = Slices.wrappedBuffer(decoded).getInput(); - BlockEncoding blockEncoding = blockEncodingSerde.readBlockEncoding(input); - return blockEncoding.readBlock(input); + return blockEncodingSerde.readBlock(input); } } }