Skip to content

Commit

Permalink
Add BlockEncodingSerde to MetadataManager
Browse files Browse the repository at this point in the history
  • Loading branch information
@haozhun
haozhun committed Apr 20, 2015
1 parent 0b580f6 commit 2a110b0
Show file tree
Hide file tree
Showing 22 changed files with 61 additions and 37 deletions.
5 changes: 4 additions & 1 deletion presto-main/src/main/java/com/facebook/presto/metadata/Metadata.java
View file
Expand Up @@ -14,9 +14,10 @@
package com.facebook.presto.metadata;

import com.facebook.presto.Session;
import com.facebook.presto.spi.ColumnMetadata;
import com.facebook.presto.spi.ColumnHandle;
import com.facebook.presto.spi.ColumnMetadata;
import com.facebook.presto.spi.Constraint;
import com.facebook.presto.spi.block.BlockEncodingSerde;
import com.facebook.presto.spi.type.Type;
import com.facebook.presto.spi.type.TypeManager;
import com.facebook.presto.spi.type.TypeSignature;
Expand Down Expand Up @@ -192,4 +193,6 @@ FunctionInfo resolveOperator(OperatorType operatorType, List<? extends Type> arg
FunctionRegistry getFunctionRegistry();

TypeManager getTypeManager();

BlockEncodingSerde getBlockEncodingSerde();
}
34 changes: 23 additions & 11 deletions presto-main/src/main/java/com/facebook/presto/metadata/MetadataManager.java
View file
Expand Up @@ -14,9 +14,10 @@
package com.facebook.presto.metadata;

import com.facebook.presto.Session;
import com.facebook.presto.block.BlockEncodingManager;
import com.facebook.presto.connector.informationSchema.InformationSchemaMetadata;
import com.facebook.presto.spi.ColumnMetadata;
import com.facebook.presto.spi.ColumnHandle;
import com.facebook.presto.spi.ColumnMetadata;
import com.facebook.presto.spi.ConnectorInsertTableHandle;
import com.facebook.presto.spi.ConnectorMetadata;
import com.facebook.presto.spi.ConnectorOutputTableHandle;
Expand All @@ -33,6 +34,7 @@
import com.facebook.presto.spi.SchemaTableName;
import com.facebook.presto.spi.SchemaTablePrefix;
import com.facebook.presto.spi.TupleDomain;
import com.facebook.presto.spi.block.BlockEncodingSerde;
import com.facebook.presto.spi.type.Type;
import com.facebook.presto.spi.type.TypeManager;
import com.facebook.presto.spi.type.TypeSignature;
Expand All @@ -42,7 +44,6 @@
import com.facebook.presto.type.TypeDeserializer;
import com.facebook.presto.type.TypeRegistry;
import com.fasterxml.jackson.databind.JsonDeserializer;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
Expand Down Expand Up @@ -92,25 +93,30 @@ public class MetadataManager
private final TypeManager typeManager;
private final JsonCodec<ViewDefinition> viewCodec;
private final SplitManager splitManager;
private final BlockEncodingSerde blockEncodingSerde;

@VisibleForTesting
public MetadataManager()
{
this(new FeaturesConfig(), new TypeRegistry(), new SplitManager());
}

public MetadataManager(FeaturesConfig featuresConfig, TypeManager typeManager, SplitManager splitManager)
public MetadataManager(FeaturesConfig featuresConfig, TypeManager typeManager, SplitManager splitManager, BlockEncodingSerde blockEncodingSerde)
{
this(featuresConfig, typeManager, createTestingViewCodec(), splitManager);
this(featuresConfig, typeManager, createTestingViewCodec(), splitManager, blockEncodingSerde);
}

@Inject
public MetadataManager(FeaturesConfig featuresConfig, TypeManager typeManager, JsonCodec<ViewDefinition> viewCodec, SplitManager splitManager)
public MetadataManager(FeaturesConfig featuresConfig, TypeManager typeManager, JsonCodec<ViewDefinition> viewCodec, SplitManager splitManager, BlockEncodingSerde blockEncodingSerde)
{
functions = new FunctionRegistry(typeManager, featuresConfig.isExperimentalSyntaxEnabled());
this.typeManager = checkNotNull(typeManager, "types is null");
this.viewCodec = checkNotNull(viewCodec, "viewCodec is null");
this.splitManager = checkNotNull(splitManager, "splitManager is null");
this.blockEncodingSerde = checkNotNull(blockEncodingSerde, "blockEncodingSerde is null");
}

public static MetadataManager createTestMetadataManager()
{
FeaturesConfig featuresConfig = new FeaturesConfig();
TypeManager typeManager = new TypeRegistry();
SplitManager splitManager = new SplitManager();
BlockEncodingSerde blockEncodingSerde = new BlockEncodingManager(typeManager);
return new MetadataManager(featuresConfig, typeManager, splitManager, blockEncodingSerde);
}

public synchronized void addConnectorMetadata(String connectorId, String catalogName, ConnectorMetadata connectorMetadata)
Expand Down Expand Up @@ -528,6 +534,12 @@ public TypeManager getTypeManager()
return typeManager;
}

@Override
public BlockEncodingSerde getBlockEncodingSerde()
{
return blockEncodingSerde;
}

private ViewDefinition deserializeView(String data)
{
try {
Expand Down
8 changes: 6 additions & 2 deletions presto-main/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java
View file
Expand Up @@ -17,14 +17,14 @@
import com.facebook.presto.Session;
import com.facebook.presto.SystemSessionProperties;
import com.facebook.presto.TaskSource;
import com.facebook.presto.block.BlockEncodingManager;
import com.facebook.presto.connector.ConnectorManager;
import com.facebook.presto.connector.system.CatalogSystemTable;
import com.facebook.presto.connector.system.NodeSystemTable;
import com.facebook.presto.connector.system.SystemConnector;
import com.facebook.presto.execution.TaskId;
import com.facebook.presto.execution.TaskManagerConfig;
import com.facebook.presto.index.IndexManager;
import com.facebook.presto.spi.ColumnHandle;
import com.facebook.presto.metadata.HandleResolver;
import com.facebook.presto.metadata.InMemoryNodeManager;
import com.facebook.presto.metadata.Metadata;
Expand All @@ -50,6 +50,7 @@
import com.facebook.presto.operator.ProjectionFunctions;
import com.facebook.presto.operator.TaskContext;
import com.facebook.presto.operator.index.IndexJoinLookupStats;
import com.facebook.presto.spi.ColumnHandle;
import com.facebook.presto.spi.ColumnMetadata;
import com.facebook.presto.spi.Connector;
import com.facebook.presto.spi.ConnectorFactory;
Expand All @@ -59,6 +60,7 @@
import com.facebook.presto.spi.RecordCursor;
import com.facebook.presto.spi.block.Block;
import com.facebook.presto.spi.block.BlockBuilder;
import com.facebook.presto.spi.block.BlockEncodingSerde;
import com.facebook.presto.spi.type.Type;
import com.facebook.presto.split.PageSinkManager;
import com.facebook.presto.split.PageSourceManager;
Expand Down Expand Up @@ -120,6 +122,7 @@ public class LocalQueryRunner
private final TypeRegistry typeRegistry;
private final MetadataManager metadata;
private final SplitManager splitManager;
private final BlockEncodingSerde blockEncodingSerde;
private final PageSourceManager pageSourceManager;
private final IndexManager indexManager;
private final PageSinkManager pageSinkManager;
Expand All @@ -143,7 +146,8 @@ public LocalQueryRunner(Session defaultSession)
this.pageSinkManager = new PageSinkManager();

this.splitManager = new SplitManager();
this.metadata = new MetadataManager(new FeaturesConfig().setExperimentalSyntaxEnabled(true), typeRegistry, splitManager);
this.blockEncodingSerde = new BlockEncodingManager(typeRegistry);
this.metadata = new MetadataManager(new FeaturesConfig().setExperimentalSyntaxEnabled(true), typeRegistry, splitManager, blockEncodingSerde);
this.pageSourceManager = new PageSourceManager();

this.compiler = new ExpressionCompiler(metadata);
Expand Down
2 changes: 1 addition & 1 deletion presto-main/src/test/java/com/facebook/presto/execution/TaskTestUtils.java
View file
Expand Up @@ -82,7 +82,7 @@ private TaskTestUtils()

public static LocalExecutionPlanner createTestingPlanner()
{
MetadataManager metadata = new MetadataManager();
MetadataManager metadata = MetadataManager.createTestMetadataManager();

PageSourceManager pageSourceManager = new PageSourceManager();
pageSourceManager.addConnectorPageSourceProvider("test", new TestingPageSourceProvider());
Expand Down
2 changes: 1 addition & 1 deletion presto-main/src/test/java/com/facebook/presto/execution/TestResetSessionTask.java
View file
Expand Up @@ -48,7 +48,7 @@ public void test()
Session session = TEST_SESSION.withSystemProperty("foo", "bar").withCatalogProperty("catalog", "baz", "blah");

QueryStateMachine stateMachine = new QueryStateMachine(new QueryId("query"), "reset foo", session, URI.create("fake://uri"), executor);
new ResetSessionTask().execute(new ResetSession(QualifiedName.of("catalog", "baz")), session, new MetadataManager(), stateMachine);
new ResetSessionTask().execute(new ResetSession(QualifiedName.of("catalog", "baz")), session, MetadataManager.createTestMetadataManager(), stateMachine);

Set<String> sessionProperties = stateMachine.getResetSessionProperties();
assertEquals(sessionProperties, ImmutableSet.of("catalog.baz"));
Expand Down
2 changes: 1 addition & 1 deletion presto-main/src/test/java/com/facebook/presto/execution/TestSetSessionTask.java
View file
Expand Up @@ -45,7 +45,7 @@ public void test()
throws Exception
{
QueryStateMachine stateMachine = new QueryStateMachine(new QueryId("query"), "set foo.bar = 'baz'", TEST_SESSION, URI.create("fake://uri"), executor);
new SetSessionTask().execute(new SetSession(QualifiedName.of("foo", "bar"), "baz"), TEST_SESSION, new MetadataManager(), stateMachine);
new SetSessionTask().execute(new SetSession(QualifiedName.of("foo", "bar"), "baz"), TEST_SESSION, MetadataManager.createTestMetadataManager(), stateMachine);

Map<String, String> sessionProperties = stateMachine.getSetSessionProperties();
assertEquals(sessionProperties, ImmutableMap.of("foo.bar", "baz"));
Expand Down
2 changes: 1 addition & 1 deletion presto-main/src/test/java/com/facebook/presto/operator/TestAggregationOperator.java
View file
Expand Up @@ -72,7 +72,7 @@ public void tearDown()
public void testAggregation()
throws Exception
{
MetadataManager metadata = new MetadataManager();
MetadataManager metadata = MetadataManager.createTestMetadataManager();
InternalAggregationFunction countVarcharColumn = metadata.resolveFunction(QualifiedName.of("count"), ImmutableList.of(parseTypeSignature(StandardTypes.VARCHAR)), false).getAggregationFunction();
InternalAggregationFunction maxVarcharColumn = metadata.resolveFunction(QualifiedName.of("max"), ImmutableList.of(parseTypeSignature(StandardTypes.VARCHAR)), false).getAggregationFunction();
List<Page> input = rowPagesBuilder(VARCHAR, BIGINT, VARCHAR, BIGINT, DOUBLE, VARCHAR)
Expand Down
4 changes: 2 additions & 2 deletions presto-main/src/test/java/com/facebook/presto/operator/TestHashAggregationOperator.java
View file
Expand Up @@ -102,7 +102,7 @@ public void tearDown()
public void testHashAggregation(boolean hashEnabled)
throws Exception
{
MetadataManager metadata = new MetadataManager();
MetadataManager metadata = MetadataManager.createTestMetadataManager();
InternalAggregationFunction countVarcharColumn = metadata.resolveFunction(QualifiedName.of("count"), ImmutableList.of(parseTypeSignature(StandardTypes.VARCHAR)), false).getAggregationFunction();
InternalAggregationFunction countBooleanColumn = metadata.resolveFunction(QualifiedName.of("count"), ImmutableList.of(parseTypeSignature(StandardTypes.BOOLEAN)), false).getAggregationFunction();
InternalAggregationFunction maxVarcharColumn = metadata.resolveFunction(QualifiedName.of("max"), ImmutableList.of(parseTypeSignature(StandardTypes.VARCHAR)), false).getAggregationFunction();
Expand Down Expand Up @@ -150,7 +150,7 @@ public void testHashAggregation(boolean hashEnabled)
@Test(dataProvider = "hashEnabledValues", expectedExceptions = ExceededMemoryLimitException.class, expectedExceptionsMessageRegExp = "Task exceeded max memory size of 10B")
public void testMemoryLimit(boolean hashEnabled)
{
MetadataManager metadata = new MetadataManager();
MetadataManager metadata = MetadataManager.createTestMetadataManager();
InternalAggregationFunction maxVarcharColumn = metadata.resolveFunction(QualifiedName.of("max"), ImmutableList.of(parseTypeSignature(StandardTypes.VARCHAR)), false).getAggregationFunction();

List<Integer> hashChannels = Ints.asList(1);
Expand Down
6 changes: 3 additions & 3 deletions presto-main/src/test/java/com/facebook/presto/operator/TestUnnestOperator.java
View file
Expand Up @@ -68,7 +68,7 @@ public void tearDown()
public void testUnnest()
throws Exception
{
MetadataManager metadata = new MetadataManager();
MetadataManager metadata = MetadataManager.createTestMetadataManager();
Type arrayType = metadata.getType(parseTypeSignature("array<bigint>"));
Type mapType = metadata.getType(parseTypeSignature("map<bigint,bigint>"));

Expand Down Expand Up @@ -99,7 +99,7 @@ public void testUnnest()
public void testUnnestWithOrdinality()
throws Exception
{
MetadataManager metadata = new MetadataManager();
MetadataManager metadata = MetadataManager.createTestMetadataManager();
Type arrayType = metadata.getType(parseTypeSignature("array<bigint>"));
Type mapType = metadata.getType(parseTypeSignature("map<bigint,bigint>"));

Expand Down Expand Up @@ -130,7 +130,7 @@ public void testUnnestWithOrdinality()
public void testUnnestNonNumericDoubles()
throws Exception
{
MetadataManager metadata = new MetadataManager();
MetadataManager metadata = MetadataManager.createTestMetadataManager();
Type arrayType = metadata.getType(parseTypeSignature("array<double>"));
Type mapType = metadata.getType(parseTypeSignature("map<bigint,double>"));

Expand Down
2 changes: 1 addition & 1 deletion ...main/src/test/java/com/facebook/presto/operator/aggregation/TestArbitraryAggregation.java
View file
Expand Up @@ -32,7 +32,7 @@

public class TestArbitraryAggregation
{
private static final MetadataManager metadata = new MetadataManager();
private static final MetadataManager metadata = MetadataManager.createTestMetadataManager();

@Test
public void testAllRegistered()
Expand Down
2 changes: 1 addition & 1 deletion presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestArrayAggregation.java
View file
Expand Up @@ -34,7 +34,7 @@

public class TestArrayAggregation
{
private static final MetadataManager metadata = new MetadataManager();
private static final MetadataManager metadata = MetadataManager.createTestMetadataManager();

@Test
public void testEmpty()
Expand Down
2 changes: 1 addition & 1 deletion ...to-main/src/test/java/com/facebook/presto/operator/aggregation/TestMapAggAggregation.java
View file
Expand Up @@ -44,7 +44,7 @@

public class TestMapAggAggregation
{
private static final MetadataManager metadata = new MetadataManager();
private static final MetadataManager metadata = MetadataManager.createTestMetadataManager();

@Test
public void testDuplicateKeysValues()
Expand Down
2 changes: 1 addition & 1 deletion presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestMaxByAggregation.java
View file
Expand Up @@ -47,7 +47,7 @@

public class TestMaxByAggregation
{
private static final MetadataManager metadata = new MetadataManager();
private static final MetadataManager metadata = MetadataManager.createTestMetadataManager();

@BeforeClass
public void setup()
Expand Down
2 changes: 1 addition & 1 deletion presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestMinByAggregation.java
View file
Expand Up @@ -38,7 +38,7 @@

public class TestMinByAggregation
{
private static final MetadataManager metadata = new MetadataManager();
private static final MetadataManager metadata = MetadataManager.createTestMetadataManager();

@BeforeClass
public void setup()
Expand Down
2 changes: 1 addition & 1 deletion presto-main/src/test/java/com/facebook/presto/sql/TestExpressionInterpreter.java
View file
Expand Up @@ -86,7 +86,7 @@ public class TestExpressionInterpreter
.build();

private static final SqlParser SQL_PARSER = new SqlParser();
private static final Metadata METADATA = new MetadataManager();
private static final Metadata METADATA = MetadataManager.createTestMetadataManager();

@Test
public void testAnd()
Expand Down
5 changes: 4 additions & 1 deletion presto-main/src/test/java/com/facebook/presto/sql/analyzer/TestAnalyzer.java
View file
Expand Up @@ -14,6 +14,7 @@
package com.facebook.presto.sql.analyzer;

import com.facebook.presto.Session;
import com.facebook.presto.block.BlockEncodingManager;
import com.facebook.presto.metadata.MetadataManager;
import com.facebook.presto.metadata.QualifiedTableName;
import com.facebook.presto.metadata.TableMetadata;
Expand All @@ -22,6 +23,7 @@
import com.facebook.presto.spi.ColumnMetadata;
import com.facebook.presto.spi.ConnectorTableMetadata;
import com.facebook.presto.spi.SchemaTableName;
import com.facebook.presto.spi.type.TypeManager;
import com.facebook.presto.split.SplitManager;
import com.facebook.presto.sql.parser.SqlParser;
import com.facebook.presto.sql.tree.Statement;
Expand Down Expand Up @@ -748,7 +750,8 @@ public void testIfInJoinClause()
public void setup()
throws Exception
{
MetadataManager metadata = new MetadataManager(new FeaturesConfig().setExperimentalSyntaxEnabled(true), new TypeRegistry(), new SplitManager());
TypeManager typeManager = new TypeRegistry();
MetadataManager metadata = new MetadataManager(new FeaturesConfig().setExperimentalSyntaxEnabled(true), typeManager, new SplitManager(), new BlockEncodingManager(typeManager));
metadata.addConnectorMetadata("tpch", "tpch", new TestingMetadata());
metadata.addConnectorMetadata("c2", "c2", new TestingMetadata());
metadata.addConnectorMetadata("c3", "c3", new TestingMetadata());
Expand Down
2 changes: 1 addition & 1 deletion presto-main/src/test/java/com/facebook/presto/sql/gen/BenchmarkPageProcessor.java
View file
Expand Up @@ -82,7 +82,7 @@ public void setup()

handCodedProcessor = new Tpch1FilterAndProject();

compiledProcessor = new ExpressionCompiler(new MetadataManager()).compilePageProcessor(FILTER, ImmutableList.of(PROJECT));
compiledProcessor = new ExpressionCompiler(MetadataManager.createTestMetadataManager()).compilePageProcessor(FILTER, ImmutableList.of(PROJECT));
}

@Benchmark
Expand Down
2 changes: 1 addition & 1 deletion presto-main/src/test/java/com/facebook/presto/sql/planner/TestDomainTranslator.java
View file
Expand Up @@ -71,7 +71,7 @@

public class TestDomainTranslator
{
private static final Metadata METADATA = new MetadataManager();
private static final Metadata METADATA = MetadataManager.createTestMetadataManager();

private static final Symbol A = new Symbol("a");
private static final Symbol B = new Symbol("b");
Expand Down
2 changes: 1 addition & 1 deletion presto-main/src/test/java/com/facebook/presto/sql/planner/TestInterpretedFilterFunction.java
View file
Expand Up @@ -30,7 +30,7 @@
public class TestInterpretedFilterFunction
{
private static final SqlParser SQL_PARSER = new SqlParser();
private static final Metadata METADATA = new MetadataManager();
private static final Metadata METADATA = MetadataManager.createTestMetadataManager();

@Test
public void testNullLiteral()
Expand Down
2 changes: 1 addition & 1 deletion ...main/src/test/java/com/facebook/presto/sql/planner/TestInterpretedProjectionFunction.java
View file
Expand Up @@ -41,7 +41,7 @@
public class TestInterpretedProjectionFunction
{
private static final SqlParser SQL_PARSER = new SqlParser();
private static final Metadata METADATA = new MetadataManager();
private static final Metadata METADATA = MetadataManager.createTestMetadataManager();

@Test
public void testBooleanExpression()
Expand Down
2 changes: 1 addition & 1 deletion presto-ml/src/test/java/com/facebook/presto/ml/TestEvaluateClassifierPredictions.java
View file
Expand Up @@ -42,7 +42,7 @@

public class TestEvaluateClassifierPredictions
{
private final Metadata metadata = new MetadataManager();
private final Metadata metadata = MetadataManager.createTestMetadataManager();

@Test
public void testEvaluateClassifierPredictions()
Expand Down
6 changes: 4 additions & 2 deletions presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestQueryFramework.java
View file
Expand Up @@ -14,10 +14,11 @@
package com.facebook.presto.tests;

import com.facebook.presto.Session;
import com.facebook.presto.block.BlockEncodingManager;
import com.facebook.presto.index.IndexManager;
import com.facebook.presto.metadata.Metadata;
import com.facebook.presto.metadata.MetadataManager;
import com.facebook.presto.spi.type.Type;
import com.facebook.presto.spi.type.TypeManager;
import com.facebook.presto.split.SplitManager;
import com.facebook.presto.sql.analyzer.FeaturesConfig;
import com.facebook.presto.sql.analyzer.QueryExplainer;
Expand Down Expand Up @@ -160,7 +161,8 @@ public String getGraphvizExplainPlan(String query, ExplainType.Type planType)

private QueryExplainer getQueryExplainer()
{
Metadata metadata = new MetadataManager(new FeaturesConfig().setExperimentalSyntaxEnabled(true), new TypeRegistry(), new SplitManager());
TypeManager typeManager = new TypeRegistry();
MetadataManager metadata = new MetadataManager(new FeaturesConfig().setExperimentalSyntaxEnabled(true), typeManager, new SplitManager(), new BlockEncodingManager(typeManager));
FeaturesConfig featuresConfig = new FeaturesConfig().setExperimentalSyntaxEnabled(true).setOptimizeHashGeneration(true);
List<PlanOptimizer> optimizers = new PlanOptimizersFactory(metadata, sqlParser, new SplitManager(), new IndexManager(), featuresConfig).get();
return new QueryExplainer(queryRunner.getDefaultSession(),
Expand Down

0 comments on commit 2a110b0

Please sign in to comment.