Permalink
Browse files

Trying out using the presto parser for SQL

  • Loading branch information...
justinsb committed Dec 18, 2013
1 parent 2d7bcb2 commit 4a42b88fc0ba6a082a5e5625ff423ccebdd7ead4
Showing with 3,111 additions and 0 deletions.
  1. +12 −0 cloudata-structured/pom.xml
  2. +70 −0 cloudata-structured/src/main/java/com/cloudata/structured/sql/SqlEngine.java
  3. +18 −0 cloudata-structured/src/main/java/com/cloudata/structured/sql/SqlSession.java
  4. +116 −0 cloudata-structured/src/main/java/com/cloudata/structured/sql/SqlStatement.java
  5. +18 −0 cloudata-structured/src/main/java/com/cloudata/structured/sql/provider/CloudataColumnHandle.java
  6. +145 −0 ...data-structured/src/main/java/com/cloudata/structured/sql/provider/CloudataConnectorMetadata.java
  7. +40 −0 cloudata-structured/src/main/java/com/cloudata/structured/sql/provider/CloudataPartition.java
  8. +71 −0 cloudata-structured/src/main/java/com/cloudata/structured/sql/provider/CloudataSplit.java
  9. +84 −0 cloudata-structured/src/main/java/com/cloudata/structured/sql/provider/CloudataSplitManager.java
  10. +74 −0 cloudata-structured/src/main/java/com/cloudata/structured/sql/provider/CloudataTableHandle.java
  11. +190 −0 cloudata-structured/src/main/java/com/cloudata/structured/sql/simple/ConvertToSimplePlanVisitor.java
  12. +27 −0 cloudata-structured/src/main/java/com/cloudata/structured/sql/simple/SimpleColumnExpression.java
  13. +7 −0 cloudata-structured/src/main/java/com/cloudata/structured/sql/simple/SimpleExpression.java
  14. +19 −0 cloudata-structured/src/main/java/com/cloudata/structured/sql/simple/SimpleExpressionPrinter.java
  15. +12 −0 cloudata-structured/src/main/java/com/cloudata/structured/sql/simple/SimpleExpressionVisitor.java
  16. +6 −0 cloudata-structured/src/main/java/com/cloudata/structured/sql/simple/SimpleNode.java
  17. +11 −0 cloudata-structured/src/main/java/com/cloudata/structured/sql/simple/SimpleNodeVisitor.java
  18. +41 −0 cloudata-structured/src/main/java/com/cloudata/structured/sql/simple/SimpleTableScan.java
  19. +45 −0 cloudata-structured/src/main/java/com/cloudata/structured/sql/simple/SimpleTreePrinter.java
  20. +165 −0 cloudata-structured/src/test/java/com/cloudata/structured/sql/FakeRemoteTaskFactory.java
  21. +18 −0 cloudata-structured/src/test/java/com/cloudata/structured/sql/MockColumnHandle.java
  22. +141 −0 cloudata-structured/src/test/java/com/cloudata/structured/sql/MockConnectorMetadata.java
  23. +11 −0 cloudata-structured/src/test/java/com/cloudata/structured/sql/MockExchangeClientSupplier.java
  24. +45 −0 cloudata-structured/src/test/java/com/cloudata/structured/sql/MockLocalStorageManager.java
  25. +65 −0 cloudata-structured/src/test/java/com/cloudata/structured/sql/MockLocationFactory.java
  26. +41 −0 cloudata-structured/src/test/java/com/cloudata/structured/sql/MockNodeManager.java
  27. +40 −0 cloudata-structured/src/test/java/com/cloudata/structured/sql/MockPartition.java
  28. +54 −0 cloudata-structured/src/test/java/com/cloudata/structured/sql/MockPeriodicImportManager.java
  29. +26 −0 cloudata-structured/src/test/java/com/cloudata/structured/sql/MockRemoteTaskFactory.java
  30. +75 −0 cloudata-structured/src/test/java/com/cloudata/structured/sql/MockShardManager.java
  31. +71 −0 cloudata-structured/src/test/java/com/cloudata/structured/sql/MockSplit.java
  32. +80 −0 cloudata-structured/src/test/java/com/cloudata/structured/sql/MockSplitManager.java
  33. +25 −0 cloudata-structured/src/test/java/com/cloudata/structured/sql/MockStorageManager.java
  34. +74 −0 cloudata-structured/src/test/java/com/cloudata/structured/sql/MockTableHandle.java
  35. +442 −0 cloudata-structured/src/test/java/com/cloudata/structured/sql/Query.java
  36. +140 −0 cloudata-structured/src/test/java/com/cloudata/structured/sql/RecursivePlanVisitor.java
  37. +458 −0 cloudata-structured/src/test/java/com/cloudata/structured/sql/SqlTest.java
  38. +15 −0 cloudata-structured/src/test/java/com/cloudata/structured/sql/TableScanCountVisitor.java
  39. +119 −0 cloudata-structured/src/test/java/com/cloudata/structured/sql/TestMetadata.java
@@ -13,6 +13,18 @@
<dependencies>
<dependency>
<groupId>com.facebook.presto</groupId>
<artifactId>presto-parser</artifactId>
<version>0.55-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>com.facebook.presto</groupId>
<artifactId>presto-main</artifactId>
<version>0.55-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>com.cloudata</groupId>
<artifactId>cloudata-server-shared</artifactId>
@@ -0,0 +1,70 @@
package com.cloudata.structured.sql;
import java.util.List;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.facebook.presto.importer.PeriodicImportManager;
import com.facebook.presto.metadata.MetadataManager;
import com.facebook.presto.sql.analyzer.Analysis;
import com.facebook.presto.sql.analyzer.Analyzer;
import com.facebook.presto.sql.analyzer.QueryExplainer;
import com.facebook.presto.sql.parser.SqlParser;
import com.facebook.presto.sql.planner.LogicalPlanner;
import com.facebook.presto.sql.planner.Plan;
import com.facebook.presto.sql.planner.PlanNodeIdAllocator;
import com.facebook.presto.sql.planner.optimizations.PlanOptimizer;
import com.facebook.presto.sql.tree.Statement;
import com.facebook.presto.storage.StorageManager;
import com.google.common.base.Optional;
public class SqlEngine {
private static final Logger log = LoggerFactory.getLogger(SqlEngine.class);
final MetadataManager metadata;
final List<PlanOptimizer> planOptimizers;
final PeriodicImportManager periodicImportManager;
final StorageManager storageManager;
// SplitManager splitManager = buildSplitManager();
public SqlEngine(MetadataManager metadata, List<PlanOptimizer> planOptimizers,
PeriodicImportManager periodicImportManager, StorageManager storageManager) {
this.metadata = metadata;
this.planOptimizers = planOptimizers;
this.periodicImportManager = periodicImportManager;
this.storageManager = storageManager;
}
public SqlStatement parse(SqlSession session, String sql) {
log.debug("Parsing sql: {}", sql);
Statement statement = SqlParser.createStatement(sql);
QueryExplainer queryExplainer = new QueryExplainer(session.prestoSession, planOptimizers, metadata,
periodicImportManager, storageManager);
// analyze query
Analyzer analyzer = new Analyzer(session.prestoSession, metadata, Optional.of(queryExplainer));
Analysis analysis = analyzer.analyze(statement);
// System.out.println("analysis: " + analysis);
PlanNodeIdAllocator idAllocator = new PlanNodeIdAllocator();
// plan query
LogicalPlanner logicalPlanner = new LogicalPlanner(session.prestoSession, planOptimizers, idAllocator,
metadata, periodicImportManager, storageManager);
Plan plan = logicalPlanner.plan(analysis);
return new SqlStatement(metadata, sql, plan);
//
// TableScanCountVisitor visitor = new TableScanCountVisitor();
// plan.getRoot().accept(visitor, 0);
// Assert.assertEquals(1, visitor.count);
// String p = PlanPrinter.textLogicalPlan(plan.getRoot(), plan.getTypes());
//
// System.out.println("plan: " + p);
}
}
@@ -0,0 +1,18 @@
package com.cloudata.structured.sql;
import com.facebook.presto.sql.analyzer.Session;
public class SqlSession {
final Session prestoSession;
public SqlSession() {
String user = "user";
String source = "source";
String catalog = "default";
String schema = "default";
String remoteUserAddress = "remoteUserAddress";
String userAgent = "userAgent";
this.prestoSession = new Session(user, source, catalog, schema, remoteUserAddress, userAgent);
}
}
@@ -0,0 +1,116 @@
package com.cloudata.structured.sql;
import com.cloudata.structured.sql.simple.ConvertToSimplePlanVisitor;
import com.cloudata.structured.sql.simple.SimpleNode;
import com.facebook.presto.metadata.Metadata;
import com.facebook.presto.sql.planner.Plan;
import com.google.common.base.Optional;
public class SqlStatement {
private final String sql;
private final Plan plan;
Optional<SimpleNode> simple;
private final Metadata metadata;
public SqlStatement(Metadata metadata, String sql, Plan plan) {
this.metadata = metadata;
this.sql = sql;
this.plan = plan;
}
public boolean isSimple() {
return getSimple() != null;
}
public SimpleNode getSimple() {
if (simple == null) {
ConvertToSimplePlanVisitor visitor = new ConvertToSimplePlanVisitor(metadata);
SimpleNode accept = plan.getRoot().accept(visitor, null);
// plan.getRoot().accept(visitor, 0);
simple = Optional.fromNullable(accept);
}
return simple.orNull();
}
// class SimplePlan {
// final Table table;
//
// final Expression[] expressions;
//
// public void evaluate() {
// // Statement: Query{queryBody=QuerySpecification{select=Select{distinct=false, selectItems=["key1" k1,
// // concat("key2", 'hello') k2]}, from=[Table{table1}], where=null, groupBy=[], having=null, orderBy=[],
// // limit=null}, orderBy=[]}
// // analysis: com.facebook.presto.sql.analyzer.Analysis@39f46204
// // plan: - Output[k1, k2]
// // k1 := key1
// // k2 := concat
// // - Project => [key1:varchar, concat:varchar]
// // concat := concat("key2", 'hello')
// // - TableScan[com.cloudata.structured.sql.MockTableHandle@737c45ee, domain={}] => [key1:varchar,
// // key2:varchar]
// // key1 := com.cloudata.structured.sql.MockColumnHandle@549448df
// // key2 := com.cloudata.structured.sql.MockColumnHandle@533c53da
//
// // ExpressionInterpreter.expressionInterpreter(expression, metadata, session)
//
// }
// }
//
// private boolean isSimple(Plan plan) {
//
// // TODO: Make this better, once we have a better grip on the logic
// PlanNode root = plan.getRoot();
// if (root instanceof OutputNode) {
// OutputNode outputNode = (OutputNode) root;
// PlanNode source = outputNode.getSource();
// if (source instanceof TableScanNode) {
// TableScanNode tableScanNode = (TableScanNode) source;
//
// List<String> columns = Lists.newArrayList();
// // List<String> columns = Lists.newArrayList();
//
// for (int i = 0; i < outputNode.getColumnNames().size(); i++) {
// String name = outputNode.getColumnNames().get(i);
// Symbol symbol = outputNode.getOutputSymbols().get(i);
//
// }
//
// // Statement: Query{queryBody=QuerySpecification{select=Select{distinct=false, selectItems=["key1" k1,
// // "key2" k2]}, from=[Table{table1}], where=null, groupBy=[], having=null, orderBy=[], limit=null},
// // orderBy=[]}
// // analysis: com.facebook.presto.sql.analyzer.Analysis@6fcc5b5d
// // plan: - Output[k1, k2]
// // k1 := key1
// // k2 := key2
// // - TableScan[com.cloudata.structured.sql.MockTableHandle@3aa92b03, domain={}] => [key1:varchar,
// // key2:varchar]
// // key1 := com.cloudata.structured.sql.MockColumnHandle@20bb82ca
// // key2 := com.cloudata.structured.sql.MockColumnHandle@7687ac8f
//
// SimpleQuery query = new SimpleQuery();
// return true;
// }
// }
// return false;
// }
// class CheckSimpleVisitor extends RecursivePlanVisitor<Integer, Boolean> {
// boolean simple = true;
//
// @Override
// public Boolean visitOutput(OutputNode node, Integer context) {
// simple = false;
// return false;
// }
//
// @Override
// protected Boolean visitPlan(PlanNode node, Integer context) {
// if (vi)
// }
//
// }
}
@@ -0,0 +1,18 @@
package com.cloudata.structured.sql.provider;
import com.facebook.presto.spi.ColumnHandle;
import com.facebook.presto.spi.ColumnMetadata;
public class CloudataColumnHandle implements ColumnHandle {
private final ColumnMetadata columnMetadata;
public CloudataColumnHandle(CloudataTableHandle tableHandle, ColumnMetadata columnMetadata) {
this.columnMetadata = columnMetadata;
}
public ColumnMetadata getColumnMetadata() {
return columnMetadata;
}
}
@@ -0,0 +1,145 @@
package com.cloudata.structured.sql.provider;
import java.util.List;
import java.util.Map;
import com.facebook.presto.spi.ColumnHandle;
import com.facebook.presto.spi.ColumnMetadata;
import com.facebook.presto.spi.ConnectorMetadata;
import com.facebook.presto.spi.ConnectorTableMetadata;
import com.facebook.presto.spi.SchemaTableName;
import com.facebook.presto.spi.SchemaTablePrefix;
import com.facebook.presto.spi.TableHandle;
import com.google.common.collect.Lists;
public class CloudataConnectorMetadata implements ConnectorMetadata {
final String connectorId;
public CloudataConnectorMetadata(String connectorId) {
this.connectorId = connectorId;
}
@Override
public boolean canHandle(TableHandle tableHandle) {
return tableHandle instanceof CloudataTableHandle
&& ((CloudataTableHandle) tableHandle).getConnectorId().equals(connectorId);
}
@Override
public List<String> listSchemaNames() {
List<String> schemas = Lists.newArrayList();
schemas.add("default");
return schemas;
}
@Override
public CloudataTableHandle getTableHandle(SchemaTableName tableName) {
if (!listSchemaNames().contains(tableName.getSchemaName())) {
return null;
}
// ExampleTable table = exampleClient.getTable(tableName.getSchemaName(), tableName.getTableName());
// if (table == null) {
// return null;
// }
return new CloudataTableHandle(connectorId, tableName.getSchemaName(), tableName.getTableName());
}
@Override
public ConnectorTableMetadata getTableMetadata(TableHandle table) {
CloudataTableHandle tableHandle = promote(table);
return tableHandle.getTableMetadata();
}
private CloudataTableHandle promote(TableHandle table) {
assert (table != null);
// checkArgument(table instanceof CloudataTableHandle, "tableHandle is not an instance of CloudataTableHandle");
assert (table instanceof CloudataTableHandle);
CloudataTableHandle tableHandle = (CloudataTableHandle) table;
assert tableHandle.getConnectorId().equals(connectorId);
// checkArgument(tableHandle.getConnectorId().equals(connectorId), "tableHandle is not for this connector");
return tableHandle;
}
@Override
public List<SchemaTableName> listTables(String schemaNameOrNull) {
throw new UnsupportedOperationException();
}
@Override
public ColumnHandle getColumnHandle(TableHandle tableHandle, String columnName) {
CloudataTableHandle exampleTableHandle = promote(tableHandle);
// ExampleTable table = exampleClient.getTable(exampleTableHandle.getSchemaName(),
// exampleTableHandle.getTableName());
// if (table == null) {
// throw new TableNotFoundException(exampleTableHandle.toSchemaTableName());
// }
// ImmutableMap.Builder<String, ColumnHandle> columnHandles = ImmutableMap.builder();
// for (ColumnMetadata columnMetadata : table.getColumnsMetadata()) {
// columnHandles.put(columnMetadata.getName(), new ExampleColumnHandle(connectorId, columnMetadata));
// }
// return columnHandles.build();
return exampleTableHandle.getColumnHandle(columnName);
}
@Override
public Map<String, ColumnHandle> getColumnHandles(TableHandle tableHandle) {
CloudataTableHandle exampleTableHandle = promote(tableHandle);
// ExampleTable table = exampleClient.getTable(exampleTableHandle.getSchemaName(),
// exampleTableHandle.getTableName());
// if (table == null) {
// throw new TableNotFoundException(exampleTableHandle.toSchemaTableName());
// }
// ImmutableMap.Builder<String, ColumnHandle> columnHandles = ImmutableMap.builder();
// for (ColumnMetadata columnMetadata : table.getColumnsMetadata()) {
// columnHandles.put(columnMetadata.getName(), new ExampleColumnHandle(connectorId, columnMetadata));
// }
// return columnHandles.build();
return exampleTableHandle.getColumnHandles();
}
@Override
public ColumnMetadata getColumnMetadata(TableHandle tableHandle, ColumnHandle column) {
// checkNotNull(tableHandle, "tableHandle is null");
// checkArgument(tableHandle instanceof MockTableHandle, "tableHandle is not an instance of MockTableHandle");
// checkArgument(((MockTableHandle) tableHandle).getConnectorId().equals(connectorId),
// "tableHandle is not for this connector");
CloudataColumnHandle columnHandle = promote(column);
return columnHandle.getColumnMetadata();
}
private CloudataColumnHandle promote(ColumnHandle column) {
// checkNotNull(columnHandle, "columnHandle is null");
assert column != null;
// checkArgument(columnHandle instanceof CloudataColumnHandle,
// "columnHandle is not an instance of CloudataColumnHandle");
assert column instanceof CloudataColumnHandle;
return (CloudataColumnHandle) column;
}
@Override
public Map<SchemaTableName, List<ColumnMetadata>> listTableColumns(SchemaTablePrefix prefix) {
throw new UnsupportedOperationException();
}
@Override
public TableHandle createTable(ConnectorTableMetadata tableMetadata) {
throw new UnsupportedOperationException();
}
@Override
public void dropTable(TableHandle tableHandle) {
throw new UnsupportedOperationException();
}
}
Oops, something went wrong.

0 comments on commit 4a42b88

Please sign in to comment.