diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestOrcPageSourceMemoryTracking.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestOrcPageSourceMemoryTracking.java index 5434015d96b4..76fe75af781a 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestOrcPageSourceMemoryTracking.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestOrcPageSourceMemoryTracking.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.hive; +import com.facebook.presto.connector.ConnectorId; import com.facebook.presto.hive.orc.OrcPageSourceFactory; import com.facebook.presto.metadata.Split; import com.facebook.presto.operator.DriverContext; @@ -405,7 +406,7 @@ public SourceOperator newTableScanOperator(DriverContext driverContext) columns.stream().map(columnHandle -> (ColumnHandle) columnHandle).collect(toList()) ); SourceOperator operator = sourceOperatorFactory.createOperator(driverContext); - operator.addSplit(new Split("test", TestingTransactionHandle.create("test"), TestingSplit.createLocalSplit())); + operator.addSplit(new Split(new ConnectorId("test"), TestingTransactionHandle.create(), TestingSplit.createLocalSplit())); return operator; } @@ -428,7 +429,7 @@ public SourceOperator newScanFilterAndProjectOperator(DriverContext driverContex types ); SourceOperator operator = sourceOperatorFactory.createOperator(driverContext); - operator.addSplit(new Split("test", TestingTransactionHandle.create("test"), TestingSplit.createLocalSplit())); + operator.addSplit(new Split(new ConnectorId("test"), TestingTransactionHandle.create(), TestingSplit.createLocalSplit())); return operator; } diff --git a/presto-main/src/main/java/com/facebook/presto/Session.java b/presto-main/src/main/java/com/facebook/presto/Session.java index fb55aeb1bad6..dc3f13fc633e 100644 --- a/presto-main/src/main/java/com/facebook/presto/Session.java +++ b/presto-main/src/main/java/com/facebook/presto/Session.java @@ -14,6 +14,7 @@ package com.facebook.presto; import com.facebook.presto.client.ClientSession; +import com.facebook.presto.connector.ConnectorId; import com.facebook.presto.metadata.SessionPropertyManager; import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.QueryId; @@ -335,17 +336,17 @@ public ConnectorSession toConnectorSession() return new FullConnectorSession(queryId.toString(), identity, timeZoneKey, locale, startTime); } - public ConnectorSession toConnectorSession(String catalog) + public ConnectorSession toConnectorSession(ConnectorId connectorId) { - requireNonNull(catalog, "catalog is null"); + requireNonNull(connectorId, "connectorId is null"); return new FullConnectorSession( queryId.toString(), identity, timeZoneKey, locale, startTime, - catalogProperties.getOrDefault(catalog, ImmutableMap.of()), - catalog, + catalogProperties.getOrDefault(connectorId.getCatalogName(), ImmutableMap.of()), + connectorId.getCatalogName(), sessionPropertyManager); } diff --git a/presto-main/src/main/java/com/facebook/presto/connector/ConnectorAwareNodeManager.java b/presto-main/src/main/java/com/facebook/presto/connector/ConnectorAwareNodeManager.java index e1712872f803..96dd87ec5c56 100644 --- a/presto-main/src/main/java/com/facebook/presto/connector/ConnectorAwareNodeManager.java +++ b/presto-main/src/main/java/com/facebook/presto/connector/ConnectorAwareNodeManager.java @@ -27,9 +27,9 @@ public class ConnectorAwareNodeManager { private final InternalNodeManager nodeManager; private final String environment; - private final String connectorId; + private final ConnectorId connectorId; - public ConnectorAwareNodeManager(InternalNodeManager nodeManager, String environment, String connectorId) + public ConnectorAwareNodeManager(InternalNodeManager nodeManager, String environment, ConnectorId connectorId) { this.nodeManager = requireNonNull(nodeManager, "nodeManager is null"); this.environment = requireNonNull(environment, "environment is null"); diff --git a/presto-main/src/main/java/com/facebook/presto/connector/ConnectorId.java b/presto-main/src/main/java/com/facebook/presto/connector/ConnectorId.java new file mode 100644 index 000000000000..bf761d5cb758 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/connector/ConnectorId.java @@ -0,0 +1,84 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.connector; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonValue; + +import java.util.Objects; + +import static com.google.common.base.Preconditions.checkArgument; +import static java.util.Objects.requireNonNull; + +public final class ConnectorId +{ + private static final String INFORMATION_SCHEMA_CONNECTOR_PREFIX = "$info_schema@"; + private static final String SYSTEM_TABLES_CONNECTOR_PREFIX = "$system@"; + + private final String catalogName; + + @JsonCreator + public ConnectorId(String catalogName) + { + this.catalogName = requireNonNull(catalogName, "catalogName is null"); + checkArgument(!catalogName.isEmpty(), "catalogName is empty"); + } + + public String getCatalogName() + { + return catalogName; + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + ConnectorId that = (ConnectorId) o; + return Objects.equals(catalogName, that.catalogName); + } + + @Override + public int hashCode() + { + return Objects.hash(catalogName); + } + + @JsonValue + @Override + public String toString() + { + return catalogName; + } + + public static boolean isInternalSystemConnector(ConnectorId connectorId) + { + return connectorId.getCatalogName().startsWith(SYSTEM_TABLES_CONNECTOR_PREFIX) || + connectorId.getCatalogName().startsWith(INFORMATION_SCHEMA_CONNECTOR_PREFIX); + } + + public static ConnectorId createInformationSchemaConnectorId(ConnectorId connectorId) + { + return new ConnectorId(INFORMATION_SCHEMA_CONNECTOR_PREFIX + connectorId.getCatalogName()); + } + + public static ConnectorId createSystemTablesConnectorId(ConnectorId connectorId) + { + return new ConnectorId(SYSTEM_TABLES_CONNECTOR_PREFIX + connectorId.getCatalogName()); + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/connector/ConnectorManager.java b/presto-main/src/main/java/com/facebook/presto/connector/ConnectorManager.java index 9e8d25c92d94..5a41f9463d7f 100644 --- a/presto-main/src/main/java/com/facebook/presto/connector/ConnectorManager.java +++ b/presto-main/src/main/java/com/facebook/presto/connector/ConnectorManager.java @@ -59,6 +59,8 @@ import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.AtomicBoolean; +import static com.facebook.presto.connector.ConnectorId.createInformationSchemaConnectorId; +import static com.facebook.presto.connector.ConnectorId.createSystemTablesConnectorId; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; import static com.google.common.collect.Sets.newConcurrentHashSet; @@ -68,9 +70,6 @@ @ThreadSafe public class ConnectorManager { - public static final String INFORMATION_SCHEMA_CONNECTOR_PREFIX = "$info_schema@"; - public static final String SYSTEM_TABLES_CONNECTOR_PREFIX = "$system@"; - private static final Logger log = Logger.get(ConnectorManager.class); private final MetadataManager metadataManager; @@ -95,7 +94,7 @@ public class ConnectorManager @GuardedBy("this") private final Set catalogs = newConcurrentHashSet(); @GuardedBy("this") - private final ConcurrentMap connectors = new ConcurrentHashMap<>(); + private final ConcurrentMap connectors = new ConcurrentHashMap<>(); private final AtomicBoolean stopped = new AtomicBoolean(); @@ -138,7 +137,7 @@ public synchronized void stop() return; } - for (Map.Entry entry : connectors.entrySet()) { + for (Map.Entry entry : connectors.entrySet()) { Connector connector = entry.getValue(); try (ThreadContextClassLoader ignored = new ThreadContextClassLoader(connector.getClass().getClassLoader())) { connector.shutdown(); @@ -163,15 +162,15 @@ public synchronized void addConnectorFactory(ConnectorFactory connectorFactory) handleResolver.addConnectorName(connectorFactory.getName(), connectorFactory.getHandleResolver()); } - public synchronized void createConnection(String catalogName, String connectorName, Map properties) + public synchronized ConnectorId createConnection(String catalogName, String connectorName, Map properties) { requireNonNull(connectorName, "connectorName is null"); ConnectorFactory connectorFactory = connectorFactories.get(connectorName); checkArgument(connectorFactory != null, "No factory for connector %s", connectorName); - createConnection(catalogName, connectorFactory, properties); + return createConnection(catalogName, connectorFactory, properties); } - private synchronized void createConnection(String catalogName, ConnectorFactory connectorFactory, Map properties) + private synchronized ConnectorId createConnection(String catalogName, ConnectorFactory connectorFactory, Map properties) { checkState(!stopped.get(), "ConnectorManager is stopped"); requireNonNull(catalogName, "catalogName is null"); @@ -179,24 +178,26 @@ private synchronized void createConnection(String catalogName, ConnectorFactory requireNonNull(connectorFactory, "connectorFactory is null"); checkArgument(!catalogs.contains(catalogName), "A catalog already exists for %s", catalogName); - String connectorId = getConnectorId(catalogName); + ConnectorId connectorId = new ConnectorId(catalogName); checkState(!connectors.containsKey(connectorId), "A connector %s already exists", connectorId); addCatalogConnector(catalogName, connectorId, connectorFactory, properties); catalogs.add(catalogName); + + return connectorId; } - private synchronized void addCatalogConnector(String catalogName, String connectorId, ConnectorFactory factory, Map properties) + private synchronized void addCatalogConnector(String catalogName, ConnectorId connectorId, ConnectorFactory factory, Map properties) { Connector connector = createConnector(connectorId, factory, properties); addConnectorInternal(ConnectorType.STANDARD, catalogName, connectorId, connector); - String informationSchemaId = makeInformationSchemaConnectorId(connectorId); + ConnectorId informationSchemaId = createInformationSchemaConnectorId(connectorId); addConnectorInternal(ConnectorType.INFORMATION_SCHEMA, catalogName, informationSchemaId, new InformationSchemaConnector(catalogName, nodeManager, metadataManager)); - String systemId = makeSystemTablesConnectorId(connectorId); + ConnectorId systemId = createSystemTablesConnectorId(connectorId); addConnectorInternal(ConnectorType.SYSTEM, catalogName, systemId, new SystemConnector( systemId, nodeManager, @@ -208,7 +209,7 @@ private synchronized void addCatalogConnector(String catalogName, String connect metadataManager.getTablePropertyManager().addProperties(catalogName, connector.getTableProperties()); } - private synchronized void addConnectorInternal(ConnectorType type, String catalogName, String connectorId, Connector connector) + private synchronized void addConnectorInternal(ConnectorType type, String catalogName, ConnectorId connectorId, Connector connector) { checkState(!stopped.get(), "ConnectorManager is stopped"); checkState(!connectors.containsKey(connectorId), "A connector %s already exists", connectorId); @@ -327,7 +328,7 @@ else if (type == ConnectorType.SYSTEM) { } } - private Connector createConnector(String connectorId, ConnectorFactory factory, Map properties) + private Connector createConnector(ConnectorId connectorId, ConnectorFactory factory, Map properties) { Class factoryClass = factory.getClass(); if (factory instanceof LegacyTransactionConnectorFactory) { @@ -341,7 +342,7 @@ private Connector createConnector(String connectorId, ConnectorFactory factory, pageIndexerFactory); try (ThreadContextClassLoader ignored = new ThreadContextClassLoader(factoryClass.getClassLoader())) { - return factory.create(connectorId, properties, context); + return factory.create(connectorId.getCatalogName(), properties, context); } } @@ -351,20 +352,4 @@ private enum ConnectorType INFORMATION_SCHEMA, SYSTEM } - - private static String makeInformationSchemaConnectorId(String connectorId) - { - return INFORMATION_SCHEMA_CONNECTOR_PREFIX + connectorId; - } - - private static String makeSystemTablesConnectorId(String connectorId) - { - return SYSTEM_TABLES_CONNECTOR_PREFIX + connectorId; - } - - private static String getConnectorId(String catalogName) - { - // for now connectorId == catalogName - return catalogName; - } } diff --git a/presto-main/src/main/java/com/facebook/presto/connector/system/CatalogSystemTable.java b/presto-main/src/main/java/com/facebook/presto/connector/system/CatalogSystemTable.java index 3c0e699638b9..13c0c32938cb 100644 --- a/presto-main/src/main/java/com/facebook/presto/connector/system/CatalogSystemTable.java +++ b/presto-main/src/main/java/com/facebook/presto/connector/system/CatalogSystemTable.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.connector.system; +import com.facebook.presto.connector.ConnectorId; import com.facebook.presto.metadata.Metadata; import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.ConnectorTableMetadata; @@ -66,8 +67,8 @@ public ConnectorTableMetadata getTableMetadata() public RecordCursor cursor(ConnectorTransactionHandle transactionHandle, ConnectorSession session, TupleDomain constraint) { Builder table = InMemoryRecordSet.builder(CATALOG_TABLE); - for (Map.Entry entry : metadata.getCatalogNames().entrySet()) { - table.addRow(entry.getKey(), entry.getValue()); + for (Map.Entry entry : metadata.getCatalogNames().entrySet()) { + table.addRow(entry.getKey(), entry.getValue().toString()); } return table.build().cursor(); } diff --git a/presto-main/src/main/java/com/facebook/presto/connector/system/SystemColumnHandle.java b/presto-main/src/main/java/com/facebook/presto/connector/system/SystemColumnHandle.java index a2f39a8c8f61..ab3343e39db0 100644 --- a/presto-main/src/main/java/com/facebook/presto/connector/system/SystemColumnHandle.java +++ b/presto-main/src/main/java/com/facebook/presto/connector/system/SystemColumnHandle.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.connector.system; +import com.facebook.presto.connector.ConnectorId; import com.facebook.presto.spi.ColumnHandle; import com.facebook.presto.spi.ColumnMetadata; import com.facebook.presto.spi.ConnectorTableMetadata; @@ -28,12 +29,12 @@ public class SystemColumnHandle implements ColumnHandle { - private final String connectorId; + private final ConnectorId connectorId; private final String columnName; @JsonCreator public SystemColumnHandle( - @JsonProperty("connectorId") String connectorId, + @JsonProperty("connectorId") ConnectorId connectorId, @JsonProperty("columnName") String columnName) { this.connectorId = requireNonNull(connectorId, "connectorId is null"); @@ -41,7 +42,7 @@ public SystemColumnHandle( } @JsonProperty - public String getConnectorId() + public ConnectorId getConnectorId() { return connectorId; } @@ -78,7 +79,7 @@ public String toString() return connectorId + ":" + columnName; } - public static Map toSystemColumnHandles(String connectorId, ConnectorTableMetadata tableMetadata) + public static Map toSystemColumnHandles(ConnectorId connectorId, ConnectorTableMetadata tableMetadata) { ImmutableMap.Builder columnHandles = ImmutableMap.builder(); for (ColumnMetadata columnMetadata : tableMetadata.getColumns()) { diff --git a/presto-main/src/main/java/com/facebook/presto/connector/system/SystemConnector.java b/presto-main/src/main/java/com/facebook/presto/connector/system/SystemConnector.java index f03e2ff754b9..b4d37f66badc 100644 --- a/presto-main/src/main/java/com/facebook/presto/connector/system/SystemConnector.java +++ b/presto-main/src/main/java/com/facebook/presto/connector/system/SystemConnector.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.connector.system; +import com.facebook.presto.connector.ConnectorId; import com.facebook.presto.metadata.InternalNodeManager; import com.facebook.presto.spi.SystemTable; import com.facebook.presto.spi.connector.ConnectorMetadata; @@ -31,14 +32,14 @@ public class SystemConnector implements InternalConnector { - private final String connectorId; + private final ConnectorId connectorId; private final ConnectorMetadata metadata; private final ConnectorSplitManager splitManager; private final ConnectorPageSourceProvider pageSourceProvider; private final Function transactionHandleFunction; public SystemConnector( - String connectorId, + ConnectorId connectorId, InternalNodeManager nodeManager, Set tables, Function transactionHandleFunction) diff --git a/presto-main/src/main/java/com/facebook/presto/connector/system/SystemSplit.java b/presto-main/src/main/java/com/facebook/presto/connector/system/SystemSplit.java index 67d7e4c0e6fd..7ec85a1b5829 100644 --- a/presto-main/src/main/java/com/facebook/presto/connector/system/SystemSplit.java +++ b/presto-main/src/main/java/com/facebook/presto/connector/system/SystemSplit.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.connector.system; +import com.facebook.presto.connector.ConnectorId; import com.facebook.presto.spi.ColumnHandle; import com.facebook.presto.spi.ConnectorSplit; import com.facebook.presto.spi.HostAddress; @@ -30,19 +31,19 @@ public class SystemSplit implements ConnectorSplit { - private final String connectorId; + private final ConnectorId connectorId; private final SystemTableHandle tableHandle; private final List addresses; private final TupleDomain constraint; - public SystemSplit(String connectorId, SystemTableHandle tableHandle, HostAddress address, TupleDomain constraint) + public SystemSplit(ConnectorId connectorId, SystemTableHandle tableHandle, HostAddress address, TupleDomain constraint) { this(connectorId, tableHandle, ImmutableList.of(requireNonNull(address, "address is null")), constraint); } @JsonCreator public SystemSplit( - @JsonProperty("connectorId") String connectorId, + @JsonProperty("connectorId") ConnectorId connectorId, @JsonProperty("tableHandle") SystemTableHandle tableHandle, @JsonProperty("addresses") List addresses, @JsonProperty("constraint") TupleDomain constraint) @@ -57,7 +58,7 @@ public SystemSplit( } @JsonProperty - public String getConnectorId() + public ConnectorId getConnectorId() { return connectorId; } diff --git a/presto-main/src/main/java/com/facebook/presto/connector/system/SystemTableHandle.java b/presto-main/src/main/java/com/facebook/presto/connector/system/SystemTableHandle.java index e64e9f0577b6..69a0e3586e9f 100644 --- a/presto-main/src/main/java/com/facebook/presto/connector/system/SystemTableHandle.java +++ b/presto-main/src/main/java/com/facebook/presto/connector/system/SystemTableHandle.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.connector.system; +import com.facebook.presto.connector.ConnectorId; import com.facebook.presto.spi.ConnectorTableHandle; import com.facebook.presto.spi.SchemaTableName; import com.fasterxml.jackson.annotation.JsonCreator; @@ -27,13 +28,13 @@ public class SystemTableHandle implements ConnectorTableHandle { - private final String connectorId; + private final ConnectorId connectorId; private final String schemaName; private final String tableName; @JsonCreator public SystemTableHandle( - @JsonProperty("connectorId") String connectorId, + @JsonProperty("connectorId") ConnectorId connectorId, @JsonProperty("schemaName") String schemaName, @JsonProperty("tableName") String tableName) { @@ -42,14 +43,14 @@ public SystemTableHandle( this.tableName = checkTableName(tableName); } - public static SystemTableHandle fromSchemaTableName(String connectorId, SchemaTableName tableName) + public static SystemTableHandle fromSchemaTableName(ConnectorId connectorId, SchemaTableName tableName) { requireNonNull(tableName, "tableName is null"); return new SystemTableHandle(connectorId, tableName.getSchemaName(), tableName.getTableName()); } @JsonProperty - public String getConnectorId() + public ConnectorId getConnectorId() { return connectorId; } diff --git a/presto-main/src/main/java/com/facebook/presto/connector/system/SystemTableLayoutHandle.java b/presto-main/src/main/java/com/facebook/presto/connector/system/SystemTableLayoutHandle.java index c8dfc52ae99b..febffc493d4e 100644 --- a/presto-main/src/main/java/com/facebook/presto/connector/system/SystemTableLayoutHandle.java +++ b/presto-main/src/main/java/com/facebook/presto/connector/system/SystemTableLayoutHandle.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.connector.system; +import com.facebook.presto.connector.ConnectorId; import com.facebook.presto.spi.ColumnHandle; import com.facebook.presto.spi.ConnectorTableLayoutHandle; import com.facebook.presto.spi.predicate.TupleDomain; @@ -24,13 +25,13 @@ public class SystemTableLayoutHandle implements ConnectorTableLayoutHandle { - private final String connectorId; + private final ConnectorId connectorId; private final SystemTableHandle table; private final TupleDomain constraint; @JsonCreator public SystemTableLayoutHandle( - @JsonProperty("connectorId") String connectorId, + @JsonProperty("connectorId") ConnectorId connectorId, @JsonProperty("table") SystemTableHandle table, @JsonProperty("constraint") TupleDomain constraint) { @@ -40,7 +41,7 @@ public SystemTableLayoutHandle( } @JsonProperty - public String getConnectorId() + public ConnectorId getConnectorId() { return connectorId; } diff --git a/presto-main/src/main/java/com/facebook/presto/connector/system/SystemTablesMetadata.java b/presto-main/src/main/java/com/facebook/presto/connector/system/SystemTablesMetadata.java index 05885e4c3cb7..4291b0da7f7b 100644 --- a/presto-main/src/main/java/com/facebook/presto/connector/system/SystemTablesMetadata.java +++ b/presto-main/src/main/java/com/facebook/presto/connector/system/SystemTablesMetadata.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.connector.system; +import com.facebook.presto.connector.ConnectorId; import com.facebook.presto.spi.ColumnHandle; import com.facebook.presto.spi.ColumnMetadata; import com.facebook.presto.spi.ConnectorSession; @@ -47,10 +48,10 @@ public class SystemTablesMetadata implements ConnectorMetadata { - private final String connectorId; + private final ConnectorId connectorId; private final Map tables; - public SystemTablesMetadata(String connectorId, Set tables) + public SystemTablesMetadata(ConnectorId connectorId, Set tables) { this.connectorId = requireNonNull(connectorId, "connectorId"); this.tables = tables.stream() diff --git a/presto-main/src/main/java/com/facebook/presto/connector/system/SystemTransactionHandle.java b/presto-main/src/main/java/com/facebook/presto/connector/system/SystemTransactionHandle.java index 00fd5e767edd..4c57b51f660b 100644 --- a/presto-main/src/main/java/com/facebook/presto/connector/system/SystemTransactionHandle.java +++ b/presto-main/src/main/java/com/facebook/presto/connector/system/SystemTransactionHandle.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.connector.system; +import com.facebook.presto.connector.ConnectorId; import com.facebook.presto.spi.connector.ConnectorTransactionHandle; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; @@ -25,12 +26,12 @@ public class SystemTransactionHandle implements ConnectorTransactionHandle { - private final String connectorId; + private final ConnectorId connectorId; private final ConnectorTransactionHandle transactionHandle; @JsonCreator public SystemTransactionHandle( - @JsonProperty("connectorId") String connectorId, + @JsonProperty("connectorId") ConnectorId connectorId, @JsonProperty("transactionHandle") ConnectorTransactionHandle transactionHandle) { this.connectorId = requireNonNull(connectorId, "connectorId is null"); @@ -38,7 +39,7 @@ public SystemTransactionHandle( } @JsonProperty - public String getConnectorId() + public ConnectorId getConnectorId() { return connectorId; } diff --git a/presto-main/src/main/java/com/facebook/presto/connector/system/TransactionsSystemTable.java b/presto-main/src/main/java/com/facebook/presto/connector/system/TransactionsSystemTable.java index a2876466b0d0..4e93e1d35848 100644 --- a/presto-main/src/main/java/com/facebook/presto/connector/system/TransactionsSystemTable.java +++ b/presto-main/src/main/java/com/facebook/presto/connector/system/TransactionsSystemTable.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.connector.system; +import com.facebook.presto.connector.ConnectorId; import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.ConnectorTableMetadata; import com.facebook.presto.spi.InMemoryRecordSet; @@ -94,22 +95,22 @@ public RecordCursor cursor(ConnectorTransactionHandle transactionHandle, Connect info.isAutoCommitContext(), info.getCreateTime().getMillis(), (long) info.getIdleTime().getValue(TimeUnit.SECONDS), - info.getWrittenConnectorId().orElse(null), + info.getWrittenConnectorId().map(ConnectorId::getCatalogName).orElse(null), createStringsBlock(info.getConnectorIds())); } return table.build().cursor(); } - private static Block createStringsBlock(List values) + private static Block createStringsBlock(List values) { VarcharType varchar = createUnboundedVarcharType(); BlockBuilder builder = varchar.createBlockBuilder(new BlockBuilderStatus(), values.size()); - for (String value : values) { + for (ConnectorId value : values) { if (value == null) { builder.appendNull(); } else { - varchar.writeString(builder, value); + varchar.writeString(builder, value.getCatalogName()); } } return builder.build(); diff --git a/presto-main/src/main/java/com/facebook/presto/event/query/QueryMonitor.java b/presto-main/src/main/java/com/facebook/presto/event/query/QueryMonitor.java index 9ed208c7885d..9a239a93315b 100644 --- a/presto-main/src/main/java/com/facebook/presto/event/query/QueryMonitor.java +++ b/presto-main/src/main/java/com/facebook/presto/event/query/QueryMonitor.java @@ -136,7 +136,7 @@ public void queryCompletedEvent(QueryInfo queryInfo) ImmutableList.Builder inputs = ImmutableList.builder(); for (Input input : queryInfo.getInputs()) { inputs.add(new QueryInputMetadata( - input.getConnectorId(), + input.getConnectorId().getCatalogName(), input.getSchema(), input.getTable(), input.getColumns().stream() @@ -147,7 +147,8 @@ public void queryCompletedEvent(QueryInfo queryInfo) Optional output = Optional.empty(); if (queryInfo.getOutput().isPresent()) { output = Optional.of( - new QueryOutputMetadata(queryInfo.getOutput().get().getConnectorId(), + new QueryOutputMetadata( + queryInfo.getOutput().get().getConnectorId().getCatalogName(), queryInfo.getOutput().get().getSchema(), queryInfo.getOutput().get().getTable())); } diff --git a/presto-main/src/main/java/com/facebook/presto/execution/CallTask.java b/presto-main/src/main/java/com/facebook/presto/execution/CallTask.java index 0373c36387b4..9657e64090a0 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/CallTask.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/CallTask.java @@ -14,6 +14,7 @@ package com.facebook.presto.execution; import com.facebook.presto.Session; +import com.facebook.presto.connector.ConnectorId; import com.facebook.presto.metadata.Metadata; import com.facebook.presto.metadata.QualifiedObjectName; import com.facebook.presto.security.AccessControl; @@ -49,6 +50,7 @@ import static com.facebook.presto.spi.StandardErrorCode.PROCEDURE_CALL_FAILED; import static com.facebook.presto.spi.type.TypeUtils.writeNativeValue; import static com.facebook.presto.sql.analyzer.SemanticErrorCode.INVALID_PROCEDURE_ARGUMENTS; +import static com.facebook.presto.sql.analyzer.SemanticErrorCode.MISSING_CATALOG; import static com.facebook.presto.sql.planner.ExpressionInterpreter.evaluateConstantExpression; import static com.google.common.base.Throwables.propagateIfInstanceOf; import static java.util.Arrays.asList; @@ -72,6 +74,10 @@ public CompletableFuture execute(Call call, TransactionManager transactionMan Session session = stateMachine.getSession(); QualifiedObjectName procedureName = createQualifiedObjectName(session, call, call.getName()); + ConnectorId connectorId = metadata.getCatalogNames().get(procedureName.getCatalogName()); + if (connectorId == null) { + throw new SemanticException(MISSING_CATALOG, call, "Catalog %s does not exist", procedureName.getCatalogName()); + } Procedure procedure = metadata.getProcedureRegistry().resolve(procedureName); // map declared argument names to positions @@ -143,7 +149,7 @@ else if (i < procedure.getArguments().size()) { Iterator valuesIterator = asList(values).iterator(); for (Class type : methodType.parameterList()) { if (ConnectorSession.class.isAssignableFrom(type)) { - arguments.add(session.toConnectorSession(procedureName.getCatalogName())); + arguments.add(session.toConnectorSession(connectorId)); } else { arguments.add(valuesIterator.next()); diff --git a/presto-main/src/main/java/com/facebook/presto/execution/CreateTableTask.java b/presto-main/src/main/java/com/facebook/presto/execution/CreateTableTask.java index e301706dba38..2dc14fff839e 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/CreateTableTask.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/CreateTableTask.java @@ -17,7 +17,6 @@ import com.facebook.presto.metadata.Metadata; import com.facebook.presto.metadata.QualifiedObjectName; import com.facebook.presto.metadata.TableHandle; -import com.facebook.presto.metadata.TableMetadata; import com.facebook.presto.security.AccessControl; import com.facebook.presto.spi.ColumnMetadata; import com.facebook.presto.spi.ConnectorTableMetadata; @@ -90,9 +89,7 @@ public CompletableFuture execute(CreateTable statement, TransactionManager tr metadata, parameters); - TableMetadata tableMetadata = new TableMetadata( - tableName.getCatalogName(), - new ConnectorTableMetadata(tableName.asSchemaTableName(), columns, properties, false)); + ConnectorTableMetadata tableMetadata = new ConnectorTableMetadata(tableName.asSchemaTableName(), columns, properties, false); metadata.createTable(session, tableName.getCatalogName(), tableMetadata); diff --git a/presto-main/src/main/java/com/facebook/presto/execution/Input.java b/presto-main/src/main/java/com/facebook/presto/execution/Input.java index f9651731e370..ceb4d8e35e00 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/Input.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/Input.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.execution; +import com.facebook.presto.connector.ConnectorId; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.collect.ImmutableList; @@ -29,7 +30,7 @@ @Immutable public final class Input { - private final String connectorId; + private final ConnectorId connectorId; private final String schema; private final String table; private final List columns; @@ -37,7 +38,7 @@ public final class Input @JsonCreator public Input( - @JsonProperty("connectorId") String connectorId, + @JsonProperty("connectorId") ConnectorId connectorId, @JsonProperty("schema") String schema, @JsonProperty("table") String table, @JsonProperty("connectorInfo") Optional connectorInfo, @@ -57,7 +58,7 @@ public Input( } @JsonProperty - public String getConnectorId() + public ConnectorId getConnectorId() { return connectorId; } diff --git a/presto-main/src/main/java/com/facebook/presto/execution/Output.java b/presto-main/src/main/java/com/facebook/presto/execution/Output.java index 5322e68ccdf9..429be7ba83b8 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/Output.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/Output.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.execution; +import com.facebook.presto.connector.ConnectorId; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; @@ -25,13 +26,13 @@ @Immutable public final class Output { - private final String connectorId; + private final ConnectorId connectorId; private final String schema; private final String table; @JsonCreator public Output( - @JsonProperty("connectorId") String connectorId, + @JsonProperty("connectorId") ConnectorId connectorId, @JsonProperty("schema") String schema, @JsonProperty("table") String table) { @@ -41,7 +42,7 @@ public Output( } @JsonProperty - public String getConnectorId() + public ConnectorId getConnectorId() { return connectorId; } diff --git a/presto-main/src/main/java/com/facebook/presto/execution/SqlStageExecution.java b/presto-main/src/main/java/com/facebook/presto/execution/SqlStageExecution.java index 69588526a452..75a74f2257f9 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/SqlStageExecution.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/SqlStageExecution.java @@ -50,6 +50,7 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; +import static com.facebook.presto.operator.ExchangeOperator.REMOTE_CONNECTOR_ID; import static com.facebook.presto.util.ImmutableCollectors.toImmutableList; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; @@ -376,7 +377,7 @@ private static Split createRemoteSplitFor(TaskId taskId, URI taskLocation) { // Fetch the results from the buffer assigned to the task based on id URI splitLocation = uriBuilderFrom(taskLocation).appendPath("results").appendPath(String.valueOf(taskId.getId())).build(); - return new Split("remote", new RemoteTransactionHandle(), new RemoteSplit(splitLocation)); + return new Split(REMOTE_CONNECTOR_ID, new RemoteTransactionHandle(), new RemoteSplit(splitLocation)); } @Override diff --git a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/NodeScheduler.java b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/NodeScheduler.java index 99439dfcfdb3..f27cc42e3b4e 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/NodeScheduler.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/NodeScheduler.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.execution.scheduler; +import com.facebook.presto.connector.ConnectorId; import com.facebook.presto.execution.NodeTaskMap; import com.facebook.presto.execution.RemoteTask; import com.facebook.presto.metadata.InternalNodeManager; @@ -116,7 +117,7 @@ public Map getTopologicalSplitCounters() return counters.build(); } - public NodeSelector createNodeSelector(String connectorId) + public NodeSelector createNodeSelector(ConnectorId connectorId) { // this supplier is thread-safe. TODO: this logic should probably move to the scheduler since the choice of which node to run in should be // done as close to when the the split is about to be scheduled diff --git a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/SqlQueryScheduler.java b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/SqlQueryScheduler.java index 1322fcf9e3aa..583b6d15d2fb 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/SqlQueryScheduler.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/SqlQueryScheduler.java @@ -16,6 +16,7 @@ import com.facebook.presto.OutputBuffers; import com.facebook.presto.OutputBuffers.OutputBufferId; import com.facebook.presto.Session; +import com.facebook.presto.connector.ConnectorId; import com.facebook.presto.execution.LocationFactory; import com.facebook.presto.execution.NodeTaskMap; import com.facebook.presto.execution.QueryState; @@ -59,8 +60,7 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Function; -import static com.facebook.presto.connector.ConnectorManager.INFORMATION_SCHEMA_CONNECTOR_PREFIX; -import static com.facebook.presto.connector.ConnectorManager.SYSTEM_TABLES_CONNECTOR_PREFIX; +import static com.facebook.presto.connector.ConnectorId.isInternalSystemConnector; import static com.facebook.presto.execution.StageState.ABORTED; import static com.facebook.presto.execution.StageState.CANCELED; import static com.facebook.presto.execution.StageState.FAILED; @@ -213,8 +213,8 @@ private List createStages( if (partitioningHandle.equals(SOURCE_DISTRIBUTION)) { // nodes are selected dynamically based on the constraints of the splits and the system load Entry entry = Iterables.getOnlyElement(plan.getSplitSources().entrySet()); - String connectorId = entry.getValue().getConnectorId(); - if (connectorId.startsWith(SYSTEM_TABLES_CONNECTOR_PREFIX) || connectorId.startsWith(INFORMATION_SCHEMA_CONNECTOR_PREFIX)) { + ConnectorId connectorId = entry.getValue().getConnectorId(); + if (isInternalSystemConnector(connectorId)) { connectorId = null; } NodeSelector nodeSelector = nodeScheduler.createNodeSelector(connectorId); diff --git a/presto-main/src/main/java/com/facebook/presto/index/IndexManager.java b/presto-main/src/main/java/com/facebook/presto/index/IndexManager.java index 33ddd9e398af..908337cb78a4 100644 --- a/presto-main/src/main/java/com/facebook/presto/index/IndexManager.java +++ b/presto-main/src/main/java/com/facebook/presto/index/IndexManager.java @@ -14,6 +14,7 @@ package com.facebook.presto.index; import com.facebook.presto.Session; +import com.facebook.presto.connector.ConnectorId; import com.facebook.presto.metadata.IndexHandle; import com.facebook.presto.spi.ColumnHandle; import com.facebook.presto.spi.ConnectorIndex; @@ -29,9 +30,9 @@ public class IndexManager { - private final ConcurrentMap providers = new ConcurrentHashMap<>(); + private final ConcurrentMap providers = new ConcurrentHashMap<>(); - public void addIndexProvider(String connectorId, ConnectorIndexProvider provider) + public void addIndexProvider(ConnectorId connectorId, ConnectorIndexProvider provider) { checkState(providers.putIfAbsent(connectorId, provider) == null, "IndexProvider for connector '%s' is already registered", connectorId); } diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/DiscoveryNodeManager.java b/presto-main/src/main/java/com/facebook/presto/metadata/DiscoveryNodeManager.java index bcc136e10cab..6deb6abf0515 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/DiscoveryNodeManager.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/DiscoveryNodeManager.java @@ -14,6 +14,7 @@ package com.facebook.presto.metadata; import com.facebook.presto.client.NodeVersion; +import com.facebook.presto.connector.ConnectorId; import com.facebook.presto.connector.system.GlobalSystemConnector; import com.facebook.presto.failureDetector.FailureDetector; import com.facebook.presto.spi.Node; @@ -77,7 +78,7 @@ public final class DiscoveryNodeManager private final ScheduledExecutorService nodeStateUpdateExecutor; @GuardedBy("this") - private SetMultimap activeNodesByConnectorId; + private SetMultimap activeNodesByConnectorId; @GuardedBy("this") private AllNodes allNodes; @@ -169,7 +170,7 @@ private synchronized PrestoNode refreshNodesInternal() ImmutableSet.Builder inactiveNodesBuilder = ImmutableSet.builder(); ImmutableSet.Builder shuttingDownNodesBuilder = ImmutableSet.builder(); ImmutableSet.Builder coordinatorsBuilder = ImmutableSet.builder(); - ImmutableSetMultimap.Builder byConnectorIdBuilder = ImmutableSetMultimap.builder(); + ImmutableSetMultimap.Builder byConnectorIdBuilder = ImmutableSetMultimap.builder(); for (ServiceDescriptor service : services) { URI uri = getHttpUri(service); @@ -197,12 +198,12 @@ private synchronized PrestoNode refreshNodesInternal() if (connectorIds != null) { connectorIds = connectorIds.toLowerCase(ENGLISH); for (String connectorId : CONNECTOR_ID_SPLITTER.split(connectorIds)) { - byConnectorIdBuilder.put(connectorId, node); + byConnectorIdBuilder.put(new ConnectorId(connectorId), node); } } // always add system connector - byConnectorIdBuilder.put(GlobalSystemConnector.NAME, node); + byConnectorIdBuilder.put(new ConnectorId(GlobalSystemConnector.NAME), node); break; case INACTIVE: inactiveNodesBuilder.add(node); @@ -303,7 +304,7 @@ public Set getNodes(NodeState state) } @Override - public synchronized Set getActiveConnectorNodes(String connectorId) + public synchronized Set getActiveConnectorNodes(ConnectorId connectorId) { refreshIfNecessary(); return activeNodesByConnectorId.get(connectorId); diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/HandleResolver.java b/presto-main/src/main/java/com/facebook/presto/metadata/HandleResolver.java index 00b2d7ae65d6..1b182fc92bab 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/HandleResolver.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/HandleResolver.java @@ -33,6 +33,7 @@ import java.util.concurrent.ConcurrentMap; import java.util.function.Function; +import static com.facebook.presto.operator.ExchangeOperator.REMOTE_CONNECTOR_ID; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; import static java.util.Objects.requireNonNull; @@ -44,7 +45,7 @@ public class HandleResolver @Inject public HandleResolver() { - handleResolvers.put("remote", new RemoteHandleResolver()); + handleResolvers.put(REMOTE_CONNECTOR_ID.toString(), new RemoteHandleResolver()); handleResolvers.put("$system", new SystemHandleResolver()); handleResolvers.put("$info_schema", new InformationSchemaHandleResolver()); } diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/InMemoryNodeManager.java b/presto-main/src/main/java/com/facebook/presto/metadata/InMemoryNodeManager.java index 9476489c993a..1e8a513923de 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/InMemoryNodeManager.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/InMemoryNodeManager.java @@ -14,6 +14,7 @@ package com.facebook.presto.metadata; import com.facebook.presto.client.NodeVersion; +import com.facebook.presto.connector.ConnectorId; import com.facebook.presto.spi.Node; import com.facebook.presto.spi.NodeState; import com.google.common.collect.HashMultimap; @@ -31,7 +32,7 @@ public class InMemoryNodeManager implements InternalNodeManager { private final Node localNode; - private final SetMultimap remoteNodes = Multimaps.synchronizedSetMultimap(HashMultimap.create()); + private final SetMultimap remoteNodes = Multimaps.synchronizedSetMultimap(HashMultimap.create()); @Inject public InMemoryNodeManager() @@ -44,17 +45,17 @@ public InMemoryNodeManager(URI localUri) localNode = new PrestoNode("local", localUri, NodeVersion.UNKNOWN, false); } - public void addCurrentNodeConnector(String connectorId) + public void addCurrentNodeConnector(ConnectorId connectorId) { addNode(connectorId, localNode); } - public void addNode(String connectorId, Node... nodes) + public void addNode(ConnectorId connectorId, Node... nodes) { addNode(connectorId, ImmutableList.copyOf(nodes)); } - public void addNode(String connectorId, Iterable nodes) + public void addNode(ConnectorId connectorId, Iterable nodes) { remoteNodes.putAll(connectorId, nodes); } @@ -75,7 +76,7 @@ public Set getNodes(NodeState state) } @Override - public Set getActiveConnectorNodes(String connectorId) + public Set getActiveConnectorNodes(ConnectorId connectorId) { return ImmutableSet.copyOf(remoteNodes.get(connectorId)); } diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/IndexHandle.java b/presto-main/src/main/java/com/facebook/presto/metadata/IndexHandle.java index 93cb3b68603d..2b5a8c3e4f7d 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/IndexHandle.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/IndexHandle.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.metadata; +import com.facebook.presto.connector.ConnectorId; import com.facebook.presto.spi.ConnectorIndexHandle; import com.facebook.presto.spi.connector.ConnectorTransactionHandle; import com.fasterxml.jackson.annotation.JsonCreator; @@ -24,13 +25,13 @@ public final class IndexHandle { - private final String connectorId; + private final ConnectorId connectorId; private final ConnectorTransactionHandle transactionHandle; private final ConnectorIndexHandle connectorHandle; @JsonCreator public IndexHandle( - @JsonProperty("connectorId") String connectorId, + @JsonProperty("connectorId") ConnectorId connectorId, @JsonProperty("transactionHandle") ConnectorTransactionHandle transactionHandle, @JsonProperty("connectorHandle") ConnectorIndexHandle connectorHandle) { @@ -40,7 +41,7 @@ public IndexHandle( } @JsonProperty - public String getConnectorId() + public ConnectorId getConnectorId() { return connectorId; } diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/InsertTableHandle.java b/presto-main/src/main/java/com/facebook/presto/metadata/InsertTableHandle.java index c94a15b813aa..df34b726250b 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/InsertTableHandle.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/InsertTableHandle.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.metadata; +import com.facebook.presto.connector.ConnectorId; import com.facebook.presto.spi.ConnectorInsertTableHandle; import com.facebook.presto.spi.connector.ConnectorTransactionHandle; import com.fasterxml.jackson.annotation.JsonCreator; @@ -24,13 +25,13 @@ public final class InsertTableHandle { - private final String connectorId; + private final ConnectorId connectorId; private final ConnectorTransactionHandle transactionHandle; private final ConnectorInsertTableHandle connectorHandle; @JsonCreator public InsertTableHandle( - @JsonProperty("connectorId") String connectorId, + @JsonProperty("connectorId") ConnectorId connectorId, @JsonProperty("transactionHandle") ConnectorTransactionHandle transactionHandle, @JsonProperty("connectorHandle") ConnectorInsertTableHandle connectorHandle) { @@ -40,7 +41,7 @@ public InsertTableHandle( } @JsonProperty - public String getConnectorId() + public ConnectorId getConnectorId() { return connectorId; } diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/InternalNodeManager.java b/presto-main/src/main/java/com/facebook/presto/metadata/InternalNodeManager.java index 4380b862e427..45932ff2928c 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/InternalNodeManager.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/InternalNodeManager.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.metadata; +import com.facebook.presto.connector.ConnectorId; import com.facebook.presto.spi.Node; import com.facebook.presto.spi.NodeState; @@ -22,7 +23,7 @@ public interface InternalNodeManager { Set getNodes(NodeState state); - Set getActiveConnectorNodes(String datasourceName); + Set getActiveConnectorNodes(ConnectorId connectorId); Node getCurrentNode(); diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/Metadata.java b/presto-main/src/main/java/com/facebook/presto/metadata/Metadata.java index 853b3b4c5c8f..dd7953da5dd4 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/Metadata.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/Metadata.java @@ -14,8 +14,10 @@ package com.facebook.presto.metadata; import com.facebook.presto.Session; +import com.facebook.presto.connector.ConnectorId; import com.facebook.presto.spi.ColumnHandle; import com.facebook.presto.spi.ColumnMetadata; +import com.facebook.presto.spi.ConnectorTableMetadata; import com.facebook.presto.spi.Constraint; import com.facebook.presto.spi.block.BlockEncodingSerde; import com.facebook.presto.spi.predicate.TupleDomain; @@ -122,7 +124,7 @@ public interface Metadata * Creates a table using the specified table metadata. */ @NotNull - void createTable(Session session, String catalogName, TableMetadata tableMetadata); + void createTable(Session session, String catalogName, ConnectorTableMetadata tableMetadata); /** * Rename the specified table. @@ -146,12 +148,12 @@ public interface Metadata */ void dropTable(Session session, TableHandle tableHandle); - Optional getNewTableLayout(Session session, String catalogName, TableMetadata tableMetadata); + Optional getNewTableLayout(Session session, String catalogName, ConnectorTableMetadata tableMetadata); /** * Begin the atomic creation of a table with data. */ - OutputTableHandle beginCreateTable(Session session, String catalogName, TableMetadata tableMetadata, Optional layout); + OutputTableHandle beginCreateTable(Session session, String catalogName, ConnectorTableMetadata tableMetadata, Optional layout); /** * Finish a table creation with data after the data is written. @@ -203,7 +205,7 @@ public interface Metadata * @return Map of catalog name to connector id */ @NotNull - Map getCatalogNames(); + Map getCatalogNames(); /** * Get the names that match the specified table prefix (never null). diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/MetadataManager.java b/presto-main/src/main/java/com/facebook/presto/metadata/MetadataManager.java index cfd87d74c807..6e20feb3fdad 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/MetadataManager.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/MetadataManager.java @@ -15,6 +15,7 @@ import com.facebook.presto.Session; import com.facebook.presto.block.BlockEncodingManager; +import com.facebook.presto.connector.ConnectorId; import com.facebook.presto.spi.ColumnHandle; import com.facebook.presto.spi.ColumnMetadata; import com.facebook.presto.spi.ConnectorInsertTableHandle; @@ -47,7 +48,6 @@ import com.facebook.presto.transaction.TransactionManager; import com.facebook.presto.type.TypeDeserializer; import com.facebook.presto.type.TypeRegistry; -import com.fasterxml.jackson.databind.JsonDeserializer; import com.google.common.base.Joiner; import com.google.common.collect.HashMultimap; import com.google.common.collect.ImmutableList; @@ -109,7 +109,7 @@ public class MetadataManager private final ConcurrentMap informationSchemasByCatalog = new ConcurrentHashMap<>(); private final ConcurrentMap systemTablesByCatalog = new ConcurrentHashMap<>(); private final ConcurrentMap connectorsByCatalog = new ConcurrentHashMap<>(); - private final ConcurrentMap connectorsById = new ConcurrentHashMap<>(); + private final ConcurrentMap connectorsById = new ConcurrentHashMap<>(); private final FunctionRegistry functions; private final ProcedureRegistry procedures; private final TypeManager typeManager; @@ -160,7 +160,7 @@ public static MetadataManager createTestMetadataManager() return new MetadataManager(featuresConfig, typeManager, blockEncodingSerde, sessionPropertyManager, new TablePropertyManager(), transactionManager); } - public synchronized void registerConnectorCatalog(String connectorId, String catalogName) + public synchronized void registerConnectorCatalog(ConnectorId connectorId, String catalogName) { checkMetadataArguments(connectorId, catalogName); checkArgument(!connectorsByCatalog.containsKey(catalogName), "Catalog '%s' is already registered", catalogName); @@ -170,7 +170,7 @@ public synchronized void registerConnectorCatalog(String connectorId, String cat connectorsByCatalog.put(catalogName, connectorEntry); } - public synchronized void registerInformationSchemaCatalog(String connectorId, String catalogName) + public synchronized void registerInformationSchemaCatalog(ConnectorId connectorId, String catalogName) { checkMetadataArguments(connectorId, catalogName); checkArgument(!informationSchemasByCatalog.containsKey(catalogName), "Information schema for catalog '%s' is already registered", catalogName); @@ -180,7 +180,7 @@ public synchronized void registerInformationSchemaCatalog(String connectorId, St informationSchemasByCatalog.put(catalogName, connectorEntry); } - public synchronized void registerSystemTablesCatalog(String connectorId, String catalogName) + public synchronized void registerSystemTablesCatalog(ConnectorId connectorId, String catalogName) { checkMetadataArguments(connectorId, catalogName); checkArgument(!systemTablesByCatalog.containsKey(catalogName), "System tables for catalog '%s' are already registered", catalogName); @@ -190,7 +190,7 @@ public synchronized void registerSystemTablesCatalog(String connectorId, String systemTablesByCatalog.put(catalogName, connectorEntry); } - private void checkMetadataArguments(String connectorId, String catalogName) + private void checkMetadataArguments(ConnectorId connectorId, String catalogName) { requireNonNull(connectorId, "connectorId is null"); requireNonNull(catalogName, "catalogName is null"); @@ -264,10 +264,13 @@ public void addFunctions(List functionInfos) @Override public boolean schemaExists(Session session, QualifiedSchemaName schema) { - ConnectorSession connectorSession = session.toConnectorSession(schema.getCatalogName()); - return allConnectorsFor(schema.getCatalogName()).stream() - .map(entry -> entry.getMetadata(session)) - .anyMatch(metadata -> metadata.schemaExists(connectorSession, schema.getSchemaName())); + for (ConnectorEntry entry : allConnectorsFor(schema.getCatalogName())) { + ConnectorMetadata metadata = entry.getMetadata(session); + if (metadata.schemaExists(session.toConnectorSession(entry.getConnectorId()), schema.getSchemaName())) { + return true; + } + } + return false; } @Override @@ -277,7 +280,7 @@ public List listSchemaNames(Session session, String catalogName) ImmutableSet.Builder schemaNames = ImmutableSet.builder(); for (ConnectorEntry entry : allConnectorsFor(catalogName)) { ConnectorMetadata metadata = entry.getMetadata(session); - schemaNames.addAll(metadata.listSchemaNames(session.toConnectorSession(entry.getCatalog()))); + schemaNames.addAll(metadata.listSchemaNames(session.toConnectorSession(entry.getConnectorId()))); } return ImmutableList.copyOf(schemaNames.build()); } @@ -291,7 +294,7 @@ public Optional getTableHandle(Session session, QualifiedObjectName if (entry != null) { ConnectorMetadata metadata = entry.getMetadata(session); - ConnectorTableHandle tableHandle = metadata.getTableHandle(session.toConnectorSession(entry.getCatalog()), table.asSchemaTableName()); + ConnectorTableHandle tableHandle = metadata.getTableHandle(session.toConnectorSession(entry.getConnectorId()), table.asSchemaTableName()); if (tableHandle != null) { return Optional.of(new TableHandle(entry.getConnectorId(), tableHandle)); @@ -308,14 +311,14 @@ public List getLayouts(Session session, TableHandle table, Co } TupleDomain summary = constraint.getSummary(); - String connectorId = table.getConnectorId(); + ConnectorId connectorId = table.getConnectorId(); ConnectorTableHandle connectorTable = table.getConnectorHandle(); Predicate> predicate = constraint.predicate(); ConnectorEntry entry = getConnectorMetadata(connectorId); ConnectorMetadata metadata = entry.getMetadata(session); ConnectorTransactionHandle transaction = entry.getTransactionHandle(session); - ConnectorSession connectorSession = session.toConnectorSession(entry.getCatalog()); + ConnectorSession connectorSession = session.toConnectorSession(entry.getConnectorId()); List layouts = metadata.getTableLayouts(connectorSession, connectorTable, new Constraint<>(summary, predicate::test), desiredColumns); return layouts.stream() @@ -326,20 +329,20 @@ public List getLayouts(Session session, TableHandle table, Co @Override public TableLayout getLayout(Session session, TableLayoutHandle handle) { - String connectorId = handle.getConnectorId(); + ConnectorId connectorId = handle.getConnectorId(); ConnectorEntry entry = getConnectorMetadata(connectorId); ConnectorMetadata metadata = entry.getMetadata(session); ConnectorTransactionHandle transaction = entry.getTransactionHandle(session); - return fromConnectorLayout(connectorId, transaction, metadata.getTableLayout(session.toConnectorSession(entry.getCatalog()), handle.getConnectorHandle())); + return fromConnectorLayout(connectorId, transaction, metadata.getTableLayout(session.toConnectorSession(entry.getConnectorId()), handle.getConnectorHandle())); } @Override public Optional getInfo(Session session, TableLayoutHandle handle) { - String connectorId = handle.getConnectorId(); + ConnectorId connectorId = handle.getConnectorId(); ConnectorEntry entry = getConnectorMetadata(connectorId); ConnectorMetadata metadata = entry.getMetadata(session); - ConnectorTableLayout tableLayout = metadata.getTableLayout(session.toConnectorSession(entry.getCatalog()), handle.getConnectorHandle()); + ConnectorTableLayout tableLayout = metadata.getTableLayout(session.toConnectorSession(entry.getConnectorId()), handle.getConnectorHandle()); return metadata.getInfo(tableLayout.getHandle()); } @@ -348,7 +351,7 @@ public TableMetadata getTableMetadata(Session session, TableHandle tableHandle) { ConnectorEntry entry = lookupConnectorFor(tableHandle); ConnectorMetadata metadata = entry.getMetadata(session); - ConnectorTableMetadata tableMetadata = metadata.getTableMetadata(session.toConnectorSession(entry.getCatalog()), tableHandle.getConnectorHandle()); + ConnectorTableMetadata tableMetadata = metadata.getTableMetadata(session.toConnectorSession(entry.getConnectorId()), tableHandle.getConnectorHandle()); return new TableMetadata(tableHandle.getConnectorId(), tableMetadata); } @@ -358,7 +361,7 @@ public Map getColumnHandles(Session session, TableHandle t { ConnectorEntry entry = lookupConnectorFor(tableHandle); ConnectorMetadata metadata = entry.getMetadata(session); - Map handles = metadata.getColumnHandles(session.toConnectorSession(entry.getCatalog()), tableHandle.getConnectorHandle()); + Map handles = metadata.getColumnHandles(session.toConnectorSession(entry.getConnectorId()), tableHandle.getConnectorHandle()); ImmutableMap.Builder map = ImmutableMap.builder(); for (Entry mapEntry : handles.entrySet()) { @@ -375,7 +378,7 @@ public ColumnMetadata getColumnMetadata(Session session, TableHandle tableHandle ConnectorEntry entry = lookupConnectorFor(tableHandle); ConnectorMetadata metadata = entry.getMetadata(session); - return metadata.getColumnMetadata(session.toConnectorSession(entry.getCatalog()), tableHandle.getConnectorHandle(), columnHandle); + return metadata.getColumnMetadata(session.toConnectorSession(entry.getConnectorId()), tableHandle.getConnectorHandle(), columnHandle); } @Override @@ -387,7 +390,7 @@ public List listTables(Session session, QualifiedTablePrefi Set tables = new LinkedHashSet<>(); for (ConnectorEntry entry : allConnectorsFor(prefix.getCatalogName())) { ConnectorMetadata metadata = entry.getMetadata(session); - ConnectorSession connectorSession = session.toConnectorSession(entry.getCatalog()); + ConnectorSession connectorSession = session.toConnectorSession(entry.getConnectorId()); for (QualifiedObjectName tableName : transform(metadata.listTables(connectorSession, schemaNameOrNull), convertFromSchemaTableName(prefix.getCatalogName()))) { tables.add(tableName); } @@ -401,7 +404,7 @@ public Optional getSampleWeightColumnHandle(Session session, Table requireNonNull(tableHandle, "tableHandle is null"); ConnectorEntry entry = lookupConnectorFor(tableHandle); ConnectorMetadata metadata = entry.getMetadata(session); - ColumnHandle handle = metadata.getSampleWeightColumnHandle(session.toConnectorSession(entry.getCatalog()), tableHandle.getConnectorHandle()); + ColumnHandle handle = metadata.getSampleWeightColumnHandle(session.toConnectorSession(entry.getConnectorId()), tableHandle.getConnectorHandle()); return Optional.ofNullable(handle); } @@ -412,7 +415,7 @@ public boolean canCreateSampledTables(Session session, String catalogName) ConnectorEntry entry = connectorsByCatalog.get(catalogName); checkArgument(entry != null, "Catalog %s does not exist", catalogName); ConnectorMetadata metadata = entry.getMetadata(session); - return metadata.canCreateSampledTables(session.toConnectorSession(entry.getCatalog())); + return metadata.canCreateSampledTables(session.toConnectorSession(entry.getConnectorId())); } @Override @@ -425,7 +428,7 @@ public Map> listTableColumns(Session s for (ConnectorEntry connectorEntry : allConnectorsFor(prefix.getCatalogName())) { ConnectorMetadata metadata = connectorEntry.getMetadata(session); - ConnectorSession connectorSession = session.toConnectorSession(connectorEntry.getCatalog()); + ConnectorSession connectorSession = session.toConnectorSession(connectorEntry.getConnectorId()); for (Entry> entry : metadata.listTableColumns(connectorSession, tablePrefix).entrySet()) { QualifiedObjectName tableName = new QualifiedObjectName( prefix.getCatalogName(), @@ -453,12 +456,12 @@ public Map> listTableColumns(Session s } @Override - public void createTable(Session session, String catalogName, TableMetadata tableMetadata) + public void createTable(Session session, String catalogName, ConnectorTableMetadata tableMetadata) { ConnectorEntry entry = connectorsByCatalog.get(catalogName); checkArgument(entry != null, "Catalog %s does not exist", catalogName); ConnectorMetadata metadata = entry.getMetadataForWrite(session); - metadata.createTable(session.toConnectorSession(entry.getCatalog()), tableMetadata.getMetadata()); + metadata.createTable(session.toConnectorSession(entry.getConnectorId()), tableMetadata); } @Override @@ -475,7 +478,7 @@ public void renameTable(Session session, TableHandle tableHandle, QualifiedObjec ConnectorEntry entry = lookupConnectorFor(tableHandle); ConnectorMetadata metadata = entry.getMetadataForWrite(session); - metadata.renameTable(session.toConnectorSession(entry.getCatalog()), tableHandle.getConnectorHandle(), newTableName.asSchemaTableName()); + metadata.renameTable(session.toConnectorSession(entry.getConnectorId()), tableHandle.getConnectorHandle(), newTableName.asSchemaTableName()); } @Override @@ -483,7 +486,7 @@ public void renameColumn(Session session, TableHandle tableHandle, ColumnHandle { ConnectorEntry entry = lookupConnectorFor(tableHandle); ConnectorMetadata metadata = entry.getMetadataForWrite(session); - metadata.renameColumn(session.toConnectorSession(entry.getCatalog()), tableHandle.getConnectorHandle(), source, target.toLowerCase(ENGLISH)); + metadata.renameColumn(session.toConnectorSession(entry.getConnectorId()), tableHandle.getConnectorHandle(), source, target.toLowerCase(ENGLISH)); } @Override @@ -491,7 +494,7 @@ public void addColumn(Session session, TableHandle tableHandle, ColumnMetadata c { ConnectorEntry entry = lookupConnectorFor(tableHandle); ConnectorMetadata metadata = entry.getMetadataForWrite(session); - metadata.addColumn(session.toConnectorSession(entry.getCatalog()), tableHandle.getConnectorHandle(), column); + metadata.addColumn(session.toConnectorSession(entry.getConnectorId()), tableHandle.getConnectorHandle(), column); } @Override @@ -499,7 +502,7 @@ public void dropTable(Session session, TableHandle tableHandle) { ConnectorEntry entry = lookupConnectorFor(tableHandle); ConnectorMetadata metadata = entry.getMetadataForWrite(session); - metadata.dropTable(session.toConnectorSession(entry.getCatalog()), tableHandle.getConnectorHandle()); + metadata.dropTable(session.toConnectorSession(entry.getConnectorId()), tableHandle.getConnectorHandle()); } @Override @@ -507,7 +510,7 @@ public Optional getInsertLayout(Session session, TableHandle tab { ConnectorEntry entry = getConnectorMetadata(table.getConnectorId()); - Optional insertLayout = entry.getMetadataForWrite(session).getInsertLayout(session.toConnectorSession(entry.getCatalog()), table.getConnectorHandle()); + Optional insertLayout = entry.getMetadataForWrite(session).getInsertLayout(session.toConnectorSession(entry.getConnectorId()), table.getConnectorHandle()); if (!insertLayout.isPresent()) { return Optional.empty(); } @@ -516,26 +519,26 @@ public Optional getInsertLayout(Session session, TableHandle tab } @Override - public Optional getNewTableLayout(Session session, String catalogName, TableMetadata tableMetadata) + public Optional getNewTableLayout(Session session, String catalogName, ConnectorTableMetadata tableMetadata) { ConnectorEntry entry = connectorsByCatalog.get(catalogName); checkArgument(entry != null, "Catalog %s does not exist", catalogName); ConnectorMetadata metadata = entry.getMetadataForWrite(session); ConnectorTransactionHandle transactionHandle = entry.getTransactionHandle(session); - ConnectorSession connectorSession = session.toConnectorSession(entry.getCatalog()); - return metadata.getNewTableLayout(connectorSession, tableMetadata.getMetadata()) + ConnectorSession connectorSession = session.toConnectorSession(entry.getConnectorId()); + return metadata.getNewTableLayout(connectorSession, tableMetadata) .map(layout -> new NewTableLayout(entry.getConnectorId(), transactionHandle, layout)); } @Override - public OutputTableHandle beginCreateTable(Session session, String catalogName, TableMetadata tableMetadata, Optional layout) + public OutputTableHandle beginCreateTable(Session session, String catalogName, ConnectorTableMetadata tableMetadata, Optional layout) { ConnectorEntry entry = connectorsByCatalog.get(catalogName); checkArgument(entry != null, "Catalog %s does not exist", catalogName); ConnectorMetadata metadata = entry.getMetadataForWrite(session); ConnectorTransactionHandle transactionHandle = entry.getTransactionHandle(session); - ConnectorSession connectorSession = session.toConnectorSession(entry.getCatalog()); - ConnectorOutputTableHandle handle = metadata.beginCreateTable(connectorSession, tableMetadata.getMetadata(), layout.map(NewTableLayout::getLayout)); + ConnectorSession connectorSession = session.toConnectorSession(entry.getConnectorId()); + ConnectorOutputTableHandle handle = metadata.beginCreateTable(connectorSession, tableMetadata, layout.map(NewTableLayout::getLayout)); return new OutputTableHandle(entry.getConnectorId(), transactionHandle, handle); } @@ -544,7 +547,7 @@ public void finishCreateTable(Session session, OutputTableHandle tableHandle, Co { ConnectorEntry entry = lookupConnectorFor(tableHandle); ConnectorMetadata metadata = entry.getMetadata(session); - metadata.finishCreateTable(session.toConnectorSession(entry.getCatalog()), tableHandle.getConnectorHandle(), fragments); + metadata.finishCreateTable(session.toConnectorSession(entry.getConnectorId()), tableHandle.getConnectorHandle(), fragments); } @Override @@ -553,7 +556,7 @@ public InsertTableHandle beginInsert(Session session, TableHandle tableHandle) ConnectorEntry entry = lookupConnectorFor(tableHandle); ConnectorMetadata metadata = entry.getMetadataForWrite(session); ConnectorTransactionHandle transactionHandle = entry.getTransactionHandle(session); - ConnectorInsertTableHandle handle = metadata.beginInsert(session.toConnectorSession(entry.getCatalog()), tableHandle.getConnectorHandle()); + ConnectorInsertTableHandle handle = metadata.beginInsert(session.toConnectorSession(entry.getConnectorId()), tableHandle.getConnectorHandle()); return new InsertTableHandle(tableHandle.getConnectorId(), transactionHandle, handle); } @@ -562,7 +565,7 @@ public void finishInsert(Session session, InsertTableHandle tableHandle, Collect { ConnectorEntry entry = lookupConnectorFor(tableHandle); ConnectorMetadata metadata = entry.getMetadata(session); - metadata.finishInsert(session.toConnectorSession(entry.getCatalog()), tableHandle.getConnectorHandle(), fragments); + metadata.finishInsert(session.toConnectorSession(entry.getConnectorId()), tableHandle.getConnectorHandle(), fragments); } @Override @@ -570,7 +573,7 @@ public ColumnHandle getUpdateRowIdColumnHandle(Session session, TableHandle tabl { ConnectorEntry entry = lookupConnectorFor(tableHandle); ConnectorMetadata metadata = entry.getMetadata(session); - return metadata.getUpdateRowIdColumnHandle(session.toConnectorSession(entry.getCatalog()), tableHandle.getConnectorHandle()); + return metadata.getUpdateRowIdColumnHandle(session.toConnectorSession(entry.getConnectorId()), tableHandle.getConnectorHandle()); } @Override @@ -579,7 +582,7 @@ public boolean supportsMetadataDelete(Session session, TableHandle tableHandle, ConnectorEntry entry = lookupConnectorFor(tableHandle); ConnectorMetadata metadata = entry.getMetadata(session); return metadata.supportsMetadataDelete( - session.toConnectorSession(entry.getCatalog()), + session.toConnectorSession(entry.getConnectorId()), tableHandle.getConnectorHandle(), tableLayoutHandle.getConnectorHandle()); } @@ -589,7 +592,7 @@ public OptionalLong metadataDelete(Session session, TableHandle tableHandle, Tab { ConnectorEntry entry = lookupConnectorFor(tableHandle); ConnectorMetadata metadata = entry.getMetadataForWrite(session); - return metadata.metadataDelete(session.toConnectorSession(entry.getCatalog()), tableHandle.getConnectorHandle(), tableLayoutHandle.getConnectorHandle()); + return metadata.metadataDelete(session.toConnectorSession(entry.getConnectorId()), tableHandle.getConnectorHandle(), tableLayoutHandle.getConnectorHandle()); } @Override @@ -597,7 +600,7 @@ public TableHandle beginDelete(Session session, TableHandle tableHandle) { ConnectorEntry entry = lookupConnectorFor(tableHandle); ConnectorMetadata metadata = entry.getMetadataForWrite(session); - ConnectorTableHandle newHandle = metadata.beginDelete(session.toConnectorSession(entry.getCatalog()), tableHandle.getConnectorHandle()); + ConnectorTableHandle newHandle = metadata.beginDelete(session.toConnectorSession(entry.getConnectorId()), tableHandle.getConnectorHandle()); return new TableHandle(tableHandle.getConnectorId(), newHandle); } @@ -606,13 +609,13 @@ public void finishDelete(Session session, TableHandle tableHandle, Collection getCatalogNames() + public Map getCatalogNames() { - ImmutableMap.Builder catalogsMap = ImmutableMap.builder(); + ImmutableMap.Builder catalogsMap = ImmutableMap.builder(); for (Map.Entry entry : connectorsByCatalog.entrySet()) { catalogsMap.put(entry.getKey(), entry.getValue().getConnectorId()); } @@ -628,7 +631,7 @@ public List listViews(Session session, QualifiedTablePrefix Set views = new LinkedHashSet<>(); for (ConnectorEntry entry : allConnectorsFor(prefix.getCatalogName())) { ConnectorMetadata metadata = entry.getMetadata(session); - ConnectorSession connectorSession = session.toConnectorSession(entry.getCatalog()); + ConnectorSession connectorSession = session.toConnectorSession(entry.getConnectorId()); for (QualifiedObjectName tableName : transform(metadata.listViews(connectorSession, schemaNameOrNull), convertFromSchemaTableName(prefix.getCatalogName()))) { views.add(tableName); } @@ -645,7 +648,7 @@ public Map getViews(Session session, Qualif Map views = new LinkedHashMap<>(); for (ConnectorEntry connectorEntry : allConnectorsFor(prefix.getCatalogName())) { ConnectorMetadata metadata = connectorEntry.getMetadata(session); - ConnectorSession connectorSession = session.toConnectorSession(connectorEntry.getCatalog()); + ConnectorSession connectorSession = session.toConnectorSession(connectorEntry.getConnectorId()); for (Entry entry : metadata.getViews(connectorSession, tablePrefix).entrySet()) { QualifiedObjectName viewName = new QualifiedObjectName( prefix.getCatalogName(), @@ -664,7 +667,7 @@ public Optional getView(Session session, QualifiedObjectName vie if (entry != null) { ConnectorMetadata metadata = entry.getMetadata(session); Map views = metadata.getViews( - session.toConnectorSession(entry.getCatalog()), + session.toConnectorSession(entry.getConnectorId()), viewName.asSchemaTableName().toSchemaTablePrefix()); ConnectorViewDefinition view = views.get(viewName.asSchemaTableName()); if (view != null) { @@ -680,7 +683,7 @@ public void createView(Session session, QualifiedObjectName viewName, String vie ConnectorEntry entry = connectorsByCatalog.get(viewName.getCatalogName()); checkArgument(entry != null, "Catalog %s does not exist", viewName.getCatalogName()); ConnectorMetadata metadata = entry.getMetadataForWrite(session); - metadata.createView(session.toConnectorSession(entry.getCatalog()), viewName.asSchemaTableName(), viewData, replace); + metadata.createView(session.toConnectorSession(entry.getConnectorId()), viewName.asSchemaTableName(), viewData, replace); } @Override @@ -689,7 +692,7 @@ public void dropView(Session session, QualifiedObjectName viewName) ConnectorEntry entry = connectorsByCatalog.get(viewName.getCatalogName()); checkArgument(entry != null, "Catalog %s does not exist", viewName.getCatalogName()); ConnectorMetadata metadata = entry.getMetadataForWrite(session); - metadata.dropView(session.toConnectorSession(entry.getCatalog()), viewName.asSchemaTableName()); + metadata.dropView(session.toConnectorSession(entry.getConnectorId()), viewName.asSchemaTableName()); } @Override @@ -698,7 +701,7 @@ public Optional resolveIndex(Session session, TableHandle tableHa ConnectorEntry entry = lookupConnectorFor(tableHandle); ConnectorMetadata metadata = entry.getMetadata(session); ConnectorTransactionHandle transaction = entry.getTransactionHandle(session); - ConnectorSession connectorSession = session.toConnectorSession(entry.getCatalog()); + ConnectorSession connectorSession = session.toConnectorSession(entry.getConnectorId()); Optional resolvedIndex = metadata.resolveIndex(connectorSession, tableHandle.getConnectorHandle(), indexableColumns, outputColumns, tupleDomain); return resolvedIndex.map(resolved -> new ResolvedIndex(tableHandle.getConnectorId(), transaction, resolved)); } @@ -709,7 +712,7 @@ public void grantTablePrivileges(Session session, QualifiedObjectName tableName, ConnectorEntry entry = connectorsByCatalog.get(tableName.getCatalogName()); checkArgument(entry != null, "Catalog %s does not exist", tableName.getCatalogName()); ConnectorMetadata metadata = entry.getMetadata(session); - metadata.grantTablePrivileges(session.toConnectorSession(entry.getCatalog()), tableName.asSchemaTableName(), privileges, grantee, grantOption); + metadata.grantTablePrivileges(session.toConnectorSession(entry.getConnectorId()), tableName.asSchemaTableName(), privileges, grantee, grantOption); } @Override @@ -718,7 +721,7 @@ public void revokeTablePrivileges(Session session, QualifiedObjectName tableName ConnectorEntry entry = connectorsByCatalog.get(tableName.getCatalogName()); checkArgument(entry != null, "Catalog %s does not exist", tableName.getCatalogName()); ConnectorMetadata metadata = entry.getMetadata(session); - metadata.revokeTablePrivileges(session.toConnectorSession(entry.getCatalog()), tableName.asSchemaTableName(), privileges, grantee, grantOption); + metadata.revokeTablePrivileges(session.toConnectorSession(entry.getConnectorId()), tableName.asSchemaTableName(), privileges, grantee, grantOption); } @Override @@ -823,7 +826,7 @@ private ConnectorEntry lookupConnectorFor(InsertTableHandle tableHandle) return getConnectorMetadata(tableHandle.getConnectorId()); } - private ConnectorEntry getConnectorMetadata(String connectorId) + private ConnectorEntry getConnectorMetadata(ConnectorId connectorId) { ConnectorEntry result = connectorsById.get(connectorId); checkArgument(result != null, "No connector for connector ID: %s", connectorId); @@ -832,16 +835,16 @@ private ConnectorEntry getConnectorMetadata(String connectorId) private class ConnectorEntry { - private final String connectorId; + private final ConnectorId connectorId; private final String catalog; - private ConnectorEntry(String connectorId, String catalog) + private ConnectorEntry(ConnectorId connectorId, String catalog) { this.connectorId = requireNonNull(connectorId, "connectorId is null"); this.catalog = requireNonNull(catalog, "catalog is null"); } - private String getConnectorId() + private ConnectorId getConnectorId() { return connectorId; } @@ -873,7 +876,7 @@ public ConnectorTransactionHandle getTransactionHandle(Session session) private static JsonCodec createTestingViewCodec() { ObjectMapperProvider provider = new ObjectMapperProvider(); - provider.setJsonDeserializers(ImmutableMap., JsonDeserializer>of(Type.class, new TypeDeserializer(new TypeRegistry()))); + provider.setJsonDeserializers(ImmutableMap.of(Type.class, new TypeDeserializer(new TypeRegistry()))); return new JsonCodecFactory(provider).jsonCodec(ViewDefinition.class); } } diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/NewTableLayout.java b/presto-main/src/main/java/com/facebook/presto/metadata/NewTableLayout.java index 5590640a1f1c..052c1643a66b 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/NewTableLayout.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/NewTableLayout.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.metadata; +import com.facebook.presto.connector.ConnectorId; import com.facebook.presto.spi.ConnectorNewTableLayout; import com.facebook.presto.spi.connector.ConnectorTransactionHandle; import com.facebook.presto.sql.planner.PartitioningHandle; @@ -28,13 +29,13 @@ public class NewTableLayout { - private final String connectorId; + private final ConnectorId connectorId; private final ConnectorTransactionHandle transactionHandle; private final ConnectorNewTableLayout layout; @JsonCreator public NewTableLayout( - @JsonProperty("connectorId") String connectorId, + @JsonProperty("connectorId") ConnectorId connectorId, @JsonProperty("transactionHandle") ConnectorTransactionHandle transactionHandle, @JsonProperty("layout") ConnectorNewTableLayout layout) { @@ -44,7 +45,7 @@ public NewTableLayout( } @JsonProperty - public String getConnectorId() + public ConnectorId getConnectorId() { return connectorId; } diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/OutputTableHandle.java b/presto-main/src/main/java/com/facebook/presto/metadata/OutputTableHandle.java index b3ef74569862..8bd148e96f88 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/OutputTableHandle.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/OutputTableHandle.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.metadata; +import com.facebook.presto.connector.ConnectorId; import com.facebook.presto.spi.ConnectorOutputTableHandle; import com.facebook.presto.spi.connector.ConnectorTransactionHandle; import com.fasterxml.jackson.annotation.JsonCreator; @@ -24,13 +25,13 @@ public final class OutputTableHandle { - private final String connectorId; + private final ConnectorId connectorId; private final ConnectorTransactionHandle transactionHandle; private final ConnectorOutputTableHandle connectorHandle; @JsonCreator public OutputTableHandle( - @JsonProperty("connectorId") String connectorId, + @JsonProperty("connectorId") ConnectorId connectorId, @JsonProperty("transactionHandle") ConnectorTransactionHandle transactionHandle, @JsonProperty("connectorHandle") ConnectorOutputTableHandle connectorHandle) { @@ -40,7 +41,7 @@ public OutputTableHandle( } @JsonProperty - public String getConnectorId() + public ConnectorId getConnectorId() { return connectorId; } diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/ResolvedIndex.java b/presto-main/src/main/java/com/facebook/presto/metadata/ResolvedIndex.java index bc38ca964f92..af6b885209ca 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/ResolvedIndex.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/ResolvedIndex.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.metadata; +import com.facebook.presto.connector.ConnectorId; import com.facebook.presto.spi.ColumnHandle; import com.facebook.presto.spi.ConnectorResolvedIndex; import com.facebook.presto.spi.connector.ConnectorTransactionHandle; @@ -25,7 +26,7 @@ public final class ResolvedIndex private final IndexHandle indexHandle; private final TupleDomain undeterminedTupleDomain; - public ResolvedIndex(String connectorId, ConnectorTransactionHandle transactionHandle, ConnectorResolvedIndex index) + public ResolvedIndex(ConnectorId connectorId, ConnectorTransactionHandle transactionHandle, ConnectorResolvedIndex index) { requireNonNull(connectorId, "connectorId is null"); requireNonNull(index, "index is null"); diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/Split.java b/presto-main/src/main/java/com/facebook/presto/metadata/Split.java index 3236ee1aba0a..e307aa6a9839 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/Split.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/Split.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.metadata; +import com.facebook.presto.connector.ConnectorId; import com.facebook.presto.spi.ConnectorSplit; import com.facebook.presto.spi.HostAddress; import com.facebook.presto.spi.connector.ConnectorTransactionHandle; @@ -26,13 +27,13 @@ public final class Split { - private final String connectorId; + private final ConnectorId connectorId; private final ConnectorTransactionHandle transactionHandle; private final ConnectorSplit connectorSplit; @JsonCreator public Split( - @JsonProperty("connectorId") String connectorId, + @JsonProperty("connectorId") ConnectorId connectorId, @JsonProperty("transactionHandle") ConnectorTransactionHandle transactionHandle, @JsonProperty("connectorSplit") ConnectorSplit connectorSplit) { @@ -42,7 +43,7 @@ public Split( } @JsonProperty - public String getConnectorId() + public ConnectorId getConnectorId() { return connectorId; } diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/TableHandle.java b/presto-main/src/main/java/com/facebook/presto/metadata/TableHandle.java index f0376cb5af48..5a63045bc3a6 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/TableHandle.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/TableHandle.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.metadata; +import com.facebook.presto.connector.ConnectorId; import com.facebook.presto.spi.ConnectorTableHandle; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; @@ -23,12 +24,12 @@ public final class TableHandle { - private final String connectorId; + private final ConnectorId connectorId; private final ConnectorTableHandle connectorHandle; @JsonCreator public TableHandle( - @JsonProperty("connectorId") String connectorId, + @JsonProperty("connectorId") ConnectorId connectorId, @JsonProperty("connectorHandle") ConnectorTableHandle connectorHandle) { this.connectorId = requireNonNull(connectorId, "connectorId is null"); @@ -36,7 +37,7 @@ public TableHandle( } @JsonProperty - public String getConnectorId() + public ConnectorId getConnectorId() { return connectorId; } diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/TableLayout.java b/presto-main/src/main/java/com/facebook/presto/metadata/TableLayout.java index 57bc93652bc3..2c6fabf6c76a 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/TableLayout.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/TableLayout.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.metadata; +import com.facebook.presto.connector.ConnectorId; import com.facebook.presto.spi.ColumnHandle; import com.facebook.presto.spi.ConnectorTableLayout; import com.facebook.presto.spi.DiscretePredicates; @@ -43,7 +44,7 @@ public TableLayout(TableLayoutHandle handle, ConnectorTableLayout layout) this.layout = layout; } - public String getConnectorId() + public ConnectorId getConnectorId() { return handle.getConnectorId(); } @@ -89,7 +90,7 @@ public Optional getDiscretePredicates() return layout.getDiscretePredicates(); } - public static TableLayout fromConnectorLayout(String connectorId, ConnectorTransactionHandle transactionHandle, ConnectorTableLayout layout) + public static TableLayout fromConnectorLayout(ConnectorId connectorId, ConnectorTransactionHandle transactionHandle, ConnectorTableLayout layout) { return new TableLayout(new TableLayoutHandle(connectorId, transactionHandle, layout.getHandle()), layout); } diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/TableLayoutHandle.java b/presto-main/src/main/java/com/facebook/presto/metadata/TableLayoutHandle.java index da65595142ed..64a834492659 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/TableLayoutHandle.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/TableLayoutHandle.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.metadata; +import com.facebook.presto.connector.ConnectorId; import com.facebook.presto.spi.ConnectorTableLayoutHandle; import com.facebook.presto.spi.connector.ConnectorTransactionHandle; import com.fasterxml.jackson.annotation.JsonCreator; @@ -24,13 +25,13 @@ public final class TableLayoutHandle { - private final String connectorId; + private final ConnectorId connectorId; private final ConnectorTransactionHandle transactionHandle; private final ConnectorTableLayoutHandle layout; @JsonCreator public TableLayoutHandle( - @JsonProperty("connectorId") String connectorId, + @JsonProperty("connectorId") ConnectorId connectorId, @JsonProperty("transactionHandle") ConnectorTransactionHandle transactionHandle, @JsonProperty("connectorHandle") ConnectorTableLayoutHandle layout) { @@ -44,7 +45,7 @@ public TableLayoutHandle( } @JsonProperty - public String getConnectorId() + public ConnectorId getConnectorId() { return connectorId; } diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/TableMetadata.java b/presto-main/src/main/java/com/facebook/presto/metadata/TableMetadata.java index acf8b347ee63..c2741b2a5ba1 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/TableMetadata.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/TableMetadata.java @@ -13,21 +13,21 @@ */ package com.facebook.presto.metadata; +import com.facebook.presto.connector.ConnectorId; import com.facebook.presto.spi.ColumnMetadata; import com.facebook.presto.spi.ConnectorTableMetadata; import com.facebook.presto.spi.SchemaTableName; import java.util.List; -import static com.facebook.presto.util.ImmutableCollectors.toImmutableList; import static java.util.Objects.requireNonNull; public class TableMetadata { - private final String connectorId; + private final ConnectorId connectorId; private final ConnectorTableMetadata metadata; - public TableMetadata(String connectorId, ConnectorTableMetadata metadata) + public TableMetadata(ConnectorId connectorId, ConnectorTableMetadata metadata) { requireNonNull(connectorId, "catalog is null"); requireNonNull(metadata, "metadata is null"); @@ -36,7 +36,7 @@ public TableMetadata(String connectorId, ConnectorTableMetadata metadata) this.metadata = metadata; } - public String getConnectorId() + public ConnectorId getConnectorId() { return connectorId; } @@ -56,21 +56,6 @@ public List getColumns() return metadata.getColumns(); } - public List getVisibleColumnNames() - { - return getColumns().stream() - .filter(column -> !column.isHidden()) - .map(ColumnMetadata::getName) - .collect(toImmutableList()); - } - - public List getVisibleColumns() - { - return getColumns().stream() - .filter(column -> !column.isHidden()) - .collect(toImmutableList()); - } - public ColumnMetadata getColumn(String name) { return getColumns().stream() diff --git a/presto-main/src/main/java/com/facebook/presto/operator/ExchangeOperator.java b/presto-main/src/main/java/com/facebook/presto/operator/ExchangeOperator.java index 078bd8a72252..952d181dd96f 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/ExchangeOperator.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/ExchangeOperator.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.operator; +import com.facebook.presto.connector.ConnectorId; import com.facebook.presto.metadata.Split; import com.facebook.presto.spi.Page; import com.facebook.presto.spi.UpdatablePageSource; @@ -34,6 +35,8 @@ public class ExchangeOperator implements SourceOperator, Closeable { + public static final ConnectorId REMOTE_CONNECTOR_ID = new ConnectorId("$remote"); + public static class ExchangeOperatorFactory implements SourceOperatorFactory { @@ -112,7 +115,7 @@ public PlanNodeId getSourceId() public Supplier> addSplit(Split split) { requireNonNull(split, "split is null"); - checkArgument(split.getConnectorId().equals("remote"), "split is not a remote split"); + checkArgument(split.getConnectorId().equals(REMOTE_CONNECTOR_ID), "split is not a remote split"); URI location = ((RemoteSplit) split.getConnectorSplit()).getLocation(); exchangeClient.addLocation(location); diff --git a/presto-main/src/main/java/com/facebook/presto/operator/index/IndexLoader.java b/presto-main/src/main/java/com/facebook/presto/operator/index/IndexLoader.java index 7afe689d9304..a498ad58aeef 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/index/IndexLoader.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/index/IndexLoader.java @@ -15,6 +15,7 @@ import com.facebook.presto.ScheduledSplit; import com.facebook.presto.TaskSource; +import com.facebook.presto.connector.ConnectorId; import com.facebook.presto.metadata.Split; import com.facebook.presto.operator.Driver; import com.facebook.presto.operator.DriverFactory; @@ -56,6 +57,7 @@ @ThreadSafe public class IndexLoader { + private static final ConnectorId INDEX_CONNECTOR_ID = new ConnectorId("$index"); private final BlockingQueue updateRequests = new LinkedBlockingQueue<>(); private final List outputTypes; @@ -237,7 +239,7 @@ public IndexedData streamIndexDataForSingleKey(UpdateRequest updateRequest) PageRecordSet pageRecordSet = new PageRecordSet(keyTypes, indexKeyTuple); PlanNodeId planNodeId = driverFactory.getSourceId().get(); - ScheduledSplit split = new ScheduledSplit(0, planNodeId, new Split("index", new ConnectorTransactionHandle() {}, new IndexSplit(pageRecordSet))); + ScheduledSplit split = new ScheduledSplit(0, planNodeId, new Split(INDEX_CONNECTOR_ID, new ConnectorTransactionHandle() {}, new IndexSplit(pageRecordSet))); driver.updateSource(new TaskSource(planNodeId, ImmutableSet.of(split), true)); return new StreamingIndexedData(outputTypes, keyTypes, indexKeyTuple, pageBuffer, driver); @@ -322,7 +324,7 @@ public boolean load(List requests) // Drive index lookup to produce the output (landing in indexSnapshotBuilder) try (Driver driver = driverFactory.createDriver(pipelineContext.addDriverContext())) { PlanNodeId sourcePlanNodeId = driverFactory.getSourceId().get(); - ScheduledSplit split = new ScheduledSplit(0, sourcePlanNodeId, new Split("index", new ConnectorTransactionHandle() {}, new IndexSplit(recordSetForLookupSource))); + ScheduledSplit split = new ScheduledSplit(0, sourcePlanNodeId, new Split(INDEX_CONNECTOR_ID, new ConnectorTransactionHandle() {}, new IndexSplit(recordSetForLookupSource))); driver.updateSource(new TaskSource(sourcePlanNodeId, ImmutableSet.of(split), true)); while (!driver.isFinished()) { ListenableFuture process = driver.process(); diff --git a/presto-main/src/main/java/com/facebook/presto/security/AccessControlManager.java b/presto-main/src/main/java/com/facebook/presto/security/AccessControlManager.java index b29779ef6609..833b489f0f0e 100644 --- a/presto-main/src/main/java/com/facebook/presto/security/AccessControlManager.java +++ b/presto-main/src/main/java/com/facebook/presto/security/AccessControlManager.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.security; +import com.facebook.presto.connector.ConnectorId; import com.facebook.presto.metadata.QualifiedObjectName; import com.facebook.presto.spi.CatalogSchemaTableName; import com.facebook.presto.spi.PrestoException; @@ -103,7 +104,7 @@ public void addSystemAccessControlFactory(SystemAccessControlFactory accessContr } } - public void addCatalogAccessControl(String connectorId, String catalogName, ConnectorAccessControl accessControl) + public void addCatalogAccessControl(ConnectorId connectorId, String catalogName, ConnectorAccessControl accessControl) { requireNonNull(connectorId, "connectorId is null"); requireNonNull(catalogName, "catalogName is null"); @@ -461,10 +462,10 @@ private static Map loadProperties(File file) private class CatalogAccessControlEntry { - private final String connectorId; + private final ConnectorId connectorId; private final ConnectorAccessControl accessControl; - public CatalogAccessControlEntry(String connectorId, ConnectorAccessControl accessControl) + public CatalogAccessControlEntry(ConnectorId connectorId, ConnectorAccessControl accessControl) { this.connectorId = requireNonNull(connectorId, "connectorId is null"); this.accessControl = requireNonNull(accessControl, "accessControl is null"); diff --git a/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java b/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java index 9038887497c3..b15305a503d4 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java +++ b/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.server; +import com.facebook.presto.connector.ConnectorId; import com.facebook.presto.discovery.EmbeddedDiscoveryModule; import com.facebook.presto.eventlistener.EventListenerManager; import com.facebook.presto.eventlistener.EventListenerModule; @@ -156,15 +157,17 @@ private static void updateConnectorIds(Announcer announcer, Metadata metadata, S // automatically build connectorIds if not configured if (connectorIds.isEmpty()) { - Set catalogs = metadata.getCatalogNames().keySet(); + Map catalogNames = metadata.getCatalogNames(); // if this is a dedicated coordinator, only add jmx if (serverConfig.isCoordinator() && !schedulerConfig.isIncludeCoordinator()) { - if (catalogs.contains("jmx")) { - connectorIds.add("jmx"); + if (catalogNames.containsKey("jmx")) { + connectorIds.add(catalogNames.get("jmx").toString()); } } else { - connectorIds.addAll(catalogs); + catalogNames.values().stream() + .map(Object::toString) + .forEach(connectorIds::add); } } diff --git a/presto-main/src/main/java/com/facebook/presto/server/testing/TestingPrestoServer.java b/presto-main/src/main/java/com/facebook/presto/server/testing/TestingPrestoServer.java index bcf0354c5e67..50d01ebc5556 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/testing/TestingPrestoServer.java +++ b/presto-main/src/main/java/com/facebook/presto/server/testing/TestingPrestoServer.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.server.testing; +import com.facebook.presto.connector.ConnectorId; import com.facebook.presto.connector.ConnectorManager; import com.facebook.presto.eventlistener.EventListenerManager; import com.facebook.presto.execution.QueryManager; @@ -291,15 +292,16 @@ public QueryManager getQueryManager() return queryManager; } - public void createCatalog(String catalogName, String connectorName) + public ConnectorId createCatalog(String catalogName, String connectorName) { - createCatalog(catalogName, connectorName, ImmutableMap.of()); + return createCatalog(catalogName, connectorName, ImmutableMap.of()); } - public void createCatalog(String catalogName, String connectorName, Map properties) + public ConnectorId createCatalog(String catalogName, String connectorName, Map properties) { - connectorManager.createConnection(catalogName, connectorName, properties); - updateConnectorIdAnnouncement(announcer, catalogName); + ConnectorId connectorId = connectorManager.createConnection(catalogName, connectorName, properties); + updateConnectorIdAnnouncement(announcer, connectorId); + return connectorId; } public Path getBaseDataDir() @@ -390,12 +392,12 @@ public final AllNodes refreshNodes() return nodeManager.getAllNodes(); } - public Set getActiveNodesWithConnector(String connectorName) + public Set getActiveNodesWithConnector(ConnectorId connectorId) { - return nodeManager.getActiveConnectorNodes(connectorName); + return nodeManager.getActiveConnectorNodes(connectorId); } - private static void updateConnectorIdAnnouncement(Announcer announcer, String connectorId) + private static void updateConnectorIdAnnouncement(Announcer announcer, ConnectorId connectorId) { // // This code was copied from PrestoServer, and is a hack that should be removed when the connectorId property is removed @@ -408,7 +410,7 @@ private static void updateConnectorIdAnnouncement(Announcer announcer, String co Map properties = new LinkedHashMap<>(announcement.getProperties()); String property = nullToEmpty(properties.get("connectorIds")); Set connectorIds = new LinkedHashSet<>(Splitter.on(',').trimResults().omitEmptyStrings().splitToList(property)); - connectorIds.add(connectorId); + connectorIds.add(connectorId.toString()); properties.put("connectorIds", Joiner.on(',').join(connectorIds)); // update announcement diff --git a/presto-main/src/main/java/com/facebook/presto/split/ConnectorAwareSplitSource.java b/presto-main/src/main/java/com/facebook/presto/split/ConnectorAwareSplitSource.java index fab84df43730..77ec512b1416 100644 --- a/presto-main/src/main/java/com/facebook/presto/split/ConnectorAwareSplitSource.java +++ b/presto-main/src/main/java/com/facebook/presto/split/ConnectorAwareSplitSource.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.split; +import com.facebook.presto.connector.ConnectorId; import com.facebook.presto.metadata.Split; import com.facebook.presto.spi.ConnectorSplitSource; import com.facebook.presto.spi.connector.ConnectorTransactionHandle; @@ -26,11 +27,11 @@ public class ConnectorAwareSplitSource implements SplitSource { - private final String connectorId; + private final ConnectorId connectorId; private final ConnectorTransactionHandle transactionHandle; private final ConnectorSplitSource source; - public ConnectorAwareSplitSource(String connectorId, ConnectorTransactionHandle transactionHandle, ConnectorSplitSource source) + public ConnectorAwareSplitSource(ConnectorId connectorId, ConnectorTransactionHandle transactionHandle, ConnectorSplitSource source) { this.connectorId = requireNonNull(connectorId, "connectorId is null"); this.transactionHandle = requireNonNull(transactionHandle, "transactionHandle is null"); @@ -38,7 +39,7 @@ public ConnectorAwareSplitSource(String connectorId, ConnectorTransactionHandle } @Override - public String getConnectorId() + public ConnectorId getConnectorId() { return connectorId; } diff --git a/presto-main/src/main/java/com/facebook/presto/split/PageSinkManager.java b/presto-main/src/main/java/com/facebook/presto/split/PageSinkManager.java index 70dfce4a22f2..c4e08fe28fea 100644 --- a/presto-main/src/main/java/com/facebook/presto/split/PageSinkManager.java +++ b/presto-main/src/main/java/com/facebook/presto/split/PageSinkManager.java @@ -14,6 +14,7 @@ package com.facebook.presto.split; import com.facebook.presto.Session; +import com.facebook.presto.connector.ConnectorId; import com.facebook.presto.metadata.InsertTableHandle; import com.facebook.presto.metadata.OutputTableHandle; import com.facebook.presto.spi.ConnectorPageSink; @@ -28,9 +29,9 @@ public class PageSinkManager implements PageSinkProvider { - private final ConcurrentMap pageSinkProviders = new ConcurrentHashMap<>(); + private final ConcurrentMap pageSinkProviders = new ConcurrentHashMap<>(); - public void addConnectorPageSinkProvider(String connectorId, ConnectorPageSinkProvider connectorPageSinkProvider) + public void addConnectorPageSinkProvider(ConnectorId connectorId, ConnectorPageSinkProvider connectorPageSinkProvider) { pageSinkProviders.put(connectorId, connectorPageSinkProvider); } @@ -51,7 +52,7 @@ public ConnectorPageSink createPageSink(Session session, InsertTableHandle table return providerFor(tableHandle.getConnectorId()).createPageSink(tableHandle.getTransactionHandle(), connectorSession, tableHandle.getConnectorHandle()); } - private ConnectorPageSinkProvider providerFor(String connectorId) + private ConnectorPageSinkProvider providerFor(ConnectorId connectorId) { ConnectorPageSinkProvider provider = pageSinkProviders.get(connectorId); checkArgument(provider != null, "No page sink provider for connector '%s'", connectorId); diff --git a/presto-main/src/main/java/com/facebook/presto/split/PageSourceManager.java b/presto-main/src/main/java/com/facebook/presto/split/PageSourceManager.java index 2366b7fbba32..d1aa212f1c9b 100644 --- a/presto-main/src/main/java/com/facebook/presto/split/PageSourceManager.java +++ b/presto-main/src/main/java/com/facebook/presto/split/PageSourceManager.java @@ -14,6 +14,7 @@ package com.facebook.presto.split; import com.facebook.presto.Session; +import com.facebook.presto.connector.ConnectorId; import com.facebook.presto.metadata.Split; import com.facebook.presto.spi.ColumnHandle; import com.facebook.presto.spi.ConnectorPageSource; @@ -30,9 +31,9 @@ public class PageSourceManager implements PageSourceProvider { - private final ConcurrentMap pageSourceProviders = new ConcurrentHashMap<>(); + private final ConcurrentMap pageSourceProviders = new ConcurrentHashMap<>(); - public void addConnectorPageSourceProvider(String connectorId, ConnectorPageSourceProvider connectorPageSourceProvider) + public void addConnectorPageSourceProvider(ConnectorId connectorId, ConnectorPageSourceProvider connectorPageSourceProvider) { pageSourceProviders.put(connectorId, connectorPageSourceProvider); } @@ -43,7 +44,6 @@ public ConnectorPageSource createPageSource(Session session, Split split, List splitManagers = new ConcurrentHashMap<>(); + private final ConcurrentMap splitManagers = new ConcurrentHashMap<>(); - public void addConnectorSplitManager(String connectorId, ConnectorSplitManager connectorSplitManager) + public void addConnectorSplitManager(ConnectorId connectorId, ConnectorSplitManager connectorSplitManager) { checkState(splitManagers.putIfAbsent(connectorId, connectorSplitManager) == null, "SplitManager for connector '%s' is already registered", connectorId); } public SplitSource getSplits(Session session, TableLayoutHandle layout) { - String connectorId = layout.getConnectorId(); + ConnectorId connectorId = layout.getConnectorId(); ConnectorSplitManager splitManager = getConnectorSplitManager(connectorId); - // assumes connectorId and catalog are the same ConnectorSession connectorSession = session.toConnectorSession(connectorId); ConnectorSplitSource source = splitManager.getSplits(layout.getTransactionHandle(), connectorSession, layout.getConnectorHandle()); @@ -47,7 +47,7 @@ public SplitSource getSplits(Session session, TableLayoutHandle layout) return new ConnectorAwareSplitSource(connectorId, layout.getTransactionHandle(), source); } - private ConnectorSplitManager getConnectorSplitManager(String connectorId) + private ConnectorSplitManager getConnectorSplitManager(ConnectorId connectorId) { ConnectorSplitManager result = splitManagers.get(connectorId); checkArgument(result != null, "No split manager for connector '%s'", connectorId); diff --git a/presto-main/src/main/java/com/facebook/presto/split/SplitSource.java b/presto-main/src/main/java/com/facebook/presto/split/SplitSource.java index e4fa79eae62d..1165075b0d38 100644 --- a/presto-main/src/main/java/com/facebook/presto/split/SplitSource.java +++ b/presto-main/src/main/java/com/facebook/presto/split/SplitSource.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.split; +import com.facebook.presto.connector.ConnectorId; import com.facebook.presto.metadata.Split; import java.io.Closeable; @@ -22,7 +23,7 @@ public interface SplitSource extends Closeable { - String getConnectorId(); + ConnectorId getConnectorId(); CompletableFuture> getNextBatch(int maxSize); diff --git a/presto-main/src/main/java/com/facebook/presto/sql/analyzer/StatementAnalyzer.java b/presto-main/src/main/java/com/facebook/presto/sql/analyzer/StatementAnalyzer.java index a61375f0e8bd..144daa607644 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/analyzer/StatementAnalyzer.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/analyzer/StatementAnalyzer.java @@ -210,7 +210,10 @@ protected Scope visitInsert(Insert insert, Scope scope) accessControl.checkCanInsertIntoTable(session.getRequiredTransactionId(), session.getIdentity(), targetTable); TableMetadata tableMetadata = metadata.getTableMetadata(session, targetTableHandle.get()); - List tableColumns = tableMetadata.getVisibleColumnNames(); + List tableColumns = tableMetadata.getColumns().stream() + .filter(column -> !column.isHidden()) + .map(ColumnMetadata::getName) + .collect(toImmutableList()); List insertColumns; if (insert.getColumns().isPresent()) { diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/LogicalPlanner.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/LogicalPlanner.java index 6b8109e39491..c1499e1c4311 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/LogicalPlanner.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/LogicalPlanner.java @@ -172,18 +172,23 @@ private RelationPlan createTableCreationPlan(Analysis analysis, Query query) RelationPlan plan = createRelationPlan(analysis, query); - TableMetadata tableMetadata = createTableMetadata(destination, getOutputTableColumns(plan), analysis.getCreateTableProperties(), plan.getSampleWeight().isPresent(), analysis.getParameters()); + ConnectorTableMetadata tableMetadata = createTableMetadata(destination, getOutputTableColumns(plan), analysis.getCreateTableProperties(), plan.getSampleWeight().isPresent(), analysis.getParameters()); if (plan.getSampleWeight().isPresent() && !metadata.canCreateSampledTables(session, destination.getCatalogName())) { throw new PrestoException(NOT_SUPPORTED, "Cannot write sampled data to a store that doesn't support sampling"); } Optional newTableLayout = metadata.getNewTableLayout(session, destination.getCatalogName(), tableMetadata); + List columnNames = tableMetadata.getColumns().stream() + .filter(column -> !column.isHidden()) + .map(ColumnMetadata::getName) + .collect(toImmutableList()); + return createTableWriterPlan( analysis, plan, new CreateName(destination.getCatalogName(), tableMetadata, newTableLayout), - tableMetadata.getVisibleColumnNames(), + columnNames, newTableLayout); } @@ -193,14 +198,21 @@ private RelationPlan createInsertPlan(Analysis analysis, Insert insertStatement) TableMetadata tableMetadata = metadata.getTableMetadata(session, insert.getTarget()); - List visibleTableColumnNames = tableMetadata.getVisibleColumnNames(); - List visibleTableColumns = tableMetadata.getVisibleColumns(); + List visibleTableColumns = tableMetadata.getColumns().stream() + .filter(column -> !column.isHidden()) + .collect(toImmutableList()); + List visibleTableColumnNames = visibleTableColumns.stream() + .map(ColumnMetadata::getName) + .collect(toImmutableList()); RelationPlan plan = createRelationPlan(analysis, insertStatement.getQuery()); Map columns = metadata.getColumnHandles(session, insert.getTarget()); ImmutableMap.Builder assignments = ImmutableMap.builder(); - for (ColumnMetadata column : tableMetadata.getVisibleColumns()) { + for (ColumnMetadata column : tableMetadata.getColumns()) { + if (column.isHidden()) { + continue; + } Symbol output = symbolAllocator.newSymbol(column.getName(), column.getType()); int index = insert.getColumns().indexOf(columns.get(column.getName())); if (index < 0) { @@ -346,7 +358,7 @@ private RelationPlan createRelationPlan(Analysis analysis, Query query) .process(query, null); } - private TableMetadata createTableMetadata(QualifiedObjectName table, List columns, Map propertyExpressions, boolean sampled, List parameters) + private ConnectorTableMetadata createTableMetadata(QualifiedObjectName table, List columns, Map propertyExpressions, boolean sampled, List parameters) { Map properties = metadata.getTablePropertyManager().getProperties( table.getCatalogName(), @@ -355,9 +367,7 @@ private TableMetadata createTableMetadata(QualifiedObjectName table, List getOutputTableColumns(RelationPlan plan) diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/NodePartitioningManager.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/NodePartitioningManager.java index e8ace8d32d07..45190ed5d00a 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/NodePartitioningManager.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/NodePartitioningManager.java @@ -14,6 +14,7 @@ package com.facebook.presto.sql.planner; import com.facebook.presto.Session; +import com.facebook.presto.connector.ConnectorId; import com.facebook.presto.execution.scheduler.NodeScheduler; import com.facebook.presto.operator.PartitionFunction; import com.facebook.presto.spi.BucketFunction; @@ -40,7 +41,7 @@ public class NodePartitioningManager { private final NodeScheduler nodeScheduler; - private final ConcurrentMap partitioningProviders = new ConcurrentHashMap<>(); + private final ConcurrentMap partitioningProviders = new ConcurrentHashMap<>(); @Inject public NodePartitioningManager(NodeScheduler nodeScheduler) @@ -48,7 +49,7 @@ public NodePartitioningManager(NodeScheduler nodeScheduler) this.nodeScheduler = requireNonNull(nodeScheduler, "nodeScheduler is null"); } - public void addPartitioningProvider(String connectorId, ConnectorNodePartitioningProvider partitioningProvider) + public void addPartitioningProvider(ConnectorId connectorId, ConnectorNodePartitioningProvider partitioningProvider) { checkArgument(partitioningProviders.putIfAbsent(connectorId, partitioningProvider) == null, "NodePartitioningProvider for connector '%s' is already registered", connectorId); } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/OutputExtractor.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/OutputExtractor.java index 4822aea0b744..37dc961c6dcc 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/OutputExtractor.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/OutputExtractor.java @@ -14,6 +14,7 @@ package com.facebook.presto.sql.planner; +import com.facebook.presto.connector.ConnectorId; import com.facebook.presto.execution.Output; import com.facebook.presto.spi.SchemaTableName; import com.facebook.presto.sql.planner.plan.PlanNode; @@ -44,7 +45,7 @@ public Optional extractOutput(PlanNode root) private class Visitor extends PlanVisitor { - private String connectorId = null; + private ConnectorId connectorId = null; private SchemaTableName schemaTableName = null; @Override @@ -82,7 +83,7 @@ protected Void visitPlan(PlanNode node, Void context) return null; } - public String getConnectorId() + public ConnectorId getConnectorId() { return connectorId; } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/PartitioningHandle.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/PartitioningHandle.java index 893b16f6b67d..56114478cf8d 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/PartitioningHandle.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/PartitioningHandle.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.sql.planner; +import com.facebook.presto.connector.ConnectorId; import com.facebook.presto.spi.connector.ConnectorPartitioningHandle; import com.facebook.presto.spi.connector.ConnectorTransactionHandle; import com.fasterxml.jackson.annotation.JsonCreator; @@ -26,13 +27,13 @@ public class PartitioningHandle { - private final Optional connectorId; + private final Optional connectorId; private final Optional transactionHandle; private final ConnectorPartitioningHandle connectorHandle; @JsonCreator public PartitioningHandle( - @JsonProperty("connectorId") Optional connectorId, + @JsonProperty("connectorId") Optional connectorId, @JsonProperty("transactionHandle") Optional transactionHandle, @JsonProperty("connectorHandle") ConnectorPartitioningHandle connectorHandle) { @@ -43,7 +44,7 @@ public PartitioningHandle( } @JsonProperty - public Optional getConnectorId() + public Optional getConnectorId() { return connectorId; } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/TableWriterNode.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/TableWriterNode.java index 14c90b90bdc7..794cb1f0ed0b 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/TableWriterNode.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/TableWriterNode.java @@ -17,7 +17,7 @@ import com.facebook.presto.metadata.NewTableLayout; import com.facebook.presto.metadata.OutputTableHandle; import com.facebook.presto.metadata.TableHandle; -import com.facebook.presto.metadata.TableMetadata; +import com.facebook.presto.spi.ConnectorTableMetadata; import com.facebook.presto.spi.SchemaTableName; import com.facebook.presto.sql.planner.PartitioningScheme; import com.facebook.presto.sql.planner.Symbol; @@ -146,10 +146,10 @@ public static class CreateName extends WriterTarget { private final String catalog; - private final TableMetadata tableMetadata; + private final ConnectorTableMetadata tableMetadata; private final Optional layout; - public CreateName(String catalog, TableMetadata tableMetadata, Optional layout) + public CreateName(String catalog, ConnectorTableMetadata tableMetadata, Optional layout) { this.catalog = requireNonNull(catalog, "catalog is null"); this.tableMetadata = requireNonNull(tableMetadata, "tableMetadata is null"); @@ -161,7 +161,7 @@ public String getCatalog() return catalog; } - public TableMetadata getTableMetadata() + public ConnectorTableMetadata getTableMetadata() { return tableMetadata; } diff --git a/presto-main/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java b/presto-main/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java index a01d4d4a111d..52baffad0dfd 100644 --- a/presto-main/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java +++ b/presto-main/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java @@ -20,6 +20,7 @@ import com.facebook.presto.TaskSource; import com.facebook.presto.block.BlockEncodingManager; import com.facebook.presto.connector.ConnectorAwareNodeManager; +import com.facebook.presto.connector.ConnectorId; import com.facebook.presto.connector.ConnectorManager; import com.facebook.presto.connector.system.CatalogSystemTable; import com.facebook.presto.connector.system.GlobalSystemConnector; @@ -350,7 +351,7 @@ public int getNodeCount() public NodeManager getNodeManager() { - return new ConnectorAwareNodeManager(nodeManager, "testenv", "test"); + return new ConnectorAwareNodeManager(nodeManager, "testenv", new ConnectorId("test")); } public TypeRegistry getTypeManager() @@ -389,7 +390,7 @@ public Session getDefaultSession() public void createCatalog(String catalogName, ConnectorFactory connectorFactory, Map properties) { - nodeManager.addCurrentNodeConnector(catalogName); + nodeManager.addCurrentNodeConnector(new ConnectorId(catalogName)); connectorManager.addConnectorFactory(connectorFactory); connectorManager.createConnection(catalogName, connectorFactory.getName(), properties); } @@ -397,7 +398,7 @@ public void createCatalog(String catalogName, ConnectorFactory connectorFactory, @Deprecated public void createCatalog(String catalogName, com.facebook.presto.spi.ConnectorFactory connectorFactory, Map properties) { - nodeManager.addCurrentNodeConnector(catalogName); + nodeManager.addCurrentNodeConnector(new ConnectorId(catalogName)); connectorManager.addConnectorFactory(connectorFactory); connectorManager.createConnection(catalogName, connectorFactory.getName(), properties); } diff --git a/presto-main/src/main/java/com/facebook/presto/testing/TestingConnectorContext.java b/presto-main/src/main/java/com/facebook/presto/testing/TestingConnectorContext.java index 93314f679c03..2fed7c668108 100644 --- a/presto-main/src/main/java/com/facebook/presto/testing/TestingConnectorContext.java +++ b/presto-main/src/main/java/com/facebook/presto/testing/TestingConnectorContext.java @@ -16,6 +16,7 @@ import com.facebook.presto.GroupByHashPageIndexerFactory; import com.facebook.presto.PagesIndexPageSorter; import com.facebook.presto.connector.ConnectorAwareNodeManager; +import com.facebook.presto.connector.ConnectorId; import com.facebook.presto.metadata.InMemoryNodeManager; import com.facebook.presto.spi.NodeManager; import com.facebook.presto.spi.PageIndexerFactory; @@ -27,7 +28,7 @@ public class TestingConnectorContext implements ConnectorContext { - private final NodeManager nodeManager = new ConnectorAwareNodeManager(new InMemoryNodeManager(), "testenv", "test"); + private final NodeManager nodeManager = new ConnectorAwareNodeManager(new InMemoryNodeManager(), "testenv", new ConnectorId("test")); private final TypeManager typeManager = new TypeRegistry(); private final PageSorter pageSorter = new PagesIndexPageSorter(); private final PageIndexerFactory pageIndexerFactory = new GroupByHashPageIndexerFactory(); diff --git a/presto-main/src/main/java/com/facebook/presto/testing/TestingTransactionHandle.java b/presto-main/src/main/java/com/facebook/presto/testing/TestingTransactionHandle.java index e5f79c0263a8..5c9d850910bf 100644 --- a/presto-main/src/main/java/com/facebook/presto/testing/TestingTransactionHandle.java +++ b/presto-main/src/main/java/com/facebook/presto/testing/TestingTransactionHandle.java @@ -26,27 +26,18 @@ public class TestingTransactionHandle implements ConnectorTransactionHandle { - private final String connectorId; private final UUID uuid; @JsonCreator public TestingTransactionHandle( - @JsonProperty("connectorId") String connectorId, @JsonProperty("uuid") UUID uuid) { - this.connectorId = requireNonNull(connectorId, "connectorId is null"); this.uuid = requireNonNull(uuid, "uuid is null"); } - public static TestingTransactionHandle create(String connectorId) + public static TestingTransactionHandle create() { - return new TestingTransactionHandle(connectorId, UUID.randomUUID()); - } - - @JsonProperty - public String getConnectorId() - { - return connectorId; + return new TestingTransactionHandle(UUID.randomUUID()); } @JsonProperty @@ -58,7 +49,7 @@ public UUID getUuid() @Override public int hashCode() { - return Objects.hash(connectorId, uuid); + return Objects.hash(uuid); } @Override @@ -71,15 +62,13 @@ public boolean equals(Object obj) return false; } final TestingTransactionHandle other = (TestingTransactionHandle) obj; - return Objects.equals(this.connectorId, other.connectorId) - && Objects.equals(this.uuid, other.uuid); + return Objects.equals(this.uuid, other.uuid); } @Override public String toString() { return toStringHelper(this) - .add("connectorId", connectorId) .add("uuid", uuid) .toString(); } diff --git a/presto-main/src/main/java/com/facebook/presto/transaction/TransactionInfo.java b/presto-main/src/main/java/com/facebook/presto/transaction/TransactionInfo.java index 220a40fe963f..f6f2fbcebd97 100644 --- a/presto-main/src/main/java/com/facebook/presto/transaction/TransactionInfo.java +++ b/presto-main/src/main/java/com/facebook/presto/transaction/TransactionInfo.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.transaction; +import com.facebook.presto.connector.ConnectorId; import com.facebook.presto.spi.transaction.IsolationLevel; import com.google.common.collect.ImmutableList; import io.airlift.units.Duration; @@ -31,10 +32,18 @@ public class TransactionInfo private final boolean autoCommitContext; private final DateTime createTime; private final Duration idleTime; - private final List connectorIds; - private final Optional writtenConnectorId; + private final List connectorIds; + private final Optional writtenConnectorId; - public TransactionInfo(TransactionId transactionId, IsolationLevel isolationLevel, boolean readOnly, boolean autoCommitContext, DateTime createTime, Duration idleTime, List connectorIds, Optional writtenConnectorId) + public TransactionInfo( + TransactionId transactionId, + IsolationLevel isolationLevel, + boolean readOnly, + boolean autoCommitContext, + DateTime createTime, + Duration idleTime, + List connectorIds, + Optional writtenConnectorId) { this.transactionId = requireNonNull(transactionId, "transactionId is null"); this.isolationLevel = requireNonNull(isolationLevel, "isolationLevel is null"); @@ -76,12 +85,12 @@ public Duration getIdleTime() return idleTime; } - public List getConnectorIds() + public List getConnectorIds() { return connectorIds; } - public Optional getWrittenConnectorId() + public Optional getWrittenConnectorId() { return writtenConnectorId; } diff --git a/presto-main/src/main/java/com/facebook/presto/transaction/TransactionManager.java b/presto-main/src/main/java/com/facebook/presto/transaction/TransactionManager.java index 44d6519eaa73..57dd612854b0 100644 --- a/presto-main/src/main/java/com/facebook/presto/transaction/TransactionManager.java +++ b/presto-main/src/main/java/com/facebook/presto/transaction/TransactionManager.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.transaction; +import com.facebook.presto.connector.ConnectorId; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.connector.Connector; import com.facebook.presto.spi.connector.ConnectorMetadata; @@ -71,7 +72,7 @@ public class TransactionManager private final Duration idleTimeout; private final int maxFinishingConcurrency; - private final ConcurrentMap connectorsById = new ConcurrentHashMap<>(); + private final ConcurrentMap connectorsById = new ConcurrentHashMap<>(); private final ConcurrentMap transactions = new ConcurrentHashMap<>(); private final Executor finishingExecutor; @@ -127,7 +128,7 @@ private synchronized void cleanUpExpiredTransactions() } } - public void addConnector(String connectorId, Connector connector) + public void addConnector(ConnectorId connectorId, Connector connector) { requireNonNull(connectorId, "connectorId is null"); requireNonNull(connector, "connector is null"); @@ -160,21 +161,21 @@ public TransactionId beginTransaction(IsolationLevel isolationLevel, boolean rea return transactionId; } - public ConnectorMetadata getMetadata(TransactionId transactionId, String connectorId) + public ConnectorMetadata getMetadata(TransactionId transactionId, ConnectorId connectorId) { TransactionMetadata transactionMetadata = getTransactionMetadata(transactionId); Connector connector = getConnector(connectorId); return transactionMetadata.getConnectorTransactionMetadata(connectorId, connector).getConnectorMetadata(); } - public ConnectorTransactionHandle getConnectorTransaction(TransactionId transactionId, String connectorId) + public ConnectorTransactionHandle getConnectorTransaction(TransactionId transactionId, ConnectorId connectorId) { TransactionMetadata transactionMetadata = getTransactionMetadata(transactionId); Connector connector = getConnector(connectorId); return transactionMetadata.getConnectorTransactionMetadata(connectorId, connector).getTransactionHandle(); } - public void checkConnectorWrite(TransactionId transactionId, String connectorId) + public void checkConnectorWrite(TransactionId transactionId, ConnectorId connectorId) { getTransactionMetadata(transactionId).checkConnectorWrite(connectorId); } @@ -196,7 +197,7 @@ public void trySetInactive(TransactionId transactionId) tryGetTransactionMetadata(transactionId).ifPresent(TransactionMetadata::setInActive); } - private Connector getConnector(String connectorId) + private Connector getConnector(ConnectorId connectorId) { Connector connector = connectorsById.get(connectorId); checkArgument(connector != null, "Unknown connector ID: %s", connectorId); @@ -257,8 +258,8 @@ private static class TransactionMetadata private final IsolationLevel isolationLevel; private final boolean readOnly; private final boolean autoCommitContext; - private final Map connectorIdToMetadata = new ConcurrentHashMap<>(); - private final AtomicReference writtenConnectorId = new AtomicReference<>(); + private final Map connectorIdToMetadata = new ConcurrentHashMap<>(); + private final AtomicReference writtenConnectorId = new AtomicReference<>(); private final Executor finishingExecutor; private final AtomicReference completedSuccessfully = new AtomicReference<>(); private final AtomicReference idleStartTime = new AtomicReference<>(); @@ -302,7 +303,7 @@ public void checkOpenTransaction() } } - public synchronized ConnectorTransactionMetadata getConnectorTransactionMetadata(String connectorId, Connector connector) + public synchronized ConnectorTransactionMetadata getConnectorTransactionMetadata(ConnectorId connectorId, Connector connector) { checkOpenTransaction(); ConnectorTransactionMetadata transactionMetadata = connectorIdToMetadata.get(connectorId); @@ -324,7 +325,7 @@ private ConnectorTransactionHandle beginTransaction(Connector connector) } } - public synchronized void checkConnectorWrite(String connectorId) + public synchronized void checkConnectorWrite(ConnectorId connectorId) { checkOpenTransaction(); ConnectorTransactionMetadata transactionMetadata = connectorIdToMetadata.get(connectorId); @@ -351,7 +352,7 @@ public synchronized CompletableFuture asyncCommit() return failedFuture(new PrestoException(TRANSACTION_ALREADY_ABORTED, "Current transaction has already been aborted")); } - String writeConnectorId = this.writtenConnectorId.get(); + ConnectorId writeConnectorId = this.writtenConnectorId.get(); if (writeConnectorId == null) { List> futures = connectorIdToMetadata.values().stream() .map(transactionMetadata -> runAsync(transactionMetadata::commit, finishingExecutor)) @@ -422,8 +423,8 @@ public TransactionInfo getTransactionInfo() Duration idleTime = Optional.ofNullable(idleStartTime.get()) .map(Duration::nanosSince) .orElse(new Duration(0, MILLISECONDS)); - Optional writtenConnectorId = Optional.ofNullable(this.writtenConnectorId.get()); - List connectorIds = ImmutableList.copyOf(connectorIdToMetadata.keySet()); + Optional writtenConnectorId = Optional.ofNullable(this.writtenConnectorId.get()); + List connectorIds = ImmutableList.copyOf(connectorIdToMetadata.keySet()); return new TransactionInfo(transactionId, isolationLevel, readOnly, autoCommitContext, createTime, idleTime, connectorIds, writtenConnectorId); } diff --git a/presto-main/src/test/java/com/facebook/presto/connector/system/TestSystemSplit.java b/presto-main/src/test/java/com/facebook/presto/connector/system/TestSystemSplit.java index f2064cb36ef7..d3682dd954f6 100644 --- a/presto-main/src/test/java/com/facebook/presto/connector/system/TestSystemSplit.java +++ b/presto-main/src/test/java/com/facebook/presto/connector/system/TestSystemSplit.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.connector.system; +import com.facebook.presto.connector.ConnectorId; import com.facebook.presto.spi.HostAddress; import com.facebook.presto.spi.predicate.TupleDomain; import io.airlift.json.JsonCodec; @@ -27,7 +28,7 @@ public class TestSystemSplit public void testSerialization() throws Exception { - String connectorId = "testid"; + ConnectorId connectorId = new ConnectorId("testid"); SystemTableHandle tableHandle = new SystemTableHandle(connectorId, "xyz", "foo"); SystemSplit expected = new SystemSplit(connectorId, tableHandle, HostAddress.fromParts("127.0.0.1", 0), TupleDomain.all()); diff --git a/presto-main/src/test/java/com/facebook/presto/execution/BenchmarkNodeScheduler.java b/presto-main/src/test/java/com/facebook/presto/execution/BenchmarkNodeScheduler.java index 7038dd09ad12..f04854029cdf 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/BenchmarkNodeScheduler.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/BenchmarkNodeScheduler.java @@ -14,6 +14,7 @@ package com.facebook.presto.execution; import com.facebook.presto.client.NodeVersion; +import com.facebook.presto.connector.ConnectorId; import com.facebook.presto.execution.scheduler.FlatNetworkTopology; import com.facebook.presto.execution.scheduler.LegacyNetworkTopology; import com.facebook.presto.execution.scheduler.NetworkLocation; @@ -87,6 +88,7 @@ public class BenchmarkNodeScheduler private static final int RACKS = DATA_NODES / 25; private static final int SPLITS = NODES * (MAX_SPLITS_PER_NODE + MAX_PENDING_SPLITS_PER_TASK_PER_NODE / 3); private static final int SPLIT_BATCH_SIZE = 100; + private static final ConnectorId CONNECTOR_ID = new ConnectorId("test_connector_id"); @Benchmark @OperationsPerInvocation(SPLITS) @@ -139,7 +141,7 @@ public static class BenchmarkData public void setup() throws NoSuchMethodException, IllegalAccessException { - TestingTransactionHandle transactionHandle = TestingTransactionHandle.create("foo"); + TestingTransactionHandle transactionHandle = TestingTransactionHandle.create(); finalizerService.start(); NodeTaskMap nodeTaskMap = new NodeTaskMap(finalizerService); @@ -154,7 +156,7 @@ public void setup() Node node = nodes.get(i); ImmutableList.Builder initialSplits = ImmutableList.builder(); for (int j = 0; j < MAX_SPLITS_PER_NODE + MAX_PENDING_SPLITS_PER_TASK_PER_NODE; j++) { - initialSplits.add(new Split("foo", transactionHandle, new TestSplitRemote(i))); + initialSplits.add(new Split(CONNECTOR_ID, transactionHandle, new TestSplitRemote(i))); } TaskId taskId = new TaskId("test", 1, i); MockRemoteTaskFactory.MockRemoteTask remoteTask = remoteTaskFactory.createTableScanTask(taskId, node, initialSplits.build(), nodeTaskMap.createPartitionedSplitCountTracker(node, taskId)); @@ -163,13 +165,13 @@ public void setup() } for (int i = 0; i < SPLITS; i++) { - splits.add(new Split("foo", transactionHandle, new TestSplitRemote(ThreadLocalRandom.current().nextInt(DATA_NODES)))); + splits.add(new Split(CONNECTOR_ID, transactionHandle, new TestSplitRemote(ThreadLocalRandom.current().nextInt(DATA_NODES)))); } InMemoryNodeManager nodeManager = new InMemoryNodeManager(); - nodeManager.addNode("foo", nodes); + nodeManager.addNode(CONNECTOR_ID, nodes); NodeScheduler nodeScheduler = new NodeScheduler(getNetworkTopology(), nodeManager, getNodeSchedulerConfig(), nodeTaskMap); - nodeSelector = nodeScheduler.createNodeSelector("foo"); + nodeSelector = nodeScheduler.createNodeSelector(CONNECTOR_ID); } @TearDown diff --git a/presto-main/src/test/java/com/facebook/presto/execution/MockRemoteTaskFactory.java b/presto-main/src/test/java/com/facebook/presto/execution/MockRemoteTaskFactory.java index 1392226f24e2..41d381292391 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/MockRemoteTaskFactory.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/MockRemoteTaskFactory.java @@ -15,6 +15,7 @@ import com.facebook.presto.OutputBuffers; import com.facebook.presto.Session; +import com.facebook.presto.connector.ConnectorId; import com.facebook.presto.execution.NodeTaskMap.PartitionedSplitCountTracker; import com.facebook.presto.execution.buffer.OutputBuffer; import com.facebook.presto.execution.buffer.SharedOutputBuffer; @@ -96,7 +97,7 @@ public MockRemoteTask createTableScanTask(TaskId taskId, Node newNode, List EMPTY_SOURCES = ImmutableList.of(); @@ -80,7 +83,7 @@ private TaskTestUtils() new PlanFragmentId("fragment"), new TableScanNode( TABLE_SCAN_NODE_ID, - new TableHandle("test", new TestingTableHandle()), + new TableHandle(CONNECTOR_ID, new TestingTableHandle()), ImmutableList.of(SYMBOL), ImmutableMap.of(SYMBOL, new TestingColumnHandle("column")), Optional.empty(), @@ -97,7 +100,7 @@ public static LocalExecutionPlanner createTestingPlanner() MetadataManager metadata = MetadataManager.createTestMetadataManager(); PageSourceManager pageSourceManager = new PageSourceManager(); - pageSourceManager.addConnectorPageSourceProvider("test", new TestingPageSourceProvider()); + pageSourceManager.addConnectorPageSourceProvider(CONNECTOR_ID, new TestingPageSourceProvider()); // we don't start the finalizer so nothing will be collected, which is ok for a test FinalizerService finalizerService = new FinalizerService(); diff --git a/presto-main/src/test/java/com/facebook/presto/execution/TestInput.java b/presto-main/src/test/java/com/facebook/presto/execution/TestInput.java index 21e79ec7ec91..c0ece350408e 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/TestInput.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/TestInput.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.execution; +import com.facebook.presto.connector.ConnectorId; import com.google.common.collect.ImmutableList; import io.airlift.json.JsonCodec; import org.testng.annotations.Test; @@ -29,7 +30,7 @@ public class TestInput public void testRoundTrip() throws Exception { - Input expected = new Input("connectorId", "schema", "table", Optional.empty(), ImmutableList.of( + Input expected = new Input(new ConnectorId("connectorId"), "schema", "table", Optional.empty(), ImmutableList.of( new Column("column1", "string"), new Column("column2", "string"), new Column("column3", "string"))); diff --git a/presto-main/src/test/java/com/facebook/presto/execution/TestNodeScheduler.java b/presto-main/src/test/java/com/facebook/presto/execution/TestNodeScheduler.java index 038166199fa3..7e6dacb9aa5a 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/TestNodeScheduler.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/TestNodeScheduler.java @@ -14,6 +14,7 @@ package com.facebook.presto.execution; import com.facebook.presto.client.NodeVersion; +import com.facebook.presto.connector.ConnectorId; import com.facebook.presto.execution.scheduler.LegacyNetworkTopology; import com.facebook.presto.execution.scheduler.NetworkLocation; import com.facebook.presto.execution.scheduler.NetworkLocationCache; @@ -65,6 +66,7 @@ @Test(singleThreaded = true) public class TestNodeScheduler { + private static final ConnectorId CONNECTOR_ID = new ConnectorId("connector_id"); private FinalizerService finalizerService; private NodeTaskMap nodeTaskMap; private InMemoryNodeManager nodeManager; @@ -85,7 +87,7 @@ public void setUp() nodeBuilder.add(new PrestoNode("other2", URI.create("http://127.0.0.1:12"), NodeVersion.UNKNOWN, false)); nodeBuilder.add(new PrestoNode("other3", URI.create("http://127.0.0.1:13"), NodeVersion.UNKNOWN, false)); ImmutableList nodes = nodeBuilder.build(); - nodeManager.addNode("foo", nodes); + nodeManager.addNode(CONNECTOR_ID, nodes); NodeSchedulerConfig nodeSchedulerConfig = new NodeSchedulerConfig() .setMaxSplitsPerNode(20) .setIncludeCoordinator(false) @@ -94,7 +96,7 @@ public void setUp() NodeScheduler nodeScheduler = new NodeScheduler(new LegacyNetworkTopology(), nodeManager, nodeSchedulerConfig, nodeTaskMap); // contents of taskMap indicate the node-task map for the current stage taskMap = new HashMap<>(); - nodeSelector = nodeScheduler.createNodeSelector("foo"); + nodeSelector = nodeScheduler.createNodeSelector(CONNECTOR_ID); remoteTaskExecutor = Executors.newCachedThreadPool(daemonThreadsNamed("remoteTaskExecutor-%s")); finalizerService.start(); @@ -112,7 +114,7 @@ public void tearDown() public void testScheduleLocal() throws Exception { - Split split = new Split("foo", TestingTransactionHandle.create("test"), new TestSplitLocal()); + Split split = new Split(CONNECTOR_ID, TestingTransactionHandle.create(), new TestSplitLocal()); Set splits = ImmutableSet.of(split); Map.Entry assignment = Iterables.getOnlyElement(nodeSelector.computeAssignments(splits, ImmutableList.copyOf(taskMap.values())).entries()); @@ -124,7 +126,7 @@ public void testScheduleLocal() public void testTopologyAwareScheduling() throws Exception { - TestingTransactionHandle transactionHandle = TestingTransactionHandle.create("foo"); + TestingTransactionHandle transactionHandle = TestingTransactionHandle.create(); NodeTaskMap nodeTaskMap = new NodeTaskMap(finalizerService); InMemoryNodeManager nodeManager = new InMemoryNodeManager(); @@ -133,7 +135,7 @@ public void testTopologyAwareScheduling() nodeBuilder.add(new PrestoNode("node2", URI.create("http://host2.rack1:12"), NodeVersion.UNKNOWN, false)); nodeBuilder.add(new PrestoNode("node3", URI.create("http://host3.rack2:13"), NodeVersion.UNKNOWN, false)); ImmutableList nodes = nodeBuilder.build(); - nodeManager.addNode("foo", nodes); + nodeManager.addNode(CONNECTOR_ID, nodes); // contents of taskMap indicate the node-task map for the current stage Map taskMap = new HashMap<>(); @@ -159,12 +161,12 @@ public NetworkLocation get(HostAddress host) } }; NodeScheduler nodeScheduler = new NodeScheduler(locationCache, topology, nodeManager, nodeSchedulerConfig, nodeTaskMap); - NodeSelector nodeSelector = nodeScheduler.createNodeSelector("foo"); + NodeSelector nodeSelector = nodeScheduler.createNodeSelector(CONNECTOR_ID); // Fill up the nodes with non-local data ImmutableSet.Builder nonRackLocalBuilder = ImmutableSet.builder(); for (int i = 0; i < 26 * 3; i++) { - nonRackLocalBuilder.add(new Split("foo", transactionHandle, new TestSplitRemote(HostAddress.fromParts("data.other_rack", 1)))); + nonRackLocalBuilder.add(new Split(CONNECTOR_ID, transactionHandle, new TestSplitRemote(HostAddress.fromParts("data.other_rack", 1)))); } Set nonRackLocalSplits = nonRackLocalBuilder.build(); Multimap assignments = nodeSelector.computeAssignments(nonRackLocalSplits, ImmutableList.copyOf(taskMap.values())); @@ -196,10 +198,10 @@ public NetworkLocation get(HostAddress host) HostAddress dataHost1 = HostAddress.fromParts("data.rack1", 1); HostAddress dataHost2 = HostAddress.fromParts("data.rack2", 1); for (int i = 0; i < 6 * 2; i++) { - rackLocalSplits.add(new Split("foo", transactionHandle, new TestSplitRemote(dataHost1))); + rackLocalSplits.add(new Split(CONNECTOR_ID, transactionHandle, new TestSplitRemote(dataHost1))); } for (int i = 0; i < 6; i++) { - rackLocalSplits.add(new Split("foo", transactionHandle, new TestSplitRemote(dataHost2))); + rackLocalSplits.add(new Split(CONNECTOR_ID, transactionHandle, new TestSplitRemote(dataHost2))); } assignments = nodeSelector.computeAssignments(rackLocalSplits.build(), ImmutableList.copyOf(taskMap.values())); for (Node node : assignments.keySet()) { @@ -251,9 +253,9 @@ public NetworkLocation get(HostAddress host) // Assign local splits ImmutableSet.Builder localSplits = ImmutableSet.builder(); - localSplits.add(new Split("foo", transactionHandle, new TestSplitRemote(HostAddress.fromParts("host1.rack1", 1)))); - localSplits.add(new Split("foo", transactionHandle, new TestSplitRemote(HostAddress.fromParts("host2.rack1", 1)))); - localSplits.add(new Split("foo", transactionHandle, new TestSplitRemote(HostAddress.fromParts("host3.rack2", 1)))); + localSplits.add(new Split(CONNECTOR_ID, transactionHandle, new TestSplitRemote(HostAddress.fromParts("host1.rack1", 1)))); + localSplits.add(new Split(CONNECTOR_ID, transactionHandle, new TestSplitRemote(HostAddress.fromParts("host2.rack1", 1)))); + localSplits.add(new Split(CONNECTOR_ID, transactionHandle, new TestSplitRemote(HostAddress.fromParts("host3.rack2", 1)))); assignments = nodeSelector.computeAssignments(localSplits.build(), ImmutableList.copyOf(taskMap.values())); assertEquals(assignments.size(), 3); assertEquals(assignments.keySet().size(), 3); @@ -268,13 +270,13 @@ public void testMultipleTasksPerNode() .setMaxPendingSplitsPerNodePerStage(10); NodeScheduler nodeScheduler = new NodeScheduler(new LegacyNetworkTopology(), nodeManager, nodeSchedulerConfig, nodeTaskMap); - NodeSelector nodeSelector = nodeScheduler.createNodeSelector("foo"); + NodeSelector nodeSelector = nodeScheduler.createNodeSelector(CONNECTOR_ID); List nodes = nodeSelector.selectRandomNodes(10); assertEquals(nodes.size(), 3); nodeSchedulerConfig.setMultipleTasksPerNodeEnabled(true); nodeScheduler = new NodeScheduler(new LegacyNetworkTopology(), nodeManager, nodeSchedulerConfig, nodeTaskMap); - nodeSelector = nodeScheduler.createNodeSelector("foo"); + nodeSelector = nodeScheduler.createNodeSelector(CONNECTOR_ID); nodes = nodeSelector.selectRandomNodes(9); assertEquals(nodes.size(), 9); Map counts = new HashMap<>(); @@ -292,7 +294,7 @@ public void testScheduleRemote() throws Exception { Set splits = new HashSet<>(); - splits.add(new Split("foo", TestingTransactionHandle.create("foo"), new TestSplitRemote())); + splits.add(new Split(CONNECTOR_ID, TestingTransactionHandle.create(), new TestSplitRemote())); Multimap assignments = nodeSelector.computeAssignments(splits, ImmutableList.copyOf(taskMap.values())); assertEquals(assignments.size(), 1); } @@ -304,11 +306,11 @@ public void testBasicAssignment() // One split for each node Set splits = new HashSet<>(); for (int i = 0; i < 3; i++) { - splits.add(new Split("foo", TestingTransactionHandle.create("foo"), new TestSplitRemote())); + splits.add(new Split(CONNECTOR_ID, TestingTransactionHandle.create(), new TestSplitRemote())); } Multimap assignments = nodeSelector.computeAssignments(splits, ImmutableList.copyOf(taskMap.values())); assertEquals(assignments.entries().size(), 3); - for (Node node : nodeManager.getActiveConnectorNodes("foo")) { + for (Node node : nodeManager.getActiveConnectorNodes(CONNECTOR_ID)) { assertTrue(assignments.keySet().contains(node)); } } @@ -317,14 +319,14 @@ public void testBasicAssignment() public void testMaxSplitsPerNode() throws Exception { - TestingTransactionHandle transactionHandle = TestingTransactionHandle.create("foo"); + TestingTransactionHandle transactionHandle = TestingTransactionHandle.create(); Node newNode = new PrestoNode("other4", URI.create("http://127.0.0.1:14"), NodeVersion.UNKNOWN, false); - nodeManager.addNode("foo", newNode); + nodeManager.addNode(CONNECTOR_ID, newNode); ImmutableList.Builder initialSplits = ImmutableList.builder(); for (int i = 0; i < 10; i++) { - initialSplits.add(new Split("foo", transactionHandle, new TestSplitRemote())); + initialSplits.add(new Split(CONNECTOR_ID, transactionHandle, new TestSplitRemote())); } MockRemoteTaskFactory remoteTaskFactory = new MockRemoteTaskFactory(remoteTaskExecutor); @@ -339,7 +341,7 @@ public void testMaxSplitsPerNode() Set splits = new HashSet<>(); for (int i = 0; i < 5; i++) { - splits.add(new Split("foo", transactionHandle, new TestSplitRemote())); + splits.add(new Split(CONNECTOR_ID, transactionHandle, new TestSplitRemote())); } Multimap assignments = nodeSelector.computeAssignments(splits, ImmutableList.copyOf(taskMap.values())); @@ -356,19 +358,19 @@ public void testMaxSplitsPerNode() public void testMaxSplitsPerNodePerTask() throws Exception { - TestingTransactionHandle transactionHandle = TestingTransactionHandle.create("foo"); + TestingTransactionHandle transactionHandle = TestingTransactionHandle.create(); Node newNode = new PrestoNode("other4", URI.create("http://127.0.0.1:14"), NodeVersion.UNKNOWN, false); - nodeManager.addNode("foo", newNode); + nodeManager.addNode(CONNECTOR_ID, newNode); ImmutableList.Builder initialSplits = ImmutableList.builder(); for (int i = 0; i < 20; i++) { - initialSplits.add(new Split("foo", transactionHandle, new TestSplitRemote())); + initialSplits.add(new Split(CONNECTOR_ID, transactionHandle, new TestSplitRemote())); } List tasks = new ArrayList<>(); MockRemoteTaskFactory remoteTaskFactory = new MockRemoteTaskFactory(remoteTaskExecutor); - for (Node node : nodeManager.getActiveConnectorNodes("foo")) { + for (Node node : nodeManager.getActiveConnectorNodes(CONNECTOR_ID)) { // Max out number of splits on node TaskId taskId = new TaskId("test", 1, 1); RemoteTask remoteTask = remoteTaskFactory.createTableScanTask(taskId, node, initialSplits.build(), nodeTaskMap.createPartitionedSplitCountTracker(node, taskId)); @@ -385,7 +387,7 @@ public void testMaxSplitsPerNodePerTask() Set splits = new HashSet<>(); for (int i = 0; i < 5; i++) { - splits.add(new Split("foo", transactionHandle, new TestSplitRemote())); + splits.add(new Split(CONNECTOR_ID, transactionHandle, new TestSplitRemote())); } Multimap assignments = nodeSelector.computeAssignments(splits, ImmutableList.copyOf(taskMap.values())); @@ -405,12 +407,12 @@ public void testTaskCompletion() throws Exception { MockRemoteTaskFactory remoteTaskFactory = new MockRemoteTaskFactory(remoteTaskExecutor); - Node chosenNode = Iterables.get(nodeManager.getActiveConnectorNodes("foo"), 0); + Node chosenNode = Iterables.get(nodeManager.getActiveConnectorNodes(CONNECTOR_ID), 0); TaskId taskId = new TaskId("test", 1, 1); RemoteTask remoteTask = remoteTaskFactory.createTableScanTask( taskId, chosenNode, - ImmutableList.of(new Split("foo", TestingTransactionHandle.create("foo"), new TestSplitRemote())), + ImmutableList.of(new Split(CONNECTOR_ID, TestingTransactionHandle.create(), new TestSplitRemote())), nodeTaskMap.createPartitionedSplitCountTracker(chosenNode, taskId)); nodeTaskMap.addTask(chosenNode, remoteTask); assertEquals(nodeTaskMap.getPartitionedSplitsOnNode(chosenNode), 1); @@ -427,19 +429,21 @@ public void testSplitCount() throws Exception { MockRemoteTaskFactory remoteTaskFactory = new MockRemoteTaskFactory(remoteTaskExecutor); - Node chosenNode = Iterables.get(nodeManager.getActiveConnectorNodes("foo"), 0); + Node chosenNode = Iterables.get(nodeManager.getActiveConnectorNodes(CONNECTOR_ID), 0); TaskId taskId1 = new TaskId("test", 1, 1); RemoteTask remoteTask1 = remoteTaskFactory.createTableScanTask(taskId1, chosenNode, - ImmutableList.of(new Split("foo", TestingTransactionHandle.create("foo"), new TestSplitRemote()), new Split("bar", TestingTransactionHandle.create("bar"), new TestSplitRemote())), + ImmutableList.of( + new Split(CONNECTOR_ID, TestingTransactionHandle.create(), new TestSplitRemote()), + new Split(CONNECTOR_ID, TestingTransactionHandle.create(), new TestSplitRemote())), nodeTaskMap.createPartitionedSplitCountTracker(chosenNode, taskId1)); TaskId taskId2 = new TaskId("test", 1, 2); RemoteTask remoteTask2 = remoteTaskFactory.createTableScanTask( taskId2, chosenNode, - ImmutableList.of(new Split("foo2", TestingTransactionHandle.create("foo2"), new TestSplitRemote())), + ImmutableList.of(new Split(CONNECTOR_ID, TestingTransactionHandle.create(), new TestSplitRemote())), nodeTaskMap.createPartitionedSplitCountTracker(chosenNode, taskId2)); nodeTaskMap.addTask(chosenNode, remoteTask1); diff --git a/presto-main/src/test/java/com/facebook/presto/execution/TestOutput.java b/presto-main/src/test/java/com/facebook/presto/execution/TestOutput.java index 3e40b9023858..555bcd9137df 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/TestOutput.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/TestOutput.java @@ -14,6 +14,7 @@ package com.facebook.presto.execution; +import com.facebook.presto.connector.ConnectorId; import io.airlift.json.JsonCodec; import org.testng.annotations.Test; @@ -27,7 +28,7 @@ public class TestOutput public void testRoundTrip() throws Exception { - Output expected = new Output("connectorId", "schema", "table"); + Output expected = new Output(new ConnectorId("connectorId"), "schema", "table"); String json = codec.toJson(expected); Output actual = codec.fromJson(json); diff --git a/presto-main/src/test/java/com/facebook/presto/execution/TestQueryStateMachine.java b/presto-main/src/test/java/com/facebook/presto/execution/TestQueryStateMachine.java index cfd31b9b7847..8dafa666ce45 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/TestQueryStateMachine.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/TestQueryStateMachine.java @@ -15,6 +15,7 @@ import com.facebook.presto.Session; import com.facebook.presto.client.FailureInfo; +import com.facebook.presto.connector.ConnectorId; import com.facebook.presto.memory.VersionedMemoryPoolId; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.QueryId; @@ -61,7 +62,7 @@ public class TestQueryStateMachine private static final String QUERY = "sql"; private static final URI LOCATION = URI.create("fake://fake-query"); private static final SQLException FAILED_CAUSE = new SQLException("FAILED"); - private static final List INPUTS = ImmutableList.of(new Input("connector", "schema", "table", Optional.empty(), ImmutableList.of(new Column("a", "varchar")))); + private static final List INPUTS = ImmutableList.of(new Input(new ConnectorId("connector"), "schema", "table", Optional.empty(), ImmutableList.of(new Column("a", "varchar")))); private static final Optional OUTPUT = Optional.empty(); private static final List OUTPUT_FIELD_NAMES = ImmutableList.of("a", "b", "c"); private static final String UPDATE_TYPE = "update type"; diff --git a/presto-main/src/test/java/com/facebook/presto/execution/scheduler/TestPhasedExecutionSchedule.java b/presto-main/src/test/java/com/facebook/presto/execution/scheduler/TestPhasedExecutionSchedule.java index bf6032cff0fb..1f5346eeccd4 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/scheduler/TestPhasedExecutionSchedule.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/scheduler/TestPhasedExecutionSchedule.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.execution.scheduler; +import com.facebook.presto.connector.ConnectorId; import com.facebook.presto.metadata.TableHandle; import com.facebook.presto.spi.predicate.TupleDomain; import com.facebook.presto.spi.type.Type; @@ -181,7 +182,7 @@ private static PlanFragment createBroadcastJoinPlanFragment(String name, PlanFra Symbol symbol = new Symbol("column"); PlanNode tableScan = new TableScanNode( new PlanNodeId(name), - new TableHandle("test", new TestingTableHandle()), + new TableHandle(new ConnectorId("test"), new TestingTableHandle()), ImmutableList.of(symbol), ImmutableMap.of(symbol, new TestingColumnHandle("column")), Optional.empty(), @@ -221,7 +222,7 @@ private static PlanFragment createTableScanPlanFragment(String name) Symbol symbol = new Symbol("column"); PlanNode planNode = new TableScanNode( new PlanNodeId(name), - new TableHandle("test", new TestingTableHandle()), + new TableHandle(new ConnectorId("test"), new TestingTableHandle()), ImmutableList.of(symbol), ImmutableMap.of(symbol, new TestingColumnHandle("column")), Optional.empty(), diff --git a/presto-main/src/test/java/com/facebook/presto/execution/scheduler/TestSourcePartitionedScheduler.java b/presto-main/src/test/java/com/facebook/presto/execution/scheduler/TestSourcePartitionedScheduler.java index c794b74c34ce..6501543c887e 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/scheduler/TestSourcePartitionedScheduler.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/scheduler/TestSourcePartitionedScheduler.java @@ -15,6 +15,7 @@ import com.facebook.presto.OutputBuffers.OutputBufferId; import com.facebook.presto.client.NodeVersion; +import com.facebook.presto.connector.ConnectorId; import com.facebook.presto.execution.LocationFactory; import com.facebook.presto.execution.MockRemoteTaskFactory; import com.facebook.presto.execution.MockRemoteTaskFactory.MockRemoteTask; @@ -88,7 +89,7 @@ public class TestSourcePartitionedScheduler { public static final OutputBufferId OUT = new OutputBufferId(0); - private static final String CONNECTOR_ID = "test"; + private static final ConnectorId CONNECTOR_ID = new ConnectorId("connector_id"); private final ExecutorService executor = newCachedThreadPool(daemonThreadsNamed("stageExecutor-%s")); private final LocationFactory locationFactory = new MockLocationFactory(); @@ -299,7 +300,7 @@ public void testNoNodes() stage, Iterables.getOnlyElement(plan.getSplitSources().keySet()), Iterables.getOnlyElement(plan.getSplitSources().values()), - new DynamicSplitPlacementPolicy(nodeScheduler.createNodeSelector("test"), stage::getAllTasks), + new DynamicSplitPlacementPolicy(nodeScheduler.createNodeSelector(CONNECTOR_ID), stage::getAllTasks), 2); scheduler.schedule(); @@ -449,7 +450,7 @@ private static StageExecutionPlan createPlan(ConnectorSplitSource splitSource) return new StageExecutionPlan( testFragment, - ImmutableMap.of(tableScanNodeId, new ConnectorAwareSplitSource(CONNECTOR_ID, TestingTransactionHandle.create(CONNECTOR_ID), splitSource)), + ImmutableMap.of(tableScanNodeId, new ConnectorAwareSplitSource(CONNECTOR_ID, TestingTransactionHandle.create(), splitSource)), ImmutableList.of()); } diff --git a/presto-main/src/test/java/com/facebook/presto/metadata/TestSystemTableHandle.java b/presto-main/src/test/java/com/facebook/presto/metadata/TestSystemTableHandle.java index a5ef71d7923b..83547aca6b14 100644 --- a/presto-main/src/test/java/com/facebook/presto/metadata/TestSystemTableHandle.java +++ b/presto-main/src/test/java/com/facebook/presto/metadata/TestSystemTableHandle.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.metadata; +import com.facebook.presto.connector.ConnectorId; import com.facebook.presto.connector.system.SystemTableHandle; import com.facebook.presto.spi.ConnectorTableHandle; import com.facebook.presto.spi.SchemaTableName; @@ -34,10 +35,10 @@ @Test(singleThreaded = true) public class TestSystemTableHandle { - private static final String CONNECTOR_ID = "system_connector_id"; - private static final Map SCHEMA_AS_MAP = ImmutableMap.of( + private static final ConnectorId CONNECTOR_ID = new ConnectorId("system_connector_id"); + private static final Map SCHEMA_AS_MAP = ImmutableMap.of( "@type", "$system", - "connectorId", CONNECTOR_ID, + "connectorId", CONNECTOR_ID.toString(), "schemaName", "system_schema", "tableName", "system_table"); diff --git a/presto-main/src/test/java/com/facebook/presto/operator/TestDriver.java b/presto-main/src/test/java/com/facebook/presto/operator/TestDriver.java index c48222368e87..d00f14c059db 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/TestDriver.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/TestDriver.java @@ -16,6 +16,7 @@ import com.facebook.presto.ScheduledSplit; import com.facebook.presto.Session; import com.facebook.presto.TaskSource; +import com.facebook.presto.connector.ConnectorId; import com.facebook.presto.metadata.Split; import com.facebook.presto.spi.ColumnHandle; import com.facebook.presto.spi.ConnectorPageSource; @@ -300,7 +301,7 @@ public Boolean call() private static Split newMockSplit() { - return new Split("test", TestingTransactionHandle.create("test"), new MockSplit()); + return new Split(new ConnectorId("test"), TestingTransactionHandle.create(), new MockSplit()); } private PageConsumerOperator createSinkOperator(Operator source) diff --git a/presto-main/src/test/java/com/facebook/presto/operator/TestExchangeOperator.java b/presto-main/src/test/java/com/facebook/presto/operator/TestExchangeOperator.java index edd8ee69a940..15afcb497a57 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/TestExchangeOperator.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/TestExchangeOperator.java @@ -59,6 +59,7 @@ import static com.facebook.presto.client.PrestoHeaders.PRESTO_PAGE_NEXT_TOKEN; import static com.facebook.presto.client.PrestoHeaders.PRESTO_PAGE_TOKEN; import static com.facebook.presto.client.PrestoHeaders.PRESTO_TASK_INSTANCE_ID; +import static com.facebook.presto.operator.ExchangeOperator.REMOTE_CONNECTOR_ID; import static com.facebook.presto.operator.PageAssertions.assertPageEquals; import static com.facebook.presto.spi.type.VarcharType.VARCHAR; import static com.facebook.presto.testing.TestingTaskContext.createTaskContext; @@ -157,7 +158,7 @@ public void testSimple() private Split newRemoteSplit(String taskId) { - return new Split("remote", new RemoteTransactionHandle(), new RemoteSplit(URI.create("http://localhost/" + taskId))); + return new Split(REMOTE_CONNECTOR_ID, new RemoteTransactionHandle(), new RemoteSplit(URI.create("http://localhost/" + taskId))); } @Test diff --git a/presto-main/src/test/java/com/facebook/presto/operator/TestScanFilterAndProjectOperator.java b/presto-main/src/test/java/com/facebook/presto/operator/TestScanFilterAndProjectOperator.java index d9ccc89e6fae..24fcfedb5d96 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/TestScanFilterAndProjectOperator.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/TestScanFilterAndProjectOperator.java @@ -15,6 +15,7 @@ import com.facebook.presto.SequencePageBuilder; import com.facebook.presto.Session; +import com.facebook.presto.connector.ConnectorId; import com.facebook.presto.metadata.Split; import com.facebook.presto.operator.index.PageRecordSet; import com.facebook.presto.spi.ColumnHandle; @@ -77,7 +78,7 @@ public ConnectorPageSource createPageSource(Session session, Split split, Listof(VARCHAR)); SourceOperator operator = factory.createOperator(driverContext); - operator.addSplit(new Split("test", TestingTransactionHandle.create("test"), TestingSplit.createLocalSplit())); + operator.addSplit(new Split(new ConnectorId("test"), TestingTransactionHandle.create(), TestingSplit.createLocalSplit())); operator.noMoreSplits(); MaterializedResult expected = toMaterializedResult(driverContext.getSession(), ImmutableList.of(VARCHAR), ImmutableList.of(input)); @@ -111,7 +112,7 @@ public ConnectorPageSource createPageSource(Session session, Split split, Listof(VARCHAR)); SourceOperator operator = factory.createOperator(driverContext); - operator.addSplit(new Split("test", TestingTransactionHandle.create("test"), TestingSplit.createLocalSplit())); + operator.addSplit(new Split(new ConnectorId("test"), TestingTransactionHandle.create(), TestingSplit.createLocalSplit())); operator.noMoreSplits(); MaterializedResult expected = toMaterializedResult(driverContext.getSession(), ImmutableList.of(VARCHAR), ImmutableList.of(input)); diff --git a/presto-main/src/test/java/com/facebook/presto/operator/TestTableWriterOperator.java b/presto-main/src/test/java/com/facebook/presto/operator/TestTableWriterOperator.java index 8d07d1b3e5ce..db763bb12cad 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/TestTableWriterOperator.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/TestTableWriterOperator.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.operator; +import com.facebook.presto.connector.ConnectorId; import com.facebook.presto.metadata.OutputTableHandle; import com.facebook.presto.spi.ConnectorInsertTableHandle; import com.facebook.presto.spi.ConnectorOutputTableHandle; @@ -48,7 +49,7 @@ public class TestTableWriterOperator { - private static final String CONNECTOR_ID = "testConnectorId"; + private static final ConnectorId CONNECTOR_ID = new ConnectorId("testConnectorId"); private ExecutorService executor; @BeforeClass diff --git a/presto-main/src/test/java/com/facebook/presto/operator/scalar/FunctionAssertions.java b/presto-main/src/test/java/com/facebook/presto/operator/scalar/FunctionAssertions.java index 91f1db0419c2..46a8e74febd9 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/scalar/FunctionAssertions.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/scalar/FunctionAssertions.java @@ -14,6 +14,7 @@ package com.facebook.presto.operator.scalar; import com.facebook.presto.Session; +import com.facebook.presto.connector.ConnectorId; import com.facebook.presto.metadata.FunctionListBuilder; import com.facebook.presto.metadata.Metadata; import com.facebook.presto.metadata.Split; @@ -730,12 +731,12 @@ static class TestSplit { static Split createRecordSetSplit() { - return new Split("test", TestingTransactionHandle.create("test"), new TestSplit(true)); + return new Split(new ConnectorId("test"), TestingTransactionHandle.create(), new TestSplit(true)); } static Split createNormalSplit() { - return new Split("test", TestingTransactionHandle.create("test"), new TestSplit(false)); + return new Split(new ConnectorId("test"), TestingTransactionHandle.create(), new TestSplit(false)); } private final boolean recordSet; diff --git a/presto-main/src/test/java/com/facebook/presto/security/TestAccessControlManager.java b/presto-main/src/test/java/com/facebook/presto/security/TestAccessControlManager.java index 3f540272ff0a..a6312ce87004 100644 --- a/presto-main/src/test/java/com/facebook/presto/security/TestAccessControlManager.java +++ b/presto-main/src/test/java/com/facebook/presto/security/TestAccessControlManager.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.security; +import com.facebook.presto.connector.ConnectorId; import com.facebook.presto.metadata.QualifiedObjectName; import com.facebook.presto.spi.CatalogSchemaTableName; import com.facebook.presto.spi.PrestoException; @@ -108,7 +109,7 @@ public void testDenyCatalogAccessControl() accessControlManager.setSystemAccessControl("test", ImmutableMap.of()); registerBogusConnector(transactionManager, "connector"); - accessControlManager.addCatalogAccessControl("connector", "catalog", new DenyConnectorAccessControl()); + accessControlManager.addCatalogAccessControl(new ConnectorId("connector"), "catalog", new DenyConnectorAccessControl()); transaction(transactionManager) .execute(transactionId -> { @@ -128,7 +129,7 @@ public void testDenySystemAccessControl() accessControlManager.setSystemAccessControl("test", ImmutableMap.of()); registerBogusConnector(transactionManager, "connector"); - accessControlManager.addCatalogAccessControl("connector", "secured_catalog", new DenyConnectorAccessControl()); + accessControlManager.addCatalogAccessControl(new ConnectorId("connector"), "secured_catalog", new DenyConnectorAccessControl()); transaction(transactionManager) .execute(transactionId -> { @@ -138,7 +139,7 @@ public void testDenySystemAccessControl() private static void registerBogusConnector(TransactionManager transactionManager, String connectorId) { - transactionManager.addConnector(connectorId, new Connector() + transactionManager.addConnector(new ConnectorId(connectorId), new Connector() { @Override public ConnectorTransactionHandle beginTransaction(IsolationLevel isolationLevel, boolean readOnly) diff --git a/presto-main/src/test/java/com/facebook/presto/sql/analyzer/TestAnalyzer.java b/presto-main/src/test/java/com/facebook/presto/sql/analyzer/TestAnalyzer.java index d4241892b874..179f6425fcff 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/analyzer/TestAnalyzer.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/analyzer/TestAnalyzer.java @@ -15,11 +15,11 @@ import com.facebook.presto.Session; import com.facebook.presto.block.BlockEncodingManager; +import com.facebook.presto.connector.ConnectorId; import com.facebook.presto.metadata.Metadata; import com.facebook.presto.metadata.MetadataManager; import com.facebook.presto.metadata.QualifiedObjectName; import com.facebook.presto.metadata.SessionPropertyManager; -import com.facebook.presto.metadata.TableMetadata; import com.facebook.presto.metadata.TablePropertyManager; import com.facebook.presto.metadata.TestingMetadata; import com.facebook.presto.metadata.ViewDefinition; @@ -91,12 +91,18 @@ @Test(singleThreaded = true) public class TestAnalyzer { + private static final String TPCH_CATALOG = "tpch"; + private static final ConnectorId TPCH_CONNECTOR_ID = new ConnectorId(TPCH_CATALOG); + private static final String SECOND_CATALOG = "c2"; + private static final ConnectorId SECOND_CONNECTOR_ID = new ConnectorId(SECOND_CATALOG); + private static final String THIRD_CATALOG = "c3"; + private static final ConnectorId THIRD_CONNECTOR_ID = new ConnectorId(THIRD_CATALOG); private static final Session SETUP_SESSION = testSessionBuilder() .setCatalog("c1") .setSchema("s1") .build(); private static final Session CLIENT_SESSION = testSessionBuilder() - .setCatalog("tpch") + .setCatalog(TPCH_CATALOG) .setSchema("s1") .build(); @@ -969,7 +975,7 @@ public void testInvalidShowTables() assertMissingInformationSchema(session, "SHOW TABLES FROM c2.s2"); session = testSessionBuilder() - .setCatalog("c2") + .setCatalog(SECOND_CATALOG) .setSchema(null) .build(); assertFails(session, SCHEMA_NOT_SPECIFIED, "SHOW TABLES"); @@ -1002,9 +1008,9 @@ public void setup() TypeManager typeManager = new TypeRegistry(); transactionManager = createTestTransactionManager(); - transactionManager.addConnector("tpch", createTestingConnector("tpch")); - transactionManager.addConnector("c2", createTestingConnector("c2")); - transactionManager.addConnector("c3", createTestingConnector("c3")); + transactionManager.addConnector(TPCH_CONNECTOR_ID, createTestingConnector(TPCH_CONNECTOR_ID)); + transactionManager.addConnector(SECOND_CONNECTOR_ID, createTestingConnector(SECOND_CONNECTOR_ID)); + transactionManager.addConnector(THIRD_CONNECTOR_ID, createTestingConnector(THIRD_CONNECTOR_ID)); MetadataManager metadata = new MetadataManager( new FeaturesConfig().setExperimentalSyntaxEnabled(true), @@ -1013,91 +1019,91 @@ public void setup() new SessionPropertyManager(), new TablePropertyManager(), transactionManager); - metadata.registerConnectorCatalog("tpch", "tpch"); - metadata.registerConnectorCatalog("c2", "c2"); - metadata.registerConnectorCatalog("c3", "c3"); + metadata.registerConnectorCatalog(TPCH_CONNECTOR_ID, TPCH_CATALOG); + metadata.registerConnectorCatalog(SECOND_CONNECTOR_ID, SECOND_CATALOG); + metadata.registerConnectorCatalog(THIRD_CONNECTOR_ID, THIRD_CATALOG); SchemaTableName table1 = new SchemaTableName("s1", "t1"); - inSetupTransaction(session -> metadata.createTable(session, "tpch", new TableMetadata("tpch", new ConnectorTableMetadata(table1, + inSetupTransaction(session -> metadata.createTable(session, TPCH_CATALOG, new ConnectorTableMetadata(table1, ImmutableList.of( new ColumnMetadata("a", BIGINT), new ColumnMetadata("b", BIGINT), new ColumnMetadata("c", BIGINT), - new ColumnMetadata("d", BIGINT)))))); + new ColumnMetadata("d", BIGINT))))); SchemaTableName table2 = new SchemaTableName("s1", "t2"); - inSetupTransaction(session -> metadata.createTable(session, "tpch", new TableMetadata("tpch", new ConnectorTableMetadata(table2, + inSetupTransaction(session -> metadata.createTable(session, TPCH_CATALOG, new ConnectorTableMetadata(table2, ImmutableList.of( new ColumnMetadata("a", BIGINT), - new ColumnMetadata("b", BIGINT)))))); + new ColumnMetadata("b", BIGINT))))); SchemaTableName table3 = new SchemaTableName("s1", "t3"); - inSetupTransaction(session -> metadata.createTable(session, "tpch", new TableMetadata("tpch", new ConnectorTableMetadata(table3, + inSetupTransaction(session -> metadata.createTable(session, TPCH_CATALOG, new ConnectorTableMetadata(table3, ImmutableList.of( new ColumnMetadata("a", BIGINT), new ColumnMetadata("b", BIGINT), - new ColumnMetadata("x", BIGINT, null, true)))))); + new ColumnMetadata("x", BIGINT, null, true))))); // table in different catalog SchemaTableName table4 = new SchemaTableName("s2", "t4"); - inSetupTransaction(session -> metadata.createTable(session, "c2", new TableMetadata("tpch", new ConnectorTableMetadata(table4, + inSetupTransaction(session -> metadata.createTable(session, SECOND_CATALOG, new ConnectorTableMetadata(table4, ImmutableList.of( - new ColumnMetadata("a", BIGINT)))))); + new ColumnMetadata("a", BIGINT))))); // table with a hidden column SchemaTableName table5 = new SchemaTableName("s1", "t5"); - inSetupTransaction(session -> metadata.createTable(session, "tpch", new TableMetadata("tpch", new ConnectorTableMetadata(table5, + inSetupTransaction(session -> metadata.createTable(session, TPCH_CATALOG, new ConnectorTableMetadata(table5, ImmutableList.of( new ColumnMetadata("a", BIGINT), - new ColumnMetadata("b", BIGINT, null, true)))))); + new ColumnMetadata("b", BIGINT, null, true))))); // table with a varchar column SchemaTableName table6 = new SchemaTableName("s1", "t6"); - inSetupTransaction(session -> metadata.createTable(session, "tpch", new TableMetadata("tpch", new ConnectorTableMetadata(table6, + inSetupTransaction(session -> metadata.createTable(session, TPCH_CATALOG, new ConnectorTableMetadata(table6, ImmutableList.of( new ColumnMetadata("a", BIGINT), new ColumnMetadata("b", VARCHAR), new ColumnMetadata("c", BIGINT), - new ColumnMetadata("d", BIGINT)))))); + new ColumnMetadata("d", BIGINT))))); // table with bigint, double, array of bigints and array of doubles column SchemaTableName table7 = new SchemaTableName("s1", "t7"); - inSetupTransaction(session -> metadata.createTable(session, "tpch", new TableMetadata("tpch", new ConnectorTableMetadata(table7, + inSetupTransaction(session -> metadata.createTable(session, TPCH_CATALOG, new ConnectorTableMetadata(table7, ImmutableList.of( new ColumnMetadata("a", BIGINT), new ColumnMetadata("b", DOUBLE), new ColumnMetadata("c", new ArrayType(BIGINT)), - new ColumnMetadata("d", new ArrayType(DOUBLE))))))); + new ColumnMetadata("d", new ArrayType(DOUBLE)))))); // valid view referencing table in same schema String viewData1 = JsonCodec.jsonCodec(ViewDefinition.class).toJson( new ViewDefinition( "select a from t1", - Optional.of("tpch"), + Optional.of(TPCH_CATALOG), Optional.of("s1"), ImmutableList.of(new ViewColumn("a", BIGINT)), Optional.of("user"))); - inSetupTransaction(session -> metadata.createView(session, new QualifiedObjectName("tpch", "s1", "v1"), viewData1, false)); + inSetupTransaction(session -> metadata.createView(session, new QualifiedObjectName(TPCH_CATALOG, "s1", "v1"), viewData1, false)); // stale view (different column type) String viewData2 = JsonCodec.jsonCodec(ViewDefinition.class).toJson( new ViewDefinition( "select a from t1", - Optional.of("tpch"), + Optional.of(TPCH_CATALOG), Optional.of("s1"), ImmutableList.of(new ViewColumn("a", VARCHAR)), Optional.of("user"))); - inSetupTransaction(session -> metadata.createView(session, new QualifiedObjectName("tpch", "s1", "v2"), viewData2, false)); + inSetupTransaction(session -> metadata.createView(session, new QualifiedObjectName(TPCH_CATALOG, "s1", "v2"), viewData2, false)); // view referencing table in different schema from itself and session String viewData3 = JsonCodec.jsonCodec(ViewDefinition.class).toJson( new ViewDefinition( "select a from t4", - Optional.of("c2"), + Optional.of(SECOND_CATALOG), Optional.of("s2"), ImmutableList.of(new ViewColumn("a", BIGINT)), Optional.of("owner"))); - inSetupTransaction(session -> metadata.createView(session, new QualifiedObjectName("c3", "s3", "v3"), viewData3, false)); + inSetupTransaction(session -> metadata.createView(session, new QualifiedObjectName(THIRD_CATALOG, "s3", "v3"), viewData3, false)); // valid view with uppercase column name String viewData4 = JsonCodec.jsonCodec(ViewDefinition.class).toJson( @@ -1194,9 +1200,10 @@ private void assertFailsWithoutExperimentalSyntax(SemanticErrorCode error, @Lang } } - private static Connector createTestingConnector(String connectorId) + @SuppressWarnings("deprecation") + private static Connector createTestingConnector(ConnectorId connectorId) { - return new LegacyTransactionConnector(connectorId, new com.facebook.presto.spi.Connector() + return new LegacyTransactionConnector(connectorId.getCatalogName(), new com.facebook.presto.spi.Connector() { private final ConnectorMetadata metadata = new TestingMetadata(); diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestEffectivePredicateExtractor.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestEffectivePredicateExtractor.java index 5b2b25836c5b..a6713c57ace1 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestEffectivePredicateExtractor.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestEffectivePredicateExtractor.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.sql.planner; +import com.facebook.presto.connector.ConnectorId; import com.facebook.presto.metadata.FunctionKind; import com.facebook.presto.metadata.Signature; import com.facebook.presto.metadata.TableHandle; @@ -78,7 +79,7 @@ @Test(singleThreaded = true) public class TestEffectivePredicateExtractor { - private static final TableHandle DUAL_TABLE_HANDLE = new TableHandle("test", new TestingTableHandle()); + private static final TableHandle DUAL_TABLE_HANDLE = new TableHandle(new ConnectorId("test"), new TestingTableHandle()); private static final Symbol A = new Symbol("a"); private static final Symbol B = new Symbol("b"); diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestTypeValidator.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestTypeValidator.java index 972ac488f5de..f4e7b1317c15 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestTypeValidator.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestTypeValidator.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.sql.planner; +import com.facebook.presto.connector.ConnectorId; import com.facebook.presto.metadata.FunctionKind; import com.facebook.presto.metadata.Signature; import com.facebook.presto.metadata.TableHandle; @@ -59,7 +60,7 @@ @Test(singleThreaded = true) public class TestTypeValidator { - private static final TableHandle TEST_TABLE_HANDLE = new TableHandle("test", new TestingTableHandle()); + private static final TableHandle TEST_TABLE_HANDLE = new TableHandle(new ConnectorId("test"), new TestingTableHandle()); private static final SqlParser SQL_PARSER = new SqlParser(); private static final TypeValidator TYPE_VALIDATOR = new TypeValidator(); diff --git a/presto-main/src/test/java/com/facebook/presto/transaction/TestTransactionManager.java b/presto-main/src/test/java/com/facebook/presto/transaction/TestTransactionManager.java index 1a92149965cf..7e016361951e 100644 --- a/presto-main/src/test/java/com/facebook/presto/transaction/TestTransactionManager.java +++ b/presto-main/src/test/java/com/facebook/presto/transaction/TestTransactionManager.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.transaction; +import com.facebook.presto.connector.ConnectorId; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.connector.Connector; import com.facebook.presto.spi.connector.ConnectorMetadata; @@ -42,6 +43,8 @@ public class TestTransactionManager { + private static final String CATALOG_NAME = "test_catalog"; + private static final ConnectorId CONNECTOR_ID = new ConnectorId(CATALOG_NAME); private final ExecutorService finishingExecutor = newCachedThreadPool(daemonThreadsNamed("transaction-%s")); @AfterClass @@ -58,8 +61,8 @@ public void testTransactionWorkflow() try (IdleCheckExecutor executor = new IdleCheckExecutor()) { TransactionManager transactionManager = TransactionManager.create(new TransactionManagerConfig(), executor.getExecutor(), finishingExecutor); - Connector c1 = new TpchConnectorFactory().create("c1", ImmutableMap.of(), new TestingConnectorContext()); - transactionManager.addConnector("c1", c1); + Connector c1 = new TpchConnectorFactory().create(CATALOG_NAME, ImmutableMap.of(), new TestingConnectorContext()); + transactionManager.addConnector(CONNECTOR_ID, c1); TransactionId transactionId = transactionManager.beginTransaction(false); @@ -69,10 +72,10 @@ public void testTransactionWorkflow() assertTrue(transactionInfo.getConnectorIds().isEmpty()); assertFalse(transactionInfo.getWrittenConnectorId().isPresent()); - ConnectorMetadata metadata = transactionManager.getMetadata(transactionId, "c1"); - metadata.listSchemaNames(TEST_SESSION.toConnectorSession("c1")); + ConnectorMetadata metadata = transactionManager.getMetadata(transactionId, CONNECTOR_ID); + metadata.listSchemaNames(TEST_SESSION.toConnectorSession(CONNECTOR_ID)); transactionInfo = transactionManager.getTransactionInfo(transactionId); - assertEquals(transactionInfo.getConnectorIds(), ImmutableList.of("c1")); + assertEquals(transactionInfo.getConnectorIds(), ImmutableList.of(CONNECTOR_ID)); assertFalse(transactionInfo.getWrittenConnectorId().isPresent()); transactionManager.asyncCommit(transactionId).join(); @@ -88,8 +91,8 @@ public void testAbortedTransactionWorkflow() try (IdleCheckExecutor executor = new IdleCheckExecutor()) { TransactionManager transactionManager = TransactionManager.create(new TransactionManagerConfig(), executor.getExecutor(), finishingExecutor); - Connector c1 = new TpchConnectorFactory().create("c1", ImmutableMap.of(), new TestingConnectorContext()); - transactionManager.addConnector("c1", c1); + Connector c1 = new TpchConnectorFactory().create(CATALOG_NAME, ImmutableMap.of(), new TestingConnectorContext()); + transactionManager.addConnector(CONNECTOR_ID, c1); TransactionId transactionId = transactionManager.beginTransaction(false); @@ -99,10 +102,10 @@ public void testAbortedTransactionWorkflow() assertTrue(transactionInfo.getConnectorIds().isEmpty()); assertFalse(transactionInfo.getWrittenConnectorId().isPresent()); - ConnectorMetadata metadata = transactionManager.getMetadata(transactionId, "c1"); - metadata.listSchemaNames(TEST_SESSION.toConnectorSession("c1")); + ConnectorMetadata metadata = transactionManager.getMetadata(transactionId, CONNECTOR_ID); + metadata.listSchemaNames(TEST_SESSION.toConnectorSession(CONNECTOR_ID)); transactionInfo = transactionManager.getTransactionInfo(transactionId); - assertEquals(transactionInfo.getConnectorIds(), ImmutableList.of("c1")); + assertEquals(transactionInfo.getConnectorIds(), ImmutableList.of(CONNECTOR_ID)); assertFalse(transactionInfo.getWrittenConnectorId().isPresent()); transactionManager.asyncAbort(transactionId).join(); @@ -118,8 +121,8 @@ public void testFailedTransactionWorkflow() try (IdleCheckExecutor executor = new IdleCheckExecutor()) { TransactionManager transactionManager = TransactionManager.create(new TransactionManagerConfig(), executor.getExecutor(), finishingExecutor); - Connector c1 = new TpchConnectorFactory().create("c1", ImmutableMap.of(), new TestingConnectorContext()); - transactionManager.addConnector("c1", c1); + Connector c1 = new TpchConnectorFactory().create(CATALOG_NAME, ImmutableMap.of(), new TestingConnectorContext()); + transactionManager.addConnector(CONNECTOR_ID, c1); TransactionId transactionId = transactionManager.beginTransaction(false); @@ -129,17 +132,17 @@ public void testFailedTransactionWorkflow() assertTrue(transactionInfo.getConnectorIds().isEmpty()); assertFalse(transactionInfo.getWrittenConnectorId().isPresent()); - ConnectorMetadata metadata = transactionManager.getMetadata(transactionId, "c1"); - metadata.listSchemaNames(TEST_SESSION.toConnectorSession("c1")); + ConnectorMetadata metadata = transactionManager.getMetadata(transactionId, CONNECTOR_ID); + metadata.listSchemaNames(TEST_SESSION.toConnectorSession(CONNECTOR_ID)); transactionInfo = transactionManager.getTransactionInfo(transactionId); - assertEquals(transactionInfo.getConnectorIds(), ImmutableList.of("c1")); + assertEquals(transactionInfo.getConnectorIds(), ImmutableList.of(CONNECTOR_ID)); assertFalse(transactionInfo.getWrittenConnectorId().isPresent()); transactionManager.fail(transactionId); assertEquals(transactionManager.getAllTransactionInfos().size(), 1); try { - transactionManager.getMetadata(transactionId, "c1"); + transactionManager.getMetadata(transactionId, CONNECTOR_ID); fail(); } catch (PrestoException e) { diff --git a/presto-tests/src/main/java/com/facebook/presto/tests/DistributedQueryRunner.java b/presto-tests/src/main/java/com/facebook/presto/tests/DistributedQueryRunner.java index 9475c686d84f..d3e21538631c 100644 --- a/presto-tests/src/main/java/com/facebook/presto/tests/DistributedQueryRunner.java +++ b/presto-tests/src/main/java/com/facebook/presto/tests/DistributedQueryRunner.java @@ -14,6 +14,7 @@ package com.facebook.presto.tests; import com.facebook.presto.Session; +import com.facebook.presto.connector.ConnectorId; import com.facebook.presto.execution.QueryInfo; import com.facebook.presto.execution.QueryManager; import com.facebook.presto.metadata.AllNodes; @@ -43,6 +44,7 @@ import java.io.IOException; import java.net.URI; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -50,7 +52,9 @@ import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; +import static com.google.common.collect.Iterables.getOnlyElement; import static io.airlift.units.Duration.nanosSince; +import static io.airlift.units.Duration.succinctNanos; import static java.util.Objects.requireNonNull; import static java.util.concurrent.TimeUnit.MILLISECONDS; import static java.util.concurrent.TimeUnit.SECONDS; @@ -248,15 +252,17 @@ public void createCatalog(String catalogName, String connectorName) public void createCatalog(String catalogName, String connectorName, Map properties) { long start = System.nanoTime(); + Set connectorIds = new HashSet<>(); for (TestingPrestoServer server : servers) { - server.createCatalog(catalogName, connectorName, properties); + connectorIds.add(server.createCatalog(catalogName, connectorName, properties)); } - log.info("Created catalog %s in %s", catalogName, nanosSince(start).convertToMostSuccinctTimeUnit()); + ConnectorId connectorId = getOnlyElement(connectorIds); + log.info("Created catalog %s (%s) in %s", catalogName, connectorId, nanosSince(start).convertToMostSuccinctTimeUnit()); // wait for all nodes to announce the new catalog start = System.nanoTime(); - while (!isConnectionVisibleToAllNodes(catalogName)) { - Assertions.assertLessThan(nanosSince(start), new Duration(100, SECONDS), "waiting form connector " + connectorName + " to be initialized in every node"); + while (!isConnectionVisibleToAllNodes(connectorId)) { + Assertions.assertLessThan(nanosSince(start), new Duration(100, SECONDS), "waiting for connector " + connectorId + " to be initialized in every node"); try { MILLISECONDS.sleep(10); } @@ -265,10 +271,10 @@ public void createCatalog(String catalogName, String connectorName, Map activeNodesWithConnector; @@ -169,7 +170,7 @@ private void refreshNodes(String catalogName) Thread.currentThread().interrupt(); break; } - activeNodesWithConnector = server.getActiveNodesWithConnector(catalogName); + activeNodesWithConnector = server.getActiveNodesWithConnector(connectorId); } while (activeNodesWithConnector.isEmpty()); } @@ -186,9 +187,9 @@ public void createCatalog(String catalogName, String connectorName) public void createCatalog(String catalogName, String connectorName, Map properties) { - server.createCatalog(catalogName, connectorName, properties); + ConnectorId connectorId = server.createCatalog(catalogName, connectorName, properties); - refreshNodes(catalogName); + refreshNodes(connectorId); } @Override