Skip to content
Permalink
Browse files
DRILL-8223: Refactor auth modes dropping DRILL_PROCESS and allowing c…
…redential providers everywhere (#2547)

* Remove CredentialedStoragePluginConfig and drill_process auth mode.

* Add a unit test of shared_user with no creds based on H2.

* Fix compile errors.
  • Loading branch information
jnturton committed May 17, 2022
1 parent 58391fe commit c688b1f241b5ac5bdd4f6a56ca3e1342bb2f76dd
Showing 17 changed files with 228 additions and 299 deletions.
@@ -22,7 +22,7 @@
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonTypeName;

import org.apache.drill.common.logical.CredentialedStoragePluginConfig;
import org.apache.drill.common.logical.StoragePluginConfig;
import org.apache.drill.exec.store.security.CredentialProviderUtils;
import org.apache.drill.common.logical.security.CredentialsProvider;
import org.apache.drill.exec.store.security.UsernamePasswordCredentials;
@@ -33,7 +33,7 @@
import java.util.Optional;

@JsonTypeName(CassandraStorageConfig.NAME)
public class CassandraStorageConfig extends CredentialedStoragePluginConfig {
public class CassandraStorageConfig extends StoragePluginConfig {
public static final String NAME = "cassandra";

private final String host;
@@ -110,9 +110,4 @@ public boolean equals(Object o) {
public int hashCode() {
return Objects.hash(host, credentialsProvider);
}

@Override
public CassandraStorageConfig updateCredentialProvider(CredentialsProvider credentialsProvider) {
return this;
}
}
@@ -24,7 +24,7 @@
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectWriter;
import org.apache.drill.common.logical.CredentialedStoragePluginConfig;
import org.apache.drill.common.logical.StoragePluginConfig;
import org.apache.drill.exec.store.security.CredentialProviderUtils;
import org.apache.drill.common.logical.security.CredentialsProvider;
import org.apache.drill.exec.store.security.UsernamePasswordCredentials;
@@ -36,7 +36,7 @@
import java.util.Optional;

@JsonTypeName(ElasticsearchStorageConfig.NAME)
public class ElasticsearchStorageConfig extends CredentialedStoragePluginConfig {
public class ElasticsearchStorageConfig extends StoragePluginConfig {
public static final String NAME = "elastic";

private static final ObjectWriter OBJECT_WRITER = new ObjectMapper().writerFor(List.class);
@@ -109,9 +109,4 @@ public boolean equals(Object o) {
public int hashCode() {
return Objects.hash(hosts, credentialsProvider);
}

@Override
public CredentialedStoragePluginConfig updateCredentialProvider(CredentialsProvider credentialsProvider) {
return this;
}
}
@@ -21,7 +21,7 @@
import org.apache.drill.common.exceptions.UserException;
import org.apache.drill.common.logical.OAuthConfig;
import org.apache.drill.common.map.CaseInsensitiveMap;
import org.apache.drill.common.logical.CredentialedStoragePluginConfig;
import org.apache.drill.common.logical.StoragePluginConfig;

import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonIgnore;
@@ -42,7 +42,7 @@


@JsonTypeName(HttpStoragePluginConfig.NAME)
public class HttpStoragePluginConfig extends CredentialedStoragePluginConfig {
public class HttpStoragePluginConfig extends StoragePluginConfig {
private static final Logger logger = LoggerFactory.getLogger(HttpStoragePluginConfig.class);
public static final String NAME = "http";

@@ -30,7 +30,6 @@
import org.apache.commons.net.util.Base64;
import org.apache.drill.common.config.DrillProperties;
import org.apache.drill.common.exceptions.UserException;
import org.apache.drill.common.logical.CredentialedStoragePluginConfig;
import org.apache.drill.common.logical.OAuthConfig;
import org.apache.drill.common.logical.StoragePluginConfig.AuthMode;
import org.apache.drill.common.logical.security.CredentialsProvider;
@@ -172,7 +171,7 @@ public void testEmptyUserCredentials() throws Exception {
// First verify that the user has no credentials
StoragePluginRegistry registry = cluster.storageRegistry();
StoragePlugin plugin = registry.getPlugin("local");
PlainCredentialsProvider credentialsProvider = (PlainCredentialsProvider) ((CredentialedStoragePluginConfig) plugin.getConfig()).getCredentialsProvider();
PlainCredentialsProvider credentialsProvider = (PlainCredentialsProvider) plugin.getConfig().getCredentialsProvider();
Map<String, String> credentials = credentialsProvider.getCredentials(TEST_USER_1);
assertNotNull(credentials);
assertNull(credentials.get("username"));
@@ -30,9 +30,9 @@
import com.fasterxml.jackson.annotation.JsonTypeName;
import org.apache.drill.common.PlanStringBuilder;
import org.apache.drill.common.exceptions.UserException;
import org.apache.drill.common.logical.CredentialedStoragePluginConfig;
import org.apache.drill.exec.proto.UserBitShared.UserCredentials;
import org.apache.drill.exec.store.security.CredentialProviderUtils;
import org.apache.drill.common.logical.StoragePluginConfig;
import org.apache.drill.common.logical.security.CredentialsProvider;
import org.apache.drill.exec.store.security.UsernamePasswordCredentials;
import org.apache.drill.shaded.guava.com.google.common.base.Preconditions;
@@ -41,7 +41,7 @@

@JsonTypeName(JdbcStorageConfig.NAME)
@JsonFilter("passwordFilter")
public class JdbcStorageConfig extends CredentialedStoragePluginConfig {
public class JdbcStorageConfig extends StoragePluginConfig {

private static final Logger logger = LoggerFactory.getLogger(JdbcStorageConfig.class);

@@ -33,12 +33,14 @@
import org.apache.calcite.sql.SqlDialectFactoryImpl;
import org.apache.drill.common.AutoCloseables;
import org.apache.drill.common.exceptions.UserException;
import org.apache.drill.common.logical.StoragePluginConfig.AuthMode;
import org.apache.drill.exec.ops.OptimizerRulesContext;
import org.apache.drill.exec.proto.UserBitShared.UserCredentials;
import org.apache.drill.exec.server.DrillbitContext;
import org.apache.drill.exec.store.AbstractStoragePlugin;
import org.apache.drill.exec.store.SchemaConfig;
import org.apache.drill.exec.store.security.UsernamePasswordCredentials;
import org.apache.drill.exec.util.ImpersonationUtil;
import org.apache.drill.shaded.guava.com.google.common.annotations.VisibleForTesting;
import org.apache.drill.shaded.guava.com.google.common.collect.ImmutableSet;
import org.slf4j.Logger;
@@ -83,19 +85,25 @@ public void registerSchemas(SchemaConfig config, SchemaPlus parent) {
public Optional<DataSource> getDataSource(UserCredentials userCredentials) {
Optional<UsernamePasswordCredentials> jdbcCreds = jdbcStorageConfig.getUsernamePasswordCredentials(userCredentials);

if (!jdbcCreds.isPresent()) {
logger.debug(
"There are no {} mode credentials in {} for query user {}",
jdbcStorageConfig.getAuthMode(),
if (!jdbcCreds.isPresent() && jdbcStorageConfig.getAuthMode() == AuthMode.USER_TRANSLATION) {
logger.info(
"There are no {} mode credentials in {} for query user {}, will not attempt to connect.",
AuthMode.USER_TRANSLATION,
getName(),
userCredentials.getUserName()
);
return Optional.<DataSource>empty();
}

// Missing creds is valid under SHARED_USER (e.g. unsecured DBs, BigQuery's OAuth)
// and we fall back to using a key of Drillbit process username in this instance.
String dsKey = jdbcCreds.isPresent()
? jdbcCreds.get().getUsername()
: ImpersonationUtil.getProcessUserName();

return Optional.of(dataSources.computeIfAbsent(
jdbcCreds.get().getUsername(),
ds -> initDataSource(this.jdbcStorageConfig, jdbcCreds.get())
dsKey,
ds -> initDataSource(this.jdbcStorageConfig, jdbcCreds.orElse(null))
));
}

@@ -55,6 +55,7 @@ public class TestJdbcPluginWithH2IT extends ClusterTest {
private static final String TABLE_PATH = "jdbcmulti/";
private static final String TABLE_NAME = String.format("%s.`%s`", StoragePluginTestUtils.DFS_PLUGIN_NAME, TABLE_PATH);
private static TimeZone defaultTimeZone;
private static URL SCRIPT_FILE = TestJdbcPluginWithH2IT.class.getClassLoader().getResource("h2-test-data.sql");

@BeforeClass
public static void init() throws Exception {
@@ -66,10 +67,10 @@ public static void init() throws Exception {
dirTestWatcher.copyResourceToRoot(Paths.get(TABLE_PATH));
Class.forName("org.h2.Driver");
String connString = "jdbc:h2:" + dirTestWatcher.getTmpDir().getCanonicalPath();
URL scriptFile = TestJdbcPluginWithH2IT.class.getClassLoader().getResource("h2-test-data.sql");
assertNotNull("Script for test tables generation 'h2-test-data.sql' cannot be found in test resources", scriptFile);

assertNotNull("Script for test tables generation 'h2-test-data.sql' cannot be found in test resources", SCRIPT_FILE);
try (Connection connection = DriverManager.getConnection(connString, "root", "root");
FileReader fileReader = new FileReader(scriptFile.getFile())) {
FileReader fileReader = new FileReader(SCRIPT_FILE.getFile())) {
RunScript.execute(connection, fileReader);
}

@@ -313,4 +314,39 @@ public void testJdbcIntermConvRuleConvention() throws Exception {
.include("mocked_enum")
.match();
}

@Test
public void testSharedUserNoCreds() throws Exception {
String connString = "jdbc:h2:" + dirTestWatcher.getTmpDir().getCanonicalPath() + "/noauth";
JdbcStorageConfig cfg = new JdbcStorageConfig(
"org.h2.Driver",
connString,
null,
null,
true,
false,
null,
null,
AuthMode.SHARED_USER.name(),
10000
);
cfg.setEnabled(true);
cluster.defineStoragePlugin("h2_noauth", cfg);

try (
Connection connection = DriverManager.getConnection(connString, null, null);
FileReader fileReader = new FileReader(SCRIPT_FILE.getFile())
) {
RunScript.execute(connection, fileReader);
}

run("USE h2_noauth");
String sql = "SHOW TABLES";
testBuilder()
.sqlQuery(sql)
.unOrdered()
.baselineColumns("TABLE_SCHEMA", "TABLE_NAME")
.baselineValues("h2_noauth.noauth.drill_h2_test_1", "PERSON")
.go();
}
}
@@ -23,15 +23,15 @@
import com.fasterxml.jackson.annotation.JsonTypeName;
import com.mongodb.ConnectionString;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.drill.common.logical.CredentialedStoragePluginConfig;
import org.apache.drill.common.logical.StoragePluginConfig;
import org.apache.drill.common.logical.security.CredentialsProvider;
import org.apache.drill.common.logical.security.PlainCredentialsProvider;

import java.util.List;
import java.util.Objects;

@JsonTypeName(MongoStoragePluginConfig.NAME)
public class MongoStoragePluginConfig extends CredentialedStoragePluginConfig {
public class MongoStoragePluginConfig extends StoragePluginConfig {

public static final String NAME = "mongo";

@@ -24,7 +24,7 @@

import org.apache.commons.lang3.StringUtils;
import org.apache.drill.common.PlanStringBuilder;
import org.apache.drill.common.logical.CredentialedStoragePluginConfig;
import org.apache.drill.common.logical.StoragePluginConfig;
import org.apache.drill.common.logical.security.CredentialsProvider;
import org.apache.drill.exec.store.security.CredentialProviderUtils;
import org.apache.drill.exec.store.security.UsernamePasswordCredentials;
@@ -35,7 +35,7 @@
import com.fasterxml.jackson.annotation.JsonTypeName;

@JsonTypeName(PhoenixStoragePluginConfig.NAME)
public class PhoenixStoragePluginConfig extends CredentialedStoragePluginConfig {
public class PhoenixStoragePluginConfig extends StoragePluginConfig {

public static final String NAME = "phoenix";
public static final String THIN_DRIVER_CLASS = "org.apache.phoenix.queryserver.client.Driver";
@@ -147,9 +147,4 @@ public String toString() {
.field("props", props)
.toString();
}

@Override
public PhoenixStoragePluginConfig updateCredentialProvider(CredentialsProvider credentialsProvider) {
return this;
}
}
@@ -23,7 +23,7 @@
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonTypeName;
import org.apache.drill.common.PlanStringBuilder;
import org.apache.drill.common.logical.CredentialedStoragePluginConfig;
import org.apache.drill.common.logical.StoragePluginConfig;
import org.apache.drill.common.logical.security.CredentialsProvider;
import org.apache.drill.common.logical.security.PlainCredentialsProvider;
import org.apache.drill.exec.store.security.CredentialProviderUtils;
@@ -33,7 +33,7 @@
import java.util.Optional;

@JsonTypeName(SplunkPluginConfig.NAME)
public class SplunkPluginConfig extends CredentialedStoragePluginConfig {
public class SplunkPluginConfig extends StoragePluginConfig {

public static final String NAME = "splunk";
public static final int DISABLED_RECONNECT_RETRIES = 1;
@@ -20,7 +20,6 @@

import io.swagger.v3.oas.annotations.ExternalDocumentation;
import io.swagger.v3.oas.annotations.Operation;
import org.apache.drill.common.logical.CredentialedStoragePluginConfig;
import org.apache.drill.common.logical.StoragePluginConfig.AuthMode;
import org.apache.drill.common.logical.StoragePluginConfig;
import org.apache.drill.common.logical.security.CredentialsProvider;
@@ -156,14 +155,7 @@ public Response createOrUpdateCredentials(@FormParam("plugin") String pluginName
}

// Get the config
StoragePluginConfig rawConfig = storage.getStoredConfig(pluginName);
if (!(rawConfig instanceof CredentialedStoragePluginConfig)) {
return Response.status(Status.INTERNAL_SERVER_ERROR)
.entity(message(pluginName + " does not support per user credentials."))
.build();
}

CredentialedStoragePluginConfig config = (CredentialedStoragePluginConfig)rawConfig;
StoragePluginConfig config = storage.getStoredConfig(pluginName);

if (config.getAuthMode() != AuthMode.USER_TRANSLATION) {
return Response.status(Status.INTERNAL_SERVER_ERROR)
@@ -177,7 +169,7 @@ public Response createOrUpdateCredentials(@FormParam("plugin") String pluginName

// Since the config classes are not accessible from java-exec, we have to serialize them,
// replace the credential provider with the updated one, and update the storage plugin registry
CredentialedStoragePluginConfig newConfig = config.updateCredentialProvider(credentialProvider);
StoragePluginConfig newConfig = config.updateCredentialProvider(credentialProvider);
newConfig.setEnabled(config.isEnabled());

try {
@@ -210,26 +202,19 @@ public Response createOrUpdatePlugin(@PathParam("pluginName") String pluginName,
cleanPluginName = pluginName.trim();
StoragePluginConfig config = storage.getStoredConfig(cleanPluginName);

if (!(config instanceof CredentialedStoragePluginConfig)) {
return Response.status(Status.INTERNAL_SERVER_ERROR)
.entity(message(cleanPluginName + " does not support user translation."))
.build();
}

if (config.getAuthMode() != AuthMode.USER_TRANSLATION) {
return Response.status(Status.INTERNAL_SERVER_ERROR)
.entity(message(cleanPluginName + " does not have user translation enabled."))
.build();
}

CredentialedStoragePluginConfig credsConfig = (CredentialedStoragePluginConfig)config;
CredentialsProvider credentialProvider = credsConfig.getCredentialsProvider();
CredentialsProvider credentialProvider = config.getCredentialsProvider();
credentialProvider.setUserCredentials(credentials.getUsername(), credentials.getPassword(), queryUser);

// Since the config classes are not accessible from java-exec, we have to serialize them,
// replace the credential provider with the updated one, and update the storage plugin registry
CredentialedStoragePluginConfig newConfig = credsConfig.updateCredentialProvider(credentialProvider);
newConfig.setEnabled(credsConfig.isEnabled());
StoragePluginConfig newConfig = config.updateCredentialProvider(credentialProvider);
newConfig.setEnabled(config.isEnabled());

try {
storage.validatedPut(cleanPluginName, newConfig);

0 comments on commit c688b1f

Please sign in to comment.