DRILL-8223: Refactor auth modes dropping DRILL_PROCESS and allowing c…

…redential providers everywhere (#2547)

* Remove CredentialedStoragePluginConfig and drill_process auth mode.

* Add a unit test of shared_user with no creds based on H2.

* Fix compile errors.

contrib/storage-cassandra/src/main/java/org/apache/drill/exec/store/cassandra/CassandraStorageConfig.java

@@ -22,7 +22,7 @@
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.annotation.JsonTypeName;
 
-import org.apache.drill.common.logical.CredentialedStoragePluginConfig;
+import org.apache.drill.common.logical.StoragePluginConfig;
 import org.apache.drill.exec.store.security.CredentialProviderUtils;
 import org.apache.drill.common.logical.security.CredentialsProvider;
 import org.apache.drill.exec.store.security.UsernamePasswordCredentials;
@@ -33,7 +33,7 @@
 import java.util.Optional;
 
 @JsonTypeName(CassandraStorageConfig.NAME)
-public class CassandraStorageConfig extends CredentialedStoragePluginConfig {
+public class CassandraStorageConfig extends StoragePluginConfig {
   public static final String NAME = "cassandra";
 
   private final String host;
@@ -110,9 +110,4 @@ public boolean equals(Object o) {
   public int hashCode() {
     return Objects.hash(host, credentialsProvider);
   }
-
-  @Override
-  public CassandraStorageConfig updateCredentialProvider(CredentialsProvider credentialsProvider) {
-    return this;
-  }
 }

contrib/storage-elasticsearch/src/main/java/org/apache/drill/exec/store/elasticsearch/ElasticsearchStorageConfig.java

@@ -24,7 +24,7 @@
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.ObjectWriter;
-import org.apache.drill.common.logical.CredentialedStoragePluginConfig;
+import org.apache.drill.common.logical.StoragePluginConfig;
 import org.apache.drill.exec.store.security.CredentialProviderUtils;
 import org.apache.drill.common.logical.security.CredentialsProvider;
 import org.apache.drill.exec.store.security.UsernamePasswordCredentials;
@@ -36,7 +36,7 @@
 import java.util.Optional;
 
 @JsonTypeName(ElasticsearchStorageConfig.NAME)
-public class ElasticsearchStorageConfig extends CredentialedStoragePluginConfig {
+public class ElasticsearchStorageConfig extends StoragePluginConfig {
   public static final String NAME = "elastic";
 
   private static final ObjectWriter OBJECT_WRITER = new ObjectMapper().writerFor(List.class);
@@ -109,9 +109,4 @@ public boolean equals(Object o) {
   public int hashCode() {
     return Objects.hash(hosts, credentialsProvider);
   }
-
-  @Override
-  public CredentialedStoragePluginConfig updateCredentialProvider(CredentialsProvider credentialsProvider) {
-    return this;
-  }
 }

contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/HttpStoragePluginConfig.java

@@ -21,7 +21,7 @@
 import org.apache.drill.common.exceptions.UserException;
 import org.apache.drill.common.logical.OAuthConfig;
 import org.apache.drill.common.map.CaseInsensitiveMap;
-import org.apache.drill.common.logical.CredentialedStoragePluginConfig;
+import org.apache.drill.common.logical.StoragePluginConfig;
 
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonIgnore;
@@ -42,7 +42,7 @@
 
 
 @JsonTypeName(HttpStoragePluginConfig.NAME)
-public class HttpStoragePluginConfig extends CredentialedStoragePluginConfig {
+public class HttpStoragePluginConfig extends StoragePluginConfig {
   private static final Logger logger = LoggerFactory.getLogger(HttpStoragePluginConfig.class);
   public static final String NAME = "http";
 

contrib/storage-http/src/test/java/org/apache/drill/exec/store/http/TestUserTranslationInHttpPlugin.java

@@ -30,7 +30,6 @@
 import org.apache.commons.net.util.Base64;
 import org.apache.drill.common.config.DrillProperties;
 import org.apache.drill.common.exceptions.UserException;
-import org.apache.drill.common.logical.CredentialedStoragePluginConfig;
 import org.apache.drill.common.logical.OAuthConfig;
 import org.apache.drill.common.logical.StoragePluginConfig.AuthMode;
 import org.apache.drill.common.logical.security.CredentialsProvider;
@@ -172,7 +171,7 @@ public void testEmptyUserCredentials() throws Exception {
     // First verify that the user has no credentials
     StoragePluginRegistry registry = cluster.storageRegistry();
     StoragePlugin plugin = registry.getPlugin("local");
-    PlainCredentialsProvider credentialsProvider = (PlainCredentialsProvider) ((CredentialedStoragePluginConfig) plugin.getConfig()).getCredentialsProvider();
+    PlainCredentialsProvider credentialsProvider = (PlainCredentialsProvider) plugin.getConfig().getCredentialsProvider();
     Map<String, String> credentials = credentialsProvider.getCredentials(TEST_USER_1);
     assertNotNull(credentials);
     assertNull(credentials.get("username"));

contrib/storage-jdbc/src/main/java/org/apache/drill/exec/store/jdbc/JdbcStorageConfig.java

@@ -30,9 +30,9 @@
 import com.fasterxml.jackson.annotation.JsonTypeName;
 import org.apache.drill.common.PlanStringBuilder;
 import org.apache.drill.common.exceptions.UserException;
-import org.apache.drill.common.logical.CredentialedStoragePluginConfig;
 import org.apache.drill.exec.proto.UserBitShared.UserCredentials;
 import org.apache.drill.exec.store.security.CredentialProviderUtils;
+import org.apache.drill.common.logical.StoragePluginConfig;
 import org.apache.drill.common.logical.security.CredentialsProvider;
 import org.apache.drill.exec.store.security.UsernamePasswordCredentials;
 import org.apache.drill.shaded.guava.com.google.common.base.Preconditions;
@@ -41,7 +41,7 @@
 
 @JsonTypeName(JdbcStorageConfig.NAME)
 @JsonFilter("passwordFilter")
-public class JdbcStorageConfig extends CredentialedStoragePluginConfig {
+public class JdbcStorageConfig extends StoragePluginConfig {
 
   private static final Logger logger = LoggerFactory.getLogger(JdbcStorageConfig.class);
 

contrib/storage-jdbc/src/main/java/org/apache/drill/exec/store/jdbc/JdbcStoragePlugin.java

@@ -33,12 +33,14 @@
 import org.apache.calcite.sql.SqlDialectFactoryImpl;
 import org.apache.drill.common.AutoCloseables;
 import org.apache.drill.common.exceptions.UserException;
+import org.apache.drill.common.logical.StoragePluginConfig.AuthMode;
 import org.apache.drill.exec.ops.OptimizerRulesContext;
 import org.apache.drill.exec.proto.UserBitShared.UserCredentials;
 import org.apache.drill.exec.server.DrillbitContext;
 import org.apache.drill.exec.store.AbstractStoragePlugin;
 import org.apache.drill.exec.store.SchemaConfig;
 import org.apache.drill.exec.store.security.UsernamePasswordCredentials;
+import org.apache.drill.exec.util.ImpersonationUtil;
 import org.apache.drill.shaded.guava.com.google.common.annotations.VisibleForTesting;
 import org.apache.drill.shaded.guava.com.google.common.collect.ImmutableSet;
 import org.slf4j.Logger;
@@ -83,19 +85,25 @@ public void registerSchemas(SchemaConfig config, SchemaPlus parent) {
   public Optional<DataSource> getDataSource(UserCredentials userCredentials) {
     Optional<UsernamePasswordCredentials> jdbcCreds = jdbcStorageConfig.getUsernamePasswordCredentials(userCredentials);
 
-    if (!jdbcCreds.isPresent()) {
-      logger.debug(
-        "There are no {} mode credentials in {} for query user {}",
-        jdbcStorageConfig.getAuthMode(),
+    if (!jdbcCreds.isPresent() && jdbcStorageConfig.getAuthMode() == AuthMode.USER_TRANSLATION) {
+      logger.info(
+        "There are no {} mode credentials in {} for query user {}, will not attempt to connect.",
+        AuthMode.USER_TRANSLATION,
         getName(),
         userCredentials.getUserName()
       );
       return Optional.<DataSource>empty();
     }
 
+    // Missing creds is valid under SHARED_USER (e.g. unsecured DBs, BigQuery's OAuth)
+    // and we fall back to using a key of Drillbit process username in this instance.
+    String dsKey = jdbcCreds.isPresent()
+      ? jdbcCreds.get().getUsername()
+      : ImpersonationUtil.getProcessUserName();
+
     return Optional.of(dataSources.computeIfAbsent(
-      jdbcCreds.get().getUsername(),
-      ds -> initDataSource(this.jdbcStorageConfig, jdbcCreds.get())
+      dsKey,
+      ds -> initDataSource(this.jdbcStorageConfig, jdbcCreds.orElse(null))
     ));
   }
 

contrib/storage-jdbc/src/test/java/org/apache/drill/exec/store/jdbc/TestJdbcPluginWithH2IT.java

@@ -55,6 +55,7 @@ public class TestJdbcPluginWithH2IT extends ClusterTest {
   private static final String TABLE_PATH = "jdbcmulti/";
   private static final String TABLE_NAME = String.format("%s.`%s`", StoragePluginTestUtils.DFS_PLUGIN_NAME, TABLE_PATH);
   private static TimeZone defaultTimeZone;
+  private static URL SCRIPT_FILE = TestJdbcPluginWithH2IT.class.getClassLoader().getResource("h2-test-data.sql");
 
   @BeforeClass
   public static void init() throws Exception {
@@ -66,10 +67,10 @@ public static void init() throws Exception {
     dirTestWatcher.copyResourceToRoot(Paths.get(TABLE_PATH));
     Class.forName("org.h2.Driver");
     String connString = "jdbc:h2:" + dirTestWatcher.getTmpDir().getCanonicalPath();
-    URL scriptFile = TestJdbcPluginWithH2IT.class.getClassLoader().getResource("h2-test-data.sql");
-    assertNotNull("Script for test tables generation 'h2-test-data.sql' cannot be found in test resources", scriptFile);
+
+    assertNotNull("Script for test tables generation 'h2-test-data.sql' cannot be found in test resources", SCRIPT_FILE);
     try (Connection connection = DriverManager.getConnection(connString, "root", "root");
-         FileReader fileReader = new FileReader(scriptFile.getFile())) {
+         FileReader fileReader = new FileReader(SCRIPT_FILE.getFile())) {
       RunScript.execute(connection, fileReader);
     }
 
@@ -313,4 +314,39 @@ public void testJdbcIntermConvRuleConvention() throws Exception {
         .include("mocked_enum")
         .match();
   }
+
+  @Test
+  public void testSharedUserNoCreds() throws Exception {
+    String connString = "jdbc:h2:" + dirTestWatcher.getTmpDir().getCanonicalPath() + "/noauth";
+    JdbcStorageConfig cfg = new JdbcStorageConfig(
+      "org.h2.Driver",
+        connString,
+        null,
+        null,
+        true,
+        false,
+        null,
+        null,
+        AuthMode.SHARED_USER.name(),
+        10000
+    );
+    cfg.setEnabled(true);
+    cluster.defineStoragePlugin("h2_noauth", cfg);
+
+    try (
+      Connection connection = DriverManager.getConnection(connString, null, null);
+      FileReader fileReader = new FileReader(SCRIPT_FILE.getFile())
+    ) {
+      RunScript.execute(connection, fileReader);
+    }
+
+    run("USE h2_noauth");
+    String sql = "SHOW TABLES";
+    testBuilder()
+      .sqlQuery(sql)
+      .unOrdered()
+      .baselineColumns("TABLE_SCHEMA", "TABLE_NAME")
+      .baselineValues("h2_noauth.noauth.drill_h2_test_1", "PERSON")
+      .go();
+  }
 }

contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/MongoStoragePluginConfig.java

@@ -23,15 +23,15 @@
 import com.fasterxml.jackson.annotation.JsonTypeName;
 import com.mongodb.ConnectionString;
 import org.apache.commons.lang3.ObjectUtils;
-import org.apache.drill.common.logical.CredentialedStoragePluginConfig;
+import org.apache.drill.common.logical.StoragePluginConfig;
 import org.apache.drill.common.logical.security.CredentialsProvider;
 import org.apache.drill.common.logical.security.PlainCredentialsProvider;
 
 import java.util.List;
 import java.util.Objects;
 
 @JsonTypeName(MongoStoragePluginConfig.NAME)
-public class MongoStoragePluginConfig extends CredentialedStoragePluginConfig {
+public class MongoStoragePluginConfig extends StoragePluginConfig {
 
   public static final String NAME = "mongo";
 

contrib/storage-phoenix/src/main/java/org/apache/drill/exec/store/phoenix/PhoenixStoragePluginConfig.java

@@ -24,7 +24,7 @@
 
 import org.apache.commons.lang3.StringUtils;
 import org.apache.drill.common.PlanStringBuilder;
-import org.apache.drill.common.logical.CredentialedStoragePluginConfig;
+import org.apache.drill.common.logical.StoragePluginConfig;
 import org.apache.drill.common.logical.security.CredentialsProvider;
 import org.apache.drill.exec.store.security.CredentialProviderUtils;
 import org.apache.drill.exec.store.security.UsernamePasswordCredentials;
@@ -35,7 +35,7 @@
 import com.fasterxml.jackson.annotation.JsonTypeName;
 
 @JsonTypeName(PhoenixStoragePluginConfig.NAME)
-public class PhoenixStoragePluginConfig extends CredentialedStoragePluginConfig {
+public class PhoenixStoragePluginConfig extends StoragePluginConfig {
 
   public static final String NAME = "phoenix";
   public static final String THIN_DRIVER_CLASS = "org.apache.phoenix.queryserver.client.Driver";
@@ -147,9 +147,4 @@ public String toString() {
         .field("props", props)
         .toString();
   }
-
-  @Override
-  public PhoenixStoragePluginConfig updateCredentialProvider(CredentialsProvider credentialsProvider) {
-    return this;
-  }
 }

contrib/storage-splunk/src/main/java/org/apache/drill/exec/store/splunk/SplunkPluginConfig.java

@@ -23,7 +23,7 @@
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.annotation.JsonTypeName;
 import org.apache.drill.common.PlanStringBuilder;
-import org.apache.drill.common.logical.CredentialedStoragePluginConfig;
+import org.apache.drill.common.logical.StoragePluginConfig;
 import org.apache.drill.common.logical.security.CredentialsProvider;
 import org.apache.drill.common.logical.security.PlainCredentialsProvider;
 import org.apache.drill.exec.store.security.CredentialProviderUtils;
@@ -33,7 +33,7 @@
 import java.util.Optional;
 
 @JsonTypeName(SplunkPluginConfig.NAME)
-public class SplunkPluginConfig extends CredentialedStoragePluginConfig {
+public class SplunkPluginConfig extends StoragePluginConfig {
 
   public static final String NAME = "splunk";
   public static final int DISABLED_RECONNECT_RETRIES = 1;

exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/CredentialResources.java

@@ -20,7 +20,6 @@
 
 import io.swagger.v3.oas.annotations.ExternalDocumentation;
 import io.swagger.v3.oas.annotations.Operation;
-import org.apache.drill.common.logical.CredentialedStoragePluginConfig;
 import org.apache.drill.common.logical.StoragePluginConfig.AuthMode;
 import org.apache.drill.common.logical.StoragePluginConfig;
 import org.apache.drill.common.logical.security.CredentialsProvider;
@@ -156,14 +155,7 @@ public Response createOrUpdateCredentials(@FormParam("plugin") String pluginName
     }
 
     // Get the config
-    StoragePluginConfig rawConfig = storage.getStoredConfig(pluginName);
-    if (!(rawConfig instanceof CredentialedStoragePluginConfig)) {
-      return Response.status(Status.INTERNAL_SERVER_ERROR)
-        .entity(message(pluginName + " does not support per user credentials."))
-        .build();
-    }
-
-    CredentialedStoragePluginConfig config = (CredentialedStoragePluginConfig)rawConfig;
+    StoragePluginConfig config = storage.getStoredConfig(pluginName);
 
     if (config.getAuthMode() != AuthMode.USER_TRANSLATION) {
       return Response.status(Status.INTERNAL_SERVER_ERROR)
@@ -177,7 +169,7 @@ public Response createOrUpdateCredentials(@FormParam("plugin") String pluginName
 
     // Since the config classes are not accessible from java-exec, we have to serialize them,
     // replace the credential provider with the updated one, and update the storage plugin registry
-    CredentialedStoragePluginConfig newConfig = config.updateCredentialProvider(credentialProvider);
+    StoragePluginConfig newConfig = config.updateCredentialProvider(credentialProvider);
     newConfig.setEnabled(config.isEnabled());
 
     try {
@@ -210,26 +202,19 @@ public Response createOrUpdatePlugin(@PathParam("pluginName") String pluginName,
     cleanPluginName = pluginName.trim();
     StoragePluginConfig config = storage.getStoredConfig(cleanPluginName);
 
-    if (!(config instanceof CredentialedStoragePluginConfig)) {
-      return Response.status(Status.INTERNAL_SERVER_ERROR)
-        .entity(message(cleanPluginName + " does not support user translation."))
-        .build();
-    }
-
     if (config.getAuthMode() != AuthMode.USER_TRANSLATION) {
       return Response.status(Status.INTERNAL_SERVER_ERROR)
         .entity(message(cleanPluginName + " does not have user translation enabled."))
         .build();
     }
 
-    CredentialedStoragePluginConfig credsConfig = (CredentialedStoragePluginConfig)config;
-    CredentialsProvider credentialProvider = credsConfig.getCredentialsProvider();
+    CredentialsProvider credentialProvider = config.getCredentialsProvider();
     credentialProvider.setUserCredentials(credentials.getUsername(), credentials.getPassword(), queryUser);
 
     // Since the config classes are not accessible from java-exec, we have to serialize them,
     // replace the credential provider with the updated one, and update the storage plugin registry
-    CredentialedStoragePluginConfig newConfig = credsConfig.updateCredentialProvider(credentialProvider);
-    newConfig.setEnabled(credsConfig.isEnabled());
+    StoragePluginConfig newConfig = config.updateCredentialProvider(credentialProvider);
+    newConfig.setEnabled(config.isEnabled());
 
     try {
       storage.validatedPut(cleanPluginName, newConfig);