Permalink
Browse files

IMPALA-595: Add CASCADE to DROP DATABASE and use it in cleanup_db

Change-Id: Idfa5b6943bc797e10d542487c31b8f1b527d8c97
Reviewed-on: http://gerrit.cloudera.org:8080/635
Reviewed-by: Vlad Berindei <vlad.berindei@cloudera.com>
Tested-by: Internal Jenkins
  • Loading branch information...
vladberindei authored and Internal Jenkins committed Aug 10, 2015
1 parent cfddbb7 commit d6664ae14881c061a2f9d32ccefa92550c57a84c
@@ -487,6 +487,9 @@ struct TDropDbParams {
// If true, no error is raised if the target db does not exist
2: required bool if_exists
+
+ // If true, drops all tables of the database
+ 3: required bool cascade
}
// Parameters of DROP TABLE/VIEW commands
@@ -232,7 +232,7 @@ parser code {:
terminal
KW_ADD, KW_AGGREGATE, KW_ALL, KW_ALTER, KW_ANALYTIC, KW_AND, KW_ANTI, KW_API_VERSION,
KW_ARRAY, KW_AS, KW_ASC, KW_AVRO, KW_BETWEEN, KW_BIGINT, KW_BINARY, KW_BOOLEAN, KW_BY,
- KW_CACHED, KW_CASE, KW_CAST, KW_CHANGE, KW_CHAR, KW_CLASS, KW_CLOSE_FN, KW_COLUMN,
+ KW_CACHED, KW_CASCADE, KW_CASE, KW_CAST, KW_CHANGE, KW_CHAR, KW_CLASS, KW_CLOSE_FN, KW_COLUMN,
KW_COLUMNS, KW_COMMENT, KW_COMPUTE, KW_CREATE, KW_CROSS, KW_CURRENT, KW_DATA,
KW_DATABASE, KW_DATABASES, KW_DATE, KW_DATETIME, KW_DECIMAL, KW_DELIMITED, KW_DESC,
KW_DESCRIBE, KW_DISTINCT, KW_DIV, KW_DOUBLE, KW_DROP, KW_ELSE, KW_END, KW_ESCAPED,
@@ -409,6 +409,7 @@ nonterminal String opt_kw_column;
// Used to simplify commands where KW_TABLE is optional
nonterminal String opt_kw_table;
nonterminal Boolean overwrite_val;
+nonterminal Boolean cascade_val;
// For GRANT/REVOKE/AUTH DDL statements
nonterminal ShowRolesStmt show_roles_stmt;
@@ -1226,6 +1227,13 @@ alter_view_stmt ::=
{: RESULT = new AlterTableOrViewRenameStmt(before_table, new_table, false); :}
;
+cascade_val ::=
+ KW_CASCADE
+ {: RESULT = true; :}
+ |
+ {: RESULT = false; :}
+ ;
+
compute_stats_stmt ::=
KW_COMPUTE KW_STATS table_name:table
{: RESULT = new ComputeStatsStmt(table); :}
@@ -1243,8 +1251,8 @@ drop_stats_stmt ::=
;
drop_db_stmt ::=
- KW_DROP db_or_schema_kw if_exists_val:if_exists IDENT:db_name
- {: RESULT = new DropDbStmt(db_name, if_exists); :}
+ KW_DROP db_or_schema_kw if_exists_val:if_exists IDENT:db_name cascade_val:cascade
+ {: RESULT = new DropDbStmt(db_name, if_exists, cascade); :}
;
drop_tbl_or_view_stmt ::=
@@ -20,38 +20,42 @@
import com.cloudera.impala.thrift.TDropDbParams;
/**
- * Represents a DROP [IF EXISTS] DATABASE statement
+ * Represents a DROP [IF EXISTS] DATABASE [CASCADE] statement
*/
public class DropDbStmt extends StatementBase {
private final String dbName_;
private final boolean ifExists_;
+ private final boolean cascade_;
/**
* Constructor for building the drop statement. If ifExists is true, an error will not
- * be thrown if the database does not exist.
+ * be thrown if the database does not exist. If cascade is true, all the tables in the
+ * database will be dropped.
*/
- public DropDbStmt(String dbName, boolean ifExists) {
+ public DropDbStmt(String dbName, boolean ifExists, boolean cascade) {
this.dbName_ = dbName;
this.ifExists_ = ifExists;
+ this.cascade_ = cascade;
}
public String getDb() { return dbName_; }
public boolean getIfExists() { return ifExists_; }
+ public boolean getCascade() { return cascade_; }
@Override
public String toSql() {
StringBuilder sb = new StringBuilder("DROP DATABASE");
- if (ifExists_) {
- sb.append(" IF EXISTS ");
- }
+ if (ifExists_) sb.append(" IF EXISTS ");
sb.append(getDb());
+ if (cascade_) sb.append(" CASCADE");
return sb.toString();
}
public TDropDbParams toThrift() {
TDropDbParams params = new TDropDbParams();
params.setDb(getDb());
params.setIf_exists(getIfExists());
+ params.setCascade(getCascade());
return params;
}
@@ -65,7 +69,7 @@ public void analyze(Analyzer analyzer) throws AnalysisException {
if (analyzer.getDefaultDb().toLowerCase().equals(dbName_.toLowerCase())) {
throw new AnalysisException("Cannot drop current default database: " + dbName_);
}
- if (db != null && db.numFunctions() > 0) {
+ if (db != null && db.numFunctions() > 0 && !cascade_) {
throw new AnalysisException("Cannot drop non-empty database: " + dbName_);
}
}
@@ -14,23 +14,20 @@
package com.cloudera.impala.service;
-import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.common.StatsSetupConst;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.MetaStoreUtils;
import org.apache.hadoop.hive.metastore.TableType;
+import org.apache.hadoop.hive.metastore.Warehouse;
import org.apache.hadoop.hive.metastore.api.AlreadyExistsException;
import org.apache.hadoop.hive.metastore.api.BooleanColumnStatsData;
import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
@@ -47,8 +44,6 @@
import org.apache.hadoop.hive.metastore.api.SerDeInfo;
import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
import org.apache.hadoop.hive.metastore.api.StringColumnStatsData;
-import org.apache.hadoop.hive.metastore.MetaStoreUtils;
-import org.apache.hadoop.hive.metastore.Warehouse;
import org.apache.log4j.Logger;
import org.apache.thrift.TException;
@@ -146,7 +141,6 @@
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
-import com.google.common.util.concurrent.SettableFuture;
/**
* Class used to execute Catalog Operations, including DDL and refresh/invalidate
@@ -902,8 +896,7 @@ private int dropTableStats(Table table) throws ImpalaRuntimeException {
/**
* Drops a database from the metastore and removes the database's metadata from the
- * internal cache. The database must be empty (contain no tables) for the drop
- * operation to succeed. Re-throws any Hive Meta Store exceptions encountered during
+ * internal cache. Re-throws any Hive Meta Store exceptions encountered during
* the drop.
*/
private void dropDatabase(TDropDbParams params, TDdlExecResponse resp)
@@ -914,15 +907,16 @@ private void dropDatabase(TDropDbParams params, TDdlExecResponse resp)
LOG.debug("Dropping database " + params.getDb());
Db db = catalog_.getDb(params.db);
- if (db != null && db.numFunctions() > 0) {
+ if (db != null && db.numFunctions() > 0 && !params.cascade) {
throw new CatalogException("Database " + db.getName() + " is not empty");
}
TCatalogObject removedObject = new TCatalogObject();
MetaStoreClient msClient = catalog_.getMetaStoreClient();
synchronized (metastoreDdlLock_) {
try {
- msClient.getHiveClient().dropDatabase(params.getDb(), true, params.if_exists);
+ msClient.getHiveClient().dropDatabase(
+ params.getDb(), true, params.if_exists, params.cascade);
} catch (TException e) {
throw new ImpalaRuntimeException(
String.format(HMS_RPC_ERROR_FORMAT_STR, "dropDatabase"), e);
@@ -69,6 +69,7 @@ import com.cloudera.impala.analysis.SqlParserSymbols;
keywordMap.put("by", new Integer(SqlParserSymbols.KW_BY));
keywordMap.put("cached", new Integer(SqlParserSymbols.KW_CACHED));
keywordMap.put("case", new Integer(SqlParserSymbols.KW_CASE));
+ keywordMap.put("cascade", new Integer(SqlParserSymbols.KW_CASCADE));
keywordMap.put("cast", new Integer(SqlParserSymbols.KW_CAST));
keywordMap.put("change", new Integer(SqlParserSymbols.KW_CHANGE));
keywordMap.put("char", new Integer(SqlParserSymbols.KW_CHAR));
@@ -739,20 +739,25 @@ public void TestDropStats() throws AnalysisException {
@Test
public void TestDrop() throws AnalysisException {
AnalyzesOk("drop database functional");
+ AnalyzesOk("drop database functional cascade");
AnalyzesOk("drop table functional.alltypes");
AnalyzesOk("drop view functional.alltypes_view");
// If the database does not exist, and the user hasn't specified "IF EXISTS",
// an analysis error should be thrown
AnalysisError("drop database db_does_not_exist",
"Database does not exist: db_does_not_exist");
+ AnalysisError("drop database db_does_not_exist cascade",
+ "Database does not exist: db_does_not_exist");
AnalysisError("drop table db_does_not_exist.alltypes",
"Database does not exist: db_does_not_exist");
AnalysisError("drop view db_does_not_exist.alltypes_view",
"Database does not exist: db_does_not_exist");
// Invalid name reports non-existence instead of invalidity.
AnalysisError("drop database `???`",
"Database does not exist: ???");
+ AnalysisError("drop database `???` cascade",
+ "Database does not exist: ???");
AnalysisError("drop table functional.`%^&`",
"Table does not exist: functional.%^&");
AnalysisError("drop view functional.`@#$%`",
@@ -767,6 +772,7 @@ public void TestDrop() throws AnalysisException {
// No error is thrown if the user specifies IF EXISTS
AnalyzesOk("drop database if exists db_does_not_exist");
+ AnalyzesOk("drop database if exists db_does_not_exist cascade");
// No error is thrown if the database does not exist
AnalyzesOk("drop table if exists db_does_not_exist.alltypes");
@@ -738,27 +738,39 @@ public void TestCreateDatabase() throws ImpalaException {
public void TestDropDatabase() throws AnalysisException, AuthorizationException {
// User has permissions.
AuthzOk("drop database tpch");
- // User has permissions, database does not exists and IF EXISTS specified
+ AuthzOk("drop database tpch cascade");
+ // User has permissions, database does not exists and IF EXISTS specified.
AuthzOk("drop database if exists newdb");
+ AuthzOk("drop database if exists newdb cascade");
// User has permission, database does not exists, IF EXISTS not specified.
try {
AuthzOk("drop database newdb");
fail("Expected analysis error");
} catch (AnalysisException e) {
Assert.assertEquals(e.getMessage(), "Database does not exist: newdb");
}
+ try {
+ AuthzOk("drop database newdb cascade");
+ fail("Expected analysis error");
+ } catch (AnalysisException e) {
+ Assert.assertEquals(e.getMessage(), "Database does not exist: newdb");
+ }
// Database exists, user doesn't have permission to drop.
AuthzError("drop database functional",
"User '%s' does not have privileges to execute 'DROP' on: functional");
AuthzError("drop database if exists functional",
"User '%s' does not have privileges to execute 'DROP' on: functional");
+ AuthzError("drop database if exists functional cascade",
+ "User '%s' does not have privileges to execute 'DROP' on: functional");
// Database does not exist, user doesn't have permission to drop.
AuthzError("drop database nodb",
"User '%s' does not have privileges to execute 'DROP' on: nodb");
AuthzError("drop database if exists nodb",
"User '%s' does not have privileges to execute 'DROP' on: nodb");
+ AuthzError("drop database if exists nodb cascade",
+ "User '%s' does not have privileges to execute 'DROP' on: nodb");
AuthzError("drop database _impala_builtins",
"Cannot modify system database.");
@@ -2346,8 +2346,10 @@ public void TestDrop() {
ParsesOk("DROP VIEW Foo.Bar");
ParsesOk("DROP VIEW IF EXISTS Foo.Bar");
ParsesOk("DROP DATABASE Foo");
+ ParsesOk("DROP DATABASE Foo CASCADE");
ParsesOk("DROP SCHEMA Foo");
ParsesOk("DROP DATABASE IF EXISTS Foo");
+ ParsesOk("DROP DATABASE IF EXISTS Foo CASCADE");
ParsesOk("DROP SCHEMA IF EXISTS Foo");
ParsesOk("DROP FUNCTION Foo()");
ParsesOk("DROP AGGREGATE FUNCTION Foo(INT)");
@@ -2361,6 +2363,8 @@ public void TestDrop() {
ParserError("DROP DATABASE Foo.Bar");
ParserError("DROP SCHEMA Foo.Bar");
ParserError("DROP DATABASE Foo Bar");
+ ParserError("DROP DATABASE CASCADE Foo");
+ ParserError("DROP CASCADE DATABASE IF EXISTS Foo");
ParserError("DROP SCHEMA Foo Bar");
ParserError("DROP TABLE IF Foo");
ParserError("DROP TABLE EXISTS Foo");
@@ -682,3 +682,62 @@ show databases like 'ddl_test_db'
---- TYPES
STRING
====
+---- QUERY
+# Test DROP DATABASE ... CASCADE
+create database if not exists test_drop_cascade_db
+====
+---- QUERY
+show databases like 'test_drop_cascade_db'
+---- RESULTS
+'test_drop_cascade_db'
+---- TYPES
+STRING
+====
+---- QUERY
+create table if not exists test_drop_cascade_db.t1 (i int);
+create table if not exists test_drop_cascade_db.t2 (i int)
+ partitioned by (year smallint, month smallint);
+insert into test_drop_cascade_db.t2 partition (year=2015, month=8) values(1);
+create external table if not exists test_drop_cascade_db.t3 like functional.alltypes
+ location '$FILESYSTEM_PREFIX/test-warehouse/alltypes_external';
+create view if not exists test_drop_cascade_db.v1 as
+ select int_col from functional.alltypes;
+create function if not exists test_drop_cascade_db.f1() returns string
+ location '$FILESYSTEM_PREFIX/test-warehouse/libTestUdfs.so' symbol='NoArgs';
+create aggregate function if not exists test_drop_cascade_db.f2(int, string) RETURNS int
+ location '$FILESYSTEM_PREFIX/test-warehouse/libTestUdas.so' UPDATE_FN='TwoArgUpdate'
+---- RESULTS
+====
+---- QUERY
+show tables in test_drop_cascade_db
+---- RESULTS
+'t1'
+'t2'
+'t3'
+'v1'
+---- TYPES
+STRING
+====
+---- QUERY
+show functions in test_drop_cascade_db
+---- RESULTS
+'STRING','f1()'
+---- TYPES
+STRING, STRING
+====
+---- QUERY
+show aggregate functions in test_drop_cascade_db
+---- RESULTS
+'INT','f2(INT, STRING)'
+---- TYPES
+STRING, STRING
+====
+---- QUERY
+# Should drop all tables and the database
+drop database test_drop_cascade_db cascade
+---- RESULTS
+====
+---- QUERY
+show databases like 'test_drop_cascade_db'
+---- RESULTS
+====
@@ -136,28 +136,9 @@ def create_hdfs_client(cls):
@classmethod
def cleanup_db(self, db_name, sync_ddl=1):
- # To drop a db, we need to first drop all the tables in that db
self.client.execute("use default")
self.client.set_configuration({'sync_ddl': sync_ddl})
-
- if db_name in self.client.execute("show databases", ).data:
- # We use quoted identifiers to avoid name clashes with keywords
- for tbl_name in self.client.execute("show tables in `" + db_name + "`").data:
- full_tbl_name = '`%s`.`%s`' % (db_name, tbl_name)
- result = self.client.execute("describe formatted " + full_tbl_name)
- if 'VIRTUAL_VIEW' in '\n'.join(result.data):
- self.client.execute("drop view " + full_tbl_name)
- else:
- self.client.execute("drop table " + full_tbl_name)
- for fn_result in self.client.execute("show functions in `" + db_name + "`").data:
- # First column is the return type, second is the function signature
- fn_name = fn_result.split('\t')[1]
- self.client.execute("drop function `%s`.%s" % (db_name, fn_name))
- for fn_result in self.client.execute(\
- "show aggregate functions in `" + db_name + "`").data:
- fn_name = fn_result.split('\t')[1]
- self.client.execute("drop function `%s`.%s" % (db_name, fn_name))
- self.client.execute("drop database `" + db_name + "`")
+ self.client.execute("drop database if exists `" + db_name + "` cascade")
def run_test_case(self, test_file_name, vector, use_db=None, multiple_impalad=False,
encoding=None):
Oops, something went wrong.

0 comments on commit d6664ae

Please sign in to comment.