Skip to content
Permalink
Browse files

[SPARK-24669][SQL] Invalidate tables in case of DROP DATABASE CASCADE

  ## What changes were proposed in this pull request?
Before dropping database refresh the tables of that database, so as to refresh all cached entries associated with those tables.
We follow the same when dropping a table.

## How was this patch tested?
UT is added

Closes #23905 from Udbhav30/SPARK-24669.

Authored-by: Udbhav30 <u.agrawal30@gmail.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
  • Loading branch information...
Udbhav30 authored and dongjoon-hyun committed Mar 6, 2019
1 parent 190a3a4 commit 9bddf7180e9e76e1cabc580eee23962dd66f84c3
@@ -218,6 +218,11 @@ class SessionCatalog(
if (dbName == DEFAULT_DATABASE) {
throw new AnalysisException(s"Can not drop default database")
}
if (cascade && databaseExists(dbName)) {
listTables(dbName).foreach { t =>
invalidateCachedTable(QualifiedTableName(dbName, t.table))
}
}
externalCatalog.dropDatabase(dbName, ignoreIfNotExists, cascade)
}

@@ -17,7 +17,7 @@

package org.apache.spark.sql.execution.command

import java.io.File
import java.io.{File, PrintWriter}
import java.net.URI
import java.util.Locale

@@ -2715,4 +2715,40 @@ abstract class DDLSuite extends QueryTest with SQLTestUtils {
}
assert(ex.getMessage.contains("Spark config"))
}

test("Refresh table before drop database cascade") {
withTempDir { tempDir =>
val file1 = new File(tempDir + "/first.csv")
val writer1 = new PrintWriter(file1)
writer1.write("first")
writer1.close()

val file2 = new File(tempDir + "/second.csv")
val writer2 = new PrintWriter(file2)
writer2.write("second")
writer2.close()

withDatabase("foo") {
withTable("foo.first") {
sql("CREATE DATABASE foo")
sql(
s"""CREATE TABLE foo.first (id STRING)
|USING csv OPTIONS (path='${file1.toURI}')
""".stripMargin)
sql("SELECT * FROM foo.first")
checkAnswer(spark.table("foo.first"), Row("first"))

// Dropping the database and again creating same table with different path
sql("DROP DATABASE foo CASCADE")
sql("CREATE DATABASE foo")
sql(
s"""CREATE TABLE foo.first (id STRING)
|USING csv OPTIONS (path='${file2.toURI}')
""".stripMargin)
sql("SELECT * FROM foo.first")
checkAnswer(spark.table("foo.first"), Row("second"))
}
}
}
}
}

0 comments on commit 9bddf71

Please sign in to comment.
You can’t perform that action at this time.