Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[SPARK-34197][SQL] SessionCatalog.refreshTable() should not invalidate the relation cache for temporary views #31265

Closed
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
Expand Up @@ -886,21 +886,25 @@ class SessionCatalog(
isTempView(nameParts.asTableIdentifier)
}

private def lookupTempView(name: TableIdentifier): Option[LogicalPlan] = {
val tableName = formatTableName(name.table)
if (name.database.isEmpty) {
tempViews.get(tableName)
} else if (formatDatabaseName(name.database.get) == globalTempViewManager.database) {
globalTempViewManager.get(tableName)
} else {
None
}
}

/**
* Return whether a table with the specified name is a temporary view.
*
* Note: The temporary view cache is checked only when database is not
* explicitly specified.
*/
def isTempView(name: TableIdentifier): Boolean = synchronized {
val table = formatTableName(name.table)
if (name.database.isEmpty) {
tempViews.contains(table)
} else if (formatDatabaseName(name.database.get) == globalTempViewManager.database) {
globalTempViewManager.get(table).isDefined
} else {
false
}
lookupTempView(name).isDefined
}

def isView(nameParts: Seq[String]): Boolean = {
Expand Down Expand Up @@ -997,21 +1001,12 @@ class SessionCatalog(
* Refresh the cache entry for a metastore table, if any.
*/
def refreshTable(name: TableIdentifier): Unit = synchronized {
val dbName = formatDatabaseName(name.database.getOrElse(currentDb))
val tableName = formatTableName(name.table)

// Go through temporary views and invalidate them.
// If the database is defined, this may be a global temporary view.
// If the database is not defined, there is a good chance this is a temp view.
if (name.database.isEmpty) {
tempViews.get(tableName).foreach(_.refresh())
} else if (dbName == globalTempViewManager.database) {
globalTempViewManager.get(tableName).foreach(_.refresh())
lookupTempView(name).map(_.refresh).getOrElse {
val dbName = formatDatabaseName(name.database.getOrElse(currentDb))
val tableName = formatTableName(name.table)
val qualifiedTableName = QualifiedTableName(dbName, tableName)
tableRelationCache.invalidate(qualifiedTableName)
}

// Also invalidate the table relation cache.
val qualifiedTableName = QualifiedTableName(dbName, tableName)
tableRelationCache.invalidate(qualifiedTableName)
}

/**
Expand Down
Expand Up @@ -1680,4 +1680,20 @@ abstract class SessionCatalogSuite extends AnalysisTest with Eventually {
}
}
}

test("SPARK-34197: refreshTable should not invalidate the relation cache for temporary views") {
withBasicCatalog { catalog =>
catalog.createTempView("tbl1", Range(1, 10, 1, 10), false)
val qualifiedName1 = QualifiedTableName("default", "tbl1")
catalog.cacheTable(qualifiedName1, Range(1, 10, 1, 10))
catalog.refreshTable(TableIdentifier("tbl1"))
assert(catalog.getCachedTable(qualifiedName1) != null)

catalog.createGlobalTempView("tbl2", Range(2, 10, 1, 10), false)
val qualifiedName2 = QualifiedTableName(catalog.globalTempViewManager.database, "tbl2")
catalog.cacheTable(qualifiedName2, Range(2, 10, 1, 10))
cloud-fan marked this conversation as resolved.
Show resolved Hide resolved
catalog.refreshTable(TableIdentifier("tbl2", Some(catalog.globalTempViewManager.database)))
assert(catalog.getCachedTable(qualifiedName2) != null)
}
}
}