From 866e8dfa32501b1853cbe3cb52fc569ec34a80c2 Mon Sep 17 00:00:00 2001 From: madhukar Date: Thu, 23 Apr 2015 13:40:47 +0530 Subject: [PATCH 1/3] [SPARK-7084] improve saveAsTable documentation --- sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala b/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala index ca6ae482eb2ab..951829ad2abb0 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala @@ -1085,6 +1085,9 @@ class DataFrame private[sql]( * there is no notion of a persisted catalog in a standard SQL context. Instead you can write * an RDD out to a parquet file, and then register that file as a table. This "table" can then * be the target of an `insertInto`. + * + * Also note that this doesn't create a hive table, but instead creates a Spark data source table. + * Here the metadata is persisted into Hive. But table will be not accessible from the hive. * @group output */ @Experimental From 2a802c6d3ce59210a04cf7c38d2f69ed7751128e Mon Sep 17 00:00:00 2001 From: madhukar Date: Thu, 23 Apr 2015 15:33:48 +0530 Subject: [PATCH 2/3] #5654 updated the doc according to comments --- sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala b/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala index 951829ad2abb0..3c2fbcdef1e69 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala @@ -1086,8 +1086,8 @@ class DataFrame private[sql]( * an RDD out to a parquet file, and then register that file as a table. This "table" can then * be the target of an `insertInto`. * - * Also note that this doesn't create a hive table, but instead creates a Spark data source table. - * Here the metadata is persisted into Hive. But table will be not accessible from the hive. + * Also note that while this function can persist the table metadata into Hive's metastore, + * the table will NOT be accessible from Hive. * @group output */ @Experimental From 386f407bad7a1b93c70194af48a34fd8eb3af229 Mon Sep 17 00:00:00 2001 From: madhukar Date: Fri, 24 Apr 2015 09:55:51 +0530 Subject: [PATCH 3/3] #5654 updated for all the methods --- .../scala/org/apache/spark/sql/DataFrame.scala | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala b/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala index 3c2fbcdef1e69..fe193c14ca035 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala @@ -1104,6 +1104,9 @@ class DataFrame private[sql]( * there is no notion of a persisted catalog in a standard SQL context. Instead you can write * an RDD out to a parquet file, and then register that file as a table. This "table" can then * be the target of an `insertInto`. + * + * Also note that while this function can persist the table metadata into Hive's metastore, + * the table will NOT be accessible from Hive. * @group output */ @Experimental @@ -1128,6 +1131,9 @@ class DataFrame private[sql]( * there is no notion of a persisted catalog in a standard SQL context. Instead you can write * an RDD out to a parquet file, and then register that file as a table. This "table" can then * be the target of an `insertInto`. + * + * Also note that while this function can persist the table metadata into Hive's metastore, + * the table will NOT be accessible from Hive. * @group output */ @Experimental @@ -1144,6 +1150,9 @@ class DataFrame private[sql]( * there is no notion of a persisted catalog in a standard SQL context. Instead you can write * an RDD out to a parquet file, and then register that file as a table. This "table" can then * be the target of an `insertInto`. + * + * Also note that while this function can persist the table metadata into Hive's metastore, + * the table will NOT be accessible from Hive. * @group output */ @Experimental @@ -1160,6 +1169,9 @@ class DataFrame private[sql]( * there is no notion of a persisted catalog in a standard SQL context. Instead you can write * an RDD out to a parquet file, and then register that file as a table. This "table" can then * be the target of an `insertInto`. + * + * Also note that while this function can persist the table metadata into Hive's metastore, + * the table will NOT be accessible from Hive. * @group output */ @Experimental @@ -1181,6 +1193,9 @@ class DataFrame private[sql]( * there is no notion of a persisted catalog in a standard SQL context. Instead you can write * an RDD out to a parquet file, and then register that file as a table. This "table" can then * be the target of an `insertInto`. + * + * Also note that while this function can persist the table metadata into Hive's metastore, + * the table will NOT be accessible from Hive. * @group output */ @Experimental