From b1bab96ff29296fd16e49ad49a31536243a4bb2e Mon Sep 17 00:00:00 2001 From: felixcheung Date: Tue, 19 Apr 2016 14:35:18 -0700 Subject: [PATCH 1/2] change unpersist blocking default value to match Scala --- python/pyspark/sql/dataframe.py | 2 +- python/pyspark/sql/tests.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/python/pyspark/sql/dataframe.py b/python/pyspark/sql/dataframe.py index b4fa8368936a4..3c9a46d9493fb 100644 --- a/python/pyspark/sql/dataframe.py +++ b/python/pyspark/sql/dataframe.py @@ -326,7 +326,7 @@ def persist(self, storageLevel=StorageLevel.MEMORY_ONLY): return self @since(1.3) - def unpersist(self, blocking=True): + def unpersist(self, blocking=False): """Marks the :class:`DataFrame` as non-persistent, and remove all blocks for it from memory and disk. """ diff --git a/python/pyspark/sql/tests.py b/python/pyspark/sql/tests.py index e4f79c911c0d9..d4c221d7125ca 100644 --- a/python/pyspark/sql/tests.py +++ b/python/pyspark/sql/tests.py @@ -362,7 +362,7 @@ def test_basic_functions(self): # cache and checkpoint self.assertFalse(df.is_cached) df.persist() - df.unpersist() + df.unpersist(True) df.cache() self.assertTrue(df.is_cached) self.assertEqual(2, df.count()) From 17318ceca0559ea2a85ce13a15923ccad74ee07f Mon Sep 17 00:00:00 2001 From: felixcheung Date: Tue, 19 Apr 2016 16:04:58 -0700 Subject: [PATCH 2/2] add docstring note --- python/pyspark/sql/dataframe.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/pyspark/sql/dataframe.py b/python/pyspark/sql/dataframe.py index 3c9a46d9493fb..328bda6601565 100644 --- a/python/pyspark/sql/dataframe.py +++ b/python/pyspark/sql/dataframe.py @@ -329,6 +329,8 @@ def persist(self, storageLevel=StorageLevel.MEMORY_ONLY): def unpersist(self, blocking=False): """Marks the :class:`DataFrame` as non-persistent, and remove all blocks for it from memory and disk. + + .. note:: `blocking` default has changed to False to match Scala in 2.0. """ self.is_cached = False self._jdf.unpersist(blocking)