From fd93eac787470aef5387b570a3d6b809a637abf5 Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Sun, 12 Nov 2017 03:26:06 -0800 Subject: [PATCH 1/4] [SPARK-22490][DOC] Add PySpark doc for SparkSession.builder --- python/docs/pyspark.sql.rst | 1 + python/pyspark/sql/session.py | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/python/docs/pyspark.sql.rst b/python/docs/pyspark.sql.rst index 09848b880194d..7c3d6bb5b1d32 100644 --- a/python/docs/pyspark.sql.rst +++ b/python/docs/pyspark.sql.rst @@ -7,6 +7,7 @@ Module Context .. automodule:: pyspark.sql :members: :undoc-members: + :exclude-members: builder pyspark.sql.types module ------------------------ diff --git a/python/pyspark/sql/session.py b/python/pyspark/sql/session.py index d1d0b8b8fe5d9..3ce507f4cffb9 100644 --- a/python/pyspark/sql/session.py +++ b/python/pyspark/sql/session.py @@ -72,6 +72,9 @@ class SparkSession(object): ... .appName("Word Count") \\ ... .config("spark.some.config.option", "some-value") \\ ... .getOrCreate() + + .. autoattribute:: builder + :annotation: """ class Builder(object): @@ -182,6 +185,7 @@ def getOrCreate(self): session.sparkContext._conf.set(key, value) return session + #: class variable for :class:`Builder` shared by all :class:`SparkSession` instances builder = Builder() _instantiatedSession = None From 6f9fc845697d898c4b481cf087c93bc005227a54 Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Sun, 12 Nov 2017 15:47:24 -0800 Subject: [PATCH 2/4] Address comments. --- python/pyspark/sql/session.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/pyspark/sql/session.py b/python/pyspark/sql/session.py index 3ce507f4cffb9..a1275ee958a76 100644 --- a/python/pyspark/sql/session.py +++ b/python/pyspark/sql/session.py @@ -185,8 +185,8 @@ def getOrCreate(self): session.sparkContext._conf.set(key, value) return session - #: class variable for :class:`Builder` shared by all :class:`SparkSession` instances builder = Builder() + """A class attribute having a :class:`Builder` for constructing :class:`SparkSession` instances""" _instantiatedSession = None From b3090e77feb954681e79c29f5af68479da037053 Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Sun, 12 Nov 2017 16:01:18 -0800 Subject: [PATCH 3/4] Reduce the length of line. --- python/pyspark/sql/session.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/pyspark/sql/session.py b/python/pyspark/sql/session.py index a1275ee958a76..33e492cc208f5 100644 --- a/python/pyspark/sql/session.py +++ b/python/pyspark/sql/session.py @@ -186,7 +186,7 @@ def getOrCreate(self): return session builder = Builder() - """A class attribute having a :class:`Builder` for constructing :class:`SparkSession` instances""" + """A class attribute having a :class:`Builder` to construct :class:`SparkSession` instances""" _instantiatedSession = None From 350edf4826356b04f4344737581541b8eba600c0 Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Tue, 14 Nov 2017 09:01:04 -0800 Subject: [PATCH 4/4] Add a comment about this workaround. --- python/docs/pyspark.sql.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/docs/pyspark.sql.rst b/python/docs/pyspark.sql.rst index 7c3d6bb5b1d32..5c3b7e274857a 100644 --- a/python/docs/pyspark.sql.rst +++ b/python/docs/pyspark.sql.rst @@ -8,6 +8,8 @@ Module Context :members: :undoc-members: :exclude-members: builder +.. We need `exclude-members` to prevent default description generations + as a workaround for old Sphinx (< 1.6.6). pyspark.sql.types module ------------------------