[doc] Integrate pyspark module into sphinx doc [skip ci] (#8066)

dmlc · Jul 17, 2022 · e28f6f6 · e28f6f6
1 parent 579ab23
commit e28f6f6
Show file tree

Hide file tree

Showing 4 changed files with 47 additions and 13 deletions.
diff --git a/doc/conf.py b/doc/conf.py
@@ -207,10 +207,11 @@
     "python": ("https://docs.python.org/3.6", None),
     "numpy": ("https://docs.scipy.org/doc/numpy/", None),
     "scipy": ("https://docs.scipy.org/doc/scipy/reference/", None),
-    "pandas": ("http://pandas-docs.github.io/pandas-docs-travis/", None),
+    "pandas": ("https://pandas.pydata.org/pandas-docs/stable/", None),
     "sklearn": ("https://scikit-learn.org/stable", None),
     "dask": ("https://docs.dask.org/en/stable/", None),
     "distributed": ("https://distributed.dask.org/en/stable/", None),
+    "pyspark": ("https://spark.apache.org/docs/latest/api/python/", None),
 }
 
 

diff --git a/doc/python/python_api.rst b/doc/python/python_api.rst
@@ -147,3 +147,29 @@ Dask API
     :members:
     :inherited-members:
     :show-inheritance:
+
+
+PySpark API
+-----------
+
+.. automodule:: xgboost.spark
+
+.. autoclass:: xgboost.spark.SparkXGBClassifier
+    :members:
+    :inherited-members:
+    :show-inheritance:
+
+.. autoclass:: xgboost.spark.SparkXGBClassifierModel
+    :members:
+    :inherited-members:
+    :show-inheritance:
+
+.. autoclass:: xgboost.spark.SparkXGBRegressor
+    :members:
+    :inherited-members:
+    :show-inheritance:
+
+.. autoclass:: xgboost.spark.SparkXGBRegressorModel
+    :members:
+    :inherited-members:
+    :show-inheritance:
diff --git a/doc/requirements.txt b/doc/requirements.txt
@@ -9,4 +9,6 @@ graphviz
 numpy
 recommonmark
 xgboost_ray
-sphinx-gallery
+sphinx-gallery
+pyspark
+cloudpickle
diff --git a/python-package/xgboost/spark/estimator.py b/python-package/xgboost/spark/estimator.py
@@ -15,12 +15,13 @@ class SparkXGBRegressor(_SparkXGBEstimator):
     """
     SparkXGBRegressor is a PySpark ML estimator. It implements the XGBoost regression
     algorithm based on XGBoost python library, and it can be used in PySpark Pipeline
-    and PySpark ML meta algorithms like CrossValidator/TrainValidationSplit/OneVsRest.
+    and PySpark ML meta algorithms like :py:class:`~pyspark.ml.tuning.CrossValidator`/
+    :py:class:`~pyspark.ml.tuning.TrainValidationSplit`/
+    :py:class:`~pyspark.ml.classification.OneVsRest`
 
     SparkXGBRegressor automatically supports most of the parameters in
     `xgboost.XGBRegressor` constructor and most of the parameters used in
-    `xgboost.XGBRegressor` fit and predict method (see `API docs <https://xgboost.readthedocs\
-    .io/en/latest/python/python_api.html#xgboost.XGBRegressor>`_ for details).
+    :py:class:`xgboost.XGBRegressor` fit and predict method.
 
     SparkXGBRegressor doesn't support setting `gpu_id` but support another param `use_gpu`,
     see doc below for more details.
@@ -65,7 +66,8 @@ class SparkXGBRegressor(_SparkXGBEstimator):
 
     .. Note:: This API is experimental.
 
-    **Examples**
+    Examples
+    --------
 
     >>> from xgboost.spark import SparkXGBRegressor
     >>> from pyspark.ml.linalg import Vectors
@@ -104,15 +106,16 @@ def _pyspark_model_cls(cls):
 
 
 class SparkXGBClassifier(_SparkXGBEstimator, HasProbabilityCol, HasRawPredictionCol):
-    """
-    SparkXGBClassifier is a PySpark ML estimator. It implements the XGBoost classification
-    algorithm based on XGBoost python library, and it can be used in PySpark Pipeline
-    and PySpark ML meta algorithms like CrossValidator/TrainValidationSplit/OneVsRest.
+    """SparkXGBClassifier is a PySpark ML estimator. It implements the XGBoost
+    classification algorithm based on XGBoost python library, and it can be used in
+    PySpark Pipeline and PySpark ML meta algorithms like
+    :py:class:`~pyspark.ml.tuning.CrossValidator`/
+    :py:class:`~pyspark.ml.tuning.TrainValidationSplit`/
+    :py:class:`~pyspark.ml.classification.OneVsRest`
 
     SparkXGBClassifier automatically supports most of the parameters in
     `xgboost.XGBClassifier` constructor and most of the parameters used in
-    `xgboost.XGBClassifier` fit and predict method (see `API docs <https://xgboost.readthedocs\
-    .io/en/latest/python/python_api.html#xgboost.XGBClassifier>`_ for details).
+    :py:class:`xgboost.XGBClassifier` fit and predict method.
 
     SparkXGBClassifier doesn't support setting `gpu_id` but support another param `use_gpu`,
     see doc below for more details.
@@ -127,6 +130,7 @@ class SparkXGBClassifier(_SparkXGBEstimator, HasProbabilityCol, HasRawPrediction
 
     Parameters
     ----------
+
     callbacks:
         The export and import of the callback functions are at best effort. For
         details, see :py:attr:`xgboost.spark.SparkXGBClassifier.callbacks` param doc.
@@ -166,7 +170,8 @@ class SparkXGBClassifier(_SparkXGBEstimator, HasProbabilityCol, HasRawPrediction
 
     .. Note:: This API is experimental.
 
-    **Examples**
+    Examples
+    --------
 
     >>> from xgboost.spark import SparkXGBClassifier
     >>> from pyspark.ml.linalg import Vectors