From f99b23a770aaece0c297691e18758db6a6ce520b Mon Sep 17 00:00:00 2001 From: Hyukjin Kwon Date: Mon, 27 Nov 2023 09:47:21 +0900 Subject: [PATCH 1/7] Deprecate pyspark.keyword_only API --- python/pyspark/__init__.py | 11 ++- python/pyspark/ml/classification.py | 20 +----- python/pyspark/ml/clustering.py | 12 +--- python/pyspark/ml/connect/classification.py | 2 - python/pyspark/ml/connect/evaluation.py | 3 - python/pyspark/ml/connect/feature.py | 4 -- python/pyspark/ml/connect/pipeline.py | 4 +- python/pyspark/ml/connect/tuning.py | 4 +- python/pyspark/ml/evaluation.py | 14 +--- python/pyspark/ml/feature.py | 76 +-------------------- python/pyspark/ml/fpm.py | 6 +- python/pyspark/ml/pipeline.py | 4 +- python/pyspark/ml/recommendation.py | 4 +- python/pyspark/ml/regression.py | 18 +---- python/pyspark/ml/tuning.py | 6 +- python/pyspark/testing/mlutils.py | 3 - python/pyspark/tests/test_util.py | 39 ----------- python/pyspark/tests/typing/test_core.yml | 2 +- 18 files changed, 22 insertions(+), 210 deletions(-) diff --git a/python/pyspark/__init__.py b/python/pyspark/__init__.py index a9a2a31702562..f859265afbab7 100644 --- a/python/pyspark/__init__.py +++ b/python/pyspark/__init__.py @@ -45,7 +45,7 @@ - :class:`InheritableThread`: A inheritable thread to use in Spark when the pinned thread mode is on. """ - +import warnings from functools import wraps import types from typing import cast, Any, Callable, Optional, TypeVar, Union @@ -121,10 +121,19 @@ def keyword_only(func: _F) -> _F: A decorator that forces keyword arguments in the wrapped method and saves actual input keyword arguments in `_input_kwargs`. + .. deprecated:: 4.0.0 + `keyword_only` method is deprecated. Use the standard + keyword-only syntax in Python instead. + Notes ----- Should only be used to wrap a method where first arg is `self` """ + warnings.warn( + "keyword_only method is deprecated. Use the standard keyword-only" + "syntax in Python instead.", + FutureWarning, + ) @wraps(func) def wrapper(self: Any, *args: Any, **kwargs: Any) -> Any: diff --git a/python/pyspark/ml/classification.py b/python/pyspark/ml/classification.py index 263a108a216dd..fb7392e138f33 100644 --- a/python/pyspark/ml/classification.py +++ b/python/pyspark/ml/classification.py @@ -37,7 +37,7 @@ TYPE_CHECKING, ) -from pyspark import keyword_only, since, SparkContext, inheritable_thread_target +from pyspark import since, SparkContext, inheritable_thread_target from pyspark.ml import Estimator, Predictor, PredictionModel, Model from pyspark.ml.param.shared import ( HasRawPredictionCol, @@ -708,7 +708,6 @@ class LinearSVC( _input_kwargs: Dict[str, Any] - @keyword_only def __init__( self, *, @@ -739,7 +738,6 @@ def __init__( kwargs = self._input_kwargs self.setParams(**kwargs) - @keyword_only @since("2.2.0") def setParams( self, @@ -1275,7 +1273,6 @@ def __init__( ): ... - @keyword_only def __init__( self, *, @@ -1374,7 +1371,6 @@ def setParams( ) -> "LogisticRegression": ... - @keyword_only @since("1.3.0") def setParams( self, @@ -1772,7 +1768,6 @@ class DecisionTreeClassifier( _input_kwargs: Dict[str, Any] - @keyword_only def __init__( self, *, @@ -1808,7 +1803,6 @@ def __init__( kwargs = self._input_kwargs self.setParams(**kwargs) - @keyword_only @since("1.4.0") def setParams( self, @@ -2064,7 +2058,6 @@ class RandomForestClassifier( _input_kwargs: Dict[str, Any] - @keyword_only def __init__( self, *, @@ -2105,7 +2098,6 @@ def __init__( kwargs = self._input_kwargs self.setParams(**kwargs) - @keyword_only @since("1.4.0") def setParams( self, @@ -2533,7 +2525,6 @@ class GBTClassifier( _input_kwargs: Dict[str, Any] - @keyword_only def __init__( self, *, @@ -2576,7 +2567,6 @@ def __init__( kwargs = self._input_kwargs self.setParams(**kwargs) - @keyword_only @since("1.4.0") def setParams( self, @@ -2930,7 +2920,6 @@ class NaiveBayes( _input_kwargs: Dict[str, Any] - @keyword_only def __init__( self, *, @@ -2956,7 +2945,6 @@ def __init__( kwargs = self._input_kwargs self.setParams(**kwargs) - @keyword_only @since("1.5.0") def setParams( self, @@ -3180,7 +3168,6 @@ class MultilayerPerceptronClassifier( _input_kwargs: Dict[str, Any] - @keyword_only def __init__( self, *, @@ -3211,7 +3198,6 @@ def __init__( kwargs = self._input_kwargs self.setParams(**kwargs) - @keyword_only @since("1.6.0") def setParams( self, @@ -3448,7 +3434,6 @@ class OneVsRest( _input_kwargs: Dict[str, Any] - @keyword_only def __init__( self, *, @@ -3469,7 +3454,6 @@ def __init__( kwargs = self._input_kwargs self._set(**kwargs) - @keyword_only @since("2.0.0") def setParams( self, @@ -4061,7 +4045,6 @@ class FMClassifier( _input_kwargs: Dict[str, Any] - @keyword_only def __init__( self, *, @@ -4097,7 +4080,6 @@ def __init__( kwargs = self._input_kwargs self.setParams(**kwargs) - @keyword_only @since("3.0.0") def setParams( self, diff --git a/python/pyspark/ml/clustering.py b/python/pyspark/ml/clustering.py index 7f9e87e612432..eb58fa1fac71b 100644 --- a/python/pyspark/ml/clustering.py +++ b/python/pyspark/ml/clustering.py @@ -21,7 +21,7 @@ import numpy as np -from pyspark import since, keyword_only +from pyspark import since from pyspark.ml.param.shared import ( HasMaxIter, HasFeaturesCol, @@ -403,7 +403,6 @@ class GaussianMixture( _input_kwargs: Dict[str, Any] - @keyword_only def __init__( self, *, @@ -432,7 +431,6 @@ def __init__( def _create_model(self, java_model: "JavaObject") -> "GaussianMixtureModel": return GaussianMixtureModel(java_model) - @keyword_only @since("2.0.0") def setParams( self, @@ -777,7 +775,6 @@ class KMeans(JavaEstimator[KMeansModel], _KMeansParams, JavaMLWritable, JavaMLRe _input_kwargs: Dict[str, Any] - @keyword_only def __init__( self, *, @@ -808,7 +805,6 @@ def __init__( def _create_model(self, java_model: "JavaObject") -> KMeansModel: return KMeansModel(java_model) - @keyword_only @since("1.5.0") def setParams( self, @@ -1129,7 +1125,6 @@ class BisectingKMeans( _input_kwargs: Dict[str, Any] - @keyword_only def __init__( self, *, @@ -1154,7 +1149,6 @@ def __init__( kwargs = self._input_kwargs self.setParams(**kwargs) - @keyword_only @since("2.0.0") def setParams( self, @@ -1681,7 +1675,6 @@ class LDA(JavaEstimator[LDAModel], _LDAParams, JavaMLReadable["LDA"], JavaMLWrit _input_kwargs: Dict[str, Any] - @keyword_only def __init__( self, *, @@ -1718,7 +1711,6 @@ def _create_model(self, java_model: "JavaObject") -> LDAModel: else: return LocalLDAModel(java_model) - @keyword_only @since("2.0.0") def setParams( self, @@ -2036,7 +2028,6 @@ class PowerIterationClustering( _input_kwargs: Dict[str, Any] - @keyword_only def __init__( self, *, @@ -2058,7 +2049,6 @@ def __init__( kwargs = self._input_kwargs self.setParams(**kwargs) - @keyword_only @since("2.4.0") def setParams( self, diff --git a/python/pyspark/ml/connect/classification.py b/python/pyspark/ml/connect/classification.py index 8b816f51ca275..261cbd5066a9a 100644 --- a/python/pyspark/ml/connect/classification.py +++ b/python/pyspark/ml/connect/classification.py @@ -22,7 +22,6 @@ import numpy as np import pandas as pd -from pyspark import keyword_only from pyspark.ml.connect.base import _PredictorParams from pyspark.ml.param.shared import HasProbabilityCol from pyspark.sql import DataFrame @@ -178,7 +177,6 @@ class LogisticRegression( _input_kwargs: Dict[str, Any] - @keyword_only def __init__( self, *, diff --git a/python/pyspark/ml/connect/evaluation.py b/python/pyspark/ml/connect/evaluation.py index 15af4dc3aa78a..eec05e03dae93 100644 --- a/python/pyspark/ml/connect/evaluation.py +++ b/python/pyspark/ml/connect/evaluation.py @@ -19,7 +19,6 @@ import numpy as np import pandas as pd -from pyspark import keyword_only from pyspark.ml.param import Param, Params, TypeConverters from pyspark.ml.param.shared import HasLabelCol, HasPredictionCol, HasProbabilityCol from pyspark.ml.connect.base import Evaluator @@ -107,7 +106,6 @@ class RegressionEvaluator(_TorchMetricEvaluator, HasLabelCol, HasPredictionCol, False """ - @keyword_only def __init__( self, *, @@ -175,7 +173,6 @@ class BinaryClassificationEvaluator( True """ - @keyword_only def __init__( self, *, diff --git a/python/pyspark/ml/connect/feature.py b/python/pyspark/ml/connect/feature.py index e618a6af5ae1e..fc094eade33d1 100644 --- a/python/pyspark/ml/connect/feature.py +++ b/python/pyspark/ml/connect/feature.py @@ -21,7 +21,6 @@ import numpy as np import pandas as pd -from pyspark import keyword_only from pyspark.sql import DataFrame from pyspark.ml.param.shared import ( HasInputCol, @@ -69,7 +68,6 @@ class MaxAbsScaler(Estimator, HasInputCol, HasOutputCol, ParamsReadWrite): _input_kwargs: Dict[str, Any] - @keyword_only def __init__(self, *, inputCol: Optional[str] = None, outputCol: Optional[str] = None) -> None: """ __init__(self, \\*, inputCol=None, outputCol=None) @@ -181,7 +179,6 @@ class StandardScaler(Estimator, HasInputCol, HasOutputCol, ParamsReadWrite): _input_kwargs: Dict[str, Any] - @keyword_only def __init__(self, inputCol: Optional[str] = None, outputCol: Optional[str] = None) -> None: """ __init__(self, \\*, inputCol=None, outputCol=None) @@ -327,7 +324,6 @@ class ArrayAssembler( typeConverter=TypeConverters.toString, ) - @keyword_only def __init__( self, *, diff --git a/python/pyspark/ml/connect/pipeline.py b/python/pyspark/ml/connect/pipeline.py index 32f44fc8007ad..b7c8e2b34804c 100644 --- a/python/pyspark/ml/connect/pipeline.py +++ b/python/pyspark/ml/connect/pipeline.py @@ -18,7 +18,7 @@ import pandas as pd -from pyspark import keyword_only, since +from pyspark import since from pyspark.ml.connect.base import Estimator, Model, Transformer from pyspark.ml.connect.io_utils import ( ParamsReadWrite, @@ -137,7 +137,6 @@ class Pipeline(Estimator["PipelineModel"], _PipelineReadWrite): _input_kwargs: Dict[str, Any] - @keyword_only def __init__(self, *, stages: Optional[List[Params]] = None): """ __init__(self, \\*, stages=None) @@ -172,7 +171,6 @@ def getStages(self) -> List[Params]: """ return self.getOrDefault(self.stages) - @keyword_only @since("3.5.0") def setParams(self, *, stages: Optional[List[Params]] = None) -> "Pipeline": """ diff --git a/python/pyspark/ml/connect/tuning.py b/python/pyspark/ml/connect/tuning.py index 97106646f74bc..e8a7e250e30a0 100644 --- a/python/pyspark/ml/connect/tuning.py +++ b/python/pyspark/ml/connect/tuning.py @@ -32,7 +32,7 @@ import numpy as np import pandas as pd -from pyspark import keyword_only, since, inheritable_thread_target +from pyspark import since, inheritable_thread_target from pyspark.ml.connect import Estimator, Model from pyspark.ml.connect.base import Evaluator from pyspark.ml.connect.io_utils import ( @@ -306,7 +306,6 @@ class CrossValidator( _input_kwargs: Dict[str, Any] - @keyword_only def __init__( self, *, @@ -327,7 +326,6 @@ def __init__( kwargs = self._input_kwargs self._set(**kwargs) - @keyword_only @since("3.5.0") def setParams( self, diff --git a/python/pyspark/ml/evaluation.py b/python/pyspark/ml/evaluation.py index c6445c7f02414..eb1e50bcadbc1 100644 --- a/python/pyspark/ml/evaluation.py +++ b/python/pyspark/ml/evaluation.py @@ -19,7 +19,7 @@ from abc import abstractmethod, ABCMeta from typing import Any, Dict, Optional, TYPE_CHECKING -from pyspark import since, keyword_only +from pyspark import since from pyspark.ml.wrapper import JavaParams from pyspark.ml.param import Param, Params, TypeConverters from pyspark.ml.param.shared import ( @@ -220,7 +220,6 @@ class BinaryClassificationEvaluator( _input_kwargs: Dict[str, Any] - @keyword_only def __init__( self, *, @@ -291,7 +290,6 @@ def setWeightCol(self, value: str) -> "BinaryClassificationEvaluator": """ return self._set(weightCol=value) - @keyword_only @since("1.4.0") def setParams( self, @@ -378,7 +376,6 @@ class RegressionEvaluator( _input_kwargs: Dict[str, Any] - @keyword_only def __init__( self, *, @@ -447,7 +444,6 @@ def setWeightCol(self, value: str) -> "RegressionEvaluator": """ return self._set(weightCol=value) - @keyword_only @since("1.4.0") def setParams( self, @@ -566,7 +562,6 @@ class MulticlassClassificationEvaluator( _input_kwargs: Dict[str, Any] - @keyword_only def __init__( self, *, @@ -676,7 +671,6 @@ def setWeightCol(self, value: str) -> "MulticlassClassificationEvaluator": """ return self._set(weightCol=value) - @keyword_only @since("1.5.0") def setParams( self, @@ -759,7 +753,6 @@ class MultilabelClassificationEvaluator( _input_kwargs: Dict[str, Any] - @keyword_only def __init__( self, *, @@ -824,7 +817,6 @@ def setPredictionCol(self, value: str) -> "MultilabelClassificationEvaluator": """ return self._set(predictionCol=value) - @keyword_only @since("3.0.0") def setParams( self, @@ -911,7 +903,6 @@ class ClusteringEvaluator( _input_kwargs: Dict[str, Any] - @keyword_only def __init__( self, *, @@ -933,7 +924,6 @@ def __init__( kwargs = self._input_kwargs self._set(**kwargs) - @keyword_only @since("2.3.0") def setParams( self, @@ -1056,7 +1046,6 @@ class RankingEvaluator( _input_kwargs: Dict[str, Any] - @keyword_only def __init__( self, *, @@ -1119,7 +1108,6 @@ def setPredictionCol(self, value: str) -> "RankingEvaluator": """ return self._set(predictionCol=value) - @keyword_only @since("3.0.0") def setParams( self, diff --git a/python/pyspark/ml/feature.py b/python/pyspark/ml/feature.py index 349b50913d7db..17d5f4e8f219c 100755 --- a/python/pyspark/ml/feature.py +++ b/python/pyspark/ml/feature.py @@ -28,7 +28,7 @@ TYPE_CHECKING, ) -from pyspark import keyword_only, since, SparkContext +from pyspark import since, SparkContext from pyspark.ml.linalg import _convert_to_vector, DenseMatrix, DenseVector, Vector from pyspark.sql.dataframe import DataFrame from pyspark.ml.param.shared import ( @@ -217,7 +217,6 @@ def __init__( ): ... - @keyword_only def __init__( self, *, @@ -258,7 +257,6 @@ def setParams( ) -> "Binarizer": ... - @keyword_only @since("1.4.0") def setParams( self, @@ -571,7 +569,6 @@ class BucketedRandomProjectionLSH( _input_kwargs: Dict[str, Any] - @keyword_only def __init__( self, *, @@ -592,7 +589,6 @@ def __init__( kwargs = self._input_kwargs self.setParams(**kwargs) - @keyword_only @since("2.2.0") def setParams( self, @@ -786,7 +782,6 @@ def __init__( ): ... - @keyword_only def __init__( self, *, @@ -830,7 +825,6 @@ def setParams( ) -> "Bucketizer": ... - @keyword_only @since("1.4.0") def setParams( self, @@ -1071,7 +1065,6 @@ class CountVectorizer( _input_kwargs: Dict[str, Any] - @keyword_only def __init__( self, *, @@ -1092,7 +1085,6 @@ def __init__( kwargs = self._input_kwargs self.setParams(**kwargs) - @keyword_only @since("1.6.0") def setParams( self, @@ -1293,7 +1285,6 @@ class DCT(JavaTransformer, HasInputCol, HasOutputCol, JavaMLReadable["DCT"], Jav typeConverter=TypeConverters.toBoolean, ) - @keyword_only def __init__( self, *, @@ -1310,7 +1301,6 @@ def __init__( kwargs = self._input_kwargs self.setParams(**kwargs) - @keyword_only @since("1.6.0") def setParams( self, @@ -1401,7 +1391,6 @@ class ElementwiseProduct( typeConverter=TypeConverters.toVector, ) - @keyword_only def __init__( self, *, @@ -1419,7 +1408,6 @@ def __init__( kwargs = self._input_kwargs self.setParams(**kwargs) - @keyword_only @since("1.5.0") def setParams( self, @@ -1537,7 +1525,6 @@ class FeatureHasher( typeConverter=TypeConverters.toListString, ) - @keyword_only def __init__( self, *, @@ -1556,7 +1543,6 @@ def __init__( kwargs = self._input_kwargs self.setParams(**kwargs) - @keyword_only @since("2.3.0") def setParams( self, @@ -1661,7 +1647,6 @@ class HashingTF( typeConverter=TypeConverters.toBoolean, ) - @keyword_only def __init__( self, *, @@ -1679,7 +1664,6 @@ def __init__( kwargs = self._input_kwargs self.setParams(**kwargs) - @keyword_only @since("1.3.0") def setParams( self, @@ -1813,7 +1797,6 @@ class IDF(JavaEstimator["IDFModel"], _IDFParams, JavaMLReadable["IDF"], JavaMLWr _input_kwargs: Dict[str, Any] - @keyword_only def __init__( self, *, @@ -1829,7 +1812,6 @@ def __init__( kwargs = self._input_kwargs self.setParams(**kwargs) - @keyword_only @since("1.4.0") def setParams( self, @@ -2101,7 +2083,6 @@ def __init__( ): ... - @keyword_only def __init__( self, *, @@ -2146,7 +2127,6 @@ def setParams( ) -> "Imputer": ... - @keyword_only @since("2.2.0") def setParams( self, @@ -2310,7 +2290,6 @@ class Interaction( _input_kwargs: Dict[str, Any] - @keyword_only def __init__(self, *, inputCols: Optional[List[str]] = None, outputCol: Optional[str] = None): """ __init__(self, \\*, inputCols=None, outputCol=None): @@ -2321,7 +2300,6 @@ def __init__(self, *, inputCols: Optional[List[str]] = None, outputCol: Optional kwargs = self._input_kwargs self.setParams(**kwargs) - @keyword_only @since("3.0.0") def setParams( self, *, inputCols: Optional[List[str]] = None, outputCol: Optional[str] = None @@ -2408,7 +2386,6 @@ class MaxAbsScaler( _input_kwargs: Dict[str, Any] - @keyword_only def __init__(self, *, inputCol: Optional[str] = None, outputCol: Optional[str] = None): """ __init__(self, \\*, inputCol=None, outputCol=None) @@ -2419,7 +2396,6 @@ def __init__(self, *, inputCol: Optional[str] = None, outputCol: Optional[str] = kwargs = self._input_kwargs self.setParams(**kwargs) - @keyword_only @since("2.0.0") def setParams( self, *, inputCol: Optional[str] = None, outputCol: Optional[str] = None @@ -2554,7 +2530,6 @@ class MinHashLSH( _input_kwargs: Dict[str, Any] - @keyword_only def __init__( self, *, @@ -2571,7 +2546,6 @@ def __init__( kwargs = self._input_kwargs self.setParams(**kwargs) - @keyword_only @since("2.2.0") def setParams( self, @@ -2719,7 +2693,6 @@ class MinMaxScaler( _input_kwargs: Dict[str, Any] - @keyword_only def __init__( self, *, @@ -2736,7 +2709,6 @@ def __init__( kwargs = self._input_kwargs self.setParams(**kwargs) - @keyword_only @since("1.6.0") def setParams( self, @@ -2891,7 +2863,6 @@ class NGram(JavaTransformer, HasInputCol, HasOutputCol, JavaMLReadable["NGram"], typeConverter=TypeConverters.toInt, ) - @keyword_only def __init__( self, *, n: int = 2, inputCol: Optional[str] = None, outputCol: Optional[str] = None ): @@ -2904,7 +2875,6 @@ def __init__( kwargs = self._input_kwargs self.setParams(**kwargs) - @keyword_only @since("1.5.0") def setParams( self, *, n: int = 2, inputCol: Optional[str] = None, outputCol: Optional[str] = None @@ -2986,7 +2956,6 @@ class Normalizer( p = Param(Params._dummy(), "p", "the p norm value.", typeConverter=TypeConverters.toFloat) - @keyword_only def __init__( self, *, p: float = 2.0, inputCol: Optional[str] = None, outputCol: Optional[str] = None ): @@ -2999,7 +2968,6 @@ def __init__( kwargs = self._input_kwargs self.setParams(**kwargs) - @keyword_only @since("1.4.0") def setParams( self, *, p: float = 2.0, inputCol: Optional[str] = None, outputCol: Optional[str] = None @@ -3170,7 +3138,6 @@ def __init__( ): ... - @keyword_only def __init__( self, *, @@ -3212,7 +3179,6 @@ def setParams( ) -> "OneHotEncoder": ... - @keyword_only @since("2.3.0") def setParams( self, @@ -3387,7 +3353,6 @@ class PolynomialExpansion( typeConverter=TypeConverters.toInt, ) - @keyword_only def __init__( self, *, degree: int = 2, inputCol: Optional[str] = None, outputCol: Optional[str] = None ): @@ -3402,7 +3367,6 @@ def __init__( kwargs = self._input_kwargs self.setParams(**kwargs) - @keyword_only @since("1.4.0") def setParams( self, *, degree: int = 2, inputCol: Optional[str] = None, outputCol: Optional[str] = None @@ -3605,7 +3569,6 @@ def __init__( ): ... - @keyword_only def __init__( self, *, @@ -3654,7 +3617,6 @@ def setParams( ) -> "QuantileDiscretizer": ... - @keyword_only @since("2.0.0") def setParams( self, @@ -3889,7 +3851,6 @@ class RobustScaler( _input_kwargs: Dict[str, Any] - @keyword_only def __init__( self, *, @@ -3910,7 +3871,6 @@ def __init__( kwargs = self._input_kwargs self.setParams(**kwargs) - @keyword_only @since("3.0.0") def setParams( self, @@ -4102,7 +4062,6 @@ class RegexTokenizer( typeConverter=TypeConverters.toBoolean, ) - @keyword_only def __init__( self, *, @@ -4123,7 +4082,6 @@ def __init__( kwargs = self._input_kwargs self.setParams(**kwargs) - @keyword_only @since("1.4.0") def setParams( self, @@ -4243,7 +4201,6 @@ class SQLTransformer(JavaTransformer, JavaMLReadable["SQLTransformer"], JavaMLWr Params._dummy(), "statement", "SQL statement", typeConverter=TypeConverters.toString ) - @keyword_only def __init__(self, *, statement: Optional[str] = None): """ __init__(self, \\*, statement=None) @@ -4253,7 +4210,6 @@ def __init__(self, *, statement: Optional[str] = None): kwargs = self._input_kwargs self.setParams(**kwargs) - @keyword_only @since("1.6.0") def setParams(self, *, statement: Optional[str] = None) -> "SQLTransformer": """ @@ -4371,7 +4327,6 @@ class StandardScaler( _input_kwargs: Dict[str, Any] - @keyword_only def __init__( self, *, @@ -4388,7 +4343,6 @@ def __init__( kwargs = self._input_kwargs self.setParams(**kwargs) - @keyword_only @since("1.4.0") def setParams( self, @@ -4631,7 +4585,6 @@ def __init__( ): ... - @keyword_only def __init__( self, *, @@ -4673,7 +4626,6 @@ def setParams( ) -> "StringIndexer": ... - @keyword_only @since("1.4.0") def setParams( self, @@ -4884,7 +4836,6 @@ class IndexToString( typeConverter=TypeConverters.toListString, ) - @keyword_only def __init__( self, *, @@ -4900,7 +4851,6 @@ def __init__( kwargs = self._input_kwargs self.setParams(**kwargs) - @keyword_only @since("1.6.0") def setParams( self, @@ -5041,7 +4991,6 @@ def __init__( ): ... - @keyword_only def __init__( self, *, @@ -5093,7 +5042,6 @@ def setParams( ) -> "StopWordsRemover": ... - @keyword_only @since("1.6.0") def setParams( self, @@ -5238,7 +5186,6 @@ class Tokenizer( _input_kwargs: Dict[str, Any] - @keyword_only def __init__(self, *, inputCol: Optional[str] = None, outputCol: Optional[str] = None): """ __init__(self, \\*, inputCol=None, outputCol=None) @@ -5248,7 +5195,6 @@ def __init__(self, *, inputCol: Optional[str] = None, outputCol: Optional[str] = kwargs = self._input_kwargs self.setParams(**kwargs) - @keyword_only @since("1.3.0") def setParams( self, *, inputCol: Optional[str] = None, outputCol: Optional[str] = None @@ -5343,7 +5289,6 @@ class VectorAssembler( typeConverter=TypeConverters.toString, ) - @keyword_only def __init__( self, *, @@ -5360,7 +5305,6 @@ def __init__( kwargs = self._input_kwargs self.setParams(**kwargs) - @keyword_only @since("1.4.0") def setParams( self, @@ -5531,7 +5475,6 @@ class VectorIndexer( _input_kwargs: Dict[str, Any] - @keyword_only def __init__( self, *, @@ -5548,7 +5491,6 @@ def __init__( kwargs = self._input_kwargs self.setParams(**kwargs) - @keyword_only @since("1.4.0") def setParams( self, @@ -5709,7 +5651,6 @@ class VectorSlicer( typeConverter=TypeConverters.toListString, ) - @keyword_only def __init__( self, *, @@ -5727,7 +5668,6 @@ def __init__( kwargs = self._input_kwargs self.setParams(**kwargs) - @keyword_only @since("1.6.0") def setParams( self, @@ -5946,7 +5886,6 @@ class Word2Vec( _input_kwargs: Dict[str, Any] - @keyword_only def __init__( self, *, @@ -5971,7 +5910,6 @@ def __init__( kwargs = self._input_kwargs self.setParams(**kwargs) - @keyword_only @since("1.4.0") def setParams( self, @@ -6187,7 +6125,6 @@ class PCA(JavaEstimator["PCAModel"], _PCAParams, JavaMLReadable["PCA"], JavaMLWr _input_kwargs: Dict[str, Any] - @keyword_only def __init__( self, *, @@ -6203,7 +6140,6 @@ def __init__( kwargs = self._input_kwargs self.setParams(**kwargs) - @keyword_only @since("1.5.0") def setParams( self, @@ -6431,7 +6367,6 @@ class RFormula( _input_kwargs: Dict[str, Any] - @keyword_only def __init__( self, *, @@ -6452,7 +6387,6 @@ def __init__( kwargs = self._input_kwargs self.setParams(**kwargs) - @keyword_only @since("1.5.0") def setParams( self, @@ -6807,7 +6741,6 @@ class ChiSqSelector( _input_kwargs: Dict[str, Any] - @keyword_only def __init__( self, *, @@ -6831,7 +6764,6 @@ def __init__( kwargs = self._input_kwargs self.setParams(**kwargs) - @keyword_only @since("2.0.0") def setParams( self, @@ -6926,7 +6858,6 @@ class VectorSizeHint( TypeConverters.toString, ) - @keyword_only def __init__( self, *, @@ -6942,7 +6873,6 @@ def __init__( self._setDefault(handleInvalid="error") self.setParams(**self._input_kwargs) - @keyword_only @since("2.3.0") def setParams( self, @@ -7058,7 +6988,6 @@ class VarianceThresholdSelector( _input_kwargs: Dict[str, Any] - @keyword_only def __init__( self, *, @@ -7077,7 +7006,6 @@ def __init__( kwargs = self._input_kwargs self.setParams(**kwargs) - @keyword_only @since("3.1.0") def setParams( self, @@ -7305,7 +7233,6 @@ class UnivariateFeatureSelector( _input_kwargs: Dict[str, Any] - @keyword_only def __init__( self, *, @@ -7325,7 +7252,6 @@ def __init__( kwargs = self._input_kwargs self.setParams(**kwargs) - @keyword_only @since("3.1.1") def setParams( self, diff --git a/python/pyspark/ml/fpm.py b/python/pyspark/ml/fpm.py index cba4219a0694b..b396826640565 100644 --- a/python/pyspark/ml/fpm.py +++ b/python/pyspark/ml/fpm.py @@ -18,7 +18,7 @@ import sys from typing import Any, Dict, Optional, TYPE_CHECKING -from pyspark import keyword_only, since +from pyspark import since from pyspark.sql import DataFrame from pyspark.ml.util import JavaMLWritable, JavaMLReadable from pyspark.ml.wrapper import JavaEstimator, JavaModel, JavaParams @@ -237,7 +237,6 @@ class FPGrowth( """ _input_kwargs: Dict[str, Any] - @keyword_only def __init__( self, *, @@ -256,7 +255,6 @@ def __init__( kwargs = self._input_kwargs self.setParams(**kwargs) - @keyword_only @since("2.2.0") def setParams( self, @@ -391,7 +389,6 @@ class PrefixSpan(JavaParams): typeConverter=TypeConverters.toString, ) - @keyword_only def __init__( self, *, @@ -412,7 +409,6 @@ def __init__( kwargs = self._input_kwargs self.setParams(**kwargs) - @keyword_only @since("2.4.0") def setParams( self, diff --git a/python/pyspark/ml/pipeline.py b/python/pyspark/ml/pipeline.py index 24653d1d919ee..71b570a5cd0ee 100644 --- a/python/pyspark/ml/pipeline.py +++ b/python/pyspark/ml/pipeline.py @@ -18,7 +18,7 @@ from typing import Any, Dict, List, Optional, Tuple, Type, Union, cast, TYPE_CHECKING -from pyspark import keyword_only, since, SparkContext +from pyspark import since, SparkContext from pyspark.ml.base import Estimator, Model, Transformer from pyspark.ml.param import Param, Params from pyspark.ml.util import ( @@ -70,7 +70,6 @@ class Pipeline(Estimator["PipelineModel"], MLReadable["Pipeline"], MLWritable): _input_kwargs: Dict[str, Any] - @keyword_only def __init__(self, *, stages: Optional[List["PipelineStage"]] = None): """ __init__(self, \\*, stages=None) @@ -105,7 +104,6 @@ def getStages(self) -> List["PipelineStage"]: """ return self.getOrDefault(self.stages) - @keyword_only @since("1.3.0") def setParams(self, *, stages: Optional[List["PipelineStage"]] = None) -> "Pipeline": """ diff --git a/python/pyspark/ml/recommendation.py b/python/pyspark/ml/recommendation.py index 873140e51afb8..114fd8dc73d3f 100644 --- a/python/pyspark/ml/recommendation.py +++ b/python/pyspark/ml/recommendation.py @@ -18,7 +18,7 @@ import sys from typing import Any, Dict, Optional, TYPE_CHECKING -from pyspark import since, keyword_only +from pyspark import since from pyspark.ml.param.shared import ( HasPredictionCol, HasBlockSize, @@ -366,7 +366,6 @@ class ALS(JavaEstimator["ALSModel"], _ALSParams, JavaMLWritable, JavaMLReadable[ _input_kwargs: Dict[str, Any] - @keyword_only def __init__( self, *, @@ -400,7 +399,6 @@ def __init__( kwargs = self._input_kwargs self.setParams(**kwargs) - @keyword_only @since("1.4.0") def setParams( self, diff --git a/python/pyspark/ml/regression.py b/python/pyspark/ml/regression.py index d08e241b41d23..bf9b004c193db 100644 --- a/python/pyspark/ml/regression.py +++ b/python/pyspark/ml/regression.py @@ -19,7 +19,7 @@ from typing import Any, Dict, Generic, List, Optional, TypeVar, TYPE_CHECKING from abc import ABCMeta -from pyspark import keyword_only, since +from pyspark import since from pyspark.ml import Predictor, PredictionModel from pyspark.ml.base import _PredictorParams from pyspark.ml.param.shared import ( @@ -296,7 +296,6 @@ class LinearRegression( _input_kwargs: Dict[str, Any] - @keyword_only def __init__( self, *, @@ -329,7 +328,6 @@ def __init__( kwargs = self._input_kwargs self.setParams(**kwargs) - @keyword_only @since("1.4.0") def setParams( self, @@ -858,7 +856,6 @@ class IsotonicRegression( _input_kwargs: Dict[str, Any] - @keyword_only def __init__( self, *, @@ -880,7 +877,6 @@ def __init__( kwargs = self._input_kwargs self.setParams(**kwargs) - @keyword_only def setParams( self, *, @@ -1108,7 +1104,6 @@ class DecisionTreeRegressor( _input_kwargs: Dict[str, Any] - @keyword_only def __init__( self, *, @@ -1143,7 +1138,6 @@ def __init__( kwargs = self._input_kwargs self.setParams(**kwargs) - @keyword_only @since("1.4.0") def setParams( self, @@ -1408,7 +1402,6 @@ class RandomForestRegressor( _input_kwargs: Dict[str, Any] - @keyword_only def __init__( self, *, @@ -1447,7 +1440,6 @@ def __init__( kwargs = self._input_kwargs self.setParams(**kwargs) - @keyword_only @since("1.4.0") def setParams( self, @@ -1751,7 +1743,6 @@ class GBTRegressor( _input_kwargs: Dict[str, Any] - @keyword_only def __init__( self, *, @@ -1792,7 +1783,6 @@ def __init__( kwargs = self._input_kwargs self.setParams(**kwargs) - @keyword_only @since("1.4.0") def setParams( self, @@ -2145,7 +2135,6 @@ class AFTSurvivalRegression( _input_kwargs: Dict[str, Any] - @keyword_only def __init__( self, *, @@ -2184,7 +2173,6 @@ def __init__( kwargs = self._input_kwargs self.setParams(**kwargs) - @keyword_only @since("1.6.0") def setParams( self, @@ -2550,7 +2538,6 @@ class GeneralizedLinearRegression( _input_kwargs: Dict[str, Any] - @keyword_only def __init__( self, *, @@ -2585,7 +2572,6 @@ def __init__( self.setParams(**kwargs) - @keyword_only @since("2.0.0") def setParams( self, @@ -3125,7 +3111,6 @@ class FMRegressor( _input_kwargs: Dict[str, Any] - @keyword_only def __init__( self, *, @@ -3155,7 +3140,6 @@ def __init__( kwargs = self._input_kwargs self.setParams(**kwargs) - @keyword_only @since("3.0.0") def setParams( self, diff --git a/python/pyspark/ml/tuning.py b/python/pyspark/ml/tuning.py index ae028b2f39969..220af8154d144 100644 --- a/python/pyspark/ml/tuning.py +++ b/python/pyspark/ml/tuning.py @@ -37,7 +37,7 @@ import numpy as np -from pyspark import keyword_only, since, SparkContext, inheritable_thread_target +from pyspark import since, SparkContext, inheritable_thread_target from pyspark.ml import Estimator, Transformer, Model from pyspark.ml.common import inherit_doc, _py2java, _java2py from pyspark.ml.evaluation import Evaluator, JavaEvaluator @@ -716,7 +716,6 @@ class CrossValidator( _input_kwargs: Dict[str, Any] - @keyword_only def __init__( self, *, @@ -738,7 +737,6 @@ def __init__( kwargs = self._input_kwargs self._set(**kwargs) - @keyword_only @since("1.4.0") def setParams( self, @@ -1347,7 +1345,6 @@ class TrainValidationSplit( _input_kwargs: Dict[str, Any] - @keyword_only def __init__( self, *, @@ -1369,7 +1366,6 @@ def __init__( self._set(**kwargs) @since("2.0.0") - @keyword_only def setParams( self, *, diff --git a/python/pyspark/testing/mlutils.py b/python/pyspark/testing/mlutils.py index 8981e97ea49ba..b7cbd9845f576 100644 --- a/python/pyspark/testing/mlutils.py +++ b/python/pyspark/testing/mlutils.py @@ -17,7 +17,6 @@ import numpy as np -from pyspark import keyword_only from pyspark.ml import Estimator, Model, Transformer, UnaryTransformer from pyspark.ml.evaluation import Evaluator from pyspark.ml.param import Param, Params, TypeConverters @@ -182,7 +181,6 @@ def setRegParam(self, value): class DummyLogisticRegression( Classifier, _DummyLogisticRegressionParams, DefaultParamsReadable, DefaultParamsWritable ): - @keyword_only def __init__( self, *, @@ -197,7 +195,6 @@ def __init__( kwargs = self._input_kwargs self.setParams(**kwargs) - @keyword_only def setParams( self, *, diff --git a/python/pyspark/tests/test_util.py b/python/pyspark/tests/test_util.py index af104d683aab8..a494d7b4dacd4 100644 --- a/python/pyspark/tests/test_util.py +++ b/python/pyspark/tests/test_util.py @@ -19,49 +19,10 @@ from py4j.protocol import Py4JJavaError -from pyspark import keyword_only from pyspark.testing.utils import PySparkTestCase, eventually from pyspark.find_spark_home import _find_spark_home -class KeywordOnlyTests(unittest.TestCase): - class Wrapped: - @keyword_only - def set(self, x=None, y=None): - if "x" in self._input_kwargs: - self._x = self._input_kwargs["x"] - if "y" in self._input_kwargs: - self._y = self._input_kwargs["y"] - return x, y - - def test_keywords(self): - w = self.Wrapped() - x, y = w.set(y=1) - self.assertEqual(y, 1) - self.assertEqual(y, w._y) - self.assertIsNone(x) - self.assertFalse(hasattr(w, "_x")) - - def test_non_keywords(self): - w = self.Wrapped() - self.assertRaises(TypeError, lambda: w.set(0, y=1)) - - def test_kwarg_ownership(self): - # test _input_kwargs is owned by each class instance and not a shared static variable - class Setter: - @keyword_only - def set(self, x=None, other=None, other_x=None): - if "other" in self._input_kwargs: - self._input_kwargs["other"].set(x=self._input_kwargs["other_x"]) - self._x = self._input_kwargs["x"] - - a = Setter() - b = Setter() - a.set(x=1, other=b, other_x=2) - self.assertEqual(a._x, 1) - self.assertEqual(b._x, 2) - - class UtilTests(PySparkTestCase): def test_py4j_str(self): with self.assertRaises(Py4JJavaError) as context: diff --git a/python/pyspark/tests/typing/test_core.yml b/python/pyspark/tests/typing/test_core.yml index ff58613492e91..1ad6b845e1306 100644 --- a/python/pyspark/tests/typing/test_core.yml +++ b/python/pyspark/tests/typing/test_core.yml @@ -17,4 +17,4 @@ - case: coreImports main: | - from pyspark import keyword_only, Row, SQLContext + from pyspark import Row, SQLContext From da4400215abcf0af4d7b6c3121ea5974aefb728e Mon Sep 17 00:00:00 2001 From: Hyukjin Kwon Date: Mon, 27 Nov 2023 09:59:14 +0900 Subject: [PATCH 2/7] Fix docs --- python/pyspark/ml/classification.py | 98 -------------- python/pyspark/ml/clustering.py | 47 ------- python/pyspark/ml/connect/feature.py | 11 -- python/pyspark/ml/connect/pipeline.py | 4 - python/pyspark/ml/connect/tuning.py | 6 - python/pyspark/ml/evaluation.py | 38 ------ python/pyspark/ml/feature.py | 186 -------------------------- python/pyspark/ml/fpm.py | 16 --- python/pyspark/ml/pipeline.py | 4 - python/pyspark/ml/recommendation.py | 12 -- python/pyspark/ml/regression.py | 90 ------------- python/pyspark/ml/tuning.py | 12 -- 12 files changed, 524 deletions(-) diff --git a/python/pyspark/ml/classification.py b/python/pyspark/ml/classification.py index fb7392e138f33..b32624ec30126 100644 --- a/python/pyspark/ml/classification.py +++ b/python/pyspark/ml/classification.py @@ -725,12 +725,6 @@ def __init__( aggregationDepth: int = 2, maxBlockSizeInMB: float = 0.0, ): - """ - __init__(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \ - maxIter=100, regParam=0.0, tol=1e-6, rawPredictionCol="rawPrediction", \ - fitIntercept=True, standardization=True, threshold=0.0, weightCol=None, \ - aggregationDepth=2, maxBlockSizeInMB=0.0): - """ super(LinearSVC, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.classification.LinearSVC", self.uid @@ -757,10 +751,6 @@ def setParams( maxBlockSizeInMB: float = 0.0, ) -> "LinearSVC": """ - setParams(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \ - maxIter=100, regParam=0.0, tol=1e-6, rawPredictionCol="rawPrediction", \ - fitIntercept=True, standardization=True, threshold=0.0, weightCol=None, \ - aggregationDepth=2, maxBlockSizeInMB=0.0): Sets params for Linear SVM Classifier. """ kwargs = self._input_kwargs @@ -1299,14 +1289,6 @@ def __init__( maxBlockSizeInMB: float = 0.0, ): """ - __init__(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \ - maxIter=100, regParam=0.0, elasticNetParam=0.0, tol=1e-6, fitIntercept=True, \ - threshold=0.5, thresholds=None, probabilityCol="probability", \ - rawPredictionCol="rawPrediction", standardization=True, weightCol=None, \ - aggregationDepth=2, family="auto", \ - lowerBoundsOnCoefficients=None, upperBoundsOnCoefficients=None, \ - lowerBoundsOnIntercepts=None, upperBoundsOnIntercepts=None, \ - maxBlockSizeInMB=0.0): If the threshold and thresholds Params are both set, they must be equivalent. """ super(LogisticRegression, self).__init__() @@ -1398,14 +1380,6 @@ def setParams( maxBlockSizeInMB: float = 0.0, ) -> "LogisticRegression": """ - setParams(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \ - maxIter=100, regParam=0.0, elasticNetParam=0.0, tol=1e-6, fitIntercept=True, \ - threshold=0.5, thresholds=None, probabilityCol="probability", \ - rawPredictionCol="rawPrediction", standardization=True, weightCol=None, \ - aggregationDepth=2, family="auto", \ - lowerBoundsOnCoefficients=None, upperBoundsOnCoefficients=None, \ - lowerBoundsOnIntercepts=None, upperBoundsOnIntercepts=None, \ - maxBlockSizeInMB=0.0): Sets params for logistic regression. If the threshold and thresholds Params are both set, they must be equivalent. """ @@ -1789,13 +1763,6 @@ def __init__( leafCol: str = "", minWeightFractionPerNode: float = 0.0, ): - """ - __init__(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \ - probabilityCol="probability", rawPredictionCol="rawPrediction", \ - maxDepth=5, maxBins=32, minInstancesPerNode=1, minInfoGain=0.0, \ - maxMemoryInMB=256, cacheNodeIds=False, checkpointInterval=10, impurity="gini", \ - seed=None, weightCol=None, leafCol="", minWeightFractionPerNode=0.0) - """ super(DecisionTreeClassifier, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.classification.DecisionTreeClassifier", self.uid @@ -1826,11 +1793,6 @@ def setParams( minWeightFractionPerNode: float = 0.0, ) -> "DecisionTreeClassifier": """ - setParams(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \ - probabilityCol="probability", rawPredictionCol="rawPrediction", \ - maxDepth=5, maxBins=32, minInstancesPerNode=1, minInfoGain=0.0, \ - maxMemoryInMB=256, cacheNodeIds=False, checkpointInterval=10, impurity="gini", \ - seed=None, weightCol=None, leafCol="", minWeightFractionPerNode=0.0) Sets params for the DecisionTreeClassifier. """ kwargs = self._input_kwargs @@ -2083,14 +2045,6 @@ def __init__( weightCol: Optional[str] = None, bootstrap: Optional[bool] = True, ): - """ - __init__(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \ - probabilityCol="probability", rawPredictionCol="rawPrediction", \ - maxDepth=5, maxBins=32, minInstancesPerNode=1, minInfoGain=0.0, \ - maxMemoryInMB=256, cacheNodeIds=False, checkpointInterval=10, impurity="gini", \ - numTrees=20, featureSubsetStrategy="auto", seed=None, subsamplingRate=1.0, \ - leafCol="", minWeightFractionPerNode=0.0, weightCol=None, bootstrap=True) - """ super(RandomForestClassifier, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.classification.RandomForestClassifier", self.uid @@ -2551,15 +2505,6 @@ def __init__( minWeightFractionPerNode: float = 0.0, weightCol: Optional[str] = None, ): - """ - __init__(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \ - maxDepth=5, maxBins=32, minInstancesPerNode=1, minInfoGain=0.0, \ - maxMemoryInMB=256, cacheNodeIds=False, checkpointInterval=10, \ - lossType="logistic", maxIter=20, stepSize=0.1, seed=None, subsamplingRate=1.0, \ - impurity="variance", featureSubsetStrategy="all", validationTol=0.01, \ - validationIndicatorCol=None, leafCol="", minWeightFractionPerNode=0.0, \ - weightCol=None) - """ super(GBTClassifier, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.classification.GBTClassifier", self.uid @@ -2595,13 +2540,6 @@ def setParams( weightCol: Optional[str] = None, ) -> "GBTClassifier": """ - setParams(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \ - maxDepth=5, maxBins=32, minInstancesPerNode=1, minInfoGain=0.0, \ - maxMemoryInMB=256, cacheNodeIds=False, checkpointInterval=10, \ - lossType="logistic", maxIter=20, stepSize=0.1, seed=None, subsamplingRate=1.0, \ - impurity="variance", featureSubsetStrategy="all", validationTol=0.01, \ - validationIndicatorCol=None, leafCol="", minWeightFractionPerNode=0.0, \ - weightCol=None) Sets params for Gradient Boosted Tree Classification. """ kwargs = self._input_kwargs @@ -2933,11 +2871,6 @@ def __init__( thresholds: Optional[List[float]] = None, weightCol: Optional[str] = None, ): - """ - __init__(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \ - probabilityCol="probability", rawPredictionCol="rawPrediction", smoothing=1.0, \ - modelType="multinomial", thresholds=None, weightCol=None) - """ super(NaiveBayes, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.classification.NaiveBayes", self.uid @@ -2960,9 +2893,6 @@ def setParams( weightCol: Optional[str] = None, ) -> "NaiveBayes": """ - setParams(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \ - probabilityCol="probability", rawPredictionCol="rawPrediction", smoothing=1.0, \ - modelType="multinomial", thresholds=None, weightCol=None) Sets params for Naive Bayes. """ kwargs = self._input_kwargs @@ -3185,12 +3115,6 @@ def __init__( probabilityCol: str = "probability", rawPredictionCol: str = "rawPrediction", ): - """ - __init__(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \ - maxIter=100, tol=1e-6, seed=None, layers=None, blockSize=128, stepSize=0.03, \ - solver="l-bfgs", initialWeights=None, probabilityCol="probability", \ - rawPredictionCol="rawPrediction") - """ super(MultilayerPerceptronClassifier, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.classification.MultilayerPerceptronClassifier", self.uid @@ -3217,10 +3141,6 @@ def setParams( rawPredictionCol: str = "rawPrediction", ) -> "MultilayerPerceptronClassifier": """ - setParams(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \ - maxIter=100, tol=1e-6, seed=None, layers=None, blockSize=128, stepSize=0.03, \ - solver="l-bfgs", initialWeights=None, probabilityCol="probability", \ - rawPredictionCol="rawPrediction"): Sets params for MultilayerPerceptronClassifier. """ kwargs = self._input_kwargs @@ -3445,10 +3365,6 @@ def __init__( weightCol: Optional[str] = None, parallelism: int = 1, ): - """ - __init__(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \ - rawPredictionCol="rawPrediction", classifier=None, weightCol=None, parallelism=1): - """ super(OneVsRest, self).__init__() self._setDefault(parallelism=1) kwargs = self._input_kwargs @@ -3467,8 +3383,6 @@ def setParams( parallelism: int = 1, ) -> "OneVsRest": """ - setParams(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \ - rawPredictionCol="rawPrediction", classifier=None, weightCol=None, parallelism=1): Sets params for OneVsRest. """ kwargs = self._input_kwargs @@ -4066,13 +3980,6 @@ def __init__( thresholds: Optional[List[float]] = None, seed: Optional[int] = None, ): - """ - __init__(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \ - probabilityCol="probability", rawPredictionCol="rawPrediction", \ - factorSize=8, fitIntercept=True, fitLinear=True, regParam=0.0, \ - miniBatchFraction=1.0, initStd=0.01, maxIter=100, stepSize=1.0, \ - tol=1e-6, solver="adamW", thresholds=None, seed=None) - """ super(FMClassifier, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.classification.FMClassifier", self.uid @@ -4103,11 +4010,6 @@ def setParams( seed: Optional[int] = None, ) -> "FMClassifier": """ - setParams(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \ - probabilityCol="probability", rawPredictionCol="rawPrediction", \ - factorSize=8, fitIntercept=True, fitLinear=True, regParam=0.0, \ - miniBatchFraction=1.0, initStd=0.01, maxIter=100, stepSize=1.0, \ - tol=1e-6, solver="adamW", thresholds=None, seed=None) Sets Params for FMClassifier. """ kwargs = self._input_kwargs diff --git a/python/pyspark/ml/clustering.py b/python/pyspark/ml/clustering.py index eb58fa1fac71b..bd95960fee4b5 100644 --- a/python/pyspark/ml/clustering.py +++ b/python/pyspark/ml/clustering.py @@ -416,11 +416,6 @@ def __init__( aggregationDepth: int = 2, weightCol: Optional[str] = None, ): - """ - __init__(self, \\*, featuresCol="features", predictionCol="prediction", k=2, \ - probabilityCol="probability", tol=0.01, maxIter=100, seed=None, \ - aggregationDepth=2, weightCol=None) - """ super(GaussianMixture, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.clustering.GaussianMixture", self.uid @@ -446,10 +441,6 @@ def setParams( weightCol: Optional[str] = None, ) -> "GaussianMixture": """ - setParams(self, \\*, featuresCol="features", predictionCol="prediction", k=2, \ - probabilityCol="probability", tol=0.01, maxIter=100, seed=None, \ - aggregationDepth=2, weightCol=None) - Sets params for GaussianMixture. """ kwargs = self._input_kwargs @@ -791,12 +782,6 @@ def __init__( solver: str = "auto", maxBlockSizeInMB: float = 0.0, ): - """ - __init__(self, \\*, featuresCol="features", predictionCol="prediction", k=2, \ - initMode="k-means||", initSteps=2, tol=1e-4, maxIter=20, seed=None, \ - distanceMeasure="euclidean", weightCol=None, solver="auto", \ - maxBlockSizeInMB=0.0) - """ super(KMeans, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.clustering.KMeans", self.uid) kwargs = self._input_kwargs @@ -823,11 +808,6 @@ def setParams( maxBlockSizeInMB: float = 0.0, ) -> "KMeans": """ - setParams(self, \\*, featuresCol="features", predictionCol="prediction", k=2, \ - initMode="k-means||", initSteps=2, tol=1e-4, maxIter=20, seed=None, \ - distanceMeasure="euclidean", weightCol=None, solver="auto", \ - maxBlockSizeInMB=0.0) - Sets params for KMeans. """ kwargs = self._input_kwargs @@ -1137,11 +1117,6 @@ def __init__( distanceMeasure: str = "euclidean", weightCol: Optional[str] = None, ): - """ - __init__(self, \\*, featuresCol="features", predictionCol="prediction", maxIter=20, \ - seed=None, k=4, minDivisibleClusterSize=1.0, distanceMeasure="euclidean", \ - weightCol=None) - """ super(BisectingKMeans, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.clustering.BisectingKMeans", self.uid @@ -1163,9 +1138,6 @@ def setParams( weightCol: Optional[str] = None, ) -> "BisectingKMeans": """ - setParams(self, \\*, featuresCol="features", predictionCol="prediction", maxIter=20, \ - seed=None, k=4, minDivisibleClusterSize=1.0, distanceMeasure="euclidean", \ - weightCol=None) Sets params for BisectingKMeans. """ kwargs = self._input_kwargs @@ -1693,13 +1665,6 @@ def __init__( topicDistributionCol: str = "topicDistribution", keepLastCheckpoint: bool = True, ): - """ - __init__(self, \\*, featuresCol="features", maxIter=20, seed=None, checkpointInterval=10,\ - k=10, optimizer="online", learningOffset=1024.0, learningDecay=0.51,\ - subsamplingRate=0.05, optimizeDocConcentration=True,\ - docConcentration=None, topicConcentration=None,\ - topicDistributionCol="topicDistribution", keepLastCheckpoint=True) - """ super(LDA, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.clustering.LDA", self.uid) kwargs = self._input_kwargs @@ -1731,12 +1696,6 @@ def setParams( keepLastCheckpoint: bool = True, ) -> "LDA": """ - setParams(self, \\*, featuresCol="features", maxIter=20, seed=None, checkpointInterval=10,\ - k=10, optimizer="online", learningOffset=1024.0, learningDecay=0.51,\ - subsamplingRate=0.05, optimizeDocConcentration=True,\ - docConcentration=None, topicConcentration=None,\ - topicDistributionCol="topicDistribution", keepLastCheckpoint=True) - Sets params for LDA. """ kwargs = self._input_kwargs @@ -2038,10 +1997,6 @@ def __init__( dstCol: str = "dst", weightCol: Optional[str] = None, ): - """ - __init__(self, \\*, k=2, maxIter=20, initMode="random", srcCol="src", dstCol="dst",\ - weightCol=None) - """ super(PowerIterationClustering, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.clustering.PowerIterationClustering", self.uid @@ -2061,8 +2016,6 @@ def setParams( weightCol: Optional[str] = None, ) -> "PowerIterationClustering": """ - setParams(self, \\*, k=2, maxIter=20, initMode="random", srcCol="src", dstCol="dst",\ - weightCol=None) Sets params for PowerIterationClustering. """ kwargs = self._input_kwargs diff --git a/python/pyspark/ml/connect/feature.py b/python/pyspark/ml/connect/feature.py index fc094eade33d1..e474f502feac0 100644 --- a/python/pyspark/ml/connect/feature.py +++ b/python/pyspark/ml/connect/feature.py @@ -69,9 +69,6 @@ class MaxAbsScaler(Estimator, HasInputCol, HasOutputCol, ParamsReadWrite): _input_kwargs: Dict[str, Any] def __init__(self, *, inputCol: Optional[str] = None, outputCol: Optional[str] = None) -> None: - """ - __init__(self, \\*, inputCol=None, outputCol=None) - """ super().__init__() kwargs = self._input_kwargs self._set(**kwargs) @@ -180,9 +177,6 @@ class StandardScaler(Estimator, HasInputCol, HasOutputCol, ParamsReadWrite): _input_kwargs: Dict[str, Any] def __init__(self, inputCol: Optional[str] = None, outputCol: Optional[str] = None) -> None: - """ - __init__(self, \\*, inputCol=None, outputCol=None) - """ super().__init__() kwargs = self._input_kwargs self._set(**kwargs) @@ -332,11 +326,6 @@ def __init__( featureSizes: Optional[List[int]] = None, handleInvalid: Optional[str] = "error", ) -> None: - """ - __init__( - self, \\*, inputCols=None, outputCol=None, featureSizes=None, handleInvalid="error" - ) - """ super().__init__() kwargs = self._input_kwargs self._set(**kwargs) diff --git a/python/pyspark/ml/connect/pipeline.py b/python/pyspark/ml/connect/pipeline.py index b7c8e2b34804c..9a5bd9372b1c2 100644 --- a/python/pyspark/ml/connect/pipeline.py +++ b/python/pyspark/ml/connect/pipeline.py @@ -138,9 +138,6 @@ class Pipeline(Estimator["PipelineModel"], _PipelineReadWrite): _input_kwargs: Dict[str, Any] def __init__(self, *, stages: Optional[List[Params]] = None): - """ - __init__(self, \\*, stages=None) - """ super(Pipeline, self).__init__() kwargs = self._input_kwargs self.setParams(**kwargs) @@ -174,7 +171,6 @@ def getStages(self) -> List[Params]: @since("3.5.0") def setParams(self, *, stages: Optional[List[Params]] = None) -> "Pipeline": """ - setParams(self, \\*, stages=None) Sets params for Pipeline. """ kwargs = self._input_kwargs diff --git a/python/pyspark/ml/connect/tuning.py b/python/pyspark/ml/connect/tuning.py index e8a7e250e30a0..81014c89abb75 100644 --- a/python/pyspark/ml/connect/tuning.py +++ b/python/pyspark/ml/connect/tuning.py @@ -317,10 +317,6 @@ def __init__( parallelism: int = 1, foldCol: str = "", ) -> None: - """ - __init__(self, \\*, estimator=None, estimatorParamMaps=None, evaluator=None, numFolds=3,\ - seed=None, parallelism=1, foldCol="") - """ super(CrossValidator, self).__init__() self._setDefault(parallelism=1) kwargs = self._input_kwargs @@ -339,8 +335,6 @@ def setParams( foldCol: str = "", ) -> "CrossValidator": """ - setParams(self, \\*, estimator=None, estimatorParamMaps=None, evaluator=None, numFolds=3,\ - seed=None, parallelism=1, collectSubModels=False, foldCol=""): Sets params for cross validator. """ kwargs = self._input_kwargs diff --git a/python/pyspark/ml/evaluation.py b/python/pyspark/ml/evaluation.py index eb1e50bcadbc1..c942685b35e3e 100644 --- a/python/pyspark/ml/evaluation.py +++ b/python/pyspark/ml/evaluation.py @@ -229,10 +229,6 @@ def __init__( weightCol: Optional[str] = None, numBins: int = 1000, ): - """ - __init__(self, \\*, rawPredictionCol="rawPrediction", labelCol="label", \ - metricName="areaUnderROC", weightCol=None, numBins=1000) - """ super(BinaryClassificationEvaluator, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.evaluation.BinaryClassificationEvaluator", self.uid @@ -301,8 +297,6 @@ def setParams( numBins: int = 1000, ) -> "BinaryClassificationEvaluator": """ - setParams(self, \\*, rawPredictionCol="rawPrediction", labelCol="label", \ - metricName="areaUnderROC", weightCol=None, numBins=1000) Sets params for binary classification evaluator. """ kwargs = self._input_kwargs @@ -385,10 +379,6 @@ def __init__( weightCol: Optional[str] = None, throughOrigin: bool = False, ): - """ - __init__(self, \\*, predictionCol="prediction", labelCol="label", \ - metricName="rmse", weightCol=None, throughOrigin=False) - """ super(RegressionEvaluator, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.evaluation.RegressionEvaluator", self.uid @@ -455,8 +445,6 @@ def setParams( throughOrigin: bool = False, ) -> "RegressionEvaluator": """ - setParams(self, \\*, predictionCol="prediction", labelCol="label", \ - metricName="rmse", weightCol=None, throughOrigin=False) Sets params for regression evaluator. """ kwargs = self._input_kwargs @@ -574,11 +562,6 @@ def __init__( probabilityCol: str = "probability", eps: float = 1e-15, ): - """ - __init__(self, \\*, predictionCol="prediction", labelCol="label", \ - metricName="f1", weightCol=None, metricLabel=0.0, beta=1.0, \ - probabilityCol="probability", eps=1e-15) - """ super(MulticlassClassificationEvaluator, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.evaluation.MulticlassClassificationEvaluator", self.uid @@ -685,9 +668,6 @@ def setParams( eps: float = 1e-15, ) -> "MulticlassClassificationEvaluator": """ - setParams(self, \\*, predictionCol="prediction", labelCol="label", \ - metricName="f1", weightCol=None, metricLabel=0.0, beta=1.0, \ - probabilityCol="probability", eps=1e-15) Sets params for multiclass classification evaluator. """ kwargs = self._input_kwargs @@ -761,10 +741,6 @@ def __init__( metricName: "MultilabelClassificationEvaluatorMetricType" = "f1Measure", metricLabel: float = 0.0, ) -> None: - """ - __init__(self, \\*, predictionCol="prediction", labelCol="label", \ - metricName="f1Measure", metricLabel=0.0) - """ super(MultilabelClassificationEvaluator, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.evaluation.MultilabelClassificationEvaluator", self.uid @@ -827,8 +803,6 @@ def setParams( metricLabel: float = 0.0, ) -> "MultilabelClassificationEvaluator": """ - setParams(self, \\*, predictionCol="prediction", labelCol="label", \ - metricName="f1Measure", metricLabel=0.0) Sets params for multilabel classification evaluator. """ kwargs = self._input_kwargs @@ -912,10 +886,6 @@ def __init__( distanceMeasure: str = "squaredEuclidean", weightCol: Optional[str] = None, ): - """ - __init__(self, \\*, predictionCol="prediction", featuresCol="features", \ - metricName="silhouette", distanceMeasure="squaredEuclidean", weightCol=None) - """ super(ClusteringEvaluator, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.evaluation.ClusteringEvaluator", self.uid @@ -935,8 +905,6 @@ def setParams( weightCol: Optional[str] = None, ) -> "ClusteringEvaluator": """ - setParams(self, \\*, predictionCol="prediction", featuresCol="features", \ - metricName="silhouette", distanceMeasure="squaredEuclidean", weightCol=None) Sets params for clustering evaluator. """ kwargs = self._input_kwargs @@ -1054,10 +1022,6 @@ def __init__( metricName: "RankingEvaluatorMetricType" = "meanAveragePrecision", k: int = 10, ): - """ - __init__(self, \\*, predictionCol="prediction", labelCol="label", \ - metricName="meanAveragePrecision", k=10) - """ super(RankingEvaluator, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.evaluation.RankingEvaluator", self.uid @@ -1118,8 +1082,6 @@ def setParams( k: int = 10, ) -> "RankingEvaluator": """ - setParams(self, \\*, predictionCol="prediction", labelCol="label", \ - metricName="meanAveragePrecision", k=10) Sets params for ranking evaluator. """ kwargs = self._input_kwargs diff --git a/python/pyspark/ml/feature.py b/python/pyspark/ml/feature.py index 17d5f4e8f219c..7a5bff9edf219 100755 --- a/python/pyspark/ml/feature.py +++ b/python/pyspark/ml/feature.py @@ -227,10 +227,6 @@ def __init__( inputCols: Optional[List[str]] = None, outputCols: Optional[List[str]] = None, ): - """ - __init__(self, \\*, threshold=0.0, inputCol=None, outputCol=None, thresholds=None, \ - inputCols=None, outputCols=None) - """ super(Binarizer, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.Binarizer", self.uid) self._setDefault(threshold=0.0) @@ -269,8 +265,6 @@ def setParams( outputCols: Optional[List[str]] = None, ) -> "Binarizer": """ - setParams(self, \\*, threshold=0.0, inputCol=None, outputCol=None, thresholds=None, \ - inputCols=None, outputCols=None) Sets params for this Binarizer. """ kwargs = self._input_kwargs @@ -578,10 +572,6 @@ def __init__( numHashTables: int = 1, bucketLength: Optional[float] = None, ): - """ - __init__(self, \\*, inputCol=None, outputCol=None, seed=None, numHashTables=1, \ - bucketLength=None) - """ super(BucketedRandomProjectionLSH, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.feature.BucketedRandomProjectionLSH", self.uid @@ -600,8 +590,6 @@ def setParams( bucketLength: Optional[float] = None, ) -> "BucketedRandomProjectionLSH": """ - setParams(self, \\*, inputCol=None, outputCol=None, seed=None, numHashTables=1, \ - bucketLength=None) Sets params for this BucketedRandomProjectionLSH. """ kwargs = self._input_kwargs @@ -793,10 +781,6 @@ def __init__( inputCols: Optional[List[str]] = None, outputCols: Optional[List[str]] = None, ): - """ - __init__(self, \\*, splits=None, inputCol=None, outputCol=None, handleInvalid="error", \ - splitsArray=None, inputCols=None, outputCols=None) - """ super(Bucketizer, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.Bucketizer", self.uid) self._setDefault(handleInvalid="error") @@ -838,8 +822,6 @@ def setParams( outputCols: Optional[List[str]] = None, ) -> "Bucketizer": """ - setParams(self, \\*, splits=None, inputCol=None, outputCol=None, handleInvalid="error", \ - splitsArray=None, inputCols=None, outputCols=None) Sets params for this Bucketizer. """ kwargs = self._input_kwargs @@ -1076,10 +1058,6 @@ def __init__( inputCol: Optional[str] = None, outputCol: Optional[str] = None, ): - """ - __init__(self, \\*, minTF=1.0, minDF=1.0, maxDF=2 ** 63 - 1, vocabSize=1 << 18,\ - binary=False, inputCol=None,outputCol=None) - """ super(CountVectorizer, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.CountVectorizer", self.uid) kwargs = self._input_kwargs @@ -1098,8 +1076,6 @@ def setParams( outputCol: Optional[str] = None, ) -> "CountVectorizer": """ - setParams(self, \\*, minTF=1.0, minDF=1.0, maxDF=2 ** 63 - 1, vocabSize=1 << 18,\ - binary=False, inputCol=None, outputCol=None) Set the params for the CountVectorizer """ kwargs = self._input_kwargs @@ -1292,9 +1268,6 @@ def __init__( inputCol: Optional[str] = None, outputCol: Optional[str] = None, ): - """ - __init__(self, \\*, inverse=False, inputCol=None, outputCol=None) - """ super(DCT, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.DCT", self.uid) self._setDefault(inverse=False) @@ -1310,7 +1283,6 @@ def setParams( outputCol: Optional[str] = None, ) -> "DCT": """ - setParams(self, \\*, inverse=False, inputCol=None, outputCol=None) Sets params for this DCT. """ kwargs = self._input_kwargs @@ -1398,9 +1370,6 @@ def __init__( inputCol: Optional[str] = None, outputCol: Optional[str] = None, ): - """ - __init__(self, \\*, scalingVec=None, inputCol=None, outputCol=None) - """ super(ElementwiseProduct, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.feature.ElementwiseProduct", self.uid @@ -1417,7 +1386,6 @@ def setParams( outputCol: Optional[str] = None, ) -> "ElementwiseProduct": """ - setParams(self, \\*, scalingVec=None, inputCol=None, outputCol=None) Sets params for this ElementwiseProduct. """ kwargs = self._input_kwargs @@ -1533,10 +1501,6 @@ def __init__( outputCol: Optional[str] = None, categoricalCols: Optional[List[str]] = None, ): - """ - __init__(self, \\*, numFeatures=1 << 18, inputCols=None, outputCol=None, \ - categoricalCols=None) - """ super(FeatureHasher, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.FeatureHasher", self.uid) self._setDefault(numFeatures=1 << 18) @@ -1553,8 +1517,6 @@ def setParams( categoricalCols: Optional[List[str]] = None, ) -> "FeatureHasher": """ - setParams(self, \\*, numFeatures=1 << 18, inputCols=None, outputCol=None, \ - categoricalCols=None) Sets params for this FeatureHasher. """ kwargs = self._input_kwargs @@ -1655,9 +1617,6 @@ def __init__( inputCol: Optional[str] = None, outputCol: Optional[str] = None, ): - """ - __init__(self, \\*, numFeatures=1 << 18, binary=False, inputCol=None, outputCol=None) - """ super(HashingTF, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.HashingTF", self.uid) self._setDefault(numFeatures=1 << 18, binary=False) @@ -1674,7 +1633,6 @@ def setParams( outputCol: Optional[str] = None, ) -> "HashingTF": """ - setParams(self, \\*, numFeatures=1 << 18, binary=False, inputCol=None, outputCol=None) Sets params for this HashingTF. """ kwargs = self._input_kwargs @@ -1804,9 +1762,6 @@ def __init__( inputCol: Optional[str] = None, outputCol: Optional[str] = None, ): - """ - __init__(self, \\*, minDocFreq=0, inputCol=None, outputCol=None) - """ super(IDF, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.IDF", self.uid) kwargs = self._input_kwargs @@ -1821,7 +1776,6 @@ def setParams( outputCol: Optional[str] = None, ) -> "IDF": """ - setParams(self, \\*, minDocFreq=0, inputCol=None, outputCol=None) Sets params for this IDF. """ kwargs = self._input_kwargs @@ -2094,10 +2048,6 @@ def __init__( outputCol: Optional[str] = None, relativeError: float = 0.001, ): - """ - __init__(self, \\*, strategy="mean", missingValue=float("nan"), inputCols=None, \ - outputCols=None, inputCol=None, outputCol=None, relativeError=0.001): - """ super(Imputer, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.Imputer", self.uid) kwargs = self._input_kwargs @@ -2140,8 +2090,6 @@ def setParams( relativeError: float = 0.001, ) -> "Imputer": """ - setParams(self, \\*, strategy="mean", missingValue=float("nan"), inputCols=None, \ - outputCols=None, inputCol=None, outputCol=None, relativeError=0.001) Sets params for this Imputer. """ kwargs = self._input_kwargs @@ -2291,9 +2239,6 @@ class Interaction( _input_kwargs: Dict[str, Any] def __init__(self, *, inputCols: Optional[List[str]] = None, outputCol: Optional[str] = None): - """ - __init__(self, \\*, inputCols=None, outputCol=None): - """ super(Interaction, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.Interaction", self.uid) self._setDefault() @@ -2305,7 +2250,6 @@ def setParams( self, *, inputCols: Optional[List[str]] = None, outputCol: Optional[str] = None ) -> "Interaction": """ - setParams(self, \\*, inputCols=None, outputCol=None) Sets params for this Interaction. """ kwargs = self._input_kwargs @@ -2387,9 +2331,6 @@ class MaxAbsScaler( _input_kwargs: Dict[str, Any] def __init__(self, *, inputCol: Optional[str] = None, outputCol: Optional[str] = None): - """ - __init__(self, \\*, inputCol=None, outputCol=None) - """ super(MaxAbsScaler, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.MaxAbsScaler", self.uid) self._setDefault() @@ -2401,7 +2342,6 @@ def setParams( self, *, inputCol: Optional[str] = None, outputCol: Optional[str] = None ) -> "MaxAbsScaler": """ - setParams(self, \\*, inputCol=None, outputCol=None) Sets params for this MaxAbsScaler. """ kwargs = self._input_kwargs @@ -2538,9 +2478,6 @@ def __init__( seed: Optional[int] = None, numHashTables: int = 1, ): - """ - __init__(self, \\*, inputCol=None, outputCol=None, seed=None, numHashTables=1) - """ super(MinHashLSH, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.MinHashLSH", self.uid) kwargs = self._input_kwargs @@ -2556,7 +2493,6 @@ def setParams( numHashTables: int = 1, ) -> "MinHashLSH": """ - setParams(self, \\*, inputCol=None, outputCol=None, seed=None, numHashTables=1) Sets params for this MinHashLSH. """ kwargs = self._input_kwargs @@ -2701,9 +2637,6 @@ def __init__( inputCol: Optional[str] = None, outputCol: Optional[str] = None, ): - """ - __init__(self, \\*, min=0.0, max=1.0, inputCol=None, outputCol=None) - """ super(MinMaxScaler, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.MinMaxScaler", self.uid) kwargs = self._input_kwargs @@ -2719,7 +2652,6 @@ def setParams( outputCol: Optional[str] = None, ) -> "MinMaxScaler": """ - setParams(self, \\*, min=0.0, max=1.0, inputCol=None, outputCol=None) Sets params for this MinMaxScaler. """ kwargs = self._input_kwargs @@ -2866,9 +2798,6 @@ class NGram(JavaTransformer, HasInputCol, HasOutputCol, JavaMLReadable["NGram"], def __init__( self, *, n: int = 2, inputCol: Optional[str] = None, outputCol: Optional[str] = None ): - """ - __init__(self, \\*, n=2, inputCol=None, outputCol=None) - """ super(NGram, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.NGram", self.uid) self._setDefault(n=2) @@ -2880,7 +2809,6 @@ def setParams( self, *, n: int = 2, inputCol: Optional[str] = None, outputCol: Optional[str] = None ) -> "NGram": """ - setParams(self, \\*, n=2, inputCol=None, outputCol=None) Sets params for this NGram. """ kwargs = self._input_kwargs @@ -2959,9 +2887,6 @@ class Normalizer( def __init__( self, *, p: float = 2.0, inputCol: Optional[str] = None, outputCol: Optional[str] = None ): - """ - __init__(self, \\*, p=2.0, inputCol=None, outputCol=None) - """ super(Normalizer, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.Normalizer", self.uid) self._setDefault(p=2.0) @@ -2973,7 +2898,6 @@ def setParams( self, *, p: float = 2.0, inputCol: Optional[str] = None, outputCol: Optional[str] = None ) -> "Normalizer": """ - setParams(self, \\*, p=2.0, inputCol=None, outputCol=None) Sets params for this Normalizer. """ kwargs = self._input_kwargs @@ -3148,10 +3072,6 @@ def __init__( inputCol: Optional[str] = None, outputCol: Optional[str] = None, ): - """ - __init__(self, \\*, inputCols=None, outputCols=None, handleInvalid="error", dropLast=True, \ - inputCol=None, outputCol=None) - """ super(OneHotEncoder, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.OneHotEncoder", self.uid) kwargs = self._input_kwargs @@ -3191,8 +3111,6 @@ def setParams( outputCol: Optional[str] = None, ) -> "OneHotEncoder": """ - setParams(self, \\*, inputCols=None, outputCols=None, handleInvalid="error", \ - dropLast=True, inputCol=None, outputCol=None) Sets params for this OneHotEncoder. """ kwargs = self._input_kwargs @@ -3356,9 +3274,6 @@ class PolynomialExpansion( def __init__( self, *, degree: int = 2, inputCol: Optional[str] = None, outputCol: Optional[str] = None ): - """ - __init__(self, \\*, degree=2, inputCol=None, outputCol=None) - """ super(PolynomialExpansion, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.feature.PolynomialExpansion", self.uid @@ -3372,7 +3287,6 @@ def setParams( self, *, degree: int = 2, inputCol: Optional[str] = None, outputCol: Optional[str] = None ) -> "PolynomialExpansion": """ - setParams(self, \\*, degree=2, inputCol=None, outputCol=None) Sets params for this PolynomialExpansion. """ kwargs = self._input_kwargs @@ -3581,10 +3495,6 @@ def __init__( inputCols: Optional[List[str]] = None, outputCols: Optional[List[str]] = None, ): - """ - __init__(self, \\*, numBuckets=2, inputCol=None, outputCol=None, relativeError=0.001, \ - handleInvalid="error", numBucketsArray=None, inputCols=None, outputCols=None) - """ super(QuantileDiscretizer, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.feature.QuantileDiscretizer", self.uid @@ -3631,8 +3541,6 @@ def setParams( outputCols: Optional[List[str]] = None, ) -> "QuantileDiscretizer": """ - setParams(self, \\*, numBuckets=2, inputCol=None, outputCol=None, relativeError=0.001, \ - handleInvalid="error", numBucketsArray=None, inputCols=None, outputCols=None) Set the params for the QuantileDiscretizer """ kwargs = self._input_kwargs @@ -3862,10 +3770,6 @@ def __init__( outputCol: Optional[str] = None, relativeError: float = 0.001, ): - """ - __init__(self, \\*, lower=0.25, upper=0.75, withCentering=False, withScaling=True, \ - inputCol=None, outputCol=None, relativeError=0.001) - """ super(RobustScaler, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.RobustScaler", self.uid) kwargs = self._input_kwargs @@ -3884,8 +3788,6 @@ def setParams( relativeError: float = 0.001, ) -> "RobustScaler": """ - setParams(self, \\*, lower=0.25, upper=0.75, withCentering=False, withScaling=True, \ - inputCol=None, outputCol=None, relativeError=0.001) Sets params for this RobustScaler. """ kwargs = self._input_kwargs @@ -4072,10 +3974,6 @@ def __init__( outputCol: Optional[str] = None, toLowercase: bool = True, ): - """ - __init__(self, \\*, minTokenLength=1, gaps=True, pattern="\\s+", inputCol=None, \ - outputCol=None, toLowercase=True) - """ super(RegexTokenizer, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.RegexTokenizer", self.uid) self._setDefault(minTokenLength=1, gaps=True, pattern="\\s+", toLowercase=True) @@ -4094,8 +3992,6 @@ def setParams( toLowercase: bool = True, ) -> "RegexTokenizer": """ - setParams(self, \\*, minTokenLength=1, gaps=True, pattern="\\s+", inputCol=None, \ - outputCol=None, toLowercase=True) Sets params for this RegexTokenizer. """ kwargs = self._input_kwargs @@ -4202,9 +4098,6 @@ class SQLTransformer(JavaTransformer, JavaMLReadable["SQLTransformer"], JavaMLWr ) def __init__(self, *, statement: Optional[str] = None): - """ - __init__(self, \\*, statement=None) - """ super(SQLTransformer, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.SQLTransformer", self.uid) kwargs = self._input_kwargs @@ -4213,7 +4106,6 @@ def __init__(self, *, statement: Optional[str] = None): @since("1.6.0") def setParams(self, *, statement: Optional[str] = None) -> "SQLTransformer": """ - setParams(self, \\*, statement=None) Sets params for this SQLTransformer. """ kwargs = self._input_kwargs @@ -4335,9 +4227,6 @@ def __init__( inputCol: Optional[str] = None, outputCol: Optional[str] = None, ): - """ - __init__(self, \\*, withMean=False, withStd=True, inputCol=None, outputCol=None) - """ super(StandardScaler, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.StandardScaler", self.uid) kwargs = self._input_kwargs @@ -4353,7 +4242,6 @@ def setParams( outputCol: Optional[str] = None, ) -> "StandardScaler": """ - setParams(self, \\*, withMean=False, withStd=True, inputCol=None, outputCol=None) Sets params for this StandardScaler. """ kwargs = self._input_kwargs @@ -4595,10 +4483,6 @@ def __init__( handleInvalid: str = "error", stringOrderType: str = "frequencyDesc", ): - """ - __init__(self, \\*, inputCol=None, outputCol=None, inputCols=None, outputCols=None, \ - handleInvalid="error", stringOrderType="frequencyDesc") - """ super(StringIndexer, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.StringIndexer", self.uid) kwargs = self._input_kwargs @@ -4638,8 +4522,6 @@ def setParams( stringOrderType: str = "frequencyDesc", ) -> "StringIndexer": """ - setParams(self, \\*, inputCol=None, outputCol=None, inputCols=None, outputCols=None, \ - handleInvalid="error", stringOrderType="frequencyDesc") Sets params for this StringIndexer. """ kwargs = self._input_kwargs @@ -4843,9 +4725,6 @@ def __init__( outputCol: Optional[str] = None, labels: Optional[List[str]] = None, ): - """ - __init__(self, \\*, inputCol=None, outputCol=None, labels=None) - """ super(IndexToString, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.IndexToString", self.uid) kwargs = self._input_kwargs @@ -4860,7 +4739,6 @@ def setParams( labels: Optional[List[str]] = None, ) -> "IndexToString": """ - setParams(self, \\*, inputCol=None, outputCol=None, labels=None) Sets params for this IndexToString. """ kwargs = self._input_kwargs @@ -5002,10 +4880,6 @@ def __init__( inputCols: Optional[List[str]] = None, outputCols: Optional[List[str]] = None, ): - """ - __init__(self, \\*, inputCol=None, outputCol=None, stopWords=None, caseSensitive=false, \ - locale=None, inputCols=None, outputCols=None) - """ super(StopWordsRemover, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.feature.StopWordsRemover", self.uid @@ -5055,8 +4929,6 @@ def setParams( outputCols: Optional[List[str]] = None, ) -> "StopWordsRemover": """ - setParams(self, \\*, inputCol=None, outputCol=None, stopWords=None, caseSensitive=false, \ - locale=None, inputCols=None, outputCols=None) Sets params for this StopWordRemover. """ kwargs = self._input_kwargs @@ -5187,9 +5059,6 @@ class Tokenizer( _input_kwargs: Dict[str, Any] def __init__(self, *, inputCol: Optional[str] = None, outputCol: Optional[str] = None): - """ - __init__(self, \\*, inputCol=None, outputCol=None) - """ super(Tokenizer, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.Tokenizer", self.uid) kwargs = self._input_kwargs @@ -5200,7 +5069,6 @@ def setParams( self, *, inputCol: Optional[str] = None, outputCol: Optional[str] = None ) -> "Tokenizer": """ - setParams(self, \\*, inputCol=None, outputCol=None) Sets params for this Tokenizer. """ kwargs = self._input_kwargs @@ -5296,9 +5164,6 @@ def __init__( outputCol: Optional[str] = None, handleInvalid: str = "error", ): - """ - __init__(self, \\*, inputCols=None, outputCol=None, handleInvalid="error") - """ super(VectorAssembler, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.VectorAssembler", self.uid) self._setDefault(handleInvalid="error") @@ -5314,7 +5179,6 @@ def setParams( handleInvalid: str = "error", ) -> "VectorAssembler": """ - setParams(self, \\*, inputCols=None, outputCol=None, handleInvalid="error") Sets params for this VectorAssembler. """ kwargs = self._input_kwargs @@ -5483,9 +5347,6 @@ def __init__( outputCol: Optional[str] = None, handleInvalid: str = "error", ): - """ - __init__(self, \\*, maxCategories=20, inputCol=None, outputCol=None, handleInvalid="error") - """ super(VectorIndexer, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.VectorIndexer", self.uid) kwargs = self._input_kwargs @@ -5501,7 +5362,6 @@ def setParams( handleInvalid: str = "error", ) -> "VectorIndexer": """ - setParams(self, \\*, maxCategories=20, inputCol=None, outputCol=None, handleInvalid="error") Sets params for this VectorIndexer. """ kwargs = self._input_kwargs @@ -5659,9 +5519,6 @@ def __init__( indices: Optional[List[int]] = None, names: Optional[List[str]] = None, ): - """ - __init__(self, \\*, inputCol=None, outputCol=None, indices=None, names=None) - """ super(VectorSlicer, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.VectorSlicer", self.uid) self._setDefault(indices=[], names=[]) @@ -5678,7 +5535,6 @@ def setParams( names: Optional[List[str]] = None, ) -> "VectorSlicer": """ - setParams(self, \\*, inputCol=None, outputCol=None, indices=None, names=None): Sets params for this VectorSlicer. """ kwargs = self._input_kwargs @@ -5900,11 +5756,6 @@ def __init__( windowSize: int = 5, maxSentenceLength: int = 1000, ): - """ - __init__(self, \\*, vectorSize=100, minCount=5, numPartitions=1, stepSize=0.025, \ - maxIter=1, seed=None, inputCol=None, outputCol=None, windowSize=5, \ - maxSentenceLength=1000) - """ super(Word2Vec, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.Word2Vec", self.uid) kwargs = self._input_kwargs @@ -5926,9 +5777,6 @@ def setParams( maxSentenceLength: int = 1000, ) -> "Word2Vec": """ - setParams(self, \\*, minCount=5, numPartitions=1, stepSize=0.025, maxIter=1, \ - seed=None, inputCol=None, outputCol=None, windowSize=5, \ - maxSentenceLength=1000) Sets params for this Word2Vec. """ kwargs = self._input_kwargs @@ -6132,9 +5980,6 @@ def __init__( inputCol: Optional[str] = None, outputCol: Optional[str] = None, ): - """ - __init__(self, \\*, k=None, inputCol=None, outputCol=None) - """ super(PCA, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.PCA", self.uid) kwargs = self._input_kwargs @@ -6149,7 +5994,6 @@ def setParams( outputCol: Optional[str] = None, ) -> "PCA": """ - setParams(self, \\*, k=None, inputCol=None, outputCol=None) Set params for this PCA. """ kwargs = self._input_kwargs @@ -6377,11 +6221,6 @@ def __init__( stringIndexerOrderType: str = "frequencyDesc", handleInvalid: str = "error", ): - """ - __init__(self, \\*, formula=None, featuresCol="features", labelCol="label", \ - forceIndexLabel=False, stringIndexerOrderType="frequencyDesc", \ - handleInvalid="error") - """ super(RFormula, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.RFormula", self.uid) kwargs = self._input_kwargs @@ -6399,9 +6238,6 @@ def setParams( handleInvalid: str = "error", ) -> "RFormula": """ - setParams(self, \\*, formula=None, featuresCol="features", labelCol="label", \ - forceIndexLabel=False, stringIndexerOrderType="frequencyDesc", \ - handleInvalid="error") Sets params for RFormula. """ kwargs = self._input_kwargs @@ -6754,11 +6590,6 @@ def __init__( fdr: float = 0.05, fwe: float = 0.05, ): - """ - __init__(self, \\*, numTopFeatures=50, featuresCol="features", outputCol=None, \ - labelCol="label", selectorType="numTopFeatures", percentile=0.1, fpr=0.05, \ - fdr=0.05, fwe=0.05) - """ super(ChiSqSelector, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.ChiSqSelector", self.uid) kwargs = self._input_kwargs @@ -6779,9 +6610,6 @@ def setParams( fwe: float = 0.05, ) -> "ChiSqSelector": """ - setParams(self, \\*, numTopFeatures=50, featuresCol="features", outputCol=None, \ - labelCol="label", selectorType="numTopFeatures", percentile=0.1, fpr=0.05, \ - fdr=0.05, fwe=0.05) Sets params for this ChiSqSelector. """ kwargs = self._input_kwargs @@ -6865,9 +6693,6 @@ def __init__( size: Optional[int] = None, handleInvalid: str = "error", ): - """ - __init__(self, \\*, inputCol=None, size=None, handleInvalid="error") - """ super(VectorSizeHint, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.VectorSizeHint", self.uid) self._setDefault(handleInvalid="error") @@ -6882,7 +6707,6 @@ def setParams( handleInvalid: str = "error", ) -> "VectorSizeHint": """ - setParams(self, \\*, inputCol=None, size=None, handleInvalid="error") Sets params for this VectorSizeHint. """ kwargs = self._input_kwargs @@ -6995,9 +6819,6 @@ def __init__( outputCol: Optional[str] = None, varianceThreshold: float = 0.0, ): - """ - __init__(self, \\*, featuresCol="features", outputCol=None, varianceThreshold=0.0) - """ super(VarianceThresholdSelector, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.feature.VarianceThresholdSelector", self.uid @@ -7015,7 +6836,6 @@ def setParams( varianceThreshold: float = 0.0, ) -> "VarianceThresholdSelector": """ - setParams(self, \\*, featuresCol="features", outputCol=None, varianceThreshold=0.0) Sets params for this VarianceThresholdSelector. """ kwargs = self._input_kwargs @@ -7241,10 +7061,6 @@ def __init__( labelCol: str = "label", selectionMode: str = "numTopFeatures", ): - """ - __init__(self, \\*, featuresCol="features", outputCol=None, \ - labelCol="label", selectionMode="numTopFeatures") - """ super(UnivariateFeatureSelector, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.feature.UnivariateFeatureSelector", self.uid @@ -7262,8 +7078,6 @@ def setParams( selectionMode: str = "numTopFeatures", ) -> "UnivariateFeatureSelector": """ - setParams(self, \\*, featuresCol="features", outputCol=None, \ - labelCol="label", selectionMode="numTopFeatures") Sets params for this UnivariateFeatureSelector. """ kwargs = self._input_kwargs diff --git a/python/pyspark/ml/fpm.py b/python/pyspark/ml/fpm.py index b396826640565..67f737b33241c 100644 --- a/python/pyspark/ml/fpm.py +++ b/python/pyspark/ml/fpm.py @@ -246,10 +246,6 @@ def __init__( predictionCol: str = "prediction", numPartitions: Optional[int] = None, ): - """ - __init__(self, \\*, minSupport=0.3, minConfidence=0.8, itemsCol="items", \ - predictionCol="prediction", numPartitions=None) - """ super(FPGrowth, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.fpm.FPGrowth", self.uid) kwargs = self._input_kwargs @@ -265,10 +261,6 @@ def setParams( predictionCol: str = "prediction", numPartitions: Optional[int] = None, ) -> "FPGrowth": - """ - setParams(self, \\*, minSupport=0.3, minConfidence=0.8, itemsCol="items", \ - predictionCol="prediction", numPartitions=None) - """ kwargs = self._input_kwargs return self._set(**kwargs) @@ -397,10 +389,6 @@ def __init__( maxLocalProjDBSize: int = 32000000, sequenceCol: str = "sequence", ): - """ - __init__(self, \\*, minSupport=0.1, maxPatternLength=10, maxLocalProjDBSize=32000000, \ - sequenceCol="sequence") - """ super(PrefixSpan, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.fpm.PrefixSpan", self.uid) self._setDefault( @@ -418,10 +406,6 @@ def setParams( maxLocalProjDBSize: int = 32000000, sequenceCol: str = "sequence", ) -> "PrefixSpan": - """ - setParams(self, \\*, minSupport=0.1, maxPatternLength=10, maxLocalProjDBSize=32000000, \ - sequenceCol="sequence") - """ kwargs = self._input_kwargs return self._set(**kwargs) diff --git a/python/pyspark/ml/pipeline.py b/python/pyspark/ml/pipeline.py index 71b570a5cd0ee..343af8827cae2 100644 --- a/python/pyspark/ml/pipeline.py +++ b/python/pyspark/ml/pipeline.py @@ -71,9 +71,6 @@ class Pipeline(Estimator["PipelineModel"], MLReadable["Pipeline"], MLWritable): _input_kwargs: Dict[str, Any] def __init__(self, *, stages: Optional[List["PipelineStage"]] = None): - """ - __init__(self, \\*, stages=None) - """ super(Pipeline, self).__init__() kwargs = self._input_kwargs self.setParams(**kwargs) @@ -107,7 +104,6 @@ def getStages(self) -> List["PipelineStage"]: @since("1.3.0") def setParams(self, *, stages: Optional[List["PipelineStage"]] = None) -> "Pipeline": """ - setParams(self, \\*, stages=None) Sets params for Pipeline. """ kwargs = self._input_kwargs diff --git a/python/pyspark/ml/recommendation.py b/python/pyspark/ml/recommendation.py index 114fd8dc73d3f..b711bd7397ffb 100644 --- a/python/pyspark/ml/recommendation.py +++ b/python/pyspark/ml/recommendation.py @@ -387,13 +387,6 @@ def __init__( coldStartStrategy: str = "nan", blockSize: int = 4096, ): - """ - __init__(self, \\*, rank=10, maxIter=10, regParam=0.1, numUserBlocks=10, - numItemBlocks=10, implicitPrefs=False, alpha=1.0, userCol="user", itemCol="item", \ - seed=None, ratingCol="rating", nonnegative=False, checkpointInterval=10, \ - intermediateStorageLevel="MEMORY_AND_DISK", \ - finalStorageLevel="MEMORY_AND_DISK", coldStartStrategy="nan", blockSize=4096) - """ super(ALS, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.recommendation.ALS", self.uid) kwargs = self._input_kwargs @@ -422,11 +415,6 @@ def setParams( blockSize: int = 4096, ) -> "ALS": """ - setParams(self, \\*, rank=10, maxIter=10, regParam=0.1, numUserBlocks=10, \ - numItemBlocks=10, implicitPrefs=False, alpha=1.0, userCol="user", itemCol="item", \ - seed=None, ratingCol="rating", nonnegative=False, checkpointInterval=10, \ - intermediateStorageLevel="MEMORY_AND_DISK", \ - finalStorageLevel="MEMORY_AND_DISK", coldStartStrategy="nan", blockSize=4096) Sets params for ALS. """ kwargs = self._input_kwargs diff --git a/python/pyspark/ml/regression.py b/python/pyspark/ml/regression.py index bf9b004c193db..93e5c3980ac31 100644 --- a/python/pyspark/ml/regression.py +++ b/python/pyspark/ml/regression.py @@ -315,12 +315,6 @@ def __init__( epsilon: float = 1.35, maxBlockSizeInMB: float = 0.0, ): - """ - __init__(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \ - maxIter=100, regParam=0.0, elasticNetParam=0.0, tol=1e-6, fitIntercept=True, \ - standardization=True, solver="auto", weightCol=None, aggregationDepth=2, \ - loss="squaredError", epsilon=1.35, maxBlockSizeInMB=0.0) - """ super(LinearRegression, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.regression.LinearRegression", self.uid @@ -349,10 +343,6 @@ def setParams( maxBlockSizeInMB: float = 0.0, ) -> "LinearRegression": """ - setParams(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \ - maxIter=100, regParam=0.0, elasticNetParam=0.0, tol=1e-6, fitIntercept=True, \ - standardization=True, solver="auto", weightCol=None, aggregationDepth=2, \ - loss="squaredError", epsilon=1.35, maxBlockSizeInMB=0.0) Sets params for linear regression. """ kwargs = self._input_kwargs @@ -866,10 +856,6 @@ def __init__( isotonic: bool = True, featureIndex: int = 0, ): - """ - __init__(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \ - weightCol=None, isotonic=True, featureIndex=0): - """ super(IsotonicRegression, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.regression.IsotonicRegression", self.uid @@ -888,8 +874,6 @@ def setParams( featureIndex: int = 0, ) -> "IsotonicRegression": """ - setParams(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \ - weightCol=None, isotonic=True, featureIndex=0): Set the params for IsotonicRegression. """ kwargs = self._input_kwargs @@ -1124,13 +1108,6 @@ def __init__( leafCol: str = "", minWeightFractionPerNode: float = 0.0, ): - """ - __init__(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \ - maxDepth=5, maxBins=32, minInstancesPerNode=1, minInfoGain=0.0, \ - maxMemoryInMB=256, cacheNodeIds=False, checkpointInterval=10, \ - impurity="variance", seed=None, varianceCol=None, weightCol=None, \ - leafCol="", minWeightFractionPerNode=0.0) - """ super(DecisionTreeRegressor, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.regression.DecisionTreeRegressor", self.uid @@ -1160,11 +1137,6 @@ def setParams( minWeightFractionPerNode: float = 0.0, ) -> "DecisionTreeRegressor": """ - setParams(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \ - maxDepth=5, maxBins=32, minInstancesPerNode=1, minInfoGain=0.0, \ - maxMemoryInMB=256, cacheNodeIds=False, checkpointInterval=10, \ - impurity="variance", seed=None, varianceCol=None, weightCol=None, \ - leafCol="", minWeightFractionPerNode=0.0) Sets params for the DecisionTreeRegressor. """ kwargs = self._input_kwargs @@ -1425,14 +1397,6 @@ def __init__( weightCol: Optional[str] = None, bootstrap: Optional[bool] = True, ): - """ - __init__(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \ - maxDepth=5, maxBins=32, minInstancesPerNode=1, minInfoGain=0.0, \ - maxMemoryInMB=256, cacheNodeIds=False, checkpointInterval=10, \ - impurity="variance", subsamplingRate=1.0, seed=None, numTrees=20, \ - featureSubsetStrategy="auto", leafCol=", minWeightFractionPerNode=0.0", \ - weightCol=None, bootstrap=True) - """ super(RandomForestRegressor, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.regression.RandomForestRegressor", self.uid @@ -1465,12 +1429,6 @@ def setParams( bootstrap: Optional[bool] = True, ) -> "RandomForestRegressor": """ - setParams(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \ - maxDepth=5, maxBins=32, minInstancesPerNode=1, minInfoGain=0.0, \ - maxMemoryInMB=256, cacheNodeIds=False, checkpointInterval=10, \ - impurity="variance", subsamplingRate=1.0, seed=None, numTrees=20, \ - featureSubsetStrategy="auto", leafCol="", minWeightFractionPerNode=0.0, \ - weightCol=None, bootstrap=True) Sets params for linear regression. """ kwargs = self._input_kwargs @@ -1769,15 +1727,6 @@ def __init__( minWeightFractionPerNode: float = 0.0, weightCol: Optional[str] = None, ): - """ - __init__(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \ - maxDepth=5, maxBins=32, minInstancesPerNode=1, minInfoGain=0.0, \ - maxMemoryInMB=256, cacheNodeIds=False, subsamplingRate=1.0, \ - checkpointInterval=10, lossType="squared", maxIter=20, stepSize=0.1, seed=None, \ - impurity="variance", featureSubsetStrategy="all", validationTol=0.01, \ - validationIndicatorCol=None, leafCol="", minWeightFractionPerNode=0.0, - weightCol=None) - """ super(GBTRegressor, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.regression.GBTRegressor", self.uid) kwargs = self._input_kwargs @@ -1811,13 +1760,6 @@ def setParams( weightCol: Optional[str] = None, ) -> "GBTRegressor": """ - setParams(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \ - maxDepth=5, maxBins=32, minInstancesPerNode=1, minInfoGain=0.0, \ - maxMemoryInMB=256, cacheNodeIds=False, subsamplingRate=1.0, \ - checkpointInterval=10, lossType="squared", maxIter=20, stepSize=0.1, seed=None, \ - impurity="variance", featureSubsetStrategy="all", validationTol=0.01, \ - validationIndicatorCol=None, leafCol="", minWeightFractionPerNode=0.0, \ - weightCol=None) Sets params for Gradient Boosted Tree Regression. """ kwargs = self._input_kwargs @@ -2160,12 +2102,6 @@ def __init__( aggregationDepth: int = 2, maxBlockSizeInMB: float = 0.0, ): - """ - __init__(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \ - fitIntercept=True, maxIter=100, tol=1E-6, censorCol="censor", \ - quantileProbabilities=[0.01, 0.05, 0.1, 0.25, 0.5, 0.75, 0.9, 0.95, 0.99], \ - quantilesCol=None, aggregationDepth=2, maxBlockSizeInMB=0.0) - """ super(AFTSurvivalRegression, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.regression.AFTSurvivalRegression", self.uid @@ -2199,12 +2135,6 @@ def setParams( aggregationDepth: int = 2, maxBlockSizeInMB: float = 0.0, ) -> "AFTSurvivalRegression": - """ - setParams(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \ - fitIntercept=True, maxIter=100, tol=1E-6, censorCol="censor", \ - quantileProbabilities=[0.01, 0.05, 0.1, 0.25, 0.5, 0.75, 0.9, 0.95, 0.99], \ - quantilesCol=None, aggregationDepth=2, maxBlockSizeInMB=0.0): - """ kwargs = self._input_kwargs return self._set(**kwargs) @@ -2558,12 +2488,6 @@ def __init__( offsetCol: Optional[str] = None, aggregationDepth: int = 2, ): - """ - __init__(self, \\*, labelCol="label", featuresCol="features", predictionCol="prediction", \ - family="gaussian", link=None, fitIntercept=True, maxIter=25, tol=1e-6, \ - regParam=0.0, weightCol=None, solver="irls", linkPredictionCol=None, \ - variancePower=0.0, linkPower=None, offsetCol=None, aggregationDepth=2) - """ super(GeneralizedLinearRegression, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.regression.GeneralizedLinearRegression", self.uid @@ -2594,10 +2518,6 @@ def setParams( aggregationDepth: int = 2, ) -> "GeneralizedLinearRegression": """ - setParams(self, \\*, labelCol="label", featuresCol="features", predictionCol="prediction", \ - family="gaussian", link=None, fitIntercept=True, maxIter=25, tol=1e-6, \ - regParam=0.0, weightCol=None, solver="irls", linkPredictionCol=None, \ - variancePower=0.0, linkPower=None, offsetCol=None, aggregationDepth=2) Sets params for generalized linear regression. """ kwargs = self._input_kwargs @@ -3129,12 +3049,6 @@ def __init__( solver: str = "adamW", seed: Optional[int] = None, ): - """ - __init__(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \ - factorSize=8, fitIntercept=True, fitLinear=True, regParam=0.0, \ - miniBatchFraction=1.0, initStd=0.01, maxIter=100, stepSize=1.0, \ - tol=1e-6, solver="adamW", seed=None) - """ super(FMRegressor, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.regression.FMRegressor", self.uid) kwargs = self._input_kwargs @@ -3160,10 +3074,6 @@ def setParams( seed: Optional[int] = None, ) -> "FMRegressor": """ - setParams(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \ - factorSize=8, fitIntercept=True, fitLinear=True, regParam=0.0, \ - miniBatchFraction=1.0, initStd=0.01, maxIter=100, stepSize=1.0, \ - tol=1e-6, solver="adamW", seed=None) Sets Params for FMRegressor. """ kwargs = self._input_kwargs diff --git a/python/pyspark/ml/tuning.py b/python/pyspark/ml/tuning.py index 220af8154d144..64f3c32253d61 100644 --- a/python/pyspark/ml/tuning.py +++ b/python/pyspark/ml/tuning.py @@ -728,10 +728,6 @@ def __init__( collectSubModels: bool = False, foldCol: str = "", ) -> None: - """ - __init__(self, \\*, estimator=None, estimatorParamMaps=None, evaluator=None, numFolds=3,\ - seed=None, parallelism=1, collectSubModels=False, foldCol="") - """ super(CrossValidator, self).__init__() self._setDefault(parallelism=1) kwargs = self._input_kwargs @@ -751,8 +747,6 @@ def setParams( foldCol: str = "", ) -> "CrossValidator": """ - setParams(self, \\*, estimator=None, estimatorParamMaps=None, evaluator=None, numFolds=3,\ - seed=None, parallelism=1, collectSubModels=False, foldCol=""): Sets params for cross validator. """ kwargs = self._input_kwargs @@ -1356,10 +1350,6 @@ def __init__( collectSubModels: bool = False, seed: Optional[int] = None, ) -> None: - """ - __init__(self, \\*, estimator=None, estimatorParamMaps=None, evaluator=None, \ - trainRatio=0.75, parallelism=1, collectSubModels=False, seed=None) - """ super(TrainValidationSplit, self).__init__() self._setDefault(parallelism=1) kwargs = self._input_kwargs @@ -1378,8 +1368,6 @@ def setParams( seed: Optional[int] = None, ) -> "TrainValidationSplit": """ - setParams(self, \\*, estimator=None, estimatorParamMaps=None, evaluator=None, \ - trainRatio=0.75, parallelism=1, collectSubModels=False, seed=None): Sets params for the train validation split. """ kwargs = self._input_kwargs From e0ed06ee9e4df136f1f9adda638e7e649815e84b Mon Sep 17 00:00:00 2001 From: Hyukjin Kwon Date: Mon, 27 Nov 2023 10:05:40 +0900 Subject: [PATCH 3/7] Fix sphinx conf --- python/docs/source/conf.py | 1 - 1 file changed, 1 deletion(-) diff --git a/python/docs/source/conf.py b/python/docs/source/conf.py index 81083c007b346..9d072fa1a1455 100644 --- a/python/docs/source/conf.py +++ b/python/docs/source/conf.py @@ -176,7 +176,6 @@ # -- Options for autodoc -------------------------------------------------- # Look at the first line of the docstring for function and method signatures. -autodoc_docstring_signature = True autosummary_generate = True # -- Options for HTML output ---------------------------------------------- From 3e88a539336b1019ed1542f619e8e3353db2dfd3 Mon Sep 17 00:00:00 2001 From: Hyukjin Kwon Date: Mon, 27 Nov 2023 12:23:36 +0900 Subject: [PATCH 4/7] Replace _input_kwargs to locals() --- python/pyspark/ml/classification.py | 96 ++--- python/pyspark/ml/clustering.py | 50 ++- python/pyspark/ml/connect/classification.py | 6 +- python/pyspark/ml/connect/feature.py | 18 +- python/pyspark/ml/connect/pipeline.py | 10 +- python/pyspark/ml/connect/tuning.py | 12 +- python/pyspark/ml/evaluation.py | 60 ++-- python/pyspark/ml/feature.py | 369 ++++++++------------ python/pyspark/ml/fpm.py | 19 +- python/pyspark/ml/pipeline.py | 10 +- python/pyspark/ml/recommendation.py | 10 +- python/pyspark/ml/regression.py | 80 ++--- python/pyspark/ml/tuning.py | 20 +- python/pyspark/testing/mlutils.py | 8 +- 14 files changed, 305 insertions(+), 463 deletions(-) diff --git a/python/pyspark/ml/classification.py b/python/pyspark/ml/classification.py index b32624ec30126..3032376f70394 100644 --- a/python/pyspark/ml/classification.py +++ b/python/pyspark/ml/classification.py @@ -706,8 +706,6 @@ class LinearSVC( True """ - _input_kwargs: Dict[str, Any] - def __init__( self, *, @@ -725,12 +723,12 @@ def __init__( aggregationDepth: int = 2, maxBlockSizeInMB: float = 0.0, ): + kwargs = locals() super(LinearSVC, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.classification.LinearSVC", self.uid ) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) @since("2.2.0") def setParams( @@ -753,8 +751,8 @@ def setParams( """ Sets params for Linear SVM Classifier. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = locals() + return self.__class__._set(**kwargs) def _create_model(self, java_model: "JavaObject") -> "LinearSVCModel": return LinearSVCModel(java_model) @@ -1207,8 +1205,6 @@ class LogisticRegression( True """ - _input_kwargs: Dict[str, Any] - @overload def __init__( self, @@ -1291,12 +1287,12 @@ def __init__( """ If the threshold and thresholds Params are both set, they must be equivalent. """ + kwargs = locals() super(LogisticRegression, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.classification.LogisticRegression", self.uid ) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) self._checkThresholdConsistency() @overload @@ -1383,8 +1379,8 @@ def setParams( Sets params for logistic regression. If the threshold and thresholds Params are both set, they must be equivalent. """ - kwargs = self._input_kwargs - self._set(**kwargs) + kwargs = locals() + self.__class__._set(**kwargs) self._checkThresholdConsistency() return self @@ -1740,8 +1736,6 @@ class DecisionTreeClassifier( DecisionTreeClassificationModel...depth=1, numNodes=3... """ - _input_kwargs: Dict[str, Any] - def __init__( self, *, @@ -1763,12 +1757,12 @@ def __init__( leafCol: str = "", minWeightFractionPerNode: float = 0.0, ): + kwargs = locals() super(DecisionTreeClassifier, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.classification.DecisionTreeClassifier", self.uid ) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) @since("1.4.0") def setParams( @@ -1795,8 +1789,8 @@ def setParams( """ Sets params for the DecisionTreeClassifier. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = locals() + return self.__class__.setParams(**kwargs) def _create_model(self, java_model: "JavaObject") -> "DecisionTreeClassificationModel": return DecisionTreeClassificationModel(java_model) @@ -2018,8 +2012,6 @@ class RandomForestClassifier( True """ - _input_kwargs: Dict[str, Any] - def __init__( self, *, @@ -2045,12 +2037,12 @@ def __init__( weightCol: Optional[str] = None, bootstrap: Optional[bool] = True, ): + kwargs = locals() super(RandomForestClassifier, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.classification.RandomForestClassifier", self.uid ) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) @since("1.4.0") def setParams( @@ -2079,16 +2071,10 @@ def setParams( bootstrap: Optional[bool] = True, ) -> "RandomForestClassifier": """ - setParams(self, featuresCol="features", labelCol="label", predictionCol="prediction", \ - probabilityCol="probability", rawPredictionCol="rawPrediction", \ - maxDepth=5, maxBins=32, minInstancesPerNode=1, minInfoGain=0.0, \ - maxMemoryInMB=256, cacheNodeIds=False, checkpointInterval=10, seed=None, \ - impurity="gini", numTrees=20, featureSubsetStrategy="auto", subsamplingRate=1.0, \ - leafCol="", minWeightFractionPerNode=0.0, weightCol=None, bootstrap=True) Sets params for linear classification. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = locals() + return self.__class__._set(**kwargs) def _create_model(self, java_model: "JavaObject") -> "RandomForestClassificationModel": return RandomForestClassificationModel(java_model) @@ -2477,8 +2463,6 @@ class GBTClassifier( 0.01 """ - _input_kwargs: Dict[str, Any] - def __init__( self, *, @@ -2505,12 +2489,12 @@ def __init__( minWeightFractionPerNode: float = 0.0, weightCol: Optional[str] = None, ): + kwargs = locals() super(GBTClassifier, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.classification.GBTClassifier", self.uid ) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) @since("1.4.0") def setParams( @@ -2542,8 +2526,8 @@ def setParams( """ Sets params for Gradient Boosted Tree Classification. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = locals() + return self.__class__._set(**kwargs) def _create_model(self, java_model: "JavaObject") -> "GBTClassificationModel": return GBTClassificationModel(java_model) @@ -2856,8 +2840,6 @@ class NaiveBayes( DenseMatrix(0, 0, [...], ...) """ - _input_kwargs: Dict[str, Any] - def __init__( self, *, @@ -2871,12 +2853,12 @@ def __init__( thresholds: Optional[List[float]] = None, weightCol: Optional[str] = None, ): + kwargs = locals() super(NaiveBayes, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.classification.NaiveBayes", self.uid ) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) @since("1.5.0") def setParams( @@ -2895,8 +2877,8 @@ def setParams( """ Sets params for Naive Bayes. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = locals() + return self.__class__._set(**kwargs) def _create_model(self, java_model: "JavaObject") -> "NaiveBayesModel": return NaiveBayesModel(java_model) @@ -3096,8 +3078,6 @@ class MultilayerPerceptronClassifier( True """ - _input_kwargs: Dict[str, Any] - def __init__( self, *, @@ -3115,12 +3095,12 @@ def __init__( probabilityCol: str = "probability", rawPredictionCol: str = "rawPrediction", ): + kwargs = locals() super(MultilayerPerceptronClassifier, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.classification.MultilayerPerceptronClassifier", self.uid ) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) @since("1.6.0") def setParams( @@ -3143,8 +3123,8 @@ def setParams( """ Sets params for MultilayerPerceptronClassifier. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = locals() + return self.__class__._set(**kwargs) def _create_model(self, java_model: "JavaObject") -> "MultilayerPerceptronClassificationModel": return MultilayerPerceptronClassificationModel(java_model) @@ -3352,8 +3332,6 @@ class OneVsRest( ['features', 'rawPrediction', 'newPrediction'] """ - _input_kwargs: Dict[str, Any] - def __init__( self, *, @@ -3365,10 +3343,10 @@ def __init__( weightCol: Optional[str] = None, parallelism: int = 1, ): + kwargs = locals() super(OneVsRest, self).__init__() self._setDefault(parallelism=1) - kwargs = self._input_kwargs - self._set(**kwargs) + self.__class__._set(**kwargs) @since("2.0.0") def setParams( @@ -3385,8 +3363,8 @@ def setParams( """ Sets params for OneVsRest. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = locals() + return self.__class__._set(**kwargs) @since("2.0.0") def setClassifier(self, value: Classifier[CM]) -> "OneVsRest": @@ -3957,8 +3935,6 @@ class FMClassifier( True """ - _input_kwargs: Dict[str, Any] - def __init__( self, *, @@ -3980,12 +3956,12 @@ def __init__( thresholds: Optional[List[float]] = None, seed: Optional[int] = None, ): + kwargs = locals() super(FMClassifier, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.classification.FMClassifier", self.uid ) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) @since("3.0.0") def setParams( @@ -4012,8 +3988,8 @@ def setParams( """ Sets Params for FMClassifier. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = locals() + return self.__class__._set(**kwargs) def _create_model(self, java_model: "JavaObject") -> "FMClassificationModel": return FMClassificationModel(java_model) diff --git a/python/pyspark/ml/clustering.py b/python/pyspark/ml/clustering.py index bd95960fee4b5..2b6d8f108dc90 100644 --- a/python/pyspark/ml/clustering.py +++ b/python/pyspark/ml/clustering.py @@ -401,8 +401,6 @@ class GaussianMixture( GaussianMixture... """ - _input_kwargs: Dict[str, Any] - def __init__( self, *, @@ -416,12 +414,12 @@ def __init__( aggregationDepth: int = 2, weightCol: Optional[str] = None, ): + kwargs = locals() super(GaussianMixture, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.clustering.GaussianMixture", self.uid ) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) def _create_model(self, java_model: "JavaObject") -> "GaussianMixtureModel": return GaussianMixtureModel(java_model) @@ -443,8 +441,8 @@ def setParams( """ Sets params for GaussianMixture. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = locals() + return self.__class__._set(**kwargs) @since("2.0.0") def setK(self, value: int) -> "GaussianMixture": @@ -764,8 +762,6 @@ class KMeans(JavaEstimator[KMeansModel], _KMeansParams, JavaMLWritable, JavaMLRe True """ - _input_kwargs: Dict[str, Any] - def __init__( self, *, @@ -782,10 +778,10 @@ def __init__( solver: str = "auto", maxBlockSizeInMB: float = 0.0, ): + kwargs = locals() super(KMeans, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.clustering.KMeans", self.uid) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) def _create_model(self, java_model: "JavaObject") -> KMeansModel: return KMeansModel(java_model) @@ -810,8 +806,8 @@ def setParams( """ Sets params for KMeans. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = locals() + return self.__class__._set(**kwargs) @since("1.5.0") def setK(self, value: int) -> "KMeans": @@ -1103,8 +1099,6 @@ class BisectingKMeans( True """ - _input_kwargs: Dict[str, Any] - def __init__( self, *, @@ -1117,12 +1111,12 @@ def __init__( distanceMeasure: str = "euclidean", weightCol: Optional[str] = None, ): + kwargs = locals() super(BisectingKMeans, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.clustering.BisectingKMeans", self.uid ) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) @since("2.0.0") def setParams( @@ -1140,8 +1134,8 @@ def setParams( """ Sets params for BisectingKMeans. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = locals() + return self.__class__._set(**kwargs) @since("2.0.0") def setK(self, value: int) -> "BisectingKMeans": @@ -1645,8 +1639,6 @@ class LDA(JavaEstimator[LDAModel], _LDAParams, JavaMLReadable["LDA"], JavaMLWrit True """ - _input_kwargs: Dict[str, Any] - def __init__( self, *, @@ -1665,10 +1657,10 @@ def __init__( topicDistributionCol: str = "topicDistribution", keepLastCheckpoint: bool = True, ): + kwargs = locals() super(LDA, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.clustering.LDA", self.uid) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) def _create_model(self, java_model: "JavaObject") -> LDAModel: if self.getOptimizer() == "em": @@ -1698,8 +1690,8 @@ def setParams( """ Sets params for LDA. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = locals() + return self.__class__._set(**kwargs) @since("2.0.0") def setCheckpointInterval(self, value: int) -> "LDA": @@ -1985,8 +1977,6 @@ class PowerIterationClustering( True """ - _input_kwargs: Dict[str, Any] - def __init__( self, *, @@ -1997,12 +1987,12 @@ def __init__( dstCol: str = "dst", weightCol: Optional[str] = None, ): + kwargs = locals() super(PowerIterationClustering, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.clustering.PowerIterationClustering", self.uid ) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) @since("2.4.0") def setParams( @@ -2018,8 +2008,8 @@ def setParams( """ Sets params for PowerIterationClustering. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = locals() + return self.__class__._set(**kwargs) @since("2.4.0") def setK(self, value: int) -> "PowerIterationClustering": diff --git a/python/pyspark/ml/connect/classification.py b/python/pyspark/ml/connect/classification.py index 261cbd5066a9a..fe0d4234a7586 100644 --- a/python/pyspark/ml/connect/classification.py +++ b/python/pyspark/ml/connect/classification.py @@ -175,8 +175,6 @@ class LogisticRegression( LogisticRegression_... """ - _input_kwargs: Dict[str, Any] - def __init__( self, *, @@ -209,9 +207,9 @@ def __init__( seed: int = 0, ) """ + kwargs = locals() super(LogisticRegression, self).__init__() - kwargs = self._input_kwargs - self._set(**kwargs) + self.__class__._set(**kwargs) def _fit(self, dataset: Union[DataFrame, pd.DataFrame]) -> "LogisticRegressionModel": if isinstance(dataset, pd.DataFrame): diff --git a/python/pyspark/ml/connect/feature.py b/python/pyspark/ml/connect/feature.py index e474f502feac0..d100b330e471c 100644 --- a/python/pyspark/ml/connect/feature.py +++ b/python/pyspark/ml/connect/feature.py @@ -66,12 +66,10 @@ class MaxAbsScaler(Estimator, HasInputCol, HasOutputCol, ParamsReadWrite): +------------+--------------------------+ """ - _input_kwargs: Dict[str, Any] - def __init__(self, *, inputCol: Optional[str] = None, outputCol: Optional[str] = None) -> None: + kwargs = locals() super().__init__() - kwargs = self._input_kwargs - self._set(**kwargs) + MaxAbsScaler._set(**kwargs) def _fit(self, dataset: Union["pd.DataFrame", "DataFrame"]) -> "MaxAbsScalerModel": input_col = self.getInputCol() @@ -174,12 +172,10 @@ class StandardScaler(Estimator, HasInputCol, HasOutputCol, ParamsReadWrite): +------------+------------------------------------------+ """ - _input_kwargs: Dict[str, Any] - def __init__(self, inputCol: Optional[str] = None, outputCol: Optional[str] = None) -> None: + kwargs = locals() super().__init__() - kwargs = self._input_kwargs - self._set(**kwargs) + StandardScaler._set(**kwargs) def _fit(self, dataset: Union[DataFrame, pd.DataFrame]) -> "StandardScalerModel": input_col = self.getInputCol() @@ -306,8 +302,6 @@ class ArrayAssembler( >>> assembler.transform(spark_df).select("out").show(truncate=False) """ - _input_kwargs: Dict[str, Any] - # Override doc of handleInvalid param. handleInvalid: Param[str] = Param( Params._dummy(), @@ -326,9 +320,9 @@ def __init__( featureSizes: Optional[List[int]] = None, handleInvalid: Optional[str] = "error", ) -> None: + kwargs = locals() super().__init__() - kwargs = self._input_kwargs - self._set(**kwargs) + ArrayAssembler._set(**kwargs) self._setDefault(handleInvalid="error") def _input_columns(self) -> List[str]: diff --git a/python/pyspark/ml/connect/pipeline.py b/python/pyspark/ml/connect/pipeline.py index 9a5bd9372b1c2..360756657001d 100644 --- a/python/pyspark/ml/connect/pipeline.py +++ b/python/pyspark/ml/connect/pipeline.py @@ -135,12 +135,10 @@ class Pipeline(Estimator["PipelineModel"], _PipelineReadWrite): Params._dummy(), "stages", "a list of pipeline stages" ) # type: ignore[assignment] - _input_kwargs: Dict[str, Any] - def __init__(self, *, stages: Optional[List[Params]] = None): + kwargs = locals() super(Pipeline, self).__init__() - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) def setStages(self, value: List[Params]) -> "Pipeline": """ @@ -173,8 +171,8 @@ def setParams(self, *, stages: Optional[List[Params]] = None) -> "Pipeline": """ Sets params for Pipeline. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = locals() + return self.__class__._set(**kwargs) def _fit(self, dataset: Union[DataFrame, pd.DataFrame]) -> "PipelineModel": stages = self.getStages() diff --git a/python/pyspark/ml/connect/tuning.py b/python/pyspark/ml/connect/tuning.py index 81014c89abb75..91a03148a8127 100644 --- a/python/pyspark/ml/connect/tuning.py +++ b/python/pyspark/ml/connect/tuning.py @@ -304,8 +304,6 @@ class CrossValidator( [0.04902833489813031, 0.05247132866444953] """ - _input_kwargs: Dict[str, Any] - def __init__( self, *, @@ -317,10 +315,10 @@ def __init__( parallelism: int = 1, foldCol: str = "", ) -> None: + kwargs = locals() super(CrossValidator, self).__init__() - self._setDefault(parallelism=1) - kwargs = self._input_kwargs - self._set(**kwargs) + CrossValidator._setDefault(parallelism=1) + self.__class__._set(**kwargs) @since("3.5.0") def setParams( @@ -337,8 +335,8 @@ def setParams( """ Sets params for cross validator. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = locals() + return self.__class__._set(**kwargs) @since("3.5.0") def setEstimator(self, value: Estimator) -> "CrossValidator": diff --git a/python/pyspark/ml/evaluation.py b/python/pyspark/ml/evaluation.py index c942685b35e3e..fc27824cd43e1 100644 --- a/python/pyspark/ml/evaluation.py +++ b/python/pyspark/ml/evaluation.py @@ -218,8 +218,6 @@ class BinaryClassificationEvaluator( typeConverter=TypeConverters.toInt, ) - _input_kwargs: Dict[str, Any] - def __init__( self, *, @@ -229,13 +227,13 @@ def __init__( weightCol: Optional[str] = None, numBins: int = 1000, ): + kwargs = locals() super(BinaryClassificationEvaluator, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.evaluation.BinaryClassificationEvaluator", self.uid ) self._setDefault(metricName="areaUnderROC", numBins=1000) - kwargs = self._input_kwargs - self._set(**kwargs) + self.__class__._set(**kwargs) @since("1.4.0") def setMetricName( @@ -299,8 +297,8 @@ def setParams( """ Sets params for binary classification evaluator. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = locals() + return self.__class__._set(**kwargs) @inherit_doc @@ -368,8 +366,6 @@ class RegressionEvaluator( typeConverter=TypeConverters.toBoolean, ) - _input_kwargs: Dict[str, Any] - def __init__( self, *, @@ -379,13 +375,13 @@ def __init__( weightCol: Optional[str] = None, throughOrigin: bool = False, ): + kwargs = locals() super(RegressionEvaluator, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.evaluation.RegressionEvaluator", self.uid ) self._setDefault(metricName="rmse", throughOrigin=False) - kwargs = self._input_kwargs - self._set(**kwargs) + self.__class__._set(**kwargs) @since("1.4.0") def setMetricName(self, value: "RegressionEvaluatorMetricType") -> "RegressionEvaluator": @@ -447,8 +443,8 @@ def setParams( """ Sets params for regression evaluator. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = locals() + return self.__class__._set(**kwargs) @inherit_doc @@ -548,8 +544,6 @@ class MulticlassClassificationEvaluator( typeConverter=TypeConverters.toFloat, ) - _input_kwargs: Dict[str, Any] - def __init__( self, *, @@ -567,8 +561,8 @@ def __init__( "org.apache.spark.ml.evaluation.MulticlassClassificationEvaluator", self.uid ) self._setDefault(metricName="f1", metricLabel=0.0, beta=1.0, eps=1e-15) - kwargs = self._input_kwargs - self._set(**kwargs) + kwargs = locals() + self.__class__._set(**kwargs) @since("1.5.0") def setMetricName( @@ -670,8 +664,8 @@ def setParams( """ Sets params for multiclass classification evaluator. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = locals() + return self.__class__._set(**kwargs) @inherit_doc @@ -731,8 +725,6 @@ class MultilabelClassificationEvaluator( typeConverter=TypeConverters.toFloat, ) - _input_kwargs: Dict[str, Any] - def __init__( self, *, @@ -741,13 +733,13 @@ def __init__( metricName: "MultilabelClassificationEvaluatorMetricType" = "f1Measure", metricLabel: float = 0.0, ) -> None: + kwargs = locals() super(MultilabelClassificationEvaluator, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.evaluation.MultilabelClassificationEvaluator", self.uid ) self._setDefault(metricName="f1Measure", metricLabel=0.0) - kwargs = self._input_kwargs - self._set(**kwargs) + self.__class__._set(**kwargs) @since("3.0.0") def setMetricName( @@ -805,8 +797,8 @@ def setParams( """ Sets params for multilabel classification evaluator. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = locals() + return self.__class__._set(**kwargs) @inherit_doc @@ -875,8 +867,6 @@ class ClusteringEvaluator( typeConverter=TypeConverters.toString, # type: ignore[arg-type] ) - _input_kwargs: Dict[str, Any] - def __init__( self, *, @@ -886,13 +876,13 @@ def __init__( distanceMeasure: str = "squaredEuclidean", weightCol: Optional[str] = None, ): + kwargs = locals() super(ClusteringEvaluator, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.evaluation.ClusteringEvaluator", self.uid ) self._setDefault(metricName="silhouette", distanceMeasure="squaredEuclidean") - kwargs = self._input_kwargs - self._set(**kwargs) + self.__class__._set(**kwargs) @since("2.3.0") def setParams( @@ -907,8 +897,8 @@ def setParams( """ Sets params for clustering evaluator. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = locals() + return self.__class__._set(**kwargs) @since("2.3.0") def setMetricName(self, value: "ClusteringEvaluatorMetricType") -> "ClusteringEvaluator": @@ -1012,8 +1002,6 @@ class RankingEvaluator( typeConverter=TypeConverters.toInt, ) - _input_kwargs: Dict[str, Any] - def __init__( self, *, @@ -1022,13 +1010,13 @@ def __init__( metricName: "RankingEvaluatorMetricType" = "meanAveragePrecision", k: int = 10, ): + kwargs = locals() super(RankingEvaluator, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.evaluation.RankingEvaluator", self.uid ) self._setDefault(metricName="meanAveragePrecision", k=10) - kwargs = self._input_kwargs - self._set(**kwargs) + self.__class__._set(**kwargs) @since("3.0.0") def setMetricName(self, value: "RankingEvaluatorMetricType") -> "RankingEvaluator": @@ -1084,8 +1072,8 @@ def setParams( """ Sets params for ranking evaluator. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = locals() + return self.__class__._set(**kwargs) if __name__ == "__main__": diff --git a/python/pyspark/ml/feature.py b/python/pyspark/ml/feature.py index 7a5bff9edf219..c743905e801ff 100755 --- a/python/pyspark/ml/feature.py +++ b/python/pyspark/ml/feature.py @@ -177,8 +177,6 @@ class Binarizer( ... """ - _input_kwargs: Dict[str, Any] - threshold: Param[float] = Param( Params._dummy(), "threshold", @@ -227,11 +225,11 @@ def __init__( inputCols: Optional[List[str]] = None, outputCols: Optional[List[str]] = None, ): + kwargs = locals() super(Binarizer, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.Binarizer", self.uid) self._setDefault(threshold=0.0) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) @overload def setParams( @@ -267,8 +265,8 @@ def setParams( """ Sets params for this Binarizer. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = locals() + return self.__class__._set(**kwargs) @since("1.4.0") def setThreshold(self, value: float) -> "Binarizer": @@ -561,8 +559,6 @@ class BucketedRandomProjectionLSH( True """ - _input_kwargs: Dict[str, Any] - def __init__( self, *, @@ -572,12 +568,12 @@ def __init__( numHashTables: int = 1, bucketLength: Optional[float] = None, ): + kwargs = locals() super(BucketedRandomProjectionLSH, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.feature.BucketedRandomProjectionLSH", self.uid ) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) @since("2.2.0") def setParams( @@ -592,8 +588,8 @@ def setParams( """ Sets params for this BucketedRandomProjectionLSH. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = locals() + return self.__class__._set(**kwargs) @since("2.2.0") def setBucketLength(self, value: float) -> "BucketedRandomProjectionLSH": @@ -705,8 +701,6 @@ class Bucketizer( ... """ - _input_kwargs: Dict[str, Any] - splits: Param[List[float]] = Param( Params._dummy(), "splits", @@ -781,11 +775,11 @@ def __init__( inputCols: Optional[List[str]] = None, outputCols: Optional[List[str]] = None, ): + kwargs = locals() super(Bucketizer, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.Bucketizer", self.uid) self._setDefault(handleInvalid="error") - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) @overload def setParams( @@ -824,8 +818,8 @@ def setParams( """ Sets params for this Bucketizer. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = locals() + return self.__class__._set(**kwargs) @since("1.4.0") def setSplits(self, value: List[float]) -> "Bucketizer": @@ -1045,8 +1039,6 @@ class CountVectorizer( ... """ - _input_kwargs: Dict[str, Any] - def __init__( self, *, @@ -1058,10 +1050,10 @@ def __init__( inputCol: Optional[str] = None, outputCol: Optional[str] = None, ): + kwargs = locals() super(CountVectorizer, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.CountVectorizer", self.uid) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) @since("1.6.0") def setParams( @@ -1078,8 +1070,8 @@ def setParams( """ Set the params for the CountVectorizer """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = locals() + return self.__class__._set(**kwargs) @since("1.6.0") def setMinTF(self, value: float) -> "CountVectorizer": @@ -1252,8 +1244,6 @@ class DCT(JavaTransformer, HasInputCol, HasOutputCol, JavaMLReadable["DCT"], Jav False """ - _input_kwargs: Dict[str, Any] - inverse: Param[bool] = Param( Params._dummy(), "inverse", @@ -1268,11 +1258,11 @@ def __init__( inputCol: Optional[str] = None, outputCol: Optional[str] = None, ): + kwargs = locals() super(DCT, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.DCT", self.uid) self._setDefault(inverse=False) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) @since("1.6.0") def setParams( @@ -1285,8 +1275,8 @@ def setParams( """ Sets params for this DCT. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = locals() + return self.__class__._set(**kwargs) @since("1.6.0") def setInverse(self, value: bool) -> "DCT": @@ -1354,8 +1344,6 @@ class ElementwiseProduct( True """ - _input_kwargs: Dict[str, Any] - scalingVec: Param[Vector] = Param( Params._dummy(), "scalingVec", @@ -1370,12 +1358,12 @@ def __init__( inputCol: Optional[str] = None, outputCol: Optional[str] = None, ): + kwargs = locals() super(ElementwiseProduct, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.feature.ElementwiseProduct", self.uid ) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) @since("1.5.0") def setParams( @@ -1388,8 +1376,8 @@ def setParams( """ Sets params for this ElementwiseProduct. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = locals() + return self.__class__._set(**kwargs) @since("2.0.0") def setScalingVec(self, value: Vector) -> "ElementwiseProduct": @@ -1484,8 +1472,6 @@ class FeatureHasher( True """ - _input_kwargs: Dict[str, Any] - categoricalCols: Param[List[str]] = Param( Params._dummy(), "categoricalCols", @@ -1501,11 +1487,11 @@ def __init__( outputCol: Optional[str] = None, categoricalCols: Optional[List[str]] = None, ): + kwargs = locals() super(FeatureHasher, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.FeatureHasher", self.uid) self._setDefault(numFeatures=1 << 18) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) @since("2.3.0") def setParams( @@ -1519,8 +1505,8 @@ def setParams( """ Sets params for this FeatureHasher. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = locals() + return self.__class__._set(**kwargs) @since("2.3.0") def setCategoricalCols(self, value: List[str]) -> "FeatureHasher": @@ -1598,8 +1584,6 @@ class HashingTF( 5 """ - _input_kwargs: Dict[str, Any] - binary: Param[bool] = Param( Params._dummy(), "binary", @@ -1617,11 +1601,11 @@ def __init__( inputCol: Optional[str] = None, outputCol: Optional[str] = None, ): + kwargs = locals() super(HashingTF, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.HashingTF", self.uid) self._setDefault(numFeatures=1 << 18, binary=False) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) @since("1.3.0") def setParams( @@ -1635,8 +1619,8 @@ def setParams( """ Sets params for this HashingTF. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = locals() + return self.__class__._set(**kwargs) @since("2.0.0") def setBinary(self, value: bool) -> "HashingTF": @@ -1753,8 +1737,6 @@ class IDF(JavaEstimator["IDFModel"], _IDFParams, JavaMLReadable["IDF"], JavaMLWr True """ - _input_kwargs: Dict[str, Any] - def __init__( self, *, @@ -1762,10 +1744,10 @@ def __init__( inputCol: Optional[str] = None, outputCol: Optional[str] = None, ): + kwargs = locals() super(IDF, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.IDF", self.uid) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) @since("1.4.0") def setParams( @@ -1778,8 +1760,8 @@ def setParams( """ Sets params for this IDF. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = locals() + return self.__class__._set(**kwargs) @since("1.4.0") def setMinDocFreq(self, value: int) -> "IDF": @@ -2011,8 +1993,6 @@ class Imputer( True """ - _input_kwargs: Dict[str, Any] - @overload def __init__( self, @@ -2048,10 +2028,10 @@ def __init__( outputCol: Optional[str] = None, relativeError: float = 0.001, ): + kwargs = locals() super(Imputer, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.Imputer", self.uid) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) @overload def setParams( @@ -2092,8 +2072,8 @@ def setParams( """ Sets params for this Imputer. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = locals() + return self.__class__._set(**kwargs) @since("2.2.0") def setStrategy(self, value: str) -> "Imputer": @@ -2236,14 +2216,12 @@ class Interaction( True """ - _input_kwargs: Dict[str, Any] - def __init__(self, *, inputCols: Optional[List[str]] = None, outputCol: Optional[str] = None): + kwargs = locals() super(Interaction, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.Interaction", self.uid) self._setDefault() - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) @since("3.0.0") def setParams( @@ -2252,8 +2230,8 @@ def setParams( """ Sets params for this Interaction. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = locals() + return self.__class__._set(**kwargs) @since("3.0.0") def setInputCols(self, value: List[str]) -> "Interaction": @@ -2328,14 +2306,12 @@ class MaxAbsScaler( True """ - _input_kwargs: Dict[str, Any] - def __init__(self, *, inputCol: Optional[str] = None, outputCol: Optional[str] = None): + kwargs = locals() super(MaxAbsScaler, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.MaxAbsScaler", self.uid) self._setDefault() - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) @since("2.0.0") def setParams( @@ -2344,8 +2320,8 @@ def setParams( """ Sets params for this MaxAbsScaler. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = locals() + return self.__class__._set(**kwargs) def setInputCol(self, value: str) -> "MaxAbsScaler": """ @@ -2468,8 +2444,6 @@ class MinHashLSH( True """ - _input_kwargs: Dict[str, Any] - def __init__( self, *, @@ -2478,10 +2452,10 @@ def __init__( seed: Optional[int] = None, numHashTables: int = 1, ): + kwargs = locals() super(MinHashLSH, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.MinHashLSH", self.uid) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) @since("2.2.0") def setParams( @@ -2495,8 +2469,8 @@ def setParams( """ Sets params for this MinHashLSH. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = locals() + return self.__class__._set(**kwargs) def setSeed(self, value: int) -> "MinHashLSH": """ @@ -2627,8 +2601,6 @@ class MinMaxScaler( True """ - _input_kwargs: Dict[str, Any] - def __init__( self, *, @@ -2637,10 +2609,10 @@ def __init__( inputCol: Optional[str] = None, outputCol: Optional[str] = None, ): + kwargs = locals() super(MinMaxScaler, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.MinMaxScaler", self.uid) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) @since("1.6.0") def setParams( @@ -2654,8 +2626,8 @@ def setParams( """ Sets params for this MinMaxScaler. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = locals() + return self.__class__._set(**kwargs) @since("1.6.0") def setMin(self, value: float) -> "MinMaxScaler": @@ -2786,8 +2758,6 @@ class NGram(JavaTransformer, HasInputCol, HasOutputCol, JavaMLReadable["NGram"], True """ - _input_kwargs: Dict[str, Any] - n: Param[int] = Param( Params._dummy(), "n", @@ -2798,11 +2768,11 @@ class NGram(JavaTransformer, HasInputCol, HasOutputCol, JavaMLReadable["NGram"], def __init__( self, *, n: int = 2, inputCol: Optional[str] = None, outputCol: Optional[str] = None ): + kwargs = locals() super(NGram, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.NGram", self.uid) self._setDefault(n=2) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) @since("1.5.0") def setParams( @@ -2811,8 +2781,8 @@ def setParams( """ Sets params for this NGram. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = locals() + return self.__class__._set(**kwargs) @since("1.5.0") def setN(self, value: int) -> "NGram": @@ -2880,18 +2850,16 @@ class Normalizer( True """ - _input_kwargs: Dict[str, Any] - p = Param(Params._dummy(), "p", "the p norm value.", typeConverter=TypeConverters.toFloat) def __init__( self, *, p: float = 2.0, inputCol: Optional[str] = None, outputCol: Optional[str] = None ): + kwargs = locals() super(Normalizer, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.Normalizer", self.uid) self._setDefault(p=2.0) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) @since("1.4.0") def setParams( @@ -2900,8 +2868,8 @@ def setParams( """ Sets params for this Normalizer. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = locals() + return self.__class__._set(**kwargs) @since("1.4.0") def setP(self, value: float) -> "Normalizer": @@ -3038,8 +3006,6 @@ class OneHotEncoder( True """ - _input_kwargs: Dict[str, Any] - @overload def __init__( self, @@ -3072,10 +3038,10 @@ def __init__( inputCol: Optional[str] = None, outputCol: Optional[str] = None, ): + kwargs = locals() super(OneHotEncoder, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.OneHotEncoder", self.uid) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) @overload def setParams( @@ -3113,8 +3079,8 @@ def setParams( """ Sets params for this OneHotEncoder. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = locals() + return self.__class__._set(**kwargs) @since("2.3.0") def setDropLast(self, value: bool) -> "OneHotEncoder": @@ -3262,8 +3228,6 @@ class PolynomialExpansion( True """ - _input_kwargs: Dict[str, Any] - degree: Param[int] = Param( Params._dummy(), "degree", @@ -3274,13 +3238,13 @@ class PolynomialExpansion( def __init__( self, *, degree: int = 2, inputCol: Optional[str] = None, outputCol: Optional[str] = None ): + kwargs = locals() super(PolynomialExpansion, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.feature.PolynomialExpansion", self.uid ) self._setDefault(degree=2) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) @since("1.4.0") def setParams( @@ -3289,8 +3253,8 @@ def setParams( """ Sets params for this PolynomialExpansion. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = locals() + return self.__class__._set(**kwargs) @since("1.4.0") def setDegree(self, value: int) -> "PolynomialExpansion": @@ -3424,8 +3388,6 @@ class QuantileDiscretizer( ... """ - _input_kwargs: Dict[str, Any] - numBuckets: Param[int] = Param( Params._dummy(), "numBuckets", @@ -3495,13 +3457,13 @@ def __init__( inputCols: Optional[List[str]] = None, outputCols: Optional[List[str]] = None, ): + kwargs = locals() super(QuantileDiscretizer, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.feature.QuantileDiscretizer", self.uid ) self._setDefault(numBuckets=2, relativeError=0.001, handleInvalid="error") - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) @overload def setParams( @@ -3543,8 +3505,8 @@ def setParams( """ Set the params for the QuantileDiscretizer """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = locals() + return self.__class__._set(**kwargs) @since("2.0.0") def setNumBuckets(self, value: int) -> "QuantileDiscretizer": @@ -3757,8 +3719,6 @@ class RobustScaler( True """ - _input_kwargs: Dict[str, Any] - def __init__( self, *, @@ -3770,10 +3730,10 @@ def __init__( outputCol: Optional[str] = None, relativeError: float = 0.001, ): + kwargs = locals() super(RobustScaler, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.RobustScaler", self.uid) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) @since("3.0.0") def setParams( @@ -3790,8 +3750,8 @@ def setParams( """ Sets params for this RobustScaler. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = locals() + return self.__class__._set(**kwargs) @since("3.0.0") def setLower(self, value: float) -> "RobustScaler": @@ -3938,8 +3898,6 @@ class RegexTokenizer( True """ - _input_kwargs: Dict[str, Any] - minTokenLength: Param[int] = Param( Params._dummy(), "minTokenLength", @@ -3974,11 +3932,11 @@ def __init__( outputCol: Optional[str] = None, toLowercase: bool = True, ): + kwargs = locals() super(RegexTokenizer, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.RegexTokenizer", self.uid) self._setDefault(minTokenLength=1, gaps=True, pattern="\\s+", toLowercase=True) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) @since("1.4.0") def setParams( @@ -3994,8 +3952,8 @@ def setParams( """ Sets params for this RegexTokenizer. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = locals() + return self.__class__._set(**kwargs) @since("1.4.0") def setMinTokenLength(self, value: int) -> "RegexTokenizer": @@ -4091,25 +4049,23 @@ class SQLTransformer(JavaTransformer, JavaMLReadable["SQLTransformer"], JavaMLWr True """ - _input_kwargs: Dict[str, Any] - statement = Param( Params._dummy(), "statement", "SQL statement", typeConverter=TypeConverters.toString ) def __init__(self, *, statement: Optional[str] = None): + kwargs = locals() super(SQLTransformer, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.SQLTransformer", self.uid) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) @since("1.6.0") def setParams(self, *, statement: Optional[str] = None) -> "SQLTransformer": """ Sets params for this SQLTransformer. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = locals() + return self.__class__._set(**kwargs) @since("1.6.0") def setStatement(self, value: str) -> "SQLTransformer": @@ -4217,8 +4173,6 @@ class StandardScaler( True """ - _input_kwargs: Dict[str, Any] - def __init__( self, *, @@ -4227,10 +4181,10 @@ def __init__( inputCol: Optional[str] = None, outputCol: Optional[str] = None, ): + kwargs = locals() super(StandardScaler, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.StandardScaler", self.uid) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) @since("1.4.0") def setParams( @@ -4244,8 +4198,8 @@ def setParams( """ Sets params for this StandardScaler. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = locals() + return self.__class__._set(**kwargs) @since("1.4.0") def setWithMean(self, value: bool) -> "StandardScaler": @@ -4449,8 +4403,6 @@ class StringIndexer( [(0, 0.0, 0.0), (1, 1.0, 1.0), (2, 2.0, 0.0), (3, 0.0, 1.0), (4, 0.0, 1.0), (5, 2.0, 1.0)] """ - _input_kwargs: Dict[str, Any] - @overload def __init__( self, @@ -4483,10 +4435,10 @@ def __init__( handleInvalid: str = "error", stringOrderType: str = "frequencyDesc", ): + kwargs = locals() super(StringIndexer, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.StringIndexer", self.uid) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) @overload def setParams( @@ -4524,8 +4476,8 @@ def setParams( """ Sets params for this StringIndexer. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = locals() + return self.__class__._set(**kwargs) def _create_model(self, java_model: "JavaObject") -> "StringIndexerModel": return StringIndexerModel(java_model) @@ -4708,8 +4660,6 @@ class IndexToString( StringIndexer : for converting categorical values into category indices """ - _input_kwargs: Dict[str, Any] - labels: Param[List[str]] = Param( Params._dummy(), "labels", @@ -4725,10 +4675,10 @@ def __init__( outputCol: Optional[str] = None, labels: Optional[List[str]] = None, ): + kwargs = locals() super(IndexToString, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.IndexToString", self.uid) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) @since("1.6.0") def setParams( @@ -4741,8 +4691,8 @@ def setParams( """ Sets params for this IndexToString. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = locals() + return self.__class__._set(**kwargs) @since("1.6.0") def setLabels(self, value: List[str]) -> "IndexToString": @@ -4824,8 +4774,6 @@ class StopWordsRemover( ... """ - _input_kwargs: Dict[str, Any] - stopWords: Param[List[str]] = Param( Params._dummy(), "stopWords", @@ -4880,6 +4828,7 @@ def __init__( inputCols: Optional[List[str]] = None, outputCols: Optional[List[str]] = None, ): + kwargs = locals() super(StopWordsRemover, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.feature.StopWordsRemover", self.uid @@ -4889,8 +4838,7 @@ def __init__( caseSensitive=False, locale=self._java_obj.getLocale(), ) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) @overload def setParams( @@ -4931,8 +4879,8 @@ def setParams( """ Sets params for this StopWordRemover. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = locals() + return self.__class__._set(**kwargs) @since("1.6.0") def setStopWords(self, value: List[str]) -> "StopWordsRemover": @@ -5056,13 +5004,11 @@ class Tokenizer( True """ - _input_kwargs: Dict[str, Any] - def __init__(self, *, inputCol: Optional[str] = None, outputCol: Optional[str] = None): + kwargs = locals() super(Tokenizer, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.Tokenizer", self.uid) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) @since("1.3.0") def setParams( @@ -5071,8 +5017,8 @@ def setParams( """ Sets params for this Tokenizer. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = locals() + return self.__class__._set(**kwargs) def setInputCol(self, value: str) -> "Tokenizer": """ @@ -5141,8 +5087,6 @@ class VectorAssembler( ... """ - _input_kwargs: Dict[str, Any] - handleInvalid: Param[str] = Param( Params._dummy(), "handleInvalid", @@ -5164,11 +5108,11 @@ def __init__( outputCol: Optional[str] = None, handleInvalid: str = "error", ): + kwargs = locals() super(VectorAssembler, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.VectorAssembler", self.uid) self._setDefault(handleInvalid="error") - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) @since("1.4.0") def setParams( @@ -5181,8 +5125,8 @@ def setParams( """ Sets params for this VectorAssembler. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = locals() + return self.__class__._set(**kwargs) def setInputCols(self, value: List[str]) -> "VectorAssembler": """ @@ -5337,8 +5281,6 @@ class VectorIndexer( DenseVector([2.0, 1.0]) """ - _input_kwargs: Dict[str, Any] - def __init__( self, *, @@ -5347,10 +5289,10 @@ def __init__( outputCol: Optional[str] = None, handleInvalid: str = "error", ): + kwargs = locals() super(VectorIndexer, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.VectorIndexer", self.uid) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) @since("1.4.0") def setParams( @@ -5364,8 +5306,8 @@ def setParams( """ Sets params for this VectorIndexer. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = locals() + return self.__class__._set(**kwargs) @since("1.4.0") def setMaxCategories(self, value: int) -> "VectorIndexer": @@ -5492,8 +5434,6 @@ class VectorSlicer( True """ - _input_kwargs: Dict[str, Any] - indices: Param[List[int]] = Param( Params._dummy(), "indices", @@ -5519,11 +5459,11 @@ def __init__( indices: Optional[List[int]] = None, names: Optional[List[str]] = None, ): + kwargs = locals() super(VectorSlicer, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.VectorSlicer", self.uid) self._setDefault(indices=[], names=[]) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) @since("1.6.0") def setParams( @@ -5537,8 +5477,8 @@ def setParams( """ Sets params for this VectorSlicer. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = locals() + return self.__class__._set(**kwargs) @since("1.6.0") def setIndices(self, value: List[int]) -> "VectorSlicer": @@ -5740,8 +5680,6 @@ class Word2Vec( True """ - _input_kwargs: Dict[str, Any] - def __init__( self, *, @@ -5756,10 +5694,10 @@ def __init__( windowSize: int = 5, maxSentenceLength: int = 1000, ): + kwargs = locals() super(Word2Vec, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.Word2Vec", self.uid) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) @since("1.4.0") def setParams( @@ -5779,8 +5717,8 @@ def setParams( """ Sets params for this Word2Vec. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = locals() + return self.__class__._set(**kwargs) @since("1.4.0") def setVectorSize(self, value: int) -> "Word2Vec": @@ -5971,8 +5909,6 @@ class PCA(JavaEstimator["PCAModel"], _PCAParams, JavaMLReadable["PCA"], JavaMLWr True """ - _input_kwargs: Dict[str, Any] - def __init__( self, *, @@ -5980,10 +5916,10 @@ def __init__( inputCol: Optional[str] = None, outputCol: Optional[str] = None, ): + kwargs = locals() super(PCA, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.PCA", self.uid) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) @since("1.5.0") def setParams( @@ -5996,8 +5932,8 @@ def setParams( """ Set params for this PCA. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = locals() + return self.__class__._set(**kwargs) @since("1.5.0") def setK(self, value: int) -> "PCA": @@ -6209,8 +6145,6 @@ class RFormula( 'RFormulaModel(ResolvedRFormula(label=y, terms=[x,s], hasIntercept=true)) (uid=...)' """ - _input_kwargs: Dict[str, Any] - def __init__( self, *, @@ -6221,10 +6155,10 @@ def __init__( stringIndexerOrderType: str = "frequencyDesc", handleInvalid: str = "error", ): + kwargs = locals() super(RFormula, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.RFormula", self.uid) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) @since("1.5.0") def setParams( @@ -6240,8 +6174,8 @@ def setParams( """ Sets params for RFormula. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = locals() + return self.__class__._set(**kwargs) @since("1.5.0") def setFormula(self, value: str) -> "RFormula": @@ -6575,8 +6509,6 @@ class ChiSqSelector( True """ - _input_kwargs: Dict[str, Any] - def __init__( self, *, @@ -6590,10 +6522,10 @@ def __init__( fdr: float = 0.05, fwe: float = 0.05, ): + kwargs = locals() super(ChiSqSelector, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.ChiSqSelector", self.uid) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) @since("2.0.0") def setParams( @@ -6612,8 +6544,8 @@ def setParams( """ Sets params for this ChiSqSelector. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = locals() + return self.__class__._set(**kwargs) def _create_model(self, java_model: "JavaObject") -> "ChiSqSelectorModel": return ChiSqSelectorModel(java_model) @@ -6669,8 +6601,6 @@ class VectorSizeHint( True """ - _input_kwargs: Dict[str, Any] - size: Param[int] = Param( Params._dummy(), "size", "Size of vectors in column.", typeConverter=TypeConverters.toInt ) @@ -6693,10 +6623,11 @@ def __init__( size: Optional[int] = None, handleInvalid: str = "error", ): + kwargs = locals() super(VectorSizeHint, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.VectorSizeHint", self.uid) self._setDefault(handleInvalid="error") - self.setParams(**self._input_kwargs) + self.__class__.setParams(**kwargs) @since("2.3.0") def setParams( @@ -6709,8 +6640,8 @@ def setParams( """ Sets params for this VectorSizeHint. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = locals() + return self.__class__._set(**kwargs) @since("2.3.0") def getSize(self) -> int: @@ -6810,8 +6741,6 @@ class VarianceThresholdSelector( True """ - _input_kwargs: Dict[str, Any] - def __init__( self, *, @@ -6819,13 +6748,13 @@ def __init__( outputCol: Optional[str] = None, varianceThreshold: float = 0.0, ): + kwargs = locals() super(VarianceThresholdSelector, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.feature.VarianceThresholdSelector", self.uid ) self._setDefault(varianceThreshold=0.0) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) @since("3.1.0") def setParams( @@ -6838,8 +6767,8 @@ def setParams( """ Sets params for this VarianceThresholdSelector. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = locals() + return self.__class__._set(**kwargs) @since("3.1.0") def setVarianceThreshold(self, value: float) -> "VarianceThresholdSelector": @@ -7051,8 +6980,6 @@ class UnivariateFeatureSelector( True """ - _input_kwargs: Dict[str, Any] - def __init__( self, *, @@ -7061,12 +6988,12 @@ def __init__( labelCol: str = "label", selectionMode: str = "numTopFeatures", ): + kwargs = locals() super(UnivariateFeatureSelector, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.feature.UnivariateFeatureSelector", self.uid ) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) @since("3.1.1") def setParams( @@ -7080,8 +7007,8 @@ def setParams( """ Sets params for this UnivariateFeatureSelector. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = locals() + return self.__class__._set(**kwargs) @since("3.1.1") def setFeatureType(self, value: str) -> "UnivariateFeatureSelector": diff --git a/python/pyspark/ml/fpm.py b/python/pyspark/ml/fpm.py index 67f737b33241c..0de59465746b8 100644 --- a/python/pyspark/ml/fpm.py +++ b/python/pyspark/ml/fpm.py @@ -235,7 +235,6 @@ class FPGrowth( >>> fpm.transform(data).take(1) == model2.transform(data).take(1) True """ - _input_kwargs: Dict[str, Any] def __init__( self, @@ -246,10 +245,10 @@ def __init__( predictionCol: str = "prediction", numPartitions: Optional[int] = None, ): + kwargs = locals() super(FPGrowth, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.fpm.FPGrowth", self.uid) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) @since("2.2.0") def setParams( @@ -261,8 +260,8 @@ def setParams( predictionCol: str = "prediction", numPartitions: Optional[int] = None, ) -> "FPGrowth": - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = locals() + return self.__class__._set(**kwargs) def setItemsCol(self, value: str) -> "FPGrowth": """ @@ -344,8 +343,6 @@ class PrefixSpan(JavaParams): ... """ - _input_kwargs: Dict[str, Any] - minSupport: Param[float] = Param( Params._dummy(), "minSupport", @@ -389,13 +386,13 @@ def __init__( maxLocalProjDBSize: int = 32000000, sequenceCol: str = "sequence", ): + kwargs = locals() super(PrefixSpan, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.fpm.PrefixSpan", self.uid) self._setDefault( minSupport=0.1, maxPatternLength=10, maxLocalProjDBSize=32000000, sequenceCol="sequence" ) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) @since("2.4.0") def setParams( @@ -406,8 +403,8 @@ def setParams( maxLocalProjDBSize: int = 32000000, sequenceCol: str = "sequence", ) -> "PrefixSpan": - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = locals() + return self.__class__._set(**kwargs) @since("3.0.0") def setMinSupport(self, value: float) -> "PrefixSpan": diff --git a/python/pyspark/ml/pipeline.py b/python/pyspark/ml/pipeline.py index 343af8827cae2..afd0ef6a82ff5 100644 --- a/python/pyspark/ml/pipeline.py +++ b/python/pyspark/ml/pipeline.py @@ -68,12 +68,10 @@ class Pipeline(Estimator["PipelineModel"], MLReadable["Pipeline"], MLWritable): Params._dummy(), "stages", "a list of pipeline stages" ) - _input_kwargs: Dict[str, Any] - def __init__(self, *, stages: Optional[List["PipelineStage"]] = None): + kwargs = locals() super(Pipeline, self).__init__() - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) def setStages(self, value: List["PipelineStage"]) -> "Pipeline": """ @@ -106,8 +104,8 @@ def setParams(self, *, stages: Optional[List["PipelineStage"]] = None) -> "Pipel """ Sets params for Pipeline. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = locals() + return self.__class__._set(**kwargs) def _fit(self, dataset: DataFrame) -> "PipelineModel": stages = self.getStages() diff --git a/python/pyspark/ml/recommendation.py b/python/pyspark/ml/recommendation.py index b711bd7397ffb..43ac1bac08b78 100644 --- a/python/pyspark/ml/recommendation.py +++ b/python/pyspark/ml/recommendation.py @@ -364,8 +364,6 @@ class ALS(JavaEstimator["ALSModel"], _ALSParams, JavaMLWritable, JavaMLReadable[ True """ - _input_kwargs: Dict[str, Any] - def __init__( self, *, @@ -387,10 +385,10 @@ def __init__( coldStartStrategy: str = "nan", blockSize: int = 4096, ): + kwargs = locals() super(ALS, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.recommendation.ALS", self.uid) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) @since("1.4.0") def setParams( @@ -417,8 +415,8 @@ def setParams( """ Sets params for ALS. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = locals() + return self.__class__._set(**kwargs) def _create_model(self, java_model: "JavaObject") -> "ALSModel": return ALSModel(java_model) diff --git a/python/pyspark/ml/regression.py b/python/pyspark/ml/regression.py index 93e5c3980ac31..e4ab28b6669fd 100644 --- a/python/pyspark/ml/regression.py +++ b/python/pyspark/ml/regression.py @@ -294,8 +294,6 @@ class LinearRegression( >>> model.write().format("pmml").save(model_path + "_2") """ - _input_kwargs: Dict[str, Any] - def __init__( self, *, @@ -315,12 +313,12 @@ def __init__( epsilon: float = 1.35, maxBlockSizeInMB: float = 0.0, ): + kwargs = locals() super(LinearRegression, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.regression.LinearRegression", self.uid ) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) @since("1.4.0") def setParams( @@ -345,8 +343,8 @@ def setParams( """ Sets params for linear regression. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = locals() + return self.__class__._set(**kwargs) def _create_model(self, java_model: "JavaObject") -> "LinearRegressionModel": return LinearRegressionModel(java_model) @@ -844,8 +842,6 @@ class IsotonicRegression( True """ - _input_kwargs: Dict[str, Any] - def __init__( self, *, @@ -856,12 +852,12 @@ def __init__( isotonic: bool = True, featureIndex: int = 0, ): + kwargs = locals() super(IsotonicRegression, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.regression.IsotonicRegression", self.uid ) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) def setParams( self, @@ -876,8 +872,8 @@ def setParams( """ Set the params for IsotonicRegression. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = locals() + return self.__class__._set(**kwargs) def _create_model(self, java_model: "JavaObject") -> "IsotonicRegressionModel": return IsotonicRegressionModel(java_model) @@ -1086,8 +1082,6 @@ class DecisionTreeRegressor( DecisionTreeRegressionModel...depth=1, numNodes=3... """ - _input_kwargs: Dict[str, Any] - def __init__( self, *, @@ -1108,12 +1102,12 @@ def __init__( leafCol: str = "", minWeightFractionPerNode: float = 0.0, ): + kwargs = locals() super(DecisionTreeRegressor, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.regression.DecisionTreeRegressor", self.uid ) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) @since("1.4.0") def setParams( @@ -1139,8 +1133,8 @@ def setParams( """ Sets params for the DecisionTreeRegressor. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = locals() + return self.__class__._set(**kwargs) def _create_model(self, java_model: "JavaObject") -> "DecisionTreeRegressionModel": return DecisionTreeRegressionModel(java_model) @@ -1372,8 +1366,6 @@ class RandomForestRegressor( True """ - _input_kwargs: Dict[str, Any] - def __init__( self, *, @@ -1397,12 +1389,12 @@ def __init__( weightCol: Optional[str] = None, bootstrap: Optional[bool] = True, ): + kwargs = locals() super(RandomForestRegressor, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.regression.RandomForestRegressor", self.uid ) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) @since("1.4.0") def setParams( @@ -1431,8 +1423,8 @@ def setParams( """ Sets params for linear regression. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = locals() + return self.__class__._set(**kwargs) def _create_model(self, java_model: "JavaObject") -> "RandomForestRegressionModel": return RandomForestRegressionModel(java_model) @@ -1699,8 +1691,6 @@ class GBTRegressor( 0.01 """ - _input_kwargs: Dict[str, Any] - def __init__( self, *, @@ -1727,10 +1717,10 @@ def __init__( minWeightFractionPerNode: float = 0.0, weightCol: Optional[str] = None, ): + kwargs = locals() super(GBTRegressor, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.regression.GBTRegressor", self.uid) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) @since("1.4.0") def setParams( @@ -1762,8 +1752,8 @@ def setParams( """ Sets params for Gradient Boosted Tree Regression. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = locals() + return self.__class__._set(**kwargs) def _create_model(self, java_model: "JavaObject") -> "GBTRegressionModel": return GBTRegressionModel(java_model) @@ -2075,8 +2065,6 @@ class AFTSurvivalRegression( .. versionadded:: 1.6.0 """ - _input_kwargs: Dict[str, Any] - def __init__( self, *, @@ -2102,12 +2090,12 @@ def __init__( aggregationDepth: int = 2, maxBlockSizeInMB: float = 0.0, ): + kwargs = locals() super(AFTSurvivalRegression, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.regression.AFTSurvivalRegression", self.uid ) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) @since("1.6.0") def setParams( @@ -2135,8 +2123,8 @@ def setParams( aggregationDepth: int = 2, maxBlockSizeInMB: float = 0.0, ) -> "AFTSurvivalRegression": - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = locals() + return self.__class__._set(**kwargs) def _create_model(self, java_model: "JavaObject") -> "AFTSurvivalRegressionModel": return AFTSurvivalRegressionModel(java_model) @@ -2466,8 +2454,6 @@ class GeneralizedLinearRegression( True """ - _input_kwargs: Dict[str, Any] - def __init__( self, *, @@ -2488,13 +2474,13 @@ def __init__( offsetCol: Optional[str] = None, aggregationDepth: int = 2, ): + kwargs = locals() super(GeneralizedLinearRegression, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.regression.GeneralizedLinearRegression", self.uid ) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) @since("2.0.0") def setParams( @@ -2520,8 +2506,8 @@ def setParams( """ Sets params for generalized linear regression. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = locals() + return self.__class__._set(**kwargs) def _create_model(self, java_model: "JavaObject") -> "GeneralizedLinearRegressionModel": return GeneralizedLinearRegressionModel(java_model) @@ -3029,8 +3015,6 @@ class FMRegressor( True """ - _input_kwargs: Dict[str, Any] - def __init__( self, *, @@ -3049,10 +3033,10 @@ def __init__( solver: str = "adamW", seed: Optional[int] = None, ): + kwargs = locals() super(FMRegressor, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.regression.FMRegressor", self.uid) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) @since("3.0.0") def setParams( @@ -3076,8 +3060,8 @@ def setParams( """ Sets Params for FMRegressor. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = locals() + return self.__class__._set(**kwargs) def _create_model(self, java_model: "JavaObject") -> "FMRegressionModel": return FMRegressionModel(java_model) diff --git a/python/pyspark/ml/tuning.py b/python/pyspark/ml/tuning.py index 64f3c32253d61..112b4172b9a0f 100644 --- a/python/pyspark/ml/tuning.py +++ b/python/pyspark/ml/tuning.py @@ -714,8 +714,6 @@ class CrossValidator( 0.8333... """ - _input_kwargs: Dict[str, Any] - def __init__( self, *, @@ -728,10 +726,10 @@ def __init__( collectSubModels: bool = False, foldCol: str = "", ) -> None: + kwargs = locals() super(CrossValidator, self).__init__() self._setDefault(parallelism=1) - kwargs = self._input_kwargs - self._set(**kwargs) + self.__class__._set(**kwargs) @since("1.4.0") def setParams( @@ -749,8 +747,8 @@ def setParams( """ Sets params for cross validator. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = locals() + return self.__class__._set(**kwargs) @since("2.0.0") def setEstimator(self, value: Estimator) -> "CrossValidator": @@ -1337,8 +1335,6 @@ class TrainValidationSplit( 0.833... """ - _input_kwargs: Dict[str, Any] - def __init__( self, *, @@ -1350,10 +1346,10 @@ def __init__( collectSubModels: bool = False, seed: Optional[int] = None, ) -> None: + kwargs = locals() super(TrainValidationSplit, self).__init__() self._setDefault(parallelism=1) - kwargs = self._input_kwargs - self._set(**kwargs) + self.__class__._set(**kwargs) @since("2.0.0") def setParams( @@ -1370,8 +1366,8 @@ def setParams( """ Sets params for the train validation split. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = locals() + return self.__class__._set(**kwargs) @since("2.0.0") def setEstimator(self, value: Estimator) -> "TrainValidationSplit": diff --git a/python/pyspark/testing/mlutils.py b/python/pyspark/testing/mlutils.py index b7cbd9845f576..2d807e6b7d172 100644 --- a/python/pyspark/testing/mlutils.py +++ b/python/pyspark/testing/mlutils.py @@ -191,9 +191,9 @@ def __init__( regParam=0.0, rawPredictionCol="rawPrediction", ): + kwargs = locals() super(DummyLogisticRegression, self).__init__() - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) def setParams( self, @@ -205,8 +205,8 @@ def setParams( regParam=0.0, rawPredictionCol="rawPrediction", ): - kwargs = self._input_kwargs - self._set(**kwargs) + kwargs = locals() + self.__class__._set(**kwargs) return self def _fit(self, dataset): From 9b1de889465d4517f947c09cee973b1d36789979 Mon Sep 17 00:00:00 2001 From: Hyukjin Kwon Date: Mon, 27 Nov 2023 12:27:32 +0900 Subject: [PATCH 5/7] Fix linter and retain the test for keyword-only decorator --- python/pyspark/ml/clustering.py | 2 +- python/pyspark/ml/connect/feature.py | 2 +- python/pyspark/ml/connect/tuning.py | 2 +- python/pyspark/ml/evaluation.py | 2 +- python/pyspark/ml/fpm.py | 2 +- python/pyspark/ml/recommendation.py | 2 +- python/pyspark/ml/regression.py | 2 +- python/pyspark/tests/test_util.py | 39 ++++++++++++++++++++++++++++ 8 files changed, 46 insertions(+), 7 deletions(-) diff --git a/python/pyspark/ml/clustering.py b/python/pyspark/ml/clustering.py index 2b6d8f108dc90..bffb43fa80934 100644 --- a/python/pyspark/ml/clustering.py +++ b/python/pyspark/ml/clustering.py @@ -17,7 +17,7 @@ import sys import warnings -from typing import Any, Dict, List, Optional, TYPE_CHECKING +from typing import Any, List, Optional, TYPE_CHECKING import numpy as np diff --git a/python/pyspark/ml/connect/feature.py b/python/pyspark/ml/connect/feature.py index d100b330e471c..23b3ff71c718d 100644 --- a/python/pyspark/ml/connect/feature.py +++ b/python/pyspark/ml/connect/feature.py @@ -16,7 +16,7 @@ # import pickle -from typing import Any, Union, List, Tuple, Callable, Dict, Optional +from typing import Any, Union, List, Tuple, Callable, Optional import numpy as np import pandas as pd diff --git a/python/pyspark/ml/connect/tuning.py b/python/pyspark/ml/connect/tuning.py index 91a03148a8127..4d870ccad323d 100644 --- a/python/pyspark/ml/connect/tuning.py +++ b/python/pyspark/ml/connect/tuning.py @@ -317,7 +317,7 @@ def __init__( ) -> None: kwargs = locals() super(CrossValidator, self).__init__() - CrossValidator._setDefault(parallelism=1) + self._setDefault(parallelism=1) self.__class__._set(**kwargs) @since("3.5.0") diff --git a/python/pyspark/ml/evaluation.py b/python/pyspark/ml/evaluation.py index fc27824cd43e1..9942a5e98be31 100644 --- a/python/pyspark/ml/evaluation.py +++ b/python/pyspark/ml/evaluation.py @@ -17,7 +17,7 @@ import sys from abc import abstractmethod, ABCMeta -from typing import Any, Dict, Optional, TYPE_CHECKING +from typing import Optional, TYPE_CHECKING from pyspark import since from pyspark.ml.wrapper import JavaParams diff --git a/python/pyspark/ml/fpm.py b/python/pyspark/ml/fpm.py index 0de59465746b8..71a45bc55a6db 100644 --- a/python/pyspark/ml/fpm.py +++ b/python/pyspark/ml/fpm.py @@ -16,7 +16,7 @@ # import sys -from typing import Any, Dict, Optional, TYPE_CHECKING +from typing import Any, Optional, TYPE_CHECKING from pyspark import since from pyspark.sql import DataFrame diff --git a/python/pyspark/ml/recommendation.py b/python/pyspark/ml/recommendation.py index 43ac1bac08b78..8b96850d1fd89 100644 --- a/python/pyspark/ml/recommendation.py +++ b/python/pyspark/ml/recommendation.py @@ -16,7 +16,7 @@ # import sys -from typing import Any, Dict, Optional, TYPE_CHECKING +from typing import Any, Optional, TYPE_CHECKING from pyspark import since from pyspark.ml.param.shared import ( diff --git a/python/pyspark/ml/regression.py b/python/pyspark/ml/regression.py index e4ab28b6669fd..21860cd7aa0be 100644 --- a/python/pyspark/ml/regression.py +++ b/python/pyspark/ml/regression.py @@ -16,7 +16,7 @@ # import sys -from typing import Any, Dict, Generic, List, Optional, TypeVar, TYPE_CHECKING +from typing import Any, Generic, List, Optional, TypeVar, TYPE_CHECKING from abc import ABCMeta from pyspark import since diff --git a/python/pyspark/tests/test_util.py b/python/pyspark/tests/test_util.py index a494d7b4dacd4..af104d683aab8 100644 --- a/python/pyspark/tests/test_util.py +++ b/python/pyspark/tests/test_util.py @@ -19,10 +19,49 @@ from py4j.protocol import Py4JJavaError +from pyspark import keyword_only from pyspark.testing.utils import PySparkTestCase, eventually from pyspark.find_spark_home import _find_spark_home +class KeywordOnlyTests(unittest.TestCase): + class Wrapped: + @keyword_only + def set(self, x=None, y=None): + if "x" in self._input_kwargs: + self._x = self._input_kwargs["x"] + if "y" in self._input_kwargs: + self._y = self._input_kwargs["y"] + return x, y + + def test_keywords(self): + w = self.Wrapped() + x, y = w.set(y=1) + self.assertEqual(y, 1) + self.assertEqual(y, w._y) + self.assertIsNone(x) + self.assertFalse(hasattr(w, "_x")) + + def test_non_keywords(self): + w = self.Wrapped() + self.assertRaises(TypeError, lambda: w.set(0, y=1)) + + def test_kwarg_ownership(self): + # test _input_kwargs is owned by each class instance and not a shared static variable + class Setter: + @keyword_only + def set(self, x=None, other=None, other_x=None): + if "other" in self._input_kwargs: + self._input_kwargs["other"].set(x=self._input_kwargs["other_x"]) + self._x = self._input_kwargs["x"] + + a = Setter() + b = Setter() + a.set(x=1, other=b, other_x=2) + self.assertEqual(a._x, 1) + self.assertEqual(b._x, 2) + + class UtilTests(PySparkTestCase): def test_py4j_str(self): with self.assertRaises(Py4JJavaError) as context: From c6b6aec0174fe7cf969543885e6367386f6cfa8e Mon Sep 17 00:00:00 2001 From: Hyukjin Kwon Date: Mon, 27 Nov 2023 13:37:40 +0900 Subject: [PATCH 6/7] Add a since version --- python/pyspark/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/python/pyspark/__init__.py b/python/pyspark/__init__.py index f859265afbab7..31ab2e4ce1b1a 100644 --- a/python/pyspark/__init__.py +++ b/python/pyspark/__init__.py @@ -76,7 +76,9 @@ def since(version: Union[str, float]) -> Callable[[_F], _F]: indent_p = re.compile(r"\n( +)") def deco(f: _F) -> _F: - assert f.__doc__ is not None + if f.__doc__ is None: + # Should still at least add a versionadded. + f.__doc__ = "" indents = indent_p.findall(f.__doc__) indent = " " * (min(len(m) for m in indents) if indents else 0) From 842c104b08ec3bfd3dd103ca1660a309e0aa6518 Mon Sep 17 00:00:00 2001 From: Hyukjin Kwon Date: Mon, 27 Nov 2023 14:36:19 +0900 Subject: [PATCH 7/7] Make it more future proof with disallowing `_` and `None` --- python/pyspark/ml/classification.py | 72 +++-- python/pyspark/ml/clustering.py | 40 ++- python/pyspark/ml/connect/classification.py | 4 +- python/pyspark/ml/connect/feature.py | 12 +- python/pyspark/ml/connect/pipeline.py | 8 +- python/pyspark/ml/connect/tuning.py | 8 +- python/pyspark/ml/evaluation.py | 48 +++- python/pyspark/ml/feature.py | 296 +++++++++++++++----- python/pyspark/ml/fpm.py | 16 +- python/pyspark/ml/pipeline.py | 8 +- python/pyspark/ml/recommendation.py | 8 +- python/pyspark/ml/regression.py | 64 +++-- python/pyspark/ml/tuning.py | 16 +- python/pyspark/testing/mlutils.py | 8 +- 14 files changed, 456 insertions(+), 152 deletions(-) diff --git a/python/pyspark/ml/classification.py b/python/pyspark/ml/classification.py index 3032376f70394..e02532b4330f6 100644 --- a/python/pyspark/ml/classification.py +++ b/python/pyspark/ml/classification.py @@ -723,7 +723,9 @@ def __init__( aggregationDepth: int = 2, maxBlockSizeInMB: float = 0.0, ): - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(LinearSVC, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.classification.LinearSVC", self.uid @@ -751,7 +753,9 @@ def setParams( """ Sets params for Linear SVM Classifier. """ - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) return self.__class__._set(**kwargs) def _create_model(self, java_model: "JavaObject") -> "LinearSVCModel": @@ -1287,7 +1291,9 @@ def __init__( """ If the threshold and thresholds Params are both set, they must be equivalent. """ - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(LogisticRegression, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.classification.LogisticRegression", self.uid @@ -1379,7 +1385,9 @@ def setParams( Sets params for logistic regression. If the threshold and thresholds Params are both set, they must be equivalent. """ - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) self.__class__._set(**kwargs) self._checkThresholdConsistency() return self @@ -1757,7 +1765,9 @@ def __init__( leafCol: str = "", minWeightFractionPerNode: float = 0.0, ): - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(DecisionTreeClassifier, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.classification.DecisionTreeClassifier", self.uid @@ -1789,7 +1799,9 @@ def setParams( """ Sets params for the DecisionTreeClassifier. """ - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) return self.__class__.setParams(**kwargs) def _create_model(self, java_model: "JavaObject") -> "DecisionTreeClassificationModel": @@ -2037,7 +2049,9 @@ def __init__( weightCol: Optional[str] = None, bootstrap: Optional[bool] = True, ): - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(RandomForestClassifier, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.classification.RandomForestClassifier", self.uid @@ -2073,7 +2087,9 @@ def setParams( """ Sets params for linear classification. """ - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) return self.__class__._set(**kwargs) def _create_model(self, java_model: "JavaObject") -> "RandomForestClassificationModel": @@ -2489,7 +2505,9 @@ def __init__( minWeightFractionPerNode: float = 0.0, weightCol: Optional[str] = None, ): - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(GBTClassifier, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.classification.GBTClassifier", self.uid @@ -2526,7 +2544,9 @@ def setParams( """ Sets params for Gradient Boosted Tree Classification. """ - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) return self.__class__._set(**kwargs) def _create_model(self, java_model: "JavaObject") -> "GBTClassificationModel": @@ -2853,7 +2873,9 @@ def __init__( thresholds: Optional[List[float]] = None, weightCol: Optional[str] = None, ): - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(NaiveBayes, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.classification.NaiveBayes", self.uid @@ -2877,7 +2899,9 @@ def setParams( """ Sets params for Naive Bayes. """ - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) return self.__class__._set(**kwargs) def _create_model(self, java_model: "JavaObject") -> "NaiveBayesModel": @@ -3095,7 +3119,9 @@ def __init__( probabilityCol: str = "probability", rawPredictionCol: str = "rawPrediction", ): - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(MultilayerPerceptronClassifier, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.classification.MultilayerPerceptronClassifier", self.uid @@ -3123,7 +3149,9 @@ def setParams( """ Sets params for MultilayerPerceptronClassifier. """ - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) return self.__class__._set(**kwargs) def _create_model(self, java_model: "JavaObject") -> "MultilayerPerceptronClassificationModel": @@ -3343,7 +3371,9 @@ def __init__( weightCol: Optional[str] = None, parallelism: int = 1, ): - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(OneVsRest, self).__init__() self._setDefault(parallelism=1) self.__class__._set(**kwargs) @@ -3363,7 +3393,9 @@ def setParams( """ Sets params for OneVsRest. """ - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) return self.__class__._set(**kwargs) @since("2.0.0") @@ -3956,7 +3988,9 @@ def __init__( thresholds: Optional[List[float]] = None, seed: Optional[int] = None, ): - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(FMClassifier, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.classification.FMClassifier", self.uid @@ -3988,7 +4022,9 @@ def setParams( """ Sets Params for FMClassifier. """ - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) return self.__class__._set(**kwargs) def _create_model(self, java_model: "JavaObject") -> "FMClassificationModel": diff --git a/python/pyspark/ml/clustering.py b/python/pyspark/ml/clustering.py index bffb43fa80934..4865418b71b62 100644 --- a/python/pyspark/ml/clustering.py +++ b/python/pyspark/ml/clustering.py @@ -414,7 +414,9 @@ def __init__( aggregationDepth: int = 2, weightCol: Optional[str] = None, ): - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(GaussianMixture, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.clustering.GaussianMixture", self.uid @@ -441,7 +443,9 @@ def setParams( """ Sets params for GaussianMixture. """ - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) return self.__class__._set(**kwargs) @since("2.0.0") @@ -778,7 +782,9 @@ def __init__( solver: str = "auto", maxBlockSizeInMB: float = 0.0, ): - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(KMeans, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.clustering.KMeans", self.uid) self.__class__.setParams(**kwargs) @@ -806,7 +812,9 @@ def setParams( """ Sets params for KMeans. """ - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) return self.__class__._set(**kwargs) @since("1.5.0") @@ -1111,7 +1119,9 @@ def __init__( distanceMeasure: str = "euclidean", weightCol: Optional[str] = None, ): - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(BisectingKMeans, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.clustering.BisectingKMeans", self.uid @@ -1134,7 +1144,9 @@ def setParams( """ Sets params for BisectingKMeans. """ - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) return self.__class__._set(**kwargs) @since("2.0.0") @@ -1657,7 +1669,9 @@ def __init__( topicDistributionCol: str = "topicDistribution", keepLastCheckpoint: bool = True, ): - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(LDA, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.clustering.LDA", self.uid) self.__class__.setParams(**kwargs) @@ -1690,7 +1704,9 @@ def setParams( """ Sets params for LDA. """ - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) return self.__class__._set(**kwargs) @since("2.0.0") @@ -1987,7 +2003,9 @@ def __init__( dstCol: str = "dst", weightCol: Optional[str] = None, ): - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(PowerIterationClustering, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.clustering.PowerIterationClustering", self.uid @@ -2008,7 +2026,9 @@ def setParams( """ Sets params for PowerIterationClustering. """ - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) return self.__class__._set(**kwargs) @since("2.4.0") diff --git a/python/pyspark/ml/connect/classification.py b/python/pyspark/ml/connect/classification.py index fe0d4234a7586..3b93973cc21ef 100644 --- a/python/pyspark/ml/connect/classification.py +++ b/python/pyspark/ml/connect/classification.py @@ -207,7 +207,9 @@ def __init__( seed: int = 0, ) """ - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(LogisticRegression, self).__init__() self.__class__._set(**kwargs) diff --git a/python/pyspark/ml/connect/feature.py b/python/pyspark/ml/connect/feature.py index 23b3ff71c718d..116fc01b61924 100644 --- a/python/pyspark/ml/connect/feature.py +++ b/python/pyspark/ml/connect/feature.py @@ -67,7 +67,9 @@ class MaxAbsScaler(Estimator, HasInputCol, HasOutputCol, ParamsReadWrite): """ def __init__(self, *, inputCol: Optional[str] = None, outputCol: Optional[str] = None) -> None: - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super().__init__() MaxAbsScaler._set(**kwargs) @@ -173,7 +175,9 @@ class StandardScaler(Estimator, HasInputCol, HasOutputCol, ParamsReadWrite): """ def __init__(self, inputCol: Optional[str] = None, outputCol: Optional[str] = None) -> None: - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super().__init__() StandardScaler._set(**kwargs) @@ -320,7 +324,9 @@ def __init__( featureSizes: Optional[List[int]] = None, handleInvalid: Optional[str] = "error", ) -> None: - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super().__init__() ArrayAssembler._set(**kwargs) self._setDefault(handleInvalid="error") diff --git a/python/pyspark/ml/connect/pipeline.py b/python/pyspark/ml/connect/pipeline.py index 360756657001d..dc3332863d690 100644 --- a/python/pyspark/ml/connect/pipeline.py +++ b/python/pyspark/ml/connect/pipeline.py @@ -136,7 +136,9 @@ class Pipeline(Estimator["PipelineModel"], _PipelineReadWrite): ) # type: ignore[assignment] def __init__(self, *, stages: Optional[List[Params]] = None): - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(Pipeline, self).__init__() self.__class__.setParams(**kwargs) @@ -171,7 +173,9 @@ def setParams(self, *, stages: Optional[List[Params]] = None) -> "Pipeline": """ Sets params for Pipeline. """ - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) return self.__class__._set(**kwargs) def _fit(self, dataset: Union[DataFrame, pd.DataFrame]) -> "PipelineModel": diff --git a/python/pyspark/ml/connect/tuning.py b/python/pyspark/ml/connect/tuning.py index 4d870ccad323d..1b26f750afa12 100644 --- a/python/pyspark/ml/connect/tuning.py +++ b/python/pyspark/ml/connect/tuning.py @@ -315,7 +315,9 @@ def __init__( parallelism: int = 1, foldCol: str = "", ) -> None: - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(CrossValidator, self).__init__() self._setDefault(parallelism=1) self.__class__._set(**kwargs) @@ -335,7 +337,9 @@ def setParams( """ Sets params for cross validator. """ - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) return self.__class__._set(**kwargs) @since("3.5.0") diff --git a/python/pyspark/ml/evaluation.py b/python/pyspark/ml/evaluation.py index 9942a5e98be31..5db5e6bc1f4d7 100644 --- a/python/pyspark/ml/evaluation.py +++ b/python/pyspark/ml/evaluation.py @@ -227,7 +227,9 @@ def __init__( weightCol: Optional[str] = None, numBins: int = 1000, ): - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(BinaryClassificationEvaluator, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.evaluation.BinaryClassificationEvaluator", self.uid @@ -297,7 +299,9 @@ def setParams( """ Sets params for binary classification evaluator. """ - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) return self.__class__._set(**kwargs) @@ -375,7 +379,9 @@ def __init__( weightCol: Optional[str] = None, throughOrigin: bool = False, ): - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(RegressionEvaluator, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.evaluation.RegressionEvaluator", self.uid @@ -443,7 +449,9 @@ def setParams( """ Sets params for regression evaluator. """ - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) return self.__class__._set(**kwargs) @@ -561,7 +569,9 @@ def __init__( "org.apache.spark.ml.evaluation.MulticlassClassificationEvaluator", self.uid ) self._setDefault(metricName="f1", metricLabel=0.0, beta=1.0, eps=1e-15) - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) self.__class__._set(**kwargs) @since("1.5.0") @@ -664,7 +674,9 @@ def setParams( """ Sets params for multiclass classification evaluator. """ - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) return self.__class__._set(**kwargs) @@ -733,7 +745,9 @@ def __init__( metricName: "MultilabelClassificationEvaluatorMetricType" = "f1Measure", metricLabel: float = 0.0, ) -> None: - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(MultilabelClassificationEvaluator, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.evaluation.MultilabelClassificationEvaluator", self.uid @@ -797,7 +811,9 @@ def setParams( """ Sets params for multilabel classification evaluator. """ - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) return self.__class__._set(**kwargs) @@ -876,7 +892,9 @@ def __init__( distanceMeasure: str = "squaredEuclidean", weightCol: Optional[str] = None, ): - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(ClusteringEvaluator, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.evaluation.ClusteringEvaluator", self.uid @@ -897,7 +915,9 @@ def setParams( """ Sets params for clustering evaluator. """ - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) return self.__class__._set(**kwargs) @since("2.3.0") @@ -1010,7 +1030,9 @@ def __init__( metricName: "RankingEvaluatorMetricType" = "meanAveragePrecision", k: int = 10, ): - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(RankingEvaluator, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.evaluation.RankingEvaluator", self.uid @@ -1072,7 +1094,9 @@ def setParams( """ Sets params for ranking evaluator. """ - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) return self.__class__._set(**kwargs) diff --git a/python/pyspark/ml/feature.py b/python/pyspark/ml/feature.py index c743905e801ff..b5d2739bf2c8a 100755 --- a/python/pyspark/ml/feature.py +++ b/python/pyspark/ml/feature.py @@ -225,7 +225,9 @@ def __init__( inputCols: Optional[List[str]] = None, outputCols: Optional[List[str]] = None, ): - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(Binarizer, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.Binarizer", self.uid) self._setDefault(threshold=0.0) @@ -265,7 +267,9 @@ def setParams( """ Sets params for this Binarizer. """ - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) return self.__class__._set(**kwargs) @since("1.4.0") @@ -568,7 +572,9 @@ def __init__( numHashTables: int = 1, bucketLength: Optional[float] = None, ): - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(BucketedRandomProjectionLSH, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.feature.BucketedRandomProjectionLSH", self.uid @@ -588,7 +594,9 @@ def setParams( """ Sets params for this BucketedRandomProjectionLSH. """ - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) return self.__class__._set(**kwargs) @since("2.2.0") @@ -775,7 +783,9 @@ def __init__( inputCols: Optional[List[str]] = None, outputCols: Optional[List[str]] = None, ): - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(Bucketizer, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.Bucketizer", self.uid) self._setDefault(handleInvalid="error") @@ -818,7 +828,9 @@ def setParams( """ Sets params for this Bucketizer. """ - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) return self.__class__._set(**kwargs) @since("1.4.0") @@ -1050,7 +1062,9 @@ def __init__( inputCol: Optional[str] = None, outputCol: Optional[str] = None, ): - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(CountVectorizer, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.CountVectorizer", self.uid) self.__class__.setParams(**kwargs) @@ -1070,7 +1084,9 @@ def setParams( """ Set the params for the CountVectorizer """ - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) return self.__class__._set(**kwargs) @since("1.6.0") @@ -1258,7 +1274,9 @@ def __init__( inputCol: Optional[str] = None, outputCol: Optional[str] = None, ): - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(DCT, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.DCT", self.uid) self._setDefault(inverse=False) @@ -1275,7 +1293,9 @@ def setParams( """ Sets params for this DCT. """ - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) return self.__class__._set(**kwargs) @since("1.6.0") @@ -1358,7 +1378,9 @@ def __init__( inputCol: Optional[str] = None, outputCol: Optional[str] = None, ): - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(ElementwiseProduct, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.feature.ElementwiseProduct", self.uid @@ -1376,7 +1398,9 @@ def setParams( """ Sets params for this ElementwiseProduct. """ - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) return self.__class__._set(**kwargs) @since("2.0.0") @@ -1487,7 +1511,9 @@ def __init__( outputCol: Optional[str] = None, categoricalCols: Optional[List[str]] = None, ): - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(FeatureHasher, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.FeatureHasher", self.uid) self._setDefault(numFeatures=1 << 18) @@ -1505,7 +1531,9 @@ def setParams( """ Sets params for this FeatureHasher. """ - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) return self.__class__._set(**kwargs) @since("2.3.0") @@ -1601,7 +1629,9 @@ def __init__( inputCol: Optional[str] = None, outputCol: Optional[str] = None, ): - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(HashingTF, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.HashingTF", self.uid) self._setDefault(numFeatures=1 << 18, binary=False) @@ -1619,7 +1649,9 @@ def setParams( """ Sets params for this HashingTF. """ - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) return self.__class__._set(**kwargs) @since("2.0.0") @@ -1744,7 +1776,9 @@ def __init__( inputCol: Optional[str] = None, outputCol: Optional[str] = None, ): - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(IDF, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.IDF", self.uid) self.__class__.setParams(**kwargs) @@ -1760,7 +1794,9 @@ def setParams( """ Sets params for this IDF. """ - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) return self.__class__._set(**kwargs) @since("1.4.0") @@ -2028,7 +2064,9 @@ def __init__( outputCol: Optional[str] = None, relativeError: float = 0.001, ): - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(Imputer, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.Imputer", self.uid) self.__class__.setParams(**kwargs) @@ -2072,7 +2110,9 @@ def setParams( """ Sets params for this Imputer. """ - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) return self.__class__._set(**kwargs) @since("2.2.0") @@ -2217,7 +2257,9 @@ class Interaction( """ def __init__(self, *, inputCols: Optional[List[str]] = None, outputCol: Optional[str] = None): - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(Interaction, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.Interaction", self.uid) self._setDefault() @@ -2230,7 +2272,9 @@ def setParams( """ Sets params for this Interaction. """ - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) return self.__class__._set(**kwargs) @since("3.0.0") @@ -2307,7 +2351,9 @@ class MaxAbsScaler( """ def __init__(self, *, inputCol: Optional[str] = None, outputCol: Optional[str] = None): - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(MaxAbsScaler, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.MaxAbsScaler", self.uid) self._setDefault() @@ -2320,7 +2366,9 @@ def setParams( """ Sets params for this MaxAbsScaler. """ - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) return self.__class__._set(**kwargs) def setInputCol(self, value: str) -> "MaxAbsScaler": @@ -2452,7 +2500,9 @@ def __init__( seed: Optional[int] = None, numHashTables: int = 1, ): - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(MinHashLSH, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.MinHashLSH", self.uid) self.__class__.setParams(**kwargs) @@ -2469,7 +2519,9 @@ def setParams( """ Sets params for this MinHashLSH. """ - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) return self.__class__._set(**kwargs) def setSeed(self, value: int) -> "MinHashLSH": @@ -2609,7 +2661,9 @@ def __init__( inputCol: Optional[str] = None, outputCol: Optional[str] = None, ): - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(MinMaxScaler, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.MinMaxScaler", self.uid) self.__class__.setParams(**kwargs) @@ -2626,7 +2680,9 @@ def setParams( """ Sets params for this MinMaxScaler. """ - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) return self.__class__._set(**kwargs) @since("1.6.0") @@ -2768,7 +2824,9 @@ class NGram(JavaTransformer, HasInputCol, HasOutputCol, JavaMLReadable["NGram"], def __init__( self, *, n: int = 2, inputCol: Optional[str] = None, outputCol: Optional[str] = None ): - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(NGram, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.NGram", self.uid) self._setDefault(n=2) @@ -2781,7 +2839,9 @@ def setParams( """ Sets params for this NGram. """ - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) return self.__class__._set(**kwargs) @since("1.5.0") @@ -2855,7 +2915,9 @@ class Normalizer( def __init__( self, *, p: float = 2.0, inputCol: Optional[str] = None, outputCol: Optional[str] = None ): - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(Normalizer, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.Normalizer", self.uid) self._setDefault(p=2.0) @@ -2868,7 +2930,9 @@ def setParams( """ Sets params for this Normalizer. """ - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) return self.__class__._set(**kwargs) @since("1.4.0") @@ -3038,7 +3102,9 @@ def __init__( inputCol: Optional[str] = None, outputCol: Optional[str] = None, ): - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(OneHotEncoder, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.OneHotEncoder", self.uid) self.__class__.setParams(**kwargs) @@ -3079,7 +3145,9 @@ def setParams( """ Sets params for this OneHotEncoder. """ - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) return self.__class__._set(**kwargs) @since("2.3.0") @@ -3238,7 +3306,9 @@ class PolynomialExpansion( def __init__( self, *, degree: int = 2, inputCol: Optional[str] = None, outputCol: Optional[str] = None ): - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(PolynomialExpansion, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.feature.PolynomialExpansion", self.uid @@ -3253,7 +3323,9 @@ def setParams( """ Sets params for this PolynomialExpansion. """ - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) return self.__class__._set(**kwargs) @since("1.4.0") @@ -3457,7 +3529,9 @@ def __init__( inputCols: Optional[List[str]] = None, outputCols: Optional[List[str]] = None, ): - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(QuantileDiscretizer, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.feature.QuantileDiscretizer", self.uid @@ -3505,7 +3579,9 @@ def setParams( """ Set the params for the QuantileDiscretizer """ - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) return self.__class__._set(**kwargs) @since("2.0.0") @@ -3730,7 +3806,9 @@ def __init__( outputCol: Optional[str] = None, relativeError: float = 0.001, ): - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(RobustScaler, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.RobustScaler", self.uid) self.__class__.setParams(**kwargs) @@ -3750,7 +3828,9 @@ def setParams( """ Sets params for this RobustScaler. """ - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) return self.__class__._set(**kwargs) @since("3.0.0") @@ -3932,7 +4012,9 @@ def __init__( outputCol: Optional[str] = None, toLowercase: bool = True, ): - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(RegexTokenizer, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.RegexTokenizer", self.uid) self._setDefault(minTokenLength=1, gaps=True, pattern="\\s+", toLowercase=True) @@ -3952,7 +4034,9 @@ def setParams( """ Sets params for this RegexTokenizer. """ - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) return self.__class__._set(**kwargs) @since("1.4.0") @@ -4054,7 +4138,9 @@ class SQLTransformer(JavaTransformer, JavaMLReadable["SQLTransformer"], JavaMLWr ) def __init__(self, *, statement: Optional[str] = None): - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(SQLTransformer, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.SQLTransformer", self.uid) self.__class__.setParams(**kwargs) @@ -4064,7 +4150,9 @@ def setParams(self, *, statement: Optional[str] = None) -> "SQLTransformer": """ Sets params for this SQLTransformer. """ - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) return self.__class__._set(**kwargs) @since("1.6.0") @@ -4181,7 +4269,9 @@ def __init__( inputCol: Optional[str] = None, outputCol: Optional[str] = None, ): - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(StandardScaler, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.StandardScaler", self.uid) self.__class__.setParams(**kwargs) @@ -4198,7 +4288,9 @@ def setParams( """ Sets params for this StandardScaler. """ - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) return self.__class__._set(**kwargs) @since("1.4.0") @@ -4435,7 +4527,9 @@ def __init__( handleInvalid: str = "error", stringOrderType: str = "frequencyDesc", ): - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(StringIndexer, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.StringIndexer", self.uid) self.__class__.setParams(**kwargs) @@ -4476,7 +4570,9 @@ def setParams( """ Sets params for this StringIndexer. """ - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) return self.__class__._set(**kwargs) def _create_model(self, java_model: "JavaObject") -> "StringIndexerModel": @@ -4675,7 +4771,9 @@ def __init__( outputCol: Optional[str] = None, labels: Optional[List[str]] = None, ): - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(IndexToString, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.IndexToString", self.uid) self.__class__.setParams(**kwargs) @@ -4691,7 +4789,9 @@ def setParams( """ Sets params for this IndexToString. """ - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) return self.__class__._set(**kwargs) @since("1.6.0") @@ -4828,7 +4928,9 @@ def __init__( inputCols: Optional[List[str]] = None, outputCols: Optional[List[str]] = None, ): - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(StopWordsRemover, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.feature.StopWordsRemover", self.uid @@ -4879,7 +4981,9 @@ def setParams( """ Sets params for this StopWordRemover. """ - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) return self.__class__._set(**kwargs) @since("1.6.0") @@ -5005,7 +5109,9 @@ class Tokenizer( """ def __init__(self, *, inputCol: Optional[str] = None, outputCol: Optional[str] = None): - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(Tokenizer, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.Tokenizer", self.uid) self.__class__.setParams(**kwargs) @@ -5017,7 +5123,9 @@ def setParams( """ Sets params for this Tokenizer. """ - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) return self.__class__._set(**kwargs) def setInputCol(self, value: str) -> "Tokenizer": @@ -5108,7 +5216,9 @@ def __init__( outputCol: Optional[str] = None, handleInvalid: str = "error", ): - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(VectorAssembler, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.VectorAssembler", self.uid) self._setDefault(handleInvalid="error") @@ -5125,7 +5235,9 @@ def setParams( """ Sets params for this VectorAssembler. """ - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) return self.__class__._set(**kwargs) def setInputCols(self, value: List[str]) -> "VectorAssembler": @@ -5289,7 +5401,9 @@ def __init__( outputCol: Optional[str] = None, handleInvalid: str = "error", ): - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(VectorIndexer, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.VectorIndexer", self.uid) self.__class__.setParams(**kwargs) @@ -5306,7 +5420,9 @@ def setParams( """ Sets params for this VectorIndexer. """ - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) return self.__class__._set(**kwargs) @since("1.4.0") @@ -5459,7 +5575,9 @@ def __init__( indices: Optional[List[int]] = None, names: Optional[List[str]] = None, ): - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(VectorSlicer, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.VectorSlicer", self.uid) self._setDefault(indices=[], names=[]) @@ -5477,7 +5595,9 @@ def setParams( """ Sets params for this VectorSlicer. """ - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) return self.__class__._set(**kwargs) @since("1.6.0") @@ -5694,7 +5814,9 @@ def __init__( windowSize: int = 5, maxSentenceLength: int = 1000, ): - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(Word2Vec, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.Word2Vec", self.uid) self.__class__.setParams(**kwargs) @@ -5717,7 +5839,9 @@ def setParams( """ Sets params for this Word2Vec. """ - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) return self.__class__._set(**kwargs) @since("1.4.0") @@ -5916,7 +6040,9 @@ def __init__( inputCol: Optional[str] = None, outputCol: Optional[str] = None, ): - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(PCA, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.PCA", self.uid) self.__class__.setParams(**kwargs) @@ -5932,7 +6058,9 @@ def setParams( """ Set params for this PCA. """ - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) return self.__class__._set(**kwargs) @since("1.5.0") @@ -6155,7 +6283,9 @@ def __init__( stringIndexerOrderType: str = "frequencyDesc", handleInvalid: str = "error", ): - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(RFormula, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.RFormula", self.uid) self.__class__.setParams(**kwargs) @@ -6174,7 +6304,9 @@ def setParams( """ Sets params for RFormula. """ - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) return self.__class__._set(**kwargs) @since("1.5.0") @@ -6522,7 +6654,9 @@ def __init__( fdr: float = 0.05, fwe: float = 0.05, ): - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(ChiSqSelector, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.ChiSqSelector", self.uid) self.__class__.setParams(**kwargs) @@ -6544,7 +6678,9 @@ def setParams( """ Sets params for this ChiSqSelector. """ - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) return self.__class__._set(**kwargs) def _create_model(self, java_model: "JavaObject") -> "ChiSqSelectorModel": @@ -6623,7 +6759,9 @@ def __init__( size: Optional[int] = None, handleInvalid: str = "error", ): - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(VectorSizeHint, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.VectorSizeHint", self.uid) self._setDefault(handleInvalid="error") @@ -6640,7 +6778,9 @@ def setParams( """ Sets params for this VectorSizeHint. """ - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) return self.__class__._set(**kwargs) @since("2.3.0") @@ -6748,7 +6888,9 @@ def __init__( outputCol: Optional[str] = None, varianceThreshold: float = 0.0, ): - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(VarianceThresholdSelector, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.feature.VarianceThresholdSelector", self.uid @@ -6767,7 +6909,9 @@ def setParams( """ Sets params for this VarianceThresholdSelector. """ - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) return self.__class__._set(**kwargs) @since("3.1.0") @@ -6988,7 +7132,9 @@ def __init__( labelCol: str = "label", selectionMode: str = "numTopFeatures", ): - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(UnivariateFeatureSelector, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.feature.UnivariateFeatureSelector", self.uid @@ -7007,7 +7153,9 @@ def setParams( """ Sets params for this UnivariateFeatureSelector. """ - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) return self.__class__._set(**kwargs) @since("3.1.1") diff --git a/python/pyspark/ml/fpm.py b/python/pyspark/ml/fpm.py index 71a45bc55a6db..1e9e9c7b4c7fc 100644 --- a/python/pyspark/ml/fpm.py +++ b/python/pyspark/ml/fpm.py @@ -245,7 +245,9 @@ def __init__( predictionCol: str = "prediction", numPartitions: Optional[int] = None, ): - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(FPGrowth, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.fpm.FPGrowth", self.uid) self.__class__.setParams(**kwargs) @@ -260,7 +262,9 @@ def setParams( predictionCol: str = "prediction", numPartitions: Optional[int] = None, ) -> "FPGrowth": - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) return self.__class__._set(**kwargs) def setItemsCol(self, value: str) -> "FPGrowth": @@ -386,7 +390,9 @@ def __init__( maxLocalProjDBSize: int = 32000000, sequenceCol: str = "sequence", ): - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(PrefixSpan, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.fpm.PrefixSpan", self.uid) self._setDefault( @@ -403,7 +409,9 @@ def setParams( maxLocalProjDBSize: int = 32000000, sequenceCol: str = "sequence", ) -> "PrefixSpan": - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) return self.__class__._set(**kwargs) @since("3.0.0") diff --git a/python/pyspark/ml/pipeline.py b/python/pyspark/ml/pipeline.py index afd0ef6a82ff5..adb0471ab8e2b 100644 --- a/python/pyspark/ml/pipeline.py +++ b/python/pyspark/ml/pipeline.py @@ -69,7 +69,9 @@ class Pipeline(Estimator["PipelineModel"], MLReadable["Pipeline"], MLWritable): ) def __init__(self, *, stages: Optional[List["PipelineStage"]] = None): - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(Pipeline, self).__init__() self.__class__.setParams(**kwargs) @@ -104,7 +106,9 @@ def setParams(self, *, stages: Optional[List["PipelineStage"]] = None) -> "Pipel """ Sets params for Pipeline. """ - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) return self.__class__._set(**kwargs) def _fit(self, dataset: DataFrame) -> "PipelineModel": diff --git a/python/pyspark/ml/recommendation.py b/python/pyspark/ml/recommendation.py index 8b96850d1fd89..9876f407c024a 100644 --- a/python/pyspark/ml/recommendation.py +++ b/python/pyspark/ml/recommendation.py @@ -385,7 +385,9 @@ def __init__( coldStartStrategy: str = "nan", blockSize: int = 4096, ): - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(ALS, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.recommendation.ALS", self.uid) self.__class__.setParams(**kwargs) @@ -415,7 +417,9 @@ def setParams( """ Sets params for ALS. """ - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) return self.__class__._set(**kwargs) def _create_model(self, java_model: "JavaObject") -> "ALSModel": diff --git a/python/pyspark/ml/regression.py b/python/pyspark/ml/regression.py index 21860cd7aa0be..f4d91053cd2c2 100644 --- a/python/pyspark/ml/regression.py +++ b/python/pyspark/ml/regression.py @@ -313,7 +313,9 @@ def __init__( epsilon: float = 1.35, maxBlockSizeInMB: float = 0.0, ): - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(LinearRegression, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.regression.LinearRegression", self.uid @@ -343,7 +345,9 @@ def setParams( """ Sets params for linear regression. """ - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) return self.__class__._set(**kwargs) def _create_model(self, java_model: "JavaObject") -> "LinearRegressionModel": @@ -852,7 +856,9 @@ def __init__( isotonic: bool = True, featureIndex: int = 0, ): - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(IsotonicRegression, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.regression.IsotonicRegression", self.uid @@ -872,7 +878,9 @@ def setParams( """ Set the params for IsotonicRegression. """ - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) return self.__class__._set(**kwargs) def _create_model(self, java_model: "JavaObject") -> "IsotonicRegressionModel": @@ -1102,7 +1110,9 @@ def __init__( leafCol: str = "", minWeightFractionPerNode: float = 0.0, ): - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(DecisionTreeRegressor, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.regression.DecisionTreeRegressor", self.uid @@ -1133,7 +1143,9 @@ def setParams( """ Sets params for the DecisionTreeRegressor. """ - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) return self.__class__._set(**kwargs) def _create_model(self, java_model: "JavaObject") -> "DecisionTreeRegressionModel": @@ -1389,7 +1401,9 @@ def __init__( weightCol: Optional[str] = None, bootstrap: Optional[bool] = True, ): - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(RandomForestRegressor, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.regression.RandomForestRegressor", self.uid @@ -1423,7 +1437,9 @@ def setParams( """ Sets params for linear regression. """ - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) return self.__class__._set(**kwargs) def _create_model(self, java_model: "JavaObject") -> "RandomForestRegressionModel": @@ -1717,7 +1733,9 @@ def __init__( minWeightFractionPerNode: float = 0.0, weightCol: Optional[str] = None, ): - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(GBTRegressor, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.regression.GBTRegressor", self.uid) self.__class__.setParams(**kwargs) @@ -1752,7 +1770,9 @@ def setParams( """ Sets params for Gradient Boosted Tree Regression. """ - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) return self.__class__._set(**kwargs) def _create_model(self, java_model: "JavaObject") -> "GBTRegressionModel": @@ -2090,7 +2110,9 @@ def __init__( aggregationDepth: int = 2, maxBlockSizeInMB: float = 0.0, ): - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(AFTSurvivalRegression, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.regression.AFTSurvivalRegression", self.uid @@ -2123,7 +2145,9 @@ def setParams( aggregationDepth: int = 2, maxBlockSizeInMB: float = 0.0, ) -> "AFTSurvivalRegression": - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) return self.__class__._set(**kwargs) def _create_model(self, java_model: "JavaObject") -> "AFTSurvivalRegressionModel": @@ -2474,7 +2498,9 @@ def __init__( offsetCol: Optional[str] = None, aggregationDepth: int = 2, ): - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(GeneralizedLinearRegression, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.regression.GeneralizedLinearRegression", self.uid @@ -2506,7 +2532,9 @@ def setParams( """ Sets params for generalized linear regression. """ - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) return self.__class__._set(**kwargs) def _create_model(self, java_model: "JavaObject") -> "GeneralizedLinearRegressionModel": @@ -3033,7 +3061,9 @@ def __init__( solver: str = "adamW", seed: Optional[int] = None, ): - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(FMRegressor, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.regression.FMRegressor", self.uid) self.__class__.setParams(**kwargs) @@ -3060,7 +3090,9 @@ def setParams( """ Sets Params for FMRegressor. """ - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) return self.__class__._set(**kwargs) def _create_model(self, java_model: "JavaObject") -> "FMRegressionModel": diff --git a/python/pyspark/ml/tuning.py b/python/pyspark/ml/tuning.py index 112b4172b9a0f..9d143949fb6b2 100644 --- a/python/pyspark/ml/tuning.py +++ b/python/pyspark/ml/tuning.py @@ -726,7 +726,9 @@ def __init__( collectSubModels: bool = False, foldCol: str = "", ) -> None: - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(CrossValidator, self).__init__() self._setDefault(parallelism=1) self.__class__._set(**kwargs) @@ -747,7 +749,9 @@ def setParams( """ Sets params for cross validator. """ - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) return self.__class__._set(**kwargs) @since("2.0.0") @@ -1346,7 +1350,9 @@ def __init__( collectSubModels: bool = False, seed: Optional[int] = None, ) -> None: - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(TrainValidationSplit, self).__init__() self._setDefault(parallelism=1) self.__class__._set(**kwargs) @@ -1366,7 +1372,9 @@ def setParams( """ Sets params for the train validation split. """ - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) return self.__class__._set(**kwargs) @since("2.0.0") diff --git a/python/pyspark/testing/mlutils.py b/python/pyspark/testing/mlutils.py index 2d807e6b7d172..51e7a7d024aa7 100644 --- a/python/pyspark/testing/mlutils.py +++ b/python/pyspark/testing/mlutils.py @@ -191,7 +191,9 @@ def __init__( regParam=0.0, rawPredictionCol="rawPrediction", ): - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(DummyLogisticRegression, self).__init__() self.__class__.setParams(**kwargs) @@ -205,7 +207,9 @@ def setParams( regParam=0.0, rawPredictionCol="rawPrediction", ): - kwargs = locals() + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) self.__class__._set(**kwargs) return self