diff --git a/python/docs/source/conf.py b/python/docs/source/conf.py index 81083c007b346..9d072fa1a1455 100644 --- a/python/docs/source/conf.py +++ b/python/docs/source/conf.py @@ -176,7 +176,6 @@ # -- Options for autodoc -------------------------------------------------- # Look at the first line of the docstring for function and method signatures. -autodoc_docstring_signature = True autosummary_generate = True # -- Options for HTML output ---------------------------------------------- diff --git a/python/pyspark/__init__.py b/python/pyspark/__init__.py index a9a2a31702562..31ab2e4ce1b1a 100644 --- a/python/pyspark/__init__.py +++ b/python/pyspark/__init__.py @@ -45,7 +45,7 @@ - :class:`InheritableThread`: A inheritable thread to use in Spark when the pinned thread mode is on. """ - +import warnings from functools import wraps import types from typing import cast, Any, Callable, Optional, TypeVar, Union @@ -76,7 +76,9 @@ def since(version: Union[str, float]) -> Callable[[_F], _F]: indent_p = re.compile(r"\n( +)") def deco(f: _F) -> _F: - assert f.__doc__ is not None + if f.__doc__ is None: + # Should still at least add a versionadded. + f.__doc__ = "" indents = indent_p.findall(f.__doc__) indent = " " * (min(len(m) for m in indents) if indents else 0) @@ -121,10 +123,19 @@ def keyword_only(func: _F) -> _F: A decorator that forces keyword arguments in the wrapped method and saves actual input keyword arguments in `_input_kwargs`. + .. deprecated:: 4.0.0 + `keyword_only` method is deprecated. Use the standard + keyword-only syntax in Python instead. + Notes ----- Should only be used to wrap a method where first arg is `self` """ + warnings.warn( + "keyword_only method is deprecated. Use the standard keyword-only" + "syntax in Python instead.", + FutureWarning, + ) @wraps(func) def wrapper(self: Any, *args: Any, **kwargs: Any) -> Any: diff --git a/python/pyspark/ml/classification.py b/python/pyspark/ml/classification.py index 263a108a216dd..e02532b4330f6 100644 --- a/python/pyspark/ml/classification.py +++ b/python/pyspark/ml/classification.py @@ -37,7 +37,7 @@ TYPE_CHECKING, ) -from pyspark import keyword_only, since, SparkContext, inheritable_thread_target +from pyspark import since, SparkContext, inheritable_thread_target from pyspark.ml import Estimator, Predictor, PredictionModel, Model from pyspark.ml.param.shared import ( HasRawPredictionCol, @@ -706,9 +706,6 @@ class LinearSVC( True """ - _input_kwargs: Dict[str, Any] - - @keyword_only def __init__( self, *, @@ -726,20 +723,15 @@ def __init__( aggregationDepth: int = 2, maxBlockSizeInMB: float = 0.0, ): - """ - __init__(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \ - maxIter=100, regParam=0.0, tol=1e-6, rawPredictionCol="rawPrediction", \ - fitIntercept=True, standardization=True, threshold=0.0, weightCol=None, \ - aggregationDepth=2, maxBlockSizeInMB=0.0): - """ + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(LinearSVC, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.classification.LinearSVC", self.uid ) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) - @keyword_only @since("2.2.0") def setParams( self, @@ -759,14 +751,12 @@ def setParams( maxBlockSizeInMB: float = 0.0, ) -> "LinearSVC": """ - setParams(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \ - maxIter=100, regParam=0.0, tol=1e-6, rawPredictionCol="rawPrediction", \ - fitIntercept=True, standardization=True, threshold=0.0, weightCol=None, \ - aggregationDepth=2, maxBlockSizeInMB=0.0): Sets params for Linear SVM Classifier. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) + return self.__class__._set(**kwargs) def _create_model(self, java_model: "JavaObject") -> "LinearSVCModel": return LinearSVCModel(java_model) @@ -1219,8 +1209,6 @@ class LogisticRegression( True """ - _input_kwargs: Dict[str, Any] - @overload def __init__( self, @@ -1275,7 +1263,6 @@ def __init__( ): ... - @keyword_only def __init__( self, *, @@ -1302,22 +1289,16 @@ def __init__( maxBlockSizeInMB: float = 0.0, ): """ - __init__(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \ - maxIter=100, regParam=0.0, elasticNetParam=0.0, tol=1e-6, fitIntercept=True, \ - threshold=0.5, thresholds=None, probabilityCol="probability", \ - rawPredictionCol="rawPrediction", standardization=True, weightCol=None, \ - aggregationDepth=2, family="auto", \ - lowerBoundsOnCoefficients=None, upperBoundsOnCoefficients=None, \ - lowerBoundsOnIntercepts=None, upperBoundsOnIntercepts=None, \ - maxBlockSizeInMB=0.0): If the threshold and thresholds Params are both set, they must be equivalent. """ + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(LogisticRegression, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.classification.LogisticRegression", self.uid ) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) self._checkThresholdConsistency() @overload @@ -1374,7 +1355,6 @@ def setParams( ) -> "LogisticRegression": ... - @keyword_only @since("1.3.0") def setParams( self, @@ -1402,19 +1382,13 @@ def setParams( maxBlockSizeInMB: float = 0.0, ) -> "LogisticRegression": """ - setParams(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \ - maxIter=100, regParam=0.0, elasticNetParam=0.0, tol=1e-6, fitIntercept=True, \ - threshold=0.5, thresholds=None, probabilityCol="probability", \ - rawPredictionCol="rawPrediction", standardization=True, weightCol=None, \ - aggregationDepth=2, family="auto", \ - lowerBoundsOnCoefficients=None, upperBoundsOnCoefficients=None, \ - lowerBoundsOnIntercepts=None, upperBoundsOnIntercepts=None, \ - maxBlockSizeInMB=0.0): Sets params for logistic regression. If the threshold and thresholds Params are both set, they must be equivalent. """ - kwargs = self._input_kwargs - self._set(**kwargs) + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) + self.__class__._set(**kwargs) self._checkThresholdConsistency() return self @@ -1770,9 +1744,6 @@ class DecisionTreeClassifier( DecisionTreeClassificationModel...depth=1, numNodes=3... """ - _input_kwargs: Dict[str, Any] - - @keyword_only def __init__( self, *, @@ -1794,21 +1765,15 @@ def __init__( leafCol: str = "", minWeightFractionPerNode: float = 0.0, ): - """ - __init__(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \ - probabilityCol="probability", rawPredictionCol="rawPrediction", \ - maxDepth=5, maxBins=32, minInstancesPerNode=1, minInfoGain=0.0, \ - maxMemoryInMB=256, cacheNodeIds=False, checkpointInterval=10, impurity="gini", \ - seed=None, weightCol=None, leafCol="", minWeightFractionPerNode=0.0) - """ + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(DecisionTreeClassifier, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.classification.DecisionTreeClassifier", self.uid ) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) - @keyword_only @since("1.4.0") def setParams( self, @@ -1832,15 +1797,12 @@ def setParams( minWeightFractionPerNode: float = 0.0, ) -> "DecisionTreeClassifier": """ - setParams(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \ - probabilityCol="probability", rawPredictionCol="rawPrediction", \ - maxDepth=5, maxBins=32, minInstancesPerNode=1, minInfoGain=0.0, \ - maxMemoryInMB=256, cacheNodeIds=False, checkpointInterval=10, impurity="gini", \ - seed=None, weightCol=None, leafCol="", minWeightFractionPerNode=0.0) Sets params for the DecisionTreeClassifier. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) + return self.__class__.setParams(**kwargs) def _create_model(self, java_model: "JavaObject") -> "DecisionTreeClassificationModel": return DecisionTreeClassificationModel(java_model) @@ -2062,9 +2024,6 @@ class RandomForestClassifier( True """ - _input_kwargs: Dict[str, Any] - - @keyword_only def __init__( self, *, @@ -2090,22 +2049,15 @@ def __init__( weightCol: Optional[str] = None, bootstrap: Optional[bool] = True, ): - """ - __init__(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \ - probabilityCol="probability", rawPredictionCol="rawPrediction", \ - maxDepth=5, maxBins=32, minInstancesPerNode=1, minInfoGain=0.0, \ - maxMemoryInMB=256, cacheNodeIds=False, checkpointInterval=10, impurity="gini", \ - numTrees=20, featureSubsetStrategy="auto", seed=None, subsamplingRate=1.0, \ - leafCol="", minWeightFractionPerNode=0.0, weightCol=None, bootstrap=True) - """ + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(RandomForestClassifier, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.classification.RandomForestClassifier", self.uid ) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) - @keyword_only @since("1.4.0") def setParams( self, @@ -2133,16 +2085,12 @@ def setParams( bootstrap: Optional[bool] = True, ) -> "RandomForestClassifier": """ - setParams(self, featuresCol="features", labelCol="label", predictionCol="prediction", \ - probabilityCol="probability", rawPredictionCol="rawPrediction", \ - maxDepth=5, maxBins=32, minInstancesPerNode=1, minInfoGain=0.0, \ - maxMemoryInMB=256, cacheNodeIds=False, checkpointInterval=10, seed=None, \ - impurity="gini", numTrees=20, featureSubsetStrategy="auto", subsamplingRate=1.0, \ - leafCol="", minWeightFractionPerNode=0.0, weightCol=None, bootstrap=True) Sets params for linear classification. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) + return self.__class__._set(**kwargs) def _create_model(self, java_model: "JavaObject") -> "RandomForestClassificationModel": return RandomForestClassificationModel(java_model) @@ -2531,9 +2479,6 @@ class GBTClassifier( 0.01 """ - _input_kwargs: Dict[str, Any] - - @keyword_only def __init__( self, *, @@ -2560,23 +2505,15 @@ def __init__( minWeightFractionPerNode: float = 0.0, weightCol: Optional[str] = None, ): - """ - __init__(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \ - maxDepth=5, maxBins=32, minInstancesPerNode=1, minInfoGain=0.0, \ - maxMemoryInMB=256, cacheNodeIds=False, checkpointInterval=10, \ - lossType="logistic", maxIter=20, stepSize=0.1, seed=None, subsamplingRate=1.0, \ - impurity="variance", featureSubsetStrategy="all", validationTol=0.01, \ - validationIndicatorCol=None, leafCol="", minWeightFractionPerNode=0.0, \ - weightCol=None) - """ + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(GBTClassifier, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.classification.GBTClassifier", self.uid ) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) - @keyword_only @since("1.4.0") def setParams( self, @@ -2605,17 +2542,12 @@ def setParams( weightCol: Optional[str] = None, ) -> "GBTClassifier": """ - setParams(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \ - maxDepth=5, maxBins=32, minInstancesPerNode=1, minInfoGain=0.0, \ - maxMemoryInMB=256, cacheNodeIds=False, checkpointInterval=10, \ - lossType="logistic", maxIter=20, stepSize=0.1, seed=None, subsamplingRate=1.0, \ - impurity="variance", featureSubsetStrategy="all", validationTol=0.01, \ - validationIndicatorCol=None, leafCol="", minWeightFractionPerNode=0.0, \ - weightCol=None) Sets params for Gradient Boosted Tree Classification. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) + return self.__class__._set(**kwargs) def _create_model(self, java_model: "JavaObject") -> "GBTClassificationModel": return GBTClassificationModel(java_model) @@ -2928,9 +2860,6 @@ class NaiveBayes( DenseMatrix(0, 0, [...], ...) """ - _input_kwargs: Dict[str, Any] - - @keyword_only def __init__( self, *, @@ -2944,19 +2873,15 @@ def __init__( thresholds: Optional[List[float]] = None, weightCol: Optional[str] = None, ): - """ - __init__(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \ - probabilityCol="probability", rawPredictionCol="rawPrediction", smoothing=1.0, \ - modelType="multinomial", thresholds=None, weightCol=None) - """ + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(NaiveBayes, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.classification.NaiveBayes", self.uid ) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) - @keyword_only @since("1.5.0") def setParams( self, @@ -2972,13 +2897,12 @@ def setParams( weightCol: Optional[str] = None, ) -> "NaiveBayes": """ - setParams(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \ - probabilityCol="probability", rawPredictionCol="rawPrediction", smoothing=1.0, \ - modelType="multinomial", thresholds=None, weightCol=None) Sets params for Naive Bayes. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) + return self.__class__._set(**kwargs) def _create_model(self, java_model: "JavaObject") -> "NaiveBayesModel": return NaiveBayesModel(java_model) @@ -3178,9 +3102,6 @@ class MultilayerPerceptronClassifier( True """ - _input_kwargs: Dict[str, Any] - - @keyword_only def __init__( self, *, @@ -3198,20 +3119,15 @@ def __init__( probabilityCol: str = "probability", rawPredictionCol: str = "rawPrediction", ): - """ - __init__(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \ - maxIter=100, tol=1e-6, seed=None, layers=None, blockSize=128, stepSize=0.03, \ - solver="l-bfgs", initialWeights=None, probabilityCol="probability", \ - rawPredictionCol="rawPrediction") - """ + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(MultilayerPerceptronClassifier, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.classification.MultilayerPerceptronClassifier", self.uid ) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) - @keyword_only @since("1.6.0") def setParams( self, @@ -3231,14 +3147,12 @@ def setParams( rawPredictionCol: str = "rawPrediction", ) -> "MultilayerPerceptronClassifier": """ - setParams(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \ - maxIter=100, tol=1e-6, seed=None, layers=None, blockSize=128, stepSize=0.03, \ - solver="l-bfgs", initialWeights=None, probabilityCol="probability", \ - rawPredictionCol="rawPrediction"): Sets params for MultilayerPerceptronClassifier. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) + return self.__class__._set(**kwargs) def _create_model(self, java_model: "JavaObject") -> "MultilayerPerceptronClassificationModel": return MultilayerPerceptronClassificationModel(java_model) @@ -3446,9 +3360,6 @@ class OneVsRest( ['features', 'rawPrediction', 'newPrediction'] """ - _input_kwargs: Dict[str, Any] - - @keyword_only def __init__( self, *, @@ -3460,16 +3371,13 @@ def __init__( weightCol: Optional[str] = None, parallelism: int = 1, ): - """ - __init__(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \ - rawPredictionCol="rawPrediction", classifier=None, weightCol=None, parallelism=1): - """ + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(OneVsRest, self).__init__() self._setDefault(parallelism=1) - kwargs = self._input_kwargs - self._set(**kwargs) + self.__class__._set(**kwargs) - @keyword_only @since("2.0.0") def setParams( self, @@ -3483,12 +3391,12 @@ def setParams( parallelism: int = 1, ) -> "OneVsRest": """ - setParams(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \ - rawPredictionCol="rawPrediction", classifier=None, weightCol=None, parallelism=1): Sets params for OneVsRest. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) + return self.__class__._set(**kwargs) @since("2.0.0") def setClassifier(self, value: Classifier[CM]) -> "OneVsRest": @@ -4059,9 +3967,6 @@ class FMClassifier( True """ - _input_kwargs: Dict[str, Any] - - @keyword_only def __init__( self, *, @@ -4083,21 +3988,15 @@ def __init__( thresholds: Optional[List[float]] = None, seed: Optional[int] = None, ): - """ - __init__(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \ - probabilityCol="probability", rawPredictionCol="rawPrediction", \ - factorSize=8, fitIntercept=True, fitLinear=True, regParam=0.0, \ - miniBatchFraction=1.0, initStd=0.01, maxIter=100, stepSize=1.0, \ - tol=1e-6, solver="adamW", thresholds=None, seed=None) - """ + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(FMClassifier, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.classification.FMClassifier", self.uid ) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) - @keyword_only @since("3.0.0") def setParams( self, @@ -4121,15 +4020,12 @@ def setParams( seed: Optional[int] = None, ) -> "FMClassifier": """ - setParams(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \ - probabilityCol="probability", rawPredictionCol="rawPrediction", \ - factorSize=8, fitIntercept=True, fitLinear=True, regParam=0.0, \ - miniBatchFraction=1.0, initStd=0.01, maxIter=100, stepSize=1.0, \ - tol=1e-6, solver="adamW", thresholds=None, seed=None) Sets Params for FMClassifier. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) + return self.__class__._set(**kwargs) def _create_model(self, java_model: "JavaObject") -> "FMClassificationModel": return FMClassificationModel(java_model) diff --git a/python/pyspark/ml/clustering.py b/python/pyspark/ml/clustering.py index 7f9e87e612432..4865418b71b62 100644 --- a/python/pyspark/ml/clustering.py +++ b/python/pyspark/ml/clustering.py @@ -17,11 +17,11 @@ import sys import warnings -from typing import Any, Dict, List, Optional, TYPE_CHECKING +from typing import Any, List, Optional, TYPE_CHECKING import numpy as np -from pyspark import since, keyword_only +from pyspark import since from pyspark.ml.param.shared import ( HasMaxIter, HasFeaturesCol, @@ -401,9 +401,6 @@ class GaussianMixture( GaussianMixture... """ - _input_kwargs: Dict[str, Any] - - @keyword_only def __init__( self, *, @@ -417,22 +414,18 @@ def __init__( aggregationDepth: int = 2, weightCol: Optional[str] = None, ): - """ - __init__(self, \\*, featuresCol="features", predictionCol="prediction", k=2, \ - probabilityCol="probability", tol=0.01, maxIter=100, seed=None, \ - aggregationDepth=2, weightCol=None) - """ + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(GaussianMixture, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.clustering.GaussianMixture", self.uid ) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) def _create_model(self, java_model: "JavaObject") -> "GaussianMixtureModel": return GaussianMixtureModel(java_model) - @keyword_only @since("2.0.0") def setParams( self, @@ -448,14 +441,12 @@ def setParams( weightCol: Optional[str] = None, ) -> "GaussianMixture": """ - setParams(self, \\*, featuresCol="features", predictionCol="prediction", k=2, \ - probabilityCol="probability", tol=0.01, maxIter=100, seed=None, \ - aggregationDepth=2, weightCol=None) - Sets params for GaussianMixture. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) + return self.__class__._set(**kwargs) @since("2.0.0") def setK(self, value: int) -> "GaussianMixture": @@ -775,9 +766,6 @@ class KMeans(JavaEstimator[KMeansModel], _KMeansParams, JavaMLWritable, JavaMLRe True """ - _input_kwargs: Dict[str, Any] - - @keyword_only def __init__( self, *, @@ -794,21 +782,16 @@ def __init__( solver: str = "auto", maxBlockSizeInMB: float = 0.0, ): - """ - __init__(self, \\*, featuresCol="features", predictionCol="prediction", k=2, \ - initMode="k-means||", initSteps=2, tol=1e-4, maxIter=20, seed=None, \ - distanceMeasure="euclidean", weightCol=None, solver="auto", \ - maxBlockSizeInMB=0.0) - """ + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(KMeans, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.clustering.KMeans", self.uid) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) def _create_model(self, java_model: "JavaObject") -> KMeansModel: return KMeansModel(java_model) - @keyword_only @since("1.5.0") def setParams( self, @@ -827,15 +810,12 @@ def setParams( maxBlockSizeInMB: float = 0.0, ) -> "KMeans": """ - setParams(self, \\*, featuresCol="features", predictionCol="prediction", k=2, \ - initMode="k-means||", initSteps=2, tol=1e-4, maxIter=20, seed=None, \ - distanceMeasure="euclidean", weightCol=None, solver="auto", \ - maxBlockSizeInMB=0.0) - Sets params for KMeans. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) + return self.__class__._set(**kwargs) @since("1.5.0") def setK(self, value: int) -> "KMeans": @@ -1127,9 +1107,6 @@ class BisectingKMeans( True """ - _input_kwargs: Dict[str, Any] - - @keyword_only def __init__( self, *, @@ -1142,19 +1119,15 @@ def __init__( distanceMeasure: str = "euclidean", weightCol: Optional[str] = None, ): - """ - __init__(self, \\*, featuresCol="features", predictionCol="prediction", maxIter=20, \ - seed=None, k=4, minDivisibleClusterSize=1.0, distanceMeasure="euclidean", \ - weightCol=None) - """ + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(BisectingKMeans, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.clustering.BisectingKMeans", self.uid ) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) - @keyword_only @since("2.0.0") def setParams( self, @@ -1169,13 +1142,12 @@ def setParams( weightCol: Optional[str] = None, ) -> "BisectingKMeans": """ - setParams(self, \\*, featuresCol="features", predictionCol="prediction", maxIter=20, \ - seed=None, k=4, minDivisibleClusterSize=1.0, distanceMeasure="euclidean", \ - weightCol=None) Sets params for BisectingKMeans. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) + return self.__class__._set(**kwargs) @since("2.0.0") def setK(self, value: int) -> "BisectingKMeans": @@ -1679,9 +1651,6 @@ class LDA(JavaEstimator[LDAModel], _LDAParams, JavaMLReadable["LDA"], JavaMLWrit True """ - _input_kwargs: Dict[str, Any] - - @keyword_only def __init__( self, *, @@ -1700,17 +1669,12 @@ def __init__( topicDistributionCol: str = "topicDistribution", keepLastCheckpoint: bool = True, ): - """ - __init__(self, \\*, featuresCol="features", maxIter=20, seed=None, checkpointInterval=10,\ - k=10, optimizer="online", learningOffset=1024.0, learningDecay=0.51,\ - subsamplingRate=0.05, optimizeDocConcentration=True,\ - docConcentration=None, topicConcentration=None,\ - topicDistributionCol="topicDistribution", keepLastCheckpoint=True) - """ + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(LDA, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.clustering.LDA", self.uid) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) def _create_model(self, java_model: "JavaObject") -> LDAModel: if self.getOptimizer() == "em": @@ -1718,7 +1682,6 @@ def _create_model(self, java_model: "JavaObject") -> LDAModel: else: return LocalLDAModel(java_model) - @keyword_only @since("2.0.0") def setParams( self, @@ -1739,16 +1702,12 @@ def setParams( keepLastCheckpoint: bool = True, ) -> "LDA": """ - setParams(self, \\*, featuresCol="features", maxIter=20, seed=None, checkpointInterval=10,\ - k=10, optimizer="online", learningOffset=1024.0, learningDecay=0.51,\ - subsamplingRate=0.05, optimizeDocConcentration=True,\ - docConcentration=None, topicConcentration=None,\ - topicDistributionCol="topicDistribution", keepLastCheckpoint=True) - Sets params for LDA. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) + return self.__class__._set(**kwargs) @since("2.0.0") def setCheckpointInterval(self, value: int) -> "LDA": @@ -2034,9 +1993,6 @@ class PowerIterationClustering( True """ - _input_kwargs: Dict[str, Any] - - @keyword_only def __init__( self, *, @@ -2047,18 +2003,15 @@ def __init__( dstCol: str = "dst", weightCol: Optional[str] = None, ): - """ - __init__(self, \\*, k=2, maxIter=20, initMode="random", srcCol="src", dstCol="dst",\ - weightCol=None) - """ + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(PowerIterationClustering, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.clustering.PowerIterationClustering", self.uid ) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) - @keyword_only @since("2.4.0") def setParams( self, @@ -2071,12 +2024,12 @@ def setParams( weightCol: Optional[str] = None, ) -> "PowerIterationClustering": """ - setParams(self, \\*, k=2, maxIter=20, initMode="random", srcCol="src", dstCol="dst",\ - weightCol=None) Sets params for PowerIterationClustering. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) + return self.__class__._set(**kwargs) @since("2.4.0") def setK(self, value: int) -> "PowerIterationClustering": diff --git a/python/pyspark/ml/connect/classification.py b/python/pyspark/ml/connect/classification.py index 8b816f51ca275..3b93973cc21ef 100644 --- a/python/pyspark/ml/connect/classification.py +++ b/python/pyspark/ml/connect/classification.py @@ -22,7 +22,6 @@ import numpy as np import pandas as pd -from pyspark import keyword_only from pyspark.ml.connect.base import _PredictorParams from pyspark.ml.param.shared import HasProbabilityCol from pyspark.sql import DataFrame @@ -176,9 +175,6 @@ class LogisticRegression( LogisticRegression_... """ - _input_kwargs: Dict[str, Any] - - @keyword_only def __init__( self, *, @@ -211,9 +207,11 @@ def __init__( seed: int = 0, ) """ + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(LogisticRegression, self).__init__() - kwargs = self._input_kwargs - self._set(**kwargs) + self.__class__._set(**kwargs) def _fit(self, dataset: Union[DataFrame, pd.DataFrame]) -> "LogisticRegressionModel": if isinstance(dataset, pd.DataFrame): diff --git a/python/pyspark/ml/connect/evaluation.py b/python/pyspark/ml/connect/evaluation.py index 15af4dc3aa78a..eec05e03dae93 100644 --- a/python/pyspark/ml/connect/evaluation.py +++ b/python/pyspark/ml/connect/evaluation.py @@ -19,7 +19,6 @@ import numpy as np import pandas as pd -from pyspark import keyword_only from pyspark.ml.param import Param, Params, TypeConverters from pyspark.ml.param.shared import HasLabelCol, HasPredictionCol, HasProbabilityCol from pyspark.ml.connect.base import Evaluator @@ -107,7 +106,6 @@ class RegressionEvaluator(_TorchMetricEvaluator, HasLabelCol, HasPredictionCol, False """ - @keyword_only def __init__( self, *, @@ -175,7 +173,6 @@ class BinaryClassificationEvaluator( True """ - @keyword_only def __init__( self, *, diff --git a/python/pyspark/ml/connect/feature.py b/python/pyspark/ml/connect/feature.py index e618a6af5ae1e..116fc01b61924 100644 --- a/python/pyspark/ml/connect/feature.py +++ b/python/pyspark/ml/connect/feature.py @@ -16,12 +16,11 @@ # import pickle -from typing import Any, Union, List, Tuple, Callable, Dict, Optional +from typing import Any, Union, List, Tuple, Callable, Optional import numpy as np import pandas as pd -from pyspark import keyword_only from pyspark.sql import DataFrame from pyspark.ml.param.shared import ( HasInputCol, @@ -67,16 +66,12 @@ class MaxAbsScaler(Estimator, HasInputCol, HasOutputCol, ParamsReadWrite): +------------+--------------------------+ """ - _input_kwargs: Dict[str, Any] - - @keyword_only def __init__(self, *, inputCol: Optional[str] = None, outputCol: Optional[str] = None) -> None: - """ - __init__(self, \\*, inputCol=None, outputCol=None) - """ + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super().__init__() - kwargs = self._input_kwargs - self._set(**kwargs) + MaxAbsScaler._set(**kwargs) def _fit(self, dataset: Union["pd.DataFrame", "DataFrame"]) -> "MaxAbsScalerModel": input_col = self.getInputCol() @@ -179,16 +174,12 @@ class StandardScaler(Estimator, HasInputCol, HasOutputCol, ParamsReadWrite): +------------+------------------------------------------+ """ - _input_kwargs: Dict[str, Any] - - @keyword_only def __init__(self, inputCol: Optional[str] = None, outputCol: Optional[str] = None) -> None: - """ - __init__(self, \\*, inputCol=None, outputCol=None) - """ + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super().__init__() - kwargs = self._input_kwargs - self._set(**kwargs) + StandardScaler._set(**kwargs) def _fit(self, dataset: Union[DataFrame, pd.DataFrame]) -> "StandardScalerModel": input_col = self.getInputCol() @@ -315,8 +306,6 @@ class ArrayAssembler( >>> assembler.transform(spark_df).select("out").show(truncate=False) """ - _input_kwargs: Dict[str, Any] - # Override doc of handleInvalid param. handleInvalid: Param[str] = Param( Params._dummy(), @@ -327,7 +316,6 @@ class ArrayAssembler( typeConverter=TypeConverters.toString, ) - @keyword_only def __init__( self, *, @@ -336,14 +324,11 @@ def __init__( featureSizes: Optional[List[int]] = None, handleInvalid: Optional[str] = "error", ) -> None: - """ - __init__( - self, \\*, inputCols=None, outputCol=None, featureSizes=None, handleInvalid="error" + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None ) - """ super().__init__() - kwargs = self._input_kwargs - self._set(**kwargs) + ArrayAssembler._set(**kwargs) self._setDefault(handleInvalid="error") def _input_columns(self) -> List[str]: diff --git a/python/pyspark/ml/connect/pipeline.py b/python/pyspark/ml/connect/pipeline.py index 32f44fc8007ad..dc3332863d690 100644 --- a/python/pyspark/ml/connect/pipeline.py +++ b/python/pyspark/ml/connect/pipeline.py @@ -18,7 +18,7 @@ import pandas as pd -from pyspark import keyword_only, since +from pyspark import since from pyspark.ml.connect.base import Estimator, Model, Transformer from pyspark.ml.connect.io_utils import ( ParamsReadWrite, @@ -135,16 +135,12 @@ class Pipeline(Estimator["PipelineModel"], _PipelineReadWrite): Params._dummy(), "stages", "a list of pipeline stages" ) # type: ignore[assignment] - _input_kwargs: Dict[str, Any] - - @keyword_only def __init__(self, *, stages: Optional[List[Params]] = None): - """ - __init__(self, \\*, stages=None) - """ + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(Pipeline, self).__init__() - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) def setStages(self, value: List[Params]) -> "Pipeline": """ @@ -172,15 +168,15 @@ def getStages(self) -> List[Params]: """ return self.getOrDefault(self.stages) - @keyword_only @since("3.5.0") def setParams(self, *, stages: Optional[List[Params]] = None) -> "Pipeline": """ - setParams(self, \\*, stages=None) Sets params for Pipeline. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) + return self.__class__._set(**kwargs) def _fit(self, dataset: Union[DataFrame, pd.DataFrame]) -> "PipelineModel": stages = self.getStages() diff --git a/python/pyspark/ml/connect/tuning.py b/python/pyspark/ml/connect/tuning.py index 97106646f74bc..1b26f750afa12 100644 --- a/python/pyspark/ml/connect/tuning.py +++ b/python/pyspark/ml/connect/tuning.py @@ -32,7 +32,7 @@ import numpy as np import pandas as pd -from pyspark import keyword_only, since, inheritable_thread_target +from pyspark import since, inheritable_thread_target from pyspark.ml.connect import Estimator, Model from pyspark.ml.connect.base import Evaluator from pyspark.ml.connect.io_utils import ( @@ -304,9 +304,6 @@ class CrossValidator( [0.04902833489813031, 0.05247132866444953] """ - _input_kwargs: Dict[str, Any] - - @keyword_only def __init__( self, *, @@ -318,16 +315,13 @@ def __init__( parallelism: int = 1, foldCol: str = "", ) -> None: - """ - __init__(self, \\*, estimator=None, estimatorParamMaps=None, evaluator=None, numFolds=3,\ - seed=None, parallelism=1, foldCol="") - """ + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(CrossValidator, self).__init__() self._setDefault(parallelism=1) - kwargs = self._input_kwargs - self._set(**kwargs) + self.__class__._set(**kwargs) - @keyword_only @since("3.5.0") def setParams( self, @@ -341,12 +335,12 @@ def setParams( foldCol: str = "", ) -> "CrossValidator": """ - setParams(self, \\*, estimator=None, estimatorParamMaps=None, evaluator=None, numFolds=3,\ - seed=None, parallelism=1, collectSubModels=False, foldCol=""): Sets params for cross validator. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) + return self.__class__._set(**kwargs) @since("3.5.0") def setEstimator(self, value: Estimator) -> "CrossValidator": diff --git a/python/pyspark/ml/evaluation.py b/python/pyspark/ml/evaluation.py index c6445c7f02414..5db5e6bc1f4d7 100644 --- a/python/pyspark/ml/evaluation.py +++ b/python/pyspark/ml/evaluation.py @@ -17,9 +17,9 @@ import sys from abc import abstractmethod, ABCMeta -from typing import Any, Dict, Optional, TYPE_CHECKING +from typing import Optional, TYPE_CHECKING -from pyspark import since, keyword_only +from pyspark import since from pyspark.ml.wrapper import JavaParams from pyspark.ml.param import Param, Params, TypeConverters from pyspark.ml.param.shared import ( @@ -218,9 +218,6 @@ class BinaryClassificationEvaluator( typeConverter=TypeConverters.toInt, ) - _input_kwargs: Dict[str, Any] - - @keyword_only def __init__( self, *, @@ -230,17 +227,15 @@ def __init__( weightCol: Optional[str] = None, numBins: int = 1000, ): - """ - __init__(self, \\*, rawPredictionCol="rawPrediction", labelCol="label", \ - metricName="areaUnderROC", weightCol=None, numBins=1000) - """ + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(BinaryClassificationEvaluator, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.evaluation.BinaryClassificationEvaluator", self.uid ) self._setDefault(metricName="areaUnderROC", numBins=1000) - kwargs = self._input_kwargs - self._set(**kwargs) + self.__class__._set(**kwargs) @since("1.4.0") def setMetricName( @@ -291,7 +286,6 @@ def setWeightCol(self, value: str) -> "BinaryClassificationEvaluator": """ return self._set(weightCol=value) - @keyword_only @since("1.4.0") def setParams( self, @@ -303,12 +297,12 @@ def setParams( numBins: int = 1000, ) -> "BinaryClassificationEvaluator": """ - setParams(self, \\*, rawPredictionCol="rawPrediction", labelCol="label", \ - metricName="areaUnderROC", weightCol=None, numBins=1000) Sets params for binary classification evaluator. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) + return self.__class__._set(**kwargs) @inherit_doc @@ -376,9 +370,6 @@ class RegressionEvaluator( typeConverter=TypeConverters.toBoolean, ) - _input_kwargs: Dict[str, Any] - - @keyword_only def __init__( self, *, @@ -388,17 +379,15 @@ def __init__( weightCol: Optional[str] = None, throughOrigin: bool = False, ): - """ - __init__(self, \\*, predictionCol="prediction", labelCol="label", \ - metricName="rmse", weightCol=None, throughOrigin=False) - """ + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(RegressionEvaluator, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.evaluation.RegressionEvaluator", self.uid ) self._setDefault(metricName="rmse", throughOrigin=False) - kwargs = self._input_kwargs - self._set(**kwargs) + self.__class__._set(**kwargs) @since("1.4.0") def setMetricName(self, value: "RegressionEvaluatorMetricType") -> "RegressionEvaluator": @@ -447,7 +436,6 @@ def setWeightCol(self, value: str) -> "RegressionEvaluator": """ return self._set(weightCol=value) - @keyword_only @since("1.4.0") def setParams( self, @@ -459,12 +447,12 @@ def setParams( throughOrigin: bool = False, ) -> "RegressionEvaluator": """ - setParams(self, \\*, predictionCol="prediction", labelCol="label", \ - metricName="rmse", weightCol=None, throughOrigin=False) Sets params for regression evaluator. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) + return self.__class__._set(**kwargs) @inherit_doc @@ -564,9 +552,6 @@ class MulticlassClassificationEvaluator( typeConverter=TypeConverters.toFloat, ) - _input_kwargs: Dict[str, Any] - - @keyword_only def __init__( self, *, @@ -579,18 +564,15 @@ def __init__( probabilityCol: str = "probability", eps: float = 1e-15, ): - """ - __init__(self, \\*, predictionCol="prediction", labelCol="label", \ - metricName="f1", weightCol=None, metricLabel=0.0, beta=1.0, \ - probabilityCol="probability", eps=1e-15) - """ super(MulticlassClassificationEvaluator, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.evaluation.MulticlassClassificationEvaluator", self.uid ) self._setDefault(metricName="f1", metricLabel=0.0, beta=1.0, eps=1e-15) - kwargs = self._input_kwargs - self._set(**kwargs) + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) + self.__class__._set(**kwargs) @since("1.5.0") def setMetricName( @@ -676,7 +658,6 @@ def setWeightCol(self, value: str) -> "MulticlassClassificationEvaluator": """ return self._set(weightCol=value) - @keyword_only @since("1.5.0") def setParams( self, @@ -691,13 +672,12 @@ def setParams( eps: float = 1e-15, ) -> "MulticlassClassificationEvaluator": """ - setParams(self, \\*, predictionCol="prediction", labelCol="label", \ - metricName="f1", weightCol=None, metricLabel=0.0, beta=1.0, \ - probabilityCol="probability", eps=1e-15) Sets params for multiclass classification evaluator. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) + return self.__class__._set(**kwargs) @inherit_doc @@ -757,9 +737,6 @@ class MultilabelClassificationEvaluator( typeConverter=TypeConverters.toFloat, ) - _input_kwargs: Dict[str, Any] - - @keyword_only def __init__( self, *, @@ -768,17 +745,15 @@ def __init__( metricName: "MultilabelClassificationEvaluatorMetricType" = "f1Measure", metricLabel: float = 0.0, ) -> None: - """ - __init__(self, \\*, predictionCol="prediction", labelCol="label", \ - metricName="f1Measure", metricLabel=0.0) - """ + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(MultilabelClassificationEvaluator, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.evaluation.MultilabelClassificationEvaluator", self.uid ) self._setDefault(metricName="f1Measure", metricLabel=0.0) - kwargs = self._input_kwargs - self._set(**kwargs) + self.__class__._set(**kwargs) @since("3.0.0") def setMetricName( @@ -824,7 +799,6 @@ def setPredictionCol(self, value: str) -> "MultilabelClassificationEvaluator": """ return self._set(predictionCol=value) - @keyword_only @since("3.0.0") def setParams( self, @@ -835,12 +809,12 @@ def setParams( metricLabel: float = 0.0, ) -> "MultilabelClassificationEvaluator": """ - setParams(self, \\*, predictionCol="prediction", labelCol="label", \ - metricName="f1Measure", metricLabel=0.0) Sets params for multilabel classification evaluator. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) + return self.__class__._set(**kwargs) @inherit_doc @@ -909,9 +883,6 @@ class ClusteringEvaluator( typeConverter=TypeConverters.toString, # type: ignore[arg-type] ) - _input_kwargs: Dict[str, Any] - - @keyword_only def __init__( self, *, @@ -921,19 +892,16 @@ def __init__( distanceMeasure: str = "squaredEuclidean", weightCol: Optional[str] = None, ): - """ - __init__(self, \\*, predictionCol="prediction", featuresCol="features", \ - metricName="silhouette", distanceMeasure="squaredEuclidean", weightCol=None) - """ + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(ClusteringEvaluator, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.evaluation.ClusteringEvaluator", self.uid ) self._setDefault(metricName="silhouette", distanceMeasure="squaredEuclidean") - kwargs = self._input_kwargs - self._set(**kwargs) + self.__class__._set(**kwargs) - @keyword_only @since("2.3.0") def setParams( self, @@ -945,12 +913,12 @@ def setParams( weightCol: Optional[str] = None, ) -> "ClusteringEvaluator": """ - setParams(self, \\*, predictionCol="prediction", featuresCol="features", \ - metricName="silhouette", distanceMeasure="squaredEuclidean", weightCol=None) Sets params for clustering evaluator. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) + return self.__class__._set(**kwargs) @since("2.3.0") def setMetricName(self, value: "ClusteringEvaluatorMetricType") -> "ClusteringEvaluator": @@ -1054,9 +1022,6 @@ class RankingEvaluator( typeConverter=TypeConverters.toInt, ) - _input_kwargs: Dict[str, Any] - - @keyword_only def __init__( self, *, @@ -1065,17 +1030,15 @@ def __init__( metricName: "RankingEvaluatorMetricType" = "meanAveragePrecision", k: int = 10, ): - """ - __init__(self, \\*, predictionCol="prediction", labelCol="label", \ - metricName="meanAveragePrecision", k=10) - """ + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(RankingEvaluator, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.evaluation.RankingEvaluator", self.uid ) self._setDefault(metricName="meanAveragePrecision", k=10) - kwargs = self._input_kwargs - self._set(**kwargs) + self.__class__._set(**kwargs) @since("3.0.0") def setMetricName(self, value: "RankingEvaluatorMetricType") -> "RankingEvaluator": @@ -1119,7 +1082,6 @@ def setPredictionCol(self, value: str) -> "RankingEvaluator": """ return self._set(predictionCol=value) - @keyword_only @since("3.0.0") def setParams( self, @@ -1130,12 +1092,12 @@ def setParams( k: int = 10, ) -> "RankingEvaluator": """ - setParams(self, \\*, predictionCol="prediction", labelCol="label", \ - metricName="meanAveragePrecision", k=10) Sets params for ranking evaluator. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) + return self.__class__._set(**kwargs) if __name__ == "__main__": diff --git a/python/pyspark/ml/feature.py b/python/pyspark/ml/feature.py index 349b50913d7db..b5d2739bf2c8a 100755 --- a/python/pyspark/ml/feature.py +++ b/python/pyspark/ml/feature.py @@ -28,7 +28,7 @@ TYPE_CHECKING, ) -from pyspark import keyword_only, since, SparkContext +from pyspark import since, SparkContext from pyspark.ml.linalg import _convert_to_vector, DenseMatrix, DenseVector, Vector from pyspark.sql.dataframe import DataFrame from pyspark.ml.param.shared import ( @@ -177,8 +177,6 @@ class Binarizer( ... """ - _input_kwargs: Dict[str, Any] - threshold: Param[float] = Param( Params._dummy(), "threshold", @@ -217,7 +215,6 @@ def __init__( ): ... - @keyword_only def __init__( self, *, @@ -228,15 +225,13 @@ def __init__( inputCols: Optional[List[str]] = None, outputCols: Optional[List[str]] = None, ): - """ - __init__(self, \\*, threshold=0.0, inputCol=None, outputCol=None, thresholds=None, \ - inputCols=None, outputCols=None) - """ + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(Binarizer, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.Binarizer", self.uid) self._setDefault(threshold=0.0) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) @overload def setParams( @@ -258,7 +253,6 @@ def setParams( ) -> "Binarizer": ... - @keyword_only @since("1.4.0") def setParams( self, @@ -271,12 +265,12 @@ def setParams( outputCols: Optional[List[str]] = None, ) -> "Binarizer": """ - setParams(self, \\*, threshold=0.0, inputCol=None, outputCol=None, thresholds=None, \ - inputCols=None, outputCols=None) Sets params for this Binarizer. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) + return self.__class__._set(**kwargs) @since("1.4.0") def setThreshold(self, value: float) -> "Binarizer": @@ -569,9 +563,6 @@ class BucketedRandomProjectionLSH( True """ - _input_kwargs: Dict[str, Any] - - @keyword_only def __init__( self, *, @@ -581,18 +572,15 @@ def __init__( numHashTables: int = 1, bucketLength: Optional[float] = None, ): - """ - __init__(self, \\*, inputCol=None, outputCol=None, seed=None, numHashTables=1, \ - bucketLength=None) - """ + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(BucketedRandomProjectionLSH, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.feature.BucketedRandomProjectionLSH", self.uid ) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) - @keyword_only @since("2.2.0") def setParams( self, @@ -604,12 +592,12 @@ def setParams( bucketLength: Optional[float] = None, ) -> "BucketedRandomProjectionLSH": """ - setParams(self, \\*, inputCol=None, outputCol=None, seed=None, numHashTables=1, \ - bucketLength=None) Sets params for this BucketedRandomProjectionLSH. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) + return self.__class__._set(**kwargs) @since("2.2.0") def setBucketLength(self, value: float) -> "BucketedRandomProjectionLSH": @@ -721,8 +709,6 @@ class Bucketizer( ... """ - _input_kwargs: Dict[str, Any] - splits: Param[List[float]] = Param( Params._dummy(), "splits", @@ -786,7 +772,6 @@ def __init__( ): ... - @keyword_only def __init__( self, *, @@ -798,15 +783,13 @@ def __init__( inputCols: Optional[List[str]] = None, outputCols: Optional[List[str]] = None, ): - """ - __init__(self, \\*, splits=None, inputCol=None, outputCol=None, handleInvalid="error", \ - splitsArray=None, inputCols=None, outputCols=None) - """ + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(Bucketizer, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.Bucketizer", self.uid) self._setDefault(handleInvalid="error") - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) @overload def setParams( @@ -830,7 +813,6 @@ def setParams( ) -> "Bucketizer": ... - @keyword_only @since("1.4.0") def setParams( self, @@ -844,12 +826,12 @@ def setParams( outputCols: Optional[List[str]] = None, ) -> "Bucketizer": """ - setParams(self, \\*, splits=None, inputCol=None, outputCol=None, handleInvalid="error", \ - splitsArray=None, inputCols=None, outputCols=None) Sets params for this Bucketizer. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) + return self.__class__._set(**kwargs) @since("1.4.0") def setSplits(self, value: List[float]) -> "Bucketizer": @@ -1069,9 +1051,6 @@ class CountVectorizer( ... """ - _input_kwargs: Dict[str, Any] - - @keyword_only def __init__( self, *, @@ -1083,16 +1062,13 @@ def __init__( inputCol: Optional[str] = None, outputCol: Optional[str] = None, ): - """ - __init__(self, \\*, minTF=1.0, minDF=1.0, maxDF=2 ** 63 - 1, vocabSize=1 << 18,\ - binary=False, inputCol=None,outputCol=None) - """ + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(CountVectorizer, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.CountVectorizer", self.uid) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) - @keyword_only @since("1.6.0") def setParams( self, @@ -1106,12 +1082,12 @@ def setParams( outputCol: Optional[str] = None, ) -> "CountVectorizer": """ - setParams(self, \\*, minTF=1.0, minDF=1.0, maxDF=2 ** 63 - 1, vocabSize=1 << 18,\ - binary=False, inputCol=None, outputCol=None) Set the params for the CountVectorizer """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) + return self.__class__._set(**kwargs) @since("1.6.0") def setMinTF(self, value: float) -> "CountVectorizer": @@ -1284,8 +1260,6 @@ class DCT(JavaTransformer, HasInputCol, HasOutputCol, JavaMLReadable["DCT"], Jav False """ - _input_kwargs: Dict[str, Any] - inverse: Param[bool] = Param( Params._dummy(), "inverse", @@ -1293,7 +1267,6 @@ class DCT(JavaTransformer, HasInputCol, HasOutputCol, JavaMLReadable["DCT"], Jav typeConverter=TypeConverters.toBoolean, ) - @keyword_only def __init__( self, *, @@ -1301,16 +1274,14 @@ def __init__( inputCol: Optional[str] = None, outputCol: Optional[str] = None, ): - """ - __init__(self, \\*, inverse=False, inputCol=None, outputCol=None) - """ + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(DCT, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.DCT", self.uid) self._setDefault(inverse=False) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) - @keyword_only @since("1.6.0") def setParams( self, @@ -1320,11 +1291,12 @@ def setParams( outputCol: Optional[str] = None, ) -> "DCT": """ - setParams(self, \\*, inverse=False, inputCol=None, outputCol=None) Sets params for this DCT. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) + return self.__class__._set(**kwargs) @since("1.6.0") def setInverse(self, value: bool) -> "DCT": @@ -1392,8 +1364,6 @@ class ElementwiseProduct( True """ - _input_kwargs: Dict[str, Any] - scalingVec: Param[Vector] = Param( Params._dummy(), "scalingVec", @@ -1401,7 +1371,6 @@ class ElementwiseProduct( typeConverter=TypeConverters.toVector, ) - @keyword_only def __init__( self, *, @@ -1409,17 +1378,15 @@ def __init__( inputCol: Optional[str] = None, outputCol: Optional[str] = None, ): - """ - __init__(self, \\*, scalingVec=None, inputCol=None, outputCol=None) - """ + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(ElementwiseProduct, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.feature.ElementwiseProduct", self.uid ) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) - @keyword_only @since("1.5.0") def setParams( self, @@ -1429,11 +1396,12 @@ def setParams( outputCol: Optional[str] = None, ) -> "ElementwiseProduct": """ - setParams(self, \\*, scalingVec=None, inputCol=None, outputCol=None) Sets params for this ElementwiseProduct. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) + return self.__class__._set(**kwargs) @since("2.0.0") def setScalingVec(self, value: Vector) -> "ElementwiseProduct": @@ -1528,8 +1496,6 @@ class FeatureHasher( True """ - _input_kwargs: Dict[str, Any] - categoricalCols: Param[List[str]] = Param( Params._dummy(), "categoricalCols", @@ -1537,7 +1503,6 @@ class FeatureHasher( typeConverter=TypeConverters.toListString, ) - @keyword_only def __init__( self, *, @@ -1546,17 +1511,14 @@ def __init__( outputCol: Optional[str] = None, categoricalCols: Optional[List[str]] = None, ): - """ - __init__(self, \\*, numFeatures=1 << 18, inputCols=None, outputCol=None, \ - categoricalCols=None) - """ + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(FeatureHasher, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.FeatureHasher", self.uid) self._setDefault(numFeatures=1 << 18) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) - @keyword_only @since("2.3.0") def setParams( self, @@ -1567,12 +1529,12 @@ def setParams( categoricalCols: Optional[List[str]] = None, ) -> "FeatureHasher": """ - setParams(self, \\*, numFeatures=1 << 18, inputCols=None, outputCol=None, \ - categoricalCols=None) Sets params for this FeatureHasher. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) + return self.__class__._set(**kwargs) @since("2.3.0") def setCategoricalCols(self, value: List[str]) -> "FeatureHasher": @@ -1650,8 +1612,6 @@ class HashingTF( 5 """ - _input_kwargs: Dict[str, Any] - binary: Param[bool] = Param( Params._dummy(), "binary", @@ -1661,7 +1621,6 @@ class HashingTF( typeConverter=TypeConverters.toBoolean, ) - @keyword_only def __init__( self, *, @@ -1670,16 +1629,14 @@ def __init__( inputCol: Optional[str] = None, outputCol: Optional[str] = None, ): - """ - __init__(self, \\*, numFeatures=1 << 18, binary=False, inputCol=None, outputCol=None) - """ + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(HashingTF, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.HashingTF", self.uid) self._setDefault(numFeatures=1 << 18, binary=False) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) - @keyword_only @since("1.3.0") def setParams( self, @@ -1690,11 +1647,12 @@ def setParams( outputCol: Optional[str] = None, ) -> "HashingTF": """ - setParams(self, \\*, numFeatures=1 << 18, binary=False, inputCol=None, outputCol=None) Sets params for this HashingTF. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) + return self.__class__._set(**kwargs) @since("2.0.0") def setBinary(self, value: bool) -> "HashingTF": @@ -1811,9 +1769,6 @@ class IDF(JavaEstimator["IDFModel"], _IDFParams, JavaMLReadable["IDF"], JavaMLWr True """ - _input_kwargs: Dict[str, Any] - - @keyword_only def __init__( self, *, @@ -1821,15 +1776,13 @@ def __init__( inputCol: Optional[str] = None, outputCol: Optional[str] = None, ): - """ - __init__(self, \\*, minDocFreq=0, inputCol=None, outputCol=None) - """ + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(IDF, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.IDF", self.uid) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) - @keyword_only @since("1.4.0") def setParams( self, @@ -1839,11 +1792,12 @@ def setParams( outputCol: Optional[str] = None, ) -> "IDF": """ - setParams(self, \\*, minDocFreq=0, inputCol=None, outputCol=None) Sets params for this IDF. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) + return self.__class__._set(**kwargs) @since("1.4.0") def setMinDocFreq(self, value: int) -> "IDF": @@ -2075,8 +2029,6 @@ class Imputer( True """ - _input_kwargs: Dict[str, Any] - @overload def __init__( self, @@ -2101,7 +2053,6 @@ def __init__( ): ... - @keyword_only def __init__( self, *, @@ -2113,14 +2064,12 @@ def __init__( outputCol: Optional[str] = None, relativeError: float = 0.001, ): - """ - __init__(self, \\*, strategy="mean", missingValue=float("nan"), inputCols=None, \ - outputCols=None, inputCol=None, outputCol=None, relativeError=0.001): - """ + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(Imputer, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.Imputer", self.uid) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) @overload def setParams( @@ -2146,7 +2095,6 @@ def setParams( ) -> "Imputer": ... - @keyword_only @since("2.2.0") def setParams( self, @@ -2160,12 +2108,12 @@ def setParams( relativeError: float = 0.001, ) -> "Imputer": """ - setParams(self, \\*, strategy="mean", missingValue=float("nan"), inputCols=None, \ - outputCols=None, inputCol=None, outputCol=None, relativeError=0.001) Sets params for this Imputer. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) + return self.__class__._set(**kwargs) @since("2.2.0") def setStrategy(self, value: str) -> "Imputer": @@ -2308,30 +2256,26 @@ class Interaction( True """ - _input_kwargs: Dict[str, Any] - - @keyword_only def __init__(self, *, inputCols: Optional[List[str]] = None, outputCol: Optional[str] = None): - """ - __init__(self, \\*, inputCols=None, outputCol=None): - """ + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(Interaction, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.Interaction", self.uid) self._setDefault() - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) - @keyword_only @since("3.0.0") def setParams( self, *, inputCols: Optional[List[str]] = None, outputCol: Optional[str] = None ) -> "Interaction": """ - setParams(self, \\*, inputCols=None, outputCol=None) Sets params for this Interaction. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) + return self.__class__._set(**kwargs) @since("3.0.0") def setInputCols(self, value: List[str]) -> "Interaction": @@ -2406,30 +2350,26 @@ class MaxAbsScaler( True """ - _input_kwargs: Dict[str, Any] - - @keyword_only def __init__(self, *, inputCol: Optional[str] = None, outputCol: Optional[str] = None): - """ - __init__(self, \\*, inputCol=None, outputCol=None) - """ + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(MaxAbsScaler, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.MaxAbsScaler", self.uid) self._setDefault() - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) - @keyword_only @since("2.0.0") def setParams( self, *, inputCol: Optional[str] = None, outputCol: Optional[str] = None ) -> "MaxAbsScaler": """ - setParams(self, \\*, inputCol=None, outputCol=None) Sets params for this MaxAbsScaler. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) + return self.__class__._set(**kwargs) def setInputCol(self, value: str) -> "MaxAbsScaler": """ @@ -2552,9 +2492,6 @@ class MinHashLSH( True """ - _input_kwargs: Dict[str, Any] - - @keyword_only def __init__( self, *, @@ -2563,15 +2500,13 @@ def __init__( seed: Optional[int] = None, numHashTables: int = 1, ): - """ - __init__(self, \\*, inputCol=None, outputCol=None, seed=None, numHashTables=1) - """ + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(MinHashLSH, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.MinHashLSH", self.uid) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) - @keyword_only @since("2.2.0") def setParams( self, @@ -2582,11 +2517,12 @@ def setParams( numHashTables: int = 1, ) -> "MinHashLSH": """ - setParams(self, \\*, inputCol=None, outputCol=None, seed=None, numHashTables=1) Sets params for this MinHashLSH. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) + return self.__class__._set(**kwargs) def setSeed(self, value: int) -> "MinHashLSH": """ @@ -2717,9 +2653,6 @@ class MinMaxScaler( True """ - _input_kwargs: Dict[str, Any] - - @keyword_only def __init__( self, *, @@ -2728,15 +2661,13 @@ def __init__( inputCol: Optional[str] = None, outputCol: Optional[str] = None, ): - """ - __init__(self, \\*, min=0.0, max=1.0, inputCol=None, outputCol=None) - """ + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(MinMaxScaler, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.MinMaxScaler", self.uid) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) - @keyword_only @since("1.6.0") def setParams( self, @@ -2747,11 +2678,12 @@ def setParams( outputCol: Optional[str] = None, ) -> "MinMaxScaler": """ - setParams(self, \\*, min=0.0, max=1.0, inputCol=None, outputCol=None) Sets params for this MinMaxScaler. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) + return self.__class__._set(**kwargs) @since("1.6.0") def setMin(self, value: float) -> "MinMaxScaler": @@ -2882,8 +2814,6 @@ class NGram(JavaTransformer, HasInputCol, HasOutputCol, JavaMLReadable["NGram"], True """ - _input_kwargs: Dict[str, Any] - n: Param[int] = Param( Params._dummy(), "n", @@ -2891,30 +2821,28 @@ class NGram(JavaTransformer, HasInputCol, HasOutputCol, JavaMLReadable["NGram"], typeConverter=TypeConverters.toInt, ) - @keyword_only def __init__( self, *, n: int = 2, inputCol: Optional[str] = None, outputCol: Optional[str] = None ): - """ - __init__(self, \\*, n=2, inputCol=None, outputCol=None) - """ + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(NGram, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.NGram", self.uid) self._setDefault(n=2) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) - @keyword_only @since("1.5.0") def setParams( self, *, n: int = 2, inputCol: Optional[str] = None, outputCol: Optional[str] = None ) -> "NGram": """ - setParams(self, \\*, n=2, inputCol=None, outputCol=None) Sets params for this NGram. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) + return self.__class__._set(**kwargs) @since("1.5.0") def setN(self, value: int) -> "NGram": @@ -2982,34 +2910,30 @@ class Normalizer( True """ - _input_kwargs: Dict[str, Any] - p = Param(Params._dummy(), "p", "the p norm value.", typeConverter=TypeConverters.toFloat) - @keyword_only def __init__( self, *, p: float = 2.0, inputCol: Optional[str] = None, outputCol: Optional[str] = None ): - """ - __init__(self, \\*, p=2.0, inputCol=None, outputCol=None) - """ + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(Normalizer, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.Normalizer", self.uid) self._setDefault(p=2.0) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) - @keyword_only @since("1.4.0") def setParams( self, *, p: float = 2.0, inputCol: Optional[str] = None, outputCol: Optional[str] = None ) -> "Normalizer": """ - setParams(self, \\*, p=2.0, inputCol=None, outputCol=None) Sets params for this Normalizer. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) + return self.__class__._set(**kwargs) @since("1.4.0") def setP(self, value: float) -> "Normalizer": @@ -3146,8 +3070,6 @@ class OneHotEncoder( True """ - _input_kwargs: Dict[str, Any] - @overload def __init__( self, @@ -3170,7 +3092,6 @@ def __init__( ): ... - @keyword_only def __init__( self, *, @@ -3181,14 +3102,12 @@ def __init__( inputCol: Optional[str] = None, outputCol: Optional[str] = None, ): - """ - __init__(self, \\*, inputCols=None, outputCols=None, handleInvalid="error", dropLast=True, \ - inputCol=None, outputCol=None) - """ + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(OneHotEncoder, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.OneHotEncoder", self.uid) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) @overload def setParams( @@ -3212,7 +3131,6 @@ def setParams( ) -> "OneHotEncoder": ... - @keyword_only @since("2.3.0") def setParams( self, @@ -3225,12 +3143,12 @@ def setParams( outputCol: Optional[str] = None, ) -> "OneHotEncoder": """ - setParams(self, \\*, inputCols=None, outputCols=None, handleInvalid="error", \ - dropLast=True, inputCol=None, outputCol=None) Sets params for this OneHotEncoder. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) + return self.__class__._set(**kwargs) @since("2.3.0") def setDropLast(self, value: bool) -> "OneHotEncoder": @@ -3378,8 +3296,6 @@ class PolynomialExpansion( True """ - _input_kwargs: Dict[str, Any] - degree: Param[int] = Param( Params._dummy(), "degree", @@ -3387,32 +3303,30 @@ class PolynomialExpansion( typeConverter=TypeConverters.toInt, ) - @keyword_only def __init__( self, *, degree: int = 2, inputCol: Optional[str] = None, outputCol: Optional[str] = None ): - """ - __init__(self, \\*, degree=2, inputCol=None, outputCol=None) - """ + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(PolynomialExpansion, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.feature.PolynomialExpansion", self.uid ) self._setDefault(degree=2) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) - @keyword_only @since("1.4.0") def setParams( self, *, degree: int = 2, inputCol: Optional[str] = None, outputCol: Optional[str] = None ) -> "PolynomialExpansion": """ - setParams(self, \\*, degree=2, inputCol=None, outputCol=None) Sets params for this PolynomialExpansion. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) + return self.__class__._set(**kwargs) @since("1.4.0") def setDegree(self, value: int) -> "PolynomialExpansion": @@ -3546,8 +3460,6 @@ class QuantileDiscretizer( ... """ - _input_kwargs: Dict[str, Any] - numBuckets: Param[int] = Param( Params._dummy(), "numBuckets", @@ -3605,7 +3517,6 @@ def __init__( ): ... - @keyword_only def __init__( self, *, @@ -3618,17 +3529,15 @@ def __init__( inputCols: Optional[List[str]] = None, outputCols: Optional[List[str]] = None, ): - """ - __init__(self, \\*, numBuckets=2, inputCol=None, outputCol=None, relativeError=0.001, \ - handleInvalid="error", numBucketsArray=None, inputCols=None, outputCols=None) - """ + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(QuantileDiscretizer, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.feature.QuantileDiscretizer", self.uid ) self._setDefault(numBuckets=2, relativeError=0.001, handleInvalid="error") - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) @overload def setParams( @@ -3654,7 +3563,6 @@ def setParams( ) -> "QuantileDiscretizer": ... - @keyword_only @since("2.0.0") def setParams( self, @@ -3669,12 +3577,12 @@ def setParams( outputCols: Optional[List[str]] = None, ) -> "QuantileDiscretizer": """ - setParams(self, \\*, numBuckets=2, inputCol=None, outputCol=None, relativeError=0.001, \ - handleInvalid="error", numBucketsArray=None, inputCols=None, outputCols=None) Set the params for the QuantileDiscretizer """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) + return self.__class__._set(**kwargs) @since("2.0.0") def setNumBuckets(self, value: int) -> "QuantileDiscretizer": @@ -3887,9 +3795,6 @@ class RobustScaler( True """ - _input_kwargs: Dict[str, Any] - - @keyword_only def __init__( self, *, @@ -3901,16 +3806,13 @@ def __init__( outputCol: Optional[str] = None, relativeError: float = 0.001, ): - """ - __init__(self, \\*, lower=0.25, upper=0.75, withCentering=False, withScaling=True, \ - inputCol=None, outputCol=None, relativeError=0.001) - """ + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(RobustScaler, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.RobustScaler", self.uid) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) - @keyword_only @since("3.0.0") def setParams( self, @@ -3924,12 +3826,12 @@ def setParams( relativeError: float = 0.001, ) -> "RobustScaler": """ - setParams(self, \\*, lower=0.25, upper=0.75, withCentering=False, withScaling=True, \ - inputCol=None, outputCol=None, relativeError=0.001) Sets params for this RobustScaler. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) + return self.__class__._set(**kwargs) @since("3.0.0") def setLower(self, value: float) -> "RobustScaler": @@ -4076,8 +3978,6 @@ class RegexTokenizer( True """ - _input_kwargs: Dict[str, Any] - minTokenLength: Param[int] = Param( Params._dummy(), "minTokenLength", @@ -4102,7 +4002,6 @@ class RegexTokenizer( typeConverter=TypeConverters.toBoolean, ) - @keyword_only def __init__( self, *, @@ -4113,17 +4012,14 @@ def __init__( outputCol: Optional[str] = None, toLowercase: bool = True, ): - """ - __init__(self, \\*, minTokenLength=1, gaps=True, pattern="\\s+", inputCol=None, \ - outputCol=None, toLowercase=True) - """ + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(RegexTokenizer, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.RegexTokenizer", self.uid) self._setDefault(minTokenLength=1, gaps=True, pattern="\\s+", toLowercase=True) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) - @keyword_only @since("1.4.0") def setParams( self, @@ -4136,12 +4032,12 @@ def setParams( toLowercase: bool = True, ) -> "RegexTokenizer": """ - setParams(self, \\*, minTokenLength=1, gaps=True, pattern="\\s+", inputCol=None, \ - outputCol=None, toLowercase=True) Sets params for this RegexTokenizer. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) + return self.__class__._set(**kwargs) @since("1.4.0") def setMinTokenLength(self, value: int) -> "RegexTokenizer": @@ -4237,31 +4133,27 @@ class SQLTransformer(JavaTransformer, JavaMLReadable["SQLTransformer"], JavaMLWr True """ - _input_kwargs: Dict[str, Any] - statement = Param( Params._dummy(), "statement", "SQL statement", typeConverter=TypeConverters.toString ) - @keyword_only def __init__(self, *, statement: Optional[str] = None): - """ - __init__(self, \\*, statement=None) - """ + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(SQLTransformer, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.SQLTransformer", self.uid) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) - @keyword_only @since("1.6.0") def setParams(self, *, statement: Optional[str] = None) -> "SQLTransformer": """ - setParams(self, \\*, statement=None) Sets params for this SQLTransformer. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) + return self.__class__._set(**kwargs) @since("1.6.0") def setStatement(self, value: str) -> "SQLTransformer": @@ -4369,9 +4261,6 @@ class StandardScaler( True """ - _input_kwargs: Dict[str, Any] - - @keyword_only def __init__( self, *, @@ -4380,15 +4269,13 @@ def __init__( inputCol: Optional[str] = None, outputCol: Optional[str] = None, ): - """ - __init__(self, \\*, withMean=False, withStd=True, inputCol=None, outputCol=None) - """ + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(StandardScaler, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.StandardScaler", self.uid) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) - @keyword_only @since("1.4.0") def setParams( self, @@ -4399,11 +4286,12 @@ def setParams( outputCol: Optional[str] = None, ) -> "StandardScaler": """ - setParams(self, \\*, withMean=False, withStd=True, inputCol=None, outputCol=None) Sets params for this StandardScaler. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) + return self.__class__._set(**kwargs) @since("1.4.0") def setWithMean(self, value: bool) -> "StandardScaler": @@ -4607,8 +4495,6 @@ class StringIndexer( [(0, 0.0, 0.0), (1, 1.0, 1.0), (2, 2.0, 0.0), (3, 0.0, 1.0), (4, 0.0, 1.0), (5, 2.0, 1.0)] """ - _input_kwargs: Dict[str, Any] - @overload def __init__( self, @@ -4631,7 +4517,6 @@ def __init__( ): ... - @keyword_only def __init__( self, *, @@ -4642,14 +4527,12 @@ def __init__( handleInvalid: str = "error", stringOrderType: str = "frequencyDesc", ): - """ - __init__(self, \\*, inputCol=None, outputCol=None, inputCols=None, outputCols=None, \ - handleInvalid="error", stringOrderType="frequencyDesc") - """ + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(StringIndexer, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.StringIndexer", self.uid) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) @overload def setParams( @@ -4673,7 +4556,6 @@ def setParams( ) -> "StringIndexer": ... - @keyword_only @since("1.4.0") def setParams( self, @@ -4686,12 +4568,12 @@ def setParams( stringOrderType: str = "frequencyDesc", ) -> "StringIndexer": """ - setParams(self, \\*, inputCol=None, outputCol=None, inputCols=None, outputCols=None, \ - handleInvalid="error", stringOrderType="frequencyDesc") Sets params for this StringIndexer. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) + return self.__class__._set(**kwargs) def _create_model(self, java_model: "JavaObject") -> "StringIndexerModel": return StringIndexerModel(java_model) @@ -4874,8 +4756,6 @@ class IndexToString( StringIndexer : for converting categorical values into category indices """ - _input_kwargs: Dict[str, Any] - labels: Param[List[str]] = Param( Params._dummy(), "labels", @@ -4884,7 +4764,6 @@ class IndexToString( typeConverter=TypeConverters.toListString, ) - @keyword_only def __init__( self, *, @@ -4892,15 +4771,13 @@ def __init__( outputCol: Optional[str] = None, labels: Optional[List[str]] = None, ): - """ - __init__(self, \\*, inputCol=None, outputCol=None, labels=None) - """ + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(IndexToString, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.IndexToString", self.uid) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) - @keyword_only @since("1.6.0") def setParams( self, @@ -4910,11 +4787,12 @@ def setParams( labels: Optional[List[str]] = None, ) -> "IndexToString": """ - setParams(self, \\*, inputCol=None, outputCol=None, labels=None) Sets params for this IndexToString. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) + return self.__class__._set(**kwargs) @since("1.6.0") def setLabels(self, value: List[str]) -> "IndexToString": @@ -4996,8 +4874,6 @@ class StopWordsRemover( ... """ - _input_kwargs: Dict[str, Any] - stopWords: Param[List[str]] = Param( Params._dummy(), "stopWords", @@ -5041,7 +4917,6 @@ def __init__( ): ... - @keyword_only def __init__( self, *, @@ -5053,10 +4928,9 @@ def __init__( inputCols: Optional[List[str]] = None, outputCols: Optional[List[str]] = None, ): - """ - __init__(self, \\*, inputCol=None, outputCol=None, stopWords=None, caseSensitive=false, \ - locale=None, inputCols=None, outputCols=None) - """ + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(StopWordsRemover, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.feature.StopWordsRemover", self.uid @@ -5066,8 +4940,7 @@ def __init__( caseSensitive=False, locale=self._java_obj.getLocale(), ) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) @overload def setParams( @@ -5093,7 +4966,6 @@ def setParams( ) -> "StopWordsRemover": ... - @keyword_only @since("1.6.0") def setParams( self, @@ -5107,12 +4979,12 @@ def setParams( outputCols: Optional[List[str]] = None, ) -> "StopWordsRemover": """ - setParams(self, \\*, inputCol=None, outputCol=None, stopWords=None, caseSensitive=false, \ - locale=None, inputCols=None, outputCols=None) Sets params for this StopWordRemover. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) + return self.__class__._set(**kwargs) @since("1.6.0") def setStopWords(self, value: List[str]) -> "StopWordsRemover": @@ -5236,29 +5108,25 @@ class Tokenizer( True """ - _input_kwargs: Dict[str, Any] - - @keyword_only def __init__(self, *, inputCol: Optional[str] = None, outputCol: Optional[str] = None): - """ - __init__(self, \\*, inputCol=None, outputCol=None) - """ + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(Tokenizer, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.Tokenizer", self.uid) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) - @keyword_only @since("1.3.0") def setParams( self, *, inputCol: Optional[str] = None, outputCol: Optional[str] = None ) -> "Tokenizer": """ - setParams(self, \\*, inputCol=None, outputCol=None) Sets params for this Tokenizer. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) + return self.__class__._set(**kwargs) def setInputCol(self, value: str) -> "Tokenizer": """ @@ -5327,8 +5195,6 @@ class VectorAssembler( ... """ - _input_kwargs: Dict[str, Any] - handleInvalid: Param[str] = Param( Params._dummy(), "handleInvalid", @@ -5343,7 +5209,6 @@ class VectorAssembler( typeConverter=TypeConverters.toString, ) - @keyword_only def __init__( self, *, @@ -5351,16 +5216,14 @@ def __init__( outputCol: Optional[str] = None, handleInvalid: str = "error", ): - """ - __init__(self, \\*, inputCols=None, outputCol=None, handleInvalid="error") - """ + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(VectorAssembler, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.VectorAssembler", self.uid) self._setDefault(handleInvalid="error") - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) - @keyword_only @since("1.4.0") def setParams( self, @@ -5370,11 +5233,12 @@ def setParams( handleInvalid: str = "error", ) -> "VectorAssembler": """ - setParams(self, \\*, inputCols=None, outputCol=None, handleInvalid="error") Sets params for this VectorAssembler. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) + return self.__class__._set(**kwargs) def setInputCols(self, value: List[str]) -> "VectorAssembler": """ @@ -5529,9 +5393,6 @@ class VectorIndexer( DenseVector([2.0, 1.0]) """ - _input_kwargs: Dict[str, Any] - - @keyword_only def __init__( self, *, @@ -5540,15 +5401,13 @@ def __init__( outputCol: Optional[str] = None, handleInvalid: str = "error", ): - """ - __init__(self, \\*, maxCategories=20, inputCol=None, outputCol=None, handleInvalid="error") - """ + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(VectorIndexer, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.VectorIndexer", self.uid) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) - @keyword_only @since("1.4.0") def setParams( self, @@ -5559,11 +5418,12 @@ def setParams( handleInvalid: str = "error", ) -> "VectorIndexer": """ - setParams(self, \\*, maxCategories=20, inputCol=None, outputCol=None, handleInvalid="error") Sets params for this VectorIndexer. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) + return self.__class__._set(**kwargs) @since("1.4.0") def setMaxCategories(self, value: int) -> "VectorIndexer": @@ -5690,8 +5550,6 @@ class VectorSlicer( True """ - _input_kwargs: Dict[str, Any] - indices: Param[List[int]] = Param( Params._dummy(), "indices", @@ -5709,7 +5567,6 @@ class VectorSlicer( typeConverter=TypeConverters.toListString, ) - @keyword_only def __init__( self, *, @@ -5718,16 +5575,14 @@ def __init__( indices: Optional[List[int]] = None, names: Optional[List[str]] = None, ): - """ - __init__(self, \\*, inputCol=None, outputCol=None, indices=None, names=None) - """ + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(VectorSlicer, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.VectorSlicer", self.uid) self._setDefault(indices=[], names=[]) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) - @keyword_only @since("1.6.0") def setParams( self, @@ -5738,11 +5593,12 @@ def setParams( names: Optional[List[str]] = None, ) -> "VectorSlicer": """ - setParams(self, \\*, inputCol=None, outputCol=None, indices=None, names=None): Sets params for this VectorSlicer. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) + return self.__class__._set(**kwargs) @since("1.6.0") def setIndices(self, value: List[int]) -> "VectorSlicer": @@ -5944,9 +5800,6 @@ class Word2Vec( True """ - _input_kwargs: Dict[str, Any] - - @keyword_only def __init__( self, *, @@ -5961,17 +5814,13 @@ def __init__( windowSize: int = 5, maxSentenceLength: int = 1000, ): - """ - __init__(self, \\*, vectorSize=100, minCount=5, numPartitions=1, stepSize=0.025, \ - maxIter=1, seed=None, inputCol=None, outputCol=None, windowSize=5, \ - maxSentenceLength=1000) - """ + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(Word2Vec, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.Word2Vec", self.uid) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) - @keyword_only @since("1.4.0") def setParams( self, @@ -5988,13 +5837,12 @@ def setParams( maxSentenceLength: int = 1000, ) -> "Word2Vec": """ - setParams(self, \\*, minCount=5, numPartitions=1, stepSize=0.025, maxIter=1, \ - seed=None, inputCol=None, outputCol=None, windowSize=5, \ - maxSentenceLength=1000) Sets params for this Word2Vec. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) + return self.__class__._set(**kwargs) @since("1.4.0") def setVectorSize(self, value: int) -> "Word2Vec": @@ -6185,9 +6033,6 @@ class PCA(JavaEstimator["PCAModel"], _PCAParams, JavaMLReadable["PCA"], JavaMLWr True """ - _input_kwargs: Dict[str, Any] - - @keyword_only def __init__( self, *, @@ -6195,15 +6040,13 @@ def __init__( inputCol: Optional[str] = None, outputCol: Optional[str] = None, ): - """ - __init__(self, \\*, k=None, inputCol=None, outputCol=None) - """ + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(PCA, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.PCA", self.uid) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) - @keyword_only @since("1.5.0") def setParams( self, @@ -6213,11 +6056,12 @@ def setParams( outputCol: Optional[str] = None, ) -> "PCA": """ - setParams(self, \\*, k=None, inputCol=None, outputCol=None) Set params for this PCA. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) + return self.__class__._set(**kwargs) @since("1.5.0") def setK(self, value: int) -> "PCA": @@ -6429,9 +6273,6 @@ class RFormula( 'RFormulaModel(ResolvedRFormula(label=y, terms=[x,s], hasIntercept=true)) (uid=...)' """ - _input_kwargs: Dict[str, Any] - - @keyword_only def __init__( self, *, @@ -6442,17 +6283,13 @@ def __init__( stringIndexerOrderType: str = "frequencyDesc", handleInvalid: str = "error", ): - """ - __init__(self, \\*, formula=None, featuresCol="features", labelCol="label", \ - forceIndexLabel=False, stringIndexerOrderType="frequencyDesc", \ - handleInvalid="error") - """ + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(RFormula, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.RFormula", self.uid) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) - @keyword_only @since("1.5.0") def setParams( self, @@ -6465,13 +6302,12 @@ def setParams( handleInvalid: str = "error", ) -> "RFormula": """ - setParams(self, \\*, formula=None, featuresCol="features", labelCol="label", \ - forceIndexLabel=False, stringIndexerOrderType="frequencyDesc", \ - handleInvalid="error") Sets params for RFormula. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) + return self.__class__._set(**kwargs) @since("1.5.0") def setFormula(self, value: str) -> "RFormula": @@ -6805,9 +6641,6 @@ class ChiSqSelector( True """ - _input_kwargs: Dict[str, Any] - - @keyword_only def __init__( self, *, @@ -6821,17 +6654,13 @@ def __init__( fdr: float = 0.05, fwe: float = 0.05, ): - """ - __init__(self, \\*, numTopFeatures=50, featuresCol="features", outputCol=None, \ - labelCol="label", selectorType="numTopFeatures", percentile=0.1, fpr=0.05, \ - fdr=0.05, fwe=0.05) - """ + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(ChiSqSelector, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.ChiSqSelector", self.uid) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) - @keyword_only @since("2.0.0") def setParams( self, @@ -6847,13 +6676,12 @@ def setParams( fwe: float = 0.05, ) -> "ChiSqSelector": """ - setParams(self, \\*, numTopFeatures=50, featuresCol="features", outputCol=None, \ - labelCol="label", selectorType="numTopFeatures", percentile=0.1, fpr=0.05, \ - fdr=0.05, fwe=0.05) Sets params for this ChiSqSelector. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) + return self.__class__._set(**kwargs) def _create_model(self, java_model: "JavaObject") -> "ChiSqSelectorModel": return ChiSqSelectorModel(java_model) @@ -6909,8 +6737,6 @@ class VectorSizeHint( True """ - _input_kwargs: Dict[str, Any] - size: Param[int] = Param( Params._dummy(), "size", "Size of vectors in column.", typeConverter=TypeConverters.toInt ) @@ -6926,7 +6752,6 @@ class VectorSizeHint( TypeConverters.toString, ) - @keyword_only def __init__( self, *, @@ -6934,15 +6759,14 @@ def __init__( size: Optional[int] = None, handleInvalid: str = "error", ): - """ - __init__(self, \\*, inputCol=None, size=None, handleInvalid="error") - """ + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(VectorSizeHint, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.VectorSizeHint", self.uid) self._setDefault(handleInvalid="error") - self.setParams(**self._input_kwargs) + self.__class__.setParams(**kwargs) - @keyword_only @since("2.3.0") def setParams( self, @@ -6952,11 +6776,12 @@ def setParams( handleInvalid: str = "error", ) -> "VectorSizeHint": """ - setParams(self, \\*, inputCol=None, size=None, handleInvalid="error") Sets params for this VectorSizeHint. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) + return self.__class__._set(**kwargs) @since("2.3.0") def getSize(self) -> int: @@ -7056,9 +6881,6 @@ class VarianceThresholdSelector( True """ - _input_kwargs: Dict[str, Any] - - @keyword_only def __init__( self, *, @@ -7066,18 +6888,16 @@ def __init__( outputCol: Optional[str] = None, varianceThreshold: float = 0.0, ): - """ - __init__(self, \\*, featuresCol="features", outputCol=None, varianceThreshold=0.0) - """ + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(VarianceThresholdSelector, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.feature.VarianceThresholdSelector", self.uid ) self._setDefault(varianceThreshold=0.0) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) - @keyword_only @since("3.1.0") def setParams( self, @@ -7087,11 +6907,12 @@ def setParams( varianceThreshold: float = 0.0, ) -> "VarianceThresholdSelector": """ - setParams(self, \\*, featuresCol="features", outputCol=None, varianceThreshold=0.0) Sets params for this VarianceThresholdSelector. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) + return self.__class__._set(**kwargs) @since("3.1.0") def setVarianceThreshold(self, value: float) -> "VarianceThresholdSelector": @@ -7303,9 +7124,6 @@ class UnivariateFeatureSelector( True """ - _input_kwargs: Dict[str, Any] - - @keyword_only def __init__( self, *, @@ -7314,18 +7132,15 @@ def __init__( labelCol: str = "label", selectionMode: str = "numTopFeatures", ): - """ - __init__(self, \\*, featuresCol="features", outputCol=None, \ - labelCol="label", selectionMode="numTopFeatures") - """ + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(UnivariateFeatureSelector, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.feature.UnivariateFeatureSelector", self.uid ) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) - @keyword_only @since("3.1.1") def setParams( self, @@ -7336,12 +7151,12 @@ def setParams( selectionMode: str = "numTopFeatures", ) -> "UnivariateFeatureSelector": """ - setParams(self, \\*, featuresCol="features", outputCol=None, \ - labelCol="label", selectionMode="numTopFeatures") Sets params for this UnivariateFeatureSelector. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) + return self.__class__._set(**kwargs) @since("3.1.1") def setFeatureType(self, value: str) -> "UnivariateFeatureSelector": diff --git a/python/pyspark/ml/fpm.py b/python/pyspark/ml/fpm.py index cba4219a0694b..1e9e9c7b4c7fc 100644 --- a/python/pyspark/ml/fpm.py +++ b/python/pyspark/ml/fpm.py @@ -16,9 +16,9 @@ # import sys -from typing import Any, Dict, Optional, TYPE_CHECKING +from typing import Any, Optional, TYPE_CHECKING -from pyspark import keyword_only, since +from pyspark import since from pyspark.sql import DataFrame from pyspark.ml.util import JavaMLWritable, JavaMLReadable from pyspark.ml.wrapper import JavaEstimator, JavaModel, JavaParams @@ -235,9 +235,7 @@ class FPGrowth( >>> fpm.transform(data).take(1) == model2.transform(data).take(1) True """ - _input_kwargs: Dict[str, Any] - @keyword_only def __init__( self, *, @@ -247,16 +245,13 @@ def __init__( predictionCol: str = "prediction", numPartitions: Optional[int] = None, ): - """ - __init__(self, \\*, minSupport=0.3, minConfidence=0.8, itemsCol="items", \ - predictionCol="prediction", numPartitions=None) - """ + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(FPGrowth, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.fpm.FPGrowth", self.uid) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) - @keyword_only @since("2.2.0") def setParams( self, @@ -267,12 +262,10 @@ def setParams( predictionCol: str = "prediction", numPartitions: Optional[int] = None, ) -> "FPGrowth": - """ - setParams(self, \\*, minSupport=0.3, minConfidence=0.8, itemsCol="items", \ - predictionCol="prediction", numPartitions=None) - """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) + return self.__class__._set(**kwargs) def setItemsCol(self, value: str) -> "FPGrowth": """ @@ -354,8 +347,6 @@ class PrefixSpan(JavaParams): ... """ - _input_kwargs: Dict[str, Any] - minSupport: Param[float] = Param( Params._dummy(), "minSupport", @@ -391,7 +382,6 @@ class PrefixSpan(JavaParams): typeConverter=TypeConverters.toString, ) - @keyword_only def __init__( self, *, @@ -400,19 +390,16 @@ def __init__( maxLocalProjDBSize: int = 32000000, sequenceCol: str = "sequence", ): - """ - __init__(self, \\*, minSupport=0.1, maxPatternLength=10, maxLocalProjDBSize=32000000, \ - sequenceCol="sequence") - """ + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(PrefixSpan, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.fpm.PrefixSpan", self.uid) self._setDefault( minSupport=0.1, maxPatternLength=10, maxLocalProjDBSize=32000000, sequenceCol="sequence" ) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) - @keyword_only @since("2.4.0") def setParams( self, @@ -422,12 +409,10 @@ def setParams( maxLocalProjDBSize: int = 32000000, sequenceCol: str = "sequence", ) -> "PrefixSpan": - """ - setParams(self, \\*, minSupport=0.1, maxPatternLength=10, maxLocalProjDBSize=32000000, \ - sequenceCol="sequence") - """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) + return self.__class__._set(**kwargs) @since("3.0.0") def setMinSupport(self, value: float) -> "PrefixSpan": diff --git a/python/pyspark/ml/pipeline.py b/python/pyspark/ml/pipeline.py index 24653d1d919ee..adb0471ab8e2b 100644 --- a/python/pyspark/ml/pipeline.py +++ b/python/pyspark/ml/pipeline.py @@ -18,7 +18,7 @@ from typing import Any, Dict, List, Optional, Tuple, Type, Union, cast, TYPE_CHECKING -from pyspark import keyword_only, since, SparkContext +from pyspark import since, SparkContext from pyspark.ml.base import Estimator, Model, Transformer from pyspark.ml.param import Param, Params from pyspark.ml.util import ( @@ -68,16 +68,12 @@ class Pipeline(Estimator["PipelineModel"], MLReadable["Pipeline"], MLWritable): Params._dummy(), "stages", "a list of pipeline stages" ) - _input_kwargs: Dict[str, Any] - - @keyword_only def __init__(self, *, stages: Optional[List["PipelineStage"]] = None): - """ - __init__(self, \\*, stages=None) - """ + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(Pipeline, self).__init__() - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) def setStages(self, value: List["PipelineStage"]) -> "Pipeline": """ @@ -105,15 +101,15 @@ def getStages(self) -> List["PipelineStage"]: """ return self.getOrDefault(self.stages) - @keyword_only @since("1.3.0") def setParams(self, *, stages: Optional[List["PipelineStage"]] = None) -> "Pipeline": """ - setParams(self, \\*, stages=None) Sets params for Pipeline. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) + return self.__class__._set(**kwargs) def _fit(self, dataset: DataFrame) -> "PipelineModel": stages = self.getStages() diff --git a/python/pyspark/ml/recommendation.py b/python/pyspark/ml/recommendation.py index 873140e51afb8..9876f407c024a 100644 --- a/python/pyspark/ml/recommendation.py +++ b/python/pyspark/ml/recommendation.py @@ -16,9 +16,9 @@ # import sys -from typing import Any, Dict, Optional, TYPE_CHECKING +from typing import Any, Optional, TYPE_CHECKING -from pyspark import since, keyword_only +from pyspark import since from pyspark.ml.param.shared import ( HasPredictionCol, HasBlockSize, @@ -364,9 +364,6 @@ class ALS(JavaEstimator["ALSModel"], _ALSParams, JavaMLWritable, JavaMLReadable[ True """ - _input_kwargs: Dict[str, Any] - - @keyword_only def __init__( self, *, @@ -388,19 +385,13 @@ def __init__( coldStartStrategy: str = "nan", blockSize: int = 4096, ): - """ - __init__(self, \\*, rank=10, maxIter=10, regParam=0.1, numUserBlocks=10, - numItemBlocks=10, implicitPrefs=False, alpha=1.0, userCol="user", itemCol="item", \ - seed=None, ratingCol="rating", nonnegative=False, checkpointInterval=10, \ - intermediateStorageLevel="MEMORY_AND_DISK", \ - finalStorageLevel="MEMORY_AND_DISK", coldStartStrategy="nan", blockSize=4096) - """ + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(ALS, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.recommendation.ALS", self.uid) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) - @keyword_only @since("1.4.0") def setParams( self, @@ -424,15 +415,12 @@ def setParams( blockSize: int = 4096, ) -> "ALS": """ - setParams(self, \\*, rank=10, maxIter=10, regParam=0.1, numUserBlocks=10, \ - numItemBlocks=10, implicitPrefs=False, alpha=1.0, userCol="user", itemCol="item", \ - seed=None, ratingCol="rating", nonnegative=False, checkpointInterval=10, \ - intermediateStorageLevel="MEMORY_AND_DISK", \ - finalStorageLevel="MEMORY_AND_DISK", coldStartStrategy="nan", blockSize=4096) Sets params for ALS. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) + return self.__class__._set(**kwargs) def _create_model(self, java_model: "JavaObject") -> "ALSModel": return ALSModel(java_model) diff --git a/python/pyspark/ml/regression.py b/python/pyspark/ml/regression.py index d08e241b41d23..f4d91053cd2c2 100644 --- a/python/pyspark/ml/regression.py +++ b/python/pyspark/ml/regression.py @@ -16,10 +16,10 @@ # import sys -from typing import Any, Dict, Generic, List, Optional, TypeVar, TYPE_CHECKING +from typing import Any, Generic, List, Optional, TypeVar, TYPE_CHECKING from abc import ABCMeta -from pyspark import keyword_only, since +from pyspark import since from pyspark.ml import Predictor, PredictionModel from pyspark.ml.base import _PredictorParams from pyspark.ml.param.shared import ( @@ -294,9 +294,6 @@ class LinearRegression( >>> model.write().format("pmml").save(model_path + "_2") """ - _input_kwargs: Dict[str, Any] - - @keyword_only def __init__( self, *, @@ -316,20 +313,15 @@ def __init__( epsilon: float = 1.35, maxBlockSizeInMB: float = 0.0, ): - """ - __init__(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \ - maxIter=100, regParam=0.0, elasticNetParam=0.0, tol=1e-6, fitIntercept=True, \ - standardization=True, solver="auto", weightCol=None, aggregationDepth=2, \ - loss="squaredError", epsilon=1.35, maxBlockSizeInMB=0.0) - """ + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(LinearRegression, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.regression.LinearRegression", self.uid ) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) - @keyword_only @since("1.4.0") def setParams( self, @@ -351,14 +343,12 @@ def setParams( maxBlockSizeInMB: float = 0.0, ) -> "LinearRegression": """ - setParams(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \ - maxIter=100, regParam=0.0, elasticNetParam=0.0, tol=1e-6, fitIntercept=True, \ - standardization=True, solver="auto", weightCol=None, aggregationDepth=2, \ - loss="squaredError", epsilon=1.35, maxBlockSizeInMB=0.0) Sets params for linear regression. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) + return self.__class__._set(**kwargs) def _create_model(self, java_model: "JavaObject") -> "LinearRegressionModel": return LinearRegressionModel(java_model) @@ -856,9 +846,6 @@ class IsotonicRegression( True """ - _input_kwargs: Dict[str, Any] - - @keyword_only def __init__( self, *, @@ -869,18 +856,15 @@ def __init__( isotonic: bool = True, featureIndex: int = 0, ): - """ - __init__(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \ - weightCol=None, isotonic=True, featureIndex=0): - """ + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(IsotonicRegression, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.regression.IsotonicRegression", self.uid ) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) - @keyword_only def setParams( self, *, @@ -892,12 +876,12 @@ def setParams( featureIndex: int = 0, ) -> "IsotonicRegression": """ - setParams(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \ - weightCol=None, isotonic=True, featureIndex=0): Set the params for IsotonicRegression. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) + return self.__class__._set(**kwargs) def _create_model(self, java_model: "JavaObject") -> "IsotonicRegressionModel": return IsotonicRegressionModel(java_model) @@ -1106,9 +1090,6 @@ class DecisionTreeRegressor( DecisionTreeRegressionModel...depth=1, numNodes=3... """ - _input_kwargs: Dict[str, Any] - - @keyword_only def __init__( self, *, @@ -1129,21 +1110,15 @@ def __init__( leafCol: str = "", minWeightFractionPerNode: float = 0.0, ): - """ - __init__(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \ - maxDepth=5, maxBins=32, minInstancesPerNode=1, minInfoGain=0.0, \ - maxMemoryInMB=256, cacheNodeIds=False, checkpointInterval=10, \ - impurity="variance", seed=None, varianceCol=None, weightCol=None, \ - leafCol="", minWeightFractionPerNode=0.0) - """ + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(DecisionTreeRegressor, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.regression.DecisionTreeRegressor", self.uid ) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) - @keyword_only @since("1.4.0") def setParams( self, @@ -1166,15 +1141,12 @@ def setParams( minWeightFractionPerNode: float = 0.0, ) -> "DecisionTreeRegressor": """ - setParams(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \ - maxDepth=5, maxBins=32, minInstancesPerNode=1, minInfoGain=0.0, \ - maxMemoryInMB=256, cacheNodeIds=False, checkpointInterval=10, \ - impurity="variance", seed=None, varianceCol=None, weightCol=None, \ - leafCol="", minWeightFractionPerNode=0.0) Sets params for the DecisionTreeRegressor. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) + return self.__class__._set(**kwargs) def _create_model(self, java_model: "JavaObject") -> "DecisionTreeRegressionModel": return DecisionTreeRegressionModel(java_model) @@ -1406,9 +1378,6 @@ class RandomForestRegressor( True """ - _input_kwargs: Dict[str, Any] - - @keyword_only def __init__( self, *, @@ -1432,22 +1401,15 @@ def __init__( weightCol: Optional[str] = None, bootstrap: Optional[bool] = True, ): - """ - __init__(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \ - maxDepth=5, maxBins=32, minInstancesPerNode=1, minInfoGain=0.0, \ - maxMemoryInMB=256, cacheNodeIds=False, checkpointInterval=10, \ - impurity="variance", subsamplingRate=1.0, seed=None, numTrees=20, \ - featureSubsetStrategy="auto", leafCol=", minWeightFractionPerNode=0.0", \ - weightCol=None, bootstrap=True) - """ + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(RandomForestRegressor, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.regression.RandomForestRegressor", self.uid ) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) - @keyword_only @since("1.4.0") def setParams( self, @@ -1473,16 +1435,12 @@ def setParams( bootstrap: Optional[bool] = True, ) -> "RandomForestRegressor": """ - setParams(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \ - maxDepth=5, maxBins=32, minInstancesPerNode=1, minInfoGain=0.0, \ - maxMemoryInMB=256, cacheNodeIds=False, checkpointInterval=10, \ - impurity="variance", subsamplingRate=1.0, seed=None, numTrees=20, \ - featureSubsetStrategy="auto", leafCol="", minWeightFractionPerNode=0.0, \ - weightCol=None, bootstrap=True) Sets params for linear regression. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) + return self.__class__._set(**kwargs) def _create_model(self, java_model: "JavaObject") -> "RandomForestRegressionModel": return RandomForestRegressionModel(java_model) @@ -1749,9 +1707,6 @@ class GBTRegressor( 0.01 """ - _input_kwargs: Dict[str, Any] - - @keyword_only def __init__( self, *, @@ -1778,21 +1733,13 @@ def __init__( minWeightFractionPerNode: float = 0.0, weightCol: Optional[str] = None, ): - """ - __init__(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \ - maxDepth=5, maxBins=32, minInstancesPerNode=1, minInfoGain=0.0, \ - maxMemoryInMB=256, cacheNodeIds=False, subsamplingRate=1.0, \ - checkpointInterval=10, lossType="squared", maxIter=20, stepSize=0.1, seed=None, \ - impurity="variance", featureSubsetStrategy="all", validationTol=0.01, \ - validationIndicatorCol=None, leafCol="", minWeightFractionPerNode=0.0, - weightCol=None) - """ + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(GBTRegressor, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.regression.GBTRegressor", self.uid) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) - @keyword_only @since("1.4.0") def setParams( self, @@ -1821,17 +1768,12 @@ def setParams( weightCol: Optional[str] = None, ) -> "GBTRegressor": """ - setParams(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \ - maxDepth=5, maxBins=32, minInstancesPerNode=1, minInfoGain=0.0, \ - maxMemoryInMB=256, cacheNodeIds=False, subsamplingRate=1.0, \ - checkpointInterval=10, lossType="squared", maxIter=20, stepSize=0.1, seed=None, \ - impurity="variance", featureSubsetStrategy="all", validationTol=0.01, \ - validationIndicatorCol=None, leafCol="", minWeightFractionPerNode=0.0, \ - weightCol=None) Sets params for Gradient Boosted Tree Regression. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) + return self.__class__._set(**kwargs) def _create_model(self, java_model: "JavaObject") -> "GBTRegressionModel": return GBTRegressionModel(java_model) @@ -2143,9 +2085,6 @@ class AFTSurvivalRegression( .. versionadded:: 1.6.0 """ - _input_kwargs: Dict[str, Any] - - @keyword_only def __init__( self, *, @@ -2171,20 +2110,15 @@ def __init__( aggregationDepth: int = 2, maxBlockSizeInMB: float = 0.0, ): - """ - __init__(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \ - fitIntercept=True, maxIter=100, tol=1E-6, censorCol="censor", \ - quantileProbabilities=[0.01, 0.05, 0.1, 0.25, 0.5, 0.75, 0.9, 0.95, 0.99], \ - quantilesCol=None, aggregationDepth=2, maxBlockSizeInMB=0.0) - """ + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(AFTSurvivalRegression, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.regression.AFTSurvivalRegression", self.uid ) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) - @keyword_only @since("1.6.0") def setParams( self, @@ -2211,14 +2145,10 @@ def setParams( aggregationDepth: int = 2, maxBlockSizeInMB: float = 0.0, ) -> "AFTSurvivalRegression": - """ - setParams(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \ - fitIntercept=True, maxIter=100, tol=1E-6, censorCol="censor", \ - quantileProbabilities=[0.01, 0.05, 0.1, 0.25, 0.5, 0.75, 0.9, 0.95, 0.99], \ - quantilesCol=None, aggregationDepth=2, maxBlockSizeInMB=0.0): - """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) + return self.__class__._set(**kwargs) def _create_model(self, java_model: "JavaObject") -> "AFTSurvivalRegressionModel": return AFTSurvivalRegressionModel(java_model) @@ -2548,9 +2478,6 @@ class GeneralizedLinearRegression( True """ - _input_kwargs: Dict[str, Any] - - @keyword_only def __init__( self, *, @@ -2571,21 +2498,16 @@ def __init__( offsetCol: Optional[str] = None, aggregationDepth: int = 2, ): - """ - __init__(self, \\*, labelCol="label", featuresCol="features", predictionCol="prediction", \ - family="gaussian", link=None, fitIntercept=True, maxIter=25, tol=1e-6, \ - regParam=0.0, weightCol=None, solver="irls", linkPredictionCol=None, \ - variancePower=0.0, linkPower=None, offsetCol=None, aggregationDepth=2) - """ + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(GeneralizedLinearRegression, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.regression.GeneralizedLinearRegression", self.uid ) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) - @keyword_only @since("2.0.0") def setParams( self, @@ -2608,14 +2530,12 @@ def setParams( aggregationDepth: int = 2, ) -> "GeneralizedLinearRegression": """ - setParams(self, \\*, labelCol="label", featuresCol="features", predictionCol="prediction", \ - family="gaussian", link=None, fitIntercept=True, maxIter=25, tol=1e-6, \ - regParam=0.0, weightCol=None, solver="irls", linkPredictionCol=None, \ - variancePower=0.0, linkPower=None, offsetCol=None, aggregationDepth=2) Sets params for generalized linear regression. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) + return self.__class__._set(**kwargs) def _create_model(self, java_model: "JavaObject") -> "GeneralizedLinearRegressionModel": return GeneralizedLinearRegressionModel(java_model) @@ -3123,9 +3043,6 @@ class FMRegressor( True """ - _input_kwargs: Dict[str, Any] - - @keyword_only def __init__( self, *, @@ -3144,18 +3061,13 @@ def __init__( solver: str = "adamW", seed: Optional[int] = None, ): - """ - __init__(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \ - factorSize=8, fitIntercept=True, fitLinear=True, regParam=0.0, \ - miniBatchFraction=1.0, initStd=0.01, maxIter=100, stepSize=1.0, \ - tol=1e-6, solver="adamW", seed=None) - """ + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(FMRegressor, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.regression.FMRegressor", self.uid) - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) - @keyword_only @since("3.0.0") def setParams( self, @@ -3176,14 +3088,12 @@ def setParams( seed: Optional[int] = None, ) -> "FMRegressor": """ - setParams(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \ - factorSize=8, fitIntercept=True, fitLinear=True, regParam=0.0, \ - miniBatchFraction=1.0, initStd=0.01, maxIter=100, stepSize=1.0, \ - tol=1e-6, solver="adamW", seed=None) Sets Params for FMRegressor. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) + return self.__class__._set(**kwargs) def _create_model(self, java_model: "JavaObject") -> "FMRegressionModel": return FMRegressionModel(java_model) diff --git a/python/pyspark/ml/tuning.py b/python/pyspark/ml/tuning.py index ae028b2f39969..9d143949fb6b2 100644 --- a/python/pyspark/ml/tuning.py +++ b/python/pyspark/ml/tuning.py @@ -37,7 +37,7 @@ import numpy as np -from pyspark import keyword_only, since, SparkContext, inheritable_thread_target +from pyspark import since, SparkContext, inheritable_thread_target from pyspark.ml import Estimator, Transformer, Model from pyspark.ml.common import inherit_doc, _py2java, _java2py from pyspark.ml.evaluation import Evaluator, JavaEvaluator @@ -714,9 +714,6 @@ class CrossValidator( 0.8333... """ - _input_kwargs: Dict[str, Any] - - @keyword_only def __init__( self, *, @@ -729,16 +726,13 @@ def __init__( collectSubModels: bool = False, foldCol: str = "", ) -> None: - """ - __init__(self, \\*, estimator=None, estimatorParamMaps=None, evaluator=None, numFolds=3,\ - seed=None, parallelism=1, collectSubModels=False, foldCol="") - """ + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(CrossValidator, self).__init__() self._setDefault(parallelism=1) - kwargs = self._input_kwargs - self._set(**kwargs) + self.__class__._set(**kwargs) - @keyword_only @since("1.4.0") def setParams( self, @@ -753,12 +747,12 @@ def setParams( foldCol: str = "", ) -> "CrossValidator": """ - setParams(self, \\*, estimator=None, estimatorParamMaps=None, evaluator=None, numFolds=3,\ - seed=None, parallelism=1, collectSubModels=False, foldCol=""): Sets params for cross validator. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) + return self.__class__._set(**kwargs) @since("2.0.0") def setEstimator(self, value: Estimator) -> "CrossValidator": @@ -1345,9 +1339,6 @@ class TrainValidationSplit( 0.833... """ - _input_kwargs: Dict[str, Any] - - @keyword_only def __init__( self, *, @@ -1359,17 +1350,14 @@ def __init__( collectSubModels: bool = False, seed: Optional[int] = None, ) -> None: - """ - __init__(self, \\*, estimator=None, estimatorParamMaps=None, evaluator=None, \ - trainRatio=0.75, parallelism=1, collectSubModels=False, seed=None) - """ + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(TrainValidationSplit, self).__init__() self._setDefault(parallelism=1) - kwargs = self._input_kwargs - self._set(**kwargs) + self.__class__._set(**kwargs) @since("2.0.0") - @keyword_only def setParams( self, *, @@ -1382,12 +1370,12 @@ def setParams( seed: Optional[int] = None, ) -> "TrainValidationSplit": """ - setParams(self, \\*, estimator=None, estimatorParamMaps=None, evaluator=None, \ - trainRatio=0.75, parallelism=1, collectSubModels=False, seed=None): Sets params for the train validation split. """ - kwargs = self._input_kwargs - return self._set(**kwargs) + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) + return self.__class__._set(**kwargs) @since("2.0.0") def setEstimator(self, value: Estimator) -> "TrainValidationSplit": diff --git a/python/pyspark/testing/mlutils.py b/python/pyspark/testing/mlutils.py index 8981e97ea49ba..51e7a7d024aa7 100644 --- a/python/pyspark/testing/mlutils.py +++ b/python/pyspark/testing/mlutils.py @@ -17,7 +17,6 @@ import numpy as np -from pyspark import keyword_only from pyspark.ml import Estimator, Model, Transformer, UnaryTransformer from pyspark.ml.evaluation import Evaluator from pyspark.ml.param import Param, Params, TypeConverters @@ -182,7 +181,6 @@ def setRegParam(self, value): class DummyLogisticRegression( Classifier, _DummyLogisticRegressionParams, DefaultParamsReadable, DefaultParamsWritable ): - @keyword_only def __init__( self, *, @@ -193,11 +191,12 @@ def __init__( regParam=0.0, rawPredictionCol="rawPrediction", ): + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) super(DummyLogisticRegression, self).__init__() - kwargs = self._input_kwargs - self.setParams(**kwargs) + self.__class__.setParams(**kwargs) - @keyword_only def setParams( self, *, @@ -208,8 +207,10 @@ def setParams( regParam=0.0, rawPredictionCol="rawPrediction", ): - kwargs = self._input_kwargs - self._set(**kwargs) + kwargs = dict( + (k, v) for k, v in locals().items() if not k.startswith("_") and v is not None + ) + self.__class__._set(**kwargs) return self def _fit(self, dataset): diff --git a/python/pyspark/tests/typing/test_core.yml b/python/pyspark/tests/typing/test_core.yml index ff58613492e91..1ad6b845e1306 100644 --- a/python/pyspark/tests/typing/test_core.yml +++ b/python/pyspark/tests/typing/test_core.yml @@ -17,4 +17,4 @@ - case: coreImports main: | - from pyspark import keyword_only, Row, SQLContext + from pyspark import Row, SQLContext