SparkML GBT Grid Search.ipynb

{
    "nbformat_minor": 1, 
    "cells": [
        {
            "execution_count": 1, 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [
                {
                    "output_type": "stream", 
                    "name": "stdout", 
                    "text": "Waiting for a Spark session to start...\nSpark Initialization Done! ApplicationId = app-20190906073242-0001\nKERNEL_ID = c41fa2d3-7578-4c35-91e0-ec025ee323e5\n"
                }
            ], 
            "source": "# The code was removed by Watson Studio for sharing."
        }, 
        {
            "execution_count": 2, 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [], 
            "source": "df.createOrReplaceTempView('df')\ndf_two_class = spark.sql(\"select * from df where class in ('Use_telephone','Standup_chair')\")"
        }, 
        {
            "execution_count": 3, 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [], 
            "source": "splits = df_two_class.randomSplit([0.8, 0.2])\ndf_train = splits[0]\ndf_test = splits[1]"
        }, 
        {
            "execution_count": 4, 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [], 
            "source": "from pyspark.ml.feature import StringIndexer, OneHotEncoder\nfrom pyspark.ml.linalg import Vectors\nfrom pyspark.ml.feature import VectorAssembler\nfrom pyspark.ml.feature import Normalizer\n\n\nindexer = StringIndexer(inputCol=\"class\", outputCol=\"label\")\nvectorAssembler = VectorAssembler(inputCols=[\"x\",\"y\",\"z\"],outputCol=\"features\")\nnormalizer = Normalizer(inputCol=\"features\", outputCol=\"features_norm\", p=1.0)"
        }, 
        {
            "execution_count": 5, 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [], 
            "source": "from pyspark.ml.classification import GBTClassifier\ngbt = GBTClassifier(labelCol=\"label\", featuresCol=\"features\", maxIter=10)"
        }, 
        {
            "execution_count": 6, 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [], 
            "source": "from pyspark.ml import Pipeline\npipeline = Pipeline(stages=[indexer, vectorAssembler, normalizer,gbt])"
        }, 
        {
            "execution_count": 8, 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [
                {
                    "output_type": "stream", 
                    "name": "stdout", 
                    "text": "CPU times: user 108 ms, sys: 52 ms, total: 160 ms\nWall time: 25.3 s\n"
                }
            ], 
            "source": "%%time\nmodel = pipeline.fit(df_train)"
        }, 
        {
            "execution_count": 9, 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [], 
            "source": "prediction = model.transform(df_train)"
        }, 
        {
            "execution_count": 10, 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [
                {
                    "execution_count": 10, 
                    "metadata": {}, 
                    "data": {
                        "text/plain": "0.9107966684212155"
                    }, 
                    "output_type": "execute_result"
                }
            ], 
            "source": "from pyspark.ml.evaluation import MulticlassClassificationEvaluator\nbinEval = MulticlassClassificationEvaluator().setMetricName(\"accuracy\") .setPredictionCol(\"prediction\").setLabelCol(\"label\")\nbinEval.evaluate(prediction) "
        }, 
        {
            "execution_count": 11, 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [], 
            "source": "prediction = model.transform(df_test)"
        }, 
        {
            "execution_count": 12, 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [
                {
                    "execution_count": 12, 
                    "metadata": {}, 
                    "data": {
                        "text/plain": "0.9085680047932894"
                    }, 
                    "output_type": "execute_result"
                }
            ], 
            "source": "binEval.evaluate(prediction) "
        }, 
        {
            "execution_count": 13, 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [], 
            "source": "from pyspark.ml.tuning import CrossValidator, ParamGridBuilder\n\nparamGrid = ParamGridBuilder() \\\n    .addGrid(normalizer.p, [1.0, 2.0, 10.0]) \\\n    .addGrid(gbt.maxBins, [2,4,8,16]) \\\n    .addGrid(gbt.maxDepth, [2,4,8,16]) \\\n    .build()"
        }, 
        {
            "execution_count": 14, 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [], 
            "source": "crossval = CrossValidator(estimator=pipeline,\n                          estimatorParamMaps=paramGrid,\n                          evaluator=MulticlassClassificationEvaluator(),\n                          numFolds=4)  \n"
        }, 
        {
            "execution_count": null, 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [], 
            "source": "%%timeit\ncvModel = crossval.fit(df_train)"
        }, 
        {
            "execution_count": 33, 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [], 
            "source": "prediction = cvModel.transform(df_test)"
        }, 
        {
            "execution_count": 34, 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [
                {
                    "execution_count": 34, 
                    "metadata": {}, 
                    "data": {
                        "text/plain": "0.9121697059918125"
                    }, 
                    "output_type": "execute_result"
                }
            ], 
            "source": "binEval.evaluate(prediction) "
        }, 
        {
            "execution_count": 43, 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [
                {
                    "execution_count": 43, 
                    "metadata": {}, 
                    "data": {
                        "text/plain": "\"cacheNodeIds: If false, the algorithm will pass trees to executors to match instances with nodes. If true, the algorithm will cache node IDs for each instance. Caching can speed up training of deeper trees. (default: False)\\ncheckpointInterval: set checkpoint interval (>= 1) or disable checkpoint (-1). E.g. 10 means that the cache will get checkpointed every 10 iterations. Note: this setting will be ignored if the checkpoint directory is not set in the SparkContext (default: 10)\\nfeatureSubsetStrategy: The number of features to consider for splits at each tree node. Supported options: auto, all, onethird, sqrt, log2, (0.0-1.0], [1-n]. (undefined)\\nfeaturesCol: features column name (default: features, current: features)\\nimpurity: Criterion used for information gain calculation (case-insensitive). Supported options: entropy, gini (undefined)\\nlabelCol: label column name (default: label, current: label)\\nlossType: Loss function which GBT tries to minimize (case-insensitive). Supported options: logistic (default: logistic)\\nmaxBins: Max number of bins for discretizing continuous features.  Must be >=2 and >= number of categories for any categorical feature. (default: 32, current: 16)\\nmaxDepth: Maximum depth of the tree. (>= 0) E.g., depth 0 means 1 leaf node; depth 1 means 1 internal node + 2 leaf nodes. (default: 5, current: 16)\\nmaxIter: maximum number of iterations (>= 0) (default: 20, current: 10)\\nmaxMemoryInMB: Maximum memory in MB allocated to histogram aggregation. (default: 256)\\nminInfoGain: Minimum information gain for a split to be considered at a tree node. (default: 0.0)\\nminInstancesPerNode: Minimum number of instances each child must have after split.  If a split causes the left or right child to have fewer than minInstancesPerNode, the split will be discarded as invalid. Should be >= 1. (default: 1)\\npredictionCol: prediction column name (default: prediction)\\nprobabilityCol: Column name for predicted class conditional probabilities. Note: Not all models output well-calibrated probability estimates! These probabilities should be treated as confidences, not precise probabilities (undefined)\\nrawPredictionCol: raw prediction (a.k.a. confidence) column name (undefined)\\nseed: random seed (default: -6370738711343826501)\\nstepSize: Step size (a.k.a. learning rate) in interval (0, 1] for shrinking the contribution of each estimator. (default: 0.1)\\nsubsamplingRate: Fraction of the training data used for learning each decision tree, in range (0, 1]. (default: 1.0)\\nthresholds: Thresholds in multi-class classification to adjust the probability of predicting each class. Array must have length equal to the number of classes, with values > 0 excepting that at most one value may be 0. The class with largest value p/t is predicted, where p is the original probability of that class and t is the class's threshold (undefined)\""
                    }, 
                    "output_type": "execute_result"
                }
            ], 
            "source": "cvModel.bestModel.stages[-1].explainParams()"
        }, 
        {
            "execution_count": null, 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [], 
            "source": ""
        }
    ], 
    "metadata": {
        "kernelspec": {
            "display_name": "Python 3.6 with Spark", 
            "name": "python36", 
            "language": "python3"
        }, 
        "language_info": {
            "mimetype": "text/x-python", 
            "nbconvert_exporter": "python", 
            "version": "3.6.8", 
            "name": "python", 
            "file_extension": ".py", 
            "pygments_lexer": "ipython3", 
            "codemirror_mode": {
                "version": 3, 
                "name": "ipython"
            }
        }
    }, 
    "nbformat": 4
}