DF 3.4.6 - Saving and Loading Models.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<pyspark.context.SparkContext at 0x7fbeaf4da790>"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "!rm -rf metastore_db/*.lck\n",
    "\n",
    "from pyspark.sql import SQLContext\n",
    "sqlc = SQLContext(sc)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Saving and Loading Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from pyspark.ml.linalg import Vectors\n",
    "\n",
    "df = sqlc.createDataFrame([(1.0, Vectors.dense(1.0, 2.0, 3.0)),\n",
    "                           (1.0, Vectors.dense(2.0, 3.0, 4.0)),\n",
    "                           (0.0, Vectors.dense(-1.0, 1.0, 2.0)),\n",
    "                           (0.0, Vectors.dense(-2.0, 3.0, 5.0))]).toDF(\"label\", \"features\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----+--------------+--------------------+--------------------+----------+\n",
      "|label|      features|       rawPrediction|         probability|prediction|\n",
      "+-----+--------------+--------------------+--------------------+----------+\n",
      "|  1.0| [1.0,2.0,3.0]|[-18.070405604445...|[1.41945802370848...|       1.0|\n",
      "|  1.0| [2.0,3.0,4.0]|[-38.987081234651...|[1.16983808020729...|       1.0|\n",
      "|  0.0|[-1.0,1.0,2.0]|[19.2085506510254...|[0.99999999545187...|       0.0|\n",
      "|  0.0|[-2.0,3.0,5.0]|[29.1902958840818...|[0.99999999999978...|       0.0|\n",
      "+-----+--------------+--------------------+--------------------+----------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from pyspark.ml.classification import LogisticRegression\n",
    "\n",
    "lr = LogisticRegression()\n",
    "\n",
    "lrModel = lr.fit(df)\n",
    "lrModel.transform(df).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lrModel.save(\"lrModel.parquet\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----+--------------+--------------------+--------------------+----------+\n",
      "|label|      features|       rawPrediction|         probability|prediction|\n",
      "+-----+--------------+--------------------+--------------------+----------+\n",
      "|  1.0| [1.0,2.0,3.0]|[-18.070405604445...|[1.41945802370848...|       1.0|\n",
      "|  1.0| [2.0,3.0,4.0]|[-38.987081234651...|[1.16983808020729...|       1.0|\n",
      "|  0.0|[-1.0,1.0,2.0]|[19.2085506510254...|[0.99999999545187...|       0.0|\n",
      "|  0.0|[-2.0,3.0,5.0]|[29.1902958840818...|[0.99999999999978...|       0.0|\n",
      "+-----+--------------+--------------------+--------------------+----------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from pyspark.ml.classification import LogisticRegressionModel\n",
    "\n",
    "sameModel = LogisticRegressionModel.load(\"lrModel.parquet\")\n",
    "sameModel.transform(df).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\"class\":\"org.apache.spark.ml.classification.LogisticRegressionModel\",\"timestamp\":1475089043445,\"sparkVersion\":\"2.0.0\",\"uid\":\"LogisticRegression_43d69252fb4e6810e320\",\"paramMap\":{\"regParam\":0.0,\"tol\":1.0E-6,\"fitIntercept\":true,\"maxIter\":100,\"standardization\":true,\"elasticNetParam\":0.0,\"probabilityCol\":\"probability\",\"rawPredictionCol\":\"rawPrediction\",\"featuresCol\":\"features\",\"labelCol\":\"label\",\"predictionCol\":\"prediction\",\"threshold\":0.5}}\r\n"
     ]
    }
   ],
   "source": [
    "!cat lrModel.parquet/metadata/part-00000 "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}