SpamEmailClassifier/SpamEmailClassifier.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"Reading a text-based dataset by pandas\"\"\"\n",
    "# 读取以制表符分隔值的文本文件，定义第一列列名为type，余下所有的列为一整体，列名为message。\n",
    "import pandas as pd\n",
    "\n",
    "sms = pd.read_table('sms.tsv', header=None, names=['type', 'message'])\n",
    "\n",
    "#查看sms Dataframe的前五行内容。\n",
    "sms.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 统计type列中不同类型的总和\n",
    "sms['type'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 将type列的内容进行数值转换，将结果存储在新的列label列中\n",
    "sms['label'] = sms['type'].map({'ham': 0, 'spam': 1})\n",
    "\n",
    "sms.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 分别获取原始数据集的特征和标签。\n",
    "X=sms['message']\n",
    "y=sms['label']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 随机将数据集划分成成70%训练集，30%测试集。\n",
    "# 设置random_state参数：get the same output the first time you make the split.\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=1, test_size=0.3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"Vectorizing our dataset：\"\"\"\n",
    "# 初始化文本矢量类实例，将训练文本转化为稀疏矩阵\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "cls = CountVectorizer()\n",
    "X_train = cls.fit_transform(X_train)\n",
    "\n",
    "# transform testing data (using fitted vocabulary) into a document-term matrix\n",
    "X_test = cls.transform(X_test)\n",
    "repr(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"Building and fit LogisticRegression model\"\"\"\n",
    "# 采用逻辑回归和贝叶斯两种模型\n",
    "import pickle\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "\n",
    "cls_dict = {\n",
    "        'LogisticRegression': LogisticRegression(),\n",
    "        'NaiveBayes': MultinomialNB()\n",
    "}\n",
    "\n",
    "# 训练并测试算法：若算法需要调优，可手动删除model序列化文件。\n",
    "for name, cls in cls_dict.items():\n",
    "    try:\n",
    "        with open('%s.pickle' % name, 'rb') as f:\n",
    "            cls = pickle.load(f)\n",
    "    except Exception, e:\n",
    "        # 训练算法\n",
    "        cls.fit(X_train, y_train)\n",
    "        print e\n",
    "\n",
    "        # 序列化算法\n",
    "        with open('%s.pickle' % name, 'wb') as f:\n",
    "            pickle.dump(cls, f)\n",
    "\n",
    "    \"\"\"Evaluating model\"\"\"\n",
    "    # 获取混淆矩阵，及F_1 Score\n",
    "    from sklearn import metrics\n",
    "    print \"%s Algorithm Accuracy: %s\\nconfusion matrix:%s\\nF1_score:%s\\n\" \\\n",
    "          % (name, cls.score(X_test, y_test), metrics.confusion_matrix(y_test, cls.predict(X_test)),\n",
    "             metrics.f1_score(y_test, cls.predict(X_test)))\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}