o_train_fasttext_using_sohudata.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import fasttext\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 载入sohu分词后的文件 - 24000个样本\n",
    "# 载入分词文件：\n",
    "cutWords_list = []\n",
    "with open('.\\sohudata\\data\\cutWords_list.txt','r',encoding = 'utf-8') as file:\n",
    "    cutWords_list  = [k.split() for k in file]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "花费时间:12.95 mins\n"
     ]
    }
   ],
   "source": [
    "t1 = time.time()\n",
    "fasttext_model = fasttext.train_supervised('.\\sohudata\\data\\cutWords_list.txt',dim = 100,lr = 0.05,word_ngrams=2,ws= 2,epoch= 1000,loss = 'softmax')\n",
    "t2 = time.time()\n",
    "print(\"花费时间:%0.2f\"%((t2-t1)/60),'mins')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# save model to local\n",
    "fasttext_model.save_model('./sohudata/data/fasttext_model.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# reload model again to use\n",
    "model = fasttext.load_model('./sohudata/data/fasttext_model.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([ 0.00392154,  0.00016244, -0.00454881, -0.00425197, -0.00378532,\n",
       "         0.00783424,  0.00268583, -0.00650554,  0.00872836, -0.00513805,\n",
       "        -0.00237193,  0.00046397, -0.00417759, -0.00429803, -0.00286202,\n",
       "        -0.00503487,  0.00994752,  0.00550235,  0.0035421 ,  0.00987214,\n",
       "        -0.00648912,  0.00254227, -0.00227253, -0.00716334, -0.0032738 ,\n",
       "        -0.00655475, -0.00913937,  0.0021687 ,  0.00018927,  0.00245532,\n",
       "         0.00339696,  0.00557171, -0.0057483 ,  0.00323862,  0.00617583,\n",
       "        -0.00276895, -0.00954516,  0.00677714,  0.00606232,  0.00776612,\n",
       "        -0.00234372, -0.00816929, -0.0089922 ,  0.00992604,  0.00999032,\n",
       "        -0.00027562,  0.00296973, -0.00527057,  0.00475217, -0.00726441,\n",
       "         0.0038992 , -0.00135373,  0.00340392,  0.00557986, -0.00931555,\n",
       "        -0.00836603, -0.00215662,  0.00969727, -0.00619385, -0.00020488,\n",
       "         0.00245811,  0.00760074,  0.00137352, -0.00127072,  0.0026076 ,\n",
       "        -0.00087646, -0.00828695, -0.00255115, -0.00640284,  0.00655462,\n",
       "        -0.00191584, -0.00137442,  0.00073851,  0.00345804,  0.00199996,\n",
       "         0.00601391, -0.00538724,  0.00218174,  0.00066983,  0.003039  ,\n",
       "         0.00356466, -0.00383788,  0.00698626, -0.00031397,  0.00580602,\n",
       "         0.00034787, -0.00289209,  0.00026612,  0.00888143,  0.00353224,\n",
       "        -0.00067222, -0.00255692,  0.00434649, -0.00382111,  0.00342258,\n",
       "        -0.0028114 , -0.00238653,  0.00537505, -0.00802103, -0.00069761], dtype=float32),\n",
       " 100)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "### 获得词向量\n",
    "model['技术'],len(model['科技'])"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda root]",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}