experiment.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from __future__ import division"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# gensim modules\n",
    "from gensim import utils\n",
    "from gensim.models.doc2vec import LabeledSentence\n",
    "from gensim.models import Doc2Vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import operator\n",
    "import time\n",
    "import glob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from random import shuffle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from collections import defaultdict"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 計算 POS, NEG 詞頻，觀察一下訓練與測試資料（都 random shuffle 過了）"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Training data (POS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "train_size = 0\n",
    "\n",
    "with open('training-data/train_POS.txt') as f:\n",
    "    words = []\n",
    "    for line in f:\n",
    "        train_size += 1\n",
    "        words += utils.to_unicode(line).split()\n",
    "\n",
    "vocab = defaultdict(lambda: 0)\n",
    "for word in words:\n",
    "    vocab[word] += 1\n",
    "\n",
    "train_POS_vocab = sorted(vocab.items(), key=operator.itemgetter(1), reverse=True)\n",
    "\n",
    "# for key, val in train_POS_vocab:\n",
    "#     print key, val"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Training data (NEG)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "with open('training-data/train_NEG.txt') as f:\n",
    "    words = []\n",
    "    for line in f:\n",
    "        train_size += 1\n",
    "        words += utils.to_unicode(line).split()\n",
    "\n",
    "vocab = defaultdict(lambda: 0)\n",
    "for word in words:\n",
    "    vocab[word] += 1\n",
    "\n",
    "train_NEG_vocab = sorted(vocab.items(), key=operator.itemgetter(1), reverse=True)\n",
    "\n",
    "# for key, val in train_NEG_vocab:\n",
    "#     print key, val"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Testing data (POS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "test_size = 0\n",
    "\n",
    "with open('training-data/test_POS.txt') as f:\n",
    "    words = []\n",
    "    for line in f:\n",
    "        test_size += 1\n",
    "        words += utils.to_unicode(line).split()\n",
    "\n",
    "vocab = defaultdict(lambda: 0)\n",
    "for word in words:\n",
    "    vocab[word] += 1\n",
    "\n",
    "test_POS_vocab = sorted(vocab.items(), key=operator.itemgetter(1), reverse=True)\n",
    "\n",
    "# for key, val in test_POS_vocab:\n",
    "#     print key, val"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Testing data (NEG)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "with open('training-data/test_NEG.txt') as f:\n",
    "    words = []\n",
    "    for line in f:\n",
    "        test_size += 1\n",
    "        words += utils.to_unicode(line).split()\n",
    "\n",
    "vocab = defaultdict(lambda: 0)\n",
    "for word in words:\n",
    "    vocab[word] += 1\n",
    "\n",
    "test_NEG_vocab = sorted(vocab.items(), key=operator.itemgetter(1), reverse=True)\n",
    "\n",
    "# for key, val in test_NEG_vocab:\n",
    "#     print key, val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "POS, NEG 訓練資料各有：658 筆。訓練資料共有：1316 筆\n",
      "POS, NEG 測試資料各有：165 筆。測試資料共有：330 筆\n"
     ]
    }
   ],
   "source": [
    "print 'POS, NEG 訓練資料各有：{} 筆。訓練資料共有：{} 筆'.format(int(train_size/2), train_size)\n",
    "print 'POS, NEG 測試資料各有：{} 筆。測試資料共有：{} 筆'.format(int(test_size/2), test_size)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 開始訓練 doc2vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class LabeledLineSentence(object):\n",
    "    def __init__(self, sources):\n",
    "        self.sources = sources\n",
    "        \n",
    "        flipped = {}\n",
    "        \n",
    "        # make sure that keys are unique\n",
    "        for key, value in sources.items():\n",
    "            if value not in flipped:\n",
    "                flipped[value] = [key]\n",
    "            else:\n",
    "                raise Exception('Non-unique prefix encountered')\n",
    "    \n",
    "    def __iter__(self):\n",
    "        for source, prefix in self.sources.items():\n",
    "            with utils.smart_open(source) as fin:\n",
    "                for item_no, line in enumerate(fin):\n",
    "                    yield LabeledSentence(utils.to_unicode(line).split(), [prefix + '_%s' % item_no])\n",
    "    \n",
    "    def to_array(self):\n",
    "        self.sentences = []\n",
    "        for source, prefix in self.sources.items():\n",
    "            with utils.smart_open(source) as fin:\n",
    "                for item_no, line in enumerate(fin):\n",
    "                    self.sentences.append(LabeledSentence(utils.to_unicode(line).split(), [prefix + '_%s' % item_no]))\n",
    "        return self.sentences\n",
    "    \n",
    "    def sentences_perm(self):\n",
    "        shuffle(self.sentences)\n",
    "        return self.sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "sources = {\n",
    "    'training-data/train_POS.txt':'TRAIN_POS',\n",
    "    'training-data/train_NEG.txt':'TRAIN_NEG',\n",
    "    'training-data/test_POS.txt': 'TEST_POS',\n",
    "    'training-data/test_NEG.txt': 'TEST_NEG'\n",
    "}\n",
    "\n",
    "sentences = LabeledLineSentence(sources)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "d2v = Doc2Vec(min_count=10, window=20, size=100, sample=1e-5, negative=15, workers=20)\n",
    "d2v.build_vocab(sentences.to_array())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 1\n",
      "--- 0.864768981934 seconds ---\n",
      "Epoch: 2\n",
      "--- 0.781358957291 seconds ---\n",
      "Epoch: 3\n",
      "--- 1.02417707443 seconds ---\n",
      "Epoch: 4\n",
      "--- 0.989632844925 seconds ---\n",
      "Epoch: 5\n",
      "--- 0.89649605751 seconds ---\n",
      "Epoch: 6\n",
      "--- 0.956152200699 seconds ---\n",
      "Epoch: 7\n",
      "--- 0.956073999405 seconds ---\n",
      "Epoch: 8\n",
      "--- 0.990180015564 seconds ---\n",
      "Epoch: 9\n",
      "--- 0.909914016724 seconds ---\n",
      "Epoch: 10\n",
      "--- 1.01475381851 seconds ---\n",
      "Epoch: 11\n",
      "--- 0.826818943024 seconds ---\n",
      "Epoch: 12\n",
      "--- 0.78685092926 seconds ---\n",
      "Epoch: 13\n",
      "--- 0.872048854828 seconds ---\n",
      "Epoch: 14\n",
      "--- 1.13171219826 seconds ---\n",
      "Epoch: 15\n",
      "--- 1.13373208046 seconds ---\n",
      "Epoch: 16\n",
      "--- 0.992563962936 seconds ---\n",
      "Epoch: 17\n",
      "--- 0.960308074951 seconds ---\n",
      "Epoch: 18\n",
      "--- 0.773078918457 seconds ---\n",
      "Epoch: 19\n",
      "--- 0.788988113403 seconds ---\n",
      "Epoch: 20\n",
      "--- 0.978390216827 seconds ---\n"
     ]
    }
   ],
   "source": [
    "for epoch in range(20):\n",
    "    print 'Epoch: {}'.format(epoch+1)\n",
    "    start_time = time.time()\n",
    "    d2v.train(sentences.sentences_perm())\n",
    "    print '--- {} seconds ---'.format(time.time() - start_time)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 看看 doc2vec 的 vocabulary (到時候要 handle unknown word)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "VOCAB = [key for key, _ in d2v.vocab.items()]\n",
    "# for vocab in VOCAB: print vocab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1865"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# vocab size\n",
    "len(d2v.vocab.items())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 觀察 word vector 品質"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "高能 0.92427611351\n",
      "预警 0.840725779533\n",
      "要来 0.827927708626\n",
      "神曲 0.764733314514\n",
      "背影 0.758471131325\n",
      "非战斗 0.729398429394\n",
      "离开 0.728282749653\n",
      "请速 0.723563849926\n",
      "人员 0.723447680473\n",
      "一段 0.696435213089\n"
     ]
    }
   ],
   "source": [
    "for key, val in d2v.most_similar(u'前方'):\n",
    "    print key, val"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Save and load doc2vec model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1209,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# d2v.save('model/doc2vec_model.d2v')\n",
    "# d2v = Doc2Vec.load('model/doc2vec_model.d2v')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train a Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "document_dict = {} # key: document tag, value: list of words of document\n",
    "for tagged_doc in sentences.to_array():\n",
    "    document_dict[tagged_doc.tags[0]] = tagged_doc.words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# 我們改以 document 中每個 word 的 vector 乘上該字出現在該 document 的字數權重\n",
    "# 再 weighted average 來表示一個 document 的 vector\n",
    "\n",
    "def convert_words_to_vector(words):\n",
    "    word_count = defaultdict(lambda: 0)\n",
    "    for word in words:\n",
    "        word_count[word] += 1\n",
    "    doc_vec = np.zeros(d2v.vector_size)\n",
    "    for word in words:\n",
    "        try:\n",
    "            doc_vec += d2v[word] #* (word_count[word]/len(words))\n",
    "        except KeyError:\n",
    "            doc_vec += d2v[u'UNK']\n",
    "    return doc_vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# 這樣來取得 tag 為 'TRAIN_POS_2' 的 document (同樣等於是 'TRAIN_POS.txt' 內第二個 document)\n",
    "# for word in document_dict['TRAIN_POS_2']:\n",
    "#     print word"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 把 training data, testing data 的 vectors, label 都各自放進 array (處理成 scikit-learn model input format)\n",
    "第 0 個 document 的 vector 就是 train_arrays[0]，其對應的 label 就是 train_labels[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def get_train_test_data(option='doc2vec'):\n",
    "    # key: 第幾個 document; value: document vector (100-D)\n",
    "    train_arrays = np.zeros((train_size, d2v.vector_size), dtype='f')\n",
    "    # POS: 1; NEG: 0\n",
    "    train_labels = np.zeros(train_size)\n",
    "\n",
    "    half_train_size = int(train_size/2)\n",
    "\n",
    "    for i in range(half_train_size):\n",
    "        prefix_train_pos = 'TRAIN_POS_' + str(i)\n",
    "        prefix_train_neg = 'TRAIN_NEG_' + str(i)\n",
    "        if option == 'doc2vec':\n",
    "            train_arrays[i] = d2v.docvecs[prefix_train_pos]\n",
    "            train_arrays[half_train_size + i] = d2v.docvecs[prefix_train_neg]\n",
    "        if option == 'word2vec':\n",
    "            train_arrays[i] = convert_words_to_vector(document_dict[prefix_train_pos])\n",
    "            train_arrays[half_train_size + i] = convert_words_to_vector(document_dict[prefix_train_neg])\n",
    "        train_labels[i] = 1\n",
    "        train_labels[half_train_size + i] = 0\n",
    "\n",
    "    test_arrays = np.zeros((test_size, d2v.vector_size))\n",
    "    test_labels = np.zeros(test_size)\n",
    "\n",
    "    half_test_size = int(test_size/2)\n",
    "\n",
    "    for i in range(half_test_size):\n",
    "        prefix_test_pos = 'TEST_POS_' + str(i)\n",
    "        prefix_test_neg = 'TEST_NEG_' + str(i)\n",
    "        if option == 'doc2vec':\n",
    "            test_arrays[i] = d2v.docvecs[prefix_test_pos]\n",
    "            test_arrays[half_test_size + i] = d2v.docvecs[prefix_test_neg]\n",
    "        if option == 'word2vec':\n",
    "            test_arrays[i] = convert_words_to_vector(document_dict[prefix_test_pos])\n",
    "            test_arrays[half_test_size + i] = convert_words_to_vector(document_dict[prefix_test_neg])\n",
    "        test_labels[i] = 1\n",
    "        test_labels[half_test_size + i] = 0\n",
    "    \n",
    "    return (train_arrays, train_labels, test_arrays, test_labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train a classification model (need keras)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from keras.models import Sequential\n",
    "from keras.models import load_model\n",
    "from keras.layers import Dense, Dropout\n",
    "from keras.metrics import precision, recall, fmeasure"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "classifier = Sequential()\n",
    "classifier.add(Dense(500, input_dim=d2v.vector_size, init='normal', activation='relu'))\n",
    "classifier.add(Dropout(0.5))\n",
    "classifier.add(Dense(500, init='normal', activation='relu'))\n",
    "classifier.add(Dropout(0.5))\n",
    "classifier.add(Dense(1, activation='sigmoid'))\n",
    "\n",
    "classifier.compile(loss='binary_crossentropy',\n",
    "              optimizer='adam',\n",
    "              metrics=[precision, recall, fmeasure])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "1316/1316 [==============================] - 0s - loss: 0.3465 - precision: 0.8874 - recall: 0.7821 - fmeasure: 0.8291     \n",
      "Epoch 2/20\n",
      "1316/1316 [==============================] - 0s - loss: 0.3417 - precision: 0.9031 - recall: 0.7648 - fmeasure: 0.8237     \n",
      "Epoch 3/20\n",
      "1316/1316 [==============================] - 0s - loss: 0.3342 - precision: 0.8946 - recall: 0.7552 - fmeasure: 0.8175     \n",
      "Epoch 4/20\n",
      "1316/1316 [==============================] - 0s - loss: 0.3453 - precision: 0.8758 - recall: 0.7548 - fmeasure: 0.8076     \n",
      "Epoch 5/20\n",
      "1316/1316 [==============================] - 0s - loss: 0.3276 - precision: 0.9042 - recall: 0.7771 - fmeasure: 0.8319     \n",
      "Epoch 6/20\n",
      "1316/1316 [==============================] - 0s - loss: 0.3408 - precision: 0.9172 - recall: 0.7423 - fmeasure: 0.8196     \n",
      "Epoch 7/20\n",
      "1316/1316 [==============================] - 0s - loss: 0.3206 - precision: 0.9111 - recall: 0.7720 - fmeasure: 0.8331     \n",
      "Epoch 8/20\n",
      "1316/1316 [==============================] - 0s - loss: 0.3365 - precision: 0.8969 - recall: 0.7754 - fmeasure: 0.8298     \n",
      "Epoch 9/20\n",
      "1316/1316 [==============================] - 0s - loss: 0.3281 - precision: 0.9095 - recall: 0.7518 - fmeasure: 0.8195     \n",
      "Epoch 10/20\n",
      "1316/1316 [==============================] - 0s - loss: 0.3280 - precision: 0.8933 - recall: 0.7882 - fmeasure: 0.8353     \n",
      "Epoch 11/20\n",
      "1316/1316 [==============================] - 0s - loss: 0.3297 - precision: 0.9206 - recall: 0.7320 - fmeasure: 0.8136     \n",
      "Epoch 12/20\n",
      "1316/1316 [==============================] - 0s - loss: 0.3119 - precision: 0.9006 - recall: 0.7725 - fmeasure: 0.8296     \n",
      "Epoch 13/20\n",
      "1316/1316 [==============================] - 0s - loss: 0.3164 - precision: 0.8933 - recall: 0.7965 - fmeasure: 0.8392     \n",
      "Epoch 14/20\n",
      "1316/1316 [==============================] - 0s - loss: 0.3035 - precision: 0.9173 - recall: 0.7807 - fmeasure: 0.8413     \n",
      "Epoch 15/20\n",
      "1316/1316 [==============================] - 0s - loss: 0.3027 - precision: 0.9143 - recall: 0.7717 - fmeasure: 0.8351     \n",
      "Epoch 16/20\n",
      "1316/1316 [==============================] - 0s - loss: 0.3050 - precision: 0.9232 - recall: 0.7717 - fmeasure: 0.8383     \n",
      "Epoch 17/20\n",
      "1316/1316 [==============================] - 0s - loss: 0.3171 - precision: 0.8776 - recall: 0.8238 - fmeasure: 0.8434     \n",
      "Epoch 18/20\n",
      "1316/1316 [==============================] - 0s - loss: 0.3170 - precision: 0.9099 - recall: 0.7936 - fmeasure: 0.8452     \n",
      "Epoch 19/20\n",
      "1316/1316 [==============================] - 0s - loss: 0.3123 - precision: 0.9003 - recall: 0.7685 - fmeasure: 0.8279     \n",
      "Epoch 20/20\n",
      "1316/1316 [==============================] - 0s - loss: 0.3032 - precision: 0.9283 - recall: 0.7634 - fmeasure: 0.8373     \n",
      "330/330 [==============================] - 0s     \n"
     ]
    }
   ],
   "source": [
    "train_arrays, train_labels, test_arrays, test_labels = get_train_test_data(option='doc2vec')\n",
    "\n",
    "classifier.fit(train_arrays, train_labels,\n",
    "          nb_epoch=20,\n",
    "          batch_size=128)\n",
    "score = classifier.evaluate(test_arrays, test_labels, batch_size=50)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Save and load model  (need h5py)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "classifier.save('model/classifier_model.h5')\n",
    "# classifier = load_model('model/classifier_model.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Make predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1437,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1439,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading classifier model ...\n",
      "Load classifier model success.\n"
     ]
    }
   ],
   "source": [
    "\"\"\"\n",
    "載入前面訓練好的 models\n",
    "\"\"\"\n",
    "print 'Loading classifier model ...'\n",
    "classifier = load_model('model/classifier_model.h5')\n",
    "print 'Load classifier model success.'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1440,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from math import floor\n",
    "\n",
    "def secondTohhmmss(second):\n",
    "  h = int(floor(second / 3600))\n",
    "  m = int(floor(second % 3600 / 60))\n",
    "  s = int(floor(second % 3600 % 60))\n",
    "  return (str(h) if (h > 9) else '0' + str(h)) + ':' + (str(m) if (m > 9) else '0' + str(m)) + ':' + (str(s) if (s > 9) else '0' + str(s))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1441,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "使用 './processed-script/split.py' 的 'toClip()'\n",
    "依照指定的時間間隔（30秒）將處理完的彈幕切割為數個片段，將每個片段中的所有彈幕串成同一列（視為一個 document）\n",
    "\"\"\"\n",
    "def toClips(filename):\n",
    "    interval = 30\n",
    "    overlap = 15\n",
    "    res = defaultdict(lambda: [])\n",
    "\n",
    "    try:\n",
    "        with open(filename) as f:\n",
    "            data = json.load(f)\n",
    "            for comment in data['comments']:\n",
    "                time = int(comment['time'])\n",
    "                # 0~30, 31~60, 61~90, ...\n",
    "                if time%interval == 0 and time != 0:\n",
    "                    res[time].append(comment)\n",
    "                else:\n",
    "                    res[(time//interval+1)*interval].append(comment)\n",
    "                # 15~45, 46~75, 76~105, ...\n",
    "                if time > 15:\n",
    "                    if (time-overlap)%interval == 0:\n",
    "                        res[time].append(comment)\n",
    "                    else:\n",
    "                        res[(time//interval+1)*interval + overlap].append(comment)\n",
    "                        \n",
    "            # Merge words to document for every comment\n",
    "            document_dict = {}\n",
    "            for time, comments in sorted(res.items()):\n",
    "                document = ' '.join([' '.join(comment['words']) for comment in comments])\n",
    "                document_dict[time] = document\n",
    "            return document_dict\n",
    "        \n",
    "    except FileNotFoundError:\n",
    "        print('Error. Missing {}.'.format(filename))\n",
    "        return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1477,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 1477,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\"\"\n",
    "剛下載未處理過的彈幕(.json)必須放在 './data/' 中\n",
    "使用 os.system 執行系統指令執行彈幕的前處理後，會把處理完的彈幕存到'./processed-data'\n",
    "\n",
    "filename: 要highlight的新電影彈幕資料\n",
    "preprocess_script_dirname: 放前處理程式(preprocess.js)的資料夾路徑\n",
    "processed_data_dirname: 放前處理完後的彈幕資料後的資料夾路徑\n",
    "\"\"\"\n",
    "filename = '逃學威龍3.json'\n",
    "preprocess_script_dirname = 'preprocess-script/'\n",
    "processed_data_dirname = 'processed-data/'\n",
    "\n",
    "os.system('cd {} && node preprocess.js -f {}'.format(preprocess_script_dirname, filename))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1478,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "document = toClips('{}{}'.format(processed_data_dirname, filename))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1495,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "highlight_time_ranges = []\n",
    "for time, doc in sorted(document.items()):\n",
    "    # reload the d2v model to solve different prediction output when input the same document.\n",
    "    d2v = Doc2Vec.load('model/doc2vec_model.d2v')\n",
    "    doc_vec = d2v.infer_vector(doc.split())\n",
    "    predict = classifier.predict(np.array([doc_vec]))[0][0]\n",
    "    if int(round(predict)) == 1:\n",
    "        highlight_time_ranges.append((time-30, time))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1496,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "00:05:30 -- 00:06:00\n",
      "00:06:15 -- 00:06:45\n",
      "00:09:45 -- 00:10:15\n",
      "00:20:30 -- 00:21:00\n",
      "00:20:45 -- 00:21:15\n",
      "00:21:15 -- 00:21:45\n",
      "00:23:30 -- 00:24:00\n",
      "00:25:30 -- 00:26:00\n",
      "00:29:30 -- 00:30:00\n",
      "00:36:30 -- 00:37:00\n",
      "00:56:00 -- 00:56:30\n",
      "00:56:15 -- 00:56:45\n",
      "01:02:15 -- 01:02:45\n",
      "01:04:00 -- 01:04:30\n",
      "01:04:15 -- 01:04:45\n",
      "01:05:30 -- 01:06:00\n",
      "01:05:45 -- 01:06:15\n",
      "01:14:00 -- 01:14:30\n",
      "01:14:15 -- 01:14:45\n",
      "01:15:00 -- 01:15:30\n",
      "01:19:45 -- 01:20:15\n",
      "01:23:00 -- 01:23:30\n",
      "01:25:15 -- 01:25:45\n"
     ]
    }
   ],
   "source": [
    "for start, end in highlight_time_ranges:\n",
    "    print '{} -- {}'.format(secondTohhmmss(start), secondTohhmmss(end))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}