7. Rule_based Search engine.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##Article searching and Restaurant Matching\n",
    "\n",
    "實作簡易的文章搜尋系統，以及標題餐廳推薦"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 第一部分  文章搜尋系統"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 引入Json檔，並且將所需資料（類別，URL，名稱）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pyspark.rdd.PipelinedRDD'>\n"
     ]
    }
   ],
   "source": [
    "import copy\n",
    "from pyspark.sql import SQLContext\n",
    "import json\n",
    "\n",
    "sqlContext = SQLContext(sc)\n",
    "df = sqlContext.jsonFile(\"./spark_tutorial_article.json\")\n",
    "\n",
    "gf = df.map(lambda x : (x[2],x[5],x[12]))\n",
    "\n",
    "print type(gf)\n",
    "#spark.read.json(sc.wholeTextFiles('./spark_tutorial_article.json').values())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#sc.textFile(\"./spark_tutorial_article.json\").map(json.loads).take(1)[0][u'author']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 用BeautifulSoup擷取內容，並套用Jieba斷詞"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "## getContent: for input aritcle, get it own word set via jieba.cut()\n",
    "def getContent(x):\n",
    "    from bs4 import BeautifulSoup\n",
    "    soup = BeautifulSoup(x)\n",
    "    text = soup.getText().replace('\\n','').replace('\\r','').replace(' ','').replace('\\t','')\n",
    "    import jieba\n",
    "    r = list()\n",
    "    for term in jieba.cut(text):\n",
    "        if len(term) > 1 and checkword(term): r.append(term)\n",
    "    return r\n",
    "\n",
    "def checkword(x):\n",
    "    return all(u'\\u4e00' <= c <= u'\\u9fff' for c in x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "text_token = gf.map(lambda x: (x[0], getContent(x[1]), x[2]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#text_token.first()\n",
    "#text_token.first()\n",
    "#text_token.count()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 計算每篇文章的TF-IDF Vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def cal_tf(tokens):\n",
    "    d = {}\n",
    "    for word in tokens:\n",
    "        if not word in d:\n",
    "            d[word] = 1\n",
    "        else:\n",
    "            d[word] += 1\n",
    "    for word in d:\n",
    "        d[word] = float(d[word])/len(tokens)\n",
    "    return d\n",
    "\n",
    "text_token_tf = text_token.map(lambda x: cal_tf(x[1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#check text_token_tf\n",
    "#text_token_tf.first()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def cal_idf(docs):\n",
    "    N = docs.count()\n",
    "    uniqueTokens = docs.map(lambda x : list(set(x[1])))\n",
    "    token_sum_tuples = uniqueTokens.flatMap(lambda x: x).map(lambda x: (x, 1)).reduceByKey(lambda x,y: x+y)\n",
    "    return token_sum_tuples.map(lambda x : (x[0], float(N)/x[1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def TFIDF(tokens, idfs):\n",
    "    tfidf_Dict = {}\n",
    "    tfs = cal_tf(tokens)\n",
    "    for tk in tfs:\n",
    "        tfs[tk] = tfs[tk]*idfs[tk]\n",
    "    tfidf_Dict = tfs\n",
    "    return tfidf_Dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "美味食記\n"
     ]
    }
   ],
   "source": [
    "doc_idfs = cal_idf(text_token)\n",
    "\n",
    "doc_c = doc_idfs.collectAsMap()  #my idf dict\n",
    "\n",
    "text_tfidf = TFIDF(text_token.collect()[0][1], doc_c)\n",
    "\n",
    "print text_token.collect()[0][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#check text_tfidf\n",
    "#text_tfidf\n",
    "#text_token.collect()[0][1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false
   },
   "source": [
    "### 計算Cosine similarity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import math\n",
    "\n",
    "def dotprod(a, b):\n",
    "    dotsum = 0\n",
    "    for tk in a:\n",
    "        if tk in b:\n",
    "            dotsum += a[tk]*b[tk]\n",
    "    return dotsum\n",
    "\n",
    "def norm(a):\n",
    "    return math.sqrt(dotprod(a,a))\n",
    "\n",
    "def cossim(a, b):\n",
    "    return dotprod(a,b)/(norm(a) * norm(b))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def cosineSimilarity(string1, string2, idfsDictionary):\n",
    "    w1 = tfidf(string1, idfsDictionary)\n",
    "    w2 = tfidf(string2, idfsDictionary)\n",
    "    return cossim(w1, w2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "### Rule One - top words in a text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def showTopWord(link):\n",
    "    tokens = text_token.filter(lambda x: x[2] == link).collect()[0][1]\n",
    "    tokens_weights = TFIDF(tokens, doc_c)\n",
    "    print type(tokens_weights)\n",
    "    tokens_weights_sorted = sorted(tokens_weights, key=tokens_weights.get, reverse=True)\n",
    "    for index in range(0,9):\n",
    "        print tokens_weights_sorted[index], tokens_weights[tokens_weights_sorted[index]]\n",
    "    print tokens_weights_sorted[:14]\n",
    "    return tokens_weights_sorted[:14]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "link = u'http://lovecc6.pixnet.net/blog/post/73513867'\n",
    "#showTopWord(link)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "urls = text_token.map(lambda x : x[2])\n",
    "\n",
    "\n",
    "#top_word_list = text_token.map(lambda x : showTopWord(x[2]))\n",
    "#top_word_list = [showTopWord(i) for i in urls]\n",
    "#top_word_list = urls.map(lambda x: showTopWord(x))\n",
    "#top_word_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#top_word_list"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false
   },
   "source": [
    "### Rule Two - Query in text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "query_input = [u'蝦球', u'辣味', u'泰式']\n",
    "\n",
    "def check_in(query, text):\n",
    "    count = 0\n",
    "    for q in query:\n",
    "        if q in text:\n",
    "            count += 1\n",
    "    return count\n",
    "\n",
    "def query_points(query):\n",
    "    query_points_table = text_token.map(lambda x : check_in(query, x[1]))\n",
    "    return query_points_table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2228"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "query_pts = query_points(query_input).collect()\n",
    "\n",
    "len(query_pts)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Rule 3 - Term Weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def term_weights(tokens):\n",
    "    d = {}\n",
    "    for word in tokens:\n",
    "        if not word in d:\n",
    "            d[word] = 1\n",
    "        else:\n",
    "            d[word] += 1\n",
    "    return d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def term_points(query, point_dict):\n",
    "    points = 0\n",
    "    for i in query:\n",
    "        if i in point_dict:\n",
    "            points += point_dict[i]\n",
    "                \n",
    "    return points\n",
    "\n",
    "tf_list = text_token.map(lambda x : term_weights(x[1])).collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2228"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "term_pts = [term_points(query_input, i) for i in tf_list]\n",
    "len(term_pts)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 計算文章分數"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def doc_points(term_weight_pts, query_pts):\n",
    "#    tw_dict = text_token.map(lambda x: term_weights(x[1])).collect()\n",
    "#    doc_point = text_token.map(lambda x : (((term_points(query_input, tw_dict))*(check_in(query_input, x[1])) , x[2])))\n",
    "    doc_point = [i*j for i,j in zip(term_weight_pts, query_pts)]\n",
    "    \n",
    "    return doc_point"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "url_list = text_token.map(lambda x : (x[2]))\n",
    "\n",
    "total_pts = zip(doc_points(term_pts, query_pts) , url_list.collect())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#print type(total_pts)\n",
    "total_pts_sort = sorted(total_pts, reverse=True)\n",
    "#total_pts_sort"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(60, u'http://changfong.pixnet.net/blog/post/40749658'),\n",
       " (34, u'http://evisko.pixnet.net/blog/post/258052708'),\n",
       " (29, u'http://justnike.pixnet.net/blog/post/61919500'),\n",
       " (28, u'http://wonderfood.pixnet.net/blog/post/198089649'),\n",
       " (28, u'http://lemonadellen.pixnet.net/blog/post/32114431'),\n",
       " (25, u'http://changfong.pixnet.net/blog/post/41828851'),\n",
       " (24,\n",
       "  u'http://sedo888.pixnet.net/blog/post/341765034-%5b%e5%8f%b0%e5%8c%97%5d-%e5%96%9c%e4%be%86%e7%99%bb%e5%a4%a7%e9%a3%af%e5%ba%97-%e2%80%a7-sukhothai%e8%98%87%e5%8f%af%e6%b3%b0%e6%b3%b0%e5%bc%8f%e6%96%99'),\n",
       " (24, u'http://lemonadellen.pixnet.net/blog/post/40637716'),\n",
       " (20, u'http://protozoa.pixnet.net/blog/post/29765279'),\n",
       " (20, u'http://infinite520visa.pixnet.net/blog/post/148834846')]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "total_pts_sort[:10]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## 第二部分 - 實作餐廳的 matching"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 用函式尋找完全配對，回傳分數"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def exact_match(restaurant, title):\n",
    "    if len(restaurant) < 3:\n",
    "        return 0 \n",
    "    exact_match_flag = 0;\n",
    "    if restaurant in title:\n",
    "        exact_match_pts = 1\n",
    "        return 1\n",
    "    else:\n",
    "        return 0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 將標題斷詞，剔除不需要字元"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def title_checkword(x):\n",
    "    return all((u'\\u4e00' <= c <= u'\\u9fff') or ('A' <= c <= 'Z') or ('a' <= c <= 'z') or (\n",
    "            '0' <= c <= '9')for c in x)\n",
    "\n",
    "def cut_title(title):\n",
    "    import jieba\n",
    "    \n",
    "    r = list()\n",
    "    for term in jieba.cut(title):\n",
    "        if title_checkword(term): r.append(term)\n",
    "    return r"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "####測試斷詞函式"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[u'\\u559c\\u4f86',\n",
       " u'\\u767b',\n",
       " u'\\u4e4b',\n",
       " u'\\u5341\\u4e8c',\n",
       " u'\\u5eda',\n",
       " u'All',\n",
       " u'u',\n",
       " u'can',\n",
       " u'eat']"
      ]
     },
     "execution_count": 181,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cut_title(article_info.first()[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###取得標題和URL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pyspark.rdd.PipelinedRDD'>\n",
      "(u'\\u559c\\u4f86\\u767b\\u4e4b\\u5341\\u4e8c\\u5eda   All u can eat ', u'http://louis740321.pixnet.net/blog/post/373737533')\n"
     ]
    }
   ],
   "source": [
    "article_info = df.map(lambda x : (x[11],x[12]))\n",
    "\n",
    "print type(article_info)\n",
    "article_info.first() "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## 將斷過的詞兩兩相接"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def bio_wordset(words):\n",
    "    try:\n",
    "        biogram_str = map(lambda x, y: x+y, words[:-1], words[1:])\n",
    "        return biogram_str\n",
    "    except:\n",
    "        return []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<type 'list'>\n",
      "[u'\\u559c\\u4f86\\u767b', u'\\u767b\\u4e4b', u'\\u4e4b\\u5341\\u4e8c', u'\\u5341\\u4e8c\\u5eda', u'\\u5edaAll', u'Allu', u'ucan', u'caneat']\n"
     ]
    }
   ],
   "source": [
    "bio_test_1 = cut_title(article_info.first()[0])\n",
    "print type(bio_test_1)\n",
    "bio_test_2 = bio_wordset(bio_test_1)\n",
    "print bio_test_2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 取得餐廳名稱資料 數字替換資料 同義詞替換資料"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "res = sc.textFile('./restaurant.csv').map(lambda line: line.split(',')).map(\n",
    "    lambda line: line[2]).collect()[1:]\n",
    "\n",
    "ex_digit = sc.textFile('./exchange_word.csv').map(lambda line: line.split(',')).map(\n",
    "    lambda line: (line[0],line[1])).filter(lambda x: x[0].isdigit()).collect()\n",
    "\n",
    "ex_word = sc.textFile('./exchange_word.csv').map(lambda line: line.split(',')).map(\n",
    "    lambda line: (line[0],line[1])).filter(lambda x: not (x[0].isdigit())).collect()\n",
    "\n",
    "res_data = list(set(res))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(u'\\u71d2\\u8089', u'\\u71d2\\u70e4'), (u'\\u98ef\\u5e97', u'\\u9152\\u5e97'), (u'\\u5403\\u5230\\u98fd', u'\\u81ea\\u52a9\\u9910\\u5ef3'), (u'\\u71d2\\u70e4', u'\\u71d2\\u8089'), (u'buffet', u'\\u81ea\\u52a9\\u9910\\u5ef3')]\n",
      "[(u'2', u'\\u4e8c'), (u'3', u'\\u4e09'), (u'4', u'\\u56db'), (u'5', u'\\u4e94'), (u'6', u'\\u516d'), (u'7', u'\\u4e03'), (u'8', u'\\u516b'), (u'9', u'\\u4e5d'), (u'10', u'\\u5341'), (u'1', u'\\u4e00')]\n"
     ]
    }
   ],
   "source": [
    "print ex_word\n",
    "print ex_digit"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 檢查是否有同義詞出現"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def check_word(title_str):\n",
    "    for w in ex_word:\n",
    "        if w[0] in title_str:\n",
    "            return True\n",
    "    return False"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 替換同義詞"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def change_word(title_str):\n",
    "    for w in ex_word:\n",
    "        if w[0] in title_str:\n",
    "            return title_str.replace(w[0], w[1])\n",
    "    return title_str"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#需要丟入斷過詞的title list\n",
    "def change_list(input_list):\n",
    "    result = ''.join([change_word(i) for i in input_list])\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "32比較公雞燒烤\n"
     ]
    }
   ],
   "source": [
    "change_list_test = u'32比較公雞燒肉'\n",
    "change_list_test = change_list(cut_title(change_list_test))\n",
    "print change_list_test"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 替換數字"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def change_num(input_str):\n",
    "    output_str = ''\n",
    "    for index, i in enumerate(input_str):\n",
    "        count = 0\n",
    "        for j in ex_digit:\n",
    "            if i == j[0]:\n",
    "                count += 1\n",
    "                output_str += j[1]\n",
    "                try:\n",
    "                    if input_str[index+1].isdigit():\n",
    "                        output_str += u'十'\n",
    "                except:\n",
    "                    continue\n",
    "        if count == 0:\n",
    "            output_str += i\n",
    "    return output_str\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "三十二比較公雞迴轉\n"
     ]
    }
   ],
   "source": [
    "change_num_test = u'32比較公雞迴轉'\n",
    "change_num_test = change_num(change_num_test)\n",
    "print change_num_test"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "####檢查餐廳種類資料"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#for i in res_data:\n",
    "#    print i"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 整理出長度大於2的詞"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def creat_long(short_str):\n",
    "    result = [p for p in short_str if (len(p) > 1)]\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[u'\\u559c\\u4f86', u'\\u5341\\u4e8c', u'All', u'can', u'eat']"
      ]
     },
     "execution_count": 191,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "creat_long(cut_title(article_info.first()[0]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 將英文詞分割出來"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 192,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def separate_eng(input_str):\n",
    "    result = list()\n",
    "    for i in input_str:\n",
    "        if i.isalpha() and (('A' <= i[0] <= 'Z') or ('a' <= i[0] <= 'z')):\n",
    "            result.append(i)\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "aa = separate_eng(cut_title(article_info.first()[0]))\n",
    "#print aa\n",
    "#print article_info.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 長詞的比較"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 194,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def long_term_compare(title, name):\n",
    "    count = 0\n",
    "    for i in title:\n",
    "        if i in name:\n",
    "            count += 1\n",
    "    return count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 195,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def bio_long_term_compare(title, name):\n",
    "    count = 0\n",
    "    for i in title:\n",
    "        if i in name:\n",
    "            count += 1\n",
    "    return count"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 包含全部詞的比較"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 196,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def term_compare(title, name):\n",
    "    term_count = 0\n",
    "    for i in title:\n",
    "        if i in name:\n",
    "            term_count += 1\n",
    "    return term_count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 197,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def bio_term_compare(title, name):\n",
    "    term_count = 0\n",
    "    for i in title:\n",
    "        if i in name:\n",
    "            term_count += 1\n",
    "    return term_count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 198,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3"
      ]
     },
     "execution_count": 198,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "q1 = ['豆腐','好臭','誰的','好香','滴油']\n",
    "q2 = ['豆腐','好臭','誰的']\n",
    "term_compare(q1, q2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 導入計分function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 200,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#將六個參數帶入計分 並且儲存計分分佈 分別為： 短詞比對，短詞相接比對，長詞比對，長詞相接比對，英文比對，完全比對\n",
    "\n",
    "def calculate_pts(short_uni, short_bio, long_uni, long_bio, eng_name, exact, pts_record):\n",
    "    a = list()\n",
    "    a.append(short_uni)\n",
    "    a.append(short_bio)\n",
    "    a.append(long_uni)\n",
    "    a.append(long_bio)\n",
    "    a.append(eng_name)\n",
    "    a.append(exact)\n",
    "    pts_record.append(a)\n",
    "    return ((1*short_uni)+(2*short_bio)+(1*long_uni)+(4*long_bio)+(4*eng_name)+(100000*exact))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 實作預測函式"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def predict_restaurant(title):\n",
    "    \n",
    "    #將需要的字串從標題斷出來\n",
    "    title_token = cut_title(change_num(title))\n",
    "    biogram_title_token = bio_wordset(title_token)\n",
    "    long_title_token = creat_long(title_token)\n",
    "    long_biogram_title_token = bio_wordset(long_title_token)\n",
    "    Eng_title_token = separate_eng(title_token)\n",
    "    \n",
    "    pts_list = list()\n",
    "    pts_record = list()\n",
    "    exact_pts = 0\n",
    "    \n",
    "    for i in res_data:\n",
    "        \n",
    "        #對每個餐廳名做需要的處理\n",
    "        exact_pts = exact_match(i, title)\n",
    "        name_token = cut_title(change_num(i))\n",
    "        biogram_name_token = bio_wordset(name_token)\n",
    "        long_name_token = creat_long(name_token)\n",
    "        long_biogram_name_token = bio_wordset(long_name_token)\n",
    "        Eng_name_token = separate_eng(name_token)\n",
    "\n",
    "        #計算參數\n",
    "        short_uni = term_compare(title_token, name_token)\n",
    "        short_bio = bio_term_compare(biogram_title_token,biogram_name_token)\n",
    "        long_uni = long_term_compare(long_title_token, long_name_token)\n",
    "        long_bio = bio_long_term_compare(long_biogram_title_token, long_biogram_name_token)\n",
    "        eng_name = term_compare(Eng_title_token, Eng_name_token)\n",
    "\n",
    "        #導入計分函式\n",
    "        pts_list.append(calculate_pts(\n",
    "            short_uni, short_bio, long_uni, long_bio, eng_name, exact_pts, pts_record))#\n",
    "    \n",
    "    #整理取前三名\n",
    "    rank_list = zip(pts_list, res_data, pts_record)\n",
    "    rank_list_sorted = sorted(rank_list, reverse = True)\n",
    "    return rank_list_sorted[:3]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 對所有文章的標題進行預測"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 201,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "last_test = article_info.map(lambda x: x[0]).map(lambda x : predict_restaurant(x))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "####快速檢查斷詞比對"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "rr = [u'【小宅食記】喜來登kitchen 12早餐吃到飽｜美好一天的開始：台北市中正區',u'雲軒西餐廳 La Rotisserie - 君品酒店',\n",
    "u'非凡大探索-吃到飽-喜來登十二廚下午茶',u'十二廚自助餐廳 - 台北喜來登大飯店',u'槿韓食堂 -韓式料理吃到飽(1F)']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def test_2(inin):\n",
    "    for i in inin:\n",
    "        print '======================'\n",
    "        title_token = cut_title(i)\n",
    "        for j in title_token:\n",
    "            print j\n",
    "        print('---------------')\n",
    "        biogram_title_token = bio_wordset(title_token)\n",
    "        for j in biogram_title_token:\n",
    "            print j\n",
    "        print('---------------')\n",
    "        long_title_token = creat_long(title_token)\n",
    "        for j in long_title_token:\n",
    "            print j\n",
    "        print('---------------')\n",
    "        long_biogram_title_token = bio_wordset(long_title_token)\n",
    "        for j in long_biogram_title_token:\n",
    "            print j\n",
    "        print('---------------')\n",
    "        Eng_title_token = separate_eng(title_token)\n",
    "        for j in Eng_title_token:\n",
    "            print j\n",
    "        print('---------------')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 202,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#test_2(rr)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 將預測結果導成list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 203,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "predict_result = last_test.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 205,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#predict_result"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 檢查每筆前三名餐廳"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#for index,i in enumerate(qqq):\n",
    "#    print index\n",
    "#    for j in i:\n",
    "#        print j[1]\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 列出每筆標題 配對餐廳 分數分佈"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 209,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def evaluation(title, prediction):\n",
    "    count = 0\n",
    "    for a, b in zip(title, prediction):\n",
    "        print count\n",
    "        print ('===========================')\n",
    "        print a\n",
    "        print b[0][1]\n",
    "        print b[0][2]\n",
    "        print ('===========================')\n",
    "        count += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 210,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "evaluation(last_test_1, qqq)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}