week_5/hw_5.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import jieba  \n",
    "import jieba.posseg as pseg  \n",
    "import os  \n",
    "import sys  \n",
    "from sklearn import feature_extraction  \n",
    "from sklearn.feature_extraction.text import TfidfTransformer  \n",
    "from sklearn.feature_extraction.text import CountVectorizer  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 匯入week4爬取的91app臉書貼文的檔案"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0                              ✨2018年依舊會是創作者大放異彩的一年!!!\n",
      "1                                   ✨影片已成為2018數位行銷主戰場。\n",
      "2    【91嚴選!】aPure，人間的臭腳終結者。\\n看看一家從解決老闆自身腳臭問題起家，\\n一步...\n",
      "3                                  ✨2017年大家都在Google什麼？\n",
      "4                               ✨連攝影師也要加入AI失業大軍的行列了是否？\n",
      "Name: message, dtype: object\n",
      "0    #91APP新零售開店系統 #全台老闆持續關注\\n2018說明會登場!!!\\n想要把握零售商...\n",
      "1                                 ✨時薪4500元欸，看到小編瞳孔都放大惹\n",
      "2                                  ✨最強廣告大集結，每則都讓人印象深刻。\n",
      "3    今天就是平安夜了!!!\\n不知大家這個聖誕周末是怎麼度過的呢？\\n上星期五 #91APP福委...\n",
      "4          ✨企業要在新零售時代找到新商機，抓住消費者的心，行銷策略與行銷工具也必須跟著調整因應。\n",
      "Name: message, dtype: object\n",
      "0    【91嚴選!每周一店】\\n「拼圖可以有很多想像！它也可以是藝術、文化、童趣...」\\n拼圖總...\n",
      "1          ✨零售商應謹記，男性和女性購物者的差異不僅體現在購買的物品上，他們購物的方式也不一樣。\n",
      "2                                               ✨醜照再見。\n",
      "3                          2018年，零售業主誰能將AI與銷售結合，誰就是贏家。\n",
      "4                     ✨Facebook出手打擊「觸及率誘餌」，以後別再認同請分享了!\n",
      "Name: message, dtype: object\n",
      "0         ✨這棟龐然大物是汽車自動販賣機。對，賣汽車的。\\n#深度體驗是消費者做購買決策中重要一環\n",
      "1    #開店專家告訴你 #說明會現場直擊\\n最新零售趨勢是什麼？智慧零售哪裡厲害？\\n最酷的銷售方...\n",
      "2                 曠世巨作，即將殺青，\\n近期公布，敬請期待。\\n#FB商店真的來啦!!!\n",
      "3                             sam真的是菜市場名。\\n#噢抱歉小編搞錯重點了\n",
      "4                      ⭐無論如何，全聯小編真的是我們的創意標竿與精神領袖，回來真好。\n",
      "Name: message, dtype: object\n",
      "0                           ✨拿回你的連結所有權吧！\\n#91APP合作店家必看\n",
      "1                                             妮可基嫚第一名。\n",
      "2                             誰說影片一定要很長？6秒鐘就能夠說好一個故事了!\n",
      "3                                   ✨看看小米如何從谷底翻身，重返榮耀。\n",
      "4    ✨瞬間知名度暴增，不代表品牌已成功建立鐵粉軍團。品牌一定要有清楚定位與價值主張，才能傳達給消...\n",
      "Name: message, dtype: object\n",
      "0    這是一個6歲小孩年收入超過三億台幣的故事。\\n#youtuber\\n#RyanToysReview\n",
      "1                                            ✨絕對一杯就掛了。\n",
      "2                                          ✨聯爺爺您回來了!!!\n",
      "3    ✨沃爾瑪目標成為「全方位零售商」，期許自己無論是在店內、網路商店或用手機App購物，都能符合...\n",
      "4    有沒有這樣的經驗，同樣是鐵鍋，\\n有的用起來得心應手，好像鍋子是活的，\\n菜炒一炒還有龍飛出...\n",
      "Name: message, dtype: object\n",
      "0                    ✨2018全美最理想就職企業，得獎的是.....!!!!!!!!!\n",
      "1    掌握準確的營運趨勢，品牌才能不再是憑”感覺”設計活動與溝通策略，並真正的往全通路經營、以會員...\n",
      "2                                          HEN有感HEN想支持\n",
      "3                    年關將近，快來瞧瞧2017最棒的APP有哪些~\\n#android\n",
      "4    ✨看到標題小編先放棄。\\n#好啦裡面還是有許多很棒的建議\\n#希望大家都能成為更好的自己\\n...\n",
      "Name: message, dtype: object\n",
      "0    去背去到被鬼笑的日子結束了。\\n#真的很強大\\n#感謝AI讚嘆AI\n",
      "1                  ✨新奇科技!!!\\n#其實hen實用阿\n",
      "Name: message, dtype: object\n"
     ]
    }
   ],
   "source": [
    "# Take each line in target_txt as an unique document\n",
    "# Cut each line for trarget_txt, and collect it in corpus.\n",
    "\n",
    "corpus = []\n",
    "sum=\"\"\n",
    "lsts=[]\n",
    "files=['1.txt','2.txt','3.txt','4.txt','5.txt','6.txt','7.txt','8.txt']\n",
    "for i in files:\n",
    "    file = pd.read_csv(i,header = 0,sep = \"\\t\")\n",
    "    f = file['message']\n",
    "    #把null弄掉\n",
    "    f = f[f.notnull()]\n",
    "    print(f)   \n",
    "    for item in f:\n",
    "        sum += item\n",
    "        lsts.append(item)\n",
    "for line in lsts:\n",
    "    corpus.append(\" \".join(jieba.cut(line.split(',')[0], cut_all=False)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "#將文本中的內容轉換成詞頻矩陣 \n",
    "vectorizer = CountVectorizer()  \n",
    "#計算每個詞語出現的次數\n",
    "num = vectorizer.fit_transform(corpus)  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 查看文本關鍵字"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['13', '2015', '2017', '2018', '4500', '5k6fwb', '91', '91app', 'ai', 'android', 'app', 'apure', 'diy', 'facebook', 'fb', 'gl', 'goo', 'google', 'hen', 'kxcm3q', 'ryantoysreview', 'sam', 'youtuber', '一個', '一定', '一家', '一年', '一店', '一杯', '一樣', '一步', '一環', '一番', '一秒', '一腳', '一致', '一鍵', '上班', '下面', '不僅', '不再', '不忍', '不知', '世界', '並以', '中華', '主張', '主戰場', '之外', '之路', '了解', '人文', '人間', '什麼', '今天', '他們', '代表', '以後別', '以會員為', '企業', '作品', '作者', '你換', '依舊', '傳說', '傳達給', '價值', '億台幣', '先放棄', '免費', '全台', '全方位', '全美', '全聯小編', '公布', '公認', '其實', '凝聚', '出來', '出手', '分享', '切入', '前十名', '創意', '加入', '動人樂章', '印象', '即將', '厲害', '參加', '可以', '台灣', '各國', '各種', '合作', '同樣', '告訴', '周末', '品牌', '哪些', '哪裡', '商品', '商店', '問號', '問題', '嚴選', '回來', '因應', '因為', '國際', '培養', '基嫚', '報名', '場名', '夜晚', '大家', '大小事', '大放', '大物', '大集', '失業', '套房', '女性', '好像', '好拉', '如何', '妮可', '官方', '定位', '實用', '將台灣', '將近', '專家', '專業', '小孩', '小米', '小編', '就掛', '就是', '工具', '巨作', '差異', '已成', '市場', '希望', '平安夜', '年收入', '年關', '店家', '度過', '廚具', '廚房', '廣告', '建立', '建議', '強力', '影師', '影片', '往全', '很多', '很強', '很棒', '很長', '得心', '得獎', '心情', '必看', '必須', '忠誠度', '快來', '快樂', '快速', '怎麼', '情感', '想像', '想要', '愛與', '感覺', '感謝', '應手', '應謹記', '成功', '成為', '我們', '或用', '所有', '所有人', '手機', '才能', '打擊', '找到', '把握', '抓住', '抱歉', '拔得頭', '拼圖', '持續', '掀起', '掌握', '推向', '擁有', '支持', '放大', '故事', '敬請', '數位', '數據', '文化', '新商機', '新奇', '方式', '方法', '日子', '日本', '明天', '明會', '星期五', '是否', '時代', '時薪', '智慧', '暴增', '曠世', '更好', '最強', '最新', '最棒', '最終', '最美', '最酷', '有感', '有點', '期待', '期許', '松果', '核心', '根本', '業主', '榮耀', '標成', '標竿', '標題', '權吧', '款式', '正確', '殺青', '每則', '每周', '決策', '汽車', '沃爾瑪目', '活動', '消費者', '深刻', '深度', '清楚', '準確', '溝通', '炒菜', '無論', '營運', '物品', '獨一無二', '現場', '理想', '生活', '男性', '異彩', '疑難', '登場', '目標', '直擊', '看到', '看看', '看見', '真好', '真正', '真的', '瞧瞧', '瞬間', '瞳孔', '知名度', '知道', '祝福', '票選', '福委會', '科技', '秒鐘', '童趣', '符合', '第一名', '第八名', '策略', '粉絲團', '精神', '系統', '結合', '結束', '結者', '絕對', '經營', '經驗', '網站', '網路', '總動員', '翻身', '聖誕', '聖誕樹', '聖誕節', '聯爺爺', '職企業', '能夠', '能將', '腳終', '腳臭', '膠來', '自動', '自己', '自身', '舉辦', '菜市', '藉由', '藝術', '行列', '行銷', '裡面', '解決', '解答', '觸及', '設計', '許多', '試試', '認同', '誕樹外', '誘餌', '調整', '講了', '變化', '谷底', '豐富', '販售', '販賣機', '購物', '購物者', '購買', '贏家', '走過', '起來', '起來慘', '起家', '超過', '趕快', '趨勢', '軍團', '近期', '這個', '這個關', '這幾棵', '這是', '這棟', '這樣', '這麼', '通路', '連攝', '連結', '進口', '邁進', '還是', '還有', '醜照', '重要', '重返', '重點', '銷售', '鍋子', '鍋底', '鐵粉', '鐵鍋', '鐵鍋害', '開場', '開始', '開店', '開放票', '關注', '除了', '雙眼', '雜症', '零售', '零售商', '需求', '領袖', '顧問', '風景', '驅動', '體現', '體驗', '魅力', '黑人', '龍飛', '龐然']\n"
     ]
    }
   ],
   "source": [
    "word = vectorizer.get_feature_names()  \n",
    "print(word)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0 0 0 ... 0 0 0]\n",
      " [0 0 0 ... 0 0 0]\n",
      " [1 0 0 ... 0 0 0]\n",
      " ...\n",
      " [0 0 0 ... 0 0 0]\n",
      " [0 0 0 ... 0 0 0]\n",
      " [0 0 0 ... 0 0 0]]\n"
     ]
    }
   ],
   "source": [
    "#看詞頻的結果\n",
    "print(num.toarray())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 把詞頻矩陣num統計成TF-IDF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TfidfTransformer(norm='l2', smooth_idf=True, sublinear_tf=False, use_idf=True)\n",
      "[[0.         0.         0.         ... 0.         0.         0.        ]\n",
      " [0.         0.         0.         ... 0.         0.         0.        ]\n",
      " [0.19029761 0.         0.         ... 0.         0.         0.        ]\n",
      " ...\n",
      " [0.         0.         0.         ... 0.         0.         0.        ]\n",
      " [0.         0.         0.         ... 0.         0.         0.        ]\n",
      " [0.         0.         0.         ... 0.         0.         0.        ]]\n"
     ]
    }
   ],
   "source": [
    "transformer = TfidfTransformer()  \n",
    "print(transformer)\n",
    "tfidf = transformer.fit_transform(num)  \n",
    "#tfidf[i][j]表示i篇文本中的tf-idf的權重  \n",
    "print(tfidf.toarray())\n",
    "weight = tfidf.toarray()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 列印每篇文章的tf-idf，第一個for是每篇文章，第二個for則是某篇文本下的詞語weight，選出tf-idf權重>0.3在每篇的字"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----Document 0----\n",
      "2018 0.3126386289923199 0 3\n",
      "一年 0.3887868304775644 0 26\n",
      "作者 0.43333067284379145 0 61\n",
      "依舊 0.43333067284379145 0 63\n",
      "大放 0.43333067284379145 0 117\n",
      "異彩 0.43333067284379145 0 267\n",
      "----Document 1----\n",
      "2018 0.31852526518444774 1 3\n",
      "主戰場 0.44148980541849325 1 47\n",
      "已成 0.44148980541849325 1 142\n",
      "影片 0.3961072522523416 1 157\n",
      "數位 0.44148980541849325 1 205\n",
      "行銷 0.3961072522523416 1 324\n",
      "----Document 2----\n",
      "結者 0.3805952118958697 2 298\n",
      "腳終 0.3805952118958697 2 313\n",
      "----Document 3----\n",
      "2017 0.49469210881631465 3 2\n",
      "google 0.5513696647095098 3 17\n",
      "什麼 0.49469210881631465 3 53\n",
      "大家 0.45447874294388946 3 115\n",
      "----Document 4----\n",
      "ai 0.31893425970447536 4 8\n",
      "加入 0.3869282745735438 4 84\n",
      "失業 0.3869282745735438 4 120\n",
      "影師 0.3869282745735438 4 156\n",
      "是否 0.3869282745735438 4 217\n",
      "行列 0.3869282745735438 4 323\n",
      "連攝 0.3869282745735438 4 363\n",
      "----Document 5----\n",
      "----Document 6----\n",
      "4500 0.4270073111273566 6 4\n",
      "小編 0.35197029943831837 6 136\n",
      "放大 0.4270073111273566 6 202\n",
      "時薪 0.4270073111273566 6 219\n",
      "看到 0.3831134731230942 6 272\n",
      "瞳孔 0.4270073111273566 6 280\n",
      "----Document 7----\n",
      "印象 0.40824829046386296 7 86\n",
      "大集 0.40824829046386296 7 119\n",
      "廣告 0.40824829046386296 7 152\n",
      "最強 0.40824829046386296 7 224\n",
      "每則 0.40824829046386296 7 246\n",
      "深刻 0.40824829046386296 7 253\n",
      "----Document 8----\n",
      "大家 0.3134460772346578 8 115\n",
      "----Document 9----\n",
      "行銷 0.47728465886679283 9 324\n",
      "----Document 10----\n",
      "拼圖 0.7385349062056805 10 195\n",
      "----Document 11----\n",
      "----Document 12----\n",
      "醜照 1.0 12 369\n",
      "----Document 13----\n",
      "ai 0.30444271797519507 13 8\n",
      "就是 0.3313805376096847 13 138\n",
      "業主 0.3693472619773538 13 237\n",
      "結合 0.3693472619773538 13 296\n",
      "能將 0.3693472619773538 13 312\n",
      "贏家 0.3693472619773538 13 345\n",
      "銷售 0.3313805376096847 13 373\n",
      "----Document 14----\n",
      "facebook 0.35794322539816636 14 13\n",
      "以後別 0.35794322539816636 14 57\n",
      "出手 0.35794322539816636 14 79\n",
      "分享 0.32114876886094723 14 80\n",
      "打擊 0.35794322539816636 14 189\n",
      "觸及 0.35794322539816636 14 328\n",
      "認同 0.35794322539816636 14 332\n",
      "誘餌 0.35794322539816636 14 334\n",
      "----Document 15----\n",
      "汽車 0.5130006335249984 15 249\n",
      "----Document 16----\n",
      "明會 0.33197507981296026 16 215\n",
      "開店 0.33197507981296026 16 381\n",
      "零售 0.38743309889603605 16 387\n",
      "----Document 17----\n",
      "fb 0.3134391458453164 17 14\n",
      "公布 0.3134391458453164 17 74\n",
      "即將 0.3134391458453164 17 87\n",
      "巨作 0.3134391458453164 17 140\n",
      "敬請 0.3134391458453164 17 204\n",
      "曠世 0.3134391458453164 17 222\n",
      "期待 0.3134391458453164 17 232\n",
      "殺青 0.3134391458453164 17 245\n",
      "近期 0.3134391458453164 17 354\n",
      "----Document 18----\n",
      "sam 0.39940010202781245 18 21\n",
      "場名 0.39940010202781245 18 113\n",
      "小編 0.3292144416339898 18 136\n",
      "抱歉 0.39940010202781245 18 193\n",
      "真的 0.30661970569499497 18 277\n",
      "菜市 0.39940010202781245 18 320\n",
      "重點 0.39940010202781245 18 372\n",
      "----Document 19----\n",
      "全聯小編 0.3228551771884164 19 73\n",
      "創意 0.3228551771884164 19 83\n",
      "標竿 0.3228551771884164 19 240\n",
      "真好 0.3228551771884164 19 275\n",
      "精神 0.3228551771884164 19 294\n",
      "領袖 0.3228551771884164 19 390\n",
      "----Document 20----\n",
      "合作 0.38956348525663753 20 94\n",
      "店家 0.38956348525663753 20 148\n",
      "必看 0.38956348525663753 20 166\n",
      "所有 0.38956348525663753 20 185\n",
      "權吧 0.38956348525663753 20 242\n",
      "連結 0.38956348525663753 20 364\n",
      "----Document 21----\n",
      "基嫚 0.5773502691896257 21 111\n",
      "妮可 0.5773502691896257 21 126\n",
      "第一名 0.5773502691896257 21 290\n",
      "----Document 22----\n",
      "一個 0.3399604247294402 22 23\n",
      "一定 0.3700409359833579 22 24\n",
      "影片 0.3700409359833579 22 157\n",
      "很長 0.41243703541205234 22 162\n",
      "故事 0.31662816752849376 22 203\n",
      "秒鐘 0.41243703541205234 22 287\n",
      "能夠 0.41243703541205234 22 311\n",
      "----Document 23----\n",
      "如何 0.3035945274499617 23 125\n",
      "小米 0.39545953174717796 23 135\n",
      "榮耀 0.39545953174717796 23 238\n",
      "看看 0.3548086197571138 23 273\n",
      "翻身 0.39545953174717796 23 305\n",
      "谷底 0.39545953174717796 23 338\n",
      "重返 0.39545953174717796 23 371\n",
      "----Document 24----\n",
      "品牌 0.49148507484040005 24 98\n",
      "----Document 25----\n",
      "ryantoysreview 0.3477595324025741 25 20\n",
      "youtuber 0.3477595324025741 25 22\n",
      "億台幣 0.3477595324025741 25 67\n",
      "小孩 0.3477595324025741 25 134\n",
      "年收入 0.3477595324025741 25 146\n",
      "超過 0.3477595324025741 25 350\n",
      "這是 0.3477595324025741 25 358\n",
      "----Document 26----\n",
      "一杯 0.5773502691896257 26 28\n",
      "就掛 0.5773502691896257 26 137\n",
      "絕對 0.5773502691896257 26 299\n",
      "----Document 27----\n",
      "回來 0.6678149073586044 27 106\n",
      "聯爺爺 0.7443273805991679 27 309\n",
      "----Document 28----\n",
      "----Document 29----\n",
      "好像 0.30473586204446795 29 123\n",
      "鍋子 0.30473586204446795 29 374\n",
      "鐵鍋 0.30473586204446795 29 377\n",
      "----Document 30----\n",
      "2018 0.33933487093036774 30 3\n",
      "全美 0.4703328197592293 30 72\n",
      "得獎 0.4703328197592293 30 164\n",
      "理想 0.4703328197592293 30 264\n",
      "職企業 0.4703328197592293 30 310\n",
      "----Document 31----\n",
      "----Document 32----\n",
      "hen 0.7853991956966059 32 18\n",
      "支持 0.4376917313584553 32 201\n",
      "有感 0.4376917313584553 32 230\n",
      "----Document 33----\n",
      "2017 0.305767715118615 33 2\n",
      "android 0.3407999431552356 33 9\n",
      "app 0.305767715118615 33 10\n",
      "哪些 0.3407999431552356 33 99\n",
      "將近 0.3407999431552356 33 131\n",
      "年關 0.3407999431552356 33 147\n",
      "快來 0.3407999431552356 33 169\n",
      "最棒 0.3407999431552356 33 226\n",
      "瞧瞧 0.3407999431552356 33 278\n",
      "----Document 34----\n",
      "----Document 35----\n",
      "ai 0.6098587509765868 35 8\n",
      "很強 0.36993767064660704 35 160\n",
      "感謝 0.36993767064660704 35 178\n",
      "日子 0.36993767064660704 35 212\n",
      "結束 0.36993767064660704 35 297\n",
      "----Document 36----\n",
      "hen 0.4093043820958175 36 18\n",
      "其實 0.4561989485951171 36 76\n",
      "實用 0.4561989485951171 36 129\n",
      "新奇 0.4561989485951171 36 209\n",
      "科技 0.4561989485951171 36 286\n"
     ]
    }
   ],
   "source": [
    "for i in range(len(weight)):\n",
    "        print('----Document {0}----'.format(i))\n",
    "        for j in range(len(words)):\n",
    "            if tfidf[i,j] > 0.3:\n",
    "                  print(words[j], tfidf[i,j], i ,j)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 結論：由tf-idf計算出來的結果，可以看到12月的貼文除了各篇聚焦的重點除了對於未來新的展望、對這年的感謝，最重要的還是在於宣傳品牌"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "參考：https://www.safaribooksonline.com/library/view/feature-engineering-for/9781491953235/ch04.html"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}