Day38_hw.ipynb

{
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  },
  "orig_nbformat": 2,
  "kernelspec": {
   "name": "python376jvsc74a57bd07b4b4feff2f24a0f0a34464dbe537a36fda679851528fb8735cb41fa49dffb2d",
   "display_name": "Python 3.7.6 64-bit ('base': conda)"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2,
 "cells": [
  {
   "source": [
    "題目：將某篇文章以上下文相同，比方三連詞(trigram) 方式修改內容  \n",
    "說明：某篇文章中我們可以找出所有的三連詞(trigram)，以及在前字與後字出現時，按照出現度隨機選出一個字去換掉中間字，這是利用三連詞修改文章內容的最基本作法。一旦字典的資料結構建立，我們就以某種機率 (比方 20% ) 去置換原文，並將置換文與原文印出來  \n",
    "延伸：可用五連詞或七連詞去取代中間字，可利用三連詞之前兩字去更換第三字，可增加加詞性的相同性(Parts Of Sentence) 提高可讀性，甚至使用 Word2Vec，Glove，或者 RNN 的  \n",
    "範例程式檔名：article_modifier_自動文件修改器.py。  \n",
    "模組：sklearn, random, numpy, nltk, bs4  \n",
    "輸入檔：./electronics/positive.review  \n",
    "成績：被置換文的合理性與可讀性  "
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import print_function, division\n",
    "from future.utils import iteritems\n",
    "from builtins import range\n",
    "\n",
    "\n",
    "\n",
    "import nltk\n",
    "import random\n",
    "import numpy as np\n",
    "\n",
    "from bs4 import BeautifulSoup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load the reviews\n",
    "positive_reviews = BeautifulSoup(open('positive(作業數據).review', encoding='ISO-8859-1').read(), \"lxml\")\n",
    "positive_reviews = positive_reviews.findAll('review_text')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 提出 三連詞 並置入字典\n",
    "# (w1, w3) 當作 key, [ w2 ] 當作值\n",
    "trigrams = {}\n",
    "for review in positive_reviews:\n",
    "    s = review.text.lower()\n",
    "    tokens = nltk.tokenize.word_tokenize(s)\n",
    "    for i in range(len(tokens) - 2):\n",
    "        k = (tokens[i], tokens[i+2])\n",
    "        if k not in trigrams:\n",
    "            trigrams[k] = []\n",
    "        trigrams[k].append(tokens[i+1])\n",
    "\n",
    "# 將中間字矩陣變成或然率向量\n",
    "trigram_probabilities = {}\n",
    "for k, words in iteritems(trigrams):\n",
    "    # 產生一個  word -> count 字典\n",
    "    if len(set(words)) > 1:\n",
    "        # 如果中間字middle word不只有一個機率 \n",
    "        d = {}\n",
    "        n = 0\n",
    "        for w in words:\n",
    "            if w not in d:\n",
    "                d[w] = 0\n",
    "            d[w] += 1\n",
    "            n += 1\n",
    "        for w, c in iteritems(d):\n",
    "            d[w] = float(c) / n\n",
    "        trigram_probabilities[k] = d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Original: \ni've used this card for a while now and i'd say that i like it overall. the power usage is great. the drivers have been vastly improved and now you are able to scan for networks. those of you who are interested in wardriving should look elsewhere. if you want to use netstumbler or pocketwarrior in windows or kismet and airsnort in linux, either go for the linksys wcf12 (prism based) or lucent orinoco based pcmcia card. if you're looking to run just 802.11b and check email, stream music, browse the web, etc, this card will be perfect plus it'll save you some battery power\n\nSpun:\ni've used this card for a year now and i'd say that i like it overall. the power usage is great. the drivers have been vastly improved and now you are able to scan for networks. those of you who are interested in wardriving should look elsewhere. if you want to use netstumbler or are in windows or kismet and airsnort in linux, either go for the linksys wcf12 ( prism based ) or lucent orinoco based pcmcia card. if you're looking to run just 802.11b and check email, stream music, browse the phone, etc. business card may be perfect plus it does save you some battery power\n"
     ]
    }
   ],
   "source": [
    "def random_sample(d):\n",
    "    # 從字典隨機選出一個帶機率值的樣本，回傳累積機率值最大的字\n",
    "    r = random.random()\n",
    "    cumulative = 0\n",
    "    for w, p in iteritems(d):\n",
    "        cumulative += p\n",
    "        if r < cumulative:\n",
    "            return w\n",
    "\n",
    "\n",
    "def test_spinner():\n",
    "    review = random.choice(positive_reviews)\n",
    "    s = review.text.lower()\n",
    "    print(\"Original:\", s)\n",
    "    tokens = nltk.tokenize.word_tokenize(s)\n",
    "    for i in range(len(tokens) - 2):\n",
    "        if random.random() < 0.2: # 20% chance of replacement\n",
    "            k = (tokens[i], tokens[i+2])\n",
    "            if k in trigram_probabilities:\n",
    "                w = random_sample(trigram_probabilities[k])\n",
    "                tokens[i+1] = w\n",
    "    print(\"Spun:\")\n",
    "    print(\" \".join(tokens).replace(\" .\", \".\").replace(\" '\", \"'\").replace(\" ,\", \",\").replace(\"$ \", \"$\").replace(\" !\", \"!\"))\n",
    "\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    test_spinner()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ]
}