Data preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Sentiment Analysis\n",
    "\n",
    "#### WEI Chen, Pierre-Yves Casanova"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Twitter2016 Dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Import Twitter2016 sentiment analysis datasets, containing training set, validation set, and test set.\n",
    "\n",
    "All @,# tags, url links in text samples will be removed.\n",
    "\n",
    "The output format will be csv files, each containing 2 columns \"label\" and \"sentence\"."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Import from the txt file."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import os\n",
    "\n",
    "def read_tweets(path):\n",
    "    labels = list([])\n",
    "    data = list([])\n",
    "    with open(path) as fp:\n",
    "        for line in fp:\n",
    "#             print(line)\n",
    "            sentence = line.split()[2:]\n",
    "            sentence = ' '.join(sentence)\n",
    "            sentence = re.sub(r'(https|http)?:\\/\\/(\\w|\\.|\\/|\\?|\\=|\\&|\\%)*\\b', '', sentence, flags=re.MULTILINE).replace(\"@\",\"\").replace(\"#\",\"\")\n",
    "            data.append(sentence)\n",
    "            label =  line.split()[1]\n",
    "            if label=='positive':\n",
    "                labels.append(1)\n",
    "            if label=='negative':\n",
    "                labels.append(-1)\n",
    "            if label=='neutral':\n",
    "                labels.append(0)\n",
    "    return data, np.array(labels)   \n",
    "\n",
    "data_dir = 'tweets_data/'\n",
    "trX, trY = read_tweets(os.path.join(data_dir, 'twitter-2016train-A.txt'))\n",
    "vaX, vaY = read_tweets(os.path.join(data_dir, 'twitter-2016dev-A.txt'))\n",
    "teX, teY = read_tweets(os.path.join(data_dir, 'twitter-2016test-A.txt'))\n",
    "\n",
    "print(len(trX),len(vaX),len(teX))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Save the trinary form to csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv\n",
    "with open('tweets2016_dev.csv', \"w\", newline='') as output:\n",
    "    writer = csv.writer(output, delimiter=',')\n",
    "    writer.writerow([\"label\",\"sentence\"])\n",
    "    for i in range(len(vaY)) :\n",
    "        writer.writerow([vaY[i],vaX[i]])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Save the binary form by eliminating all neutral samples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv\n",
    "with open('tweets2016_binary_dev.csv', \"w\", newline='') as output:\n",
    "    writer = csv.writer(output, delimiter=',')\n",
    "    writer.writerow([\"label\",\"sentence\"])\n",
    "    for i in range(len(vaY)) :\n",
    "        if teY[i]!=0:\n",
    "            writer.writerow([vaY[i],vaX[i]])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## OpinMind Dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Import from txt file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import os\n",
    "\n",
    "def read_opinmind(path):\n",
    "    labels = list([])\n",
    "    data = list([])\n",
    "    with open(path, encoding='utf8') as fp:\n",
    "        for line in fp:\n",
    "#             print(line)\n",
    "            sentence = line.split()[1:]\n",
    "            sentence = ' '.join(sentence)\n",
    "            data.append(sentence)\n",
    "            label = line.split()[0]\n",
    "#             print(type(label),label)\n",
    "            if label=='1' or label=='\\ufeff1':\n",
    "                labels.append(1)\n",
    "            elif label=='0':\n",
    "                labels.append(-1)\n",
    "            else:\n",
    "                labels.append(label)\n",
    "    return data, np.array(labels)\n",
    "            \n",
    "\n",
    "data_dir = 'opinmind_data/'\n",
    "trX, trY = read_opinmind(os.path.join(data_dir, 'trainingdata.txt'))\n",
    "teX, teY = read_opinmind(os.path.join(data_dir, 'testdata.txt'))\n",
    "\n",
    "print(len(trX),trY.shape)\n",
    "print(len(teX),teY.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "De-duplicate the training set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "trX, unique_indices = np.unique(trX, return_index=True)\n",
    "trX = trX.tolist()\n",
    "trY = trY[unique_indices]\n",
    "\n",
    "print(len(trX),trY.shape)\n",
    "print(len(teX),teY.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## CNN corpus"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load the file names respectively for training, validation and test sets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "train_id = pd.read_csv('./cnn/train_story_ids.csv')\n",
    "dev_id = pd.read_csv('./cnn/dev_story_ids.csv')\n",
    "test_id = pd.read_csv('./cnn/test_story_ids.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Read CNN news articles, remove the document header and footer."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_cnn(path):\n",
    "    labels = list([])\n",
    "    data = list([])\n",
    "    with open(path, encoding='utf8') as fp:\n",
    "        for sentence in fp:\n",
    "            if sentence != '\\n' and sentence != '@highlight\\n':\n",
    "#                 print(sentence)\n",
    "                data.append(sentence.strip())\n",
    "    return data\n",
    "\n",
    "trX = list([])\n",
    "for path in train_id['story_id']:\n",
    "    trX = trX + read_cnn(path)\n",
    "veX = list([])\n",
    "for path in dev_id['story_id']:\n",
    "    veX = veX + read_cnn(path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Save training and validation corpus in two seperate txt files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "text_file = open(\"cnn_training.txt\", \"w\", encoding=\"utf-8\")\n",
    "text_file.write(trX)\n",
    "text_file.close()\n",
    "\n",
    "text_file = open(\"cnn_validation.txt\", \"w\", encoding=\"utf-8\")\n",
    "text_file.write(veX)\n",
    "text_file.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Bloomberg corpus"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Integrate seperate news article files into a single txt file, remove the document header and footer."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import os\n",
    "import pickle\n",
    "\n",
    "file_num = 0\n",
    "ignore_num = 0\n",
    "with open('G:\\\\2017-2018 courses\\\\PRIM2017\\\\sentiment-discovery-master\\\\data\\\\Bloomberg2.txt', 'w') as f:\n",
    "    for root, dirs, files in os.walk('G:\\\\2017-2018 courses\\\\PRIM2017\\\\financial-news-dataset-master\\\\20061020_20131126_bloomberg_news'):\n",
    "        for file in files:\n",
    "        # assert os.path.exists(path)\n",
    "        # Count bytes\n",
    "            with open(os.path.join(root, file), 'r', encoding='utf8') as lines:\n",
    "                article = ''\n",
    "                for i, line in enumerate(lines):\n",
    "                    article += line.strip()+' '\n",
    "#                 print(article)\n",
    "                try:\n",
    "#                     title = article.split(\".html\")[0].split(\"--\")[1]\n",
    "                    content = article.split(\".html\")[1].split(\"To contact the reporter on this story:\")[0].strip()\n",
    "                    f.write(content)\n",
    "                    f.write(\"\\n\")\n",
    "                except:\n",
    "                    ignore_num += 1\n",
    "                    if ignore_num%100 == 0:\n",
    "                        print(\"* ignored file: \", ignore_num)\n",
    "            file_num += 1\n",
    "            if file_num%1000 == 0:\n",
    "                print(\"* number of file: \", file_num)\n",
    "print(\"* number of file: \", file_num)\n",
    "print(\"* ignored file: \", ignore_num)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## MPQA2.0 Dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load the documents, find the target sentence with contextual polarity tag"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import os\n",
    "import numpy as np\n",
    "import nltk\n",
    "\n",
    "df = pd.read_pickle(\"G:\\\\2017-2018 courses\\\\PRIM2017\\\\database.mpqa.2.0\\\\mpqa_features.pickle\")\n",
    "paths = df.index.values\n",
    "keywords = df[\"context_\"].values\n",
    "sentiments = df['c_pol'].map(lambda k: 0 if k < 0 else 1).values\n",
    "\n",
    "mpqa_senti = list()\n",
    "\n",
    "for i, p in enumerate(paths):\n",
    "    path = os.path.join(\"G:\\\\2017-2018 courses\\\\PRIM2017\\\\database.mpqa.2.0\\\\docs\",p[0],p[1])\n",
    "    with open(path, 'r', encoding='utf8') as file:\n",
    "        document = file.read().replace('\\n', '')\n",
    "        sentences = nltk.sent_tokenize(document)\n",
    "        for j, line in enumerate(sentences):\n",
    "            if keywords[i][0] in line and keywords[i][1] in line and keywords[i][2] in line:\n",
    "                mpqa_senti.append((sentiments[i],line))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Split the whole dataset as training(0.7), validation(0.2), test(0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "mpqa_senti_train, mpqa_senti_test = train_test_split(mpqa_senti, test_size=0.3, random_state=42)\n",
    "mpqa_senti_dev, mpqa_senti_test = train_test_split(mpqa_senti_test, test_size=0.33, random_state=42)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Save to csv file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv \n",
    "with open(\"mpqa2.0_test.csv\", \"w\", newline='', encoding=\"utf8\") as output:\n",
    "    writer = csv.writer(output, delimiter=',')\n",
    "    writer.writerow([\"label\",\"sentence\"])\n",
    "    for i in range(len(mpqa_senti_test)) :\n",
    "        writer.writerow(mpqa_senti_test[i])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## IMDb dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "imdb dataset, containing 50000 movie reviews.\n",
    "\n",
    "Load from tsv file."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd       \n",
    "data = pd.read_csv(\"imdb50000/labeledTrainData.tsv\", header=0, delimiter=\"\\t\", quoting=3)\n",
    "\n",
    "X = data[\"review\"]\n",
    "y = data[\"sentiment\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Split the whole dataset as training(0.7), validation(0.2), test(0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42)\n",
    "X_dev, X_test, y_dev, y_test = train_test_split(X_test, y_test, test_size=0.33, random_state=42)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Save to csv file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv \n",
    "\n",
    "with open(\"imdb_dev.csv\", \"w\", newline='', encoding=\"utf8\") as output:\n",
    "    writer = csv.writer(output, delimiter=',')\n",
    "    writer.writerow([\"label\",\"sentence\"])\n",
    "    for i in range(len(X_dev)) :\n",
    "        writer.writerow((y_dev.values[i], X_dev.values[i].replace(\"<br />\", \" \")))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}