data_reading.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_dir = 'data/stance'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "stances =['abortion', 'gayRights', 'marijuana', 'obama']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import defaultdict\n",
    "ids = defaultdict(list)\n",
    "\n",
    "for stance in os.listdir(data_dir):\n",
    "    if stance in stances:\n",
    "        dir_ = os.path.join(data_dir, stance)\n",
    "        for path in os.listdir(dir_):\n",
    "            number, info = path.split('.')\n",
    "            if number not in ids[stance]:\n",
    "                ids[stance].append(number)\n",
    "            \n",
    "            \n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentences = defaultdict(dict)\n",
    "for topic, idx in ids.items():\n",
    "    topic_path = os.path.join(data_dir, topic)\n",
    "    for id_ in idx:\n",
    "        if len(id_)>0:\n",
    "            data_path = os.path.join(topic_path, id_+'.data')\n",
    "            meta_path =os.path.join(topic_path, id_+'.meta')\n",
    "            sentences[topic][id_] = dict()\n",
    "            with open(meta_path) as file:\n",
    "                for line in file:\n",
    "                    line=line.replace('\\n','')\n",
    "                    x,y = line.split('=')\n",
    "                    sentences[topic][id_][x] =y\n",
    "            with open(data_path)as file:\n",
    "                sentences[topic][id_]['data']= file.read().replace('\\n','')\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1376, 1915)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(sentences['gayRights']), len(sentences['abortion'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(626, 985)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(sentences['marijuana']), len(sentences['obama'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentences['marijuana']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "abortion_df = pd.DataFrame.from_dict(sentences['abortion'], orient='index', columns=['ID','PID','Stance', 'rebuttal', 'data'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "dir_ = 'processed_data/stance/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "abortion_df.to_csv(dir_+'abortion.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "for stance in sentences:\n",
    "    df = pd.DataFrame.from_dict(sentences[stance], orient='index', columns=['ID','PID','Stance', 'rebuttal', 'data'])\n",
    "    df.to_csv(dir_+stance+'.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "showing info https://raw.githubusercontent.com/nltk/nltk_data/gh-pages/index.xml\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nltk.download()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk\n",
    "data_dict=defaultdict(dict)\n",
    "for topic, value in sentences.items():\n",
    "    data_dict[topic]=defaultdict(list)\n",
    "    for key in value:\n",
    "        v= value[key]\n",
    "        data = v['data']\n",
    "        st = v['Stance']\n",
    "        \n",
    "        sents = nltk.sent_tokenize(data)\n",
    "        data_dict[topic][st].extend(sents)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "count =0 \n",
    "for topic, data in data_dict.items():\n",
    "    for stance, sentences in data.items():\n",
    "        with open(dir_+topic+stance+'.txt', 'w') as file:\n",
    "            for sentence in sentences:\n",
    "                file.write(str(count)+','+sentence+'\\n')\n",
    "                count+=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(dir_+'abortion-1.txt')as file:\n",
    "    for line in file:\n",
    "        id, data = line.split(',',1)\n",
    "        print(id)\n",
    "        print(data)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "nlp",
   "language": "python",
   "name": "nlp"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}