data_reorganize_pheme.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9b150a35",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import os\n",
    "import json\n",
    "import numpy as np\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "from utils import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "61b7abaf",
   "metadata": {},
   "outputs": [],
   "source": [
    "config_data_type = 'text_comments ' # text_comments, text_only, comments_only\n",
    "base_dir = './data/PHEME-SEG/' + config_data_type\n",
    "if not os.path.exists(base_dir):\n",
    "    os.mkdir(base_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "034f28a2",
   "metadata": {},
   "outputs": [],
   "source": [
    "raw_data = pd.read_csv('./data/pheme_raw_data.csv')\n",
    "raw_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a04233d5",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = raw_data[['text_comments','label']]\n",
    "data = data.rename(columns = {'text_comments':'text'})\n",
    "data['label'] = LabelEncoder().fit_transform(data['label'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6ce1d149",
   "metadata": {},
   "outputs": [],
   "source": [
    "train, tmp_val = train_test_split(data, test_size=0.1, random_state=35)\n",
    "_, tmp_test = train_test_split(train, test_size=0.3, random_state=35)\n",
    "\n",
    "test = pd.concat([tmp_val, tmp_test])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0fac9a51",
   "metadata": {},
   "outputs": [],
   "source": [
    "train.reset_index(drop=True, inplace=True)\n",
    "test.reset_index(drop=True, inplace=True)\n",
    "\n",
    "print(len(train),len(test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0e6ba940",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_label = train['label'].values.tolist()\n",
    "test_label = test['label'].values.tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b1be3e99",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(base_dir + '/train_label.txt','w',encoding='utf-8') as file:\n",
    "    for each_index in train_label:\n",
    "        file.write(str(each_index))\n",
    "        file.write('\\n')\n",
    "\n",
    "with open(base_dir + '/test_label.txt','w',encoding='utf-8') as file:\n",
    "    for each_index in test_label:\n",
    "        file.write(str(each_index))\n",
    "        file.write('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "18ee9595",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_tmp = train.copy()\n",
    "train_tmp['text_split'] = train['text'].apply(get_split)\n",
    "train = train_tmp\n",
    "\n",
    "test_tmp = test.copy()\n",
    "test_tmp['text_split'] = test['text'].apply(get_split)\n",
    "test = test_tmp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "293a139d",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_l = []  # Segmented Text\n",
    "label_l = []  # Label of Each Text\n",
    "index_l =[]   # The Index of Each Text Before Segmentation\n",
    "for idx,row in train.iterrows():\n",
    "  for l in row['text_split']:\n",
    "    train_l.append(l)\n",
    "    label_l.append(row['label'])\n",
    "    index_l.append(idx)\n",
    "len(train_l), len(label_l), len(index_l)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1496fcb2",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_l = []\n",
    "test_label_l = []\n",
    "test_index_l =[]\n",
    "for idx,row in test.iterrows():\n",
    "  for l in row['text_split']:\n",
    "    test_l.append(l)\n",
    "    test_label_l.append(row['label'])\n",
    "    test_index_l.append(idx)\n",
    "len(test_l), len(test_label_l), len(test_index_l)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3c84aee1",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(base_dir + '/train_index_list.txt','w',encoding='utf-8') as file:\n",
    "    for each_index in index_l:\n",
    "        file.write(str(each_index))\n",
    "        file.write('\\n')\n",
    "\n",
    "with open(base_dir + '/test_index_list.txt','w',encoding='utf-8') as file:\n",
    "    for each_index in test_index_l:\n",
    "        file.write(str(each_index))\n",
    "        file.write('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5539893a",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df = pd.DataFrame({'text':train_l, 'label':label_l})\n",
    "train_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "71f74570",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_df = pd.DataFrame({'text':test_l, 'label':test_label_l})\n",
    "test_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "936e65bc",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(base_dir + '/train.txt', 'w', encoding='utf-8') as file:\n",
    "    for i in range(len(train_df)):\n",
    "        file.write(str(train_df.iloc[i,1]))\n",
    "        file.write(' ')\n",
    "        string = train_df.iloc[i,0].replace('\\n',' ').replace('[SEP]',' ').replace('\\t',' ').replace('\\r',' ')\n",
    "        file.write(string)\n",
    "        file.write('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "16dd0721",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(base_dir + '/test.txt', 'w', encoding='utf-8') as file:\n",
    "    for i in range(len(test_df)):\n",
    "        file.write(str(test_df.iloc[i,1]))\n",
    "        file.write(' ')\n",
    "        string = test_df.iloc[i,0].replace('\\n',' ').replace('[SEP]',' ').replace('\\t',' ').replace('\\r',' ')\n",
    "        file.write(string)\n",
    "        file.write('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cb4dc2a1",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}