notebooks/11.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "269d5137",
   "metadata": {},
   "source": [
    "# Aggregate all generated texts into one table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "131a5508",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "\n",
    "PATH800 = \"../data/additional-features-v2/new/4_GPT-2_Generated_Text/\"\n",
    "PATH800_Categories = os.listdir(PATH800)\n",
    "PATHReplies = \"../data/additional-features-v2/new/10_Replies/\"\n",
    "PATHRepliesN = [n for n in sorted(os.listdir(PATHReplies)) if not n.startswith('.')]\n",
    "\n",
    "li = []\n",
    "for category in PATH800_Categories:\n",
    "    with os.scandir(os.path.join(PATH800, category)) as it:\n",
    "        for email in it:\n",
    "            if not email.name.startswith('.') and email.is_file():\n",
    "                with open(os.path.join(PATH800, category, email.name)) as f:\n",
    "                    email_content = [f.read(), 0, category, email.name.split('.')[0]]\n",
    "                    li.append(email_content)          "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "027155f6",
   "metadata": {},
   "outputs": [],
   "source": [
    "for i, replyN in enumerate(PATHRepliesN):\n",
    "    with os.scandir(os.path.join(PATHReplies, replyN)) as it:\n",
    "        for email in it:\n",
    "            if not email.name.startswith('.') and email.is_file():\n",
    "                with open(os.path.join(PATHReplies, replyN, email.name)) as f:\n",
    "                    name_split = email.name.rsplit(\"_\", 2)\n",
    "                    email_content = [f.read(), i+1, name_split[0], name_split[1]]\n",
    "                    li.append(email_content)     "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "c4718f22",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(li, columns=['content', 'reply #', 'category', 'index'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "40eaabb7",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.sort_values(['reply #','category', 'index']).reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "c01fbdb6",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv('../data/additional-features-v2/new/assignment2.tsv', sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d5789c3d",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "DSCI-550-Assignment-2",
   "language": "python",
   "name": "dsci-550-assignment-2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}