datasets/winning-args-corpus/stats.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import convokit\n",
    "from convokit import Corpus, Parser, PolitenessStrategies, download\n",
    "import timeit\n",
    "import re\n",
    "from numpy import mean\n",
    "from scipy import stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "corpus = Corpus(download(\"winning-args-corpus\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "utterance_ids = corpus.get_utterance_ids()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "293297"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(utterance_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "u = 0\n",
    "s = 0\n",
    "for each in utterance_ids:\n",
    "    if corpus.get_utterance(each).meta[\"success\"] == 0:\n",
    "        u = u + 1\n",
    "    if corpus.get_utterance(each).meta[\"success\"] == 1:\n",
    "        s = s + 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number of successful comments: 12420\n",
      "number of unsuccessful comments: 7294\n"
     ]
    }
   ],
   "source": [
    "print(\"number of successful comments: \" + str(s))\n",
    "print(\"number of unsuccessful comments: \" + str(u))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the number of unique pair_ids is: 4263\n"
     ]
    }
   ],
   "source": [
    "z = []\n",
    "for iD in utterance_ids:\n",
    "    a = corpus.get_utterance(iD)\n",
    "    b = a.meta\n",
    "    for every in b[\"pair_ids\"]:\n",
    "        z.append(every)\n",
    "print(\"the number of unique pair_ids is: \" + str(len(list(set(z)))))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Since this corpus is too large to parse with my laptop, we select the utterance ids for the groups of utterances that we are interested in. The following code finds the utterance ids for OP's comments (including the original post) and the challenger's comments (for both successful and unsuccessful arguments). Every other comment in the thread is excluded.\n",
    "\n",
    "Note: this subset of data is still larger than the 'pair_data.json' in the data provided by the changemyview paper (see readme for citation) because I have also matched the OP replies to the challenger's comments."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# we want the original post made by op, the challenger's comments and all of OP's responses to the challengers\n",
    "# these three lists are utterance ids for the original post, challenger comments and op replies respectively\n",
    "\n",
    "opPost = []\n",
    "challengerComments = []\n",
    "opReplies = []\n",
    "for iD in utterance_ids:\n",
    "    if corpus.get_utterance(iD).id == corpus.get_utterance(iD).root:\n",
    "        opPost.append(iD)\n",
    "    if (\n",
    "        corpus.get_utterance(iD).speaker.id\n",
    "        != corpus.get_utterance(corpus.get_utterance(iD).root).speaker.id\n",
    "        and corpus.get_utterance(iD).meta[\"success\"] == 0\n",
    "    ):\n",
    "        challengerComments.append(iD)\n",
    "\n",
    "    if (\n",
    "        corpus.get_utterance(iD).speaker.id\n",
    "        != corpus.get_utterance(corpus.get_utterance(iD).root).speaker.id\n",
    "        and corpus.get_utterance(iD).meta[\"success\"] == 1\n",
    "    ):\n",
    "        challengerComments.append(iD)\n",
    "\n",
    "    if (\n",
    "        corpus.get_utterance(iD).id != corpus.get_utterance(iD).root\n",
    "        and corpus.get_utterance(iD).speaker.id\n",
    "        == corpus.get_utterance(corpus.get_utterance(iD).root).speaker.id\n",
    "        and corpus.get_utterance(iD).meta[\"success\"] == 0\n",
    "    ):\n",
    "        opReplies.append(iD)\n",
    "    if (\n",
    "        corpus.get_utterance(iD).id != corpus.get_utterance(iD).root\n",
    "        and corpus.get_utterance(iD).speaker.id\n",
    "        == corpus.get_utterance(corpus.get_utterance(iD).root).speaker.id\n",
    "        and corpus.get_utterance(iD).meta[\"success\"] == 1\n",
    "    ):\n",
    "        opReplies.append(iD)\n",
    "\n",
    "# subset challenger and op replies for later use (into successful and unsuccessful arguments)\n",
    "challengerPos = []\n",
    "challengerNeg = []\n",
    "for iD in challengerComments:\n",
    "    if corpus.get_utterance(iD).meta[\"success\"] == 1:\n",
    "        challengerPos.append(iD)\n",
    "    if corpus.get_utterance(iD).meta[\"success\"] == 0:\n",
    "        challengerNeg.append(iD)\n",
    "# these are OP's replies to successful and unsuccessful challengers\n",
    "opReplyPos = []\n",
    "opReplyNeg = []\n",
    "for iD in opReplies:\n",
    "    if corpus.get_utterance(iD).meta[\"success\"] == 1:\n",
    "        opReplyPos.append(iD)\n",
    "    if corpus.get_utterance(iD).meta[\"success\"] == 0:\n",
    "        opReplyNeg.append(iD)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "subset = opPost + challengerComments + opReplies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# collect utterance dict given the subset of ids\n",
    "utterance_list = []\n",
    "for iD in subset:\n",
    "    utterance_list.append(corpus.get_utterance(iD))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# collect challenger comments to count pairID (data validation check: there should be 3456 unique pair IDs)\n",
    "challenger_utterance_list = []\n",
    "for iD in challengerComments:\n",
    "    challenger_utterance_list.append(corpus.get_utterance(iD))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20536\n",
      "4263\n"
     ]
    }
   ],
   "source": [
    "pair_idz = []\n",
    "for utt in utterance_list:\n",
    "    for each in utt.meta[\"pair_ids\"]:\n",
    "        pair_idz.append(each)\n",
    "print(len(pair_idz))\n",
    "print(len(list(set(pair_idz))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "11567\n",
      "4263\n"
     ]
    }
   ],
   "source": [
    "pair_idz = []\n",
    "for utt in challenger_utterance_list:\n",
    "    for each in utt.meta[\"pair_ids\"]:\n",
    "        pair_idz.append(each)\n",
    "print(len(pair_idz))\n",
    "print(len(list(set(pair_idz))))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create the subset corpus that we are interested (note: the original data from the paper only contained the challenger replies and the original post, nothing else was included -- I collected OP's replies from the 'all' data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# this subset separates OP comments and challenger utterances from all other comments in every conversation (thread)\n",
    "corpus = convokit.Corpus(utterances=utterance_list, version=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of Speakers: 6210\n",
      "Number of Utterances: 22765\n",
      "Number of Conversations: 3051\n"
     ]
    }
   ],
   "source": [
    "corpus.print_summary_stats()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "11020"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(challengerComments)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8694"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(opReplies)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Simple statistics:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Note: the averages below are for challenger comments only\n",
      "Number of Speakers: 6210\n",
      "Number of Utterances: 22765\n",
      "Number of Conversations: 3051\n",
      "Average number of words in a succesful comment is 257.3235489220564\n",
      "Average number of words in an unsuccesful comment is 197.236873747495\n",
      "p-value for number of words is 5.38614002411782e-57\n",
      "Average number of words in a succesful root comment is 276.7655660377358\n",
      "Average number of words in an unsuccesful rootcomment is 214.2785825142266\n",
      "p-value for number of words in root comments is 7.645306791441279e-44\n",
      "Average number of deltas assigned to an author of a succesful comment is 13.61011608623549\n",
      "Average number of deltas assigned to an author of a unsuccesful comment is 8.446492985971943\n",
      "p-value for number of deltas assigned to commenters is 3.355536404744011e-38\n",
      "Average number of deltas assigned to an author of a succesful root comment is 12.837028301886793\n",
      "Average number of deltas assigned to an author of a unsuccesful root comment is 8.184945680289704\n",
      "p-value for number of deltas assigned to root commenters is 3.489655850372796e-24\n"
     ]
    }
   ],
   "source": [
    "print(\"Note: the averages below are for challenger comments only\")\n",
    "\n",
    "corpus.print_summary_stats()\n",
    "\n",
    "utts = list(corpus.iter_utterances())  # list of all uterrance objects in the corpus\n",
    "succ_length = []  # length of all comments in succesful threads\n",
    "root_succ_length = []  # length of successful root replies\n",
    "succ_deltas = []  # num deltas given to users commenting in all succesful threads\n",
    "root_succ_deltas = []  # num deltas given to root commenters in succesful threads\n",
    "unsucc_length = []  # length of all comments in unsuccesful threads\n",
    "root_unsucc_length = []  # length of unsuccessful root replies\n",
    "unsucc_deltas = []  # num deltas given to users commenting in all unsuccesful threads\n",
    "root_unsucc_deltas = []  # num deltas given to root commenters in unsuccessful threads\n",
    "\n",
    "for i in utts:\n",
    "    if (\n",
    "        i.root != i.id and i.speaker.id != corpus.get_utterance(i.root).speaker.id\n",
    "    ):  # exclude the original post and exlcude comments made by op\n",
    "        if i.meta[\"success\"] == 1:  # if succesful\n",
    "            succ_length.append(len((i.text).split()))  # num words\n",
    "            if i.reply_to == i.root:  # it's a root comment\n",
    "                root_succ_length.append(len((i.text).split()))\n",
    "\n",
    "            if i.meta[\"author_flair_text\"]:\n",
    "                r = re.search(\n",
    "                    r\"\\d+\", i.meta[\"author_flair_text\"]\n",
    "                )  # number of delta given to author. Note: have not checked this regex\n",
    "                if r:  # ignore weird cases that don't fit pattern, e.g. inf\n",
    "                    succ_deltas.append(int(r.group()))\n",
    "                    if i.reply_to == i.root:  # it's a root comment\n",
    "                        root_succ_deltas.append(int(r.group()))\n",
    "            else:\n",
    "                succ_deltas.append(0)\n",
    "                if i.reply_to == i.root:  # it's a root comment\n",
    "                    root_succ_deltas.append(0)\n",
    "        elif i.meta[\"success\"] == 0:  # if unsuccesful:\n",
    "            unsucc_length.append(len((i.text).split()))  # num words\n",
    "            if i.reply_to == i.root:  # it's a root comment\n",
    "                root_unsucc_length.append(len((i.text).split()))\n",
    "\n",
    "            if i.meta[\"author_flair_text\"]:\n",
    "                r = re.search(r\"\\d+\", i.meta[\"author_flair_text\"])\n",
    "                if r:  # ignore weird cases that don't fit pattern, e.g. inf\n",
    "                    unsucc_deltas.append(int(r.group()))\n",
    "                    if i.reply_to == i.root:  # it's a root comment\n",
    "                        root_unsucc_deltas.append(int(r.group()))\n",
    "            else:\n",
    "                unsucc_deltas.append(0)\n",
    "                if i.reply_to == i.root:  # it's a root comment\n",
    "                    root_unsucc_deltas.append(0)\n",
    "\n",
    "\n",
    "# length of comments\n",
    "print(\"Average number of words in a succesful comment is \" + str(mean(succ_length)))\n",
    "print(\"Average number of words in an unsuccesful comment is \" + str(mean(unsucc_length)))\n",
    "p_val = stats.ttest_ind(succ_length, unsucc_length, equal_var=False)[\n",
    "    1\n",
    "]  # using Welch's t-test, because I have no reason to assume variances are the same.\n",
    "print(\"p-value for number of words is \" + str(p_val))\n",
    "\n",
    "# length of root comments\n",
    "print(\"Average number of words in a succesful root comment is \" + str(mean(root_succ_length)))\n",
    "print(\"Average number of words in an unsuccesful rootcomment is \" + str(mean(root_unsucc_length)))\n",
    "p_val = stats.ttest_ind(root_succ_length, root_unsucc_length, equal_var=False)[\n",
    "    1\n",
    "]  # using Welch's t-test, because I have no reason to assume variances are the same.\n",
    "print(\"p-value for number of words in root comments is \" + str(p_val))\n",
    "\n",
    "# deltas to commenters\n",
    "print(\n",
    "    \"Average number of deltas assigned to an author of a succesful comment is \"\n",
    "    + str(mean(succ_deltas))\n",
    ")\n",
    "print(\n",
    "    \"Average number of deltas assigned to an author of a unsuccesful comment is \"\n",
    "    + str(mean(unsucc_deltas))\n",
    ")\n",
    "p_val = stats.ttest_ind(succ_deltas, unsucc_deltas, equal_var=False)[\n",
    "    1\n",
    "]  # using Welch's t-test, because I have no reason to assume variances are the same.\n",
    "print(\"p-value for number of deltas assigned to commenters is \" + str(p_val))\n",
    "\n",
    "# deltas to root commenters\n",
    "print(\n",
    "    \"Average number of deltas assigned to an author of a succesful root comment is \"\n",
    "    + str(mean(root_succ_deltas))\n",
    ")\n",
    "print(\n",
    "    \"Average number of deltas assigned to an author of a unsuccesful root comment is \"\n",
    "    + str(mean(root_unsucc_deltas))\n",
    ")\n",
    "p_val = stats.ttest_ind(root_succ_deltas, root_unsucc_deltas, equal_var=False)[\n",
    "    1\n",
    "]  # using Welch's t-test, because I have no reason to assume variances are the same.\n",
    "print(\"p-value for number of deltas assigned to root commenters is \" + str(p_val))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "pycharm": {
   "stem_cell": {
    "cell_type": "raw",
    "metadata": {
     "collapsed": false
    },
    "source": []
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}